From 7358d102eb1310bf9252395dfda0f1b56b83d4f0 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 20:07:43 -0500 Subject: [PATCH 01/51] add startup taint --- main.tf | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/main.tf b/main.tf index 638cfdc..1303812 100644 --- a/main.tf +++ b/main.tf @@ -87,4 +87,63 @@ resource "helm_release" "nodepool_resources" { vpc_id = var.vpc_id }) ] + #### Default values overrides + set { + name = "name" + value = var.cluster_name + } + set { + name = "clusterName" + value = var.cluster_name + } + set { + name = "ec2NodeClass.karpenterRole" + value = aws_iam_instance_profile.karpenter_node.name + } + set { + name = "ec2NodeClass.amiFamily" + value = local.amd_ami_family + } + set { + name = "ec2NodeClass.amiSelectorTerms[0].alias" + value = local.amd_ami_alias + } + set { + name = "ec2NodeClass.securityGroupSelectorTerms[0].tags.Name" + value = "${var.cluster_name}-node" + } + set { + name = "ec2NodeClass.subnetSelectorTerms[0].tags.Name" + value = var.subnets_name + } + set { + name = "ec2NodeClass.subnetSelectorTerms[0].tags.kubernetes\\.io/cluster/${var.cluster_name}" + value = "shared" + } + ### Node Class Inputs + set { + name = "nodePool.nodeClassRef.name" + value = "${var.cluster_name}-node-class" + } + set { + name = "nodePool.name" + value = "${var.cluster_name}-karpenter-node-pool" + } + set { + name = "nodeClass.cluster_name" + value = var.cluster_name + } + set { + name = "nodePool.limits.cpu" + value = "1000" + } + set { + name = "nodePool.limits.memory" + value = "1000Gi" + } + set { + name = "ec2NodeClass.tags" + value = jsonencode(var.tags) + } + } From 07d456fad573da5592db8d839ad79af57efed0a2 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 20:28:27 -0500 Subject: [PATCH 02/51] add namespace and startup taint to template --- main.tf | 4 ++-- variables.tf | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/main.tf b/main.tf index 1303812..aa40854 100644 --- a/main.tf +++ b/main.tf @@ -43,7 +43,7 @@ resource "helm_release" "karpenter" { namespace = var.namespace description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" repository = var.karpenter_helm_repo - create_namespace = true + create_namespace = false values = [ templatefile("${path.module}/karpenter/values.yaml.tpl", { @@ -123,7 +123,7 @@ resource "helm_release" "nodepool_resources" { ### Node Class Inputs set { name = "nodePool.nodeClassRef.name" - value = "${var.cluster_name}-node-class" + value = "${var.cluster_name}-karpenter-node-class" } set { name = "nodePool.name" diff --git a/variables.tf b/variables.tf index 72bd4d8..388139d 100644 --- a/variables.tf +++ b/variables.tf @@ -65,8 +65,3 @@ variable "namespace" { type = string default = "namespace" } - -variable "vpc_id" { - description = "The ID of the VPC where the resources will be deployed" - type = string -} From ea60e939e096910a6f57374ff064e824575040f5 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 22:33:41 -0500 Subject: [PATCH 03/51] templatlize --- karpenter-resources/values.yaml.tpl | 16 +++--- karpenter/values.yaml.tpl | 49 +++++++++++++++++ main.tf | 82 ++++++++--------------------- 3 files changed, 81 insertions(+), 66 deletions(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 747ba17..9dc3841 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -5,17 +5,17 @@ clusterName: ${cluster_name} # EC2NodeClass specific values ec2NodeClass: enabled: true - amiFamily: ${amd_ami_family} + amiFamily: Bottlerocket amiSelectorTerms: - - alias: ${amd_ami_alias} - vpcId: ${vpc_id} + - alias: "bottlerocket@latest" subnetSelectorTerms: - tags: - Name: "${subnets_name}" + Name: ${subnets_name} + "kubernetes.io/cluster/${cluster_name}": "shared" securityGroupSelectorTerms: - tags: Name: "${cluster_name}-node" - karpenterRole: ${karpenter_node_profile} + karpenterRole: ${karpenter_node_profile_name} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: enabled @@ -34,15 +34,17 @@ ec2NodeClass: # NodePool specific values nodePool: - name: "${cluster_name}-worker-node-pool" + name: "${cluster_name}-karpenter-node-pool" enabled: true nodeClassRef: - name: "${cluster_name}-worker-node-class" + name: "${cluster_name}-karpenter-node-class" group: karpenter.k8s.aws kind: EC2NodeClass disruption: consolidationPolicy: "WhenEmptyOrUnderutilized" consolidateAfter: "5m" + budgets: + maxUnavailable: 1 limits: cpu: "1000" memory: 1000Gi diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 1c0b489..9d405ac 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -1,3 +1,4 @@ +<<<<<<< HEAD settings: clusterName: ${cluster_name} clusterEndpoint: ${cluster_endpoint} @@ -8,10 +9,13 @@ settings: reservedCapacity: true spotToSpotConsolidation: true +======= +>>>>>>> bc776f3 (templatlize) serviceAccount: annotations: eks.amazonaws.com/role-arn: ${iam_role_arn} +<<<<<<< HEAD replicas: 2 serviceMonitor: @@ -26,10 +30,22 @@ controller: limits: cpu: 500m memory: 1Gi +======= +settings: + clusterName: ${cluster_name} + clusterEndpoint: ${cluster_endpoint} + interruptionQueue: ${queue_name} + +aws: + defaultInstanceProfile: ${karpenter_node_group_name} + +controller: +>>>>>>> bc776f3 (templatlize) image: repository: ${repository} tag: ${tag} digest: ${digest} +<<<<<<< HEAD env: - name: AWS_REGION value: ${region} @@ -39,14 +55,47 @@ controller: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" effect: "NoSchedule" +======= + resources: + requests: + cpu: 0.5 + memory: 1Gi + limits: + cpu: 1 + memory: 1Gi + env: + - name: AWS_REGION + value: ${region} + +replicas: 2 + +nodeSelector: + kubernetes.io/os: linux + +tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule +>>>>>>> bc776f3 (templatlize) affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: +<<<<<<< HEAD - matchExpressions: - key: karpenter.sh/nodepool operator: DoesNotExist podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" +======= + - matchExpressions: + - key: eks.amazonaws.com/nodegroup + operator: Exists +>>>>>>> bc776f3 (templatlize) diff --git a/main.tf b/main.tf index aa40854..cbc8b5e 100644 --- a/main.tf +++ b/main.tf @@ -47,6 +47,7 @@ resource "helm_release" "karpenter" { values = [ templatefile("${path.module}/karpenter/values.yaml.tpl", { +<<<<<<< HEAD cluster_endpoint = var.cluster_endpoint cluster_name = var.cluster_name digest = data.aws_ecr_image.karpenter_image.id @@ -60,6 +61,19 @@ resource "helm_release" "karpenter" { module.images.images[local.karpenter_key].dest_repository ) tag = module.images.images[local.karpenter_key].tag +======= + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + karpenter_node_group_name = var.karpenter_node_group_name + queue_name = module.karpenter_resources.queue_name + repository = format("%v/%v", + module.images.images[local.karpenter_key].dest_registry, + module.images.images[local.karpenter_key].dest_repository) + tag = module.images.images[local.karpenter_key].tag + digest = data.aws_ecr_image.karpenter_image.id + region = var.region + iam_role_arn = module.karpenter_resources.iam_role_arn +>>>>>>> bc776f3 (templatlize) }) ] @@ -78,6 +92,7 @@ resource "helm_release" "nodepool_resources" { values = [ templatefile("${path.module}/karpenter-resources/values.yaml.tpl", { +<<<<<<< HEAD amd_ami_alias = local.amd_ami_alias amd_ami_family = local.amd_ami_family cluster_name = var.cluster_name @@ -85,65 +100,14 @@ resource "helm_release" "nodepool_resources" { subnets_name = var.subnets_name tags = var.tags vpc_id = var.vpc_id +======= + cluster_name = var.cluster_name + karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name + amd_ami_family = local.amd_ami_family + amd_ami_alias = local.amd_ami_alias + subnets_name = var.subnets_name + tags = var.tags +>>>>>>> bc776f3 (templatlize) }) ] - #### Default values overrides - set { - name = "name" - value = var.cluster_name - } - set { - name = "clusterName" - value = var.cluster_name - } - set { - name = "ec2NodeClass.karpenterRole" - value = aws_iam_instance_profile.karpenter_node.name - } - set { - name = "ec2NodeClass.amiFamily" - value = local.amd_ami_family - } - set { - name = "ec2NodeClass.amiSelectorTerms[0].alias" - value = local.amd_ami_alias - } - set { - name = "ec2NodeClass.securityGroupSelectorTerms[0].tags.Name" - value = "${var.cluster_name}-node" - } - set { - name = "ec2NodeClass.subnetSelectorTerms[0].tags.Name" - value = var.subnets_name - } - set { - name = "ec2NodeClass.subnetSelectorTerms[0].tags.kubernetes\\.io/cluster/${var.cluster_name}" - value = "shared" - } - ### Node Class Inputs - set { - name = "nodePool.nodeClassRef.name" - value = "${var.cluster_name}-karpenter-node-class" - } - set { - name = "nodePool.name" - value = "${var.cluster_name}-karpenter-node-pool" - } - set { - name = "nodeClass.cluster_name" - value = var.cluster_name - } - set { - name = "nodePool.limits.cpu" - value = "1000" - } - set { - name = "nodePool.limits.memory" - value = "1000Gi" - } - set { - name = "ec2NodeClass.tags" - value = jsonencode(var.tags) - } - } From 40ed7b6441aee3f50f146f0255f97109f1715f16 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 22:45:24 -0500 Subject: [PATCH 04/51] fix vals --- karpenter-resources/values.yaml.tpl | 2 +- karpenter/values.yaml.tpl | 2 +- main.tf | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 9dc3841..1a5a180 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -15,7 +15,7 @@ ec2NodeClass: securityGroupSelectorTerms: - tags: Name: "${cluster_name}-node" - karpenterRole: ${karpenter_node_profile_name} + karpenterRole: ${karpenter_node_profile} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: enabled diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 9d405ac..894daaa 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -37,7 +37,7 @@ settings: interruptionQueue: ${queue_name} aws: - defaultInstanceProfile: ${karpenter_node_group_name} + defaultInstanceProfile: ${karpenter_node_profile} controller: >>>>>>> bc776f3 (templatlize) diff --git a/main.tf b/main.tf index cbc8b5e..c286cd4 100644 --- a/main.tf +++ b/main.tf @@ -65,6 +65,7 @@ resource "helm_release" "karpenter" { cluster_name = var.cluster_name cluster_endpoint = var.cluster_endpoint karpenter_node_group_name = var.karpenter_node_group_name + karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name queue_name = module.karpenter_resources.queue_name repository = format("%v/%v", module.images.images[local.karpenter_key].dest_registry, From 0d8a8f6e79f8750e3cbce602d96b91029c853173 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 23:14:53 -0500 Subject: [PATCH 05/51] fewer things --- karpenter/values.yaml.tpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 894daaa..a8db047 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -1,4 +1,5 @@ <<<<<<< HEAD +<<<<<<< HEAD settings: clusterName: ${cluster_name} clusterEndpoint: ${cluster_endpoint} @@ -31,6 +32,8 @@ controller: cpu: 500m memory: 1Gi ======= +======= +>>>>>>> 4e10249 (fewer things) settings: clusterName: ${cluster_name} clusterEndpoint: ${cluster_endpoint} @@ -45,6 +48,7 @@ controller: repository: ${repository} tag: ${tag} digest: ${digest} +<<<<<<< HEAD <<<<<<< HEAD env: - name: AWS_REGION @@ -82,6 +86,8 @@ tolerations: operator: Exists effect: NoSchedule >>>>>>> bc776f3 (templatlize) +======= +>>>>>>> 4e10249 (fewer things) affinity: nodeAffinity: From 501aacd4f29f1957ac3fefbebe773795197c3fc7 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 23:41:40 -0500 Subject: [PATCH 06/51] add a few missing values --- karpenter/values.yaml.tpl | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index a8db047..b972528 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -38,9 +38,14 @@ settings: clusterName: ${cluster_name} clusterEndpoint: ${cluster_endpoint} interruptionQueue: ${queue_name} + isolatedVPC: true + featureGates: + nodeRepair: true + spotToSpotConsolidation: true -aws: - defaultInstanceProfile: ${karpenter_node_profile} +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: ${iam_role_arn} controller: >>>>>>> bc776f3 (templatlize) @@ -49,6 +54,7 @@ controller: tag: ${tag} digest: ${digest} <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD env: - name: AWS_REGION @@ -88,6 +94,11 @@ tolerations: >>>>>>> bc776f3 (templatlize) ======= >>>>>>> 4e10249 (fewer things) +======= + env: + - name: AWS_REGION + value: ${region} +>>>>>>> fe6559f (add a few missing values) affinity: nodeAffinity: @@ -102,6 +113,14 @@ affinity: - topologyKey: "kubernetes.io/hostname" ======= - matchExpressions: + - key: karpenter.sh/nodepool + operator: DoesNotExist - key: eks.amazonaws.com/nodegroup +<<<<<<< HEAD operator: Exists >>>>>>> bc776f3 (templatlize) +======= + operator: In + values: + - ${karpenter_node_group_name} +>>>>>>> fe6559f (add a few missing values) From 19cf7e65383b6032d2bd2960bb84dc697e117121 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 23:56:35 -0500 Subject: [PATCH 07/51] create ns --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index c286cd4..10a2c2b 100644 --- a/main.tf +++ b/main.tf @@ -43,7 +43,7 @@ resource "helm_release" "karpenter" { namespace = var.namespace description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" repository = var.karpenter_helm_repo - create_namespace = false + create_namespace = true values = [ templatefile("${path.module}/karpenter/values.yaml.tpl", { From 1a547b5d77bc19d68e14cc421db720421b7db805 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 12:49:55 -0500 Subject: [PATCH 08/51] update values --- karpenter/values.yaml.tpl | 33 +++++++++++++++++++++++++++++++++ main.tf | 19 ++----------------- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index b972528..18f80a7 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -41,13 +41,27 @@ settings: isolatedVPC: true featureGates: nodeRepair: true + reservedCapacity: true spotToSpotConsolidation: true serviceAccount: annotations: eks.amazonaws.com/role-arn: ${iam_role_arn} +replicas: 2 + +podDisruptionBudget: + name: karpenter + maxUnavailable: 1 +podSecurityContext: + fsGroup: 65532 +priorityClassName: system-cluster-critical +serviceMonitor: + enabled: true +logLevel: info + controller: +<<<<<<< HEAD >>>>>>> bc776f3 (templatlize) image: repository: ${repository} @@ -56,6 +70,19 @@ controller: <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +======= + resources: + requests: + cpu: 1 + memory: 1Gi + limits: + cpu: 1 + memory: 1Gi + image: + repository: ${repository} + tag: ${tag} + # digest: ${digest} +>>>>>>> 8bb3bc4 (update values) env: - name: AWS_REGION value: ${region} @@ -123,4 +150,10 @@ affinity: operator: In values: - ${karpenter_node_group_name} +<<<<<<< HEAD >>>>>>> fe6559f (add a few missing values) +======= + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" +>>>>>>> 8bb3bc4 (update values) diff --git a/main.tf b/main.tf index 10a2c2b..ea7daf6 100644 --- a/main.tf +++ b/main.tf @@ -47,7 +47,6 @@ resource "helm_release" "karpenter" { values = [ templatefile("${path.module}/karpenter/values.yaml.tpl", { -<<<<<<< HEAD cluster_endpoint = var.cluster_endpoint cluster_name = var.cluster_name digest = data.aws_ecr_image.karpenter_image.id @@ -61,20 +60,16 @@ resource "helm_release" "karpenter" { module.images.images[local.karpenter_key].dest_repository ) tag = module.images.images[local.karpenter_key].tag -======= - cluster_name = var.cluster_name - cluster_endpoint = var.cluster_endpoint karpenter_node_group_name = var.karpenter_node_group_name karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name queue_name = module.karpenter_resources.queue_name repository = format("%v/%v", module.images.images[local.karpenter_key].dest_registry, module.images.images[local.karpenter_key].dest_repository) - tag = module.images.images[local.karpenter_key].tag - digest = data.aws_ecr_image.karpenter_image.id + tag = module.images.images[local.karpenter_key].tag + # digest = data.aws_ecr_image.karpenter_image.id region = var.region iam_role_arn = module.karpenter_resources.iam_role_arn ->>>>>>> bc776f3 (templatlize) }) ] @@ -93,22 +88,12 @@ resource "helm_release" "nodepool_resources" { values = [ templatefile("${path.module}/karpenter-resources/values.yaml.tpl", { -<<<<<<< HEAD - amd_ami_alias = local.amd_ami_alias - amd_ami_family = local.amd_ami_family - cluster_name = var.cluster_name - karpenter_node_profile = module.karpenter_resources.instance_profile_name - subnets_name = var.subnets_name - tags = var.tags - vpc_id = var.vpc_id -======= cluster_name = var.cluster_name karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name amd_ami_family = local.amd_ami_family amd_ami_alias = local.amd_ami_alias subnets_name = var.subnets_name tags = var.tags ->>>>>>> bc776f3 (templatlize) }) ] } From d445b1c36590522d99dad3fc1f481daae579bd3f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 12:54:42 -0500 Subject: [PATCH 09/51] add nodeSelector and tolerations --- karpenter/values.yaml.tpl | 88 +-------------------------------------- 1 file changed, 2 insertions(+), 86 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 18f80a7..f346b60 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -1,39 +1,3 @@ -<<<<<<< HEAD -<<<<<<< HEAD -settings: - clusterName: ${cluster_name} - clusterEndpoint: ${cluster_endpoint} - interruptionQueue: ${queue_name} - isolatedVPC: true - featureGates: - nodeRepair: true - reservedCapacity: true - spotToSpotConsolidation: true - -======= ->>>>>>> bc776f3 (templatlize) -serviceAccount: - annotations: - eks.amazonaws.com/role-arn: ${iam_role_arn} - -<<<<<<< HEAD -replicas: 2 - -serviceMonitor: - enabled: false -logLevel: debug - -controller: - resources: - requests: - cpu: 1m - memory: 1Mi - limits: - cpu: 500m - memory: 1Gi -======= -======= ->>>>>>> 4e10249 (fewer things) settings: clusterName: ${cluster_name} clusterEndpoint: ${cluster_endpoint} @@ -61,50 +25,26 @@ serviceMonitor: logLevel: info controller: -<<<<<<< HEAD ->>>>>>> bc776f3 (templatlize) image: repository: ${repository} tag: ${tag} digest: ${digest} -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -======= resources: requests: - cpu: 1 + cpu: 0.5 memory: 1Gi limits: cpu: 1 memory: 1Gi - image: - repository: ${repository} - tag: ${tag} - # digest: ${digest} ->>>>>>> 8bb3bc4 (update values) env: - name: AWS_REGION value: ${region} nodeSelector: - kubernetes.io/os: linux + kubernetes.io/os: linux # Adjust based on your available nodes tolerations: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" effect: "NoSchedule" -======= - resources: - requests: - cpu: 0.5 - memory: 1Gi - limits: - cpu: 1 - memory: 1Gi - env: - - name: AWS_REGION - value: ${region} - -replicas: 2 nodeSelector: kubernetes.io/os: linux @@ -118,42 +58,18 @@ tolerations: - key: node-role.kubernetes.io/control-plane operator: Exists effect: NoSchedule ->>>>>>> bc776f3 (templatlize) -======= ->>>>>>> 4e10249 (fewer things) -======= - env: - - name: AWS_REGION - value: ${region} ->>>>>>> fe6559f (add a few missing values) affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: -<<<<<<< HEAD - matchExpressions: - key: karpenter.sh/nodepool operator: DoesNotExist - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - topologyKey: "kubernetes.io/hostname" -======= - - matchExpressions: - - key: karpenter.sh/nodepool - operator: DoesNotExist - key: eks.amazonaws.com/nodegroup -<<<<<<< HEAD operator: Exists ->>>>>>> bc776f3 (templatlize) -======= - operator: In values: - ${karpenter_node_group_name} -<<<<<<< HEAD ->>>>>>> fe6559f (add a few missing values) -======= podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" ->>>>>>> 8bb3bc4 (update values) From 81a8b5662a93a7a80cc1eeef88a9745d678d28de Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 12:56:25 -0500 Subject: [PATCH 10/51] remove digest --- karpenter/values.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index f346b60..b6a6a0b 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -40,7 +40,7 @@ controller: - name: AWS_REGION value: ${region} nodeSelector: - kubernetes.io/os: linux # Adjust based on your available nodes + kubernetes.io/os: linux tolerations: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" From 449243a29f32b4b14b808a9983a70cb1d231dd37 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 13:33:12 -0500 Subject: [PATCH 11/51] node group --- karpenter/values.yaml.tpl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index b6a6a0b..2a64150 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -67,9 +67,8 @@ affinity: - key: karpenter.sh/nodepool operator: DoesNotExist - key: eks.amazonaws.com/nodegroup - operator: Exists - values: - - ${karpenter_node_group_name} + operator: In + values: ${karpenter_node_group_name} podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" From b0c7ec8a7b3e6a2d252b64a98701c31625ed9e73 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 13:36:50 -0500 Subject: [PATCH 12/51] fmt --- main.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.tf b/main.tf index ea7daf6..b327ff4 100644 --- a/main.tf +++ b/main.tf @@ -66,8 +66,7 @@ resource "helm_release" "karpenter" { repository = format("%v/%v", module.images.images[local.karpenter_key].dest_registry, module.images.images[local.karpenter_key].dest_repository) - tag = module.images.images[local.karpenter_key].tag - # digest = data.aws_ecr_image.karpenter_image.id + tag = module.images.images[local.karpenter_key].tag region = var.region iam_role_arn = module.karpenter_resources.iam_role_arn }) From 0d06502fee2393f1014d8b39bcd8fb2b0b2018c5 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 13:47:31 -0500 Subject: [PATCH 13/51] syntax and startupTaints --- karpenter-resources/values.yaml.tpl | 3 --- karpenter/values.yaml.tpl | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 1a5a180..8604e10 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -51,9 +51,6 @@ nodePool: weight: 1 labels: {} taints: [] - startupTaints: - - key: ebs.csi.aws.com/agent-not-ready - effect: NoExecute requirements: - key: "kubernetes.io/arch" operator: "In" diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 2a64150..64723c3 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -68,7 +68,7 @@ affinity: operator: DoesNotExist - key: eks.amazonaws.com/nodegroup operator: In - values: ${karpenter_node_group_name} + values: ["${karpenter_node_group_name}"] podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" From 2b4650433e0b63eb1088d68503c6592464f27fe4 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 14:42:14 -0500 Subject: [PATCH 14/51] fix syntax --- karpenter/values.yaml.tpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 64723c3..31ad710 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -68,7 +68,8 @@ affinity: operator: DoesNotExist - key: eks.amazonaws.com/nodegroup operator: In - values: ["${karpenter_node_group_name}"] + values: + - ${karpenter_node_group_name} podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" From 69e8502f7283f74c562a64c0d877deb689e49112 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 14:50:21 -0500 Subject: [PATCH 15/51] update values --- karpenter/values.yaml.tpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 31ad710..64723c3 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -68,8 +68,7 @@ affinity: operator: DoesNotExist - key: eks.amazonaws.com/nodegroup operator: In - values: - - ${karpenter_node_group_name} + values: ["${karpenter_node_group_name}"] podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" From 7c5fa11826d2e19c75ac63334abfc59437de419a Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 14:54:51 -0500 Subject: [PATCH 16/51] remove ng selector --- karpenter/values.yaml.tpl | 3 --- 1 file changed, 3 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 64723c3..2a24463 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -66,9 +66,6 @@ affinity: - matchExpressions: - key: karpenter.sh/nodepool operator: DoesNotExist - - key: eks.amazonaws.com/nodegroup - operator: In - values: ["${karpenter_node_group_name}"] podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" From 9c910c31449be01ac8fadc311920b39d266134ba Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 14:59:02 -0500 Subject: [PATCH 17/51] lower cpu requests --- karpenter/values.yaml.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 2a24463..b355f52 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -31,10 +31,10 @@ controller: digest: ${digest} resources: requests: - cpu: 0.5 + cpu: 100m memory: 1Gi limits: - cpu: 1 + cpu: 500m memory: 1Gi env: - name: AWS_REGION From ec02f16cf406cc1643f6a38aaaa9658930fd3ab2 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:06:42 -0500 Subject: [PATCH 18/51] use digest again --- main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/main.tf b/main.tf index b327ff4..1efeaba 100644 --- a/main.tf +++ b/main.tf @@ -66,6 +66,7 @@ resource "helm_release" "karpenter" { repository = format("%v/%v", module.images.images[local.karpenter_key].dest_registry, module.images.images[local.karpenter_key].dest_repository) + digest = data.aws_ecr_image.karpenter_image.id tag = module.images.images[local.karpenter_key].tag region = var.region iam_role_arn = module.karpenter_resources.iam_role_arn From 116fdff6689d30b4931a1a5e9bcfc33153e78ec0 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:18:25 -0500 Subject: [PATCH 19/51] simplify repository --- main.tf | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/main.tf b/main.tf index 1efeaba..c74d202 100644 --- a/main.tf +++ b/main.tf @@ -63,13 +63,11 @@ resource "helm_release" "karpenter" { karpenter_node_group_name = var.karpenter_node_group_name karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name queue_name = module.karpenter_resources.queue_name - repository = format("%v/%v", - module.images.images[local.karpenter_key].dest_registry, - module.images.images[local.karpenter_key].dest_repository) - digest = data.aws_ecr_image.karpenter_image.id - tag = module.images.images[local.karpenter_key].tag - region = var.region - iam_role_arn = module.karpenter_resources.iam_role_arn + repository = module.images.images[local.karpenter_key].dest_repository + digest = data.aws_ecr_image.karpenter_image.id + tag = module.images.images[local.karpenter_key].tag + region = var.region + iam_role_arn = module.karpenter_resources.iam_role_arn }) ] From 671a726cf7fe39123fed015b4af7f26c69165c24 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:30:08 -0500 Subject: [PATCH 20/51] fix repository --- main.tf | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/main.tf b/main.tf index c74d202..1eecbc0 100644 --- a/main.tf +++ b/main.tf @@ -63,11 +63,14 @@ resource "helm_release" "karpenter" { karpenter_node_group_name = var.karpenter_node_group_name karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name queue_name = module.karpenter_resources.queue_name - repository = module.images.images[local.karpenter_key].dest_repository - digest = data.aws_ecr_image.karpenter_image.id - tag = module.images.images[local.karpenter_key].tag - region = var.region - iam_role_arn = module.karpenter_resources.iam_role_arn + repository = format("%v/%v", + module.images.images[local.karpenter_key].dest_registry, + module.images.images[local.karpenter_key].dest_repository + ) + digest = data.aws_ecr_image.karpenter_image.id + tag = module.images.images[local.karpenter_key].tag + region = var.region + iam_role_arn = module.karpenter_resources.iam_role_arn }) ] From 8c8cbf17928b8e194bc7f5ccb52642fa3592f76b Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:37:22 -0500 Subject: [PATCH 21/51] escape the . --- karpenter-resources/values.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 8604e10..5fb2180 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -11,7 +11,7 @@ ec2NodeClass: subnetSelectorTerms: - tags: Name: ${subnets_name} - "kubernetes.io/cluster/${cluster_name}": "shared" + "kubernetes\\.io/cluster/${cluster_name}": "shared" securityGroupSelectorTerms: - tags: Name: "${cluster_name}-node" From 762af8a2e71f64294ea95242182574cf727a5d72 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:42:39 -0500 Subject: [PATCH 22/51] fix subnets_name --- karpenter-resources/values.yaml.tpl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 5fb2180..dc6ca1c 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -10,7 +10,7 @@ ec2NodeClass: - alias: "bottlerocket@latest" subnetSelectorTerms: - tags: - Name: ${subnets_name} + Name: "${subnets_name}" "kubernetes\\.io/cluster/${cluster_name}": "shared" securityGroupSelectorTerms: - tags: @@ -51,6 +51,9 @@ nodePool: weight: 1 labels: {} taints: [] + startupTaints: + - key: ebs.csi.aws.com/agent-not-ready + effect: NoExecute requirements: - key: "kubernetes.io/arch" operator: "In" From bc4911c2a44f91c947d7a78fc2c242cce98de3d3 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:51:36 -0500 Subject: [PATCH 23/51] subnet selector --- karpenter-resources/values.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index dc6ca1c..6ba5387 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -11,7 +11,7 @@ ec2NodeClass: subnetSelectorTerms: - tags: Name: "${subnets_name}" - "kubernetes\\.io/cluster/${cluster_name}": "shared" + "kubernetes.io/cluster/${cluster_name}": "shared" securityGroupSelectorTerms: - tags: Name: "${cluster_name}-node" From 91b59f7fc5bcde6ba0921010aaf79811899cd1aa Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 16:18:41 -0500 Subject: [PATCH 24/51] use the outputs from karpenter module --- main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index 1eecbc0..55e7103 100644 --- a/main.tf +++ b/main.tf @@ -61,7 +61,7 @@ resource "helm_release" "karpenter" { ) tag = module.images.images[local.karpenter_key].tag karpenter_node_group_name = var.karpenter_node_group_name - karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name + karpenter_node_profile = module.karpenter_resources.instance_profile_name queue_name = module.karpenter_resources.queue_name repository = format("%v/%v", module.images.images[local.karpenter_key].dest_registry, @@ -70,7 +70,7 @@ resource "helm_release" "karpenter" { digest = data.aws_ecr_image.karpenter_image.id tag = module.images.images[local.karpenter_key].tag region = var.region - iam_role_arn = module.karpenter_resources.iam_role_arn + iam_role_arn = module.karpenter_resources.node_iam_role_arn }) ] @@ -90,7 +90,7 @@ resource "helm_release" "nodepool_resources" { values = [ templatefile("${path.module}/karpenter-resources/values.yaml.tpl", { cluster_name = var.cluster_name - karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name + karpenter_node_profile = module.karpenter_resources.instance_profile_name amd_ami_family = local.amd_ami_family amd_ami_alias = local.amd_ami_alias subnets_name = var.subnets_name From 38dab0b92d7fa1c84cec94b6ac1f924c23efd24e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 16:27:26 -0500 Subject: [PATCH 25/51] create instance profile true --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 55e7103..b8777ff 100644 --- a/main.tf +++ b/main.tf @@ -11,7 +11,7 @@ module "karpenter_resources" { cluster_name = var.cluster_name enable_irsa = true irsa_oidc_provider_arn = var.oidc_provider_arn - irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] + irsa_namespace_service_accounts = ["${local.namespace}:karpenter"] create_pod_identity_association = true enable_v1_permissions = true create_instance_profile = true From 687523ddebd62caa580aa623607c24b5270d82d1 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 17:29:10 -0500 Subject: [PATCH 26/51] cleanup --- karpenter-resources/values.yaml.tpl | 10 ++++------ karpenter/values.yaml.tpl | 6 ------ main.tf | 15 ++------------- 3 files changed, 6 insertions(+), 25 deletions(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 6ba5387..f998e20 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -5,9 +5,9 @@ clusterName: ${cluster_name} # EC2NodeClass specific values ec2NodeClass: enabled: true - amiFamily: Bottlerocket + amiFamily: ${amd_ami_family} amiSelectorTerms: - - alias: "bottlerocket@latest" + - alias: ${amd_ami_alias} subnetSelectorTerms: - tags: Name: "${subnets_name}" @@ -34,17 +34,15 @@ ec2NodeClass: # NodePool specific values nodePool: - name: "${cluster_name}-karpenter-node-pool" + name: "${cluster_name}-worker-node-pool" enabled: true nodeClassRef: - name: "${cluster_name}-karpenter-node-class" + name: "${cluster_name}-worker-node-class" group: karpenter.k8s.aws kind: EC2NodeClass disruption: consolidationPolicy: "WhenEmptyOrUnderutilized" consolidateAfter: "5m" - budgets: - maxUnavailable: 1 limits: cpu: "1000" memory: 1000Gi diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index b355f52..5b98152 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -14,12 +14,6 @@ serviceAccount: replicas: 2 -podDisruptionBudget: - name: karpenter - maxUnavailable: 1 -podSecurityContext: - fsGroup: 65532 -priorityClassName: system-cluster-critical serviceMonitor: enabled: true logLevel: info diff --git a/main.tf b/main.tf index b8777ff..6f43c9e 100644 --- a/main.tf +++ b/main.tf @@ -60,17 +60,6 @@ resource "helm_release" "karpenter" { module.images.images[local.karpenter_key].dest_repository ) tag = module.images.images[local.karpenter_key].tag - karpenter_node_group_name = var.karpenter_node_group_name - karpenter_node_profile = module.karpenter_resources.instance_profile_name - queue_name = module.karpenter_resources.queue_name - repository = format("%v/%v", - module.images.images[local.karpenter_key].dest_registry, - module.images.images[local.karpenter_key].dest_repository - ) - digest = data.aws_ecr_image.karpenter_image.id - tag = module.images.images[local.karpenter_key].tag - region = var.region - iam_role_arn = module.karpenter_resources.node_iam_role_arn }) ] @@ -89,10 +78,10 @@ resource "helm_release" "nodepool_resources" { values = [ templatefile("${path.module}/karpenter-resources/values.yaml.tpl", { + amd_ami_alias = local.amd_ami_alias + amd_ami_family = local.amd_ami_family cluster_name = var.cluster_name karpenter_node_profile = module.karpenter_resources.instance_profile_name - amd_ami_family = local.amd_ami_family - amd_ami_alias = local.amd_ami_alias subnets_name = var.subnets_name tags = var.tags }) From e49cb0dca55aeccdbd1cdbbe97ab59a9cd8aaf5a Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 17:59:20 -0500 Subject: [PATCH 27/51] without cluster tag --- karpenter-resources/values.yaml.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index f998e20..c4004b1 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -11,7 +11,6 @@ ec2NodeClass: subnetSelectorTerms: - tags: Name: "${subnets_name}" - "kubernetes.io/cluster/${cluster_name}": "shared" securityGroupSelectorTerms: - tags: Name: "${cluster_name}-node" From 64c71a1e89b4b61e71dd2e3b10010c8bd1c2a573 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 18:27:19 -0500 Subject: [PATCH 28/51] pass in vpc --- karpenter-resources/values.yaml.tpl | 1 + main.tf | 1 + variables.tf | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index c4004b1..747ba17 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -8,6 +8,7 @@ ec2NodeClass: amiFamily: ${amd_ami_family} amiSelectorTerms: - alias: ${amd_ami_alias} + vpcId: ${vpc_id} subnetSelectorTerms: - tags: Name: "${subnets_name}" diff --git a/main.tf b/main.tf index 6f43c9e..bca2130 100644 --- a/main.tf +++ b/main.tf @@ -84,6 +84,7 @@ resource "helm_release" "nodepool_resources" { karpenter_node_profile = module.karpenter_resources.instance_profile_name subnets_name = var.subnets_name tags = var.tags + vpc_id = var.vpc_id }) ] } diff --git a/variables.tf b/variables.tf index 388139d..72bd4d8 100644 --- a/variables.tf +++ b/variables.tf @@ -65,3 +65,8 @@ variable "namespace" { type = string default = "namespace" } + +variable "vpc_id" { + description = "The ID of the VPC where the resources will be deployed" + type = string +} From a7d4d443c71db71a8a2bb948e2d7f72664932502 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 19:10:09 -0500 Subject: [PATCH 29/51] enable debug and disable serviceMonitor --- karpenter/values.yaml.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 5b98152..be083c0 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -15,8 +15,8 @@ serviceAccount: replicas: 2 serviceMonitor: - enabled: true -logLevel: info + enabled: false +logLevel: debug controller: image: From 4ef14ef3c38ae55bd03a5be7106a6877fa96edb2 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 11 Mar 2025 18:18:04 -0400 Subject: [PATCH 30/51] fix merge issues --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index bca2130..638cfdc 100644 --- a/main.tf +++ b/main.tf @@ -11,7 +11,7 @@ module "karpenter_resources" { cluster_name = var.cluster_name enable_irsa = true irsa_oidc_provider_arn = var.oidc_provider_arn - irsa_namespace_service_accounts = ["${local.namespace}:karpenter"] + irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] create_pod_identity_association = true enable_v1_permissions = true create_instance_profile = true From 7e9c2b68a23abb7eeb0ea22dbef916d6e0d9dc36 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 21 Mar 2025 20:25:39 -0400 Subject: [PATCH 31/51] update resources and set eksControlPlane true --- karpenter/values.yaml.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index be083c0..3228846 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -3,6 +3,7 @@ settings: clusterEndpoint: ${cluster_endpoint} interruptionQueue: ${queue_name} isolatedVPC: true + eksControlPlane: true featureGates: nodeRepair: true reservedCapacity: true From 92bb9caa3a579061be98f8074097a84dff7b0eae Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 16:15:10 -0400 Subject: [PATCH 32/51] try enable_pod_identity --- main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/main.tf b/main.tf index 638cfdc..42f48e9 100644 --- a/main.tf +++ b/main.tf @@ -14,6 +14,7 @@ module "karpenter_resources" { irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] create_pod_identity_association = true enable_v1_permissions = true + enable_pod_identity = true create_instance_profile = true # Attach additional IAM policies to the Karpenter node IAM role From 7652208aebc2cb1f4f30bdfa5c1c312c18dced1f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 16:16:22 -0400 Subject: [PATCH 33/51] update updated upstream --- README.md | 2 +- main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 31d8199..373e59f 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Source | Version | |------|--------|---------| | [images](#module\_images) | git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/ | tf-upgrade | -| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | v20.33.1 | +| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | v20.34.0 | ## Resources diff --git a/main.tf b/main.tf index 42f48e9..6b11879 100644 --- a/main.tf +++ b/main.tf @@ -6,7 +6,7 @@ locals { # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter # Karpenter IRSA roles and policies module "karpenter_resources" { - source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.33.1" + source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.34.0" cluster_name = var.cluster_name enable_irsa = true From 6ca575663c4c298fa0a44f210fa25617e2149217 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 16:27:38 -0400 Subject: [PATCH 34/51] more options for namings --- main.tf | 4 +++- prefixes.tf | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 prefixes.tf diff --git a/main.tf b/main.tf index 6b11879..0abe99f 100644 --- a/main.tf +++ b/main.tf @@ -16,7 +16,9 @@ module "karpenter_resources" { enable_v1_permissions = true enable_pod_identity = true create_instance_profile = true - + create_access_entry = true + rule_name_prefix = format("%v%v%v", local.prefixes["eks-policy"], var.cluster_name, "karpenter") + node_iam_role_name = format("%v%v%v", local.prefixes["eks-role"], var.cluster_name, "karpenter-node-role") # Attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = format("arn:%v:iam::%v:%v", data.aws_arn.current.partition, "aws", "policy/AmazonSSMManagedInstanceCore") diff --git a/prefixes.tf b/prefixes.tf new file mode 100644 index 0000000..e9a6127 --- /dev/null +++ b/prefixes.tf @@ -0,0 +1,35 @@ +locals { + prefixes = { + "efs" = "v-efs-" + "s3" = "v-s3-" + "ebs" = "v-ebs-" + "kms" = "k-kms-" + "role" = "r-" + "policy" = "p-" + "group" = "g-" + "security-group" = "" # "sg-" + # VPC + "vpc" = "" + "dhcp-options" = "" + "vpc-peer" = "vpcp-" + "route-table" = "route-" + "subnet" = "" + "vpc-endpoint" = "vpce-" + "elastic-ip" = "eip-" + "nat-gateway" = "nat-" + "internet-gateway" = "igw-" + "network-acl" = "nacl-" + "customer-gateway" = "cgw-" + "vpn-gateway" = "vpcg-" + "vpn-connection" = "vpn_" + "log-group" = "lg-" + "log-stream" = "lgs-" + # EKS + "eks" = "eks-" + "eks-s3" = "v-s3-eks-" + "eks-user" = "s-eks-" + "eks-role" = "r-eks-" + "eks-policy" = "p-eks-" + "eks-security-group" = "eks-sg-" # "sg-eks-" + } +} From 1d20564371e7011f91bf4c832c58e481ab2470da Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 16:33:00 -0400 Subject: [PATCH 35/51] shorter --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 0abe99f..9b5f430 100644 --- a/main.tf +++ b/main.tf @@ -17,8 +17,8 @@ module "karpenter_resources" { enable_pod_identity = true create_instance_profile = true create_access_entry = true - rule_name_prefix = format("%v%v%v", local.prefixes["eks-policy"], var.cluster_name, "karpenter") - node_iam_role_name = format("%v%v%v", local.prefixes["eks-role"], var.cluster_name, "karpenter-node-role") + rule_name_prefix = format("%v%v-%v", local.prefixes["eks-policy"], var.cluster_name, "karpenter") + node_iam_role_name = format("%v%v-%v", local.prefixes["eks-role"], var.cluster_name, "karpenter") # Attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = format("arn:%v:iam::%v:%v", data.aws_arn.current.partition, "aws", "policy/AmazonSSMManagedInstanceCore") From 33b69674bd106891e011304eec4eb7172ff610f8 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 16:37:00 -0400 Subject: [PATCH 36/51] validate role name length --- main.tf | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 9b5f430..8175280 100644 --- a/main.tf +++ b/main.tf @@ -1,6 +1,14 @@ locals { amd_ami_family = "Bottlerocket" amd_ami_alias = "bottlerocket@latest" + + # Calculate the role name with prefix + role_name_raw = format("%v%v-%v", local.prefixes["eks-role"], var.cluster_name, "karpenter-node-") + + # Ensure role name stays within AWS limits (38 chars) when used as name_prefix + # (needs to account for the hyphen that gets added) + max_role_name_length = 37 # 38 - 1 for the hyphen added by name_prefix + node_iam_role_name = length(local.role_name_raw) > local.max_role_name_length ? substr(local.role_name_raw, 0, local.max_role_name_length) : local.role_name_raw } # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter @@ -17,8 +25,8 @@ module "karpenter_resources" { enable_pod_identity = true create_instance_profile = true create_access_entry = true - rule_name_prefix = format("%v%v-%v", local.prefixes["eks-policy"], var.cluster_name, "karpenter") - node_iam_role_name = format("%v%v-%v", local.prefixes["eks-role"], var.cluster_name, "karpenter") + rule_name_prefix = format("%v%v-%v", local.prefixes["eks-policy"], var.cluster_name, "karpenter-") + node_iam_role_name = local.node_iam_role_name # Attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = format("arn:%v:iam::%v:%v", data.aws_arn.current.partition, "aws", "policy/AmazonSSMManagedInstanceCore") From 8a57be2e04a66479227bca37faa36c6244154fb9 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 17:08:22 -0400 Subject: [PATCH 37/51] validate event rules name length --- main.tf | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 8175280..22c2716 100644 --- a/main.tf +++ b/main.tf @@ -9,6 +9,13 @@ locals { # (needs to account for the hyphen that gets added) max_role_name_length = 37 # 38 - 1 for the hyphen added by name_prefix node_iam_role_name = length(local.role_name_raw) > local.max_role_name_length ? substr(local.role_name_raw, 0, local.max_role_name_length) : local.role_name_raw + + # EventBridge rule name validation + # Leave enough space for suffixes like "SpotInterrupt" and unique IDs generated by AWS + # EventBridge has a max length of 64 chars + rule_name_raw = format("%v%v-%v", local.prefixes["eks-policy"], var.cluster_name, "karpenter") + max_rule_name_length = 30 # Conservative limit to allow for suffixes and unique IDs + rule_name_prefix = length(local.rule_name_raw) > local.max_rule_name_length ? substr(local.rule_name_raw, 0, local.max_rule_name_length) : local.rule_name_raw } # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter @@ -25,7 +32,7 @@ module "karpenter_resources" { enable_pod_identity = true create_instance_profile = true create_access_entry = true - rule_name_prefix = format("%v%v-%v", local.prefixes["eks-policy"], var.cluster_name, "karpenter-") + rule_name_prefix = local.rule_name_prefix node_iam_role_name = local.node_iam_role_name # Attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { From a10b32e4103af36d90ffe587c728ac65e19e071f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 17:22:20 -0400 Subject: [PATCH 38/51] shorter --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 22c2716..b25c04d 100644 --- a/main.tf +++ b/main.tf @@ -14,7 +14,7 @@ locals { # Leave enough space for suffixes like "SpotInterrupt" and unique IDs generated by AWS # EventBridge has a max length of 64 chars rule_name_raw = format("%v%v-%v", local.prefixes["eks-policy"], var.cluster_name, "karpenter") - max_rule_name_length = 30 # Conservative limit to allow for suffixes and unique IDs + max_rule_name_length = 28 # Conservative limit to allow for suffixes and unique IDs rule_name_prefix = length(local.rule_name_raw) > local.max_rule_name_length ? substr(local.rule_name_raw, 0, local.max_rule_name_length) : local.rule_name_raw } From 82c4bfbaca4eec36e47be3e1bc953aee5c65aac5 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 17:23:32 -0400 Subject: [PATCH 39/51] shorter still --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index b25c04d..11cc682 100644 --- a/main.tf +++ b/main.tf @@ -14,7 +14,7 @@ locals { # Leave enough space for suffixes like "SpotInterrupt" and unique IDs generated by AWS # EventBridge has a max length of 64 chars rule_name_raw = format("%v%v-%v", local.prefixes["eks-policy"], var.cluster_name, "karpenter") - max_rule_name_length = 28 # Conservative limit to allow for suffixes and unique IDs + max_rule_name_length = 25 # Conservative limit to allow for suffixes and unique IDs rule_name_prefix = length(local.rule_name_raw) > local.max_rule_name_length ? substr(local.rule_name_raw, 0, local.max_rule_name_length) : local.rule_name_raw } From b31af4f4461cf72e6a67321787b91c255ddbc5c9 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 17:27:02 -0400 Subject: [PATCH 40/51] fix prefix --- main.tf | 2 +- prefixes.tf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 11cc682..c8923be 100644 --- a/main.tf +++ b/main.tf @@ -13,7 +13,7 @@ locals { # EventBridge rule name validation # Leave enough space for suffixes like "SpotInterrupt" and unique IDs generated by AWS # EventBridge has a max length of 64 chars - rule_name_raw = format("%v%v-%v", local.prefixes["eks-policy"], var.cluster_name, "karpenter") + rule_name_raw = format("%v%v", local.prefixes["event-rule"], var.cluster_name) max_rule_name_length = 25 # Conservative limit to allow for suffixes and unique IDs rule_name_prefix = length(local.rule_name_raw) > local.max_rule_name_length ? substr(local.rule_name_raw, 0, local.max_rule_name_length) : local.rule_name_raw } diff --git a/prefixes.tf b/prefixes.tf index e9a6127..1031f3b 100644 --- a/prefixes.tf +++ b/prefixes.tf @@ -31,5 +31,6 @@ locals { "eks-role" = "r-eks-" "eks-policy" = "p-eks-" "eks-security-group" = "eks-sg-" # "sg-eks-" + "event-rule" = "eks-ev-" } } From d03299a12ba056f0ee9bae5baee0410f1e934f55 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 17:35:58 -0400 Subject: [PATCH 41/51] name the queue --- main.tf | 16 +++++++++------- prefixes.tf | 1 + 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/main.tf b/main.tf index c8923be..cb1b026 100644 --- a/main.tf +++ b/main.tf @@ -16,6 +16,7 @@ locals { rule_name_raw = format("%v%v", local.prefixes["event-rule"], var.cluster_name) max_rule_name_length = 25 # Conservative limit to allow for suffixes and unique IDs rule_name_prefix = length(local.rule_name_raw) > local.max_rule_name_length ? substr(local.rule_name_raw, 0, local.max_rule_name_length) : local.rule_name_raw + queue_name = format("%v%v", local.prefixes["eks-queue"], var.cluster_name) } # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter @@ -24,16 +25,17 @@ module "karpenter_resources" { source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.34.0" cluster_name = var.cluster_name - enable_irsa = true - irsa_oidc_provider_arn = var.oidc_provider_arn - irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] + create_access_entry = true + create_instance_profile = true create_pod_identity_association = true - enable_v1_permissions = true + enable_irsa = true enable_pod_identity = true - create_instance_profile = true - create_access_entry = true - rule_name_prefix = local.rule_name_prefix + enable_v1_permissions = true + irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] + irsa_oidc_provider_arn = var.oidc_provider_arn node_iam_role_name = local.node_iam_role_name + queue_name = local.queue_name + rule_name_prefix = local.rule_name_prefix # Attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = format("arn:%v:iam::%v:%v", data.aws_arn.current.partition, "aws", "policy/AmazonSSMManagedInstanceCore") diff --git a/prefixes.tf b/prefixes.tf index 1031f3b..51073d7 100644 --- a/prefixes.tf +++ b/prefixes.tf @@ -32,5 +32,6 @@ locals { "eks-policy" = "p-eks-" "eks-security-group" = "eks-sg-" # "sg-eks-" "event-rule" = "eks-ev-" + "eks-queue" = "eks-q-" } } From 591720cf25b9d80e3e1338f6522d2e6a6baf4bbb Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 17:53:32 -0400 Subject: [PATCH 42/51] 66 - 3 = 63, limit 64 --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index cb1b026..6a8ec49 100644 --- a/main.tf +++ b/main.tf @@ -14,7 +14,7 @@ locals { # Leave enough space for suffixes like "SpotInterrupt" and unique IDs generated by AWS # EventBridge has a max length of 64 chars rule_name_raw = format("%v%v", local.prefixes["event-rule"], var.cluster_name) - max_rule_name_length = 25 # Conservative limit to allow for suffixes and unique IDs + max_rule_name_length = 22 rule_name_prefix = length(local.rule_name_raw) > local.max_rule_name_length ? substr(local.rule_name_raw, 0, local.max_rule_name_length) : local.rule_name_raw queue_name = format("%v%v", local.prefixes["eks-queue"], var.cluster_name) } From c37722a00d6a40db567599321d70aeba8db01bd3 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 17:55:55 -0400 Subject: [PATCH 43/51] longest event name is InstanceStateChange --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 6a8ec49..8af723c 100644 --- a/main.tf +++ b/main.tf @@ -14,7 +14,7 @@ locals { # Leave enough space for suffixes like "SpotInterrupt" and unique IDs generated by AWS # EventBridge has a max length of 64 chars rule_name_raw = format("%v%v", local.prefixes["event-rule"], var.cluster_name) - max_rule_name_length = 22 + max_rule_name_length = 8 rule_name_prefix = length(local.rule_name_raw) > local.max_rule_name_length ? substr(local.rule_name_raw, 0, local.max_rule_name_length) : local.rule_name_raw queue_name = format("%v%v", local.prefixes["eks-queue"], var.cluster_name) } From d07da3cf368c90570fd2e52f6769051a210f59de Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 21:54:07 -0400 Subject: [PATCH 44/51] another --- main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index 8af723c..902b7ab 100644 --- a/main.tf +++ b/main.tf @@ -3,11 +3,10 @@ locals { amd_ami_alias = "bottlerocket@latest" # Calculate the role name with prefix - role_name_raw = format("%v%v-%v", local.prefixes["eks-role"], var.cluster_name, "karpenter-node-") + role_name_raw = format("%v%v-%v", local.prefixes["eks-role"], var.cluster_name, "karpenter-") # Ensure role name stays within AWS limits (38 chars) when used as name_prefix - # (needs to account for the hyphen that gets added) - max_role_name_length = 37 # 38 - 1 for the hyphen added by name_prefix + max_role_name_length = 37 node_iam_role_name = length(local.role_name_raw) > local.max_role_name_length ? substr(local.role_name_raw, 0, local.max_role_name_length) : local.role_name_raw # EventBridge rule name validation @@ -31,6 +30,7 @@ module "karpenter_resources" { enable_irsa = true enable_pod_identity = true enable_v1_permissions = true + iam_role_name_prefix = local.role_name_raw irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] irsa_oidc_provider_arn = var.oidc_provider_arn node_iam_role_name = local.node_iam_role_name From 47cf922a29db1144fa1be068a6b49617ddd8813f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 22:02:01 -0400 Subject: [PATCH 45/51] maybe --- main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 902b7ab..cc2655b 100644 --- a/main.tf +++ b/main.tf @@ -26,11 +26,12 @@ module "karpenter_resources" { cluster_name = var.cluster_name create_access_entry = true create_instance_profile = true + create_node_iam_role = true create_pod_identity_association = true enable_irsa = true enable_pod_identity = true enable_v1_permissions = true - iam_role_name_prefix = local.role_name_raw + role_name_prefix = local.role_name_raw irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] irsa_oidc_provider_arn = var.oidc_provider_arn node_iam_role_name = local.node_iam_role_name From a76b71e2efe1ae842e116c2dd378270019b1c7c0 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 22:08:08 -0400 Subject: [PATCH 46/51] name --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index cc2655b..4298750 100644 --- a/main.tf +++ b/main.tf @@ -31,7 +31,7 @@ module "karpenter_resources" { enable_irsa = true enable_pod_identity = true enable_v1_permissions = true - role_name_prefix = local.role_name_raw + iam_role_name = format("%v%v", local.role_name_raw, "role") irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] irsa_oidc_provider_arn = var.oidc_provider_arn node_iam_role_name = local.node_iam_role_name From 6a49954ed39664f544a4891dc58b808fb5f0c31f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 22:11:09 -0400 Subject: [PATCH 47/51] name_prefix --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 4298750..d0b7bae 100644 --- a/main.tf +++ b/main.tf @@ -31,7 +31,7 @@ module "karpenter_resources" { enable_irsa = true enable_pod_identity = true enable_v1_permissions = true - iam_role_name = format("%v%v", local.role_name_raw, "role") + name_prefix = local.role_name_raw irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] irsa_oidc_provider_arn = var.oidc_provider_arn node_iam_role_name = local.node_iam_role_name From c6d8a706d20a2b50d29afbd49d7a57cc6041710e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 24 Mar 2025 22:12:15 -0400 Subject: [PATCH 48/51] or_not --- main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/main.tf b/main.tf index d0b7bae..efb76da 100644 --- a/main.tf +++ b/main.tf @@ -31,7 +31,6 @@ module "karpenter_resources" { enable_irsa = true enable_pod_identity = true enable_v1_permissions = true - name_prefix = local.role_name_raw irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] irsa_oidc_provider_arn = var.oidc_provider_arn node_iam_role_name = local.node_iam_role_name From bb56bad216fb0725f064baa11d4cfeeec680472b Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 1 Apr 2025 11:48:44 -0400 Subject: [PATCH 49/51] add module release process --- .github/workflows/terraform-release.yaml | 73 ++++++++++++++++ .github/workflows/terraform-validate.yaml | 42 +++++++++ .github/workflows/terragrunt-cicd.yml | 101 ---------------------- 3 files changed, 115 insertions(+), 101 deletions(-) create mode 100644 .github/workflows/terraform-release.yaml create mode 100644 .github/workflows/terraform-validate.yaml delete mode 100644 .github/workflows/terragrunt-cicd.yml diff --git a/.github/workflows/terraform-release.yaml b/.github/workflows/terraform-release.yaml new file mode 100644 index 0000000..90910bc --- /dev/null +++ b/.github/workflows/terraform-release.yaml @@ -0,0 +1,73 @@ +name: Terraform CI/CD +on: + workflow_dispatch: + pull_request: + types: [closed] + branches: + - main +jobs: + terraform-ci-cd: + runs-on: 229685449397 + permissions: + contents: write + + steps: + - name: Checkout code + uses: CSVD/gh-actions-checkout@v4 + + - name: Setup Terraform + uses: CSVD/gh-actions-setup-terraform@v3 + with: + terraform_version: "1.9.1" + + - name: Setup GITHUB Credentials + id: github_credentials + uses: CSVD/gh-auth@main + with: + github_app_pem_file: ${{ secrets.GH_APP_PEM_FILE }} + github_app_installation_id: ${{ vars.GH_APP_INSTALLATION_ID }} + github_app_id: ${{ vars.GH_APP_ID }} + + + - name: Debug Authentication + run: | + # Print the GitHub server URL + echo "GitHub Server URL: ${{ github.server_url }}" + + # Extract the host from the URL + HOST="${{ github.server_url }}" + HOST="${HOST#*//}" + HOST="${HOST%%/*}" + echo "GitHub Host: $HOST" + + # Check if token exists + if [[ -n "${{ steps.github_credentials.outputs.github_token }}" ]]; then + echo "Token generated successfully" + # Test the token with a simple GitHub API call (without exposing the token) + STATUS=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer ${{ steps.github_credentials.outputs.github_token }}" "${{ github.server_url }}/api/v3/user") + echo "API Test Status Code: $STATUS" + else + echo "No token was generated!" + fi + + - name: Setup GitHub CLI + run: | + # Force manual authentication since setup-git might not work with GitHub Enterprise + echo "${{ steps.github_credentials.outputs.github_token }}" > /tmp/token.txt + gh auth login --with-token --hostname "github.e.it.census.gov" < /tmp/token.txt + rm /tmp/token.txt + + # Test GitHub CLI auth status + gh auth status || echo "GitHub CLI authentication failed" + + - name: AWS Auth + id: aws_auth + uses: CSVD/aws-auth@main + with: + ecs: true + + - name: Run Terraform Module Release Action + uses: CSVD/terraform-module-release@main + with: + github-token: ${{ steps.github_credentials.outputs.github_token }} + working-directory: '.' diff --git a/.github/workflows/terraform-validate.yaml b/.github/workflows/terraform-validate.yaml new file mode 100644 index 0000000..72829d8 --- /dev/null +++ b/.github/workflows/terraform-validate.yaml @@ -0,0 +1,42 @@ +name: Terraform Validate +on: + pull_request: + workflow_dispatch: + +jobs: + + terraform-validate: + runs-on: "229685449397" + permissions: + contents: write + steps: + - name: Checkout code + uses: CSVD/gh-actions-checkout@v4 + + - name: Setup Terraform + uses: CSVD/gh-actions-setup-terraform@v2 + with: + terraform_version: '1.7.3' + + - name: Validate Terraform Configuration + id: validate + uses: CSVD/terraform-validate@main + + - name: Check Validation/Test Results + if: always() + run: | + # Set default values if outputs are empty + IS_VALID="${{ steps.validate.outputs.is_valid }}" + TESTS_PASSED="${{ steps.validate.outputs.tests_passed }}" + + # If outputs are empty, set them to false + [ -z "$IS_VALID" ] && IS_VALID="false" + [ -z "$TESTS_PASSED" ] && TESTS_PASSED="false" + + if [[ "$IS_VALID" != "true" || "$TESTS_PASSED" != "true" ]]; then + echo "Validation or test errors found:" + echo "${{ steps.validate.outputs.stderr }}" + exit 1 + else + echo "All validations and tests passed successfully!" + fi diff --git a/.github/workflows/terragrunt-cicd.yml b/.github/workflows/terragrunt-cicd.yml deleted file mode 100644 index a78523e..0000000 --- a/.github/workflows/terragrunt-cicd.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: 'Terraform Module CI' - -on: - push: - branches: - - main - paths: - - '**/*.hcl' - - '**/*.tf' - pull_request: - branches: - - main - paths: - - '**/*.hcl' - - '**/*.tf' - -permissions: - contents: read - pull-requests: write - -jobs: - validate: - name: 'Validate Module' - runs-on: self-hosted - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 1.5.0 - - - name: Terraform Init - run: | - terraform init -backend=false - - - name: Terraform Format - run: | - terraform fmt -check - - - name: Terraform Validate - run: | - terraform validate - - - name: Run tflint - uses: terraform-linters/setup-tflint@v3 - if: github.event_name == 'pull_request' - - - name: Lint Terraform - if: github.event_name == 'pull_request' - run: | - tflint --format compact - - release: - name: 'Create Release' - needs: validate - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - runs-on: self-hosted - permissions: - contents: write - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Commitizen - run: | - pip install commitizen - - - name: Configure Git - run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - - - name: Bump Version and Generate Changelog - id: cz - run: | - cz bump --yes - echo "new_version=$(cz version --project)" >> $GITHUB_OUTPUT - echo "changelog=$(cz changelog --dry-run)" >> $GITHUB_OUTPUT - - - name: Create Release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: v${{ steps.cz.outputs.new_version }} - release_name: Release v${{ steps.cz.outputs.new_version }} - draft: false - prerelease: false - body: ${{ steps.cz.outputs.changelog }} From f203a8066a9660905608fb534616330cf5ea35dc Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 1 Apr 2025 18:13:32 -0400 Subject: [PATCH 50/51] update module source --- README.md | 4 ++-- copy_images.tf | 2 +- main.tf | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 373e59f..1def36b 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,8 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Source | Version | |------|--------|---------| -| [images](#module\_images) | git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/ | tf-upgrade | -| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | v20.34.0 | +| [images](#module\_images) | git::https://github.e.it.census.gov/terraform-modules/aws-ecr-copy-images.git/ | tf-upgrade | +| [karpenter\_resources](#module\_karpenter\_resources) | git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks.git//modules/karpenter | v20.35.0 | ## Resources diff --git a/copy_images.tf b/copy_images.tf index 8febc1b..7e90fc5 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -17,7 +17,7 @@ locals { # 224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter:0.37.0 # map[repository:224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter tag:0.37.0] module "images" { - source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" + source = "git::https://github.e.it.census.gov/terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" profile = var.profile application_name = var.cluster_name diff --git a/main.tf b/main.tf index efb76da..90b9b03 100644 --- a/main.tf +++ b/main.tf @@ -21,7 +21,7 @@ locals { # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter # Karpenter IRSA roles and policies module "karpenter_resources" { - source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.34.0" + source = "git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.35.0" cluster_name = var.cluster_name create_access_entry = true From 37fe374704e4ae4d5f60f580d87aea91d2b3496f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 3 Apr 2025 13:06:25 -0400 Subject: [PATCH 51/51] update release actions --- .github/workflows/terraform-release.yaml | 37 ++---------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/.github/workflows/terraform-release.yaml b/.github/workflows/terraform-release.yaml index 90910bc..04b96db 100644 --- a/.github/workflows/terraform-release.yaml +++ b/.github/workflows/terraform-release.yaml @@ -15,11 +15,6 @@ jobs: - name: Checkout code uses: CSVD/gh-actions-checkout@v4 - - name: Setup Terraform - uses: CSVD/gh-actions-setup-terraform@v3 - with: - terraform_version: "1.9.1" - - name: Setup GITHUB Credentials id: github_credentials uses: CSVD/gh-auth@main @@ -28,28 +23,6 @@ jobs: github_app_installation_id: ${{ vars.GH_APP_INSTALLATION_ID }} github_app_id: ${{ vars.GH_APP_ID }} - - - name: Debug Authentication - run: | - # Print the GitHub server URL - echo "GitHub Server URL: ${{ github.server_url }}" - - # Extract the host from the URL - HOST="${{ github.server_url }}" - HOST="${HOST#*//}" - HOST="${HOST%%/*}" - echo "GitHub Host: $HOST" - - # Check if token exists - if [[ -n "${{ steps.github_credentials.outputs.github_token }}" ]]; then - echo "Token generated successfully" - # Test the token with a simple GitHub API call (without exposing the token) - STATUS=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer ${{ steps.github_credentials.outputs.github_token }}" "${{ github.server_url }}/api/v3/user") - echo "API Test Status Code: $STATUS" - else - echo "No token was generated!" - fi - - name: Setup GitHub CLI run: | # Force manual authentication since setup-git might not work with GitHub Enterprise @@ -60,14 +33,8 @@ jobs: # Test GitHub CLI auth status gh auth status || echo "GitHub CLI authentication failed" - - name: AWS Auth - id: aws_auth - uses: CSVD/aws-auth@main - with: - ecs: true - - - name: Run Terraform Module Release Action - uses: CSVD/terraform-module-release@main + - name: Run Release Action + uses: CSVD/releaser@main with: github-token: ${{ steps.github_credentials.outputs.github_token }} working-directory: '.'