From cc9251f3099202e117b5f7c38eec659bda3113c5 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 28 Feb 2025 18:09:51 -0500 Subject: [PATCH 01/33] autoscaling --- .github/workflows/terragrunt-cicd.yml | 101 ++++++++++++++++++++++++++ karpenter-resources/values.yaml | 7 +- 2 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/terragrunt-cicd.yml diff --git a/.github/workflows/terragrunt-cicd.yml b/.github/workflows/terragrunt-cicd.yml new file mode 100644 index 0000000..a78523e --- /dev/null +++ b/.github/workflows/terragrunt-cicd.yml @@ -0,0 +1,101 @@ +name: 'Terraform Module CI' + +on: + push: + branches: + - main + paths: + - '**/*.hcl' + - '**/*.tf' + pull_request: + branches: + - main + paths: + - '**/*.hcl' + - '**/*.tf' + +permissions: + contents: read + pull-requests: write + +jobs: + validate: + name: 'Validate Module' + runs-on: self-hosted + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.5.0 + + - name: Terraform Init + run: | + terraform init -backend=false + + - name: Terraform Format + run: | + terraform fmt -check + + - name: Terraform Validate + run: | + terraform validate + + - name: Run tflint + uses: terraform-linters/setup-tflint@v3 + if: github.event_name == 'pull_request' + + - name: Lint Terraform + if: github.event_name == 'pull_request' + run: | + tflint --format compact + + release: + name: 'Create Release' + needs: validate + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + runs-on: self-hosted + permissions: + contents: write + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install Commitizen + run: | + pip install commitizen + + - name: Configure Git + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + - name: Bump Version and Generate Changelog + id: cz + run: | + cz bump --yes + echo "new_version=$(cz version --project)" >> $GITHUB_OUTPUT + echo "changelog=$(cz changelog --dry-run)" >> $GITHUB_OUTPUT + + - name: Create Release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ steps.cz.outputs.new_version }} + release_name: Release v${{ steps.cz.outputs.new_version }} + draft: false + prerelease: false + body: ${{ steps.cz.outputs.changelog }} diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index cc9886d..d228918 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -59,11 +59,6 @@ nodePool: - key: karpenter.sh/capacity-type operator: In values: ["on-demand", "spot"] - - key: karpenter.k8s.aws/instance-generation - operator: Gt - values: ["2"] - - key: "karpenter.k8s.aws/instance-hypervisor" - operator: In - values: ["nitro"] + expireAfter: "72h" terminationGracePeriod: "4h" From d2b761fd0c4087181280500ebb23538a0217785f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 20:07:43 -0500 Subject: [PATCH 02/33] add startup taint --- .tflint.hcl | 26 ++++++++++---------- README.md | 2 +- karpenter-resources/templates/node-pool.yaml | 4 +++ karpenter-resources/values.yaml | 3 +++ main.tf | 8 +++--- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/.tflint.hcl b/.tflint.hcl index 684d807..ab8ea66 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -4,18 +4,18 @@ config { disabled_by_default = false } -rule "aws_instance_invalid_type" { - enabled = true -} +# rule "aws_instance_invalid_type" { +# enabled = true +# } -plugin "aws" { - enabled = true - version = "0.32.0" - source = "github.com/terraform-linters/tflint-ruleset-aws" -} +# plugin "aws" { +# enabled = true +# version = "0.32.0" +# source = "github.com/terraform-linters/tflint-ruleset-aws" +# } -plugin "terraform" { - enabled = true - version = "0.9.0" - source = "github.com/terraform-linters/tflint-ruleset-terraform" -} +# plugin "terraform" { +# enabled = true +# version = "0.9.0" +# source = "github.com/terraform-linters/tflint-ruleset-terraform" +# } diff --git a/README.md b/README.md index 09d0981..53091e5 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Version | |------|---------| -| [aws](#provider\_aws) | 5.84.0 | +| [aws](#provider\_aws) | 5.89.0 | | [helm](#provider\_helm) | 2.17.0 | ## Modules diff --git a/karpenter-resources/templates/node-pool.yaml b/karpenter-resources/templates/node-pool.yaml index ac8e368..347a194 100644 --- a/karpenter-resources/templates/node-pool.yaml +++ b/karpenter-resources/templates/node-pool.yaml @@ -25,6 +25,10 @@ spec: taints: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.nodePool.startupTaints }} + startupTaints: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodePool.requirements }} requirements: {{- toYaml . | nindent 8 }} diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index d228918..148bc87 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -49,6 +49,9 @@ nodePool: weight: 1 labels: {} taints: [] + startupTaints: + - key: ebs.csi.aws.com/agent-not-ready + effect: NoExecute requirements: - key: "kubernetes.io/arch" operator: "In" diff --git a/main.tf b/main.tf index 59c8cd0..f929c81 100644 --- a/main.tf +++ b/main.tf @@ -173,10 +173,10 @@ resource "helm_release" "nodepool_resources" { name = "ec2NodeClass.subnetSelectorTerms[0].tags.Name" value = var.subnets_name } - # set { - # name = "ec2NodeClass.subnetSelectorTerms[0].tags.\"kubernetes.io/cluster/${var.cluster_name}\"" - # value = "shared" - # } + set { + name = "ec2NodeClass.subnetSelectorTerms[0].tags.kubernetes\\.io/cluster/${var.cluster_name}" + value = "shared" + } ### Node Class Inputs set { name = "nodePool.nodeClassRef.name" From 75c3c16e2d0270a06d53092611ef8ea0da894c5e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 20:28:27 -0500 Subject: [PATCH 03/33] add namespace and startup taint to template --- README.md | 1 + karpenter-resources/values.yaml | 1 - main.tf | 4 ++-- variables.tf | 6 ++++++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 53091e5..2f7cb1b 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [karpenter\_helm\_repo](#input\_karpenter\_helm\_repo) | Helm repo for official karpenter chart | `string` | `"oci://public.ecr.aws/karpenter"` | no | | [karpenter\_node\_group\_name](#input\_karpenter\_node\_group\_name) | The cluster node group that will host karpenter, should not be a karpenter managed node group | `string` | n/a | yes | | [karpenter\_tag](#input\_karpenter\_tag) | Which tag of karpenter image | `string` | `"1.1.1"` | no | +| [namespace](#input\_namespace) | Namespace for Karpenter | `string` | `"namespace"` | no | | [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes | | [profile](#input\_profile) | AWS config profile | `string` | `""` | no | | [region](#input\_region) | AWS region | `string` | n/a | yes | diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 148bc87..29f2394 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -62,6 +62,5 @@ nodePool: - key: karpenter.sh/capacity-type operator: In values: ["on-demand", "spot"] - expireAfter: "72h" terminationGracePeriod: "4h" diff --git a/main.tf b/main.tf index f929c81..b291dfd 100644 --- a/main.tf +++ b/main.tf @@ -55,7 +55,7 @@ resource "helm_release" "karpenter" { namespace = local.namespace description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" repository = var.karpenter_helm_repo - create_namespace = true + create_namespace = false set { name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key" @@ -180,7 +180,7 @@ resource "helm_release" "nodepool_resources" { ### Node Class Inputs set { name = "nodePool.nodeClassRef.name" - value = "${var.cluster_name}-node-class" + value = "${var.cluster_name}-karpenter-node-class" } set { name = "nodePool.name" diff --git a/variables.tf b/variables.tf index 85e1724..98c54a8 100644 --- a/variables.tf +++ b/variables.tf @@ -65,3 +65,9 @@ variable "tags" { type = map(string) default = {} } + +variable "namespace" { + description = "Namespace for Karpenter" + type = string + default = "namespace" +} From bc776f39e7eb208a452eca62835e17c57b2b6902 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 22:33:41 -0500 Subject: [PATCH 04/33] templatlize --- .../{values.yaml => values.yaml.tpl} | 28 +-- karpenter/values.yaml.tpl | 50 ++++++ main.tf | 160 +++--------------- 3 files changed, 93 insertions(+), 145 deletions(-) rename karpenter-resources/{values.yaml => values.yaml.tpl} (67%) create mode 100644 karpenter/values.yaml.tpl diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml.tpl similarity index 67% rename from karpenter-resources/values.yaml rename to karpenter-resources/values.yaml.tpl index 29f2394..9dc3841 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml.tpl @@ -1,20 +1,21 @@ # Shared values -name: default -clusterName: test-cluster +name: ${cluster_name} +clusterName: ${cluster_name} # EC2NodeClass specific values ec2NodeClass: enabled: true - amiFamily: bottlerocket + amiFamily: Bottlerocket amiSelectorTerms: - - alias: "Bottlerocket@latest" + - alias: "bottlerocket@latest" subnetSelectorTerms: - tags: - Name: subnet-name + Name: ${subnets_name} + "kubernetes.io/cluster/${cluster_name}": "shared" securityGroupSelectorTerms: - tags: - Name: security-group-name - karpenterRole: karpenter-role + Name: "${cluster_name}-node" + karpenterRole: ${karpenter_node_profile_name} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: enabled @@ -27,14 +28,16 @@ ec2NodeClass: volumeType: gp3 detailedMonitoring: true instanceStorePolicy: RAID0 - + tags: + # These would be populated from var.tags in Terraform + "karpenter.sh/discovery": "${cluster_name}" # NodePool specific values nodePool: - name: example-cluster-node-pool + name: "${cluster_name}-karpenter-node-pool" enabled: true nodeClassRef: - name: default + name: "${cluster_name}-karpenter-node-class" group: karpenter.k8s.aws kind: EC2NodeClass disruption: @@ -42,7 +45,6 @@ nodePool: consolidateAfter: "5m" budgets: maxUnavailable: 1 - maxUnavailablePercentage: 10 limits: cpu: "1000" memory: 1000Gi @@ -64,3 +66,7 @@ nodePool: values: ["on-demand", "spot"] expireAfter: "72h" terminationGracePeriod: "4h" + +# Extra value used in node-class.yaml template +nodeClass: + cluster_name: ${cluster_name} diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl new file mode 100644 index 0000000..c54a0bf --- /dev/null +++ b/karpenter/values.yaml.tpl @@ -0,0 +1,50 @@ +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: ${iam_role_arn} + +settings: + clusterName: ${cluster_name} + clusterEndpoint: ${cluster_endpoint} + interruptionQueue: ${queue_name} + +aws: + defaultInstanceProfile: ${karpenter_node_group_name} + +controller: + image: + repository: ${repository} + tag: ${tag} + digest: ${digest} + resources: + requests: + cpu: 0.5 + memory: 1Gi + limits: + cpu: 1 + memory: 1Gi + env: + - name: AWS_REGION + value: ${region} + +replicas: 2 + +nodeSelector: + kubernetes.io/os: linux + +tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/nodegroup + operator: Exists diff --git a/main.tf b/main.tf index b291dfd..f8d620d 100644 --- a/main.tf +++ b/main.tf @@ -1,5 +1,4 @@ locals { - namespace = var.namespace amd_ami_family = "Bottlerocket" amd_ami_alias = "bottlerocket@latest" } @@ -52,82 +51,27 @@ resource "helm_release" "karpenter" { name = "karpenter" chart = "karpenter" version = var.karpenter_helm_chart - namespace = local.namespace + namespace = var.namespace description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" repository = var.karpenter_helm_repo create_namespace = false - set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key" - value = "karpenter.sh/nodepool" - } - set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator" - value = "DoesNotExist" - } - set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].key" - value = "eks.amazonaws.com/nodegroup" - } - set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].operator" - value = "In" - } - set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].values[0]" - value = var.karpenter_node_group_name - } - set { - name = "affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey" - value = "kubernetes.io/hostname" - } - set { - name = "settings.clusterName" - value = var.cluster_name - } - set { - name = "settings.clusterEndpoint" - value = var.cluster_endpoint - } - set { - name = "settings.featureGates.spotToSpotConsolidation" - value = "true" - } - set { - name = "settings.interruptionQueue" - value = module.karpenter_resources.queue_name - } - set { - name = "settings.isolatedVPC" - value = "true" - } - set { - name = "controller.image.repository" - value = format("%v/%v", - module.images.images[local.karpenter_key].dest_registry, - module.images.images[local.karpenter_key].dest_repository - ) - } - set { - name = "controller.image.tag" - value = module.images.images[local.karpenter_key].tag - } - set { - name = "controller.image.digest" - value = data.aws_ecr_image.karpenter_image.id - } - set { - name = "controller.env[0].name" - value = "AWS_REGION" - } - set { - name = "controller.env[0].value" - value = var.region - } - set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.karpenter_resources.iam_role_arn - } + values = [ + templatefile("${path.module}/karpenter/values.yaml.tpl", { + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + karpenter_node_group_name = var.karpenter_node_group_name + queue_name = module.karpenter_resources.queue_name + repository = format("%v/%v", + module.images.images[local.karpenter_key].dest_registry, + module.images.images[local.karpenter_key].dest_repository) + tag = module.images.images[local.karpenter_key].tag + digest = data.aws_ecr_image.karpenter_image.id + region = var.region + iam_role_arn = module.karpenter_resources.iam_role_arn + }) + ] + timeout = 300 } @@ -139,68 +83,16 @@ resource "helm_release" "nodepool_resources" { description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" chart = "./karpenter-resources" version = local.module_version - namespace = local.namespace + namespace = var.namespace values = [ - file("${path.module}/karpenter-resources/values.yaml") + templatefile("${path.module}/karpenter-resources/values.yaml.tpl", { + cluster_name = var.cluster_name + karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name + amd_ami_family = local.amd_ami_family + amd_ami_alias = local.amd_ami_alias + subnets_name = var.subnets_name + tags = var.tags + }) ] - #### Default values overrides - set { - name = "name" - value = var.cluster_name - } - set { - name = "clusterName" - value = var.cluster_name - } - set { - name = "ec2NodeClass.karpenterRole" - value = aws_iam_instance_profile.karpenter_node.name - } - set { - name = "ec2NodeClass.amiFamily" - value = local.amd_ami_family - } - set { - name = "ec2NodeClass.amiSelectorTerms[0].alias" - value = local.amd_ami_alias - } - set { - name = "ec2NodeClass.securityGroupSelectorTerms[0].tags.Name" - value = "${var.cluster_name}-node" - } - set { - name = "ec2NodeClass.subnetSelectorTerms[0].tags.Name" - value = var.subnets_name - } - set { - name = "ec2NodeClass.subnetSelectorTerms[0].tags.kubernetes\\.io/cluster/${var.cluster_name}" - value = "shared" - } - ### Node Class Inputs - set { - name = "nodePool.nodeClassRef.name" - value = "${var.cluster_name}-karpenter-node-class" - } - set { - name = "nodePool.name" - value = "${var.cluster_name}-karpenter-node-pool" - } - set { - name = "nodeClass.cluster_name" - value = var.cluster_name - } - set { - name = "nodePool.limits.cpu" - value = "1000" - } - set { - name = "nodePool.limits.memory" - value = "1000Gi" - } - set { - name = "ec2NodeClass.tags" - value = jsonencode(var.tags) - } - } From 151997f804eb3c836b18521bbff00deabde30ead Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 22:45:24 -0500 Subject: [PATCH 05/33] fix vals --- karpenter-resources/values.yaml.tpl | 2 +- karpenter/values.yaml.tpl | 2 +- main.tf | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 9dc3841..1a5a180 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -15,7 +15,7 @@ ec2NodeClass: securityGroupSelectorTerms: - tags: Name: "${cluster_name}-node" - karpenterRole: ${karpenter_node_profile_name} + karpenterRole: ${karpenter_node_profile} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: enabled diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index c54a0bf..f92f538 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -8,7 +8,7 @@ settings: interruptionQueue: ${queue_name} aws: - defaultInstanceProfile: ${karpenter_node_group_name} + defaultInstanceProfile: ${karpenter_node_profile} controller: image: diff --git a/main.tf b/main.tf index f8d620d..b177b6c 100644 --- a/main.tf +++ b/main.tf @@ -61,6 +61,7 @@ resource "helm_release" "karpenter" { cluster_name = var.cluster_name cluster_endpoint = var.cluster_endpoint karpenter_node_group_name = var.karpenter_node_group_name + karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name queue_name = module.karpenter_resources.queue_name repository = format("%v/%v", module.images.images[local.karpenter_key].dest_registry, From 4e10249da1fc2434733759fadd6fb608d37f2f6d Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 23:14:53 -0500 Subject: [PATCH 06/33] fewer things --- karpenter/values.yaml.tpl | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index f92f538..6ce3b98 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -1,7 +1,3 @@ -serviceAccount: - annotations: - eks.amazonaws.com/role-arn: ${iam_role_arn} - settings: clusterName: ${cluster_name} clusterEndpoint: ${cluster_endpoint} @@ -15,31 +11,6 @@ controller: repository: ${repository} tag: ${tag} digest: ${digest} - resources: - requests: - cpu: 0.5 - memory: 1Gi - limits: - cpu: 1 - memory: 1Gi - env: - - name: AWS_REGION - value: ${region} - -replicas: 2 - -nodeSelector: - kubernetes.io/os: linux - -tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule affinity: nodeAffinity: From fe6559faa0e5084e5ecad9e5a5d0b57e91b6ee51 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 23:41:40 -0500 Subject: [PATCH 07/33] add a few missing values --- karpenter/values.yaml.tpl | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 6ce3b98..dc2fbc6 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -2,20 +2,32 @@ settings: clusterName: ${cluster_name} clusterEndpoint: ${cluster_endpoint} interruptionQueue: ${queue_name} + isolatedVPC: true + featureGates: + nodeRepair: true + spotToSpotConsolidation: true -aws: - defaultInstanceProfile: ${karpenter_node_profile} +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: ${iam_role_arn} controller: image: repository: ${repository} tag: ${tag} digest: ${digest} + env: + - name: AWS_REGION + value: ${region} affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: + - key: karpenter.sh/nodepool + operator: DoesNotExist - key: eks.amazonaws.com/nodegroup - operator: Exists + operator: In + values: + - ${karpenter_node_group_name} From ceb85f544843056aca3f640dbbb5fdfbedfe518a Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 4 Mar 2025 23:56:35 -0500 Subject: [PATCH 08/33] create ns --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index b177b6c..58be0e6 100644 --- a/main.tf +++ b/main.tf @@ -54,7 +54,7 @@ resource "helm_release" "karpenter" { namespace = var.namespace description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" repository = var.karpenter_helm_repo - create_namespace = false + create_namespace = true values = [ templatefile("${path.module}/karpenter/values.yaml.tpl", { From 8bb3bc4bd247c1535caae76d8fee623dc92ff4f4 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 12:49:55 -0500 Subject: [PATCH 09/33] update values --- karpenter/full_karpenter_values.yml | 195 ++++++++++++++++++++++++++++ karpenter/values.yaml.tpl | 25 +++- main.tf | 9 +- 3 files changed, 224 insertions(+), 5 deletions(-) create mode 100644 karpenter/full_karpenter_values.yml diff --git a/karpenter/full_karpenter_values.yml b/karpenter/full_karpenter_values.yml new file mode 100644 index 0000000..1b6995c --- /dev/null +++ b/karpenter/full_karpenter_values.yml @@ -0,0 +1,195 @@ +# -- Overrides the chart's name. +nameOverride: "" +# -- Overrides the chart's computed fullname. +fullnameOverride: "" +# -- Additional labels to add into metadata. +additionalLabels: {} +# app: karpenter + +# -- Additional annotations to add into metadata. +additionalAnnotations: {} +# -- Image pull policy for Docker images. +imagePullPolicy: IfNotPresent +# -- Image pull secrets for Docker images. +imagePullSecrets: [] +service: + # -- Additional annotations for the Service. + annotations: {} +serviceAccount: + # -- Specifies if a ServiceAccount should be created. + create: true + # -- The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template. + name: "" + # -- Additional annotations for the ServiceAccount. + annotations: {} +# -- Specifies additional rules for the core ClusterRole. +additionalClusterRoleRules: [] +serviceMonitor: + # -- Specifies whether a ServiceMonitor should be created. + enabled: false + # -- Additional labels for the ServiceMonitor. + additionalLabels: {} + # -- Configuration on `http-metrics` endpoint for the ServiceMonitor. + # Not to be used to add additional endpoints. + # See the Prometheus operator documentation for configurable fields https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint + endpointConfig: {} +# -- Number of replicas. +replicas: 2 +# -- The number of old ReplicaSets to retain to allow rollback. +revisionHistoryLimit: 10 +# -- Strategy for updating the pod. +strategy: + rollingUpdate: + maxUnavailable: 1 +# -- Additional labels for the pod. +podLabels: {} +# -- Additional annotations for the pod. +podAnnotations: {} +podDisruptionBudget: + name: karpenter + maxUnavailable: 1 +# -- SecurityContext for the pod. +podSecurityContext: + fsGroup: 65532 +# -- PriorityClass name for the pod. +priorityClassName: system-cluster-critical +# -- Override the default termination grace period for the pod. +terminationGracePeriodSeconds: +# -- Bind the pod to the host network. +# This is required when using a custom CNI. +hostNetwork: false +# -- Specify which Kubernetes scheduler should dispatch the pod. +schedulerName: default-scheduler +# -- Configure the DNS Policy for the pod +dnsPolicy: ClusterFirst +# -- Configure DNS Config for the pod +dnsConfig: {} +# options: +# - name: ndots +# value: "1" +# -- Node selectors to schedule the pod to nodes with labels. +nodeSelector: + kubernetes.io/os: linux +# -- Affinity rules for scheduling the pod. If an explicit label selector is not provided for pod affinity or pod anti-affinity one will be created from the pod selector labels. +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: karpenter.sh/nodepool + operator: DoesNotExist + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" +# -- Topology spread constraints to increase the controller resilience by distributing pods across the cluster zones. If an explicit label selector is not provided one will be created from the pod selector labels. +topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule +# -- Tolerations to allow the pod to be scheduled to nodes with taints. +tolerations: + - key: CriticalAddonsOnly + operator: Exists +# -- Additional volumes for the pod. +extraVolumes: [] +# - name: aws-iam-token +# projected: +# defaultMode: 420 +# sources: +# - serviceAccountToken: +# audience: sts.amazonaws.com +# expirationSeconds: 86400 +# path: token +controller: + # -- Distinguishing container name (containerName: karpenter-controller). + containerName: controller + image: + # -- Repository path to the controller image. + repository: public.ecr.aws/karpenter/controller + # -- Tag of the controller image. + tag: 1.3.0 + # -- SHA256 digest of the controller image. + digest: sha256:23876d27c39f4b99ad41ee245319fc2a2fa499183131e9cfccb550658d003045 + # -- Additional environment variables for the controller pod. + env: [] + # - name: AWS_REGION + # value: eu-west-1 + envFrom: [] + # -- Resources for the controller pod. + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 1 + # memory: 1Gi + + # -- Additional volumeMounts for the controller pod. + extraVolumeMounts: [] + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # -- Additional sidecarContainer config + sidecarContainer: [] + # -- Additional volumeMounts for the sidecar - this will be added to the volume mounts on top of extraVolumeMounts + sidecarVolumeMounts: [] + metrics: + # -- The container port to use for metrics. + port: 8080 + healthProbe: + # -- The container port to use for http health probe. + port: 8081 +# -- Global log level, defaults to 'info' +logLevel: info +# -- Log outputPaths - defaults to stdout only +logOutputPaths: + - stdout +# -- Log errorOutputPaths - defaults to stderr only +logErrorOutputPaths: + - stderr +# -- Global Settings to configure Karpenter +settings: + # -- The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one + # time which usually results in fewer but larger nodes. + batchMaxDuration: 10s + # -- The maximum amount of time with no new ending pods that if exceeded ends the current batching window. If pods arrive + # faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods + # will be batched separately. + batchIdleDuration: 1s + # -- Cluster CA bundle for TLS configuration of provisioned nodes. If not set, this is taken from the controller's TLS configuration for the API server. + clusterCABundle: "" + # -- Cluster name. + clusterName: "" + # -- Cluster endpoint. If not set, will be discovered during startup (EKS only) + clusterEndpoint: "" + # -- If true then assume we can't reach AWS services which don't have a VPC endpoint + # This also has the effect of disabling look-ups to the AWS pricing endpoint + isolatedVPC: false + # Marking this true means that your cluster is running with an EKS control plane and Karpenter should attempt to discover cluster details from the DescribeCluster API + eksControlPlane: false + # -- The VM memory overhead as a percent that will be subtracted from the total memory for all instance types. The value of `0.075` equals to 7.5%. + vmMemoryOverheadPercent: 0.075 + # -- Interruption queue is the name of the SQS queue used for processing interruption events from EC2 + # Interruption handling is disabled if not specified. Enabling interruption handling may + # require additional permissions on the controller service account. Additional permissions are outlined in the docs. + interruptionQueue: "" + # -- Reserved ENIs are not included in the calculations for max-pods or kube-reserved + # This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html + reservedENIs: "0" + # -- Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates + # in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features + featureGates: + # -- nodeRepair is ALPHA and is disabled by default. + # Setting this to true will enable node repair. + nodeRepair: false + # -- reservedCapacity is ALPHA and is disabled by default. + # Setting this will enable native on-demand capacity reservation support. + reservedCapacity: false + # -- spotToSpotConsolidation is ALPHA and is disabled by default. + # Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation. + spotToSpotConsolidation: false diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index dc2fbc6..6b32133 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -5,17 +5,37 @@ settings: isolatedVPC: true featureGates: nodeRepair: true + reservedCapacity: true spotToSpotConsolidation: true serviceAccount: annotations: eks.amazonaws.com/role-arn: ${iam_role_arn} +replicas: 2 + +podDisruptionBudget: + name: karpenter + maxUnavailable: 1 +podSecurityContext: + fsGroup: 65532 +priorityClassName: system-cluster-critical +serviceMonitor: + enabled: true +logLevel: info + controller: + resources: + requests: + cpu: 1 + memory: 1Gi + limits: + cpu: 1 + memory: 1Gi image: repository: ${repository} tag: ${tag} - digest: ${digest} + # digest: ${digest} env: - name: AWS_REGION value: ${region} @@ -31,3 +51,6 @@ affinity: operator: In values: - ${karpenter_node_group_name} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" diff --git a/main.tf b/main.tf index 58be0e6..320d346 100644 --- a/main.tf +++ b/main.tf @@ -58,16 +58,17 @@ resource "helm_release" "karpenter" { values = [ templatefile("${path.module}/karpenter/values.yaml.tpl", { - cluster_name = var.cluster_name - cluster_endpoint = var.cluster_endpoint + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + karpenter_node_group_name = var.karpenter_node_group_name karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name queue_name = module.karpenter_resources.queue_name repository = format("%v/%v", module.images.images[local.karpenter_key].dest_registry, module.images.images[local.karpenter_key].dest_repository) - tag = module.images.images[local.karpenter_key].tag - digest = data.aws_ecr_image.karpenter_image.id + tag = module.images.images[local.karpenter_key].tag + # digest = data.aws_ecr_image.karpenter_image.id region = var.region iam_role_arn = module.karpenter_resources.iam_role_arn }) From 4703425277afbd35bc8f765702822e49c7428100 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 12:54:42 -0500 Subject: [PATCH 10/33] add nodeSelector and tolerations --- karpenter/values.yaml.tpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 6b32133..d1fca5b 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -39,6 +39,12 @@ controller: env: - name: AWS_REGION value: ${region} + nodeSelector: + kubernetes.io/os: linux # Adjust based on your available nodes + tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" affinity: nodeAffinity: From f0005dfdd9baae58d78db7ae89a0cd45c2f3d7d3 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 12:56:25 -0500 Subject: [PATCH 11/33] remove digest --- karpenter/values.yaml.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index d1fca5b..8f9b711 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -35,7 +35,6 @@ controller: image: repository: ${repository} tag: ${tag} - # digest: ${digest} env: - name: AWS_REGION value: ${region} From 1aee35ce6857a73193c8de8216311812770458fb Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 13:33:12 -0500 Subject: [PATCH 12/33] node group --- karpenter/values.yaml.tpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 8f9b711..a3e77df 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -54,8 +54,7 @@ affinity: operator: DoesNotExist - key: eks.amazonaws.com/nodegroup operator: In - values: - - ${karpenter_node_group_name} + values: ${karpenter_node_group_name} podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" From a5ef77b6fa5e8a3100d0565076b5118eb0bd5ea5 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 13:36:50 -0500 Subject: [PATCH 13/33] fmt --- main.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 320d346..bc1a389 100644 --- a/main.tf +++ b/main.tf @@ -67,8 +67,7 @@ resource "helm_release" "karpenter" { repository = format("%v/%v", module.images.images[local.karpenter_key].dest_registry, module.images.images[local.karpenter_key].dest_repository) - tag = module.images.images[local.karpenter_key].tag - # digest = data.aws_ecr_image.karpenter_image.id + tag = module.images.images[local.karpenter_key].tag region = var.region iam_role_arn = module.karpenter_resources.iam_role_arn }) From ed747760da40d2a6d72b53203566731741883e72 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 13:47:31 -0500 Subject: [PATCH 14/33] syntax and startupTaints --- karpenter-resources/values.yaml.tpl | 3 --- karpenter/values.yaml.tpl | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 1a5a180..8604e10 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -51,9 +51,6 @@ nodePool: weight: 1 labels: {} taints: [] - startupTaints: - - key: ebs.csi.aws.com/agent-not-ready - effect: NoExecute requirements: - key: "kubernetes.io/arch" operator: "In" diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index a3e77df..35409d4 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -54,7 +54,7 @@ affinity: operator: DoesNotExist - key: eks.amazonaws.com/nodegroup operator: In - values: ${karpenter_node_group_name} + values: ["${karpenter_node_group_name}"] podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" From cc134b4230d34c92332c2674c742e251b0e6174e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 14:42:14 -0500 Subject: [PATCH 15/33] fix syntax --- karpenter/values.yaml.tpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 35409d4..8f9b711 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -54,7 +54,8 @@ affinity: operator: DoesNotExist - key: eks.amazonaws.com/nodegroup operator: In - values: ["${karpenter_node_group_name}"] + values: + - ${karpenter_node_group_name} podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" From b3a45c8e37ee2ae8119f5cb488115113e1364c71 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 14:50:21 -0500 Subject: [PATCH 16/33] update values --- karpenter/values.yaml.tpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 8f9b711..35409d4 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -54,8 +54,7 @@ affinity: operator: DoesNotExist - key: eks.amazonaws.com/nodegroup operator: In - values: - - ${karpenter_node_group_name} + values: ["${karpenter_node_group_name}"] podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" From 960f663c0d6e3a3953836a8e8d73874cd0358188 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 14:54:51 -0500 Subject: [PATCH 17/33] remove ng selector --- karpenter/values.yaml.tpl | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 35409d4..4d64cf5 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -39,7 +39,7 @@ controller: - name: AWS_REGION value: ${region} nodeSelector: - kubernetes.io/os: linux # Adjust based on your available nodes + kubernetes.io/os: linux # Adjust based on your available nodes tolerations: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" @@ -49,12 +49,9 @@ affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: karpenter.sh/nodepool - operator: DoesNotExist - - key: eks.amazonaws.com/nodegroup - operator: In - values: ["${karpenter_node_group_name}"] + - matchExpressions: + - key: karpenter.sh/nodepool + operator: DoesNotExist podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" From b4dcd79c5eb191761a4c5f5001640a6f2e4d2925 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 14:59:02 -0500 Subject: [PATCH 18/33] lower cpu requests --- karpenter/values.yaml.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 4d64cf5..faab597 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -27,10 +27,10 @@ logLevel: info controller: resources: requests: - cpu: 1 + cpu: 1m memory: 1Gi limits: - cpu: 1 + cpu: 500m memory: 1Gi image: repository: ${repository} From d8785b2dc3c47d84a3e753bf3bc09498194d1f4e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:06:42 -0500 Subject: [PATCH 19/33] use digest again --- main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/main.tf b/main.tf index bc1a389..ac57cc3 100644 --- a/main.tf +++ b/main.tf @@ -67,6 +67,7 @@ resource "helm_release" "karpenter" { repository = format("%v/%v", module.images.images[local.karpenter_key].dest_registry, module.images.images[local.karpenter_key].dest_repository) + digest = data.aws_ecr_image.karpenter_image.id tag = module.images.images[local.karpenter_key].tag region = var.region iam_role_arn = module.karpenter_resources.iam_role_arn From 940948099f1f800e250da22620828f71596ebd2e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:18:25 -0500 Subject: [PATCH 20/33] simplify repository --- copy_images.tf | 8 ++++---- main.tf | 12 +++++------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/copy_images.tf b/copy_images.tf index 4c05c03..8febc1b 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -16,7 +16,6 @@ locals { # 224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter:0.37.0 # map[repository:224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter tag:0.37.0] - module "images" { source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" @@ -25,7 +24,8 @@ module "images" { image_config = local.image_config tags = {} - enable_lifecycle_policy = true - lifecycle_policy_all = true - force_delete = true + enable_lifecycle_policy = true + lifecycle_policy_all = true + force_delete = true + lifecycle_policy_keep_count = 3 } diff --git a/main.tf b/main.tf index ac57cc3..618516b 100644 --- a/main.tf +++ b/main.tf @@ -64,13 +64,11 @@ resource "helm_release" "karpenter" { karpenter_node_group_name = var.karpenter_node_group_name karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name queue_name = module.karpenter_resources.queue_name - repository = format("%v/%v", - module.images.images[local.karpenter_key].dest_registry, - module.images.images[local.karpenter_key].dest_repository) - digest = data.aws_ecr_image.karpenter_image.id - tag = module.images.images[local.karpenter_key].tag - region = var.region - iam_role_arn = module.karpenter_resources.iam_role_arn + repository = module.images.images[local.karpenter_key].dest_repository + digest = data.aws_ecr_image.karpenter_image.id + tag = module.images.images[local.karpenter_key].tag + region = var.region + iam_role_arn = module.karpenter_resources.iam_role_arn }) ] From d6586400a3ce80757692d6a6c8281b28d7bcdcfa Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:30:08 -0500 Subject: [PATCH 21/33] fix repository --- main.tf | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/main.tf b/main.tf index 618516b..3003636 100644 --- a/main.tf +++ b/main.tf @@ -64,11 +64,14 @@ resource "helm_release" "karpenter" { karpenter_node_group_name = var.karpenter_node_group_name karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name queue_name = module.karpenter_resources.queue_name - repository = module.images.images[local.karpenter_key].dest_repository - digest = data.aws_ecr_image.karpenter_image.id - tag = module.images.images[local.karpenter_key].tag - region = var.region - iam_role_arn = module.karpenter_resources.iam_role_arn + repository = format("%v/%v", + module.images.images[local.karpenter_key].dest_registry, + module.images.images[local.karpenter_key].dest_repository + ) + digest = data.aws_ecr_image.karpenter_image.id + tag = module.images.images[local.karpenter_key].tag + region = var.region + iam_role_arn = module.karpenter_resources.iam_role_arn }) ] From ca03492318384d8d3739b52baa7f6a5ab74c9269 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:34:41 -0500 Subject: [PATCH 22/33] add digest value --- karpenter/values.yaml.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index faab597..31b922d 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -35,6 +35,7 @@ controller: image: repository: ${repository} tag: ${tag} + digest: ${digest} env: - name: AWS_REGION value: ${region} From cfa779863579c5cd3e25f74bbd2d5bac05e4ae78 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:37:22 -0500 Subject: [PATCH 23/33] escape the . --- karpenter-resources/values.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 8604e10..5fb2180 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -11,7 +11,7 @@ ec2NodeClass: subnetSelectorTerms: - tags: Name: ${subnets_name} - "kubernetes.io/cluster/${cluster_name}": "shared" + "kubernetes\\.io/cluster/${cluster_name}": "shared" securityGroupSelectorTerms: - tags: Name: "${cluster_name}-node" From b3104800b82926108a44c6a21180d6d163bcd812 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:42:39 -0500 Subject: [PATCH 24/33] fix subnets_name --- karpenter-resources/values.yaml.tpl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 5fb2180..dc6ca1c 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -10,7 +10,7 @@ ec2NodeClass: - alias: "bottlerocket@latest" subnetSelectorTerms: - tags: - Name: ${subnets_name} + Name: "${subnets_name}" "kubernetes\\.io/cluster/${cluster_name}": "shared" securityGroupSelectorTerms: - tags: @@ -51,6 +51,9 @@ nodePool: weight: 1 labels: {} taints: [] + startupTaints: + - key: ebs.csi.aws.com/agent-not-ready + effect: NoExecute requirements: - key: "kubernetes.io/arch" operator: "In" From e1ccf85beb1ec3404d69cb7ce1666b632842f235 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 15:51:36 -0500 Subject: [PATCH 25/33] subnet selector --- karpenter-resources/values.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index dc6ca1c..6ba5387 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -11,7 +11,7 @@ ec2NodeClass: subnetSelectorTerms: - tags: Name: "${subnets_name}" - "kubernetes\\.io/cluster/${cluster_name}": "shared" + "kubernetes.io/cluster/${cluster_name}": "shared" securityGroupSelectorTerms: - tags: Name: "${cluster_name}-node" From 705ebc0c3c12b42f430b7512672d0902a657b69d Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 16:18:41 -0500 Subject: [PATCH 26/33] use the outputs from karpenter module --- README.md | 1 - main.tf | 20 +++----------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2f7cb1b..061a258 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Type | |------|------| -| [aws_iam_instance_profile.karpenter_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.nodepool_resources](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | diff --git a/main.tf b/main.tf index 3003636..b7180bb 100644 --- a/main.tf +++ b/main.tf @@ -21,20 +21,6 @@ module "karpenter_resources" { tags = var.tags } -# Create IAM Instance Profile for Karpenter Nodes -resource "aws_iam_instance_profile" "karpenter_node" { - name = "${var.cluster_name}-karpenter-node-profile" - role = module.karpenter_resources.node_iam_role_name - - tags = merge( - var.tags, - { - "karpenter.sh/managed-by" = var.cluster_name - "kubernetes.io/cluster/${var.cluster_name}" = "owned" - } - ) -} - # Install Karpenter data "aws_ecr_image" "karpenter_image" { depends_on = [ @@ -62,7 +48,7 @@ resource "helm_release" "karpenter" { cluster_endpoint = var.cluster_endpoint karpenter_node_group_name = var.karpenter_node_group_name - karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name + karpenter_node_profile = module.karpenter_resources.instance_profile_name queue_name = module.karpenter_resources.queue_name repository = format("%v/%v", module.images.images[local.karpenter_key].dest_registry, @@ -71,7 +57,7 @@ resource "helm_release" "karpenter" { digest = data.aws_ecr_image.karpenter_image.id tag = module.images.images[local.karpenter_key].tag region = var.region - iam_role_arn = module.karpenter_resources.iam_role_arn + iam_role_arn = module.karpenter_resources.node_iam_role_arn }) ] @@ -91,7 +77,7 @@ resource "helm_release" "nodepool_resources" { values = [ templatefile("${path.module}/karpenter-resources/values.yaml.tpl", { cluster_name = var.cluster_name - karpenter_node_profile = aws_iam_instance_profile.karpenter_node.name + karpenter_node_profile = module.karpenter_resources.instance_profile_name amd_ami_family = local.amd_ami_family amd_ami_alias = local.amd_ami_alias subnets_name = var.subnets_name From c992ddc43dce7ab5e6dacd3adc5ef2bf7eea5928 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 16:27:26 -0500 Subject: [PATCH 27/33] create instance profile true --- main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.tf b/main.tf index b7180bb..09ed892 100644 --- a/main.tf +++ b/main.tf @@ -12,6 +12,9 @@ module "karpenter_resources" { enable_irsa = true irsa_oidc_provider_arn = var.oidc_provider_arn irsa_namespace_service_accounts = ["${local.namespace}:karpenter"] + create_pod_identity_association = true + enable_v1_permissions = true + create_instance_profile = true # Attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { From 8e091d86fb8874e682228f7c45f5987e64ec522e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 17:29:10 -0500 Subject: [PATCH 28/33] cleanup --- karpenter-resources/values.yaml.tpl | 10 ++++------ karpenter/values.yaml.tpl | 8 +------- main.tf | 17 ++++++++--------- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 6ba5387..f998e20 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -5,9 +5,9 @@ clusterName: ${cluster_name} # EC2NodeClass specific values ec2NodeClass: enabled: true - amiFamily: Bottlerocket + amiFamily: ${amd_ami_family} amiSelectorTerms: - - alias: "bottlerocket@latest" + - alias: ${amd_ami_alias} subnetSelectorTerms: - tags: Name: "${subnets_name}" @@ -34,17 +34,15 @@ ec2NodeClass: # NodePool specific values nodePool: - name: "${cluster_name}-karpenter-node-pool" + name: "${cluster_name}-worker-node-pool" enabled: true nodeClassRef: - name: "${cluster_name}-karpenter-node-class" + name: "${cluster_name}-worker-node-class" group: karpenter.k8s.aws kind: EC2NodeClass disruption: consolidationPolicy: "WhenEmptyOrUnderutilized" consolidateAfter: "5m" - budgets: - maxUnavailable: 1 limits: cpu: "1000" memory: 1000Gi diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 31b922d..20026a0 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -14,12 +14,6 @@ serviceAccount: replicas: 2 -podDisruptionBudget: - name: karpenter - maxUnavailable: 1 -podSecurityContext: - fsGroup: 65532 -priorityClassName: system-cluster-critical serviceMonitor: enabled: true logLevel: info @@ -40,7 +34,7 @@ controller: - name: AWS_REGION value: ${region} nodeSelector: - kubernetes.io/os: linux # Adjust based on your available nodes + kubernetes.io/os: linux tolerations: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" diff --git a/main.tf b/main.tf index 09ed892..6f43c9e 100644 --- a/main.tf +++ b/main.tf @@ -47,20 +47,19 @@ resource "helm_release" "karpenter" { values = [ templatefile("${path.module}/karpenter/values.yaml.tpl", { - cluster_name = var.cluster_name - cluster_endpoint = var.cluster_endpoint - + cluster_endpoint = var.cluster_endpoint + cluster_name = var.cluster_name + digest = data.aws_ecr_image.karpenter_image.id + iam_role_arn = module.karpenter_resources.iam_role_arn karpenter_node_group_name = var.karpenter_node_group_name karpenter_node_profile = module.karpenter_resources.instance_profile_name queue_name = module.karpenter_resources.queue_name + region = var.region repository = format("%v/%v", module.images.images[local.karpenter_key].dest_registry, module.images.images[local.karpenter_key].dest_repository ) - digest = data.aws_ecr_image.karpenter_image.id - tag = module.images.images[local.karpenter_key].tag - region = var.region - iam_role_arn = module.karpenter_resources.node_iam_role_arn + tag = module.images.images[local.karpenter_key].tag }) ] @@ -79,10 +78,10 @@ resource "helm_release" "nodepool_resources" { values = [ templatefile("${path.module}/karpenter-resources/values.yaml.tpl", { + amd_ami_alias = local.amd_ami_alias + amd_ami_family = local.amd_ami_family cluster_name = var.cluster_name karpenter_node_profile = module.karpenter_resources.instance_profile_name - amd_ami_family = local.amd_ami_family - amd_ami_alias = local.amd_ami_alias subnets_name = var.subnets_name tags = var.tags }) From 0522914124eca14ccf8bbbda12797c92c8dc2cae Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 17:59:20 -0500 Subject: [PATCH 29/33] without cluster tag --- karpenter-resources/values.yaml.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index f998e20..c4004b1 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -11,7 +11,6 @@ ec2NodeClass: subnetSelectorTerms: - tags: Name: "${subnets_name}" - "kubernetes.io/cluster/${cluster_name}": "shared" securityGroupSelectorTerms: - tags: Name: "${cluster_name}-node" From 6be6c794f8ffd923228b9008708148a47199bcda Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 18:27:19 -0500 Subject: [PATCH 30/33] pass in vpc --- README.md | 1 + karpenter-resources/values.yaml.tpl | 1 + main.tf | 1 + variables.tf | 5 +++++ 4 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 061a258..31d8199 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [region](#input\_region) | AWS region | `string` | n/a | yes | | [subnets\_name](#input\_subnets\_name) | Define the name of the subnets to be used by this cluster | `string` | `"*-container-*"` | no | | [tags](#input\_tags) | AWS Tags to apply to appropriate resources | `map(string)` | `{}` | no | +| [vpc\_id](#input\_vpc\_id) | The ID of the VPC where the resources will be deployed | `string` | n/a | yes | ## Outputs diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index c4004b1..747ba17 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -8,6 +8,7 @@ ec2NodeClass: amiFamily: ${amd_ami_family} amiSelectorTerms: - alias: ${amd_ami_alias} + vpcId: ${vpc_id} subnetSelectorTerms: - tags: Name: "${subnets_name}" diff --git a/main.tf b/main.tf index 6f43c9e..bca2130 100644 --- a/main.tf +++ b/main.tf @@ -84,6 +84,7 @@ resource "helm_release" "nodepool_resources" { karpenter_node_profile = module.karpenter_resources.instance_profile_name subnets_name = var.subnets_name tags = var.tags + vpc_id = var.vpc_id }) ] } diff --git a/variables.tf b/variables.tf index 98c54a8..1f30170 100644 --- a/variables.tf +++ b/variables.tf @@ -71,3 +71,8 @@ variable "namespace" { type = string default = "namespace" } + +variable "vpc_id" { + description = "The ID of the VPC where the resources will be deployed" + type = string +} From b7aa5f1566da12f6139f40eefdfa6ab76a092861 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 5 Mar 2025 19:10:09 -0500 Subject: [PATCH 31/33] enable debug and disable serviceMonitor --- karpenter/values.yaml.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 20026a0..07a5a97 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -15,8 +15,8 @@ serviceAccount: replicas: 2 serviceMonitor: - enabled: true -logLevel: info + enabled: false +logLevel: debug controller: resources: From e2f7086833223b8ec9dc5bfeb0ef1425dc80f8aa Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 6 Mar 2025 20:09:12 -0500 Subject: [PATCH 32/33] lower resources --- karpenter/values.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 07a5a97..1c0b489 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -22,7 +22,7 @@ controller: resources: requests: cpu: 1m - memory: 1Gi + memory: 1Mi limits: cpu: 500m memory: 1Gi From c131b23df8227df5593f9a4b5d3472dd439f9d08 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 11 Mar 2025 18:18:04 -0400 Subject: [PATCH 33/33] fix merge issues --- main.tf | 2 +- variables.tf | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/main.tf b/main.tf index bca2130..638cfdc 100644 --- a/main.tf +++ b/main.tf @@ -11,7 +11,7 @@ module "karpenter_resources" { cluster_name = var.cluster_name enable_irsa = true irsa_oidc_provider_arn = var.oidc_provider_arn - irsa_namespace_service_accounts = ["${local.namespace}:karpenter"] + irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] create_pod_identity_association = true enable_v1_permissions = true create_instance_profile = true diff --git a/variables.tf b/variables.tf index 1f30170..72bd4d8 100644 --- a/variables.tf +++ b/variables.tf @@ -48,12 +48,6 @@ variable "karpenter_tag" { default = "1.1.1" } -variable "namespace" { - description = "Namespace in which to install cert manager" - type = string - default = "kube-system" -} - variable "subnets_name" { description = "Define the name of the subnets to be used by this cluster" type = string