From bd2365cbffd9e4a911ddfc96aa10c37108f6a1dd Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 28 Jan 2025 20:58:23 -0500 Subject: [PATCH 01/69] convert to helm charts --- .pre-commit-config.yaml | 2 +- README.md | 5 +-- karpenter/Chart.yaml | 4 ++ karpenter/templates/node_class.yaml | 45 ++++++++++++++++++++ karpenter/templates/node_pool.yaml | 38 +++++++++++++++++ karpenter/values.yaml | 65 +++++++++++++++++++++++++++++ main.tf | 54 ++++++++++++++++++++++++ node_class.tf => node_class.tf.off | 0 node_pool.tf => node_pool.tf.off | 0 requirements.tf | 8 ++-- 10 files changed, 212 insertions(+), 9 deletions(-) create mode 100644 karpenter/Chart.yaml create mode 100644 karpenter/templates/node_class.yaml create mode 100644 karpenter/templates/node_pool.yaml create mode 100644 karpenter/values.yaml rename node_class.tf => node_class.tf.off (100%) rename node_pool.tf => node_pool.tf.off (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85adacb..c93668a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: args: [--markdown-linebreak-ext=md] exclude: CHANGELOG.md - - id: check-yaml + # - id: check-yaml - id: check-merge-conflict - id: check-executables-have-shebangs diff --git a/README.md b/README.md index fd79403..68b6390 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [terraform](#requirement\_terraform) | >= 0.13 | | [aws](#requirement\_aws) | >= 5.14.0 | | [helm](#requirement\_helm) | >= 2.11.0 | -| [kubernetes](#requirement\_kubernetes) | >= 2.23.0 | ## Providers @@ -28,7 +27,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|---------| | [aws](#provider\_aws) | 5.84.0 | | [helm](#provider\_helm) | 2.17.0 | -| [kubernetes](#provider\_kubernetes) | 2.35.1 | ## Modules @@ -43,8 +41,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|------| | [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [kubernetes_manifest.karpenter_node_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | -| [kubernetes_manifest.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | +| [helm_release.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | diff --git a/karpenter/Chart.yaml b/karpenter/Chart.yaml new file mode 100644 index 0000000..5b9ab30 --- /dev/null +++ b/karpenter/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: karpenter +version: 0.1.0 +description: A Helm chart for Karpenter resources diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml new file mode 100644 index 0000000..1f7b22d --- /dev/null +++ b/karpenter/templates/node_class.yaml @@ -0,0 +1,45 @@ +apiVersion: {{ .Values.nodeClass.apiVersion }} +kind: EC2NodeClass +metadata: + name: {{ .Values.nodeClass.metadata.name }} + annotations: + kubernetes.io/description: "EC2NodeClass for running {{ .Values.nodeClass.metadata.annotations.description }}" +spec: + amiFamily: {{ .Values.nodeClass.spec.amiFamily }} + amiSelectorTerms: + - alias: {{ .Values.nodeClass.spec.amiSelectorTerms.alias }} + detailedMonitoring: {{ .Values.nodeClass.spec.detailedMonitoring }} + instanceProfile: {{ .Values.nodeClass.spec.instanceProfile }} + instanceStorePolicy: {{ .Values.nodeClass.spec.instanceStorePolicy }} + kubelet: + evictionHard: + memory.available: {{ .Values.nodeClass.spec.kubelet.evictionHard.memoryAvailable }} + nodefs.available: {{ .Values.nodeClass.spec.kubelet.evictionHard.nodefsAvailable }} + nodefs.inodesFree: {{ .Values.nodeClass.spec.kubelet.evictionHard.nodefsInodesFree }} + evictionMaxPodGracePeriod: {{ .Values.nodeClass.spec.kubelet.evictionMaxPodGracePeriod }} + evictionSoft: + memory.available: {{ .Values.nodeClass.spec.kubelet.evictionSoft.memoryAvailable }} + nodefs.available: {{ .Values.nodeClass.spec.kubelet.evictionSoft.nodefsAvailable }} + nodefs.inodesFree: {{ .Values.nodeClass.spec.kubelet.evictionSoft.nodefsInodesFree }} + evictionSoftGracePeriod: + memory.available: {{ .Values.nodeClass.spec.kubelet.evictionSoftGracePeriod.memoryAvailable }} + nodefs.available: {{ .Values.nodeClass.spec.kubelet.evictionSoftGracePeriod.nodefsAvailable }} + nodefs.inodesFree: {{ .Values.nodeClass.spec.kubelet.evictionSoftGracePeriod.nodefsInodesFree }} + imageGCHighThresholdPercent: {{ .Values.nodeClass.spec.kubelet.imageGCHighThresholdPercent }} + imageGCLowThresholdPercent: {{ .Values.nodeClass.spec.kubelet.imageGCLowThresholdPercent }} + metadataOptions: + httpEndpoint: {{ .Values.nodeClass.spec.metadataOptions.httpEndpoint }} + httpProtocolIPv6: {{ .Values.nodeClass.spec.metadataOptions.httpProtocolIPv6 }} + httpPutResponseHopLimit: {{ .Values.nodeClass.spec.metadataOptions.httpPutResponseHopLimit }} + httpTokens: {{ .Values.nodeClass.spec.metadataOptions.httpTokens }} + securityGroupSelectorTerms: + - tags: + name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} + subnetSelectorTerms: + - tags: + name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} + "kubernetes.io/cluster/{{ .Values.nodeClass.cluster_name }}": "shared" + tags: + {{- range $key, $value := .Values.nodeClass.spec.tags }} + {{ $key }}: {{ $value }} + {{- end }} diff --git a/karpenter/templates/node_pool.yaml b/karpenter/templates/node_pool.yaml new file mode 100644 index 0000000..e0f93c9 --- /dev/null +++ b/karpenter/templates/node_pool.yaml @@ -0,0 +1,38 @@ +apiVersion: {{ .Values.nodePool.apiVersion }} +kind: NodePool +metadata: + name: {{ .Values.nodePool.metadata.name }} + annotations: + kubernetes.io/description: General purpose NodePool for generic workloads +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: [amd64] + - key: kubernetes.io/os + operator: In + values: [linux] + - key: {{ .Values.nodePool.spec.template.spec.requirements.instanceHypervisorKey }} + operator: In + values: [nitro] + - key: {{ .Values.nodePool.spec.template.spec.requirements.capacityTypeKey }} + operator: In + values: [spot, on-demand] + - key: {{ .Values.nodePool.spec.template.spec.requirements.instanceGenerationKey }} + operator: Gt + values: [2] + nodeClassRef: + group: {{ .Values.nodePool.spec.template.spec.nodeClassRef.group }} + kind: EC2NodeClass + name: {{ .Values.nodePool.spec.template.spec.nodeClassRef.name }} + expireAfter: {{ .Values.nodePool.spec.template.spec.expireAfter }} + terminationGracePeriod: {{ .Values.nodePool.spec.template.spec.terminationGracePeriod }} + disruption: + consolidationPolicy: {{ .Values.nodePool.spec.disruption.consolidationPolicy }} + consolidateAfter: {{ .Values.nodePool.spec.disruption.consolidateAfter }} + tags: + {{- range $key, $value := .Values.nodePool.spec.tags }} + {{ $key }}: {{ $value }} + {{- end }} diff --git a/karpenter/values.yaml b/karpenter/values.yaml new file mode 100644 index 0000000..b164dd9 --- /dev/null +++ b/karpenter/values.yaml @@ -0,0 +1,65 @@ +nodePool: + apiVersion: "karpenter.sh/v1" + metadata: + name: "example-node-pool" + spec: + template: + spec: + requirements: + instanceHypervisorKey: "karpenter.k8s.aws/instance-hypervisor" + capacityTypeKey: "karpenter.sh/capacity-type" + instanceGenerationKey: "karpenter.k8s.aws/instance-generation" + nodeClassRef: + group: "karpenter.k8s.aws" + name: "example-node-class" + expireAfter: "72h" + terminationGracePeriod: "8h" + disruption: + consolidationPolicy: "WhenEmptyOrUnderutilized" + consolidateAfter: "30s" + tags: + environment: "dev" + team: "platform" + +nodeClass: + apiVersion: "karpenter.k8s.aws/v1" + metadata: + name: "example-node-class" + annotations: + description: "example-node-class" + spec: + amiFamily: "AL2" + amiSelectorTerms: + alias: "amazon-linux-2" + detailedMonitoring: true + instanceProfile: "example-instance-profile" + instanceStorePolicy: "RAID0" + kubelet: + evictionHard: + memoryAvailable: "5%" + nodefsAvailable: "10%" + nodefsInodesFree: "10%" + evictionMaxPodGracePeriod: 60 + evictionSoft: + memoryAvailable: "500Mi" + nodefsAvailable: "15%" + nodefsInodesFree: "15%" + evictionSoftGracePeriod: + memoryAvailable: "1m" + nodefsAvailable: "1m30s" + nodefsInodesFree: "2m" + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + metadataOptions: + httpEndpoint: "enabled" + httpProtocolIPv6: "enabled" + httpPutResponseHopLimit: 2 + httpTokens: "required" + securityGroupSelectorTerms: + name: "example-security-group" + subnetSelectorTerms: + name: "example-subnet" + tags: + environment: "dev" + team: "platform" + cluster_name: "example-cluster" diff --git a/main.tf b/main.tf index 4393215..03db18f 100644 --- a/main.tf +++ b/main.tf @@ -121,3 +121,57 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { name = "${var.cluster_name}-karpenter-instance-profile" role = module.karpenter_resources.node_iam_role_name } + +resource "helm_release" "karpenter_node_pool" { + depends_on = [ + helm_release.karpenter + ] + name = "karpenter-node-pool" + chart = "./karpenter" + version = local.module_version + namespace = local.namespace + description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" + create_namespace = true + + values = [ + file("${path.module}/values.yaml") + ] + #### Node Pool Inputs + set { + name = "nodePool.metadata.name" + value = "${var.cluster_name}-node-pool" + } + set { + name = "nodePool.spec.template.spec.nodeClassRef.name" + value = "${var.cluster_name}-node-class" + } + ### Node Class Inputs + set { + name = "nodeClass.metadata.name" + value = "${var.cluster_name}-node-class" + } + set { + name = "nodeClass.spec.amiFamily" + value = local.amd_ami_family + } + set { + name = "nodeClass.spec.amiSelectorTerms.alias" + value = local.amd_ami_alias + } + set { + name = "nodeClass.spec.instanceProfile" + value = aws_iam_instance_profile.karpenter_node_instance_profile.name + } + set { + name = "nodeClass.spec.securityGroupSelectorTerms.name" + value = "${var.cluster_name}-node" + } + set { + name = "nodeClass.spec.subnetSelectorTerms.name" + value = var.subnets_name + } + set { + name = "nodeClass.cluster_name" + value = var.cluster_name + } +} diff --git a/node_class.tf b/node_class.tf.off similarity index 100% rename from node_class.tf rename to node_class.tf.off diff --git a/node_pool.tf b/node_pool.tf.off similarity index 100% rename from node_pool.tf rename to node_pool.tf.off diff --git a/requirements.tf b/requirements.tf index ae62e15..c190a2e 100644 --- a/requirements.tf +++ b/requirements.tf @@ -10,9 +10,9 @@ terraform { source = "hashicorp/helm" version = ">= 2.11.0" } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.23.0" - } + # kubernetes = { + # source = "hashicorp/kubernetes" + # version = ">= 2.23.0" + # } } } From c8418e7d03c81cb00c1f3f829224c158a8bd9962 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 28 Jan 2025 21:01:43 -0500 Subject: [PATCH 02/69] update pathing --- main.tf | 2 +- requirements.tf | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/main.tf b/main.tf index 03db18f..f8711c9 100644 --- a/main.tf +++ b/main.tf @@ -134,7 +134,7 @@ resource "helm_release" "karpenter_node_pool" { create_namespace = true values = [ - file("${path.module}/values.yaml") + file("${path.module}/karpenter/values.yaml") ] #### Node Pool Inputs set { diff --git a/requirements.tf b/requirements.tf index c190a2e..2e678e4 100644 --- a/requirements.tf +++ b/requirements.tf @@ -10,9 +10,5 @@ terraform { source = "hashicorp/helm" version = ">= 2.11.0" } - # kubernetes = { - # source = "hashicorp/kubernetes" - # version = ">= 2.23.0" - # } } } From ac6898d260941f60160f75ae484daf34c16d1ab9 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 28 Jan 2025 21:06:49 -0500 Subject: [PATCH 03/69] cruft and tags --- karpenter/values.yaml | 6 ---- main.tf | 4 +++ node_class.tf.off | 67 ------------------------------------------- node_pool.tf.off | 58 ------------------------------------- 4 files changed, 4 insertions(+), 131 deletions(-) delete mode 100644 node_class.tf.off delete mode 100644 node_pool.tf.off diff --git a/karpenter/values.yaml b/karpenter/values.yaml index b164dd9..66f245a 100644 --- a/karpenter/values.yaml +++ b/karpenter/values.yaml @@ -17,9 +17,6 @@ nodePool: disruption: consolidationPolicy: "WhenEmptyOrUnderutilized" consolidateAfter: "30s" - tags: - environment: "dev" - team: "platform" nodeClass: apiVersion: "karpenter.k8s.aws/v1" @@ -59,7 +56,4 @@ nodeClass: name: "example-security-group" subnetSelectorTerms: name: "example-subnet" - tags: - environment: "dev" - team: "platform" cluster_name: "example-cluster" diff --git a/main.tf b/main.tf index f8711c9..0521ca4 100644 --- a/main.tf +++ b/main.tf @@ -174,4 +174,8 @@ resource "helm_release" "karpenter_node_pool" { name = "nodeClass.cluster_name" value = var.cluster_name } + set { + name = "nodeClass.tags" + value = var.tags + } } diff --git a/node_class.tf.off b/node_class.tf.off deleted file mode 100644 index e5b981e..0000000 --- a/node_class.tf.off +++ /dev/null @@ -1,67 +0,0 @@ -resource "kubernetes_manifest" "karpenter_node_class" { - depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile] - - manifest = { - apiVersion = "${helm_release.karpenter.name}.k8s.aws/v1" - kind = "EC2NodeClass" - metadata = { - name = "${var.cluster_name}-${helm_release.karpenter.name}-node-class" - annotations = { - "kubernetes.io/description" = "EC2NodeClass for running ${local.amd_ami_family} nodes" - } - } - spec = { - amiFamily = local.amd_ami_family - amiSelectorTerms = [ - { - alias = "${local.amd_ami_alias}@latest" - } - ] - detailedMonitoring = true - instanceProfile = aws_iam_instance_profile.karpenter_node_instance_profile.name - instanceStorePolicy = "RAID0" - kubelet = { - evictionHard = { - "memory.available" = "5%" - "nodefs.available" = "10%" - "nodefs.inodesFree" = "10%" - } - evictionMaxPodGracePeriod = 60 - evictionSoft = { - "memory.available" = "500Mi" - "nodefs.available" = "15%" - "nodefs.inodesFree" = "15%" - } - evictionSoftGracePeriod = { - "memory.available" = "1m" - "nodefs.available" = "1m30s" - "nodefs.inodesFree" = "2m" - } - imageGCHighThresholdPercent = 85 - imageGCLowThresholdPercent = 80 - } - metadataOptions = { - httpEndpoint = "enabled" - httpProtocolIPv6 = "enabled" - httpPutResponseHopLimit = 2 - httpTokens = "required" - } - securityGroupSelectorTerms = [ - { - tags = { - Name = "${var.cluster_name}-node" - } - } - ] - subnetSelectorTerms = [ - { - tags = { - Name = var.subnets_name - "kubernetes.io/cluster/${var.cluster_name}" = "shared" - } - } - ] - tags = var.tags - } - } -} diff --git a/node_pool.tf.off b/node_pool.tf.off deleted file mode 100644 index a270888..0000000 --- a/node_pool.tf.off +++ /dev/null @@ -1,58 +0,0 @@ -resource "kubernetes_manifest" "karpenter_node_pool" { - depends_on = [kubernetes_manifest.karpenter_node_class] - manifest = { - apiVersion = "${helm_release.karpenter.name}.sh/v1" - kind = "NodePool" - metadata = { - name = "${var.cluster_name}-${helm_release.karpenter.name}-node-pool" - annotations = { - "kubernetes.io/description" = "General purpose NodePool for generic workloads" - } - } - spec = { - template = { - spec = { - requirements = [ - { - key = "kubernetes.io/arch" - operator = "In" - values = ["amd64"] - }, - { - key = "kubernetes.io/os" - operator = "In" - values = ["linux"] - }, - { - key = "${helm_release.karpenter.name}.k8s.aws/instance-hypervisor" - operator = "In" - values = ["nitro"] - }, - { - key = "${helm_release.karpenter.name}.sh/capacity-type" - operator = "In" - values = ["spot", "on-demand"] - }, - { - key = "${helm_release.karpenter.name}.k8s.aws/instance-generation" - operator = "Gt" - values = ["2"] - } - ] - nodeClassRef = { - group = "${helm_release.karpenter.name}.k8s.aws" - kind = "EC2NodeClass" - name = "${var.cluster_name}-${helm_release.karpenter.name}-node-class" - } - expireAfter = "72h" - terminationGracePeriod = "8h" - } - } - disruption = { - consolidationPolicy = "WhenEmptyOrUnderutilized" - consolidateAfter = "30s" - } - tags = var.tags - } - } -} From 3ce422259410c2b0c8d4e83648f740e4b8f766d1 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 28 Jan 2025 21:08:40 -0500 Subject: [PATCH 04/69] jsonit --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 0521ca4..395101c 100644 --- a/main.tf +++ b/main.tf @@ -176,6 +176,6 @@ resource "helm_release" "karpenter_node_pool" { } set { name = "nodeClass.tags" - value = var.tags + value = jsonencode(var.tags) } } From 550b10fc0d6e2d45fa7677ba6fb010082272450d Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 28 Jan 2025 21:14:12 -0500 Subject: [PATCH 05/69] tags --- main.tf | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/main.tf b/main.tf index 395101c..3b9058c 100644 --- a/main.tf +++ b/main.tf @@ -174,8 +174,5 @@ resource "helm_release" "karpenter_node_pool" { name = "nodeClass.cluster_name" value = var.cluster_name } - set { - name = "nodeClass.tags" - value = jsonencode(var.tags) - } + } From 81ddbc4b304e37b330eabbe1f385d3e85829fa36 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 28 Jan 2025 21:19:07 -0500 Subject: [PATCH 06/69] formatting --- karpenter/templates/node_class.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index 1f7b22d..b1cf702 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -34,11 +34,11 @@ spec: httpTokens: {{ .Values.nodeClass.spec.metadataOptions.httpTokens }} securityGroupSelectorTerms: - tags: - name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} + name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} subnetSelectorTerms: - tags: - name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} - "kubernetes.io/cluster/{{ .Values.nodeClass.cluster_name }}": "shared" + name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} + "kubernetes.io/cluster/{{ .Values.nodeClass.cluster_name }}": "shared" tags: {{- range $key, $value := .Values.nodeClass.spec.tags }} {{ $key }}: {{ $value }} From b1e7e01eb8f4667bbd2699d441c1ba1f3d8613ae Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 28 Jan 2025 21:20:56 -0500 Subject: [PATCH 07/69] fmt --- karpenter/templates/node_class.yaml | 1 + karpenter/templates/node_pool.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index b1cf702..9e03f3b 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -1,3 +1,4 @@ +--- apiVersion: {{ .Values.nodeClass.apiVersion }} kind: EC2NodeClass metadata: diff --git a/karpenter/templates/node_pool.yaml b/karpenter/templates/node_pool.yaml index e0f93c9..4bc003c 100644 --- a/karpenter/templates/node_pool.yaml +++ b/karpenter/templates/node_pool.yaml @@ -1,3 +1,4 @@ +--- apiVersion: {{ .Values.nodePool.apiVersion }} kind: NodePool metadata: From 1e006ea3bac907f423f68fcf3a62028c4344f8a5 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 28 Jan 2025 21:25:05 -0500 Subject: [PATCH 08/69] comment --- karpenter/templates/node_pool.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/karpenter/templates/node_pool.yaml b/karpenter/templates/node_pool.yaml index 4bc003c..acc5a06 100644 --- a/karpenter/templates/node_pool.yaml +++ b/karpenter/templates/node_pool.yaml @@ -33,7 +33,7 @@ spec: disruption: consolidationPolicy: {{ .Values.nodePool.spec.disruption.consolidationPolicy }} consolidateAfter: {{ .Values.nodePool.spec.disruption.consolidateAfter }} - tags: - {{- range $key, $value := .Values.nodePool.spec.tags }} - {{ $key }}: {{ $value }} - {{- end }} + # tags: + # {{- range $key, $value := .Values.nodePool.spec.tags }} + # {{ $key }}: {{ $value }} + # {{- end }} From 87404a8ee28317aa4b9b831b0c48b81f6718aa5d Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 12:33:58 -0500 Subject: [PATCH 09/69] tags2 --- karpenter/templates/node_class.yaml | 4 ++-- karpenter/templates/node_pool.yaml | 8 ++++---- main.tf | 4 ++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index 9e03f3b..b6c3224 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -41,6 +41,6 @@ spec: name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} "kubernetes.io/cluster/{{ .Values.nodeClass.cluster_name }}": "shared" tags: - {{- range $key, $value := .Values.nodeClass.spec.tags }} + {{- range $key, $value := .Values.tags }} {{ $key }}: {{ $value }} - {{- end }} + {{- end }} diff --git a/karpenter/templates/node_pool.yaml b/karpenter/templates/node_pool.yaml index acc5a06..b0e19d4 100644 --- a/karpenter/templates/node_pool.yaml +++ b/karpenter/templates/node_pool.yaml @@ -33,7 +33,7 @@ spec: disruption: consolidationPolicy: {{ .Values.nodePool.spec.disruption.consolidationPolicy }} consolidateAfter: {{ .Values.nodePool.spec.disruption.consolidateAfter }} - # tags: - # {{- range $key, $value := .Values.nodePool.spec.tags }} - # {{ $key }}: {{ $value }} - # {{- end }} + tags: + {{- range $k, $v := .Values.tags }} + {{ $k }}: {{ print $.Values.prefix ":" $v | quote }} + {{- end }} diff --git a/main.tf b/main.tf index 3b9058c..4c9912b 100644 --- a/main.tf +++ b/main.tf @@ -145,6 +145,10 @@ resource "helm_release" "karpenter_node_pool" { name = "nodePool.spec.template.spec.nodeClassRef.name" value = "${var.cluster_name}-node-class" } + set { + name = "tags" + value = var.tags + } ### Node Class Inputs set { name = "nodeClass.metadata.name" From 9bd4d9aaefe5a6a0b86cfc3fafe796d610a512e7 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 12:35:31 -0500 Subject: [PATCH 10/69] add json --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 4c9912b..ddc0be3 100644 --- a/main.tf +++ b/main.tf @@ -147,7 +147,7 @@ resource "helm_release" "karpenter_node_pool" { } set { name = "tags" - value = var.tags + value = jsonencode(var.tags) } ### Node Class Inputs set { From f9ea1a4bc4fcded045afdb47d2025d71809aca9e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 12:38:41 -0500 Subject: [PATCH 11/69] fix fmt --- karpenter/templates/node_class.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index b6c3224..a0b2d4b 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -34,10 +34,10 @@ spec: httpPutResponseHopLimit: {{ .Values.nodeClass.spec.metadataOptions.httpPutResponseHopLimit }} httpTokens: {{ .Values.nodeClass.spec.metadataOptions.httpTokens }} securityGroupSelectorTerms: - - tags: + tags: name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} subnetSelectorTerms: - - tags: + tags: name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} "kubernetes.io/cluster/{{ .Values.nodeClass.cluster_name }}": "shared" tags: From 37553d949484a3b79188a20dfee44d8495f07746 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 12:41:44 -0500 Subject: [PATCH 12/69] formatting v2 --- karpenter/templates/node_class.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index a0b2d4b..f289ced 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -41,6 +41,6 @@ spec: name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} "kubernetes.io/cluster/{{ .Values.nodeClass.cluster_name }}": "shared" tags: - {{- range $key, $value := .Values.tags }} + {{- range $key, $value := .Values.tags }} {{ $key }}: {{ $value }} - {{- end }} + {{- end }} From dcbd3bcdaaf02862dc9f4310aca24dad94186d84 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 12:48:04 -0500 Subject: [PATCH 13/69] quotes --- karpenter/templates/node_class.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index f289ced..8a2a28e 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -39,7 +39,7 @@ spec: subnetSelectorTerms: tags: name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} - "kubernetes.io/cluster/{{ .Values.nodeClass.cluster_name }}": "shared" + kubernetes.io/cluster/{{ .Values.nodeClass.cluster_name }}: "shared" tags: {{- range $key, $value := .Values.tags }} {{ $key }}: {{ $value }} From e6413dc99b4cbe7d9bba98b29dc8ec08ea95f636 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 12:51:37 -0500 Subject: [PATCH 14/69] escape the slashes --- karpenter/templates/node_class.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index 8a2a28e..33a4aa5 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -39,7 +39,7 @@ spec: subnetSelectorTerms: tags: name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} - kubernetes.io/cluster/{{ .Values.nodeClass.cluster_name }}: "shared" + "kubernetes.io\/cluster\/{{ .Values.nodeClass.cluster_name }}": "shared" tags: {{- range $key, $value := .Values.tags }} {{ $key }}: {{ $value }} From 1a27fca7ff1b644667612a76bab09e1f8c75d648 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 12:56:10 -0500 Subject: [PATCH 15/69] fmt3 --- karpenter/templates/node_class.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index 33a4aa5..e08931d 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -39,7 +39,7 @@ spec: subnetSelectorTerms: tags: name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} - "kubernetes.io\/cluster\/{{ .Values.nodeClass.cluster_name }}": "shared" + "{{ printf "kubernetes.io/cluster/%s" .Values.nodeClass.cluster_name }}": "shared" tags: {{- range $key, $value := .Values.tags }} {{ $key }}: {{ $value }} From 887959e08887a7d9d78a968577c42fc238363916 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 13:00:57 -0500 Subject: [PATCH 16/69] fmt4 --- karpenter/templates/node_class.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index e08931d..3cad393 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -34,12 +34,12 @@ spec: httpPutResponseHopLimit: {{ .Values.nodeClass.spec.metadataOptions.httpPutResponseHopLimit }} httpTokens: {{ .Values.nodeClass.spec.metadataOptions.httpTokens }} securityGroupSelectorTerms: - tags: - name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} + - tags: + name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} subnetSelectorTerms: - tags: - name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} - "{{ printf "kubernetes.io/cluster/%s" .Values.nodeClass.cluster_name }}": "shared" + - tags: + name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} + "{{ printf "kubernetes.io/cluster/%s" .Values.nodeClass.cluster_name }}": "shared" tags: {{- range $key, $value := .Values.tags }} {{ $key }}: {{ $value }} From 67692170a0867146f6a6e51d1521cc26877558ba Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 14:09:37 -0500 Subject: [PATCH 17/69] subnet selectors --- karpenter/templates/node_class.yaml | 4 ++-- main.tf | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index 3cad393..fd9cb24 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -38,8 +38,8 @@ spec: name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} subnetSelectorTerms: - tags: - name: {{ .Values.nodeClass.spec.subnetSelectorTerms.name }} - "{{ printf "kubernetes.io/cluster/%s" .Values.nodeClass.cluster_name }}": "shared" + name: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.name }} + {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name }}: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value }} tags: {{- range $key, $value := .Values.tags }} {{ $key }}: {{ $value }} diff --git a/main.tf b/main.tf index ddc0be3..ef5355b 100644 --- a/main.tf +++ b/main.tf @@ -171,9 +171,17 @@ resource "helm_release" "karpenter_node_pool" { value = "${var.cluster_name}-node" } set { - name = "nodeClass.spec.subnetSelectorTerms.name" + name = "nodeClass.spec.subnetSelectorTerms.tags.name" value = var.subnets_name } + set { + name = "nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name" + value = "kubernetes.io/cluster/${var.cluster_name}" + } + set { + name = "nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value" + value = "shared" + } set { name = "nodeClass.cluster_name" value = var.cluster_name From 26cf47a4558e3598a0f0244e69423c8c943224d6 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 14:42:19 -0500 Subject: [PATCH 18/69] fmt4 --- karpenter/templates/node_class.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index fd9cb24..c9fcaa7 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -34,12 +34,12 @@ spec: httpPutResponseHopLimit: {{ .Values.nodeClass.spec.metadataOptions.httpPutResponseHopLimit }} httpTokens: {{ .Values.nodeClass.spec.metadataOptions.httpTokens }} securityGroupSelectorTerms: - - tags: - name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} + tags: + name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} subnetSelectorTerms: - - tags: - name: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.name }} - {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name }}: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value }} + tags: + name: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.name }} + {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name }}: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value }} tags: {{- range $key, $value := .Values.tags }} {{ $key }}: {{ $value }} From c8daa08f9210fb3683b310085794d522ab76a477 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 14:45:04 -0500 Subject: [PATCH 19/69] quotes --- karpenter/templates/node_class.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index c9fcaa7..ea5d189 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -39,7 +39,7 @@ spec: subnetSelectorTerms: tags: name: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.name }} - {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name }}: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value }} + {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name | quote }}: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value }} tags: {{- range $key, $value := .Values.tags }} {{ $key }}: {{ $value }} From 03002e9fe5ce9a0f2a0212a938aaf4784c72f378 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 14:47:13 -0500 Subject: [PATCH 20/69] more quotes --- karpenter/templates/node_class.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index ea5d189..f42df6a 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -39,7 +39,7 @@ spec: subnetSelectorTerms: tags: name: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.name }} - {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name | quote }}: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value }} + {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name | quote }}: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value | quote }} tags: {{- range $key, $value := .Values.tags }} {{ $key }}: {{ $value }} From 4ae7bf2a670a2ac0c582524a08c46ff71293c676 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 14:52:44 -0500 Subject: [PATCH 21/69] remove values defaults --- karpenter/values.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/karpenter/values.yaml b/karpenter/values.yaml index 66f245a..ee9f8c9 100644 --- a/karpenter/values.yaml +++ b/karpenter/values.yaml @@ -52,8 +52,4 @@ nodeClass: httpProtocolIPv6: "enabled" httpPutResponseHopLimit: 2 httpTokens: "required" - securityGroupSelectorTerms: - name: "example-security-group" - subnetSelectorTerms: - name: "example-subnet" cluster_name: "example-cluster" From 77eab483a57f59268aba6f23c15930617afd6414 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 15:45:40 -0500 Subject: [PATCH 22/69] remove problem lines --- karpenter/templates/node_class.yaml | 8 -------- karpenter/templates/node_pool.yaml | 4 ---- 2 files changed, 12 deletions(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index f42df6a..f105f26 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -36,11 +36,3 @@ spec: securityGroupSelectorTerms: tags: name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} - subnetSelectorTerms: - tags: - name: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.name }} - {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name | quote }}: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value | quote }} - tags: - {{- range $key, $value := .Values.tags }} - {{ $key }}: {{ $value }} - {{- end }} diff --git a/karpenter/templates/node_pool.yaml b/karpenter/templates/node_pool.yaml index b0e19d4..483fd10 100644 --- a/karpenter/templates/node_pool.yaml +++ b/karpenter/templates/node_pool.yaml @@ -33,7 +33,3 @@ spec: disruption: consolidationPolicy: {{ .Values.nodePool.spec.disruption.consolidationPolicy }} consolidateAfter: {{ .Values.nodePool.spec.disruption.consolidateAfter }} - tags: - {{- range $k, $v := .Values.tags }} - {{ $k }}: {{ print $.Values.prefix ":" $v | quote }} - {{- end }} From 2cf00ec894c9153580de406248bdcc3b5f88ab9d Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 21:37:21 -0500 Subject: [PATCH 23/69] add subnetSelectorTerms again --- karpenter/templates/node_class.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index f105f26..7475065 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -36,3 +36,7 @@ spec: securityGroupSelectorTerms: tags: name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} + subnetSelectorTerms: + tags: + name: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.name }} + {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name | quote }}: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value }} From 3e5c0e86dc3351c4e7845b09e00d2c55addf5a4b Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 21:42:35 -0500 Subject: [PATCH 24/69] ergh --- karpenter/templates/node_class.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index 7475065..f105f26 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -36,7 +36,3 @@ spec: securityGroupSelectorTerms: tags: name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} - subnetSelectorTerms: - tags: - name: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.name }} - {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name | quote }}: {{ .Values.nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value }} From 621171325c88c918a3913424e1e1b966ce576a7e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 22:02:44 -0500 Subject: [PATCH 25/69] index --- karpenter/templates/node_class.yaml | 4 ++-- karpenter/values.yaml | 3 +++ main.tf | 8 ++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index f105f26..7b823c5 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -34,5 +34,5 @@ spec: httpPutResponseHopLimit: {{ .Values.nodeClass.spec.metadataOptions.httpPutResponseHopLimit }} httpTokens: {{ .Values.nodeClass.spec.metadataOptions.httpTokens }} securityGroupSelectorTerms: - tags: - name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.name }} + - tags: + name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.tags.sgname }} diff --git a/karpenter/values.yaml b/karpenter/values.yaml index ee9f8c9..8380085 100644 --- a/karpenter/values.yaml +++ b/karpenter/values.yaml @@ -52,4 +52,7 @@ nodeClass: httpProtocolIPv6: "enabled" httpPutResponseHopLimit: 2 httpTokens: "required" + securityGroupSelectorTerms: + - tags: + sgname: "example-cluster-node" cluster_name: "example-cluster" diff --git a/main.tf b/main.tf index ef5355b..b4d96fb 100644 --- a/main.tf +++ b/main.tf @@ -167,19 +167,19 @@ resource "helm_release" "karpenter_node_pool" { value = aws_iam_instance_profile.karpenter_node_instance_profile.name } set { - name = "nodeClass.spec.securityGroupSelectorTerms.name" + name = "nodeClass.spec.securityGroupSelectorTerms.tags[0].name" value = "${var.cluster_name}-node" } set { - name = "nodeClass.spec.subnetSelectorTerms.tags.name" + name = "nodeClass.spec.subnetSelectorTerms.tags[0].name" value = var.subnets_name } set { - name = "nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name" + name = "nodeClass.spec.subnetSelectorTerms.tags[0].cluster_tag_name" value = "kubernetes.io/cluster/${var.cluster_name}" } set { - name = "nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value" + name = "nodeClass.spec.subnetSelectorTerms.tags[0].cluster_tag_value" value = "shared" } set { From cc37cba8eed4355ee53c81a4e49e17ddbea34b94 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 22:11:57 -0500 Subject: [PATCH 26/69] use the tools --- karpenter/Chart.yaml | 1 + karpenter/templates/node_class.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/karpenter/Chart.yaml b/karpenter/Chart.yaml index 5b9ab30..a874125 100644 --- a/karpenter/Chart.yaml +++ b/karpenter/Chart.yaml @@ -2,3 +2,4 @@ apiVersion: v2 name: karpenter version: 0.1.0 description: A Helm chart for Karpenter resources +icon: https://repository-images.githubusercontent.com/278480393/dab059c8-caa1-4b55-aaa7-3d30e47a5616 diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index 7b823c5..e4a9e07 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -35,4 +35,4 @@ spec: httpTokens: {{ .Values.nodeClass.spec.metadataOptions.httpTokens }} securityGroupSelectorTerms: - tags: - name: {{ .Values.nodeClass.spec.securityGroupSelectorTerms.tags.sgname }} + name: {{ .Values.nodeClass.cluster_name }} From 4cc579a03db9a7490e7677a0a56e4595cddfe16f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 22:23:17 -0500 Subject: [PATCH 27/69] Name? --- karpenter/templates/node_class.yaml | 2 +- karpenter/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml index e4a9e07..74dc586 100644 --- a/karpenter/templates/node_class.yaml +++ b/karpenter/templates/node_class.yaml @@ -35,4 +35,4 @@ spec: httpTokens: {{ .Values.nodeClass.spec.metadataOptions.httpTokens }} securityGroupSelectorTerms: - tags: - name: {{ .Values.nodeClass.cluster_name }} + Name: {{ .Values.nodeClass.cluster_name }} diff --git a/karpenter/values.yaml b/karpenter/values.yaml index 8380085..f259ee2 100644 --- a/karpenter/values.yaml +++ b/karpenter/values.yaml @@ -54,5 +54,5 @@ nodeClass: httpTokens: "required" securityGroupSelectorTerms: - tags: - sgname: "example-cluster-node" + Name: "example-cluster-node" cluster_name: "example-cluster" From 6c684ad3ef172864eb0f379c86aaa937ee58b700 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 29 Jan 2025 22:25:54 -0500 Subject: [PATCH 28/69] pass in same --- main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main.tf b/main.tf index b4d96fb..b52aba3 100644 --- a/main.tf +++ b/main.tf @@ -167,19 +167,19 @@ resource "helm_release" "karpenter_node_pool" { value = aws_iam_instance_profile.karpenter_node_instance_profile.name } set { - name = "nodeClass.spec.securityGroupSelectorTerms.tags[0].name" + name = "nodeClass.spec.securityGroupSelectorTerms.tags.Name" value = "${var.cluster_name}-node" } set { - name = "nodeClass.spec.subnetSelectorTerms.tags[0].name" + name = "nodeClass.spec.subnetSelectorTerms.tags.Name" value = var.subnets_name } set { - name = "nodeClass.spec.subnetSelectorTerms.tags[0].cluster_tag_name" + name = "nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name" value = "kubernetes.io/cluster/${var.cluster_name}" } set { - name = "nodeClass.spec.subnetSelectorTerms.tags[0].cluster_tag_value" + name = "nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value" value = "shared" } set { From 1822d20b52c383b949fb6b95ede6a85d47bfbd4a Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 30 Jan 2025 00:32:26 -0500 Subject: [PATCH 29/69] wip --- README.md | 2 +- karpenter-resources/Chart.yaml | 6 + karpenter-resources/README.md | 71 ++++++++++ karpenter-resources/templates/_helpers.tpl | 125 ++++++++++++++++++ karpenter-resources/templates/node-class.yaml | 58 ++++++++ karpenter-resources/templates/node-pool.yaml | 43 ++++++ karpenter-resources/values.yaml | 64 +++++++++ karpenter/Chart.yaml | 5 - karpenter/templates/node_class.yaml | 38 ------ karpenter/templates/node_pool.yaml | 35 ----- karpenter/values.yaml | 58 -------- main.tf | 65 +++++---- 12 files changed, 400 insertions(+), 170 deletions(-) create mode 100644 karpenter-resources/Chart.yaml create mode 100644 karpenter-resources/README.md create mode 100644 karpenter-resources/templates/_helpers.tpl create mode 100644 karpenter-resources/templates/node-class.yaml create mode 100644 karpenter-resources/templates/node-pool.yaml create mode 100644 karpenter-resources/values.yaml delete mode 100644 karpenter/Chart.yaml delete mode 100644 karpenter/templates/node_class.yaml delete mode 100644 karpenter/templates/node_pool.yaml delete mode 100644 karpenter/values.yaml diff --git a/README.md b/README.md index 68b6390..dc20882 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|------| | [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.karpenter_resources](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | diff --git a/karpenter-resources/Chart.yaml b/karpenter-resources/Chart.yaml new file mode 100644 index 0000000..cf46148 --- /dev/null +++ b/karpenter-resources/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: karpenter-resources +description: Helm chart for configuring custom resources for Karpenter on the cluster +version: 0.0.2 +appVersion: 0.0.2 +icon: https://repository-images.githubusercontent.com/278480393/dab059c8-caa1-4b55-aaa7-3d30e47a5616 diff --git a/karpenter-resources/README.md b/karpenter-resources/README.md new file mode 100644 index 0000000..e95f582 --- /dev/null +++ b/karpenter-resources/README.md @@ -0,0 +1,71 @@ +# Karpenter Resources Helm Chart + +## Overview + +This Helm chart is an abstraction layer designed for deploying various configurations of Karpenter nodes in a Kubernetes cluster managed by EKS. It integrates seamlessly with Terraform, allowing users to define different node pools and settings for their Kubernetes cluster. + +## Prerequisites + +- Helm 3.x or later installed +- Terraform installed +- Access to an AWS EKS cluster + +## Configuration + +The chart is configured to be used with Terraform. Here is an example of how you might define Helm releases for different Karpenter configurations in your Terraform files using EKS Blueprints add-ons: + +```hcl +module "eks_blueprints_addons" { + # ... other configurations ... + helm_releases = { + karpenter-resources-default = { + name = "default" + description = "A Helm chart for default node pool" + chart = "${path.module}/helm-values/karpenter-resources" + values = [ + <<-EOT + clusterName: ${module.eks.cluster_name} + ec2NodeClass: + karpenterRole: ${split("/", module.eks_blueprints_addons.karpenter.node_iam_role_arn)[1]} + nodePool: + labels: + - provisioner: default + - workload: rayhead + EOT + ] + } + } +} +``` + +## Testing the Chart with Helm Template + +To review the Kubernetes manifests that will be generated by the Helm chart based on your configuration, you can use the `helm template` command. This is especially useful for validating your Terraform configurations before applying them. + +1. **Generate the Manifests** + + Navigate to the directory where your Helm chart is located. + + ```sh + cd path/to/helm-chart + ``` + +2. **Run Helm Template** + + Use the `helm template` command with your custom values. For example: + + ```sh + helm template my-release-name . --values values.yaml + ``` + + Replace `my-release-name` with a name for your release, and `values.yaml` with the path to your custom values file. + + To test specific configurations defined in your Terraform file, you can create a temporary values file with the configuration snippet from your Terraform definition: + + ```sh + echo '' > temp-values.yaml + helm template my-release-name . --values temp-values.yaml + rm temp-values.yaml + ``` + + This will output the Kubernetes manifests to your terminal, allowing you to review them. diff --git a/karpenter-resources/templates/_helpers.tpl b/karpenter-resources/templates/_helpers.tpl new file mode 100644 index 0000000..19c2446 --- /dev/null +++ b/karpenter-resources/templates/_helpers.tpl @@ -0,0 +1,125 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "karpenter-node-pool.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "karpenter-node-pool.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "karpenter-node-pool.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "karpenter-node-pool.labels" -}} +helm.sh/chart: {{ include "karpenter-node-pool.chart" . }} +{{ include "karpenter-node-pool.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "karpenter-node-pool.selectorLabels" -}} +app.kubernetes.io/name: {{ include "karpenter-node-pool.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "karpenter-node-pool.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "karpenter-node-pool.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Expand the name of the chart. +*/}} +{{- define "karpenter-node-class.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "karpenter-node-class.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "karpenter-node-class.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "karpenter-node-class.labels" -}} +helm.sh/chart: {{ include "karpenter-node-class.chart" . }} +{{ include "karpenter-node-class.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "karpenter-node-class.selectorLabels" -}} +app.kubernetes.io/name: {{ include "karpenter-node-class.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "karpenter-node-class.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "karpenter-node-class.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/karpenter-resources/templates/node-class.yaml b/karpenter-resources/templates/node-class.yaml new file mode 100644 index 0000000..98b798e --- /dev/null +++ b/karpenter-resources/templates/node-class.yaml @@ -0,0 +1,58 @@ +{{- if .Values.ec2NodeClass.enabled }} +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass +metadata: + name: {{ .Values.name }} +spec: + {{- if .Values.ec2NodeClass.amiFamily }} + amiFamily: {{ .Values.ec2NodeClass.amiFamily }} + {{- end }} + {{- if .Values.ec2NodeClass.amiSelectorTerms }} + amiSelectorTerms: + {{- toYaml .Values.ec2NodeClass.amiSelectorTerms | nindent 4 }} + {{- end }} + subnetSelectorTerms: + {{- if and (.Values.ec2NodeClass.subnetSelectorTerms) (hasKey .Values.ec2NodeClass.subnetSelectorTerms "tags") }} + - tags: + {{- range $key, $value := .Values.ec2NodeClass.subnetSelectorTerms.tags }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if and (.Values.ec2NodeClass.subnetSelectorTerms.id) (hasKey .Values.ec2NodeClass.subnetSelectorTerms "id") }} + - id: {{ .Values.ec2NodeClass.subnetSelectorTerms.id }} + {{- end }} + securityGroupSelectorTerms: + {{- if and (.Values.ec2NodeClass.securityGroupSelectorTerms) (hasKey .Values.ec2NodeClass.securityGroupSelectorTerms "name") }} + - name: {{ .Values.ec2NodeClass.securityGroupSelectorTerms.name }} + {{- end }} + {{- if and (.Values.ec2NodeClass.securityGroupSelectorTerms) (hasKey .Values.ec2NodeClass.securityGroupSelectorTerms "id") }} + - id: {{ .Values.ec2NodeClass.securityGroupSelectorTerms.id }} + {{- end }} + {{- if and (.Values.ec2NodeClass.securityGroupSelectorTerms) (hasKey .Values.ec2NodeClass.securityGroupSelectorTerms "tags") }} + - tags: + {{- range $key, $value := .Values.ec2NodeClass.securityGroupSelectorTerms.tags }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + role: {{ .Values.ec2NodeClass.karpenterRole }} + tags: + Name: karpenter-{{ .Values.name }} + metadataOptions: + httpEndpoint: {{ .Values.ec2NodeClass.metadataOptions.httpEndpoint }} + httpProtocolIPv6: {{ .Values.ec2NodeClass.metadataOptions.httpProtocolIPv6 }} + httpPutResponseHopLimit: {{ .Values.ec2NodeClass.metadataOptions.httpPutResponseHopLimit }} + httpTokens: {{ .Values.ec2NodeClass.metadataOptions.httpTokens }} + {{- with .Values.ec2NodeClass.blockDeviceMappings }} + blockDeviceMappings: + {{- toYaml . | nindent 4 }} + {{- end }} + + detailedMonitoring: {{ .Values.ec2NodeClass.detailedMonitoring }} + {{- if .Values.ec2NodeClass.instanceStorePolicy }} + instanceStorePolicy: {{ .Values.ec2NodeClass.instanceStorePolicy }} + {{- end }} + {{- if .Values.ec2NodeClass.userData }} + userData: | + {{- .Values.ec2NodeClass.userData | nindent 4 }} + {{- end }} +{{- end }} diff --git a/karpenter-resources/templates/node-pool.yaml b/karpenter-resources/templates/node-pool.yaml new file mode 100644 index 0000000..60c80aa --- /dev/null +++ b/karpenter-resources/templates/node-pool.yaml @@ -0,0 +1,43 @@ +{{- if .Values.nodePool.enabled }} +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: {{ .Values.name }} +spec: + template: + metadata: + labels: + NodePool: {{ .Values.name }} + NodeGroupType: {{ .Values.name }} + {{- with .Values.nodePool.labels }} + {{- range . }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + spec: + nodeClassRef: + name: {{ .Values.name }} + group: karpenter.k8s.aws + kind: EC2NodeClass + {{- with .Values.nodePool.taints }} + taints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodePool.requirements }} + requirements: + {{- toYaml . | nindent 8 }} + {{- end }} + expireAfter: {{ .Values.nodePool.expireAfter }} + terminationGracePeriod: {{ .Values.nodePool.terminationGracePeriod }} + disruption: + consolidationPolicy: {{ .Values.nodePool.disruption.consolidationPolicy }} + consolidateAfter: {{ .Values.nodePool.disruption.consolidateAfter }} + {{- with .Values.nodePool.disruption.budgets }} + budgets: + {{- toYaml . | nindent 4 }} + {{- end }} + limits: + cpu: {{ .Values.nodePool.limits.cpu }} + memory: {{ .Values.nodePool.limits.memory }} + weight: {{ .Values.nodePool.weight }} +{{- end }} diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml new file mode 100644 index 0000000..10df2f4 --- /dev/null +++ b/karpenter-resources/values.yaml @@ -0,0 +1,64 @@ +# Shared values +name: default +clusterName: test-cluster + +# EC2NodeClass specific values +ec2NodeClass: + enabled: true + amiFamily: "AL2" + amiSelectorTerms: + - alias: "bottlerocket@latest" + subnetSelectorTerms: + tags: + Name: "subnet-name" + # id: "subnet-id" + securityGroupSelectorTerms: + name: "security-group-name" + # id: "security-group-id" + tags: + key1: "value1" + key2: "value2" + karpenterRole: "karpenter-role" + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: required + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 100Gi + volumeType: gp3 + encrypted: true + deleteOnTermination: true + detailedMonitoring: true + userData: | + #!/bin/bash + echo "User data script" + instanceStorePolicy: "instance-store-policy" + +# NodePool specific values +nodePool: + enabled: true + labels: + environment: production + team: devops + taints: + - key: "key1" + value: "value1" + effect: "NoSchedule" + requirements: + - key: "instance-type" + operator: "In" + values: ["m5.large", "m5.xlarge"] + expireAfter: 720h + terminationGracePeriod: 48h + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 30s + budgets: + - nodes: 10% + limits: + cpu: "1000" + memory: 1000Gi + weight: 10 diff --git a/karpenter/Chart.yaml b/karpenter/Chart.yaml deleted file mode 100644 index a874125..0000000 --- a/karpenter/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v2 -name: karpenter -version: 0.1.0 -description: A Helm chart for Karpenter resources -icon: https://repository-images.githubusercontent.com/278480393/dab059c8-caa1-4b55-aaa7-3d30e47a5616 diff --git a/karpenter/templates/node_class.yaml b/karpenter/templates/node_class.yaml deleted file mode 100644 index 74dc586..0000000 --- a/karpenter/templates/node_class.yaml +++ /dev/null @@ -1,38 +0,0 @@ ---- -apiVersion: {{ .Values.nodeClass.apiVersion }} -kind: EC2NodeClass -metadata: - name: {{ .Values.nodeClass.metadata.name }} - annotations: - kubernetes.io/description: "EC2NodeClass for running {{ .Values.nodeClass.metadata.annotations.description }}" -spec: - amiFamily: {{ .Values.nodeClass.spec.amiFamily }} - amiSelectorTerms: - - alias: {{ .Values.nodeClass.spec.amiSelectorTerms.alias }} - detailedMonitoring: {{ .Values.nodeClass.spec.detailedMonitoring }} - instanceProfile: {{ .Values.nodeClass.spec.instanceProfile }} - instanceStorePolicy: {{ .Values.nodeClass.spec.instanceStorePolicy }} - kubelet: - evictionHard: - memory.available: {{ .Values.nodeClass.spec.kubelet.evictionHard.memoryAvailable }} - nodefs.available: {{ .Values.nodeClass.spec.kubelet.evictionHard.nodefsAvailable }} - nodefs.inodesFree: {{ .Values.nodeClass.spec.kubelet.evictionHard.nodefsInodesFree }} - evictionMaxPodGracePeriod: {{ .Values.nodeClass.spec.kubelet.evictionMaxPodGracePeriod }} - evictionSoft: - memory.available: {{ .Values.nodeClass.spec.kubelet.evictionSoft.memoryAvailable }} - nodefs.available: {{ .Values.nodeClass.spec.kubelet.evictionSoft.nodefsAvailable }} - nodefs.inodesFree: {{ .Values.nodeClass.spec.kubelet.evictionSoft.nodefsInodesFree }} - evictionSoftGracePeriod: - memory.available: {{ .Values.nodeClass.spec.kubelet.evictionSoftGracePeriod.memoryAvailable }} - nodefs.available: {{ .Values.nodeClass.spec.kubelet.evictionSoftGracePeriod.nodefsAvailable }} - nodefs.inodesFree: {{ .Values.nodeClass.spec.kubelet.evictionSoftGracePeriod.nodefsInodesFree }} - imageGCHighThresholdPercent: {{ .Values.nodeClass.spec.kubelet.imageGCHighThresholdPercent }} - imageGCLowThresholdPercent: {{ .Values.nodeClass.spec.kubelet.imageGCLowThresholdPercent }} - metadataOptions: - httpEndpoint: {{ .Values.nodeClass.spec.metadataOptions.httpEndpoint }} - httpProtocolIPv6: {{ .Values.nodeClass.spec.metadataOptions.httpProtocolIPv6 }} - httpPutResponseHopLimit: {{ .Values.nodeClass.spec.metadataOptions.httpPutResponseHopLimit }} - httpTokens: {{ .Values.nodeClass.spec.metadataOptions.httpTokens }} - securityGroupSelectorTerms: - - tags: - Name: {{ .Values.nodeClass.cluster_name }} diff --git a/karpenter/templates/node_pool.yaml b/karpenter/templates/node_pool.yaml deleted file mode 100644 index 483fd10..0000000 --- a/karpenter/templates/node_pool.yaml +++ /dev/null @@ -1,35 +0,0 @@ ---- -apiVersion: {{ .Values.nodePool.apiVersion }} -kind: NodePool -metadata: - name: {{ .Values.nodePool.metadata.name }} - annotations: - kubernetes.io/description: General purpose NodePool for generic workloads -spec: - template: - spec: - requirements: - - key: kubernetes.io/arch - operator: In - values: [amd64] - - key: kubernetes.io/os - operator: In - values: [linux] - - key: {{ .Values.nodePool.spec.template.spec.requirements.instanceHypervisorKey }} - operator: In - values: [nitro] - - key: {{ .Values.nodePool.spec.template.spec.requirements.capacityTypeKey }} - operator: In - values: [spot, on-demand] - - key: {{ .Values.nodePool.spec.template.spec.requirements.instanceGenerationKey }} - operator: Gt - values: [2] - nodeClassRef: - group: {{ .Values.nodePool.spec.template.spec.nodeClassRef.group }} - kind: EC2NodeClass - name: {{ .Values.nodePool.spec.template.spec.nodeClassRef.name }} - expireAfter: {{ .Values.nodePool.spec.template.spec.expireAfter }} - terminationGracePeriod: {{ .Values.nodePool.spec.template.spec.terminationGracePeriod }} - disruption: - consolidationPolicy: {{ .Values.nodePool.spec.disruption.consolidationPolicy }} - consolidateAfter: {{ .Values.nodePool.spec.disruption.consolidateAfter }} diff --git a/karpenter/values.yaml b/karpenter/values.yaml deleted file mode 100644 index f259ee2..0000000 --- a/karpenter/values.yaml +++ /dev/null @@ -1,58 +0,0 @@ -nodePool: - apiVersion: "karpenter.sh/v1" - metadata: - name: "example-node-pool" - spec: - template: - spec: - requirements: - instanceHypervisorKey: "karpenter.k8s.aws/instance-hypervisor" - capacityTypeKey: "karpenter.sh/capacity-type" - instanceGenerationKey: "karpenter.k8s.aws/instance-generation" - nodeClassRef: - group: "karpenter.k8s.aws" - name: "example-node-class" - expireAfter: "72h" - terminationGracePeriod: "8h" - disruption: - consolidationPolicy: "WhenEmptyOrUnderutilized" - consolidateAfter: "30s" - -nodeClass: - apiVersion: "karpenter.k8s.aws/v1" - metadata: - name: "example-node-class" - annotations: - description: "example-node-class" - spec: - amiFamily: "AL2" - amiSelectorTerms: - alias: "amazon-linux-2" - detailedMonitoring: true - instanceProfile: "example-instance-profile" - instanceStorePolicy: "RAID0" - kubelet: - evictionHard: - memoryAvailable: "5%" - nodefsAvailable: "10%" - nodefsInodesFree: "10%" - evictionMaxPodGracePeriod: 60 - evictionSoft: - memoryAvailable: "500Mi" - nodefsAvailable: "15%" - nodefsInodesFree: "15%" - evictionSoftGracePeriod: - memoryAvailable: "1m" - nodefsAvailable: "1m30s" - nodefsInodesFree: "2m" - imageGCHighThresholdPercent: 85 - imageGCLowThresholdPercent: 80 - metadataOptions: - httpEndpoint: "enabled" - httpProtocolIPv6: "enabled" - httpPutResponseHopLimit: 2 - httpTokens: "required" - securityGroupSelectorTerms: - - tags: - Name: "example-cluster-node" - cluster_name: "example-cluster" diff --git a/main.tf b/main.tf index b52aba3..9d22f6c 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,7 @@ locals { namespace = "karpenter" amd_ami_family = "Bottlerocket" - amd_ami_alias = "bottlerocket" + amd_ami_alias = "bottlerocket@latest" } # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter @@ -122,65 +122,64 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { role = module.karpenter_resources.node_iam_role_name } -resource "helm_release" "karpenter_node_pool" { +resource "helm_release" "karpenter_resources" { depends_on = [ helm_release.karpenter ] - name = "karpenter-node-pool" - chart = "./karpenter" - version = local.module_version - namespace = local.namespace - description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" - create_namespace = true + name = "karpenter-resources" + description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" + chart = "./karpenter-resources" + version = local.module_version + namespace = local.namespace values = [ - file("${path.module}/karpenter/values.yaml") + file("${path.module}/karpenter-resources/values.yaml") ] - #### Node Pool Inputs + #### Default values overrides set { - name = "nodePool.metadata.name" - value = "${var.cluster_name}-node-pool" + name = "clusterName" + value = var.cluster_name } set { - name = "nodePool.spec.template.spec.nodeClassRef.name" - value = "${var.cluster_name}-node-class" + name = "ec2NodeClass.karpenterRole" + value = aws_iam_instance_profile.karpenter_node_instance_profile.name } set { - name = "tags" - value = jsonencode(var.tags) + name = "ec2NodeClass.amiFamily" + value = local.amd_ami_family } - ### Node Class Inputs set { - name = "nodeClass.metadata.name" - value = "${var.cluster_name}-node-class" + name = "ec2NodeClass.amiSelectorTerms[0].alias" + value = local.amd_ami_alias } set { - name = "nodeClass.spec.amiFamily" - value = local.amd_ami_family + name = "ec2NodeClass.subnetSelectorTerms.tags.Name" + value = var.subnets_name } set { - name = "nodeClass.spec.amiSelectorTerms.alias" - value = local.amd_ami_alias + name = "ec2NodeClassc.subnetSelectorTerms.tags" + value = "kubernetes.io/cluster/${var.cluster_name}" } set { - name = "nodeClass.spec.instanceProfile" - value = aws_iam_instance_profile.karpenter_node_instance_profile.name + name = "nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value" + value = "shared" } set { - name = "nodeClass.spec.securityGroupSelectorTerms.tags.Name" - value = "${var.cluster_name}-node" + name = "tags" + value = jsonencode(var.tags) } + ### Node Class Inputs set { - name = "nodeClass.spec.subnetSelectorTerms.tags.Name" - value = var.subnets_name + name = "nodeClass.metadata.name" + value = "${var.cluster_name}-node-class" } set { - name = "nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_name" - value = "kubernetes.io/cluster/${var.cluster_name}" + name = "nodeClass.spec.instanceProfile" + value = aws_iam_instance_profile.karpenter_node_instance_profile.name } set { - name = "nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value" - value = "shared" + name = "nodeClass.spec.securityGroupSelectorTerms.tags.Name" + value = "${var.cluster_name}-node" } set { name = "nodeClass.cluster_name" From 6cbdedd2e9c062556fc1438a3c810c7730ad3749 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 30 Jan 2025 17:12:20 -0500 Subject: [PATCH 30/69] add if around labels --- karpenter-resources/templates/node-class.yaml | 9 ++- karpenter-resources/templates/node-pool.yaml | 28 +++++-- karpenter-resources/values.yaml | 79 +++++++++++-------- main.tf | 32 ++++---- 4 files changed, 91 insertions(+), 57 deletions(-) diff --git a/karpenter-resources/templates/node-class.yaml b/karpenter-resources/templates/node-class.yaml index 98b798e..4fce705 100644 --- a/karpenter-resources/templates/node-class.yaml +++ b/karpenter-resources/templates/node-class.yaml @@ -2,7 +2,7 @@ apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: - name: {{ .Values.name }} + name: {{ .Values.nodePool.nodeClassRef.name }} spec: {{- if .Values.ec2NodeClass.amiFamily }} amiFamily: {{ .Values.ec2NodeClass.amiFamily }} @@ -35,8 +35,12 @@ spec: {{- end }} {{- end }} role: {{ .Values.ec2NodeClass.karpenterRole }} + {{- if and (.Values.ec2NodeClass.tags) (hasKey .Values.ec2NodeClass "tags") }} tags: - Name: karpenter-{{ .Values.name }} + {{- range $key, $value := .Values.ec2NodeClass.tags }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} metadataOptions: httpEndpoint: {{ .Values.ec2NodeClass.metadataOptions.httpEndpoint }} httpProtocolIPv6: {{ .Values.ec2NodeClass.metadataOptions.httpProtocolIPv6 }} @@ -46,7 +50,6 @@ spec: blockDeviceMappings: {{- toYaml . | nindent 4 }} {{- end }} - detailedMonitoring: {{ .Values.ec2NodeClass.detailedMonitoring }} {{- if .Values.ec2NodeClass.instanceStorePolicy }} instanceStorePolicy: {{ .Values.ec2NodeClass.instanceStorePolicy }} diff --git a/karpenter-resources/templates/node-pool.yaml b/karpenter-resources/templates/node-pool.yaml index 60c80aa..98f8f11 100644 --- a/karpenter-resources/templates/node-pool.yaml +++ b/karpenter-resources/templates/node-pool.yaml @@ -2,23 +2,25 @@ apiVersion: karpenter.sh/v1 kind: NodePool metadata: - name: {{ .Values.name }} + name: {{ .Values.nodePool.name }} spec: template: metadata: labels: - NodePool: {{ .Values.name }} - NodeGroupType: {{ .Values.name }} + NodePool: {{ .Values.nodePool.name }} + NodeGroupType: {{ .Values.nodePool.name }} + {{- if .Values.nodePool.labels }} {{- with .Values.nodePool.labels }} {{- range . }} {{- toYaml . | nindent 8 }} {{- end }} {{- end }} + {{- end }} spec: nodeClassRef: - name: {{ .Values.name }} - group: karpenter.k8s.aws - kind: EC2NodeClass + name: {{ .Values.nodePool.nodeClassRef.name }} + group: {{ .Values.nodePool.nodeClassRef.group }} + kind: {{ .Values.nodePool.nodeClassRef.kind }} {{- with .Values.nodePool.taints }} taints: {{- toYaml . | nindent 8 }} @@ -27,8 +29,12 @@ spec: requirements: {{- toYaml . | nindent 8 }} {{- end }} + {{- if .Values.nodePool.expireAfter }} expireAfter: {{ .Values.nodePool.expireAfter }} + {{- end }} + {{- if .Values.nodePool.terminationGracePeriod }} terminationGracePeriod: {{ .Values.nodePool.terminationGracePeriod }} + {{- end }} disruption: consolidationPolicy: {{ .Values.nodePool.disruption.consolidationPolicy }} consolidateAfter: {{ .Values.nodePool.disruption.consolidateAfter }} @@ -36,8 +42,16 @@ spec: budgets: {{- toYaml . | nindent 4 }} {{- end }} + {{- with .Values.nodePool.limits }} limits: + {{- if .cpu }} cpu: {{ .Values.nodePool.limits.cpu }} - memory: {{ .Values.nodePool.limits.memory }} + {{- end }} + {{- if .memory }} + memory: {{ .memory }} + {{- end }} + {{- end }} + {{- if .Values.nodePool.weight }} weight: {{ .Values.nodePool.weight }} + {{- end }} {{- end }} diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 10df2f4..9a09d9f 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -5,23 +5,23 @@ clusterName: test-cluster # EC2NodeClass specific values ec2NodeClass: enabled: true - amiFamily: "AL2" + amiFamily: bottlerocket amiSelectorTerms: - - alias: "bottlerocket@latest" + - alias: "Bottlerocket@latest" subnetSelectorTerms: tags: - Name: "subnet-name" + Name: subnet-name # id: "subnet-id" securityGroupSelectorTerms: - name: "security-group-name" + # name: security-group-name # id: "security-group-id" tags: - key1: "value1" - key2: "value2" - karpenterRole: "karpenter-role" + Name: example-cluster-node + # key2: "value2" + karpenterRole: karpenter-role metadataOptions: httpEndpoint: enabled - httpProtocolIPv6: disabled + httpProtocolIPv6: enabled httpPutResponseHopLimit: 2 httpTokens: required blockDeviceMappings: @@ -32,33 +32,50 @@ ec2NodeClass: encrypted: true deleteOnTermination: true detailedMonitoring: true - userData: | - #!/bin/bash - echo "User data script" - instanceStorePolicy: "instance-store-policy" + # userData: | + # #!/bin/bash + # echo "User data script" + instanceStorePolicy: RAID0 # NodePool specific values nodePool: + name: example-cluster-node-pool enabled: true - labels: - environment: production - team: devops - taints: - - key: "key1" - value: "value1" - effect: "NoSchedule" + # labels: + # environment: production + # team: devops + # taints: + # - key: "key1" + # value: "value1" + # effect: "NoSchedule" requirements: - - key: "instance-type" - operator: "In" - values: ["m5.large", "m5.xlarge"] - expireAfter: 720h - terminationGracePeriod: 48h + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: karpenter.io/os + operator: In + values: ["linux"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + - key: karpenter.k8s.aws/instance-hypervisor + operator: In + values: ["nitro"] + - key: karpenter.sh/capacity-type + operator: In + values: ["spot", "on-demand"] + nodeClassRef: + group: karpenter.k8s.aws + kind: Ec2NodeClass + name: example-clsuter-karpenter-node-class + expireAfter: 72h + terminationGracePeriod: 4h disruption: - consolidationPolicy: WhenEmpty + consolidationPolicy: WhenEmptyOrUnderutilized consolidateAfter: 30s - budgets: - - nodes: 10% - limits: - cpu: "1000" - memory: 1000Gi - weight: 10 + # budgets: + # - nodes: 10% + # limits: + # cpu: "1000" + # memory: 1000Gi + # weight: 10 diff --git a/main.tf b/main.tf index 9d22f6c..345e5cd 100644 --- a/main.tf +++ b/main.tf @@ -136,6 +136,10 @@ resource "helm_release" "karpenter_resources" { file("${path.module}/karpenter-resources/values.yaml") ] #### Default values overrides + set { + name = "ec2NodeClass.metadata.name" + value = var.cluster_name + } set { name = "clusterName" value = var.cluster_name @@ -157,33 +161,29 @@ resource "helm_release" "karpenter_resources" { value = var.subnets_name } set { - name = "ec2NodeClassc.subnetSelectorTerms.tags" - value = "kubernetes.io/cluster/${var.cluster_name}" - } - set { - name = "nodeClass.spec.subnetSelectorTerms.tags.cluster_tag_value" - value = "shared" - } - set { - name = "tags" - value = jsonencode(var.tags) + name = "ec2NodeClass.subnetSelectorTerms.tags" + value = "kubernetes.io/cluster/${var.cluster_name}: shared" } ### Node Class Inputs set { - name = "nodeClass.metadata.name" - value = "${var.cluster_name}-node-class" + name = "ec2NodeClass.securityGroupSelectorTerms.tags.Name" + value = "${var.cluster_name}-node" } set { - name = "nodeClass.spec.instanceProfile" - value = aws_iam_instance_profile.karpenter_node_instance_profile.name + name = "nodePool.nodeClassRef.name" + value = "${var.cluster_name}-node-class" } set { - name = "nodeClass.spec.securityGroupSelectorTerms.tags.Name" - value = "${var.cluster_name}-node" + name = "nodePool.name" + value = "${var.cluster_name}-karpenter-node-pool" } set { name = "nodeClass.cluster_name" value = var.cluster_name } + set { + name = "tags" + value = jsonencode(var.tags) + } } From fe05dc1e4c2f86a4861873e24c27329fe598682e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 30 Jan 2025 17:26:44 -0500 Subject: [PATCH 31/69] lints --- karpenter-resources/templates/node-class.yaml | 4 ++-- karpenter-resources/values.yaml | 3 ++- main.tf | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/karpenter-resources/templates/node-class.yaml b/karpenter-resources/templates/node-class.yaml index 4fce705..5a2a820 100644 --- a/karpenter-resources/templates/node-class.yaml +++ b/karpenter-resources/templates/node-class.yaml @@ -13,9 +13,9 @@ spec: {{- end }} subnetSelectorTerms: {{- if and (.Values.ec2NodeClass.subnetSelectorTerms) (hasKey .Values.ec2NodeClass.subnetSelectorTerms "tags") }} - - tags: + tags: {{- range $key, $value := .Values.ec2NodeClass.subnetSelectorTerms.tags }} - {{ $key }}: {{ $value | quote }} + {{ $key }}: {{ $value | quote }} {{- end }} {{- end }} {{- if and (.Values.ec2NodeClass.subnetSelectorTerms.id) (hasKey .Values.ec2NodeClass.subnetSelectorTerms "id") }} diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 9a09d9f..f985fe2 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -11,13 +11,14 @@ ec2NodeClass: subnetSelectorTerms: tags: Name: subnet-name + kubernetes.io/cluster/example-cluster: shared # id: "subnet-id" securityGroupSelectorTerms: # name: security-group-name # id: "security-group-id" tags: Name: example-cluster-node - # key2: "value2" + # key2: "value2" karpenterRole: karpenter-role metadataOptions: httpEndpoint: enabled diff --git a/main.tf b/main.tf index 345e5cd..79f14d1 100644 --- a/main.tf +++ b/main.tf @@ -161,8 +161,8 @@ resource "helm_release" "karpenter_resources" { value = var.subnets_name } set { - name = "ec2NodeClass.subnetSelectorTerms.tags" - value = "kubernetes.io/cluster/${var.cluster_name}: shared" + name = "ec2NodeClass.subnetSelectorTerms.tags.kubernetes.io/cluster/${var.cluster_name}" + value = "shared" } ### Node Class Inputs set { From ee201dd865900929425cb07f43f3f6070f045517 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 30 Jan 2025 17:35:35 -0500 Subject: [PATCH 32/69] data structures --- karpenter-resources/templates/node-class.yaml | 38 ++++++++++--------- karpenter-resources/values.yaml | 14 +++---- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/karpenter-resources/templates/node-class.yaml b/karpenter-resources/templates/node-class.yaml index 5a2a820..9afd362 100644 --- a/karpenter-resources/templates/node-class.yaml +++ b/karpenter-resources/templates/node-class.yaml @@ -7,31 +7,35 @@ spec: {{- if .Values.ec2NodeClass.amiFamily }} amiFamily: {{ .Values.ec2NodeClass.amiFamily }} {{- end }} - {{- if .Values.ec2NodeClass.amiSelectorTerms }} +{{- if .Values.ec2NodeClass.amiSelectorTerms }} amiSelectorTerms: {{- toYaml .Values.ec2NodeClass.amiSelectorTerms | nindent 4 }} {{- end }} subnetSelectorTerms: - {{- if and (.Values.ec2NodeClass.subnetSelectorTerms) (hasKey .Values.ec2NodeClass.subnetSelectorTerms "tags") }} - tags: - {{- range $key, $value := .Values.ec2NodeClass.subnetSelectorTerms.tags }} - {{ $key }}: {{ $value | quote }} + {{- range .Values.ec2NodeClass.subnetSelectorTerms }} + - {{- if .tags }} + tags: + {{- range $key, $value := .tags }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .id }} + id: {{ .id }} {{- end }} - {{- end }} - {{- if and (.Values.ec2NodeClass.subnetSelectorTerms.id) (hasKey .Values.ec2NodeClass.subnetSelectorTerms "id") }} - - id: {{ .Values.ec2NodeClass.subnetSelectorTerms.id }} {{- end }} securityGroupSelectorTerms: - {{- if and (.Values.ec2NodeClass.securityGroupSelectorTerms) (hasKey .Values.ec2NodeClass.securityGroupSelectorTerms "name") }} - - name: {{ .Values.ec2NodeClass.securityGroupSelectorTerms.name }} - {{- end }} - {{- if and (.Values.ec2NodeClass.securityGroupSelectorTerms) (hasKey .Values.ec2NodeClass.securityGroupSelectorTerms "id") }} - - id: {{ .Values.ec2NodeClass.securityGroupSelectorTerms.id }} - {{- end }} - {{- if and (.Values.ec2NodeClass.securityGroupSelectorTerms) (hasKey .Values.ec2NodeClass.securityGroupSelectorTerms "tags") }} - - tags: - {{- range $key, $value := .Values.ec2NodeClass.securityGroupSelectorTerms.tags }} + {{- range .Values.ec2NodeClass.securityGroupSelectorTerms }} + - {{- if .tags }} + tags: + {{- range $key, $value := .tags }} {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .id }} + id: {{ .id }} + {{- end }} + {{- if .name }} + name: {{ .name }} {{- end }} {{- end }} role: {{ .Values.ec2NodeClass.karpenterRole }} diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index f985fe2..151aba9 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -9,16 +9,14 @@ ec2NodeClass: amiSelectorTerms: - alias: "Bottlerocket@latest" subnetSelectorTerms: - tags: - Name: subnet-name - kubernetes.io/cluster/example-cluster: shared + - tags: + Name: subnet-name + kubernetes.io/cluster/example-cluster: shared # id: "subnet-id" securityGroupSelectorTerms: - # name: security-group-name - # id: "security-group-id" - tags: - Name: example-cluster-node - # key2: "value2" + - tags: + Name: example-cluster-node + # key2: "value2" karpenterRole: karpenter-role metadataOptions: httpEndpoint: enabled From 1f9a27c4494c63e56aa4a813fe78e3443536f95c Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 30 Jan 2025 17:42:58 -0500 Subject: [PATCH 33/69] cleanup --- karpenter-resources/values.yaml | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 151aba9..1ba534b 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -10,13 +10,11 @@ ec2NodeClass: - alias: "Bottlerocket@latest" subnetSelectorTerms: - tags: - Name: subnet-name + name: subnet-name kubernetes.io/cluster/example-cluster: shared - # id: "subnet-id" securityGroupSelectorTerms: - tags: - Name: example-cluster-node - # key2: "value2" + name: example-cluster-node karpenterRole: karpenter-role metadataOptions: httpEndpoint: enabled @@ -31,22 +29,12 @@ ec2NodeClass: encrypted: true deleteOnTermination: true detailedMonitoring: true - # userData: | - # #!/bin/bash - # echo "User data script" instanceStorePolicy: RAID0 # NodePool specific values nodePool: name: example-cluster-node-pool enabled: true - # labels: - # environment: production - # team: devops - # taints: - # - key: "key1" - # value: "value1" - # effect: "NoSchedule" requirements: - key: kubernetes.io/arch operator: In @@ -72,9 +60,3 @@ nodePool: disruption: consolidationPolicy: WhenEmptyOrUnderutilized consolidateAfter: 30s - # budgets: - # - nodes: 10% - # limits: - # cpu: "1000" - # memory: 1000Gi - # weight: 10 From 0e05d6f1ec64356e7eff98286a04e4b96e680908 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 30 Jan 2025 17:49:15 -0500 Subject: [PATCH 34/69] fmt --- karpenter-resources/templates/node-class.yaml | 10 +++++----- karpenter-resources/values.yaml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/karpenter-resources/templates/node-class.yaml b/karpenter-resources/templates/node-class.yaml index 9afd362..962cdae 100644 --- a/karpenter-resources/templates/node-class.yaml +++ b/karpenter-resources/templates/node-class.yaml @@ -7,7 +7,7 @@ spec: {{- if .Values.ec2NodeClass.amiFamily }} amiFamily: {{ .Values.ec2NodeClass.amiFamily }} {{- end }} -{{- if .Values.ec2NodeClass.amiSelectorTerms }} + {{- if .Values.ec2NodeClass.amiSelectorTerms }} amiSelectorTerms: {{- toYaml .Values.ec2NodeClass.amiSelectorTerms | nindent 4 }} {{- end }} @@ -39,10 +39,10 @@ spec: {{- end }} {{- end }} role: {{ .Values.ec2NodeClass.karpenterRole }} - {{- if and (.Values.ec2NodeClass.tags) (hasKey .Values.ec2NodeClass "tags") }} + {{- if .Values.ec2NodeClass.tags }} tags: {{- range $key, $value := .Values.ec2NodeClass.tags }} - {{ $key }}: {{ $value | quote }} + {{ $key }}: {{ $value | quote }} {{- end }} {{- end }} metadataOptions: @@ -50,9 +50,9 @@ spec: httpProtocolIPv6: {{ .Values.ec2NodeClass.metadataOptions.httpProtocolIPv6 }} httpPutResponseHopLimit: {{ .Values.ec2NodeClass.metadataOptions.httpPutResponseHopLimit }} httpTokens: {{ .Values.ec2NodeClass.metadataOptions.httpTokens }} - {{- with .Values.ec2NodeClass.blockDeviceMappings }} + {{- if .Values.ec2NodeClass.blockDeviceMappings }} blockDeviceMappings: - {{- toYaml . | nindent 4 }} + {{- toYaml .Values.ec2NodeClass.blockDeviceMappings | nindent 4 }} {{- end }} detailedMonitoring: {{ .Values.ec2NodeClass.detailedMonitoring }} {{- if .Values.ec2NodeClass.instanceStorePolicy }} diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 1ba534b..85b6c76 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -10,11 +10,11 @@ ec2NodeClass: - alias: "Bottlerocket@latest" subnetSelectorTerms: - tags: - name: subnet-name + Name: subnet-name kubernetes.io/cluster/example-cluster: shared securityGroupSelectorTerms: - tags: - name: example-cluster-node + Name: example-cluster-node karpenterRole: karpenter-role metadataOptions: httpEndpoint: enabled From a359fda4e8d2ff6df50b5b7e98339fc57633edb8 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 30 Jan 2025 17:55:38 -0500 Subject: [PATCH 35/69] index on securityGroup --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 79f14d1..e35f84f 100644 --- a/main.tf +++ b/main.tf @@ -166,7 +166,7 @@ resource "helm_release" "karpenter_resources" { } ### Node Class Inputs set { - name = "ec2NodeClass.securityGroupSelectorTerms.tags.Name" + name = "ec2NodeClass.securityGroupSelectorTerms[0].tags.Name" value = "${var.cluster_name}-node" } set { From aafe5d3576e94862ebcd0b0fb9fec30d09ef5552 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 30 Jan 2025 17:58:23 -0500 Subject: [PATCH 36/69] indexes --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index e35f84f..0b34b1f 100644 --- a/main.tf +++ b/main.tf @@ -157,11 +157,11 @@ resource "helm_release" "karpenter_resources" { value = local.amd_ami_alias } set { - name = "ec2NodeClass.subnetSelectorTerms.tags.Name" + name = "ec2NodeClass.subnetSelectorTerms[0].tags.Name" value = var.subnets_name } set { - name = "ec2NodeClass.subnetSelectorTerms.tags.kubernetes.io/cluster/${var.cluster_name}" + name = "ec2NodeClass.subnetSelectorTerms[0].tags.kubernetes.io/cluster/${var.cluster_name}" value = "shared" } ### Node Class Inputs From e3ce602aac5ded710fdbf6fa4d20e012bbf4949a Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 30 Jan 2025 18:09:13 -0500 Subject: [PATCH 37/69] swap role for instanceProfile --- karpenter-resources/templates/node-class.yaml | 2 +- main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/karpenter-resources/templates/node-class.yaml b/karpenter-resources/templates/node-class.yaml index 962cdae..d81908e 100644 --- a/karpenter-resources/templates/node-class.yaml +++ b/karpenter-resources/templates/node-class.yaml @@ -38,7 +38,7 @@ spec: name: {{ .name }} {{- end }} {{- end }} - role: {{ .Values.ec2NodeClass.karpenterRole }} + instanceProfile: {{ .Values.ec2NodeClass.karpenterRole }} {{- if .Values.ec2NodeClass.tags }} tags: {{- range $key, $value := .Values.ec2NodeClass.tags }} diff --git a/main.tf b/main.tf index 0b34b1f..179b052 100644 --- a/main.tf +++ b/main.tf @@ -146,7 +146,7 @@ resource "helm_release" "karpenter_resources" { } set { name = "ec2NodeClass.karpenterRole" - value = aws_iam_instance_profile.karpenter_node_instance_profile.name + value = aws_iam_instance_profile.karpenter_node_instance_profile } set { name = "ec2NodeClass.amiFamily" From 7c010ed59bbbfa22b5fe6abba54bbc3dd740679d Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 30 Jan 2025 18:10:37 -0500 Subject: [PATCH 38/69] .name --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 179b052..0b34b1f 100644 --- a/main.tf +++ b/main.tf @@ -146,7 +146,7 @@ resource "helm_release" "karpenter_resources" { } set { name = "ec2NodeClass.karpenterRole" - value = aws_iam_instance_profile.karpenter_node_instance_profile + value = aws_iam_instance_profile.karpenter_node_instance_profile.name } set { name = "ec2NodeClass.amiFamily" From 586c33f3c5500d4ddfcc1646c7c975fc02cf4577 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 3 Feb 2025 17:10:20 -0500 Subject: [PATCH 39/69] comment out bad default value --- karpenter-resources/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 85b6c76..8e69757 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -11,7 +11,7 @@ ec2NodeClass: subnetSelectorTerms: - tags: Name: subnet-name - kubernetes.io/cluster/example-cluster: shared + # kubernetes.io/cluster/example-cluster: shared securityGroupSelectorTerms: - tags: Name: example-cluster-node From f77bce82674e0989eb758a32e51afdd228288c27 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 6 Feb 2025 14:45:37 -0500 Subject: [PATCH 40/69] not a map --- .pre-commit-config.yaml | 2 +- .tflint.hcl | 26 +++++++++++++------------- main.tf | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c93668a..6dcd874 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,7 +50,7 @@ repos: # Terraform Hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.96.1 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases + rev: v1.97.3 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases hooks: - id: terraform_fmt args: diff --git a/.tflint.hcl b/.tflint.hcl index 684d807..553e56f 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -4,18 +4,18 @@ config { disabled_by_default = false } -rule "aws_instance_invalid_type" { - enabled = true -} +# rule "aws_instance_invalid_type" { +# enabled = true +# } -plugin "aws" { - enabled = true - version = "0.32.0" - source = "github.com/terraform-linters/tflint-ruleset-aws" -} +# plugin "aws" { +# enabled = true +# version = "0.37.0" +# source = "github.com/terraform-linters/tflint-ruleset-aws" +# } -plugin "terraform" { - enabled = true - version = "0.9.0" - source = "github.com/terraform-linters/tflint-ruleset-terraform" -} +# plugin "terraform" { +# enabled = true +# version = "0.10.0" +# source = "github.com/terraform-linters/tflint-ruleset-terraform" +# } diff --git a/main.tf b/main.tf index 0b34b1f..76ab2d5 100644 --- a/main.tf +++ b/main.tf @@ -161,7 +161,7 @@ resource "helm_release" "karpenter_resources" { value = var.subnets_name } set { - name = "ec2NodeClass.subnetSelectorTerms[0].tags.kubernetes.io/cluster/${var.cluster_name}" + name = "ec2NodeClass.subnetSelectorTerms[0].tags.\"kubernetes.io/cluster/${var.cluster_name}\"" value = "shared" } ### Node Class Inputs From c2f6a6864118a4319dd1a724fa96c365b3227d50 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 6 Feb 2025 14:45:53 -0500 Subject: [PATCH 41/69] not a map --- .tflint.hcl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.tflint.hcl b/.tflint.hcl index 553e56f..8a31a41 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -14,8 +14,8 @@ config { # source = "github.com/terraform-linters/tflint-ruleset-aws" # } -# plugin "terraform" { -# enabled = true -# version = "0.10.0" -# source = "github.com/terraform-linters/tflint-ruleset-terraform" -# } +plugin "terraform" { + enabled = true + version = "0.10.0" + source = "github.com/terraform-linters/tflint-ruleset-terraform" +} From 73030722231b85c7d24f82cc2389b97344fb7fa1 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 6 Feb 2025 14:46:07 -0500 Subject: [PATCH 42/69] not a map --- .tflint.hcl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.tflint.hcl b/.tflint.hcl index 8a31a41..ac37999 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -8,11 +8,11 @@ config { # enabled = true # } -# plugin "aws" { -# enabled = true -# version = "0.37.0" -# source = "github.com/terraform-linters/tflint-ruleset-aws" -# } +plugin "aws" { + enabled = true + version = "0.37.0" + source = "github.com/terraform-linters/tflint-ruleset-aws" +} plugin "terraform" { enabled = true From 4b61f0f496f24a068b7e18d01c9dd0013f7e52f7 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 6 Feb 2025 14:46:24 -0500 Subject: [PATCH 43/69] not a map --- .tflint.hcl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.tflint.hcl b/.tflint.hcl index ac37999..8eb8249 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -4,9 +4,9 @@ config { disabled_by_default = false } -# rule "aws_instance_invalid_type" { -# enabled = true -# } +rule "aws_instance_invalid_type" { + enabled = true +} plugin "aws" { enabled = true From d5aefcf93b47845d86bbe3fb6c2b303952f4686e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 6 Feb 2025 14:55:32 -0500 Subject: [PATCH 44/69] remove commented value --- karpenter-resources/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 8e69757..d12cf97 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -11,7 +11,6 @@ ec2NodeClass: subnetSelectorTerms: - tags: Name: subnet-name - # kubernetes.io/cluster/example-cluster: shared securityGroupSelectorTerms: - tags: Name: example-cluster-node From 1103cc1a63b85499ba0115d9d09cafae4841c9e0 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 6 Feb 2025 20:56:40 -0500 Subject: [PATCH 45/69] weird tflint errors --- .tflint.hcl | 24 ++++++++++-------------- README.md | 3 +-- main.tf | 11 +++-------- 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/.tflint.hcl b/.tflint.hcl index 8eb8249..daa0227 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -4,18 +4,14 @@ config { disabled_by_default = false } -rule "aws_instance_invalid_type" { - enabled = true -} - -plugin "aws" { - enabled = true - version = "0.37.0" - source = "github.com/terraform-linters/tflint-ruleset-aws" -} +# plugin "aws" { +# enabled = true +# version = "0.37.0" +# source = "github.com/terraform-linters/tflint-ruleset-aws" +# } -plugin "terraform" { - enabled = true - version = "0.10.0" - source = "github.com/terraform-linters/tflint-ruleset-terraform" -} +# plugin "terraform" { +# enabled = true +# version = "0.10.0" +# source = "github.com/terraform-linters/tflint-ruleset-terraform" +# } diff --git a/README.md b/README.md index dc20882..536b2cd 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,8 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Type | |------|------| -| [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.karpenter_resources](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.nodepool_resources](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | diff --git a/main.tf b/main.tf index 76ab2d5..9f6816e 100644 --- a/main.tf +++ b/main.tf @@ -117,16 +117,11 @@ resource "helm_release" "karpenter" { timeout = 300 } -resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { - name = "${var.cluster_name}-karpenter-instance-profile" - role = module.karpenter_resources.node_iam_role_name -} - -resource "helm_release" "karpenter_resources" { +resource "helm_release" "nodepool_resources" { depends_on = [ helm_release.karpenter ] - name = "karpenter-resources" + name = "nodepool-resources" description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" chart = "./karpenter-resources" version = local.module_version @@ -146,7 +141,7 @@ resource "helm_release" "karpenter_resources" { } set { name = "ec2NodeClass.karpenterRole" - value = aws_iam_instance_profile.karpenter_node_instance_profile.name + value = module.karpenter_resources.node_iam_role_name } set { name = "ec2NodeClass.amiFamily" From 52414a1099717cc83549515076dea3a208bb54de Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 6 Feb 2025 20:58:14 -0500 Subject: [PATCH 46/69] plugins --- .tflint.hcl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.tflint.hcl b/.tflint.hcl index daa0227..02a846b 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -4,14 +4,14 @@ config { disabled_by_default = false } -# plugin "aws" { -# enabled = true -# version = "0.37.0" -# source = "github.com/terraform-linters/tflint-ruleset-aws" -# } +plugin "aws" { + enabled = true + version = "0.37.0" + source = "github.com/terraform-linters/tflint-ruleset-aws" +} -# plugin "terraform" { -# enabled = true -# version = "0.10.0" -# source = "github.com/terraform-linters/tflint-ruleset-terraform" -# } +plugin "terraform" { + enabled = true + version = "0.10.0" + source = "github.com/terraform-linters/tflint-ruleset-terraform" +} From 5d4265b4679ffca5159774031f9e777e69653b3b Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 6 Feb 2025 21:43:21 -0500 Subject: [PATCH 47/69] retry --- .tflint.hcl | 17 ----------------- karpenter-resources/values.yaml | 6 +++--- main.tf | 10 +++++----- 3 files changed, 8 insertions(+), 25 deletions(-) delete mode 100644 .tflint.hcl diff --git a/.tflint.hcl b/.tflint.hcl deleted file mode 100644 index 02a846b..0000000 --- a/.tflint.hcl +++ /dev/null @@ -1,17 +0,0 @@ -config { - module = true - force = false - disabled_by_default = false -} - -plugin "aws" { - enabled = true - version = "0.37.0" - source = "github.com/terraform-linters/tflint-ruleset-aws" -} - -plugin "terraform" { - enabled = true - version = "0.10.0" - source = "github.com/terraform-linters/tflint-ruleset-terraform" -} diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index d12cf97..68515b4 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -8,12 +8,12 @@ ec2NodeClass: amiFamily: bottlerocket amiSelectorTerms: - alias: "Bottlerocket@latest" - subnetSelectorTerms: - - tags: - Name: subnet-name securityGroupSelectorTerms: - tags: Name: example-cluster-node + subnetSelectorTerms: + - tags: + Name: subnet-name karpenterRole: karpenter-role metadataOptions: httpEndpoint: enabled diff --git a/main.tf b/main.tf index 9f6816e..22b6bc6 100644 --- a/main.tf +++ b/main.tf @@ -132,7 +132,7 @@ resource "helm_release" "nodepool_resources" { ] #### Default values overrides set { - name = "ec2NodeClass.metadata.name" + name = "name" value = var.cluster_name } set { @@ -151,6 +151,10 @@ resource "helm_release" "nodepool_resources" { name = "ec2NodeClass.amiSelectorTerms[0].alias" value = local.amd_ami_alias } + set { + name = "ec2NodeClass.securityGroupSelectorTerms[0].tags.Name" + value = "${var.cluster_name}-node" + } set { name = "ec2NodeClass.subnetSelectorTerms[0].tags.Name" value = var.subnets_name @@ -160,10 +164,6 @@ resource "helm_release" "nodepool_resources" { value = "shared" } ### Node Class Inputs - set { - name = "ec2NodeClass.securityGroupSelectorTerms[0].tags.Name" - value = "${var.cluster_name}-node" - } set { name = "nodePool.nodeClassRef.name" value = "${var.cluster_name}-node-class" From a2f721d4705faceb73692e60a85775cd58a11333 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 12:45:41 -0500 Subject: [PATCH 48/69] parsing errors --- karpenter-resources/templates/node-class.yaml | 8 ++++ karpenter-resources/values.yaml | 45 ++++++------------- 2 files changed, 21 insertions(+), 32 deletions(-) diff --git a/karpenter-resources/templates/node-class.yaml b/karpenter-resources/templates/node-class.yaml index d81908e..9610c63 100644 --- a/karpenter-resources/templates/node-class.yaml +++ b/karpenter-resources/templates/node-class.yaml @@ -11,6 +11,7 @@ spec: amiSelectorTerms: {{- toYaml .Values.ec2NodeClass.amiSelectorTerms | nindent 4 }} {{- end }} + {{- if .Values.ec2NodeClass.subnetSelectorTerms }} subnetSelectorTerms: {{- range .Values.ec2NodeClass.subnetSelectorTerms }} - {{- if .tags }} @@ -23,6 +24,8 @@ spec: id: {{ .id }} {{- end }} {{- end }} + {{- end }} + {{- if .Values.ec2NodeClass.securityGroupSelectorTerms }} securityGroupSelectorTerms: {{- range .Values.ec2NodeClass.securityGroupSelectorTerms }} - {{- if .tags }} @@ -38,6 +41,7 @@ spec: name: {{ .name }} {{- end }} {{- end }} + {{- end }} instanceProfile: {{ .Values.ec2NodeClass.karpenterRole }} {{- if .Values.ec2NodeClass.tags }} tags: @@ -47,8 +51,12 @@ spec: {{- end }} metadataOptions: httpEndpoint: {{ .Values.ec2NodeClass.metadataOptions.httpEndpoint }} + {{- if .Values.ec2NodeClass.metadataOptions.httpProtocolIPv6 }} httpProtocolIPv6: {{ .Values.ec2NodeClass.metadataOptions.httpProtocolIPv6 }} + {{- end }} + {{- if .Values.ec2NodeClass.metadataOptions.httpPutResponseHopLimit }} httpPutResponseHopLimit: {{ .Values.ec2NodeClass.metadataOptions.httpPutResponseHopLimit }} + {{- end }} httpTokens: {{ .Values.ec2NodeClass.metadataOptions.httpTokens }} {{- if .Values.ec2NodeClass.blockDeviceMappings }} blockDeviceMappings: diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 68515b4..b4ad2c1 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -8,54 +8,35 @@ ec2NodeClass: amiFamily: bottlerocket amiSelectorTerms: - alias: "Bottlerocket@latest" - securityGroupSelectorTerms: - - tags: - Name: example-cluster-node subnetSelectorTerms: - tags: Name: subnet-name + securityGroupSelectorTerms: + - tags: + Name: security-group-name karpenterRole: karpenter-role metadataOptions: httpEndpoint: enabled - httpProtocolIPv6: enabled + httpProtocolIPv6: disabled httpPutResponseHopLimit: 2 httpTokens: required blockDeviceMappings: - deviceName: /dev/xvda ebs: - volumeSize: 100Gi - volumeType: gp3 - encrypted: true - deleteOnTermination: true + volumeSize: 20 + volumeType: gp2 detailedMonitoring: true instanceStorePolicy: RAID0 + tags: + Environment: production + Project: karpenter + userData: | + #!/bin/bash + echo "Hello, World!" # NodePool specific values nodePool: name: example-cluster-node-pool enabled: true - requirements: - - key: kubernetes.io/arch - operator: In - values: ["amd64"] - - key: karpenter.io/os - operator: In - values: ["linux"] - - key: karpenter.k8s.aws/instance-generation - operator: Gt - values: ["2"] - - key: karpenter.k8s.aws/instance-hypervisor - operator: In - values: ["nitro"] - - key: karpenter.sh/capacity-type - operator: In - values: ["spot", "on-demand"] nodeClassRef: - group: karpenter.k8s.aws - kind: Ec2NodeClass - name: example-clsuter-karpenter-node-class - expireAfter: 72h - terminationGracePeriod: 4h - disruption: - consolidationPolicy: WhenEmptyOrUnderutilized - consolidateAfter: 30s + name: default From dde25098a02b15ecf7dfd2bf9bf243253861b441 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 12:49:39 -0500 Subject: [PATCH 49/69] update values --- karpenter-resources/values.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index b4ad2c1..137c8f8 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -40,3 +40,13 @@ nodePool: enabled: true nodeClassRef: name: default + disruption: + consolidationPolicy: "Consolidate" + consolidateAfter: "5m" + budgets: + maxUnavailable: 1 + maxUnavailablePercentage: 10 + limits: + cpu: "1000m" + memory: "1Gi" + weight: 1 From 30e69c0c0eac3c863e30f1cba7ab1f6f9b74f203 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 12:51:42 -0500 Subject: [PATCH 50/69] add budgets --- karpenter-resources/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 137c8f8..79e0fdf 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -40,6 +40,8 @@ nodePool: enabled: true nodeClassRef: name: default + group: karpenter.k8s.aws + kind: EC2NodeClass disruption: consolidationPolicy: "Consolidate" consolidateAfter: "5m" From ec3de741101e2cee0e70bf8faff3e4d506fd841c Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 12:54:28 -0500 Subject: [PATCH 51/69] update nodepool.yaml --- karpenter-resources/templates/node-pool.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/karpenter-resources/templates/node-pool.yaml b/karpenter-resources/templates/node-pool.yaml index 98f8f11..ac8e368 100644 --- a/karpenter-resources/templates/node-pool.yaml +++ b/karpenter-resources/templates/node-pool.yaml @@ -11,8 +11,8 @@ spec: NodeGroupType: {{ .Values.nodePool.name }} {{- if .Values.nodePool.labels }} {{- with .Values.nodePool.labels }} - {{- range . }} - {{- toYaml . | nindent 8 }} + {{- range $key, $value := . }} + {{ $key }}: {{ $value | quote }} {{- end }} {{- end }} {{- end }} @@ -45,7 +45,7 @@ spec: {{- with .Values.nodePool.limits }} limits: {{- if .cpu }} - cpu: {{ .Values.nodePool.limits.cpu }} + cpu: {{ .cpu }} {{- end }} {{- if .memory }} memory: {{ .memory }} From a1603ff3a012ab525be13a57f0c82bd2018a7c09 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 14:16:38 -0500 Subject: [PATCH 52/69] set limits --- main.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/main.tf b/main.tf index 22b6bc6..296e006 100644 --- a/main.tf +++ b/main.tf @@ -176,6 +176,14 @@ resource "helm_release" "nodepool_resources" { name = "nodeClass.cluster_name" value = var.cluster_name } + set { + name = "nodePool.limits.cpu" + value = "1000m" + } + set { + name = "nodePool.limits.memory" + value = "1000Gi" + } set { name = "tags" value = jsonencode(var.tags) From 89161d5080ed858a27c4d466f12fd6cbd0cc72fb Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 14:29:24 -0500 Subject: [PATCH 53/69] more values --- karpenter-resources/values.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 79e0fdf..0ed6d07 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -52,3 +52,8 @@ nodePool: cpu: "1000m" memory: "1Gi" weight: 1 + labels: {} + taints: [] + requirements: [] + expireAfter: "" + terminationGracePeriod: "" From d2a223ed0c7e1c57eed2fdb9005371061218b2d1 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 14:41:13 -0500 Subject: [PATCH 54/69] fmt --- karpenter-resources/templates/node-class.yaml | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/karpenter-resources/templates/node-class.yaml b/karpenter-resources/templates/node-class.yaml index 9610c63..c66a8b4 100644 --- a/karpenter-resources/templates/node-class.yaml +++ b/karpenter-resources/templates/node-class.yaml @@ -14,35 +14,37 @@ spec: {{- if .Values.ec2NodeClass.subnetSelectorTerms }} subnetSelectorTerms: {{- range .Values.ec2NodeClass.subnetSelectorTerms }} - - {{- if .tags }} - tags: - {{- range $key, $value := .tags }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} - {{- if .id }} - id: {{ .id }} + {{- if .tags }} + - tags: + {{- range $key, $value := .tags }} + {{ $key }}: {{ $value | quote }} {{- end }} {{- end }} + {{- if .id }} + id: {{ .id }} + {{- end }} + {{- end }} {{- end }} {{- if .Values.ec2NodeClass.securityGroupSelectorTerms }} securityGroupSelectorTerms: {{- range .Values.ec2NodeClass.securityGroupSelectorTerms }} - - {{- if .tags }} - tags: - {{- range $key, $value := .tags }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} - {{- if .id }} - id: {{ .id }} - {{- end }} - {{- if .name }} - name: {{ .name }} + {{- if .tags }} + - tags: + {{- range $key, $value := .tags }} + {{ $key }}: {{ $value | quote }} {{- end }} {{- end }} + {{- if .id }} + id: {{ .id }} + {{- end }} + {{- if .name }} + name: {{ .name }} + {{- end }} + {{- end }} {{- end }} + {{- if .Values.ec2NodeClass.karpenterRole }} instanceProfile: {{ .Values.ec2NodeClass.karpenterRole }} + {{- end }} {{- if .Values.ec2NodeClass.tags }} tags: {{- range $key, $value := .Values.ec2NodeClass.tags }} From ac8b3594a8c056f04bd7fc4bcce212e30ecac0a8 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 15:32:40 -0500 Subject: [PATCH 55/69] comment likely problem --- main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main.tf b/main.tf index 296e006..66f3764 100644 --- a/main.tf +++ b/main.tf @@ -159,10 +159,10 @@ resource "helm_release" "nodepool_resources" { name = "ec2NodeClass.subnetSelectorTerms[0].tags.Name" value = var.subnets_name } - set { - name = "ec2NodeClass.subnetSelectorTerms[0].tags.\"kubernetes.io/cluster/${var.cluster_name}\"" - value = "shared" - } + # set { + # name = "ec2NodeClass.subnetSelectorTerms[0].tags.\"kubernetes.io/cluster/${var.cluster_name}\"" + # value = "shared" + # } ### Node Class Inputs set { name = "nodePool.nodeClassRef.name" From 4fe14405cf9fb8b36c0228798e2efe0004abb992 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 15:38:03 -0500 Subject: [PATCH 56/69] update values --- karpenter-resources/values.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 0ed6d07..cd5c4ab 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -17,18 +17,17 @@ ec2NodeClass: karpenterRole: karpenter-role metadataOptions: httpEndpoint: enabled - httpProtocolIPv6: disabled + httpProtocolIPv6: enabled httpPutResponseHopLimit: 2 httpTokens: required blockDeviceMappings: - deviceName: /dev/xvda ebs: - volumeSize: 20 - volumeType: gp2 + volumeSize: "40" + volumeType: gp3 detailedMonitoring: true instanceStorePolicy: RAID0 tags: - Environment: production Project: karpenter userData: | #!/bin/bash @@ -43,7 +42,7 @@ nodePool: group: karpenter.k8s.aws kind: EC2NodeClass disruption: - consolidationPolicy: "Consolidate" + consolidationPolicy: "WhenEmptyOrUnderutilized" consolidateAfter: "5m" budgets: maxUnavailable: 1 @@ -54,6 +53,9 @@ nodePool: weight: 1 labels: {} taints: [] - requirements: [] - expireAfter: "" - terminationGracePeriod: "" + requirements: + - key: "kubernetes.io/arch" + operator: "In" + values: ["amd64"] + expireAfter: "30s" + terminationGracePeriod: "4h" From 58d7f95e4cd5b0637b75e6298b5fcac884cc05e5 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 15:51:26 -0500 Subject: [PATCH 57/69] more --- karpenter-resources/values.yaml | 2 +- main.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index cd5c4ab..3181a09 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -23,7 +23,7 @@ ec2NodeClass: blockDeviceMappings: - deviceName: /dev/xvda ebs: - volumeSize: "40" + volumeSize: "40Gi" volumeType: gp3 detailedMonitoring: true instanceStorePolicy: RAID0 diff --git a/main.tf b/main.tf index 66f3764..adbcb68 100644 --- a/main.tf +++ b/main.tf @@ -152,11 +152,11 @@ resource "helm_release" "nodepool_resources" { value = local.amd_ami_alias } set { - name = "ec2NodeClass.securityGroupSelectorTerms[0].tags.Name" + name = "ec2NodeClass.securityGroupSelectorTerms[0].tags.name" value = "${var.cluster_name}-node" } set { - name = "ec2NodeClass.subnetSelectorTerms[0].tags.Name" + name = "ec2NodeClass.subnetSelectorTerms[0].tags.name" value = var.subnets_name } # set { From 14f6a11b0db52809861ef59756a1a0895fa31fd2 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 15:53:55 -0500 Subject: [PATCH 58/69] even more --- karpenter-resources/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 3181a09..e058825 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -10,10 +10,10 @@ ec2NodeClass: - alias: "Bottlerocket@latest" subnetSelectorTerms: - tags: - Name: subnet-name + name: subnet-name securityGroupSelectorTerms: - tags: - Name: security-group-name + name: security-group-name karpenterRole: karpenter-role metadataOptions: httpEndpoint: enabled From fab33ab76f4296e922a13cd1169ffded6874bbf9 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 15:58:20 -0500 Subject: [PATCH 59/69] undo --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index adbcb68..66f3764 100644 --- a/main.tf +++ b/main.tf @@ -152,11 +152,11 @@ resource "helm_release" "nodepool_resources" { value = local.amd_ami_alias } set { - name = "ec2NodeClass.securityGroupSelectorTerms[0].tags.name" + name = "ec2NodeClass.securityGroupSelectorTerms[0].tags.Name" value = "${var.cluster_name}-node" } set { - name = "ec2NodeClass.subnetSelectorTerms[0].tags.name" + name = "ec2NodeClass.subnetSelectorTerms[0].tags.Name" value = var.subnets_name } # set { From 9edcc20a57044ef1283fd1fa8c7a57f372ccf680 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 16:00:58 -0500 Subject: [PATCH 60/69] redo --- karpenter-resources/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index e058825..3181a09 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -10,10 +10,10 @@ ec2NodeClass: - alias: "Bottlerocket@latest" subnetSelectorTerms: - tags: - name: subnet-name + Name: subnet-name securityGroupSelectorTerms: - tags: - name: security-group-name + Name: security-group-name karpenterRole: karpenter-role metadataOptions: httpEndpoint: enabled From b556fbb91a41cecbcb3768c69df985ebe859d9bc Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 7 Feb 2025 16:02:36 -0500 Subject: [PATCH 61/69] undo --- karpenter-resources/templates/node-class.yaml | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/karpenter-resources/templates/node-class.yaml b/karpenter-resources/templates/node-class.yaml index c66a8b4..9610c63 100644 --- a/karpenter-resources/templates/node-class.yaml +++ b/karpenter-resources/templates/node-class.yaml @@ -14,37 +14,35 @@ spec: {{- if .Values.ec2NodeClass.subnetSelectorTerms }} subnetSelectorTerms: {{- range .Values.ec2NodeClass.subnetSelectorTerms }} - {{- if .tags }} - - tags: - {{- range $key, $value := .tags }} - {{ $key }}: {{ $value | quote }} + - {{- if .tags }} + tags: + {{- range $key, $value := .tags }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .id }} + id: {{ .id }} {{- end }} - {{- end }} - {{- if .id }} - id: {{ .id }} - {{- end }} {{- end }} {{- end }} {{- if .Values.ec2NodeClass.securityGroupSelectorTerms }} securityGroupSelectorTerms: {{- range .Values.ec2NodeClass.securityGroupSelectorTerms }} - {{- if .tags }} - - tags: - {{- range $key, $value := .tags }} - {{ $key }}: {{ $value | quote }} + - {{- if .tags }} + tags: + {{- range $key, $value := .tags }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .id }} + id: {{ .id }} + {{- end }} + {{- if .name }} + name: {{ .name }} {{- end }} - {{- end }} - {{- if .id }} - id: {{ .id }} - {{- end }} - {{- if .name }} - name: {{ .name }} - {{- end }} {{- end }} {{- end }} - {{- if .Values.ec2NodeClass.karpenterRole }} instanceProfile: {{ .Values.ec2NodeClass.karpenterRole }} - {{- end }} {{- if .Values.ec2NodeClass.tags }} tags: {{- range $key, $value := .Values.ec2NodeClass.tags }} From ec3ae8c4675b24eacd4697a6b3db0d77f4086291 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 11 Feb 2025 19:36:01 -0500 Subject: [PATCH 62/69] dont set default limits --- karpenter-resources/values.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 3181a09..145a43d 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -47,9 +47,7 @@ nodePool: budgets: maxUnavailable: 1 maxUnavailablePercentage: 10 - limits: - cpu: "1000m" - memory: "1Gi" + limits: {} weight: 1 labels: {} taints: [] From 76f61a4445241769f12b4f327af59b462192af08 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 11 Feb 2025 19:44:52 -0500 Subject: [PATCH 63/69] more reqs --- karpenter-resources/values.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 145a43d..f96d93a 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -55,5 +55,14 @@ nodePool: - key: "kubernetes.io/arch" operator: "In" values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] expireAfter: "30s" terminationGracePeriod: "4h" From ed1c8bf118e9cefe3d49f3034e5c50eb00e802e7 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 11 Feb 2025 20:10:11 -0500 Subject: [PATCH 64/69] update default limits --- karpenter-resources/values.yaml | 7 ++++++- main.tf | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index f96d93a..cb9c4a5 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -47,7 +47,9 @@ nodePool: budgets: maxUnavailable: 1 maxUnavailablePercentage: 10 - limits: {} + limits: + cpu: "1000" + memory: 1000Gi weight: 1 labels: {} taints: [] @@ -64,5 +66,8 @@ nodePool: - key: karpenter.k8s.aws/instance-generation operator: Gt values: ["2"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] expireAfter: "30s" terminationGracePeriod: "4h" diff --git a/main.tf b/main.tf index 66f3764..759383a 100644 --- a/main.tf +++ b/main.tf @@ -178,7 +178,7 @@ resource "helm_release" "nodepool_resources" { } set { name = "nodePool.limits.cpu" - value = "1000m" + value = "1000" } set { name = "nodePool.limits.memory" From c99fa51b5c0112bb1a2700d0c3cb37d7a673cfab Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 11 Feb 2025 20:12:35 -0500 Subject: [PATCH 65/69] update default limits --- karpenter-resources/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index cb9c4a5..35a3480 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -69,5 +69,5 @@ nodePool: - key: "karpenter.k8s.aws/instance-hypervisor" operator: In values: ["nitro"] - expireAfter: "30s" + expireAfter: "72h" terminationGracePeriod: "4h" From eccf52be08d1e6ea9ccf471da45a9dad9c6efc2f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 11 Feb 2025 20:17:06 -0500 Subject: [PATCH 66/69] tags maybe --- karpenter-resources/values.yaml | 6 +----- main.tf | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml index 35a3480..cc9886d 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml @@ -27,11 +27,7 @@ ec2NodeClass: volumeType: gp3 detailedMonitoring: true instanceStorePolicy: RAID0 - tags: - Project: karpenter - userData: | - #!/bin/bash - echo "Hello, World!" + # NodePool specific values nodePool: diff --git a/main.tf b/main.tf index 759383a..591d23c 100644 --- a/main.tf +++ b/main.tf @@ -185,7 +185,7 @@ resource "helm_release" "nodepool_resources" { value = "1000Gi" } set { - name = "tags" + name = "ec2NodeClass.tags" value = jsonencode(var.tags) } From 5d271f25ab8f8ca1ca7e6c5694eea33221a393e0 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 11 Feb 2025 20:25:34 -0500 Subject: [PATCH 67/69] dont look up the value guess it --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 591d23c..72a1388 100644 --- a/main.tf +++ b/main.tf @@ -141,7 +141,7 @@ resource "helm_release" "nodepool_resources" { } set { name = "ec2NodeClass.karpenterRole" - value = module.karpenter_resources.node_iam_role_name + value = module.karpenter_resources.iam_role_arn } set { name = "ec2NodeClass.amiFamily" From 4a676fceede3a0caac374872d88079d91554d3c3 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 11 Feb 2025 20:37:29 -0500 Subject: [PATCH 68/69] add instance profile again because govcloud --- README.md | 1 + main.tf | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 536b2cd..09d0981 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Type | |------|------| +| [aws_iam_instance_profile.karpenter_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.nodepool_resources](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | diff --git a/main.tf b/main.tf index 72a1388..5e5818e 100644 --- a/main.tf +++ b/main.tf @@ -22,6 +22,20 @@ module "karpenter_resources" { tags = var.tags } +# Create IAM Instance Profile for Karpenter Nodes +resource "aws_iam_instance_profile" "karpenter_node" { + name = "${var.cluster_name}-karpenter-node-profile" + role = module.karpenter_resources.node_iam_role_name + + tags = merge( + var.tags, + { + "karpenter.sh/managed-by" = var.cluster_name + "kubernetes.io/cluster/${var.cluster_name}" = "owned" + } + ) +} + # Install Karpenter data "aws_ecr_image" "karpenter_image" { depends_on = [ @@ -141,7 +155,7 @@ resource "helm_release" "nodepool_resources" { } set { name = "ec2NodeClass.karpenterRole" - value = module.karpenter_resources.iam_role_arn + value = aws_iam_instance_profile.karpenter_node.name } set { name = "ec2NodeClass.amiFamily" From 0c0e6ffbbc257fd914df4395f8ea559b08b36c78 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 12 Feb 2025 16:41:26 -0500 Subject: [PATCH 69/69] restore tflint config --- .tflint.hcl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .tflint.hcl diff --git a/.tflint.hcl b/.tflint.hcl new file mode 100644 index 0000000..684d807 --- /dev/null +++ b/.tflint.hcl @@ -0,0 +1,21 @@ +config { + module = true + force = false + disabled_by_default = false +} + +rule "aws_instance_invalid_type" { + enabled = true +} + +plugin "aws" { + enabled = true + version = "0.32.0" + source = "github.com/terraform-linters/tflint-ruleset-aws" +} + +plugin "terraform" { + enabled = true + version = "0.9.0" + source = "github.com/terraform-linters/tflint-ruleset-terraform" +}