From 581f06ff07bf524420312cc99603580fe6b3f003 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 10 Oct 2024 21:09:26 -0400 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8=20feat(nodepool):=20even=20more?= =?UTF-8?q?=20generic=20workload=20nodepools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 - ec2_node_pool.yaml.tpl | 43 +++++++++++++++++++++++++++++++----- main.tf | 49 ++++++++++++++++++++---------------------- 3 files changed, 61 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index ee72a80..dc22017 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_default_tags.default_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | | [aws_ecr_image.kubectl_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | diff --git a/ec2_node_pool.yaml.tpl b/ec2_node_pool.yaml.tpl index 0da0ee8..4a9a138 100644 --- a/ec2_node_pool.yaml.tpl +++ b/ec2_node_pool.yaml.tpl @@ -11,16 +11,19 @@ spec: requirements: - key: kubernetes.io/arch operator: In - values: ["amd64"] + values: ["amd64", "arm64"] - key: kubernetes.io/os operator: In - values: ["linux"] + values: ["linux", "windows"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] - key: karpenter.sh/capacity-type operator: In - values: ["on-demand", "spot"] + values: ["spot", "on-demand"] - key: karpenter.k8s.aws/instance-category operator: In - values: ["c", "m", "r", "t"] + values: ["c", "m", "r", "i", "d", "t"] - key: karpenter.k8s.aws/instance-generation operator: Gt values: ["2"] @@ -28,7 +31,37 @@ spec: group: karpenter.k8s.aws kind: EC2NodeClass name: ${cluster_name}-karpenter-node-class + kubelet: + containerRuntime: containerd + systemReserved: + cpu: 100m + memory: 100Mi disruption: consolidationPolicy: WhenEmptyOrUnderutilized consolidateAfter: 1h - expireAfter: 1h + expireAfter: 72h + limits: + cpu: 1000 + memory: 1000Gi + nvidia.com/gpu: 2 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "topology.kubernetes.io/zone" + whenUnsatisfiable: ScheduleAnyway + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: ScheduleAnyway + - maxSkew: 1 + topologyKey: "karpenter.sh/capacity-type" + whenUnsatisfiable: ScheduleAnyway + budgets: + - nodes: "50%" + reasons: + - "Empty" + - "Drifted" + - nodes: "5" + - nodes: "0" + schedule: "@daily" + duration: 10m + reasons: + - "Underutilized" diff --git a/main.tf b/main.tf index 3343453..b2ad5a0 100644 --- a/main.tf +++ b/main.tf @@ -51,27 +51,27 @@ resource "helm_release" "karpenter" { create_namespace = true set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key" + name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key" value = "karpenter.sh/nodepool" } set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator" + name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator" value = "DoesNotExist" } set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].key" + name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].key" value = "eks.amazonaws.com/nodegroup" } set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].operator" + name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].operator" value = "In" } set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].values[0]" + name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].values[0]" value = var.karpenter_node_group_name } set { - name = "affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey" + name = "affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].topologyKey" value = "kubernetes.io/hostname" } set { @@ -132,22 +132,22 @@ resource "helm_release" "karpenter" { name = "controller.env[0].value" value = var.region } - # set { - # name = "resources.requests.cpu" - # value = 1 - # } - # set { - # name = "resources.requests.memory" - # value = "1Gi" - # } - # set { - # name = "resources.limits.cpu" - # value = 1 - # } - # set { - # name = "resources.limits.memory" - # value = "1Gi" - # } + set { + name = "resources.requests.cpu" + value = 1 + } + set { + name = "resources.requests.memory" + value = "1Gi" + } + set { + name = "resources.limits.cpu" + value = 1 + } + set { + name = "resources.limits.memory" + value = "1Gi" + } set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" value = module.karpenter_resources.iam_role_arn @@ -161,9 +161,6 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { role = module.karpenter_resources.node_iam_role_name } -# Get the default tags from the provider -data "aws_default_tags" "default_tags" {} - # SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART # Create karpenter default resource "kubectl_manifest" "karpenter_ec2_node_class" { @@ -172,7 +169,7 @@ resource "kubectl_manifest" "karpenter_ec2_node_class" { ] yaml_body = templatefile("${path.module}/ec2_node_class.yaml.tpl", { cluster_name = var.cluster_name - tags = jsonencode(data.aws_default_tags.default_tags.tags) + tags = var.tags amd_ami_family = "Bottlerocket" amd_ami_alias = "bottlerocket" subnets_name = local.container_subnets_name From 447eb593e32afc1faa9dc2ae0ab3d512d2d2cfdb Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 10 Oct 2024 21:34:25 -0400 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=90=9B=20fix(tags):=20revert=20tag=20?= =?UTF-8?q?changes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + aws_data.tf | 2 ++ main.tf | 14 +++++++------- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index dc22017..ee72a80 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_default_tags.default_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | | [aws_ecr_image.kubectl_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | diff --git a/aws_data.tf b/aws_data.tf index 2c6aade..e7211e8 100644 --- a/aws_data.tf +++ b/aws_data.tf @@ -3,3 +3,5 @@ data "aws_caller_identity" "current" {} data "aws_arn" "current" { arn = data.aws_caller_identity.current.arn } + +data "aws_default_tags" "default_tags" {} diff --git a/main.tf b/main.tf index b2ad5a0..6ec527d 100644 --- a/main.tf +++ b/main.tf @@ -51,27 +51,27 @@ resource "helm_release" "karpenter" { create_namespace = true set { - name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key" + name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key" value = "karpenter.sh/nodepool" } set { - name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator" + name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator" value = "DoesNotExist" } set { - name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].key" + name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].key" value = "eks.amazonaws.com/nodegroup" } set { - name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].operator" + name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].operator" value = "In" } set { - name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].values[0]" + name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].values[0]" value = var.karpenter_node_group_name } set { - name = "affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].topologyKey" + name = "affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey" value = "kubernetes.io/hostname" } set { @@ -169,7 +169,7 @@ resource "kubectl_manifest" "karpenter_ec2_node_class" { ] yaml_body = templatefile("${path.module}/ec2_node_class.yaml.tpl", { cluster_name = var.cluster_name - tags = var.tags + tags = jsonencode(data.aws_default_tags.default_tags.tags) amd_ami_family = "Bottlerocket" amd_ami_alias = "bottlerocket" subnets_name = local.container_subnets_name From 44ed1ef411f7b249207e268bd747eaf19ac12d3e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 10 Oct 2024 22:09:11 -0400 Subject: [PATCH 3/3] fewer constraints on instance types --- ec2_node_pool.yaml.tpl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ec2_node_pool.yaml.tpl b/ec2_node_pool.yaml.tpl index 4a9a138..1fcb874 100644 --- a/ec2_node_pool.yaml.tpl +++ b/ec2_node_pool.yaml.tpl @@ -21,12 +21,12 @@ spec: - key: karpenter.sh/capacity-type operator: In values: ["spot", "on-demand"] - - key: karpenter.k8s.aws/instance-category - operator: In - values: ["c", "m", "r", "i", "d", "t"] - - key: karpenter.k8s.aws/instance-generation - operator: Gt - values: ["2"] + # - key: karpenter.k8s.aws/instance-category + # operator: In + # values: ["c", "m", "r", "i", "d", "t"] + # - key: karpenter.k8s.aws/instance-generation + # operator: Gt + # values: ["2"] nodeClassRef: group: karpenter.k8s.aws kind: EC2NodeClass