From 581f06ff07bf524420312cc99603580fe6b3f003 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 10 Oct 2024 21:09:26 -0400 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(nodepool):=20even=20more=20gen?= =?UTF-8?q?eric=20workload=20nodepools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 - ec2_node_pool.yaml.tpl | 43 +++++++++++++++++++++++++++++++----- main.tf | 49 ++++++++++++++++++++---------------------- 3 files changed, 61 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index ee72a80..dc22017 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_default_tags.default_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | | [aws_ecr_image.kubectl_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | diff --git a/ec2_node_pool.yaml.tpl b/ec2_node_pool.yaml.tpl index 0da0ee8..4a9a138 100644 --- a/ec2_node_pool.yaml.tpl +++ b/ec2_node_pool.yaml.tpl @@ -11,16 +11,19 @@ spec: requirements: - key: kubernetes.io/arch operator: In - values: ["amd64"] + values: ["amd64", "arm64"] - key: kubernetes.io/os operator: In - values: ["linux"] + values: ["linux", "windows"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] - key: karpenter.sh/capacity-type operator: In - values: ["on-demand", "spot"] + values: ["spot", "on-demand"] - key: karpenter.k8s.aws/instance-category operator: In - values: ["c", "m", "r", "t"] + values: ["c", "m", "r", "i", "d", "t"] - key: karpenter.k8s.aws/instance-generation operator: Gt values: ["2"] @@ -28,7 +31,37 @@ spec: group: karpenter.k8s.aws kind: EC2NodeClass name: ${cluster_name}-karpenter-node-class + kubelet: + containerRuntime: containerd + systemReserved: + cpu: 100m + memory: 100Mi disruption: consolidationPolicy: WhenEmptyOrUnderutilized consolidateAfter: 1h - expireAfter: 1h + expireAfter: 72h + limits: + cpu: 1000 + memory: 1000Gi + nvidia.com/gpu: 2 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "topology.kubernetes.io/zone" + whenUnsatisfiable: ScheduleAnyway + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: ScheduleAnyway + - maxSkew: 1 + topologyKey: "karpenter.sh/capacity-type" + whenUnsatisfiable: ScheduleAnyway + budgets: + - nodes: "50%" + reasons: + - "Empty" + - "Drifted" + - nodes: "5" + - nodes: "0" + schedule: "@daily" + duration: 10m + reasons: + - "Underutilized" diff --git a/main.tf b/main.tf index 3343453..b2ad5a0 100644 --- a/main.tf +++ b/main.tf @@ -51,27 +51,27 @@ resource "helm_release" "karpenter" { create_namespace = true set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key" + name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key" value = "karpenter.sh/nodepool" } set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator" + name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator" value = "DoesNotExist" } set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].key" + name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].key" value = "eks.amazonaws.com/nodegroup" } set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].operator" + name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].operator" value = "In" } set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].values[0]" + name = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].values[0]" value = var.karpenter_node_group_name } set { - name = "affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey" + name = "affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].topologyKey" value = "kubernetes.io/hostname" } set { @@ -132,22 +132,22 @@ resource "helm_release" "karpenter" { name = "controller.env[0].value" value = var.region } - # set { - # name = "resources.requests.cpu" - # value = 1 - # } - # set { - # name = "resources.requests.memory" - # value = "1Gi" - # } - # set { - # name = "resources.limits.cpu" - # value = 1 - # } - # set { - # name = "resources.limits.memory" - # value = "1Gi" - # } + set { + name = "resources.requests.cpu" + value = 1 + } + set { + name = "resources.requests.memory" + value = "1Gi" + } + set { + name = "resources.limits.cpu" + value = 1 + } + set { + name = "resources.limits.memory" + value = "1Gi" + } set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" value = module.karpenter_resources.iam_role_arn @@ -161,9 +161,6 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { role = module.karpenter_resources.node_iam_role_name } -# Get the default tags from the provider -data "aws_default_tags" "default_tags" {} - # SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART # Create karpenter default resource "kubectl_manifest" "karpenter_ec2_node_class" { @@ -172,7 +169,7 @@ resource "kubectl_manifest" "karpenter_ec2_node_class" { ] yaml_body = templatefile("${path.module}/ec2_node_class.yaml.tpl", { cluster_name = var.cluster_name - tags = jsonencode(data.aws_default_tags.default_tags.tags) + tags = var.tags amd_ami_family = "Bottlerocket" amd_ami_alias = "bottlerocket" subnets_name = local.container_subnets_name