✨ feat(nodepool): even more generic workload nodepools

SCT-Engineering · Oct 11, 2024 · 581f06f · 581f06f
1 parent 39beb8e
commit 581f06f
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 32 deletions.
diff --git a/README.md b/README.md
@@ -47,7 +47,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the
 | [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
 | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
-| [aws_default_tags.default_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source |
 | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source |
 | [aws_ecr_image.kubectl_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source |
 

diff --git a/ec2_node_pool.yaml.tpl b/ec2_node_pool.yaml.tpl
@@ -11,24 +11,57 @@ spec:
       requirements:
         - key: kubernetes.io/arch
           operator: In
-          values: ["amd64"]
+          values: ["amd64", "arm64"]
         - key: kubernetes.io/os
           operator: In
-          values: ["linux"]
+          values: ["linux", "windows"]
+        - key: "karpenter.k8s.aws/instance-hypervisor"
+          operator: In
+          values: ["nitro"]
         - key: karpenter.sh/capacity-type
           operator: In
-          values: ["on-demand", "spot"]
+          values: ["spot", "on-demand"]
         - key: karpenter.k8s.aws/instance-category
           operator: In
-          values: ["c", "m", "r", "t"]
+          values: ["c", "m", "r", "i", "d", "t"]
         - key: karpenter.k8s.aws/instance-generation
           operator: Gt
           values: ["2"]
       nodeClassRef:
         group: karpenter.k8s.aws
         kind: EC2NodeClass
         name: ${cluster_name}-karpenter-node-class
+      kubelet:
+        containerRuntime: containerd
+        systemReserved:
+          cpu: 100m
+          memory: 100Mi
   disruption:
     consolidationPolicy: WhenEmptyOrUnderutilized
     consolidateAfter: 1h
-    expireAfter: 1h
+    expireAfter: 72h
+  limits:
+    cpu: 1000
+    memory: 1000Gi
+    nvidia.com/gpu: 2
+  topologySpreadConstraints:
+    - maxSkew: 1
+      topologyKey: "topology.kubernetes.io/zone"
+      whenUnsatisfiable: ScheduleAnyway
+    - maxSkew: 1
+      topologyKey: "kubernetes.io/hostname"
+      whenUnsatisfiable: ScheduleAnyway
+    - maxSkew: 1
+      topologyKey: "karpenter.sh/capacity-type"
+      whenUnsatisfiable: ScheduleAnyway
+  budgets:
+    - nodes: "50%"
+      reasons:
+      - "Empty"
+      - "Drifted"
+    - nodes: "5"
+    - nodes: "0"
+      schedule: "@daily"
+      duration: 10m
+      reasons:
+      - "Underutilized"
diff --git a/main.tf b/main.tf
@@ -51,27 +51,27 @@ resource "helm_release" "karpenter" {
   create_namespace = true
 
   set {
-    name  = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key"
+    name  = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key"
     value = "karpenter.sh/nodepool"
   }
   set {
-    name  = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator"
+    name  = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator"
     value = "DoesNotExist"
   }
   set {
-    name  = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].key"
+    name  = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].key"
     value = "eks.amazonaws.com/nodegroup"
   }
   set {
-    name  = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].operator"
+    name  = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].operator"
     value = "In"
   }
   set {
-    name  = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].values[0]"
+    name  = "affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].values[0]"
     value = var.karpenter_node_group_name
   }
   set {
-    name  = "affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey"
+    name  = "affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].topologyKey"
     value = "kubernetes.io/hostname"
   }
   set {
@@ -132,22 +132,22 @@ resource "helm_release" "karpenter" {
     name  = "controller.env[0].value"
     value = var.region
   }
-  # set {
-  #   name  = "resources.requests.cpu"
-  #   value = 1
-  # }
-  # set {
-  #   name  = "resources.requests.memory"
-  #   value = "1Gi"
-  # }
-  # set {
-  #   name  = "resources.limits.cpu"
-  #   value = 1
-  # }
-  # set {
-  #   name  = "resources.limits.memory"
-  #   value = "1Gi"
-  # }
+  set {
+    name  = "resources.requests.cpu"
+    value = 1
+  }
+  set {
+    name  = "resources.requests.memory"
+    value = "1Gi"
+  }
+  set {
+    name  = "resources.limits.cpu"
+    value = 1
+  }
+  set {
+    name  = "resources.limits.memory"
+    value = "1Gi"
+  }
   set {
     name  = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
     value = module.karpenter_resources.iam_role_arn
@@ -161,9 +161,6 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" {
   role = module.karpenter_resources.node_iam_role_name
 }
 
-# Get the default tags from the provider
-data "aws_default_tags" "default_tags" {}
-
 # SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART
 # Create karpenter default
 resource "kubectl_manifest" "karpenter_ec2_node_class" {
@@ -172,7 +169,7 @@ resource "kubectl_manifest" "karpenter_ec2_node_class" {
   ]
   yaml_body = templatefile("${path.module}/ec2_node_class.yaml.tpl", {
     cluster_name   = var.cluster_name
-    tags           = jsonencode(data.aws_default_tags.default_tags.tags)
+    tags           = var.tags
     amd_ami_family = "Bottlerocket"
     amd_ami_alias  = "bottlerocket"
     subnets_name   = local.container_subnets_name