Merge pull request #10 from SCT-Engineering/spots

Spots
SCT-Engineering · Oct 15, 2024 · 622d81b · 622d81b
2 parents 39beb8e + 44ed1ef
commit 622d81b
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 29 deletions.
diff --git a/aws_data.tf b/aws_data.tf
@@ -3,3 +3,5 @@ data "aws_caller_identity" "current" {}
 data "aws_arn" "current" {
   arn = data.aws_caller_identity.current.arn
 }
+
+data "aws_default_tags" "default_tags" {}
diff --git a/ec2_node_pool.yaml.tpl b/ec2_node_pool.yaml.tpl
@@ -11,24 +11,57 @@ spec:
       requirements:
         - key: kubernetes.io/arch
           operator: In
-          values: ["amd64"]
+          values: ["amd64", "arm64"]
         - key: kubernetes.io/os
           operator: In
-          values: ["linux"]
-        - key: karpenter.sh/capacity-type
+          values: ["linux", "windows"]
+        - key: "karpenter.k8s.aws/instance-hypervisor"
           operator: In
-          values: ["on-demand", "spot"]
-        - key: karpenter.k8s.aws/instance-category
+          values: ["nitro"]
+        - key: karpenter.sh/capacity-type
           operator: In
-          values: ["c", "m", "r", "t"]
-        - key: karpenter.k8s.aws/instance-generation
-          operator: Gt
-          values: ["2"]
+          values: ["spot", "on-demand"]
+        # - key: karpenter.k8s.aws/instance-category
+        #   operator: In
+        #   values: ["c", "m", "r", "i", "d", "t"]
+        # - key: karpenter.k8s.aws/instance-generation
+        #   operator: Gt
+        #   values: ["2"]
       nodeClassRef:
         group: karpenter.k8s.aws
         kind: EC2NodeClass
         name: ${cluster_name}-karpenter-node-class
+      kubelet:
+        containerRuntime: containerd
+        systemReserved:
+          cpu: 100m
+          memory: 100Mi
   disruption:
     consolidationPolicy: WhenEmptyOrUnderutilized
     consolidateAfter: 1h
-    expireAfter: 1h
+    expireAfter: 72h
+  limits:
+    cpu: 1000
+    memory: 1000Gi
+    nvidia.com/gpu: 2
+  topologySpreadConstraints:
+    - maxSkew: 1
+      topologyKey: "topology.kubernetes.io/zone"
+      whenUnsatisfiable: ScheduleAnyway
+    - maxSkew: 1
+      topologyKey: "kubernetes.io/hostname"
+      whenUnsatisfiable: ScheduleAnyway
+    - maxSkew: 1
+      topologyKey: "karpenter.sh/capacity-type"
+      whenUnsatisfiable: ScheduleAnyway
+  budgets:
+    - nodes: "50%"
+      reasons:
+      - "Empty"
+      - "Drifted"
+    - nodes: "5"
+    - nodes: "0"
+      schedule: "@daily"
+      duration: 10m
+      reasons:
+      - "Underutilized"
diff --git a/main.tf b/main.tf
@@ -132,22 +132,22 @@ resource "helm_release" "karpenter" {
     name  = "controller.env[0].value"
     value = var.region
   }
-  # set {
-  #   name  = "resources.requests.cpu"
-  #   value = 1
-  # }
-  # set {
-  #   name  = "resources.requests.memory"
-  #   value = "1Gi"
-  # }
-  # set {
-  #   name  = "resources.limits.cpu"
-  #   value = 1
-  # }
-  # set {
-  #   name  = "resources.limits.memory"
-  #   value = "1Gi"
-  # }
+  set {
+    name  = "resources.requests.cpu"
+    value = 1
+  }
+  set {
+    name  = "resources.requests.memory"
+    value = "1Gi"
+  }
+  set {
+    name  = "resources.limits.cpu"
+    value = 1
+  }
+  set {
+    name  = "resources.limits.memory"
+    value = "1Gi"
+  }
   set {
     name  = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
     value = module.karpenter_resources.iam_role_arn
@@ -161,9 +161,6 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" {
   role = module.karpenter_resources.node_iam_role_name
 }
 
-# Get the default tags from the provider
-data "aws_default_tags" "default_tags" {}
-
 # SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART
 # Create karpenter default
 resource "kubectl_manifest" "karpenter_ec2_node_class" {