🐛 fix(stabilty): cluster was most stable when the least constraints w…

…ere placed on karpenter so basically, let karpenter figure it out
SCT-Engineering · Oct 23, 2024 · 02c4b90 · 02c4b90
1 parent 2c169a9
commit 02c4b90
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 51 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.6.0
+  rev: v5.0.0
   hooks:
     # Git style
     - id: check-added-large-files
@@ -49,7 +49,7 @@ repos:
 
 # Terraform Hooks
 - repo: https://github.com/antonbabenko/pre-commit-terraform
-  rev: v1.92.1 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases
+  rev: v1.96.1 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases
   hooks:
     - id: terraform_fmt
       args:
@@ -106,6 +106,6 @@ repos:
     #     - --hook-config=--parallelism-ci-cpu-cores=2
 
 - repo: https://github.com/ljnsn/cz-conventional-gitmoji
-  rev: v0.3.2
+  rev: v0.6.1
   hooks:
     - id: conventional-gitmoji
diff --git a/ec2_node_pool.yaml.tpl b/ec2_node_pool.yaml.tpl
@@ -31,37 +31,37 @@ spec:
         group: karpenter.k8s.aws
         kind: EC2NodeClass
         name: ${cluster_name}-karpenter-node-class
-      kubelet:
-        containerRuntime: containerd
-        systemReserved:
-          cpu: 100m
-          memory: 100Mi
+      # kubelet:
+      #   containerRuntime: containerd
+      #   systemReserved:
+      #     cpu: 100m
+      #     memory: 100Mi
   disruption:
     consolidationPolicy: WhenEmptyOrUnderutilized
-    consolidateAfter: 0s
+    consolidateAfter: 5m
     expireAfter: 72h
-  limits:
-    cpu: 1000
-    memory: 1000Gi
-    nvidia.com/gpu: 2
-  topologySpreadConstraints:
-    - maxSkew: 99
-      topologyKey: "topology.kubernetes.io/zone"
-      whenUnsatisfiable: ScheduleAnyway
-    - maxSkew: 99
-      topologyKey: "kubernetes.io/hostname"
-      whenUnsatisfiable: ScheduleAnyway
-    - maxSkew: 99
-      topologyKey: "karpenter.sh/capacity-type"
-      whenUnsatisfiable: ScheduleAnyway
-  budgets:
-    - nodes: "50%"
-      reasons:
-      - "Empty"
-      - "Drifted"
-    - nodes: "5"
-    - nodes: "0"
-      schedule: "@daily"
-      duration: 10m
-      reasons:
-      - "Underutilized"
+  # limits:
+  #   cpu: 1000
+  #   memory: 1000Gi
+  #   nvidia.com/gpu: 2
+  # topologySpreadConstraints:
+  #   - maxSkew: 99
+  #     topologyKey: "topology.kubernetes.io/zone"
+  #     whenUnsatisfiable: ScheduleAnyway
+  #   - maxSkew: 99
+  #     topologyKey: "kubernetes.io/hostname"
+  #     whenUnsatisfiable: ScheduleAnyway
+  #   - maxSkew: 99
+  #     topologyKey: "karpenter.sh/capacity-type"
+  #     whenUnsatisfiable: ScheduleAnyway
+  # budgets:
+  #   - nodes: "50%"
+  #     reasons:
+  #     - "Empty"
+  #     - "Drifted"
+  #   - nodes: "5"
+  #   - nodes: "0"
+  #     schedule: "@daily"
+  #     duration: 10m
+  #     reasons:
+  #     - "Underutilized"
diff --git a/examples/simple/README.md b/examples/simple/README.md
@@ -48,12 +48,16 @@ terraform destroy
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.14.0 |
+| <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.11.0 |
+| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | >= 1.14.0 |
+| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.23.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | 5.57.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.14.0 |
 
 ## Modules
 

diff --git a/main.tf b/main.tf
@@ -132,22 +132,22 @@ resource "helm_release" "karpenter" {
     name  = "controller.env[0].value"
     value = var.region
   }
-  set {
-    name  = "resources.requests.cpu"
-    value = 1
-  }
-  set {
-    name  = "resources.requests.memory"
-    value = "1Gi"
-  }
-  set {
-    name  = "resources.limits.cpu"
-    value = 1
-  }
-  set {
-    name  = "resources.limits.memory"
-    value = "1Gi"
-  }
+  # set {
+  #   name  = "resources.requests.cpu"
+  #   value = 1
+  # }
+  # set {
+  #   name  = "resources.requests.memory"
+  #   value = "1Gi"
+  # }
+  # set {
+  #   name  = "resources.limits.cpu"
+  #   value = 1
+  # }
+  # set {
+  #   name  = "resources.limits.memory"
+  #   value = "1Gi"
+  # }
   set {
     name  = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
     value = module.karpenter_resources.iam_role_arn