diff --git a/aws_data.tf b/aws_data.tf index 2c6aade..e7211e8 100644 --- a/aws_data.tf +++ b/aws_data.tf @@ -3,3 +3,5 @@ data "aws_caller_identity" "current" {} data "aws_arn" "current" { arn = data.aws_caller_identity.current.arn } + +data "aws_default_tags" "default_tags" {} diff --git a/ec2_node_pool.yaml.tpl b/ec2_node_pool.yaml.tpl index 0da0ee8..1fcb874 100644 --- a/ec2_node_pool.yaml.tpl +++ b/ec2_node_pool.yaml.tpl @@ -11,24 +11,57 @@ spec: requirements: - key: kubernetes.io/arch operator: In - values: ["amd64"] + values: ["amd64", "arm64"] - key: kubernetes.io/os operator: In - values: ["linux"] - - key: karpenter.sh/capacity-type + values: ["linux", "windows"] + - key: "karpenter.k8s.aws/instance-hypervisor" operator: In - values: ["on-demand", "spot"] - - key: karpenter.k8s.aws/instance-category + values: ["nitro"] + - key: karpenter.sh/capacity-type operator: In - values: ["c", "m", "r", "t"] - - key: karpenter.k8s.aws/instance-generation - operator: Gt - values: ["2"] + values: ["spot", "on-demand"] + # - key: karpenter.k8s.aws/instance-category + # operator: In + # values: ["c", "m", "r", "i", "d", "t"] + # - key: karpenter.k8s.aws/instance-generation + # operator: Gt + # values: ["2"] nodeClassRef: group: karpenter.k8s.aws kind: EC2NodeClass name: ${cluster_name}-karpenter-node-class + kubelet: + containerRuntime: containerd + systemReserved: + cpu: 100m + memory: 100Mi disruption: consolidationPolicy: WhenEmptyOrUnderutilized consolidateAfter: 1h - expireAfter: 1h + expireAfter: 72h + limits: + cpu: 1000 + memory: 1000Gi + nvidia.com/gpu: 2 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "topology.kubernetes.io/zone" + whenUnsatisfiable: ScheduleAnyway + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: ScheduleAnyway + - maxSkew: 1 + topologyKey: "karpenter.sh/capacity-type" + whenUnsatisfiable: ScheduleAnyway + budgets: + - nodes: "50%" + reasons: + - "Empty" + - "Drifted" + - nodes: "5" + - nodes: "0" + schedule: "@daily" + duration: 10m + reasons: + - "Underutilized" diff --git a/main.tf b/main.tf index 3343453..6ec527d 100644 --- a/main.tf +++ b/main.tf @@ -132,22 +132,22 @@ resource "helm_release" "karpenter" { name = "controller.env[0].value" value = var.region } - # set { - # name = "resources.requests.cpu" - # value = 1 - # } - # set { - # name = "resources.requests.memory" - # value = "1Gi" - # } - # set { - # name = "resources.limits.cpu" - # value = 1 - # } - # set { - # name = "resources.limits.memory" - # value = "1Gi" - # } + set { + name = "resources.requests.cpu" + value = 1 + } + set { + name = "resources.requests.memory" + value = "1Gi" + } + set { + name = "resources.limits.cpu" + value = 1 + } + set { + name = "resources.limits.memory" + value = "1Gi" + } set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" value = module.karpenter_resources.iam_role_arn @@ -161,9 +161,6 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { role = module.karpenter_resources.node_iam_role_name } -# Get the default tags from the provider -data "aws_default_tags" "default_tags" {} - # SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART # Create karpenter default resource "kubectl_manifest" "karpenter_ec2_node_class" {