From feb202ef6eb48dedf9df8538496069dec4cf34f5 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 3 Sep 2024 18:18:25 -0400 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=90=9B=20fix(main.tf):=20fix=20servic?= =?UTF-8?q?e-linked-role=20creation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 - main.tf | 8 -------- 2 files changed, 9 deletions(-) diff --git a/README.md b/README.md index bd6a95e..a191961 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | | [aws_ecr_image.kubectl_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | -| [aws_iam_roles.roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_roles) | data source | | [aws_subnets.container_subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | ## Inputs diff --git a/main.tf b/main.tf index e81a6e8..539a995 100644 --- a/main.tf +++ b/main.tf @@ -181,19 +181,11 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { role = module.karpenter_resources.node_iam_role_name } -# Query to see if service-linked role exists -data "aws_iam_roles" "roles" { - path_prefix = "/aws-reserved/spot.amazonaws.com/${var.cluster_name}" -} - # Permission for spot resource "aws_iam_service_linked_role" "ec2_spot" { - count = data.aws_iam_roles.roles == null ? 1 : 0 aws_service_name = "spot.amazonaws.com" - custom_suffix = var.cluster_name } - # SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART # Create karpenter default resource "kubectl_manifest" "karpenter_ec2_node_class" { From b0a84a4f2f43ca22bc790b461ee2113a2a50402c Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 3 Sep 2024 21:53:28 -0400 Subject: [PATCH 2/3] fix(main.tf): custom_suffix does not apply for spot --- README.md | 1 + main.tf | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index a191961..bd6a95e 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | | [aws_ecr_image.kubectl_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | +| [aws_iam_roles.roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_roles) | data source | | [aws_subnets.container_subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | ## Inputs diff --git a/main.tf b/main.tf index 539a995..52f216c 100644 --- a/main.tf +++ b/main.tf @@ -181,11 +181,18 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { role = module.karpenter_resources.node_iam_role_name } +# Query to see if service-linked role exists +data "aws_iam_roles" "roles" { + path_prefix = "/aws-reserved/spot.amazonaws.com" +} + # Permission for spot resource "aws_iam_service_linked_role" "ec2_spot" { + count = data.aws_iam_roles.roles == null ? 1 : 0 aws_service_name = "spot.amazonaws.com" } + # SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART # Create karpenter default resource "kubectl_manifest" "karpenter_ec2_node_class" { From fb550193a8d0c2539c2e2fac2cf9763b3efeb193 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 3 Sep 2024 23:24:19 -0400 Subject: [PATCH 3/3] fix(main.tf): update node class to remove resource reservations for min scale --- ec2_node_class.yaml.tpl | 15 ++------------- main.tf | 34 +++++++++++++++++----------------- 2 files changed, 19 insertions(+), 30 deletions(-) diff --git a/ec2_node_class.yaml.tpl b/ec2_node_class.yaml.tpl index 4251292..74c4172 100644 --- a/ec2_node_class.yaml.tpl +++ b/ec2_node_class.yaml.tpl @@ -4,19 +4,9 @@ kind: EC2NodeClass metadata: name: ${cluster_name}-karpenter-node-class annotations: - kubernetes.io/description: "EC2NodeClass for running Bottlerocket nodes" + kubernetes.io/description: "EC2NodeClass for running ${amd_ami_family} nodes" spec: kubelet: - podsPerCore: 2 - maxPods: 20 - systemReserved: - cpu: 100m - memory: 100Mi - ephemeral-storage: 1Gi - kubeReserved: - cpu: 200m - memory: 100Mi - ephemeral-storage: 3Gi evictionHard: memory.available: 5% nodefs.available: 10% @@ -32,12 +22,11 @@ spec: evictionMaxPodGracePeriod: 60 imageGCHighThresholdPercent: 85 imageGCLowThresholdPercent: 80 - cpuCFSQuota: true # Required, resolves a default ami and userdata amiFamily: ${amd_ami_family} amiSelectorTerms: - - alias: bottlerocket@latest # Bottlerocket + - alias: ${amd_ami_alias}@latest # Bottlerocket # Required, discovers subnets to attach to instances # Each term in the array of subnetSelectorTerms is ORed together # Within a single term, all conditions are ANDed diff --git a/main.tf b/main.tf index 52f216c..8831008 100644 --- a/main.tf +++ b/main.tf @@ -152,22 +152,22 @@ resource "helm_release" "karpenter" { name = "controller.env[0].value" value = var.region } - set { - name = "resources.requests.cpu" - value = 1 - } - set { - name = "resources.requests.memory" - value = "1Gi" - } - set { - name = "resources.limits.cpu" - value = 1 - } - set { - name = "resources.limits.memory" - value = "1Gi" - } + # set { + # name = "resources.requests.cpu" + # value = 1 + # } + # set { + # name = "resources.requests.memory" + # value = "1Gi" + # } + # set { + # name = "resources.limits.cpu" + # value = 1 + # } + # set { + # name = "resources.limits.memory" + # value = "1Gi" + # } set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" value = module.karpenter_resources.iam_role_arn @@ -192,7 +192,6 @@ resource "aws_iam_service_linked_role" "ec2_spot" { aws_service_name = "spot.amazonaws.com" } - # SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART # Create karpenter default resource "kubectl_manifest" "karpenter_ec2_node_class" { @@ -202,6 +201,7 @@ resource "kubectl_manifest" "karpenter_ec2_node_class" { yaml_body = templatefile("${path.module}/ec2_node_class.yaml.tpl", { cluster_name = var.cluster_name amd_ami_family = "Bottlerocket" + amd_ami_alias = "bottlerocket" }) }