From d80edc4898c96e410107f7c59c4764eaf2067cae Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 27 Jan 2025 18:20:11 -0500 Subject: [PATCH 1/6] since karpenter is later in the cycle, add taint tolerations to cluster addons --- main.tf | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/main.tf b/main.tf index 6fb7632..ce56ea9 100644 --- a/main.tf +++ b/main.tf @@ -36,6 +36,17 @@ locals { "boc:created_by" = "terragrunt" "karpenter.sh/discovery" = var.cluster_name } + karpenter_taint = jsonencode({ + tolerations = [ + # Allow CoreDNS to run on the same nodes as the Karpenter controller + # for use during cluster creation when Karpenter nodes do not yet exist + { + key = "karpenter.sh/controller" + value = "true" + effect = "NoSchedule" + } + ] + }) max_tag_count = 45 ng_name = format("%v%v-nodegroup", local.prefixes["eks"], var.cluster_name) subnets = [for k, v in data.aws_subnet.subnets : v.id if length(regexall("us-east-1e", v.availability_zone)) == 0] @@ -67,41 +78,38 @@ module "cluster" { cluster_addons = { amazon-cloudwatch-observability = { most_recent = true + configuration_values = local.karpenter_taint service_account_role_arn = module.cloudwatch_observability_irsa_role.iam_role_arn } aws-ebs-csi-driver = { most_recent = true + configuration_values = local.karpenter_taint service_account_role_arn = module.ebs_csi_irsa_role.iam_role_arn } aws-efs-csi-driver = { most_recent = true service_account_role_arn = module.efs_csi_irsa_role.iam_role_arn + configuration_values = local.karpenter_taint } coredns = { - most_recent = true - configuration_values = jsonencode({ - tolerations = [ - # Allow CoreDNS to run on the same nodes as the Karpenter controller - # for use during cluster creation when Karpenter nodes do not yet exist - { - key = "karpenter.sh/controller" - value = "true" - effect = "NoSchedule" - } - ] - }) + most_recent = true + configuration_values = local.karpenter_taint } eks-pod-identity-agent = { - most_recent = true + most_recent = true + configuration_values = local.karpenter_taint } kube-proxy = { - most_recent = true + most_recent = true + configuration_values = local.karpenter_taint } snapshot-controller = { - most_recent = true + most_recent = true + configuration_values = local.karpenter_taint } vpc-cni = { most_recent = true + configuration_values = local.karpenter_taint service_account_role_arn = module.vpc_cni_irsa_role.iam_role_arn } } From bdcc7eec959a14c7f26f6702c94e3412e74961d4 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 27 Jan 2025 18:45:54 -0500 Subject: [PATCH 2/6] not all addons support taints --- main.tf | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/main.tf b/main.tf index ce56ea9..48675ef 100644 --- a/main.tf +++ b/main.tf @@ -78,18 +78,15 @@ module "cluster" { cluster_addons = { amazon-cloudwatch-observability = { most_recent = true - configuration_values = local.karpenter_taint service_account_role_arn = module.cloudwatch_observability_irsa_role.iam_role_arn } aws-ebs-csi-driver = { most_recent = true - configuration_values = local.karpenter_taint service_account_role_arn = module.ebs_csi_irsa_role.iam_role_arn } aws-efs-csi-driver = { most_recent = true service_account_role_arn = module.efs_csi_irsa_role.iam_role_arn - configuration_values = local.karpenter_taint } coredns = { most_recent = true @@ -100,8 +97,7 @@ module "cluster" { configuration_values = local.karpenter_taint } kube-proxy = { - most_recent = true - configuration_values = local.karpenter_taint + most_recent = true } snapshot-controller = { most_recent = true From 289964b2bff6f753deb3da3d3164c391fbff1c2b Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 27 Jan 2025 19:14:57 -0500 Subject: [PATCH 3/6] add note and comment --- main.tf | 40 +++++++++++++--------------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/main.tf b/main.tf index 48675ef..982bc32 100644 --- a/main.tf +++ b/main.tf @@ -36,17 +36,6 @@ locals { "boc:created_by" = "terragrunt" "karpenter.sh/discovery" = var.cluster_name } - karpenter_taint = jsonencode({ - tolerations = [ - # Allow CoreDNS to run on the same nodes as the Karpenter controller - # for use during cluster creation when Karpenter nodes do not yet exist - { - key = "karpenter.sh/controller" - value = "true" - effect = "NoSchedule" - } - ] - }) max_tag_count = 45 ng_name = format("%v%v-nodegroup", local.prefixes["eks"], var.cluster_name) subnets = [for k, v in data.aws_subnet.subnets : v.id if length(regexall("us-east-1e", v.availability_zone)) == 0] @@ -89,23 +78,19 @@ module "cluster" { service_account_role_arn = module.efs_csi_irsa_role.iam_role_arn } coredns = { - most_recent = true - configuration_values = local.karpenter_taint + most_recent = true } eks-pod-identity-agent = { - most_recent = true - configuration_values = local.karpenter_taint + most_recent = true } kube-proxy = { most_recent = true } snapshot-controller = { - most_recent = true - configuration_values = local.karpenter_taint + most_recent = true } vpc-cni = { most_recent = true - configuration_values = local.karpenter_taint service_account_role_arn = module.vpc_cni_irsa_role.iam_role_arn } } @@ -147,15 +132,16 @@ module "cluster" { labels = { intent = "control-apps" } - taints = { - # The pods that do not tolerate this taint should run on nodes - # created by Karpenter - karpenter = { - key = "karpenter.sh/controller" - value = "true" - effect = "NO_SCHEDULE" - } - } + # This cannot be enabled until karpenter is availabe. + # taints = { + # # The pods that do not tolerate this taint should run on nodes + # # created by Karpenter + # karpenter = { + # key = "karpenter.sh/controller" + # value = "true" + # effect = "NO_SCHEDULE" + # } + # } } } tags = local.tags From eee3aeca8fa95106d48193e2b3c063984541e243 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 27 Jan 2025 19:27:31 -0500 Subject: [PATCH 4/6] typo --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 982bc32..0f06f1e 100644 --- a/main.tf +++ b/main.tf @@ -132,7 +132,7 @@ module "cluster" { labels = { intent = "control-apps" } - # This cannot be enabled until karpenter is availabe. + # This cannot be enabled until karpenter is available. # taints = { # # The pods that do not tolerate this taint should run on nodes # # created by Karpenter From c2f6f92a69dd0e58489faf289df73432d659d113 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 27 Jan 2025 20:23:05 -0500 Subject: [PATCH 5/6] update upstream --- README.md | 2 +- main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bf9d153..d0313b0 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ Change logs are auto-generated with commitizen. | Name | Source | Version | |------|--------|---------| | [cloudwatch\_observability\_irsa\_role](#module\_cloudwatch\_observability\_irsa\_role) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | n/a | -| [cluster](#module\_cluster) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git | v20.31.1 | +| [cluster](#module\_cluster) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git | v20.33.1 | | [ebs\_csi\_irsa\_role](#module\_ebs\_csi\_irsa\_role) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | n/a | | [efs\_csi\_irsa\_role](#module\_efs\_csi\_irsa\_role) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | n/a | | [vpc\_cni\_irsa\_role](#module\_vpc\_cni\_irsa\_role) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | n/a | diff --git a/main.tf b/main.tf index 0f06f1e..db86298 100644 --- a/main.tf +++ b/main.tf @@ -45,7 +45,7 @@ locals { } module "cluster" { - source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git?ref=v20.31.1" + source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git?ref=v20.33.1" cluster_name = var.cluster_name cluster_version = var.cluster_version From a58b1c39bad7a34648b2775e95707afb189f80d7 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 12 Feb 2025 16:29:22 -0500 Subject: [PATCH 6/6] pre-commit updated --- .pre-commit-config.yaml | 2 +- main.tf | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a5c34b..ef52d70 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: # Terraform Hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.96.1 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases + rev: v1.97.3 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases hooks: - id: terraform_fmt args: diff --git a/main.tf b/main.tf index db86298..068d433 100644 --- a/main.tf +++ b/main.tf @@ -33,7 +33,6 @@ locals { "boc:eks_cluster_name" = var.cluster_name "boc:tf_module_name" = local.module_name "boc:tf_module_version" = local.module_version - "boc:created_by" = "terragrunt" "karpenter.sh/discovery" = var.cluster_name } max_tag_count = 45