From d65655dcea218dd8b7e94fee59861bb7220ed986 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 26 Jul 2024 16:21:32 -0400 Subject: [PATCH 01/13] minor changes --- EC2NodePool.yaml.tpl | 22 ++++++++++++++++++++++ copy_images.tf | 6 +++++- main.tf | 28 ++-------------------------- version.tf | 4 ++++ 4 files changed, 33 insertions(+), 27 deletions(-) create mode 100644 EC2NodePool.yaml.tpl create mode 100644 version.tf diff --git a/EC2NodePool.yaml.tpl b/EC2NodePool.yaml.tpl new file mode 100644 index 0000000..538f282 --- /dev/null +++ b/EC2NodePool.yaml.tpl @@ -0,0 +1,22 @@ +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot", "on-demand"] + - key: "node.kubernetes.io/instance-type" + operator: In + values: ["t3a.xlarge", "t3.xlarge", "c5.large", "m5.large", "r5.large", "m5.xlarge"] + nodeClassRef: + name: default + limits: + cpu: 1000 + memory: 1000Gi + disruption: + consolidationPolicy: WhenUnderutilized + expireAfter: 24h # 30 * 24h = 720h diff --git a/copy_images.tf b/copy_images.tf index 00f6212..4c05c03 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -18,10 +18,14 @@ locals { # map[repository:224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter tag:0.37.0] module "images" { - source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=2.0.2" + source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" profile = var.profile application_name = var.cluster_name image_config = local.image_config tags = {} + + enable_lifecycle_policy = true + lifecycle_policy_all = true + force_delete = true } diff --git a/main.tf b/main.tf index cee21c8..2eeb0e8 100644 --- a/main.tf +++ b/main.tf @@ -5,7 +5,7 @@ locals { # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter # Karpenter IRSA roles and policies module "karpenter_resources" { - source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.8.5" + source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.20.0" cluster_name = var.cluster_name enable_irsa = true @@ -166,29 +166,5 @@ resource "kubectl_manifest" "karpenter_NodePool" { depends_on = [ kubectl_manifest.karpenter_EC2NodeClass, ] - - yaml_body = < Date: Fri, 26 Jul 2024 16:23:28 -0400 Subject: [PATCH 02/13] null args --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 2eeb0e8..d90dc05 100644 --- a/main.tf +++ b/main.tf @@ -166,5 +166,5 @@ resource "kubectl_manifest" "karpenter_NodePool" { depends_on = [ kubectl_manifest.karpenter_EC2NodeClass, ] - yaml_body = templatefile("${path.module}/EC2NodePool.yaml.tpl") + yaml_body = templatefile("${path.module}/EC2NodePool.yaml.tpl", "") } From 94e2eef4e7ed3408925aec4f85290983040c6dd1 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 26 Jul 2024 16:24:43 -0400 Subject: [PATCH 03/13] empty map --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index d90dc05..e529968 100644 --- a/main.tf +++ b/main.tf @@ -166,5 +166,5 @@ resource "kubectl_manifest" "karpenter_NodePool" { depends_on = [ kubectl_manifest.karpenter_EC2NodeClass, ] - yaml_body = templatefile("${path.module}/EC2NodePool.yaml.tpl", "") + yaml_body = templatefile("${path.module}/EC2NodePool.yaml.tpl", {}) } From fd0919359cded4baac44ed1ccd8772aa9a83d684 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 26 Jul 2024 16:31:57 -0400 Subject: [PATCH 04/13] pass region to copy_images --- copy_images.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/copy_images.tf b/copy_images.tf index 4c05c03..0593510 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -24,6 +24,7 @@ module "images" { application_name = var.cluster_name image_config = local.image_config tags = {} + region = var.region enable_lifecycle_policy = true lifecycle_policy_all = true From d4597d2976c6489aec8adb5c07a2e75083f43606 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 26 Jul 2024 17:49:57 -0400 Subject: [PATCH 05/13] maybe --- variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/variables.tf b/variables.tf index 6cd9f7e..a7b3ed2 100644 --- a/variables.tf +++ b/variables.tf @@ -32,7 +32,7 @@ variable "karpenter_node_group_name" { variable "karpenter_helm_repo" { description = "Helm repo for official karpenter chart" type = string - default = "oci://public.ecr.aws/karpenter" + default = "oci://public.ecr.aws/karpenter/karpenter" # default = "https://charts.karpenter.sh/" public.ecr.aws/karpenter/karpenter:0.37.0 } @@ -52,4 +52,4 @@ variable "tags" { description = "AWS Tags to apply to appropriate resources" type = map(string) default = {} -} \ No newline at end of file +} From 71b4a794c2a1fd1846374d578bf0a50ca4495877 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 26 Jul 2024 19:24:35 -0400 Subject: [PATCH 06/13] comment more --- main.tf | 23 ++++++++++++----------- variables.tf | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/main.tf b/main.tf index e529968..f6eadbf 100644 --- a/main.tf +++ b/main.tf @@ -20,13 +20,13 @@ module "karpenter_resources" { } # Install Karpenter -data "aws_ecr_image" "karpenter_image" { - depends_on = [ - module.images, - ] - repository_name = module.images.images[local.karpenter_key].dest_repository - image_tag = module.images.images[local.karpenter_key].tag -} +# data "aws_ecr_image" "karpenter_image" { +# depends_on = [ +# module.images, +# ] +# repository_name = module.images.images[local.karpenter_key].dest_repository +# image_tag = module.images.images[local.karpenter_key].tag +# } resource "helm_release" "karpenter" { depends_on = [ @@ -92,10 +92,10 @@ resource "helm_release" "karpenter" { name = "controller.image.tag" value = module.images.images[local.karpenter_key].tag } - set { - name = "controller.image.digest" - value = data.aws_ecr_image.karpenter_image.id - } + # set { + # name = "controller.image.digest" + # value = data.aws_ecr_image.karpenter_image.id + # } set { name = "controller.env[0].name" value = "AWS_REGION" @@ -166,5 +166,6 @@ resource "kubectl_manifest" "karpenter_NodePool" { depends_on = [ kubectl_manifest.karpenter_EC2NodeClass, ] + yaml_body = templatefile("${path.module}/EC2NodePool.yaml.tpl", {}) } diff --git a/variables.tf b/variables.tf index a7b3ed2..1150802 100644 --- a/variables.tf +++ b/variables.tf @@ -32,7 +32,7 @@ variable "karpenter_node_group_name" { variable "karpenter_helm_repo" { description = "Helm repo for official karpenter chart" type = string - default = "oci://public.ecr.aws/karpenter/karpenter" + default = "oci://public.ecr.aws/karpenter" # default = "https://charts.karpenter.sh/" public.ecr.aws/karpenter/karpenter:0.37.0 } From 35a808c5c651502d405e12259c0e03b1be86f6bc Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 26 Jul 2024 19:32:05 -0400 Subject: [PATCH 07/13] no region --- copy_images.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/copy_images.tf b/copy_images.tf index 0593510..4c05c03 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -24,7 +24,6 @@ module "images" { application_name = var.cluster_name image_config = local.image_config tags = {} - region = var.region enable_lifecycle_policy = true lifecycle_policy_all = true From 7db0a9acc7bd1691c8f9ddf6aeea03abc78f8d10 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 26 Jul 2024 19:46:05 -0400 Subject: [PATCH 08/13] gallery --- copy_images.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy_images.tf b/copy_images.tf index 4c05c03..9a9d39c 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -7,7 +7,7 @@ locals { dest_path = null name = "karpenter/controller" source_image = "karpenter/controller" - source_registry = "public.ecr.aws" + source_registry = "gallery.ecr.aws" source_tag = var.karpenter_tag tag = var.karpenter_tag }, From 5741bd959d76f2ec7c8a95c41bff168e72174d8f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 26 Jul 2024 20:35:45 -0400 Subject: [PATCH 09/13] public again --- copy_images.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy_images.tf b/copy_images.tf index 9a9d39c..4c05c03 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -7,7 +7,7 @@ locals { dest_path = null name = "karpenter/controller" source_image = "karpenter/controller" - source_registry = "gallery.ecr.aws" + source_registry = "public.ecr.aws" source_tag = var.karpenter_tag tag = var.karpenter_tag }, From 4101081086456da83d76b27df7d433ab98f6e620 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 26 Jul 2024 21:16:49 -0400 Subject: [PATCH 10/13] add _module_providers --- version.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/version.tf b/version.tf index 1abbce2..9f8a44a 100644 --- a/version.tf +++ b/version.tf @@ -1,4 +1,5 @@ locals { _module_name = "tfmod-karpenter" _module_version = "0.0.1" + _module_providers = ["helm", "kubernetes"] } From 63c247f36897aeb0281bc2d13d11c2965ac1d7d2 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 26 Jul 2024 23:27:53 -0400 Subject: [PATCH 11/13] add outputs --- outputs.tf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 outputs.tf diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..f3bba7c --- /dev/null +++ b/outputs.tf @@ -0,0 +1,18 @@ +################################################################################ +# Module information +################################################################################ + +output "module_name" { + description = "The name of this module." + value = local._module_name +} + +output "module_version" { + description = "The version of this module." + value = local._module_version +} + +output "_module_providers" { + description = "The providers used in this module." + value = local._module_providers +} From 556e9eeda2757cdbc4831b1d94634bc968466209 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 30 Jul 2024 16:38:31 -0400 Subject: [PATCH 12/13] fmt --- README.md | 5 +++-- main.tf | 24 +++++++++++------------- outputs.tf | 5 ----- version.tf | 3 +-- 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 5fb1238..bbe1d75 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@ NOTE: Deploying Karpenter after deploying Istio is not recommended because it di The module deploys Karpenter needed AWS resources, namely in IAM. It copies the Karpenter controller from the public registry to local ECR, deploys Karpenter via Helm and deploys a single default Karpenter Nodepool with EC2NodeClass. # CHANGELOG - +* 0.0.2 -- 2024-07-30 + - update after removing cluster-autoscaler * 0.0.1 -- 2024-07-26 - Genesis/Initial development @@ -71,4 +72,4 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the ## Outputs No outputs. - \ No newline at end of file + diff --git a/main.tf b/main.tf index f6eadbf..94ff312 100644 --- a/main.tf +++ b/main.tf @@ -20,19 +20,18 @@ module "karpenter_resources" { } # Install Karpenter -# data "aws_ecr_image" "karpenter_image" { -# depends_on = [ -# module.images, -# ] -# repository_name = module.images.images[local.karpenter_key].dest_repository -# image_tag = module.images.images[local.karpenter_key].tag -# } +data "aws_ecr_image" "karpenter_image" { + depends_on = [ + module.images, + ] + repository_name = module.images.images[local.karpenter_key].dest_repository + image_tag = module.images.images[local.karpenter_key].tag +} resource "helm_release" "karpenter" { depends_on = [ module.images, ] - name = "karpenter" chart = "karpenter" version = var.karpenter_helm_chart @@ -92,10 +91,10 @@ resource "helm_release" "karpenter" { name = "controller.image.tag" value = module.images.images[local.karpenter_key].tag } - # set { - # name = "controller.image.digest" - # value = data.aws_ecr_image.karpenter_image.id - # } + set { + name = "controller.image.digest" + value = data.aws_ecr_image.karpenter_image.id + } set { name = "controller.env[0].name" value = "AWS_REGION" @@ -166,6 +165,5 @@ resource "kubectl_manifest" "karpenter_NodePool" { depends_on = [ kubectl_manifest.karpenter_EC2NodeClass, ] - yaml_body = templatefile("${path.module}/EC2NodePool.yaml.tpl", {}) } diff --git a/outputs.tf b/outputs.tf index f3bba7c..0349496 100644 --- a/outputs.tf +++ b/outputs.tf @@ -11,8 +11,3 @@ output "module_version" { description = "The version of this module." value = local._module_version } - -output "_module_providers" { - description = "The providers used in this module." - value = local._module_providers -} diff --git a/version.tf b/version.tf index 9f8a44a..5970ee3 100644 --- a/version.tf +++ b/version.tf @@ -1,5 +1,4 @@ locals { _module_name = "tfmod-karpenter" - _module_version = "0.0.1" - _module_providers = ["helm", "kubernetes"] + _module_version = "0.0.2" } From 2b207deb5257f102222f964120ed8a0ed41fd6a1 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 30 Jul 2024 16:42:53 -0400 Subject: [PATCH 13/13] notes --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index bbe1d75..acf93ce 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the # CHANGELOG * 0.0.2 -- 2024-07-30 - update after removing cluster-autoscaler + - add second yaml template + - update images module * 0.0.1 -- 2024-07-26 - Genesis/Initial development