From ac4f0e3fbc54bbb35c8ad9544862ce0767dbc2df Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 16:25:24 -0500 Subject: [PATCH 01/41] try kubernetes_manifest --- README.md | 16 ++++++-------- copy_images.tf | 10 --------- main.tf | 58 ++++++++++++++++++++++++------------------------- requirements.tf | 6 ++--- variables.tf | 6 ----- 5 files changed, 39 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 9bcb169..545ff2b 100644 --- a/README.md +++ b/README.md @@ -20,22 +20,22 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [terraform](#requirement\_terraform) | >= 0.13 | | [aws](#requirement\_aws) | >= 5.14.0 | | [helm](#requirement\_helm) | >= 2.11.0 | -| [kubectl](#requirement\_kubectl) | >= 1.14.0 | +| [kubernetes](#requirement\_kubernetes) | >= 2.23.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | 5.73.0 | -| [helm](#provider\_helm) | 2.16.1 | -| [kubectl](#provider\_kubectl) | 1.14.0 | +| [aws](#provider\_aws) | >= 5.14.0 | +| [helm](#provider\_helm) | >= 2.11.0 | +| [kubernetes](#provider\_kubernetes) | >= 2.23.0 | ## Modules | Name | Source | Version | |------|--------|---------| | [images](#module\_images) | git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/ | tf-upgrade | -| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | v20.24.3 | +| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | 20.31.6 | ## Resources @@ -43,13 +43,12 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|------| | [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [kubectl_manifest.karpenter_ec2_node_class](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubernetes_manifest.karpenter_ec2_node_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | +| [kubernetes_manifest.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_default_tags.default_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | -| [aws_ecr_image.kubectl_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | ## Inputs @@ -61,7 +60,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [karpenter\_helm\_repo](#input\_karpenter\_helm\_repo) | Helm repo for official karpenter chart | `string` | `"oci://public.ecr.aws/karpenter"` | no | | [karpenter\_node\_group\_name](#input\_karpenter\_node\_group\_name) | The cluster node group that will host karpenter, should not be a karpenter managed node group | `string` | n/a | yes | | [karpenter\_tag](#input\_karpenter\_tag) | Which tag of karpenter image | `string` | `"1.0.6"` | no | -| [kubectl\_tag](#input\_kubectl\_tag) | Which tag of kubectl image | `string` | `"1.30.4"` | no | | [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes | | [profile](#input\_profile) | AWS config profile | `string` | `""` | no | | [region](#input\_region) | AWS region | `string` | n/a | yes | diff --git a/copy_images.tf b/copy_images.tf index c98c243..4c05c03 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -1,6 +1,5 @@ locals { karpenter_key = format("%v#%v", "karpenter/controller", var.karpenter_tag) - kubectl_key = format("%v#%v", "bitnami/kubectl", var.kubectl_tag) image_config = [ { @@ -12,15 +11,6 @@ locals { source_tag = var.karpenter_tag tag = var.karpenter_tag }, - { - enabled = true - dest_path = null - name = "bitnami/kubectl" - source_image = "bitnami/kubectl" - source_registry = "docker.io" - source_tag = var.kubectl_tag - tag = var.kubectl_tag - }, ] } diff --git a/main.tf b/main.tf index d1ce7e9..ef8d2db 100644 --- a/main.tf +++ b/main.tf @@ -6,7 +6,7 @@ locals { # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter # Karpenter IRSA roles and policies module "karpenter_resources" { - source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.24.3" + source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=20.31.6" cluster_name = var.cluster_name enable_irsa = true @@ -30,13 +30,13 @@ data "aws_ecr_image" "karpenter_image" { image_tag = module.images.images[local.karpenter_key].tag } -data "aws_ecr_image" "kubectl_image" { - depends_on = [ - module.images, - ] - repository_name = module.images.images[local.kubectl_key].dest_repository - image_tag = module.images.images[local.kubectl_key].tag -} +# data "aws_ecr_image" "kubectl_image" { +# depends_on = [ +# module.images, +# ] +# repository_name = module.images.images[local.kubectl_key].dest_repository +# image_tag = module.images.images[local.kubectl_key].tag +# } resource "helm_release" "karpenter" { depends_on = [ @@ -109,21 +109,21 @@ resource "helm_release" "karpenter" { name = "controller.image.digest" value = data.aws_ecr_image.karpenter_image.id } - set { - name = "postInstallHook.image.repository" - value = format("%v/%v", - module.images.images[local.kubectl_key].dest_registry, - module.images.images[local.kubectl_key].dest_repository - ) - } - set { - name = "postInstallHook.image.tag" - value = module.images.images[local.kubectl_key].tag - } - set { - name = "postInstallHook.image.digest" - value = data.aws_ecr_image.kubectl_image.id - } + # set { + # name = "postInstallHook.image.repository" + # value = format("%v/%v", + # module.images.images[local.kubectl_key].dest_registry, + # module.images.images[local.kubectl_key].dest_repository + # ) + # } + # set { + # name = "postInstallHook.image.tag" + # value = module.images.images[local.kubectl_key].tag + # } + # set { + # name = "postInstallHook.image.digest" + # value = data.aws_ecr_image.kubectl_image.id + # } set { name = "controller.env[0].name" value = "AWS_REGION" @@ -163,25 +163,25 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { # SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART # Create karpenter default -resource "kubectl_manifest" "karpenter_ec2_node_class" { +resource "kubernetes_manifest" "karpenter_ec2_node_class" { depends_on = [ helm_release.karpenter, ] - yaml_body = templatefile("${path.module}/ec2_node_class.yaml.tpl", { + manifest = yamldecode(templatefile("${path.module}/ec2_node_class.yaml.tpl", { cluster_name = var.cluster_name tags = jsonencode(data.aws_default_tags.default_tags.tags) amd_ami_family = "Bottlerocket" amd_ami_alias = "bottlerocket" subnets_name = local.container_subnets_name - }) + })) } # Create karpenter default NodePool -resource "kubectl_manifest" "karpenter_node_pool" { +resource "kubernetes_manifest" "karpenter_node_pool" { depends_on = [ kubectl_manifest.karpenter_ec2_node_class, ] - yaml_body = templatefile("${path.module}/ec2_node_pool.yaml.tpl", { + manifest = yamldecode(templatefile("${path.module}/ec2_node_pool.yaml.tpl", { cluster_name = var.cluster_name - }) + })) } diff --git a/requirements.tf b/requirements.tf index 4eab89e..ae62e15 100644 --- a/requirements.tf +++ b/requirements.tf @@ -10,9 +10,9 @@ terraform { source = "hashicorp/helm" version = ">= 2.11.0" } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.14.0" + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.23.0" } } } diff --git a/variables.tf b/variables.tf index d73f57e..ca1e6f8 100644 --- a/variables.tf +++ b/variables.tf @@ -48,12 +48,6 @@ variable "karpenter_tag" { default = "1.0.6" } -variable "kubectl_tag" { - description = "Which tag of kubectl image" - type = string - default = "1.30.4" -} - variable "subnets_name" { description = "Define the name of the subnets to be used by this cluster" type = string From 39e8c27866218686cd33634802e443ca2803100e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 16:27:56 -0500 Subject: [PATCH 02/41] minor --- main.tf | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/main.tf b/main.tf index ef8d2db..e6dd698 100644 --- a/main.tf +++ b/main.tf @@ -30,14 +30,6 @@ data "aws_ecr_image" "karpenter_image" { image_tag = module.images.images[local.karpenter_key].tag } -# data "aws_ecr_image" "kubectl_image" { -# depends_on = [ -# module.images, -# ] -# repository_name = module.images.images[local.kubectl_key].dest_repository -# image_tag = module.images.images[local.kubectl_key].tag -# } - resource "helm_release" "karpenter" { depends_on = [ data.aws_ecr_image.karpenter_image @@ -179,7 +171,7 @@ resource "kubernetes_manifest" "karpenter_ec2_node_class" { # Create karpenter default NodePool resource "kubernetes_manifest" "karpenter_node_pool" { depends_on = [ - kubectl_manifest.karpenter_ec2_node_class, + kubernetes_manifest.karpenter_ec2_node_class, ] manifest = yamldecode(templatefile("${path.module}/ec2_node_pool.yaml.tpl", { cluster_name = var.cluster_name From 1d8c1aa3c38d3f2493889a7e62d96f957fa2693e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 16:31:34 -0500 Subject: [PATCH 03/41] updated ver --- README.md | 2 +- main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 545ff2b..8b72630 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Source | Version | |------|--------|---------| | [images](#module\_images) | git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/ | tf-upgrade | -| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | 20.31.6 | +| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | 20.33.1 | ## Resources diff --git a/main.tf b/main.tf index e6dd698..fcf4ac6 100644 --- a/main.tf +++ b/main.tf @@ -6,7 +6,7 @@ locals { # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter # Karpenter IRSA roles and policies module "karpenter_resources" { - source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=20.31.6" + source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=20.33.1" cluster_name = var.cluster_name enable_irsa = true From 88176402179e5237dceae1821cebd210cd5863f2 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 16:45:24 -0500 Subject: [PATCH 04/41] add v --- README.md | 2 +- main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8b72630..8bcb9e1 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Source | Version | |------|--------|---------| | [images](#module\_images) | git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/ | tf-upgrade | -| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | 20.33.1 | +| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | v20.33.1 | ## Resources diff --git a/main.tf b/main.tf index fcf4ac6..cc0c007 100644 --- a/main.tf +++ b/main.tf @@ -6,7 +6,7 @@ locals { # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter # Karpenter IRSA roles and policies module "karpenter_resources" { - source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=20.33.1" + source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.33.1" cluster_name = var.cluster_name enable_irsa = true From 58ed4f6dca3fac3e35fe5be1b83766794b4727ad Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 17:24:31 -0500 Subject: [PATCH 05/41] add custom helm charts --- .pre-commit-config.yaml | 2 +- .pre-commit-hooks.yaml | 170 ------------------ README.md | 7 +- karpenter-node-class/.helmignore | 23 +++ karpenter-node-class/Chart.yaml | 21 +++ .../${cluster_name}-karpenter-node-class.yaml | 44 +++++ karpenter-node-class/templates/_helpers.tpl | 62 +++++++ karpenter-node-class/values.yaml | 1 + karpenter-node-pool/.helmignore | 23 +++ karpenter-node-pool/Chart.yaml | 21 +++ .../${cluster_name}-node-pool-general.yaml | 37 ++++ karpenter-node-pool/templates/_helpers.tpl | 62 +++++++ karpenter-node-pool/values.yaml | 1 + main.tf | 85 ++++++--- requirements.tf | 4 - variables.tf | 5 + 16 files changed, 368 insertions(+), 200 deletions(-) delete mode 100644 .pre-commit-hooks.yaml create mode 100644 karpenter-node-class/.helmignore create mode 100644 karpenter-node-class/Chart.yaml create mode 100644 karpenter-node-class/templates/${cluster_name}-karpenter-node-class.yaml create mode 100644 karpenter-node-class/templates/_helpers.tpl create mode 100644 karpenter-node-class/values.yaml create mode 100644 karpenter-node-pool/.helmignore create mode 100644 karpenter-node-pool/Chart.yaml create mode 100644 karpenter-node-pool/templates/${cluster_name}-node-pool-general.yaml create mode 100644 karpenter-node-pool/templates/_helpers.tpl create mode 100644 karpenter-node-pool/values.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f451b69..41952f0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] exclude: CHANGELOG.md - - id: check-yaml + # - id: check-yaml - id: check-merge-conflict - id: check-executables-have-shebangs diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml deleted file mode 100644 index 266f0c7..0000000 --- a/.pre-commit-hooks.yaml +++ /dev/null @@ -1,170 +0,0 @@ -#### THESE ARE NOT ENABLED, THEY ARE FOR REFERENCE -# - id: infracost_breakdown -# name: Infracost breakdown -# description: Check terraform infrastructure cost -# entry: hooks/infracost_breakdown.sh -# language: script -# require_serial: true -# files: \.(tf(vars)?|hcl)$ -# exclude: \.terraform/.*$ - -# - id: terraform_fmt -# name: Terraform fmt -# description: Rewrites all Terraform configuration files to a canonical format. -# entry: hooks/terraform_fmt.sh -# language: script -# files: (\.tf|\.tfvars)$ -# exclude: \.terraform/.*$ - -# - id: terraform_docs -# name: Terraform docs -# description: Inserts input and output documentation into README.md (using terraform-docs). -# require_serial: true -# entry: hooks/terraform_docs.sh -# language: script -# files: (\.tf|\.terraform\.lock\.hcl)$ -# exclude: \.terraform/.*$ - -# - id: terraform_docs_without_aggregate_type_defaults -# name: Terraform docs (without aggregate type defaults) -# description: Inserts input and output documentation into README.md (using terraform-docs). Identical to terraform_docs. -# require_serial: true -# entry: hooks/terraform_docs.sh -# language: script -# files: (\.tf)$ -# exclude: \.terraform/.*$ - -# - id: terraform_docs_replace -# name: Terraform docs (overwrite README.md) -# description: Overwrite content of README.md with terraform-docs. -# require_serial: true -# entry: terraform_docs_replace -# language: python -# files: (\.tf)$ -# exclude: \.terraform/.*$ - -# - id: terraform_validate -# name: Terraform validate -# description: Validates all Terraform configuration files. -# require_serial: true -# entry: hooks/terraform_validate.sh -# language: script -# files: \.(tf(vars)?|terraform\.lock\.hcl)$ -# exclude: \.terraform/.*$ - -# - id: terraform_providers_lock -# name: Lock terraform provider versions -# description: Updates provider signatures in dependency lock files. -# require_serial: true -# entry: hooks/terraform_providers_lock.sh -# language: script -# files: (\.terraform\.lock\.hcl)$ -# exclude: \.terraform/.*$ - -# - id: terraform_tflint -# name: Terraform validate with tflint -# description: Validates all Terraform configuration files with TFLint. -# require_serial: true -# entry: hooks/terraform_tflint.sh -# language: script -# files: (\.tf|\.tfvars)$ -# exclude: \.terraform/.*$ - -# - id: terragrunt_fmt -# name: Terragrunt fmt -# description: Rewrites all Terragrunt configuration files to a canonical format. -# entry: hooks/terragrunt_fmt.sh -# language: script -# files: (\.hcl)$ -# exclude: \.terraform/.*$ - -# - id: terragrunt_validate -# name: Terragrunt validate -# description: Validates all Terragrunt configuration files. -# entry: hooks/terragrunt_validate.sh -# language: script -# files: (\.hcl)$ -# exclude: \.terraform/.*$ - -# - id: terragrunt_validate_inputs -# name: Terragrunt validate inputs -# description: Validates Terragrunt unused and undefined inputs. -# entry: hooks/terragrunt_validate_inputs.sh -# language: script -# files: (\.hcl)$ -# exclude: \.terraform/.*$ - -# - id: terragrunt_providers_lock -# name: Terragrunt providers lock -# description: Updates provider signatures in dependency lock files using terragrunt. -# entry: hooks/terragrunt_providers_lock.sh -# language: script -# files: (terragrunt|\.terraform\.lock)\.hcl$ -# exclude: \.(terraform/.*|terragrunt-cache)$ - -# - id: terraform_tfsec -# name: Terraform validate with tfsec (deprecated, use "terraform_trivy") -# description: Static analysis of Terraform templates to spot potential security issues. -# require_serial: true -# entry: hooks/terraform_tfsec.sh -# files: \.tf(vars)?$ -# language: script - -# - id: terraform_trivy -# name: Terraform validate with trivy -# description: Static analysis of Terraform templates to spot potential security issues. -# require_serial: true -# entry: hooks/terraform_trivy.sh -# files: \.tf(vars)?$ -# language: script - -# - id: checkov -# name: checkov (deprecated, use "terraform_checkov") -# description: Runs checkov on Terraform templates. -# entry: checkov -d . -# language: python -# pass_filenames: false -# always_run: false -# files: \.tf$ -# exclude: \.terraform/.*$ -# require_serial: true - -# - id: terraform_checkov -# name: Checkov -# description: Runs checkov on Terraform templates. -# entry: hooks/terraform_checkov.sh -# language: script -# always_run: false -# files: \.tf$ -# exclude: \.terraform/.*$ -# require_serial: true - -# - id: terraform_wrapper_module_for_each -# name: Terraform wrapper with for_each in module -# description: Generate Terraform wrappers with for_each in module. -# entry: hooks/terraform_wrapper_module_for_each.sh -# language: script -# pass_filenames: false -# always_run: false -# require_serial: true -# files: \.tf$ -# exclude: \.terraform/.*$ - -# - id: terrascan -# name: terrascan -# description: Runs terrascan on Terraform templates. -# language: script -# entry: hooks/terrascan.sh -# files: \.tf$ -# exclude: \.terraform/.*$ -# require_serial: true - -# - id: tfupdate -# name: tfupdate -# description: Runs tfupdate on Terraform templates. -# language: script -# entry: hooks/tfupdate.sh -# args: -# - --args=terraform -# files: \.tf$ -# require_serial: true diff --git a/README.md b/README.md index 8bcb9e1..7b5fa82 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [terraform](#requirement\_terraform) | >= 0.13 | | [aws](#requirement\_aws) | >= 5.14.0 | | [helm](#requirement\_helm) | >= 2.11.0 | -| [kubernetes](#requirement\_kubernetes) | >= 2.23.0 | ## Providers @@ -28,7 +27,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|---------| | [aws](#provider\_aws) | >= 5.14.0 | | [helm](#provider\_helm) | >= 2.11.0 | -| [kubernetes](#provider\_kubernetes) | >= 2.23.0 | ## Modules @@ -43,8 +41,8 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|------| | [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [kubernetes_manifest.karpenter_ec2_node_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | -| [kubernetes_manifest.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | +| [helm_release.karpenter_node_class](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_default_tags.default_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | @@ -63,6 +61,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes | | [profile](#input\_profile) | AWS config profile | `string` | `""` | no | | [region](#input\_region) | AWS region | `string` | n/a | yes | +| [release\_version](#input\_release\_version) | Which version of platform-eng packages to use | `string` | n/a | yes | | [subnets\_name](#input\_subnets\_name) | Define the name of the subnets to be used by this cluster | `string` | `"*-container-*"` | no | | [tags](#input\_tags) | AWS Tags to apply to appropriate resources | `map(string)` | `{}` | no | diff --git a/karpenter-node-class/.helmignore b/karpenter-node-class/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/karpenter-node-class/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/karpenter-node-class/Chart.yaml b/karpenter-node-class/Chart.yaml new file mode 100644 index 0000000..b580371 --- /dev/null +++ b/karpenter-node-class/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: karpenter-node-class +description: A Helm chart for Kubernetes +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.1.0" diff --git a/karpenter-node-class/templates/${cluster_name}-karpenter-node-class.yaml b/karpenter-node-class/templates/${cluster_name}-karpenter-node-class.yaml new file mode 100644 index 0000000..bdb94d6 --- /dev/null +++ b/karpenter-node-class/templates/${cluster_name}-karpenter-node-class.yaml @@ -0,0 +1,44 @@ +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass +metadata: + name: {{ include "karpenter-node-class.fullname" . }}-${cluster_name}-karpenter-node-class + labels: + {{- include "karpenter-node-class.labels" . | nindent 4 }} + annotations: + kubernetes.io/description: EC2NodeClass for running ${amd_ami_family} nodes +spec: + amiFamily: ${amd_ami_family} + amiSelectorTerms: + - alias: ${amd_ami_alias}@latest + detailedMonitoring: true + instanceProfile: ${cluster_name}-karpenter-instance-profile + instanceStorePolicy: RAID0 + kubelet: + evictionHard: + memory.available: 5% + nodefs.available: 10% + nodefs.inodesFree: 10% + evictionMaxPodGracePeriod: 60 + evictionSoft: + memory.available: 500Mi + nodefs.available: 15% + nodefs.inodesFree: 15% + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: enabled + httpPutResponseHopLimit: 2 + httpTokens: required + securityGroupSelectorTerms: + - tags: + Name: ${cluster_name}-node + subnetSelectorTerms: + - tags: + Name: ${subnets_name} + kubernetes.io/cluster/${cluster_name}: shared + tags: ${tags} diff --git a/karpenter-node-class/templates/_helpers.tpl b/karpenter-node-class/templates/_helpers.tpl new file mode 100644 index 0000000..9bfbc91 --- /dev/null +++ b/karpenter-node-class/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "karpenter-node-class.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "karpenter-node-class.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "karpenter-node-class.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "karpenter-node-class.labels" -}} +helm.sh/chart: {{ include "karpenter-node-class.chart" . }} +{{ include "karpenter-node-class.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "karpenter-node-class.selectorLabels" -}} +app.kubernetes.io/name: {{ include "karpenter-node-class.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "karpenter-node-class.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "karpenter-node-class.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/karpenter-node-class/values.yaml b/karpenter-node-class/values.yaml new file mode 100644 index 0000000..e4779cd --- /dev/null +++ b/karpenter-node-class/values.yaml @@ -0,0 +1 @@ +kubernetesClusterDomain: cluster.local diff --git a/karpenter-node-pool/.helmignore b/karpenter-node-pool/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/karpenter-node-pool/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/karpenter-node-pool/Chart.yaml b/karpenter-node-pool/Chart.yaml new file mode 100644 index 0000000..24f8073 --- /dev/null +++ b/karpenter-node-pool/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: karpenter-node-pool +description: A Helm chart for Kubernetes +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.1.0" diff --git a/karpenter-node-pool/templates/${cluster_name}-node-pool-general.yaml b/karpenter-node-pool/templates/${cluster_name}-node-pool-general.yaml new file mode 100644 index 0000000..798fb49 --- /dev/null +++ b/karpenter-node-pool/templates/${cluster_name}-node-pool-general.yaml @@ -0,0 +1,37 @@ +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: {{ include "karpenter-node-pool.fullname" . }}-${cluster_name}-node-pool-general + labels: + {{- include "karpenter-node-pool.labels" . | nindent 4 }} + annotations: + kubernetes.io/description: General purpose NodePool for generic workloads +spec: + disruption: + consolidateAfter: 5m + consolidationPolicy: WhenEmptyOrUnderutilized + expireAfter: 72h + template: + spec: + nodeClassRef: + group: karpenter.k8s.aws + kind: EC2NodeClass + name: ${cluster_name}-karpenter-node-class + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: kubernetes.io/os + operator: In + values: + - linux + - key: karpenter.k8s.aws/instance-hypervisor + operator: In + values: + - nitro + - key: karpenter.sh/capacity-type + operator: In + values: + - spot + - on-demand diff --git a/karpenter-node-pool/templates/_helpers.tpl b/karpenter-node-pool/templates/_helpers.tpl new file mode 100644 index 0000000..0b50a67 --- /dev/null +++ b/karpenter-node-pool/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "karpenter-node-pool.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "karpenter-node-pool.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "karpenter-node-pool.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "karpenter-node-pool.labels" -}} +helm.sh/chart: {{ include "karpenter-node-pool.chart" . }} +{{ include "karpenter-node-pool.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "karpenter-node-pool.selectorLabels" -}} +app.kubernetes.io/name: {{ include "karpenter-node-pool.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "karpenter-node-pool.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "karpenter-node-pool.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/karpenter-node-pool/values.yaml b/karpenter-node-pool/values.yaml new file mode 100644 index 0000000..e4779cd --- /dev/null +++ b/karpenter-node-pool/values.yaml @@ -0,0 +1 @@ +kubernetesClusterDomain: cluster.local diff --git a/main.tf b/main.tf index cc0c007..8e7aff6 100644 --- a/main.tf +++ b/main.tf @@ -153,27 +153,70 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { role = module.karpenter_resources.node_iam_role_name } -# SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART -# Create karpenter default -resource "kubernetes_manifest" "karpenter_ec2_node_class" { - depends_on = [ - helm_release.karpenter, - ] - manifest = yamldecode(templatefile("${path.module}/ec2_node_class.yaml.tpl", { - cluster_name = var.cluster_name - tags = jsonencode(data.aws_default_tags.default_tags.tags) - amd_ami_family = "Bottlerocket" - amd_ami_alias = "bottlerocket" - subnets_name = local.container_subnets_name - })) +# # SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART +# # Create karpenter default +# resource "kubernetes_manifest" "karpenter_ec2_node_class" { +# depends_on = [ +# helm_release.karpenter, +# ] +# manifest = yamldecode(templatefile("${path.module}/ec2_node_class.yaml.tpl", { +# cluster_name = var.cluster_name +# tags = jsonencode(data.aws_default_tags.default_tags.tags) +# amd_ami_family = "Bottlerocket" +# amd_ami_alias = "bottlerocket" +# subnets_name = local.container_subnets_name +# })) +# } + +# # Create karpenter default NodePool +# resource "kubernetes_manifest" "karpenter_node_pool" { +# depends_on = [ +# kubernetes_manifest.karpenter_ec2_node_class, +# ] +# manifest = yamldecode(templatefile("${path.module}/ec2_node_pool.yaml.tpl", { +# cluster_name = var.cluster_name +# })) +# } + +resource "helm_release" "karpenter_node_class" { + chart = "karpenter-node-class" + name = "karpenter-node-class" + namespace = local.namespace + version = var.release_version + repository = "./" + + set { + name = "cluster_name" + value = var.cluster_name + } + set { + name = "tags" + value = jsonencode(data.aws_default_tags.default_tags.tags) + } + set { + name = "amd_ami_family" + value = "Bottlerocket" + } + set { + name = "amd_ami_alias" + value = "bottlerocket" + } + set { + name = "subnets_name" + value = local.container_subnets_name + } + } -# Create karpenter default NodePool -resource "kubernetes_manifest" "karpenter_node_pool" { - depends_on = [ - kubernetes_manifest.karpenter_ec2_node_class, - ] - manifest = yamldecode(templatefile("${path.module}/ec2_node_pool.yaml.tpl", { - cluster_name = var.cluster_name - })) +resource "helm_release" "karpenter_node_pool" { + chart = "karpenter-node-pool" + name = "karpenter-node-pool" + namespace = local.namespace + version = var.release_version + repository = "./" + + set { + name = "cluster_name" + value = var.cluster_name + } } diff --git a/requirements.tf b/requirements.tf index ae62e15..2e678e4 100644 --- a/requirements.tf +++ b/requirements.tf @@ -10,9 +10,5 @@ terraform { source = "hashicorp/helm" version = ">= 2.11.0" } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.23.0" - } } } diff --git a/variables.tf b/variables.tf index ca1e6f8..f9162ce 100644 --- a/variables.tf +++ b/variables.tf @@ -48,6 +48,11 @@ variable "karpenter_tag" { default = "1.0.6" } +variable "release_version" { + description = "Which version of platform-eng packages to use" + type = string +} + variable "subnets_name" { description = "Define the name of the subnets to be used by this cluster" type = string From d429bf8aae585d8a876bbcb6993a3133e03d7221 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 19:43:08 -0500 Subject: [PATCH 06/41] vars --- ec2_node_class.yaml.tpl | 18 +++++++++--------- ec2_node_pool.yaml.tpl | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ec2_node_class.yaml.tpl b/ec2_node_class.yaml.tpl index baf320e..4f77f29 100644 --- a/ec2_node_class.yaml.tpl +++ b/ec2_node_class.yaml.tpl @@ -2,9 +2,9 @@ apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: - name: ${cluster_name}-karpenter-node-class + name: "{{ .Values.cluster_name }}-karpenter-node-class" annotations: - kubernetes.io/description: "EC2NodeClass for running ${amd_ami_family} nodes" + kubernetes.io/description: "EC2NodeClass for running {{ .Values.amd_ami_family }} nodes" spec: kubelet: evictionHard: @@ -24,27 +24,27 @@ spec: imageGCLowThresholdPercent: 80 # Required, resolves a default ami and userdata - amiFamily: ${amd_ami_family} + amiFamily: {{ .Values.amd_ami_family }} amiSelectorTerms: - - alias: ${amd_ami_alias}@latest # Bottlerocket + - alias: "{{ .Values.amd_ami_alias }}@latest" # Bottlerocket # Required, discovers subnets to attach to instances # Each term in the array of subnetSelectorTerms is ORed together # Within a single term, all conditions are ANDed subnetSelectorTerms: - tags: - Name: "${subnets_name}" - "kubernetes.io/cluster/${cluster_name}": "shared" + Name: {{ .Values.subnets_name }} + "kubernetes.io/cluster/{{ .Values.cluster_name }}": "shared" # Required, discovers security groups to attach to instances # Each term in the array of securityGroupSelectorTerms is ORed together # Within a single term, all conditions are ANDed securityGroupSelectorTerms: - tags: - Name: "${cluster_name}-node" + Name: "{{ .Values.cluster_name }}-node" # Optional, IAM instance profile to use for the node identity. # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes - instanceProfile: "${cluster_name}-karpenter-instance-profile" + instanceProfile: "{{ .Values.cluster_name }}-karpenter-instance-profile" # Optional, use instance-store volumes for node ephemeral-storage instanceStorePolicy: RAID0 @@ -60,4 +60,4 @@ spec: detailedMonitoring: true # Add finops tags to karpenter provisioned nodes - tags: ${tags} + tags: {{ .Values.tags }} diff --git a/ec2_node_pool.yaml.tpl b/ec2_node_pool.yaml.tpl index 5062239..73c4a68 100644 --- a/ec2_node_pool.yaml.tpl +++ b/ec2_node_pool.yaml.tpl @@ -2,7 +2,7 @@ apiVersion: karpenter.sh/v1 kind: NodePool metadata: - name: ${cluster_name}-node-pool-general + name: {{ .Values.cluster_name }}-node-pool-general annotations: kubernetes.io/description: "General purpose NodePool for generic workloads" spec: @@ -30,7 +30,7 @@ spec: nodeClassRef: group: karpenter.k8s.aws kind: EC2NodeClass - name: ${cluster_name}-karpenter-node-class + name: {{ .Values.cluster_name }}-karpenter-node-class # kubelet: # containerRuntime: containerd # systemReserved: From bc59b27fdf6530a26b39017340baa6383a610bc4 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 19:58:37 -0500 Subject: [PATCH 07/41] more renames --- ...de-class.yaml => karpenter-node-class.yaml} | 18 +++++++++--------- ...ool-general.yaml => node-pool-general.yaml} | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) rename karpenter-node-class/templates/{${cluster_name}-karpenter-node-class.yaml => karpenter-node-class.yaml} (62%) rename karpenter-node-pool/templates/{${cluster_name}-node-pool-general.yaml => node-pool-general.yaml} (83%) diff --git a/karpenter-node-class/templates/${cluster_name}-karpenter-node-class.yaml b/karpenter-node-class/templates/karpenter-node-class.yaml similarity index 62% rename from karpenter-node-class/templates/${cluster_name}-karpenter-node-class.yaml rename to karpenter-node-class/templates/karpenter-node-class.yaml index bdb94d6..94a393d 100644 --- a/karpenter-node-class/templates/${cluster_name}-karpenter-node-class.yaml +++ b/karpenter-node-class/templates/karpenter-node-class.yaml @@ -1,17 +1,17 @@ apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: - name: {{ include "karpenter-node-class.fullname" . }}-${cluster_name}-karpenter-node-class + name: {{ include "karpenter-node-class.fullname" . }}-{{ .Values.cluster_name }}-karpenter-node-class labels: {{- include "karpenter-node-class.labels" . | nindent 4 }} annotations: - kubernetes.io/description: EC2NodeClass for running ${amd_ami_family} nodes + kubernetes.io/description: EC2NodeClass for running {{ .Values.amd_ami_family }} nodes spec: - amiFamily: ${amd_ami_family} + amiFamily: {{ .Values.amd_ami_family }} amiSelectorTerms: - - alias: ${amd_ami_alias}@latest + - alias: {{ .Values.amd_ami_alias }}@latest detailedMonitoring: true - instanceProfile: ${cluster_name}-karpenter-instance-profile + instanceProfile: {{ .Values.cluster_name }}-karpenter-instance-profile instanceStorePolicy: RAID0 kubelet: evictionHard: @@ -36,9 +36,9 @@ spec: httpTokens: required securityGroupSelectorTerms: - tags: - Name: ${cluster_name}-node + Name: {{ .Values.cluster_name }}-node subnetSelectorTerms: - tags: - Name: ${subnets_name} - kubernetes.io/cluster/${cluster_name}: shared - tags: ${tags} + Name: {{ .Values.subnets_name }} + kubernetes.io/cluster/{{ .Values.cluster_name }}: shared + tags: {{ .Values.tags }} diff --git a/karpenter-node-pool/templates/${cluster_name}-node-pool-general.yaml b/karpenter-node-pool/templates/node-pool-general.yaml similarity index 83% rename from karpenter-node-pool/templates/${cluster_name}-node-pool-general.yaml rename to karpenter-node-pool/templates/node-pool-general.yaml index 798fb49..bd11d90 100644 --- a/karpenter-node-pool/templates/${cluster_name}-node-pool-general.yaml +++ b/karpenter-node-pool/templates/node-pool-general.yaml @@ -1,7 +1,7 @@ apiVersion: karpenter.sh/v1 kind: NodePool metadata: - name: {{ include "karpenter-node-pool.fullname" . }}-${cluster_name}-node-pool-general + name: {{ include "karpenter-node-pool.fullname" . }}-{{ .Values.cluster_name }}-node-pool-general labels: {{- include "karpenter-node-pool.labels" . | nindent 4 }} annotations: @@ -16,7 +16,7 @@ spec: nodeClassRef: group: karpenter.k8s.aws kind: EC2NodeClass - name: ${cluster_name}-karpenter-node-class + name: {{ .Values.cluster_name }}-karpenter-node-class requirements: - key: kubernetes.io/arch operator: In From d37c9cafa323fd01bc3ecdb1ac7e17cec1e0d823 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 20:03:18 -0500 Subject: [PATCH 08/41] yaml --- karpenter-node-class/templates/karpenter-node-class.yaml | 3 ++- main.tf | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/karpenter-node-class/templates/karpenter-node-class.yaml b/karpenter-node-class/templates/karpenter-node-class.yaml index 94a393d..008ae59 100644 --- a/karpenter-node-class/templates/karpenter-node-class.yaml +++ b/karpenter-node-class/templates/karpenter-node-class.yaml @@ -41,4 +41,5 @@ spec: - tags: Name: {{ .Values.subnets_name }} kubernetes.io/cluster/{{ .Values.cluster_name }}: shared - tags: {{ .Values.tags }} + tags: + {{ .Values.tags }} diff --git a/main.tf b/main.tf index 8e7aff6..c0159e4 100644 --- a/main.tf +++ b/main.tf @@ -191,7 +191,7 @@ resource "helm_release" "karpenter_node_class" { } set { name = "tags" - value = jsonencode(data.aws_default_tags.default_tags.tags) + value = yamlencode(data.aws_default_tags.default_tags.tags) } set { name = "amd_ami_family" From f815a13888bc94c81d0894cc75ea183e500593ee Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 20:08:35 -0500 Subject: [PATCH 09/41] simpler --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index c0159e4..13664bf 100644 --- a/main.tf +++ b/main.tf @@ -191,7 +191,7 @@ resource "helm_release" "karpenter_node_class" { } set { name = "tags" - value = yamlencode(data.aws_default_tags.default_tags.tags) + value = data.aws_default_tags.default_tags.tags } set { name = "amd_ami_family" From 5d830dca23db3a852682cd38af3116bfdcd6dcfe Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 20:44:06 -0500 Subject: [PATCH 10/41] tasg --- README.md | 4 ++-- karpenter-node-class/templates/karpenter-node-class.yaml | 3 +-- main.tf | 2 +- variables.tf | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7b5fa82..c77949f 100644 --- a/README.md +++ b/README.md @@ -54,10 +54,10 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|-------------|------|---------|:--------:| | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint for your Kubernetes API server | `string` | n/a | yes | | [cluster\_name](#input\_cluster\_name) | EKS cluster name name component used through out the EKS cluster describing its purpose (ex: dice-dev) | `string` | n/a | yes | -| [karpenter\_helm\_chart](#input\_karpenter\_helm\_chart) | Which helm chart of karpenter | `string` | `"1.0.6"` | no | +| [karpenter\_helm\_chart](#input\_karpenter\_helm\_chart) | Which helm chart of karpenter | `string` | `"1.1.1"` | no | | [karpenter\_helm\_repo](#input\_karpenter\_helm\_repo) | Helm repo for official karpenter chart | `string` | `"oci://public.ecr.aws/karpenter"` | no | | [karpenter\_node\_group\_name](#input\_karpenter\_node\_group\_name) | The cluster node group that will host karpenter, should not be a karpenter managed node group | `string` | n/a | yes | -| [karpenter\_tag](#input\_karpenter\_tag) | Which tag of karpenter image | `string` | `"1.0.6"` | no | +| [karpenter\_tag](#input\_karpenter\_tag) | Which tag of karpenter image | `string` | `"1.1.1"` | no | | [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes | | [profile](#input\_profile) | AWS config profile | `string` | `""` | no | | [region](#input\_region) | AWS region | `string` | n/a | yes | diff --git a/karpenter-node-class/templates/karpenter-node-class.yaml b/karpenter-node-class/templates/karpenter-node-class.yaml index 008ae59..94a393d 100644 --- a/karpenter-node-class/templates/karpenter-node-class.yaml +++ b/karpenter-node-class/templates/karpenter-node-class.yaml @@ -41,5 +41,4 @@ spec: - tags: Name: {{ .Values.subnets_name }} kubernetes.io/cluster/{{ .Values.cluster_name }}: shared - tags: - {{ .Values.tags }} + tags: {{ .Values.tags }} diff --git a/main.tf b/main.tf index 13664bf..8e7aff6 100644 --- a/main.tf +++ b/main.tf @@ -191,7 +191,7 @@ resource "helm_release" "karpenter_node_class" { } set { name = "tags" - value = data.aws_default_tags.default_tags.tags + value = jsonencode(data.aws_default_tags.default_tags.tags) } set { name = "amd_ami_family" diff --git a/variables.tf b/variables.tf index f9162ce..8d055cc 100644 --- a/variables.tf +++ b/variables.tf @@ -39,13 +39,13 @@ variable "karpenter_helm_repo" { variable "karpenter_helm_chart" { description = "Which helm chart of karpenter" type = string - default = "1.0.6" + default = "1.1.1" } variable "karpenter_tag" { description = "Which tag of karpenter image" type = string - default = "1.0.6" + default = "1.1.1" } variable "release_version" { From 29c4242d0c982b1fa206c8439d1b01c45a4f55e9 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 20:48:53 -0500 Subject: [PATCH 11/41] fromJson --- karpenter-node-class/templates/karpenter-node-class.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karpenter-node-class/templates/karpenter-node-class.yaml b/karpenter-node-class/templates/karpenter-node-class.yaml index 94a393d..b935b28 100644 --- a/karpenter-node-class/templates/karpenter-node-class.yaml +++ b/karpenter-node-class/templates/karpenter-node-class.yaml @@ -41,4 +41,4 @@ spec: - tags: Name: {{ .Values.subnets_name }} kubernetes.io/cluster/{{ .Values.cluster_name }}: shared - tags: {{ .Values.tags }} + tags: {{ .Values.tags | fromJson }} From 6b31095202cd2ed8d9ff9a60d3c87d6ea9a70be1 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 21:35:57 -0500 Subject: [PATCH 12/41] native tf and lints --- README.md | 8 +-- aws_data.tf | 2 - main.tf | 159 ++++++++++++++++++++++++++++++++++++------------ requirements.tf | 4 ++ variables.tf | 5 -- 5 files changed, 129 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index c77949f..a40e82b 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [terraform](#requirement\_terraform) | >= 0.13 | | [aws](#requirement\_aws) | >= 5.14.0 | | [helm](#requirement\_helm) | >= 2.11.0 | +| [kubernetes](#requirement\_kubernetes) | >= 2.23.0 | ## Providers @@ -27,6 +28,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|---------| | [aws](#provider\_aws) | >= 5.14.0 | | [helm](#provider\_helm) | >= 2.11.0 | +| [kubernetes](#provider\_kubernetes) | >= 2.23.0 | ## Modules @@ -41,11 +43,10 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|------| | [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.karpenter_node_class](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_manifest.karpenter_node_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | +| [kubernetes_manifest.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_default_tags.default_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | ## Inputs @@ -61,7 +62,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes | | [profile](#input\_profile) | AWS config profile | `string` | `""` | no | | [region](#input\_region) | AWS region | `string` | n/a | yes | -| [release\_version](#input\_release\_version) | Which version of platform-eng packages to use | `string` | n/a | yes | | [subnets\_name](#input\_subnets\_name) | Define the name of the subnets to be used by this cluster | `string` | `"*-container-*"` | no | | [tags](#input\_tags) | AWS Tags to apply to appropriate resources | `map(string)` | `{}` | no | diff --git a/aws_data.tf b/aws_data.tf index e7211e8..2c6aade 100644 --- a/aws_data.tf +++ b/aws_data.tf @@ -3,5 +3,3 @@ data "aws_caller_identity" "current" {} data "aws_arn" "current" { arn = data.aws_caller_identity.current.arn } - -data "aws_default_tags" "default_tags" {} diff --git a/main.tf b/main.tf index 8e7aff6..85a1a46 100644 --- a/main.tf +++ b/main.tf @@ -1,6 +1,7 @@ locals { - namespace = "karpenter" - container_subnets_name = var.subnets_name + namespace = "karpenter" + amd_ami_family = "Bottlerocket" + amd_ami_alias = "bottlerocket" } # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter @@ -178,45 +179,127 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { # })) # } -resource "helm_release" "karpenter_node_class" { - chart = "karpenter-node-class" - name = "karpenter-node-class" - namespace = local.namespace - version = var.release_version - repository = "./" +resource "kubernetes_manifest" "karpenter_node_class" { + depends_on = [helm_release.karpenter] - set { - name = "cluster_name" - value = var.cluster_name - } - set { - name = "tags" - value = jsonencode(data.aws_default_tags.default_tags.tags) - } - set { - name = "amd_ami_family" - value = "Bottlerocket" - } - set { - name = "amd_ami_alias" - value = "bottlerocket" - } - set { - name = "subnets_name" - value = local.container_subnets_name + manifest = { + apiVersion = "karpenter.k8s.aws/v1" + kind = "EC2NodeClass" + metadata = { + name = "${var.cluster_name}-karpenter-node-class" + annotations = { + "kubernetes.io/description" = "EC2NodeClass for running ${local.amd_ami_family} nodes" + } + } + spec = { + amiFamily = local.amd_ami_family + amiSelectorTerms = [ + { + alias = "${local.amd_ami_alias}@latest" + } + ] + detailedMonitoring = true + instanceProfile = "${var.cluster_name}-karpenter-instance-profile" + instanceStorePolicy = "RAID0" + kubelet = { + evictionHard = { + "memory.available" = "5%" + "nodefs.available" = "10%" + "nodefs.inodesFree" = "10%" + } + evictionMaxPodGracePeriod = 60 + evictionSoft = { + "memory.available" = "500Mi" + "nodefs.available" = "15%" + "nodefs.inodesFree" = "15%" + } + evictionSoftGracePeriod = { + "memory.available" = "1m" + "nodefs.available" = "1m30s" + "nodefs.inodesFree" = "2m" + } + imageGCHighThresholdPercent = 85 + imageGCLowThresholdPercent = 80 + } + metadataOptions = { + httpEndpoint = "enabled" + httpProtocolIPv6 = "enabled" + httpPutResponseHopLimit = 2 + httpTokens = "required" + } + securityGroupSelectorTerms = [ + { + tags = { + Name = "${var.cluster_name}-node" + } + } + ] + subnetSelectorTerms = [ + { + tags = { + Name = var.subnets_name + "kubernetes.io/cluster/${var.cluster_name}" = "shared" + } + } + ] + tags = var.tags + } } - } -resource "helm_release" "karpenter_node_pool" { - chart = "karpenter-node-pool" - name = "karpenter-node-pool" - namespace = local.namespace - version = var.release_version - repository = "./" - - set { - name = "cluster_name" - value = var.cluster_name +resource "kubernetes_manifest" "karpenter_node_pool" { + depends_on = [helm_release.karpenter] + manifest = { + apiVersion = "karpenter.sh/v1" + kind = "NodePool" + metadata = { + name = "${var.cluster_name}-node-pool-general" + annotations = { + "kubernetes.io/description" = "General purpose NodePool for generic workloads" + } + } + spec = { + template = { + spec = { + requirements = [ + { + key = "kubernetes.io/arch" + operator = "In" + values = ["amd64"] + }, + { + key = "kubernetes.io/os" + operator = "In" + values = ["linux"] + }, + { + key = "karpenter.k8s.aws/instance-hypervisor" + operator = "In" + values = ["nitro"] + }, + { + key = "karpenter.k8s.aws/capacity-type" + operator = "In" + values = ["spot", "on-demand"] + }, + { + key = "karpenter.k8s.aws/instance-generation" + operator = "Gt" + values = ["2"] + } + ] + nodeClassRef = { + group = "karpenter.k8s.aws" + kind = "EC2NodeClass" + name = "${var.cluster_name}-karpenter-node-class" + } + } + } + } + disruption = { + consolidationPolicy = "WhenEmptyOrUnderutilized" + consolidateAfter = "5m" + expireAfter = "72h" + } } } diff --git a/requirements.tf b/requirements.tf index 2e678e4..ae62e15 100644 --- a/requirements.tf +++ b/requirements.tf @@ -10,5 +10,9 @@ terraform { source = "hashicorp/helm" version = ">= 2.11.0" } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.23.0" + } } } diff --git a/variables.tf b/variables.tf index 8d055cc..bd99185 100644 --- a/variables.tf +++ b/variables.tf @@ -48,11 +48,6 @@ variable "karpenter_tag" { default = "1.1.1" } -variable "release_version" { - description = "Which version of platform-eng packages to use" - type = string -} - variable "subnets_name" { description = "Define the name of the subnets to be used by this cluster" type = string From 9bbcad0964983deb24594cda7d5f48d6aeb82c14 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 21:39:23 -0500 Subject: [PATCH 13/41] up one --- main.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/main.tf b/main.tf index 85a1a46..59e4356 100644 --- a/main.tf +++ b/main.tf @@ -295,11 +295,11 @@ resource "kubernetes_manifest" "karpenter_node_pool" { } } } - } - disruption = { - consolidationPolicy = "WhenEmptyOrUnderutilized" - consolidateAfter = "5m" - expireAfter = "72h" + disruption = { + consolidationPolicy = "WhenEmptyOrUnderutilized" + consolidateAfter = "5m" + expireAfter = "72h" + } } } } From 2939ea64482149e08248d08041894714ddaf2ee8 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 21:46:37 -0500 Subject: [PATCH 14/41] separate files --- main.tf | 182 -------------------------------------------------- node_class.tf | 67 +++++++++++++++++++ node_pool.tf | 56 ++++++++++++++++ 3 files changed, 123 insertions(+), 182 deletions(-) create mode 100644 node_class.tf create mode 100644 node_pool.tf diff --git a/main.tf b/main.tf index 59e4356..4393215 100644 --- a/main.tf +++ b/main.tf @@ -102,21 +102,6 @@ resource "helm_release" "karpenter" { name = "controller.image.digest" value = data.aws_ecr_image.karpenter_image.id } - # set { - # name = "postInstallHook.image.repository" - # value = format("%v/%v", - # module.images.images[local.kubectl_key].dest_registry, - # module.images.images[local.kubectl_key].dest_repository - # ) - # } - # set { - # name = "postInstallHook.image.tag" - # value = module.images.images[local.kubectl_key].tag - # } - # set { - # name = "postInstallHook.image.digest" - # value = data.aws_ecr_image.kubectl_image.id - # } set { name = "controller.env[0].name" value = "AWS_REGION" @@ -125,27 +110,10 @@ resource "helm_release" "karpenter" { name = "controller.env[0].value" value = var.region } - # set { - # name = "resources.requests.cpu" - # value = 1 - # } - # set { - # name = "resources.requests.memory" - # value = "1Gi" - # } - # set { - # name = "resources.limits.cpu" - # value = 1 - # } - # set { - # name = "resources.limits.memory" - # value = "1Gi" - # } set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" value = module.karpenter_resources.iam_role_arn } - timeout = 300 } @@ -153,153 +121,3 @@ resource "aws_iam_instance_profile" "karpenter_node_instance_profile" { name = "${var.cluster_name}-karpenter-instance-profile" role = module.karpenter_resources.node_iam_role_name } - -# # SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART -# # Create karpenter default -# resource "kubernetes_manifest" "karpenter_ec2_node_class" { -# depends_on = [ -# helm_release.karpenter, -# ] -# manifest = yamldecode(templatefile("${path.module}/ec2_node_class.yaml.tpl", { -# cluster_name = var.cluster_name -# tags = jsonencode(data.aws_default_tags.default_tags.tags) -# amd_ami_family = "Bottlerocket" -# amd_ami_alias = "bottlerocket" -# subnets_name = local.container_subnets_name -# })) -# } - -# # Create karpenter default NodePool -# resource "kubernetes_manifest" "karpenter_node_pool" { -# depends_on = [ -# kubernetes_manifest.karpenter_ec2_node_class, -# ] -# manifest = yamldecode(templatefile("${path.module}/ec2_node_pool.yaml.tpl", { -# cluster_name = var.cluster_name -# })) -# } - -resource "kubernetes_manifest" "karpenter_node_class" { - depends_on = [helm_release.karpenter] - - manifest = { - apiVersion = "karpenter.k8s.aws/v1" - kind = "EC2NodeClass" - metadata = { - name = "${var.cluster_name}-karpenter-node-class" - annotations = { - "kubernetes.io/description" = "EC2NodeClass for running ${local.amd_ami_family} nodes" - } - } - spec = { - amiFamily = local.amd_ami_family - amiSelectorTerms = [ - { - alias = "${local.amd_ami_alias}@latest" - } - ] - detailedMonitoring = true - instanceProfile = "${var.cluster_name}-karpenter-instance-profile" - instanceStorePolicy = "RAID0" - kubelet = { - evictionHard = { - "memory.available" = "5%" - "nodefs.available" = "10%" - "nodefs.inodesFree" = "10%" - } - evictionMaxPodGracePeriod = 60 - evictionSoft = { - "memory.available" = "500Mi" - "nodefs.available" = "15%" - "nodefs.inodesFree" = "15%" - } - evictionSoftGracePeriod = { - "memory.available" = "1m" - "nodefs.available" = "1m30s" - "nodefs.inodesFree" = "2m" - } - imageGCHighThresholdPercent = 85 - imageGCLowThresholdPercent = 80 - } - metadataOptions = { - httpEndpoint = "enabled" - httpProtocolIPv6 = "enabled" - httpPutResponseHopLimit = 2 - httpTokens = "required" - } - securityGroupSelectorTerms = [ - { - tags = { - Name = "${var.cluster_name}-node" - } - } - ] - subnetSelectorTerms = [ - { - tags = { - Name = var.subnets_name - "kubernetes.io/cluster/${var.cluster_name}" = "shared" - } - } - ] - tags = var.tags - } - } -} - -resource "kubernetes_manifest" "karpenter_node_pool" { - depends_on = [helm_release.karpenter] - manifest = { - apiVersion = "karpenter.sh/v1" - kind = "NodePool" - metadata = { - name = "${var.cluster_name}-node-pool-general" - annotations = { - "kubernetes.io/description" = "General purpose NodePool for generic workloads" - } - } - spec = { - template = { - spec = { - requirements = [ - { - key = "kubernetes.io/arch" - operator = "In" - values = ["amd64"] - }, - { - key = "kubernetes.io/os" - operator = "In" - values = ["linux"] - }, - { - key = "karpenter.k8s.aws/instance-hypervisor" - operator = "In" - values = ["nitro"] - }, - { - key = "karpenter.k8s.aws/capacity-type" - operator = "In" - values = ["spot", "on-demand"] - }, - { - key = "karpenter.k8s.aws/instance-generation" - operator = "Gt" - values = ["2"] - } - ] - nodeClassRef = { - group = "karpenter.k8s.aws" - kind = "EC2NodeClass" - name = "${var.cluster_name}-karpenter-node-class" - } - } - } - disruption = { - consolidationPolicy = "WhenEmptyOrUnderutilized" - consolidateAfter = "5m" - expireAfter = "72h" - } - } - } -} diff --git a/node_class.tf b/node_class.tf new file mode 100644 index 0000000..280b159 --- /dev/null +++ b/node_class.tf @@ -0,0 +1,67 @@ +resource "kubernetes_manifest" "karpenter_node_class" { + depends_on = [helm_release.karpenter] + + manifest = { + apiVersion = "karpenter.k8s.aws/v1" + kind = "EC2NodeClass" + metadata = { + name = "${var.cluster_name}-karpenter-node-class" + annotations = { + "kubernetes.io/description" = "EC2NodeClass for running ${local.amd_ami_family} nodes" + } + } + spec = { + amiFamily = local.amd_ami_family + amiSelectorTerms = [ + { + alias = "${local.amd_ami_alias}@latest" + } + ] + detailedMonitoring = true + instanceProfile = "${var.cluster_name}-karpenter-instance-profile" + instanceStorePolicy = "RAID0" + kubelet = { + evictionHard = { + "memory.available" = "5%" + "nodefs.available" = "10%" + "nodefs.inodesFree" = "10%" + } + evictionMaxPodGracePeriod = 60 + evictionSoft = { + "memory.available" = "500Mi" + "nodefs.available" = "15%" + "nodefs.inodesFree" = "15%" + } + evictionSoftGracePeriod = { + "memory.available" = "1m" + "nodefs.available" = "1m30s" + "nodefs.inodesFree" = "2m" + } + imageGCHighThresholdPercent = 85 + imageGCLowThresholdPercent = 80 + } + metadataOptions = { + httpEndpoint = "enabled" + httpProtocolIPv6 = "enabled" + httpPutResponseHopLimit = 2 + httpTokens = "required" + } + securityGroupSelectorTerms = [ + { + tags = { + Name = "${var.cluster_name}-node" + } + } + ] + subnetSelectorTerms = [ + { + tags = { + Name = var.subnets_name + "kubernetes.io/cluster/${var.cluster_name}" = "shared" + } + } + ] + tags = var.tags + } + } +} diff --git a/node_pool.tf b/node_pool.tf new file mode 100644 index 0000000..cff667e --- /dev/null +++ b/node_pool.tf @@ -0,0 +1,56 @@ +resource "kubernetes_manifest" "karpenter_node_pool" { + depends_on = [helm_release.karpenter] + manifest = { + apiVersion = "karpenter.sh/v1" + kind = "NodePool" + metadata = { + name = "${var.cluster_name}-node-pool-general" + annotations = { + "kubernetes.io/description" = "General purpose NodePool for generic workloads" + } + } + spec = { + template = { + spec = { + requirements = [ + { + key = "kubernetes.io/arch" + operator = "In" + values = ["amd64"] + }, + { + key = "kubernetes.io/os" + operator = "In" + values = ["linux"] + }, + { + key = "karpenter.k8s.aws/instance-hypervisor" + operator = "In" + values = ["nitro"] + }, + { + key = "karpenter.k8s.aws/capacity-type" + operator = "In" + values = ["spot", "on-demand"] + }, + { + key = "karpenter.k8s.aws/instance-generation" + operator = "Gt" + values = ["2"] + } + ] + nodeClassRef = { + group = "karpenter.k8s.aws" + kind = "EC2NodeClass" + name = "${var.cluster_name}-karpenter-node-class" + } + } + } + disruption = { + consolidationPolicy = "WhenEmptyOrUnderutilized" + consolidateAfter = "5m" + expireAfter = "72h" + } + } + } +} From 241ab9602019e40be77b21bd9fa4ded3484cba6b Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 21:59:11 -0500 Subject: [PATCH 15/41] expireAfter --- node_pool.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node_pool.tf b/node_pool.tf index cff667e..ca50498 100644 --- a/node_pool.tf +++ b/node_pool.tf @@ -48,9 +48,9 @@ resource "kubernetes_manifest" "karpenter_node_pool" { } disruption = { consolidationPolicy = "WhenEmptyOrUnderutilized" - consolidateAfter = "5m" - expireAfter = "72h" + consolidateAfter = "0" } + expireAfter = "72h" } } } From 44d6127eb7f3683716d5dc947fb1033f57b9402a Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 22:01:10 -0500 Subject: [PATCH 16/41] spec.template.spec.expireAfter --- node_pool.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_pool.tf b/node_pool.tf index ca50498..93b0159 100644 --- a/node_pool.tf +++ b/node_pool.tf @@ -44,13 +44,13 @@ resource "kubernetes_manifest" "karpenter_node_pool" { kind = "EC2NodeClass" name = "${var.cluster_name}-karpenter-node-class" } + expireAfter = "72h" } } disruption = { consolidationPolicy = "WhenEmptyOrUnderutilized" consolidateAfter = "0" } - expireAfter = "72h" } } } From ddcca28b1aee6fbd35bcebede2a6deb9acdd9e56 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 22:34:37 -0500 Subject: [PATCH 17/41] .sh not .k8s.aws --- ec2_node_pool.yaml.tpl | 27 +++++++++++++-------------- node_pool.tf | 6 +++--- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/ec2_node_pool.yaml.tpl b/ec2_node_pool.yaml.tpl index 73c4a68..e813d7c 100644 --- a/ec2_node_pool.yaml.tpl +++ b/ec2_node_pool.yaml.tpl @@ -2,9 +2,9 @@ apiVersion: karpenter.sh/v1 kind: NodePool metadata: - name: {{ .Values.cluster_name }}-node-pool-general + name: "{{ .Values.cluster_name }}-node-pool-general" annotations: - kubernetes.io/description: "General purpose NodePool for generic workloads" + kubernetes.io/description: 'General purpose NodePool for generic workloads' spec: template: spec: @@ -15,30 +15,29 @@ spec: - key: kubernetes.io/os operator: In values: ["linux"] - - key: "karpenter.k8s.aws/instance-hypervisor" + - key: karpenter.k8s.aws/instance-hypervisor operator: In values: ["nitro"] - key: karpenter.sh/capacity-type operator: In values: ["spot", "on-demand"] - # - key: karpenter.k8s.aws/instance-category - # operator: In - # values: ["c", "m", "r", "i", "d", "t"] - # - key: karpenter.k8s.aws/instance-generation - # operator: Gt - # values: ["2"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] nodeClassRef: group: karpenter.k8s.aws kind: EC2NodeClass name: {{ .Values.cluster_name }}-karpenter-node-class - # kubelet: - # containerRuntime: containerd - # systemReserved: - # cpu: 100m - # memory: 100Mi + # The disruption section defines policies for consolidating and expiring nodes. + # Time format: , where unit can be 's' (seconds), 'm' (minutes), 'h' (hours), 'd' (days) disruption: + # consolidationPolicy: Defines the policy for consolidating EC2 instances in the node pool. + # This policy helps in optimizing resource usage and cost by consolidating workloads onto fewer instances. + # Possible values include 'None', 'OptimizeUtilization', and 'OptimizeCost'. consolidationPolicy: WhenEmptyOrUnderutilized + # consolidateAfter specifies the duration after which nodes should be consolidated. consolidateAfter: 5m + # expireAfter specifies the duration after which nodes should be expired. expireAfter: 72h # limits: # cpu: 1000 diff --git a/node_pool.tf b/node_pool.tf index 93b0159..b2afca3 100644 --- a/node_pool.tf +++ b/node_pool.tf @@ -29,7 +29,7 @@ resource "kubernetes_manifest" "karpenter_node_pool" { values = ["nitro"] }, { - key = "karpenter.k8s.aws/capacity-type" + key = "karpenter.sh/capacity-type" operator = "In" values = ["spot", "on-demand"] }, @@ -44,12 +44,12 @@ resource "kubernetes_manifest" "karpenter_node_pool" { kind = "EC2NodeClass" name = "${var.cluster_name}-karpenter-node-class" } - expireAfter = "72h" } } disruption = { consolidationPolicy = "WhenEmptyOrUnderutilized" - consolidateAfter = "0" + consolidateAfter = "0s" + expireAfter = "72h" } } } From c7156e2ff46a2df63f33e1f5c256784b0accae94 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 22:38:25 -0500 Subject: [PATCH 18/41] expireAfter issues --- node_pool.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/node_pool.tf b/node_pool.tf index b2afca3..fb82884 100644 --- a/node_pool.tf +++ b/node_pool.tf @@ -44,12 +44,13 @@ resource "kubernetes_manifest" "karpenter_node_pool" { kind = "EC2NodeClass" name = "${var.cluster_name}-karpenter-node-class" } + expireAfter = "72h" + terminationGracePeriod = "8h" } } disruption = { consolidationPolicy = "WhenEmptyOrUnderutilized" - consolidateAfter = "0s" - expireAfter = "72h" + consolidateAfter = "30s" } } } From 9ec633acbed083f098e34d919e1b3d56292d39c5 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 22 Jan 2025 23:30:37 -0500 Subject: [PATCH 19/41] update depends --- node_class.tf | 2 +- node_pool.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node_class.tf b/node_class.tf index 280b159..e248dea 100644 --- a/node_class.tf +++ b/node_class.tf @@ -1,5 +1,5 @@ resource "kubernetes_manifest" "karpenter_node_class" { - depends_on = [helm_release.karpenter] + depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile] manifest = { apiVersion = "karpenter.k8s.aws/v1" diff --git a/node_pool.tf b/node_pool.tf index fb82884..bf8e51a 100644 --- a/node_pool.tf +++ b/node_pool.tf @@ -1,5 +1,5 @@ resource "kubernetes_manifest" "karpenter_node_pool" { - depends_on = [helm_release.karpenter] + depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile] manifest = { apiVersion = "karpenter.sh/v1" kind = "NodePool" From 88349fb63d0614bac947c96d12ffd1039aa40f90 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 23 Jan 2025 15:59:39 -0500 Subject: [PATCH 20/41] add chart for crds --- README.md | 1 + main.tf | 10 ++++++++++ node_class.tf | 2 +- node_pool.tf | 2 +- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a40e82b..c866d64 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|------| | [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.karpenter_crds](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubernetes_manifest.karpenter_node_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | | [kubernetes_manifest.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | diff --git a/main.tf b/main.tf index 4393215..8f8846c 100644 --- a/main.tf +++ b/main.tf @@ -25,12 +25,22 @@ module "karpenter_resources" { # Install Karpenter data "aws_ecr_image" "karpenter_image" { depends_on = [ + helm_release.karpenter_crds, module.images, ] repository_name = module.images.images[local.karpenter_key].dest_repository image_tag = module.images.images[local.karpenter_key].tag } +resource "helm_release" "karpenter_crds" { + name = "karpenter-crd" + chart = "karpentercrd" + version = var.karpenter_helm_chart + namespace = local.namespace + description = "installs the crds for karpenter so our manifest resources can apply" + repository = "oci://public.ecr.aws/karpenter/karpenter-crd" +} + resource "helm_release" "karpenter" { depends_on = [ data.aws_ecr_image.karpenter_image diff --git a/node_class.tf b/node_class.tf index e248dea..15ca207 100644 --- a/node_class.tf +++ b/node_class.tf @@ -1,5 +1,5 @@ resource "kubernetes_manifest" "karpenter_node_class" { - depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile] + depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile, helm_release.karpenter] manifest = { apiVersion = "karpenter.k8s.aws/v1" diff --git a/node_pool.tf b/node_pool.tf index bf8e51a..57ddce7 100644 --- a/node_pool.tf +++ b/node_pool.tf @@ -1,5 +1,5 @@ resource "kubernetes_manifest" "karpenter_node_pool" { - depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile] + depends_on = [kubernetes_manifest.karpenter_node_class, helm_release.karpenter] manifest = { apiVersion = "karpenter.sh/v1" kind = "NodePool" From f8606b7a7d07ef174d11c76530f4e089abc16c36 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 23 Jan 2025 16:01:30 -0500 Subject: [PATCH 21/41] update depends --- node_class.tf | 2 +- node_pool.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node_class.tf b/node_class.tf index 15ca207..ee18100 100644 --- a/node_class.tf +++ b/node_class.tf @@ -1,5 +1,5 @@ resource "kubernetes_manifest" "karpenter_node_class" { - depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile, helm_release.karpenter] + depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile, helm_release.karpenter_crds] manifest = { apiVersion = "karpenter.k8s.aws/v1" diff --git a/node_pool.tf b/node_pool.tf index 57ddce7..2b00cef 100644 --- a/node_pool.tf +++ b/node_pool.tf @@ -1,5 +1,5 @@ resource "kubernetes_manifest" "karpenter_node_pool" { - depends_on = [kubernetes_manifest.karpenter_node_class, helm_release.karpenter] + depends_on = [kubernetes_manifest.karpenter_node_class, helm_release.karpenter_crds] manifest = { apiVersion = "karpenter.sh/v1" kind = "NodePool" From 0d45c90cfd9fdef32dccae6884fbd6e21280d967 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 15:50:49 -0500 Subject: [PATCH 22/41] update depends --- README.md | 1 - karpenter-node-class/.helmignore | 23 ------- karpenter-node-class/Chart.yaml | 21 ------- karpenter-node-class/templates/_helpers.tpl | 62 ------------------- .../templates/karpenter-node-class.yaml | 44 ------------- karpenter-node-class/values.yaml | 1 - karpenter-node-pool/.helmignore | 23 ------- karpenter-node-pool/Chart.yaml | 21 ------- karpenter-node-pool/templates/_helpers.tpl | 62 ------------------- .../templates/node-pool-general.yaml | 37 ----------- karpenter-node-pool/values.yaml | 1 - main.tf | 16 ++--- node_class.tf | 2 +- 13 files changed, 9 insertions(+), 305 deletions(-) delete mode 100644 karpenter-node-class/.helmignore delete mode 100644 karpenter-node-class/Chart.yaml delete mode 100644 karpenter-node-class/templates/_helpers.tpl delete mode 100644 karpenter-node-class/templates/karpenter-node-class.yaml delete mode 100644 karpenter-node-class/values.yaml delete mode 100644 karpenter-node-pool/.helmignore delete mode 100644 karpenter-node-pool/Chart.yaml delete mode 100644 karpenter-node-pool/templates/_helpers.tpl delete mode 100644 karpenter-node-pool/templates/node-pool-general.yaml delete mode 100644 karpenter-node-pool/values.yaml diff --git a/README.md b/README.md index c866d64..a40e82b 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|------| | [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.karpenter_crds](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubernetes_manifest.karpenter_node_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | | [kubernetes_manifest.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | diff --git a/karpenter-node-class/.helmignore b/karpenter-node-class/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/karpenter-node-class/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/karpenter-node-class/Chart.yaml b/karpenter-node-class/Chart.yaml deleted file mode 100644 index b580371..0000000 --- a/karpenter-node-class/Chart.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v2 -name: karpenter-node-class -description: A Helm chart for Kubernetes -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "0.1.0" diff --git a/karpenter-node-class/templates/_helpers.tpl b/karpenter-node-class/templates/_helpers.tpl deleted file mode 100644 index 9bfbc91..0000000 --- a/karpenter-node-class/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "karpenter-node-class.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "karpenter-node-class.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "karpenter-node-class.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "karpenter-node-class.labels" -}} -helm.sh/chart: {{ include "karpenter-node-class.chart" . }} -{{ include "karpenter-node-class.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "karpenter-node-class.selectorLabels" -}} -app.kubernetes.io/name: {{ include "karpenter-node-class.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "karpenter-node-class.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "karpenter-node-class.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/karpenter-node-class/templates/karpenter-node-class.yaml b/karpenter-node-class/templates/karpenter-node-class.yaml deleted file mode 100644 index b935b28..0000000 --- a/karpenter-node-class/templates/karpenter-node-class.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: karpenter.k8s.aws/v1 -kind: EC2NodeClass -metadata: - name: {{ include "karpenter-node-class.fullname" . }}-{{ .Values.cluster_name }}-karpenter-node-class - labels: - {{- include "karpenter-node-class.labels" . | nindent 4 }} - annotations: - kubernetes.io/description: EC2NodeClass for running {{ .Values.amd_ami_family }} nodes -spec: - amiFamily: {{ .Values.amd_ami_family }} - amiSelectorTerms: - - alias: {{ .Values.amd_ami_alias }}@latest - detailedMonitoring: true - instanceProfile: {{ .Values.cluster_name }}-karpenter-instance-profile - instanceStorePolicy: RAID0 - kubelet: - evictionHard: - memory.available: 5% - nodefs.available: 10% - nodefs.inodesFree: 10% - evictionMaxPodGracePeriod: 60 - evictionSoft: - memory.available: 500Mi - nodefs.available: 15% - nodefs.inodesFree: 15% - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - imageGCHighThresholdPercent: 85 - imageGCLowThresholdPercent: 80 - metadataOptions: - httpEndpoint: enabled - httpProtocolIPv6: enabled - httpPutResponseHopLimit: 2 - httpTokens: required - securityGroupSelectorTerms: - - tags: - Name: {{ .Values.cluster_name }}-node - subnetSelectorTerms: - - tags: - Name: {{ .Values.subnets_name }} - kubernetes.io/cluster/{{ .Values.cluster_name }}: shared - tags: {{ .Values.tags | fromJson }} diff --git a/karpenter-node-class/values.yaml b/karpenter-node-class/values.yaml deleted file mode 100644 index e4779cd..0000000 --- a/karpenter-node-class/values.yaml +++ /dev/null @@ -1 +0,0 @@ -kubernetesClusterDomain: cluster.local diff --git a/karpenter-node-pool/.helmignore b/karpenter-node-pool/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/karpenter-node-pool/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/karpenter-node-pool/Chart.yaml b/karpenter-node-pool/Chart.yaml deleted file mode 100644 index 24f8073..0000000 --- a/karpenter-node-pool/Chart.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v2 -name: karpenter-node-pool -description: A Helm chart for Kubernetes -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "0.1.0" diff --git a/karpenter-node-pool/templates/_helpers.tpl b/karpenter-node-pool/templates/_helpers.tpl deleted file mode 100644 index 0b50a67..0000000 --- a/karpenter-node-pool/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "karpenter-node-pool.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "karpenter-node-pool.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "karpenter-node-pool.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "karpenter-node-pool.labels" -}} -helm.sh/chart: {{ include "karpenter-node-pool.chart" . }} -{{ include "karpenter-node-pool.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "karpenter-node-pool.selectorLabels" -}} -app.kubernetes.io/name: {{ include "karpenter-node-pool.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "karpenter-node-pool.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "karpenter-node-pool.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/karpenter-node-pool/templates/node-pool-general.yaml b/karpenter-node-pool/templates/node-pool-general.yaml deleted file mode 100644 index bd11d90..0000000 --- a/karpenter-node-pool/templates/node-pool-general.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: karpenter.sh/v1 -kind: NodePool -metadata: - name: {{ include "karpenter-node-pool.fullname" . }}-{{ .Values.cluster_name }}-node-pool-general - labels: - {{- include "karpenter-node-pool.labels" . | nindent 4 }} - annotations: - kubernetes.io/description: General purpose NodePool for generic workloads -spec: - disruption: - consolidateAfter: 5m - consolidationPolicy: WhenEmptyOrUnderutilized - expireAfter: 72h - template: - spec: - nodeClassRef: - group: karpenter.k8s.aws - kind: EC2NodeClass - name: {{ .Values.cluster_name }}-karpenter-node-class - requirements: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: kubernetes.io/os - operator: In - values: - - linux - - key: karpenter.k8s.aws/instance-hypervisor - operator: In - values: - - nitro - - key: karpenter.sh/capacity-type - operator: In - values: - - spot - - on-demand diff --git a/karpenter-node-pool/values.yaml b/karpenter-node-pool/values.yaml deleted file mode 100644 index e4779cd..0000000 --- a/karpenter-node-pool/values.yaml +++ /dev/null @@ -1 +0,0 @@ -kubernetesClusterDomain: cluster.local diff --git a/main.tf b/main.tf index 8f8846c..2dd3e93 100644 --- a/main.tf +++ b/main.tf @@ -32,14 +32,14 @@ data "aws_ecr_image" "karpenter_image" { image_tag = module.images.images[local.karpenter_key].tag } -resource "helm_release" "karpenter_crds" { - name = "karpenter-crd" - chart = "karpentercrd" - version = var.karpenter_helm_chart - namespace = local.namespace - description = "installs the crds for karpenter so our manifest resources can apply" - repository = "oci://public.ecr.aws/karpenter/karpenter-crd" -} +# resource "helm_release" "karpenter_crds" { +# name = "karpenter-crd" +# chart = "karpentercrd" +# version = var.karpenter_helm_chart +# namespace = local.namespace +# description = "installs the crds for karpenter so our manifest resources can apply" +# repository = "oci://public.ecr.aws/karpenter/karpenter-crd" +# } resource "helm_release" "karpenter" { depends_on = [ diff --git a/node_class.tf b/node_class.tf index ee18100..1ecc4bf 100644 --- a/node_class.tf +++ b/node_class.tf @@ -1,5 +1,5 @@ resource "kubernetes_manifest" "karpenter_node_class" { - depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile, helm_release.karpenter_crds] + depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile, helm_release.karpenter.cluster_endpoint] manifest = { apiVersion = "karpenter.k8s.aws/v1" From b0ce8237bb87c904de987735bc5784a5bb78309c Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 15:52:09 -0500 Subject: [PATCH 23/41] not this --- main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/main.tf b/main.tf index 2dd3e93..58a38fd 100644 --- a/main.tf +++ b/main.tf @@ -25,7 +25,6 @@ module "karpenter_resources" { # Install Karpenter data "aws_ecr_image" "karpenter_image" { depends_on = [ - helm_release.karpenter_crds, module.images, ] repository_name = module.images.images[local.karpenter_key].dest_repository From 64cfbd10b72e92d6867b91c133047728b56827a1 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 15:53:23 -0500 Subject: [PATCH 24/41] maybe --- node_class.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_class.tf b/node_class.tf index 1ecc4bf..e248dea 100644 --- a/node_class.tf +++ b/node_class.tf @@ -1,5 +1,5 @@ resource "kubernetes_manifest" "karpenter_node_class" { - depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile, helm_release.karpenter.cluster_endpoint] + depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile] manifest = { apiVersion = "karpenter.k8s.aws/v1" From f172757547c9a8b4424e25ce4f1c1bedd8aa00e3 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 15:54:21 -0500 Subject: [PATCH 25/41] not this --- node_pool.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_pool.tf b/node_pool.tf index 2b00cef..fb7f45f 100644 --- a/node_pool.tf +++ b/node_pool.tf @@ -1,5 +1,5 @@ resource "kubernetes_manifest" "karpenter_node_pool" { - depends_on = [kubernetes_manifest.karpenter_node_class, helm_release.karpenter_crds] + depends_on = [kubernetes_manifest.karpenter_node_class] manifest = { apiVersion = "karpenter.sh/v1" kind = "NodePool" From afc0fed18c995b0cec8b522fa26b17f1d9859d10 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 15:58:45 -0500 Subject: [PATCH 26/41] add wait --- node_class.tf | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/node_class.tf b/node_class.tf index e248dea..351b601 100644 --- a/node_class.tf +++ b/node_class.tf @@ -64,4 +64,25 @@ resource "kubernetes_manifest" "karpenter_node_class" { tags = var.tags } } + wait { + fields = { + # Check the phase of a pod + "status.phase" = "Running" + + # Check a container's status + "status.containerStatuses[0].ready" = "true", + + # Check an ingress has an IP + "status.loadBalancer.ingress[0].ip" = "^(\\d+(\\.|$)){4}" + + # Check the replica count of a Deployment + "status.readyReplicas" = "2" + } + } + + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } } From 9ad0f6b421907bb96b1ee88dd5b083c753e6ffc4 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 15:59:51 -0500 Subject: [PATCH 27/41] rollout true --- node_class.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/node_class.tf b/node_class.tf index 351b601..5ce2404 100644 --- a/node_class.tf +++ b/node_class.tf @@ -65,6 +65,7 @@ resource "kubernetes_manifest" "karpenter_node_class" { } } wait { + rollout = true fields = { # Check the phase of a pod "status.phase" = "Running" From 49fd0f0af6218716a0d08e0216ee8322f24a6302 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 16:01:10 -0500 Subject: [PATCH 28/41] fields and rollout are mutually exclusive --- node_class.tf | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/node_class.tf b/node_class.tf index 5ce2404..d6882dd 100644 --- a/node_class.tf +++ b/node_class.tf @@ -66,19 +66,19 @@ resource "kubernetes_manifest" "karpenter_node_class" { } wait { rollout = true - fields = { - # Check the phase of a pod - "status.phase" = "Running" + # fields = { + # # Check the phase of a pod + # "status.phase" = "Running" - # Check a container's status - "status.containerStatuses[0].ready" = "true", + # # Check a container's status + # "status.containerStatuses[0].ready" = "true", - # Check an ingress has an IP - "status.loadBalancer.ingress[0].ip" = "^(\\d+(\\.|$)){4}" + # # Check an ingress has an IP + # "status.loadBalancer.ingress[0].ip" = "^(\\d+(\\.|$)){4}" - # Check the replica count of a Deployment - "status.readyReplicas" = "2" - } + # # Check the replica count of a Deployment + # "status.readyReplicas" = "2" + # } } timeouts { From 2b0977eabcfdbd63c66bc8b75d595bc4f43b3953 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 16:26:14 -0500 Subject: [PATCH 29/41] comment ftm --- README.md | 5 -- main.tf | 4 +- node_class.tf | 154 +++++++++++++++++++++--------------------------- node_pool.tf | 114 +++++++++++++++++------------------ requirements.tf | 8 +-- variables.tf | 6 -- 6 files changed, 128 insertions(+), 163 deletions(-) diff --git a/README.md b/README.md index a40e82b..1d044c3 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [terraform](#requirement\_terraform) | >= 0.13 | | [aws](#requirement\_aws) | >= 5.14.0 | | [helm](#requirement\_helm) | >= 2.11.0 | -| [kubernetes](#requirement\_kubernetes) | >= 2.23.0 | ## Providers @@ -28,7 +27,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|---------| | [aws](#provider\_aws) | >= 5.14.0 | | [helm](#provider\_helm) | >= 2.11.0 | -| [kubernetes](#provider\_kubernetes) | >= 2.23.0 | ## Modules @@ -43,8 +41,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|------| | [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [kubernetes_manifest.karpenter_node_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | -| [kubernetes_manifest.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | @@ -62,7 +58,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes | | [profile](#input\_profile) | AWS config profile | `string` | `""` | no | | [region](#input\_region) | AWS region | `string` | n/a | yes | -| [subnets\_name](#input\_subnets\_name) | Define the name of the subnets to be used by this cluster | `string` | `"*-container-*"` | no | | [tags](#input\_tags) | AWS Tags to apply to appropriate resources | `map(string)` | `{}` | no | ## Outputs diff --git a/main.tf b/main.tf index 58a38fd..9a30400 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,5 @@ locals { - namespace = "karpenter" - amd_ami_family = "Bottlerocket" - amd_ami_alias = "bottlerocket" + namespace = "karpenter" } # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter diff --git a/node_class.tf b/node_class.tf index d6882dd..440537d 100644 --- a/node_class.tf +++ b/node_class.tf @@ -1,89 +1,67 @@ -resource "kubernetes_manifest" "karpenter_node_class" { - depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile] +# resource "kubernetes_manifest" "karpenter_node_class" { +# depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile] - manifest = { - apiVersion = "karpenter.k8s.aws/v1" - kind = "EC2NodeClass" - metadata = { - name = "${var.cluster_name}-karpenter-node-class" - annotations = { - "kubernetes.io/description" = "EC2NodeClass for running ${local.amd_ami_family} nodes" - } - } - spec = { - amiFamily = local.amd_ami_family - amiSelectorTerms = [ - { - alias = "${local.amd_ami_alias}@latest" - } - ] - detailedMonitoring = true - instanceProfile = "${var.cluster_name}-karpenter-instance-profile" - instanceStorePolicy = "RAID0" - kubelet = { - evictionHard = { - "memory.available" = "5%" - "nodefs.available" = "10%" - "nodefs.inodesFree" = "10%" - } - evictionMaxPodGracePeriod = 60 - evictionSoft = { - "memory.available" = "500Mi" - "nodefs.available" = "15%" - "nodefs.inodesFree" = "15%" - } - evictionSoftGracePeriod = { - "memory.available" = "1m" - "nodefs.available" = "1m30s" - "nodefs.inodesFree" = "2m" - } - imageGCHighThresholdPercent = 85 - imageGCLowThresholdPercent = 80 - } - metadataOptions = { - httpEndpoint = "enabled" - httpProtocolIPv6 = "enabled" - httpPutResponseHopLimit = 2 - httpTokens = "required" - } - securityGroupSelectorTerms = [ - { - tags = { - Name = "${var.cluster_name}-node" - } - } - ] - subnetSelectorTerms = [ - { - tags = { - Name = var.subnets_name - "kubernetes.io/cluster/${var.cluster_name}" = "shared" - } - } - ] - tags = var.tags - } - } - wait { - rollout = true - # fields = { - # # Check the phase of a pod - # "status.phase" = "Running" - - # # Check a container's status - # "status.containerStatuses[0].ready" = "true", - - # # Check an ingress has an IP - # "status.loadBalancer.ingress[0].ip" = "^(\\d+(\\.|$)){4}" - - # # Check the replica count of a Deployment - # "status.readyReplicas" = "2" - # } - } - - timeouts { - create = "10m" - update = "10m" - delete = "30s" - } -} +# manifest = { +# apiVersion = "karpenter.k8s.aws/v1" +# kind = "EC2NodeClass" +# metadata = { +# name = "${var.cluster_name}-karpenter-node-class" +# annotations = { +# "kubernetes.io/description" = "EC2NodeClass for running ${local.amd_ami_family} nodes" +# } +# } +# spec = { +# amiFamily = local.amd_ami_family +# amiSelectorTerms = [ +# { +# alias = "${local.amd_ami_alias}@latest" +# } +# ] +# detailedMonitoring = true +# instanceProfile = aws_iam_instance_profile.karpenter_node_instance_profile.name +# instanceStorePolicy = "RAID0" +# kubelet = { +# evictionHard = { +# "memory.available" = "5%" +# "nodefs.available" = "10%" +# "nodefs.inodesFree" = "10%" +# } +# evictionMaxPodGracePeriod = 60 +# evictionSoft = { +# "memory.available" = "500Mi" +# "nodefs.available" = "15%" +# "nodefs.inodesFree" = "15%" +# } +# evictionSoftGracePeriod = { +# "memory.available" = "1m" +# "nodefs.available" = "1m30s" +# "nodefs.inodesFree" = "2m" +# } +# imageGCHighThresholdPercent = 85 +# imageGCLowThresholdPercent = 80 +# } +# metadataOptions = { +# httpEndpoint = "enabled" +# httpProtocolIPv6 = "enabled" +# httpPutResponseHopLimit = 2 +# httpTokens = "required" +# } +# securityGroupSelectorTerms = [ +# { +# tags = { +# Name = "${var.cluster_name}-node" +# } +# } +# ] +# subnetSelectorTerms = [ +# { +# tags = { +# Name = var.subnets_name +# "kubernetes.io/cluster/${var.cluster_name}" = "shared" +# } +# } +# ] +# tags = var.tags +# } +# } +# } diff --git a/node_pool.tf b/node_pool.tf index fb7f45f..5d04296 100644 --- a/node_pool.tf +++ b/node_pool.tf @@ -1,57 +1,57 @@ -resource "kubernetes_manifest" "karpenter_node_pool" { - depends_on = [kubernetes_manifest.karpenter_node_class] - manifest = { - apiVersion = "karpenter.sh/v1" - kind = "NodePool" - metadata = { - name = "${var.cluster_name}-node-pool-general" - annotations = { - "kubernetes.io/description" = "General purpose NodePool for generic workloads" - } - } - spec = { - template = { - spec = { - requirements = [ - { - key = "kubernetes.io/arch" - operator = "In" - values = ["amd64"] - }, - { - key = "kubernetes.io/os" - operator = "In" - values = ["linux"] - }, - { - key = "karpenter.k8s.aws/instance-hypervisor" - operator = "In" - values = ["nitro"] - }, - { - key = "karpenter.sh/capacity-type" - operator = "In" - values = ["spot", "on-demand"] - }, - { - key = "karpenter.k8s.aws/instance-generation" - operator = "Gt" - values = ["2"] - } - ] - nodeClassRef = { - group = "karpenter.k8s.aws" - kind = "EC2NodeClass" - name = "${var.cluster_name}-karpenter-node-class" - } - expireAfter = "72h" - terminationGracePeriod = "8h" - } - } - disruption = { - consolidationPolicy = "WhenEmptyOrUnderutilized" - consolidateAfter = "30s" - } - } - } -} +# resource "kubernetes_manifest" "karpenter_node_pool" { +# depends_on = [kubernetes_manifest.karpenter_node_class] +# manifest = { +# apiVersion = "karpenter.sh/v1" +# kind = "NodePool" +# metadata = { +# name = "${var.cluster_name}-node-pool-general" +# annotations = { +# "kubernetes.io/description" = "General purpose NodePool for generic workloads" +# } +# } +# spec = { +# template = { +# spec = { +# requirements = [ +# { +# key = "kubernetes.io/arch" +# operator = "In" +# values = ["amd64"] +# }, +# { +# key = "kubernetes.io/os" +# operator = "In" +# values = ["linux"] +# }, +# { +# key = "karpenter.k8s.aws/instance-hypervisor" +# operator = "In" +# values = ["nitro"] +# }, +# { +# key = "karpenter.sh/capacity-type" +# operator = "In" +# values = ["spot", "on-demand"] +# }, +# { +# key = "karpenter.k8s.aws/instance-generation" +# operator = "Gt" +# values = ["2"] +# } +# ] +# nodeClassRef = { +# group = "karpenter.k8s.aws" +# kind = "EC2NodeClass" +# name = "${var.cluster_name}-karpenter-node-class" +# } +# expireAfter = "72h" +# terminationGracePeriod = "8h" +# } +# } +# disruption = { +# consolidationPolicy = "WhenEmptyOrUnderutilized" +# consolidateAfter = "30s" +# } +# } +# } +# } diff --git a/requirements.tf b/requirements.tf index ae62e15..c190a2e 100644 --- a/requirements.tf +++ b/requirements.tf @@ -10,9 +10,9 @@ terraform { source = "hashicorp/helm" version = ">= 2.11.0" } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.23.0" - } + # kubernetes = { + # source = "hashicorp/kubernetes" + # version = ">= 2.23.0" + # } } } diff --git a/variables.tf b/variables.tf index bd99185..ad020f3 100644 --- a/variables.tf +++ b/variables.tf @@ -48,12 +48,6 @@ variable "karpenter_tag" { default = "1.1.1" } -variable "subnets_name" { - description = "Define the name of the subnets to be used by this cluster" - type = string - default = "*-container-*" -} - variable "tags" { description = "AWS Tags to apply to appropriate resources" type = map(string) From 8395be00610bda063adbcd7592f57c66910939a1 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 16:33:08 -0500 Subject: [PATCH 30/41] add ref to helm chart --- README.md | 3 ++ node_class.tf | 132 ++++++++++++++++++++++++------------------------ requirements.tf | 8 +-- 3 files changed, 73 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 1d044c3..fb20772 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [terraform](#requirement\_terraform) | >= 0.13 | | [aws](#requirement\_aws) | >= 5.14.0 | | [helm](#requirement\_helm) | >= 2.11.0 | +| [kubernetes](#requirement\_kubernetes) | >= 2.23.0 | ## Providers @@ -27,6 +28,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|---------| | [aws](#provider\_aws) | >= 5.14.0 | | [helm](#provider\_helm) | >= 2.11.0 | +| [kubernetes](#provider\_kubernetes) | >= 2.23.0 | ## Modules @@ -41,6 +43,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|------| | [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_manifest.karpenter_node_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | diff --git a/node_class.tf b/node_class.tf index 440537d..a95402a 100644 --- a/node_class.tf +++ b/node_class.tf @@ -1,67 +1,67 @@ -# resource "kubernetes_manifest" "karpenter_node_class" { -# depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile] +resource "kubernetes_manifest" "karpenter_node_class" { + depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile] -# manifest = { -# apiVersion = "karpenter.k8s.aws/v1" -# kind = "EC2NodeClass" -# metadata = { -# name = "${var.cluster_name}-karpenter-node-class" -# annotations = { -# "kubernetes.io/description" = "EC2NodeClass for running ${local.amd_ami_family} nodes" -# } -# } -# spec = { -# amiFamily = local.amd_ami_family -# amiSelectorTerms = [ -# { -# alias = "${local.amd_ami_alias}@latest" -# } -# ] -# detailedMonitoring = true -# instanceProfile = aws_iam_instance_profile.karpenter_node_instance_profile.name -# instanceStorePolicy = "RAID0" -# kubelet = { -# evictionHard = { -# "memory.available" = "5%" -# "nodefs.available" = "10%" -# "nodefs.inodesFree" = "10%" -# } -# evictionMaxPodGracePeriod = 60 -# evictionSoft = { -# "memory.available" = "500Mi" -# "nodefs.available" = "15%" -# "nodefs.inodesFree" = "15%" -# } -# evictionSoftGracePeriod = { -# "memory.available" = "1m" -# "nodefs.available" = "1m30s" -# "nodefs.inodesFree" = "2m" -# } -# imageGCHighThresholdPercent = 85 -# imageGCLowThresholdPercent = 80 -# } -# metadataOptions = { -# httpEndpoint = "enabled" -# httpProtocolIPv6 = "enabled" -# httpPutResponseHopLimit = 2 -# httpTokens = "required" -# } -# securityGroupSelectorTerms = [ -# { -# tags = { -# Name = "${var.cluster_name}-node" -# } -# } -# ] -# subnetSelectorTerms = [ -# { -# tags = { -# Name = var.subnets_name -# "kubernetes.io/cluster/${var.cluster_name}" = "shared" -# } -# } -# ] -# tags = var.tags -# } -# } -# } + manifest = { + apiVersion = "karpenter.k8s.aws/v1" + kind = "EC2NodeClass" + metadata = { + name = "${helm_release.karpenter.metadata.settings.clusterName}-karpenter-node-class" + annotations = { + "kubernetes.io/description" = "EC2NodeClass for running ${local.amd_ami_family} nodes" + } + } + spec = { + amiFamily = local.amd_ami_family + amiSelectorTerms = [ + { + alias = "${local.amd_ami_alias}@latest" + } + ] + detailedMonitoring = true + instanceProfile = aws_iam_instance_profile.karpenter_node_instance_profile.name + instanceStorePolicy = "RAID0" + kubelet = { + evictionHard = { + "memory.available" = "5%" + "nodefs.available" = "10%" + "nodefs.inodesFree" = "10%" + } + evictionMaxPodGracePeriod = 60 + evictionSoft = { + "memory.available" = "500Mi" + "nodefs.available" = "15%" + "nodefs.inodesFree" = "15%" + } + evictionSoftGracePeriod = { + "memory.available" = "1m" + "nodefs.available" = "1m30s" + "nodefs.inodesFree" = "2m" + } + imageGCHighThresholdPercent = 85 + imageGCLowThresholdPercent = 80 + } + metadataOptions = { + httpEndpoint = "enabled" + httpProtocolIPv6 = "enabled" + httpPutResponseHopLimit = 2 + httpTokens = "required" + } + securityGroupSelectorTerms = [ + { + tags = { + Name = "${helm_release.karpenter.metadata.settings.clusterName}-node" + } + } + ] + subnetSelectorTerms = [ + { + tags = { + Name = var.subnets_name + "kubernetes.io/cluster/${helm_release.karpenter.metadata.settings.clusterName}" = "shared" + } + } + ] + tags = var.tags + } + } +} diff --git a/requirements.tf b/requirements.tf index c190a2e..ae62e15 100644 --- a/requirements.tf +++ b/requirements.tf @@ -10,9 +10,9 @@ terraform { source = "hashicorp/helm" version = ">= 2.11.0" } - # kubernetes = { - # source = "hashicorp/kubernetes" - # version = ">= 2.23.0" - # } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.23.0" + } } } From 1c2717d2fc98b85d450dd5c6877dec7e41e0b40e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 16:35:10 -0500 Subject: [PATCH 31/41] add values --- node_class.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/node_class.tf b/node_class.tf index a95402a..02c3a2f 100644 --- a/node_class.tf +++ b/node_class.tf @@ -5,7 +5,7 @@ resource "kubernetes_manifest" "karpenter_node_class" { apiVersion = "karpenter.k8s.aws/v1" kind = "EC2NodeClass" metadata = { - name = "${helm_release.karpenter.metadata.settings.clusterName}-karpenter-node-class" + name = "${helm_release.karpenter.metadata.values.settings.clusterName}-karpenter-node-class" annotations = { "kubernetes.io/description" = "EC2NodeClass for running ${local.amd_ami_family} nodes" } @@ -49,15 +49,15 @@ resource "kubernetes_manifest" "karpenter_node_class" { securityGroupSelectorTerms = [ { tags = { - Name = "${helm_release.karpenter.metadata.settings.clusterName}-node" + Name = "${helm_release.karpenter.metadata.values.settings.clusterName}-node" } } ] subnetSelectorTerms = [ { tags = { - Name = var.subnets_name - "kubernetes.io/cluster/${helm_release.karpenter.metadata.settings.clusterName}" = "shared" + Name = var.subnets_name + "kubernetes.io/cluster/${helm_release.karpenter.metadata.values.settings.clusterName}" = "shared" } } ] From 6771246afa6c3e8c0c766718b467559caa824d20 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 16:37:22 -0500 Subject: [PATCH 32/41] add index to metadata --- node_class.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/node_class.tf b/node_class.tf index 02c3a2f..60c8c39 100644 --- a/node_class.tf +++ b/node_class.tf @@ -5,7 +5,7 @@ resource "kubernetes_manifest" "karpenter_node_class" { apiVersion = "karpenter.k8s.aws/v1" kind = "EC2NodeClass" metadata = { - name = "${helm_release.karpenter.metadata.values.settings.clusterName}-karpenter-node-class" + name = "${helm_release.karpenter.metadata[0].values.settings.clusterName}-karpenter-node-class" annotations = { "kubernetes.io/description" = "EC2NodeClass for running ${local.amd_ami_family} nodes" } @@ -49,15 +49,15 @@ resource "kubernetes_manifest" "karpenter_node_class" { securityGroupSelectorTerms = [ { tags = { - Name = "${helm_release.karpenter.metadata.values.settings.clusterName}-node" + Name = "${helm_release.karpenter.metadata[0].values.settings.clusterName}-node" } } ] subnetSelectorTerms = [ { tags = { - Name = var.subnets_name - "kubernetes.io/cluster/${helm_release.karpenter.metadata.values.settings.clusterName}" = "shared" + Name = var.subnets_name + "kubernetes.io/cluster/${helm_release.karpenter.metadata[0].values.settings.clusterName}" = "shared" } } ] From ba3304b80be0ee886ad93bb0da073e318ba0d159 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 16:50:34 -0500 Subject: [PATCH 33/41] use chart name --- node_class.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/node_class.tf b/node_class.tf index 60c8c39..e5b981e 100644 --- a/node_class.tf +++ b/node_class.tf @@ -2,10 +2,10 @@ resource "kubernetes_manifest" "karpenter_node_class" { depends_on = [aws_iam_instance_profile.karpenter_node_instance_profile] manifest = { - apiVersion = "karpenter.k8s.aws/v1" + apiVersion = "${helm_release.karpenter.name}.k8s.aws/v1" kind = "EC2NodeClass" metadata = { - name = "${helm_release.karpenter.metadata[0].values.settings.clusterName}-karpenter-node-class" + name = "${var.cluster_name}-${helm_release.karpenter.name}-node-class" annotations = { "kubernetes.io/description" = "EC2NodeClass for running ${local.amd_ami_family} nodes" } @@ -49,15 +49,15 @@ resource "kubernetes_manifest" "karpenter_node_class" { securityGroupSelectorTerms = [ { tags = { - Name = "${helm_release.karpenter.metadata[0].values.settings.clusterName}-node" + Name = "${var.cluster_name}-node" } } ] subnetSelectorTerms = [ { tags = { - Name = var.subnets_name - "kubernetes.io/cluster/${helm_release.karpenter.metadata[0].values.settings.clusterName}" = "shared" + Name = var.subnets_name + "kubernetes.io/cluster/${var.cluster_name}" = "shared" } } ] From d9dbf7907d684d17d19a86aee083ba7ae0e1bbd6 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 16:52:52 -0500 Subject: [PATCH 34/41] how about now --- README.md | 1 + variables.tf | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index fb20772..4b1cf27 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes | | [profile](#input\_profile) | AWS config profile | `string` | `""` | no | | [region](#input\_region) | AWS region | `string` | n/a | yes | +| [subnets\_name](#input\_subnets\_name) | Define the name of the subnets to be used by this cluster | `string` | `"*-container-*"` | no | | [tags](#input\_tags) | AWS Tags to apply to appropriate resources | `map(string)` | `{}` | no | ## Outputs diff --git a/variables.tf b/variables.tf index ad020f3..bd99185 100644 --- a/variables.tf +++ b/variables.tf @@ -48,6 +48,12 @@ variable "karpenter_tag" { default = "1.1.1" } +variable "subnets_name" { + description = "Define the name of the subnets to be used by this cluster" + type = string + default = "*-container-*" +} + variable "tags" { description = "AWS Tags to apply to appropriate resources" type = map(string) From 32f77ee577a9903b345a4ccc2eebb210d3f8b768 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 16:54:02 -0500 Subject: [PATCH 35/41] locals? --- main.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 9a30400..58a38fd 100644 --- a/main.tf +++ b/main.tf @@ -1,5 +1,7 @@ locals { - namespace = "karpenter" + namespace = "karpenter" + amd_ami_family = "Bottlerocket" + amd_ami_alias = "bottlerocket" } # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter From 96e18f282cb9025756f3d3e9e561cf9cd20527f2 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 17:00:26 -0500 Subject: [PATCH 36/41] add pool back --- README.md | 1 + node_pool.tf | 114 +++++++++++++++++++++++++-------------------------- 2 files changed, 58 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 4b1cf27..a40e82b 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubernetes_manifest.karpenter_node_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | +| [kubernetes_manifest.karpenter_node_pool](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | diff --git a/node_pool.tf b/node_pool.tf index 5d04296..ef3a9d9 100644 --- a/node_pool.tf +++ b/node_pool.tf @@ -1,57 +1,57 @@ -# resource "kubernetes_manifest" "karpenter_node_pool" { -# depends_on = [kubernetes_manifest.karpenter_node_class] -# manifest = { -# apiVersion = "karpenter.sh/v1" -# kind = "NodePool" -# metadata = { -# name = "${var.cluster_name}-node-pool-general" -# annotations = { -# "kubernetes.io/description" = "General purpose NodePool for generic workloads" -# } -# } -# spec = { -# template = { -# spec = { -# requirements = [ -# { -# key = "kubernetes.io/arch" -# operator = "In" -# values = ["amd64"] -# }, -# { -# key = "kubernetes.io/os" -# operator = "In" -# values = ["linux"] -# }, -# { -# key = "karpenter.k8s.aws/instance-hypervisor" -# operator = "In" -# values = ["nitro"] -# }, -# { -# key = "karpenter.sh/capacity-type" -# operator = "In" -# values = ["spot", "on-demand"] -# }, -# { -# key = "karpenter.k8s.aws/instance-generation" -# operator = "Gt" -# values = ["2"] -# } -# ] -# nodeClassRef = { -# group = "karpenter.k8s.aws" -# kind = "EC2NodeClass" -# name = "${var.cluster_name}-karpenter-node-class" -# } -# expireAfter = "72h" -# terminationGracePeriod = "8h" -# } -# } -# disruption = { -# consolidationPolicy = "WhenEmptyOrUnderutilized" -# consolidateAfter = "30s" -# } -# } -# } -# } +resource "kubernetes_manifest" "karpenter_node_pool" { + depends_on = [kubernetes_manifest.karpenter_node_class] + manifest = { + apiVersion = "${helm_release.karpenter.name}.sh/v1" + kind = "NodePool" + metadata = { + name = "${var.cluster_name}-${helm_release.karpenter.name}-node-pool" + annotations = { + "kubernetes.io/description" = "General purpose NodePool for generic workloads" + } + } + spec = { + template = { + spec = { + requirements = [ + { + key = "kubernetes.io/arch" + operator = "In" + values = ["amd64"] + }, + { + key = "kubernetes.io/os" + operator = "In" + values = ["linux"] + }, + { + key = "${helm_release.karpenter.name}.k8s.aws/instance-hypervisor" + operator = "In" + values = ["nitro"] + }, + { + key = "${helm_release.karpenter.name}.sh/capacity-type" + operator = "In" + values = ["spot", "on-demand"] + }, + { + key = "${helm_release.karpenter.name}.k8s.aws/instance-generation" + operator = "Gt" + values = ["2"] + } + ] + nodeClassRef = { + group = "${helm_release.karpenter.name}.k8s.aws" + kind = "EC2NodeClass" + name = "${var.cluster_name}-${helm_release.karpenter.name}-node-class" + } + expireAfter = "72h" + terminationGracePeriod = "8h" + } + } + disruption = { + consolidationPolicy = "WhenEmptyOrUnderutilized" + consolidateAfter = "30s" + } + } + } +} From 13d863771149a7b638db4a8e5bbcc64da1c3473d Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 19:20:44 -0500 Subject: [PATCH 37/41] readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a40e82b..fd79403 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,9 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.14.0 | -| [helm](#provider\_helm) | >= 2.11.0 | -| [kubernetes](#provider\_kubernetes) | >= 2.23.0 | +| [aws](#provider\_aws) | 5.84.0 | +| [helm](#provider\_helm) | 2.17.0 | +| [kubernetes](#provider\_kubernetes) | 2.35.1 | ## Modules From b6206c4d85e687ac17c008cc6ede500ccc109c42 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 19:22:39 -0500 Subject: [PATCH 38/41] cruft --- .pre-commit-config.yaml | 2 +- ec2_node_class.yaml.tpl | 63 --------------------------------------- ec2_node_pool.yaml.tpl | 66 ----------------------------------------- 3 files changed, 1 insertion(+), 130 deletions(-) delete mode 100644 ec2_node_class.yaml.tpl delete mode 100644 ec2_node_pool.yaml.tpl diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41952f0..f451b69 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] exclude: CHANGELOG.md - # - id: check-yaml + - id: check-yaml - id: check-merge-conflict - id: check-executables-have-shebangs diff --git a/ec2_node_class.yaml.tpl b/ec2_node_class.yaml.tpl deleted file mode 100644 index 4f77f29..0000000 --- a/ec2_node_class.yaml.tpl +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: karpenter.k8s.aws/v1 -kind: EC2NodeClass -metadata: - name: "{{ .Values.cluster_name }}-karpenter-node-class" - annotations: - kubernetes.io/description: "EC2NodeClass for running {{ .Values.amd_ami_family }} nodes" -spec: - kubelet: - evictionHard: - memory.available: 5% - nodefs.available: 10% - nodefs.inodesFree: 10% - evictionSoft: - memory.available: 500Mi - nodefs.available: 15% - nodefs.inodesFree: 15% - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - evictionMaxPodGracePeriod: 60 - imageGCHighThresholdPercent: 85 - imageGCLowThresholdPercent: 80 - - # Required, resolves a default ami and userdata - amiFamily: {{ .Values.amd_ami_family }} - amiSelectorTerms: - - alias: "{{ .Values.amd_ami_alias }}@latest" # Bottlerocket - # Required, discovers subnets to attach to instances - # Each term in the array of subnetSelectorTerms is ORed together - # Within a single term, all conditions are ANDed - subnetSelectorTerms: - - tags: - Name: {{ .Values.subnets_name }} - "kubernetes.io/cluster/{{ .Values.cluster_name }}": "shared" - - # Required, discovers security groups to attach to instances - # Each term in the array of securityGroupSelectorTerms is ORed together - # Within a single term, all conditions are ANDed - securityGroupSelectorTerms: - - tags: - Name: "{{ .Values.cluster_name }}-node" - - # Optional, IAM instance profile to use for the node identity. - # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes - instanceProfile: "{{ .Values.cluster_name }}-karpenter-instance-profile" - - # Optional, use instance-store volumes for node ephemeral-storage - instanceStorePolicy: RAID0 - - # Optional, configures IMDS for the instance - metadataOptions: - httpEndpoint: enabled - httpProtocolIPv6: enabled - httpPutResponseHopLimit: 2 - httpTokens: required - - # Optional, configures detailed monitoring for the instance - detailedMonitoring: true - - # Add finops tags to karpenter provisioned nodes - tags: {{ .Values.tags }} diff --git a/ec2_node_pool.yaml.tpl b/ec2_node_pool.yaml.tpl deleted file mode 100644 index e813d7c..0000000 --- a/ec2_node_pool.yaml.tpl +++ /dev/null @@ -1,66 +0,0 @@ ---- -apiVersion: karpenter.sh/v1 -kind: NodePool -metadata: - name: "{{ .Values.cluster_name }}-node-pool-general" - annotations: - kubernetes.io/description: 'General purpose NodePool for generic workloads' -spec: - template: - spec: - requirements: - - key: kubernetes.io/arch - operator: In - values: ["amd64"] - - key: kubernetes.io/os - operator: In - values: ["linux"] - - key: karpenter.k8s.aws/instance-hypervisor - operator: In - values: ["nitro"] - - key: karpenter.sh/capacity-type - operator: In - values: ["spot", "on-demand"] - - key: karpenter.k8s.aws/instance-generation - operator: Gt - values: ["2"] - nodeClassRef: - group: karpenter.k8s.aws - kind: EC2NodeClass - name: {{ .Values.cluster_name }}-karpenter-node-class - # The disruption section defines policies for consolidating and expiring nodes. - # Time format: , where unit can be 's' (seconds), 'm' (minutes), 'h' (hours), 'd' (days) - disruption: - # consolidationPolicy: Defines the policy for consolidating EC2 instances in the node pool. - # This policy helps in optimizing resource usage and cost by consolidating workloads onto fewer instances. - # Possible values include 'None', 'OptimizeUtilization', and 'OptimizeCost'. - consolidationPolicy: WhenEmptyOrUnderutilized - # consolidateAfter specifies the duration after which nodes should be consolidated. - consolidateAfter: 5m - # expireAfter specifies the duration after which nodes should be expired. - expireAfter: 72h - # limits: - # cpu: 1000 - # memory: 1000Gi - # nvidia.com/gpu: 2 - # topologySpreadConstraints: - # - maxSkew: 99 - # topologyKey: "topology.kubernetes.io/zone" - # whenUnsatisfiable: ScheduleAnyway - # - maxSkew: 99 - # topologyKey: "kubernetes.io/hostname" - # whenUnsatisfiable: ScheduleAnyway - # - maxSkew: 99 - # topologyKey: "karpenter.sh/capacity-type" - # whenUnsatisfiable: ScheduleAnyway - # budgets: - # - nodes: "50%" - # reasons: - # - "Empty" - # - "Drifted" - # - nodes: "5" - # - nodes: "0" - # schedule: "@daily" - # duration: 10m - # reasons: - # - "Underutilized" From 00d4e41ac247119f0c6a536822221682fa5a75c7 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 19:26:22 -0500 Subject: [PATCH 39/41] fix pre-commit --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f451b69..85adacb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,6 +25,7 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] exclude: CHANGELOG.md + - id: check-yaml - id: check-merge-conflict - id: check-executables-have-shebangs From 7b4303276092763e31186d1b3ffd2bc309029425 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 19:31:14 -0500 Subject: [PATCH 40/41] remove commented code --- main.tf | 9 --------- 1 file changed, 9 deletions(-) diff --git a/main.tf b/main.tf index 58a38fd..4393215 100644 --- a/main.tf +++ b/main.tf @@ -31,15 +31,6 @@ data "aws_ecr_image" "karpenter_image" { image_tag = module.images.images[local.karpenter_key].tag } -# resource "helm_release" "karpenter_crds" { -# name = "karpenter-crd" -# chart = "karpentercrd" -# version = var.karpenter_helm_chart -# namespace = local.namespace -# description = "installs the crds for karpenter so our manifest resources can apply" -# repository = "oci://public.ecr.aws/karpenter/karpenter-crd" -# } - resource "helm_release" "karpenter" { depends_on = [ data.aws_ecr_image.karpenter_image From 699a05df50b85e4084d7568fac87b5cded44e0dc Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 24 Jan 2025 19:38:52 -0500 Subject: [PATCH 41/41] add tags --- node_pool.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/node_pool.tf b/node_pool.tf index ef3a9d9..a270888 100644 --- a/node_pool.tf +++ b/node_pool.tf @@ -52,6 +52,7 @@ resource "kubernetes_manifest" "karpenter_node_pool" { consolidationPolicy = "WhenEmptyOrUnderutilized" consolidateAfter = "30s" } + tags = var.tags } } }