From 3cb4bb502ced7eeac28744e4425b7e8cb9f5e59a Mon Sep 17 00:00:00 2001 From: Matthew Creal Morgan Date: Tue, 22 Apr 2025 11:34:40 -0700 Subject: [PATCH] [Major] Code Freeze PR (#21) * add startup taint * add namespace and startup taint to template * templatlize * fix vals * fewer things * add a few missing values * create ns * update values * add nodeSelector and tolerations * remove digest * node group * fmt * syntax and startupTaints * fix syntax * update values * remove ng selector * lower cpu requests * use digest again * simplify repository * fix repository * escape the . * fix subnets_name * subnet selector * use the outputs from karpenter module * create instance profile true * cleanup * without cluster tag * pass in vpc * enable debug and disable serviceMonitor * fix merge issues * update resources and set eksControlPlane true * try enable_pod_identity * update updated upstream * more options for namings * shorter * validate role name length * validate event rules name length * shorter * shorter still * fix prefix * name the queue * 66 - 3 = 63, limit 64 * longest event name is InstanceStateChange * another * maybe * name * name_prefix * or_not * add module release process * update module source * update release actions * pull from ent-ecr * Clean up unused variables and values (#20) * Add module release process (#19) * add startup taint * add namespace and startup taint to template * templatlize * fix vals * fewer things * add a few missing values * create ns * update values * add nodeSelector and tolerations * remove digest * node group * fmt * syntax and startupTaints * fix syntax * update values * remove ng selector * lower cpu requests * use digest again * simplify repository * fix repository * escape the . * fix subnets_name * subnet selector * use the outputs from karpenter module * create instance profile true * cleanup * without cluster tag * pass in vpc * enable debug and disable serviceMonitor * fix merge issues * update resources and set eksControlPlane true * try enable_pod_identity * update updated upstream * more options for namings * shorter * validate role name length * validate event rules name length * shorter * shorter still * fix prefix * name the queue * 66 - 3 = 63, limit 64 * longest event name is InstanceStateChange * another * maybe * name * name_prefix * or_not * add module release process * update module source * update release actions * Comment out unused values * Comment out vpc_id variable definition * Disable IRSA --------- Co-authored-by: Matthew Creal Morgan * use eecr images * pull across accounts from central ecr * pull across accounts from central ecr * add irsa back * remove dupes from bad merge * dynamic version * remove eecr data item as it is in the provider * update copy images for eecr pulling * update var and ent_ecr_source * ensure workflows are current * ensure workflows are current * ensure committed --------- Co-authored-by: Delong Yang --- .github/workflows/terraform-release.yaml | 6 +- .github/workflows/terraform-validate.yaml | 2 +- README.md | 13 +++- copy_images.tf | 30 +++++++-- karpenter-resources/values.yaml.tpl | 1 - karpenter/values.yaml.tpl | 6 -- main.tf | 5 +- requirements.tf | 4 ++ variables.tf | 77 ++++++++++++++--------- version.tf | 14 ++++- 10 files changed, 106 insertions(+), 52 deletions(-) diff --git a/.github/workflows/terraform-release.yaml b/.github/workflows/terraform-release.yaml index 04b96db..3f67574 100644 --- a/.github/workflows/terraform-release.yaml +++ b/.github/workflows/terraform-release.yaml @@ -1,4 +1,4 @@ -name: Terraform CI/CD +name: Terraform Module Release on: workflow_dispatch: pull_request: @@ -6,8 +6,8 @@ on: branches: - main jobs: - terraform-ci-cd: - runs-on: 229685449397 + terraform-release: + runs-on: "229685449397" permissions: contents: write diff --git a/.github/workflows/terraform-validate.yaml b/.github/workflows/terraform-validate.yaml index 72829d8..ac349eb 100644 --- a/.github/workflows/terraform-validate.yaml +++ b/.github/workflows/terraform-validate.yaml @@ -16,7 +16,7 @@ jobs: - name: Setup Terraform uses: CSVD/gh-actions-setup-terraform@v2 with: - terraform_version: '1.7.3' + terraform_version: '1.10.5' - name: Validate Terraform Configuration id: validate diff --git a/README.md b/README.md index 1def36b..aaae1a6 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,16 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [terraform](#requirement\_terraform) | >= 0.13 | | [aws](#requirement\_aws) | >= 5.14.0 | | [helm](#requirement\_helm) | >= 2.11.0 | +| [null](#requirement\_null) | >= 3.2.1 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | 5.89.0 | +| [aws](#provider\_aws) | 5.94.1 | +| [aws.eecr](#provider\_aws.eecr) | 5.94.1 | | [helm](#provider\_helm) | 2.17.0 | +| [null](#provider\_null) | 3.2.3 | ## Modules @@ -41,27 +44,31 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the |------|------| | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.nodepool_resources](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [null_resource.git_version](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_ecr_authorization_token.ecr_token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_authorization_token) | data source | +| [aws_ecr_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_authorization_token) | data source | | [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [account\_id](#input\_account\_id) | aws account number | `string` | `""` | no | | [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint for your Kubernetes API server | `string` | n/a | yes | | [cluster\_name](#input\_cluster\_name) | EKS cluster name name component used through out the EKS cluster describing its purpose (ex: dice-dev) | `string` | n/a | yes | +| [eecr\_info](#input\_eecr\_info) | Enterprise ECR source information | 
object({
    account_id = string
    alias      = string
    profile    = string
    region     = string
  })
| 
{
  "account_id": "269222635945",
  "alias": "lab-gov-shared-nonprod",
  "profile": "269222635945-lab-gov-shared-nonprod",
  "region": "us-gov-east-1"
}
| no | | [karpenter\_helm\_chart](#input\_karpenter\_helm\_chart) | Which helm chart of karpenter | `string` | `"1.1.1"` | no | | [karpenter\_helm\_repo](#input\_karpenter\_helm\_repo) | Helm repo for official karpenter chart | `string` | `"oci://public.ecr.aws/karpenter"` | no | | [karpenter\_node\_group\_name](#input\_karpenter\_node\_group\_name) | The cluster node group that will host karpenter, should not be a karpenter managed node group | `string` | n/a | yes | | [karpenter\_tag](#input\_karpenter\_tag) | Which tag of karpenter image | `string` | `"1.1.1"` | no | | [namespace](#input\_namespace) | Namespace for Karpenter | `string` | `"namespace"` | no | -| [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes | +| [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | `""` | no | | [profile](#input\_profile) | AWS config profile | `string` | `""` | no | | [region](#input\_region) | AWS region | `string` | n/a | yes | | [subnets\_name](#input\_subnets\_name) | Define the name of the subnets to be used by this cluster | `string` | `"*-container-*"` | no | | [tags](#input\_tags) | AWS Tags to apply to appropriate resources | `map(string)` | `{}` | no | -| [vpc\_id](#input\_vpc\_id) | The ID of the VPC where the resources will be deployed | `string` | n/a | yes | ## Outputs diff --git a/copy_images.tf b/copy_images.tf index 7e90fc5..2d7e7c1 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -1,5 +1,6 @@ locals { - karpenter_key = format("%v#%v", "karpenter/controller", var.karpenter_tag) + karpenter_key = format("%v#%v", "karpenter/controller", var.karpenter_tag) + ent_ecr_source = format("%v.%v.%v.%v", var.eecr_info.account_id, "dkr.ecr", var.region, "amazonaws.com/ent-images") image_config = [ { @@ -7,15 +8,13 @@ locals { dest_path = null name = "karpenter/controller" source_image = "karpenter/controller" - source_registry = "public.ecr.aws" + source_registry = format("%v/%v", local.ent_ecr_source, "public-ecr") source_tag = var.karpenter_tag tag = var.karpenter_tag }, ] } -# 224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter:0.37.0 -# map[repository:224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter tag:0.37.0] module "images" { source = "git::https://github.e.it.census.gov/terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" @@ -27,5 +26,26 @@ module "images" { enable_lifecycle_policy = true lifecycle_policy_all = true force_delete = true - lifecycle_policy_keep_count = 3 + lifecycle_policy_keep_count = 5 + + source_username = data.aws_ecr_authorization_token.ecr_token.user_name + source_password = data.aws_ecr_authorization_token.ecr_token.password + + destination_username = data.aws_ecr_authorization_token.token.user_name + destination_password = data.aws_ecr_authorization_token.token.password +} + +data "aws_ecr_authorization_token" "token" { + registry_id = var.account_id +} + +data "aws_ecr_authorization_token" "ecr_token" { + provider = aws.eecr + registry_id = var.eecr_info.account_id +} + +provider "aws" { + alias = "eecr" + profile = var.eecr_info.profile + region = var.eecr_info.region } diff --git a/karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml.tpl index 747ba17..c4004b1 100644 --- a/karpenter-resources/values.yaml.tpl +++ b/karpenter-resources/values.yaml.tpl @@ -8,7 +8,6 @@ ec2NodeClass: amiFamily: ${amd_ami_family} amiSelectorTerms: - alias: ${amd_ami_alias} - vpcId: ${vpc_id} subnetSelectorTerms: - tags: Name: "${subnets_name}" diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 3228846..7585898 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -34,12 +34,6 @@ controller: env: - name: AWS_REGION value: ${region} - nodeSelector: - kubernetes.io/os: linux - tolerations: - - key: "node-role.kubernetes.io/control-plane" - operator: "Exists" - effect: "NoSchedule" nodeSelector: kubernetes.io/os: linux diff --git a/main.tf b/main.tf index 90b9b03..c4f1a05 100644 --- a/main.tf +++ b/main.tf @@ -29,10 +29,10 @@ module "karpenter_resources" { create_node_iam_role = true create_pod_identity_association = true enable_irsa = true - enable_pod_identity = true - enable_v1_permissions = true irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] irsa_oidc_provider_arn = var.oidc_provider_arn + enable_pod_identity = true + enable_v1_permissions = true node_iam_role_name = local.node_iam_role_name queue_name = local.queue_name rule_name_prefix = local.rule_name_prefix @@ -104,7 +104,6 @@ resource "helm_release" "nodepool_resources" { karpenter_node_profile = module.karpenter_resources.instance_profile_name subnets_name = var.subnets_name tags = var.tags - vpc_id = var.vpc_id }) ] } diff --git a/requirements.tf b/requirements.tf index 2e678e4..5cd741d 100644 --- a/requirements.tf +++ b/requirements.tf @@ -10,5 +10,9 @@ terraform { source = "hashicorp/helm" version = ">= 2.11.0" } + null = { + source = "hashicorp/null" + version = ">= 3.2.1" + } } } diff --git a/variables.tf b/variables.tf index 72bd4d8..1cd76ec 100644 --- a/variables.tf +++ b/variables.tf @@ -1,12 +1,12 @@ -variable "region" { - description = "AWS region" +variable "account_id" { + description = "aws account number" type = string + default = "" } -variable "profile" { - description = "AWS config profile" +variable "cluster_endpoint" { + description = "Endpoint for your Kubernetes API server" type = string - default = "" } variable "cluster_name" { @@ -14,19 +14,26 @@ variable "cluster_name" { type = string } -variable "cluster_endpoint" { - description = "Endpoint for your Kubernetes API server" - type = string +variable "eecr_info" { + description = "Enterprise ECR source information" + type = object({ + account_id = string + alias = string + profile = string + region = string + }) + default = { + account_id = "269222635945" + alias = "lab-gov-shared-nonprod" + profile = "269222635945-lab-gov-shared-nonprod" + region = "us-gov-east-1" + } } -variable "oidc_provider_arn" { - description = "The ARN of the OIDC Provider if `enable_irsa = true`" - type = string -} - -variable "karpenter_node_group_name" { - description = "The cluster node group that will host karpenter, should not be a karpenter managed node group" +variable "karpenter_helm_chart" { + description = "Which helm chart of karpenter" type = string + default = "1.1.1" } variable "karpenter_helm_repo" { @@ -36,10 +43,9 @@ variable "karpenter_helm_repo" { # default = "https://charts.karpenter.sh/" public.ecr.aws/karpenter/karpenter:0.37.0 } -variable "karpenter_helm_chart" { - description = "Which helm chart of karpenter" +variable "karpenter_node_group_name" { + description = "The cluster node group that will host karpenter, should not be a karpenter managed node group" type = string - default = "1.1.1" } variable "karpenter_tag" { @@ -48,6 +54,30 @@ variable "karpenter_tag" { default = "1.1.1" } +variable "namespace" { + description = "Namespace for Karpenter" + type = string + default = "namespace" +} + +# tflint-ignore: terraform_unused_declarations +variable "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + type = string + default = "" +} + +variable "profile" { + description = "AWS config profile" + type = string + default = "" +} + +variable "region" { + description = "AWS region" + type = string +} + variable "subnets_name" { description = "Define the name of the subnets to be used by this cluster" type = string @@ -59,14 +89,3 @@ variable "tags" { type = map(string) default = {} } - -variable "namespace" { - description = "Namespace for Karpenter" - type = string - default = "namespace" -} - -variable "vpc_id" { - description = "The ID of the VPC where the resources will be deployed" - type = string -} diff --git a/version.tf b/version.tf index 5e91d6d..df199ec 100644 --- a/version.tf +++ b/version.tf @@ -1,4 +1,16 @@ +resource "null_resource" "git_version" { + triggers = { + # Force this to run on every apply to get the latest tag value + always_run = timestamp() + } + + provisioner "local-exec" { + command = "git describe --tags --abbrev=0 2>/dev/null || echo 'unknown' > ${path.module}/.git_tag" + on_failure = continue + } +} + locals { module_name = "tfmod-karpenter" - module_version = "0.1.1" + module_version = fileexists("${path.module}/.git_tag") ? trimspace(file("${path.module}/.git_tag")) : "latest" }