From 2b84ab913012673d59eb0a8ce1c40a62d3f8f990 Mon Sep 17 00:00:00 2001 From: Matthew Creal Morgan Date: Thu, 3 Apr 2025 10:31:42 -0700 Subject: [PATCH] Add module release process (#19) * add startup taint * add namespace and startup taint to template * templatlize * fix vals * fewer things * add a few missing values * create ns * update values * add nodeSelector and tolerations * remove digest * node group * fmt * syntax and startupTaints * fix syntax * update values * remove ng selector * lower cpu requests * use digest again * simplify repository * fix repository * escape the . * fix subnets_name * subnet selector * use the outputs from karpenter module * create instance profile true * cleanup * without cluster tag * pass in vpc * enable debug and disable serviceMonitor * fix merge issues * update resources and set eksControlPlane true * try enable_pod_identity * update updated upstream * more options for namings * shorter * validate role name length * validate event rules name length * shorter * shorter still * fix prefix * name the queue * 66 - 3 = 63, limit 64 * longest event name is InstanceStateChange * another * maybe * name * name_prefix * or_not * add module release process * update module source * update release actions --- .github/workflows/terraform-release.yaml | 40 +++++++++ .github/workflows/terraform-validate.yaml | 42 +++++++++ .github/workflows/terragrunt-cicd.yml | 101 ---------------------- README.md | 4 +- copy_images.tf | 2 +- karpenter/values.yaml.tpl | 26 ++++-- main.tf | 32 +++++-- prefixes.tf | 37 ++++++++ 8 files changed, 168 insertions(+), 116 deletions(-) create mode 100644 .github/workflows/terraform-release.yaml create mode 100644 .github/workflows/terraform-validate.yaml delete mode 100644 .github/workflows/terragrunt-cicd.yml create mode 100644 prefixes.tf diff --git a/.github/workflows/terraform-release.yaml b/.github/workflows/terraform-release.yaml new file mode 100644 index 0000000..04b96db --- /dev/null +++ b/.github/workflows/terraform-release.yaml @@ -0,0 +1,40 @@ +name: Terraform CI/CD +on: + workflow_dispatch: + pull_request: + types: [closed] + branches: + - main +jobs: + terraform-ci-cd: + runs-on: 229685449397 + permissions: + contents: write + + steps: + - name: Checkout code + uses: CSVD/gh-actions-checkout@v4 + + - name: Setup GITHUB Credentials + id: github_credentials + uses: CSVD/gh-auth@main + with: + github_app_pem_file: ${{ secrets.GH_APP_PEM_FILE }} + github_app_installation_id: ${{ vars.GH_APP_INSTALLATION_ID }} + github_app_id: ${{ vars.GH_APP_ID }} + + - name: Setup GitHub CLI + run: | + # Force manual authentication since setup-git might not work with GitHub Enterprise + echo "${{ steps.github_credentials.outputs.github_token }}" > /tmp/token.txt + gh auth login --with-token --hostname "github.e.it.census.gov" < /tmp/token.txt + rm /tmp/token.txt + + # Test GitHub CLI auth status + gh auth status || echo "GitHub CLI authentication failed" + + - name: Run Release Action + uses: CSVD/releaser@main + with: + github-token: ${{ steps.github_credentials.outputs.github_token }} + working-directory: '.' diff --git a/.github/workflows/terraform-validate.yaml b/.github/workflows/terraform-validate.yaml new file mode 100644 index 0000000..72829d8 --- /dev/null +++ b/.github/workflows/terraform-validate.yaml @@ -0,0 +1,42 @@ +name: Terraform Validate +on: + pull_request: + workflow_dispatch: + +jobs: + + terraform-validate: + runs-on: "229685449397" + permissions: + contents: write + steps: + - name: Checkout code + uses: CSVD/gh-actions-checkout@v4 + + - name: Setup Terraform + uses: CSVD/gh-actions-setup-terraform@v2 + with: + terraform_version: '1.7.3' + + - name: Validate Terraform Configuration + id: validate + uses: CSVD/terraform-validate@main + + - name: Check Validation/Test Results + if: always() + run: | + # Set default values if outputs are empty + IS_VALID="${{ steps.validate.outputs.is_valid }}" + TESTS_PASSED="${{ steps.validate.outputs.tests_passed }}" + + # If outputs are empty, set them to false + [ -z "$IS_VALID" ] && IS_VALID="false" + [ -z "$TESTS_PASSED" ] && TESTS_PASSED="false" + + if [[ "$IS_VALID" != "true" || "$TESTS_PASSED" != "true" ]]; then + echo "Validation or test errors found:" + echo "${{ steps.validate.outputs.stderr }}" + exit 1 + else + echo "All validations and tests passed successfully!" + fi diff --git a/.github/workflows/terragrunt-cicd.yml b/.github/workflows/terragrunt-cicd.yml deleted file mode 100644 index a78523e..0000000 --- a/.github/workflows/terragrunt-cicd.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: 'Terraform Module CI' - -on: - push: - branches: - - main - paths: - - '**/*.hcl' - - '**/*.tf' - pull_request: - branches: - - main - paths: - - '**/*.hcl' - - '**/*.tf' - -permissions: - contents: read - pull-requests: write - -jobs: - validate: - name: 'Validate Module' - runs-on: self-hosted - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 1.5.0 - - - name: Terraform Init - run: | - terraform init -backend=false - - - name: Terraform Format - run: | - terraform fmt -check - - - name: Terraform Validate - run: | - terraform validate - - - name: Run tflint - uses: terraform-linters/setup-tflint@v3 - if: github.event_name == 'pull_request' - - - name: Lint Terraform - if: github.event_name == 'pull_request' - run: | - tflint --format compact - - release: - name: 'Create Release' - needs: validate - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - runs-on: self-hosted - permissions: - contents: write - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install Commitizen - run: | - pip install commitizen - - - name: Configure Git - run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - - - name: Bump Version and Generate Changelog - id: cz - run: | - cz bump --yes - echo "new_version=$(cz version --project)" >> $GITHUB_OUTPUT - echo "changelog=$(cz changelog --dry-run)" >> $GITHUB_OUTPUT - - - name: Create Release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: v${{ steps.cz.outputs.new_version }} - release_name: Release v${{ steps.cz.outputs.new_version }} - draft: false - prerelease: false - body: ${{ steps.cz.outputs.changelog }} diff --git a/README.md b/README.md index 31d8199..1def36b 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,8 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Source | Version | |------|--------|---------| -| [images](#module\_images) | git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/ | tf-upgrade | -| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | v20.33.1 | +| [images](#module\_images) | git::https://github.e.it.census.gov/terraform-modules/aws-ecr-copy-images.git/ | tf-upgrade | +| [karpenter\_resources](#module\_karpenter\_resources) | git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks.git//modules/karpenter | v20.35.0 | ## Resources diff --git a/copy_images.tf b/copy_images.tf index 8febc1b..7e90fc5 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -17,7 +17,7 @@ locals { # 224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter:0.37.0 # map[repository:224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter tag:0.37.0] module "images" { - source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" + source = "git::https://github.e.it.census.gov/terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" profile = var.profile application_name = var.cluster_name diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl index 1c0b489..3228846 100644 --- a/karpenter/values.yaml.tpl +++ b/karpenter/values.yaml.tpl @@ -3,6 +3,7 @@ settings: clusterEndpoint: ${cluster_endpoint} interruptionQueue: ${queue_name} isolatedVPC: true + eksControlPlane: true featureGates: nodeRepair: true reservedCapacity: true @@ -19,17 +20,17 @@ serviceMonitor: logLevel: debug controller: + image: + repository: ${repository} + tag: ${tag} + digest: ${digest} resources: requests: - cpu: 1m - memory: 1Mi + cpu: 100m + memory: 1Gi limits: cpu: 500m memory: 1Gi - image: - repository: ${repository} - tag: ${tag} - digest: ${digest} env: - name: AWS_REGION value: ${region} @@ -40,6 +41,19 @@ controller: operator: "Exists" effect: "NoSchedule" +nodeSelector: + kubernetes.io/os: linux + +tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/main.tf b/main.tf index 638cfdc..90b9b03 100644 --- a/main.tf +++ b/main.tf @@ -1,21 +1,41 @@ locals { amd_ami_family = "Bottlerocket" amd_ami_alias = "bottlerocket@latest" + + # Calculate the role name with prefix + role_name_raw = format("%v%v-%v", local.prefixes["eks-role"], var.cluster_name, "karpenter-") + + # Ensure role name stays within AWS limits (38 chars) when used as name_prefix + max_role_name_length = 37 + node_iam_role_name = length(local.role_name_raw) > local.max_role_name_length ? substr(local.role_name_raw, 0, local.max_role_name_length) : local.role_name_raw + + # EventBridge rule name validation + # Leave enough space for suffixes like "SpotInterrupt" and unique IDs generated by AWS + # EventBridge has a max length of 64 chars + rule_name_raw = format("%v%v", local.prefixes["event-rule"], var.cluster_name) + max_rule_name_length = 8 + rule_name_prefix = length(local.rule_name_raw) > local.max_rule_name_length ? substr(local.rule_name_raw, 0, local.max_rule_name_length) : local.rule_name_raw + queue_name = format("%v%v", local.prefixes["eks-queue"], var.cluster_name) } # Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter # Karpenter IRSA roles and policies module "karpenter_resources" { - source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.33.1" + source = "git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.35.0" cluster_name = var.cluster_name - enable_irsa = true - irsa_oidc_provider_arn = var.oidc_provider_arn - irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] + create_access_entry = true + create_instance_profile = true + create_node_iam_role = true create_pod_identity_association = true + enable_irsa = true + enable_pod_identity = true enable_v1_permissions = true - create_instance_profile = true - + irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] + irsa_oidc_provider_arn = var.oidc_provider_arn + node_iam_role_name = local.node_iam_role_name + queue_name = local.queue_name + rule_name_prefix = local.rule_name_prefix # Attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = format("arn:%v:iam::%v:%v", data.aws_arn.current.partition, "aws", "policy/AmazonSSMManagedInstanceCore") diff --git a/prefixes.tf b/prefixes.tf new file mode 100644 index 0000000..51073d7 --- /dev/null +++ b/prefixes.tf @@ -0,0 +1,37 @@ +locals { + prefixes = { + "efs" = "v-efs-" + "s3" = "v-s3-" + "ebs" = "v-ebs-" + "kms" = "k-kms-" + "role" = "r-" + "policy" = "p-" + "group" = "g-" + "security-group" = "" # "sg-" + # VPC + "vpc" = "" + "dhcp-options" = "" + "vpc-peer" = "vpcp-" + "route-table" = "route-" + "subnet" = "" + "vpc-endpoint" = "vpce-" + "elastic-ip" = "eip-" + "nat-gateway" = "nat-" + "internet-gateway" = "igw-" + "network-acl" = "nacl-" + "customer-gateway" = "cgw-" + "vpn-gateway" = "vpcg-" + "vpn-connection" = "vpn_" + "log-group" = "lg-" + "log-stream" = "lgs-" + # EKS + "eks" = "eks-" + "eks-s3" = "v-s3-eks-" + "eks-user" = "s-eks-" + "eks-role" = "r-eks-" + "eks-policy" = "p-eks-" + "eks-security-group" = "eks-sg-" # "sg-eks-" + "event-rule" = "eks-ev-" + "eks-queue" = "eks-q-" + } +}