autoscaling (#18)

* autoscaling * add startup taint * add namespace and startup taint to template * templatlize * fix vals * fewer things * add a few missing values * create ns * update values * add nodeSelector and tolerations * remove digest * node group * fmt * syntax and startupTaints * fix syntax * update values * remove ng selector * lower cpu requests * use digest again * simplify repository * fix repository * add digest value * escape the . * fix subnets_name * subnet selector * use the outputs from karpenter module * create instance profile true * cleanup * without cluster tag * pass in vpc * enable debug and disable serviceMonitor * lower resources * fix merge issues
SCT-Engineering · Mar 17, 2025 · b159e31 · b159e31
1 parent 5cf4b69
commit b159e31
Show file tree

Hide file tree

Showing 10 changed files with 436 additions and 193 deletions.
diff --git a/.github/workflows/terragrunt-cicd.yml b/.github/workflows/terragrunt-cicd.yml
@@ -0,0 +1,101 @@
+name: 'Terraform Module CI'
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - '**/*.hcl'
+      - '**/*.tf'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - '**/*.hcl'
+      - '**/*.tf'
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  validate:
+    name: 'Validate Module'
+    runs-on: self-hosted
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v2
+      with:
+        terraform_version: 1.5.0
+
+    - name: Terraform Init
+      run: |
+        terraform init -backend=false
+
+    - name: Terraform Format
+      run: |
+        terraform fmt -check
+
+    - name: Terraform Validate
+      run: |
+        terraform validate
+
+    - name: Run tflint
+      uses: terraform-linters/setup-tflint@v3
+      if: github.event_name == 'pull_request'
+
+    - name: Lint Terraform
+      if: github.event_name == 'pull_request'
+      run: |
+        tflint --format compact
+
+  release:
+    name: 'Create Release'
+    needs: validate
+    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+    runs-on: self-hosted
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install Commitizen
+        run: |
+          pip install commitizen
+
+      - name: Configure Git
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+
+      - name: Bump Version and Generate Changelog
+        id: cz
+        run: |
+          cz bump --yes
+          echo "new_version=$(cz version --project)" >> $GITHUB_OUTPUT
+          echo "changelog=$(cz changelog --dry-run)" >> $GITHUB_OUTPUT
+
+      - name: Create Release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: v${{ steps.cz.outputs.new_version }}
+          release_name: Release v${{ steps.cz.outputs.new_version }}
+          draft: false
+          prerelease: false
+          body: ${{ steps.cz.outputs.changelog }}
diff --git a/.tflint.hcl b/.tflint.hcl
@@ -4,18 +4,18 @@ config {
   disabled_by_default = false
 }
 
-rule "aws_instance_invalid_type" {
-  enabled = true
-}
+# rule "aws_instance_invalid_type" {
+#   enabled = true
+# }
 
-plugin "aws" {
-  enabled = true
-  version = "0.32.0"
-  source  = "github.com/terraform-linters/tflint-ruleset-aws"
-}
+# plugin "aws" {
+#   enabled = true
+#   version = "0.32.0"
+#   source  = "github.com/terraform-linters/tflint-ruleset-aws"
+# }
 
-plugin "terraform" {
-  enabled = true
-  version = "0.9.0"
-  source  = "github.com/terraform-linters/tflint-ruleset-terraform"
-}
+# plugin "terraform" {
+#   enabled = true
+#   version = "0.9.0"
+#   source  = "github.com/terraform-linters/tflint-ruleset-terraform"
+# }
diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | 5.84.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | 5.89.0 |
 | <a name="provider_helm"></a> [helm](#provider\_helm) | 2.17.0 |
 
 ## Modules
@@ -39,7 +39,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the
 
 | Name | Type |
 |------|------|
-| [aws_iam_instance_profile.karpenter_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource |
 | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.nodepool_resources](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source |
@@ -56,11 +55,13 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the
 | <a name="input_karpenter_helm_repo"></a> [karpenter\_helm\_repo](#input\_karpenter\_helm\_repo) | Helm repo for official karpenter chart | `string` | `"oci://public.ecr.aws/karpenter"` | no |
 | <a name="input_karpenter_node_group_name"></a> [karpenter\_node\_group\_name](#input\_karpenter\_node\_group\_name) | The cluster node group that will host karpenter, should not be a karpenter managed node group | `string` | n/a | yes |
 | <a name="input_karpenter_tag"></a> [karpenter\_tag](#input\_karpenter\_tag) | Which tag of karpenter image | `string` | `"1.1.1"` | no |
+| <a name="input_namespace"></a> [namespace](#input\_namespace) | Namespace for Karpenter | `string` | `"namespace"` | no |
 | <a name="input_oidc_provider_arn"></a> [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes |
 | <a name="input_profile"></a> [profile](#input\_profile) | AWS config profile | `string` | `""` | no |
 | <a name="input_region"></a> [region](#input\_region) | AWS region | `string` | n/a | yes |
 | <a name="input_subnets_name"></a> [subnets\_name](#input\_subnets\_name) | Define the name of the subnets to be used by this cluster | `string` | `"*-container-*"` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | AWS Tags to apply to appropriate resources | `map(string)` | `{}` | no |
+| <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | The ID of the VPC where the resources will be deployed | `string` | n/a | yes |
 
 ## Outputs
 

diff --git a/copy_images.tf b/copy_images.tf
@@ -16,7 +16,6 @@ locals {
 
 # 224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter:0.37.0
 #  map[repository:224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter tag:0.37.0]
-
 module "images" {
   source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade"
 
@@ -25,7 +24,8 @@ module "images" {
   image_config     = local.image_config
   tags             = {}
 
-  enable_lifecycle_policy = true
-  lifecycle_policy_all    = true
-  force_delete            = true
+  enable_lifecycle_policy     = true
+  lifecycle_policy_all        = true
+  force_delete                = true
+  lifecycle_policy_keep_count = 3
 }
diff --git a/karpenter-resources/templates/node-pool.yaml b/karpenter-resources/templates/node-pool.yaml
@@ -25,6 +25,10 @@ spec:
       taints:
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.nodePool.startupTaints }}
+      startupTaints:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.nodePool.requirements }}
       requirements:
       {{- toYaml . | nindent 8 }}

diff --git a/karpenter-resources/values.yaml → karpenter-resources/values.yaml.tpl b/karpenter-resources/values.yaml → karpenter-resources/values.yaml.tpl
@@ -1,20 +1,21 @@
 # Shared values
-name: default
-clusterName: test-cluster
+name: ${cluster_name}
+clusterName: ${cluster_name}
 
 # EC2NodeClass specific values
 ec2NodeClass:
   enabled: true
-  amiFamily: bottlerocket
+  amiFamily: ${amd_ami_family}
   amiSelectorTerms:
-    - alias: "Bottlerocket@latest"
+    - alias: ${amd_ami_alias}
+  vpcId: ${vpc_id}
   subnetSelectorTerms:
     - tags:
-        Name: subnet-name
+        Name: "${subnets_name}"
   securityGroupSelectorTerms:
     - tags:
-        Name: security-group-name
-  karpenterRole: karpenter-role
+        Name: "${cluster_name}-node"
+  karpenterRole: ${karpenter_node_profile}
   metadataOptions:
     httpEndpoint: enabled
     httpProtocolIPv6: enabled
@@ -27,28 +28,30 @@ ec2NodeClass:
         volumeType: gp3
   detailedMonitoring: true
   instanceStorePolicy: RAID0
-
+  tags:
+    # These would be populated from var.tags in Terraform
+    "karpenter.sh/discovery": "${cluster_name}"
 
 # NodePool specific values
 nodePool:
-  name: example-cluster-node-pool
+  name: "${cluster_name}-worker-node-pool"
   enabled: true
   nodeClassRef:
-    name: default
+    name: "${cluster_name}-worker-node-class"
     group: karpenter.k8s.aws
     kind: EC2NodeClass
   disruption:
     consolidationPolicy: "WhenEmptyOrUnderutilized"
     consolidateAfter: "5m"
-    budgets:
-      maxUnavailable: 1
-      maxUnavailablePercentage: 10
   limits:
     cpu: "1000"
     memory: 1000Gi
   weight: 1
   labels: {}
   taints: []
+  startupTaints:
+    - key: ebs.csi.aws.com/agent-not-ready
+      effect: NoExecute
   requirements:
     - key: "kubernetes.io/arch"
       operator: "In"
@@ -59,11 +62,9 @@ nodePool:
     - key: karpenter.sh/capacity-type
       operator: In
       values: ["on-demand", "spot"]
-    - key: karpenter.k8s.aws/instance-generation
-      operator: Gt
-      values: ["2"]
-    - key: "karpenter.k8s.aws/instance-hypervisor"
-      operator: In
-      values: ["nitro"]
   expireAfter: "72h"
   terminationGracePeriod: "4h"
+
+# Extra value used in node-class.yaml template
+nodeClass:
+  cluster_name: ${cluster_name}