From b159e31587f079031a73f66a6b37929d55c605f3 Mon Sep 17 00:00:00 2001 From: Matthew Creal Morgan Date: Mon, 17 Mar 2025 14:04:49 -0700 Subject: [PATCH] autoscaling (#18) * autoscaling * add startup taint * add namespace and startup taint to template * templatlize * fix vals * fewer things * add a few missing values * create ns * update values * add nodeSelector and tolerations * remove digest * node group * fmt * syntax and startupTaints * fix syntax * update values * remove ng selector * lower cpu requests * use digest again * simplify repository * fix repository * add digest value * escape the . * fix subnets_name * subnet selector * use the outputs from karpenter module * create instance profile true * cleanup * without cluster tag * pass in vpc * enable debug and disable serviceMonitor * lower resources * fix merge issues --- .github/workflows/terragrunt-cicd.yml | 101 +++++++++ .tflint.hcl | 26 +-- README.md | 5 +- copy_images.tf | 8 +- karpenter-resources/templates/node-pool.yaml | 4 + .../{values.yaml => values.yaml.tpl} | 39 ++-- karpenter/full_karpenter_values.yml | 195 ++++++++++++++++++ karpenter/values.yaml.tpl | 52 +++++ main.tf | 182 +++------------- variables.tf | 17 +- 10 files changed, 436 insertions(+), 193 deletions(-) create mode 100644 .github/workflows/terragrunt-cicd.yml rename karpenter-resources/{values.yaml => values.yaml.tpl} (63%) create mode 100644 karpenter/full_karpenter_values.yml create mode 100644 karpenter/values.yaml.tpl diff --git a/.github/workflows/terragrunt-cicd.yml b/.github/workflows/terragrunt-cicd.yml new file mode 100644 index 0000000..a78523e --- /dev/null +++ b/.github/workflows/terragrunt-cicd.yml @@ -0,0 +1,101 @@ +name: 'Terraform Module CI' + +on: + push: + branches: + - main + paths: + - '**/*.hcl' + - '**/*.tf' + pull_request: + branches: + - main + paths: + - '**/*.hcl' + - '**/*.tf' + +permissions: + contents: read + pull-requests: write + +jobs: + validate: + name: 'Validate Module' + runs-on: self-hosted + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.5.0 + + - name: Terraform Init + run: | + terraform init -backend=false + + - name: Terraform Format + run: | + terraform fmt -check + + - name: Terraform Validate + run: | + terraform validate + + - name: Run tflint + uses: terraform-linters/setup-tflint@v3 + if: github.event_name == 'pull_request' + + - name: Lint Terraform + if: github.event_name == 'pull_request' + run: | + tflint --format compact + + release: + name: 'Create Release' + needs: validate + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + runs-on: self-hosted + permissions: + contents: write + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install Commitizen + run: | + pip install commitizen + + - name: Configure Git + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + - name: Bump Version and Generate Changelog + id: cz + run: | + cz bump --yes + echo "new_version=$(cz version --project)" >> $GITHUB_OUTPUT + echo "changelog=$(cz changelog --dry-run)" >> $GITHUB_OUTPUT + + - name: Create Release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ steps.cz.outputs.new_version }} + release_name: Release v${{ steps.cz.outputs.new_version }} + draft: false + prerelease: false + body: ${{ steps.cz.outputs.changelog }} diff --git a/.tflint.hcl b/.tflint.hcl index 684d807..ab8ea66 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -4,18 +4,18 @@ config { disabled_by_default = false } -rule "aws_instance_invalid_type" { - enabled = true -} +# rule "aws_instance_invalid_type" { +# enabled = true +# } -plugin "aws" { - enabled = true - version = "0.32.0" - source = "github.com/terraform-linters/tflint-ruleset-aws" -} +# plugin "aws" { +# enabled = true +# version = "0.32.0" +# source = "github.com/terraform-linters/tflint-ruleset-aws" +# } -plugin "terraform" { - enabled = true - version = "0.9.0" - source = "github.com/terraform-linters/tflint-ruleset-terraform" -} +# plugin "terraform" { +# enabled = true +# version = "0.9.0" +# source = "github.com/terraform-linters/tflint-ruleset-terraform" +# } diff --git a/README.md b/README.md index 09d0981..31d8199 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Version | |------|---------| -| [aws](#provider\_aws) | 5.84.0 | +| [aws](#provider\_aws) | 5.89.0 | | [helm](#provider\_helm) | 2.17.0 | ## Modules @@ -39,7 +39,6 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | Name | Type | |------|------| -| [aws_iam_instance_profile.karpenter_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.nodepool_resources](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | @@ -56,11 +55,13 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the | [karpenter\_helm\_repo](#input\_karpenter\_helm\_repo) | Helm repo for official karpenter chart | `string` | `"oci://public.ecr.aws/karpenter"` | no | | [karpenter\_node\_group\_name](#input\_karpenter\_node\_group\_name) | The cluster node group that will host karpenter, should not be a karpenter managed node group | `string` | n/a | yes | | [karpenter\_tag](#input\_karpenter\_tag) | Which tag of karpenter image | `string` | `"1.1.1"` | no | +| [namespace](#input\_namespace) | Namespace for Karpenter | `string` | `"namespace"` | no | | [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes | | [profile](#input\_profile) | AWS config profile | `string` | `""` | no | | [region](#input\_region) | AWS region | `string` | n/a | yes | | [subnets\_name](#input\_subnets\_name) | Define the name of the subnets to be used by this cluster | `string` | `"*-container-*"` | no | | [tags](#input\_tags) | AWS Tags to apply to appropriate resources | `map(string)` | `{}` | no | +| [vpc\_id](#input\_vpc\_id) | The ID of the VPC where the resources will be deployed | `string` | n/a | yes | ## Outputs diff --git a/copy_images.tf b/copy_images.tf index 4c05c03..8febc1b 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -16,7 +16,6 @@ locals { # 224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter:0.37.0 # map[repository:224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter tag:0.37.0] - module "images" { source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" @@ -25,7 +24,8 @@ module "images" { image_config = local.image_config tags = {} - enable_lifecycle_policy = true - lifecycle_policy_all = true - force_delete = true + enable_lifecycle_policy = true + lifecycle_policy_all = true + force_delete = true + lifecycle_policy_keep_count = 3 } diff --git a/karpenter-resources/templates/node-pool.yaml b/karpenter-resources/templates/node-pool.yaml index ac8e368..347a194 100644 --- a/karpenter-resources/templates/node-pool.yaml +++ b/karpenter-resources/templates/node-pool.yaml @@ -25,6 +25,10 @@ spec: taints: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.nodePool.startupTaints }} + startupTaints: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodePool.requirements }} requirements: {{- toYaml . | nindent 8 }} diff --git a/karpenter-resources/values.yaml b/karpenter-resources/values.yaml.tpl similarity index 63% rename from karpenter-resources/values.yaml rename to karpenter-resources/values.yaml.tpl index cc9886d..747ba17 100644 --- a/karpenter-resources/values.yaml +++ b/karpenter-resources/values.yaml.tpl @@ -1,20 +1,21 @@ # Shared values -name: default -clusterName: test-cluster +name: ${cluster_name} +clusterName: ${cluster_name} # EC2NodeClass specific values ec2NodeClass: enabled: true - amiFamily: bottlerocket + amiFamily: ${amd_ami_family} amiSelectorTerms: - - alias: "Bottlerocket@latest" + - alias: ${amd_ami_alias} + vpcId: ${vpc_id} subnetSelectorTerms: - tags: - Name: subnet-name + Name: "${subnets_name}" securityGroupSelectorTerms: - tags: - Name: security-group-name - karpenterRole: karpenter-role + Name: "${cluster_name}-node" + karpenterRole: ${karpenter_node_profile} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: enabled @@ -27,28 +28,30 @@ ec2NodeClass: volumeType: gp3 detailedMonitoring: true instanceStorePolicy: RAID0 - + tags: + # These would be populated from var.tags in Terraform + "karpenter.sh/discovery": "${cluster_name}" # NodePool specific values nodePool: - name: example-cluster-node-pool + name: "${cluster_name}-worker-node-pool" enabled: true nodeClassRef: - name: default + name: "${cluster_name}-worker-node-class" group: karpenter.k8s.aws kind: EC2NodeClass disruption: consolidationPolicy: "WhenEmptyOrUnderutilized" consolidateAfter: "5m" - budgets: - maxUnavailable: 1 - maxUnavailablePercentage: 10 limits: cpu: "1000" memory: 1000Gi weight: 1 labels: {} taints: [] + startupTaints: + - key: ebs.csi.aws.com/agent-not-ready + effect: NoExecute requirements: - key: "kubernetes.io/arch" operator: "In" @@ -59,11 +62,9 @@ nodePool: - key: karpenter.sh/capacity-type operator: In values: ["on-demand", "spot"] - - key: karpenter.k8s.aws/instance-generation - operator: Gt - values: ["2"] - - key: "karpenter.k8s.aws/instance-hypervisor" - operator: In - values: ["nitro"] expireAfter: "72h" terminationGracePeriod: "4h" + +# Extra value used in node-class.yaml template +nodeClass: + cluster_name: ${cluster_name} diff --git a/karpenter/full_karpenter_values.yml b/karpenter/full_karpenter_values.yml new file mode 100644 index 0000000..1b6995c --- /dev/null +++ b/karpenter/full_karpenter_values.yml @@ -0,0 +1,195 @@ +# -- Overrides the chart's name. +nameOverride: "" +# -- Overrides the chart's computed fullname. +fullnameOverride: "" +# -- Additional labels to add into metadata. +additionalLabels: {} +# app: karpenter + +# -- Additional annotations to add into metadata. +additionalAnnotations: {} +# -- Image pull policy for Docker images. +imagePullPolicy: IfNotPresent +# -- Image pull secrets for Docker images. +imagePullSecrets: [] +service: + # -- Additional annotations for the Service. + annotations: {} +serviceAccount: + # -- Specifies if a ServiceAccount should be created. + create: true + # -- The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template. + name: "" + # -- Additional annotations for the ServiceAccount. + annotations: {} +# -- Specifies additional rules for the core ClusterRole. +additionalClusterRoleRules: [] +serviceMonitor: + # -- Specifies whether a ServiceMonitor should be created. + enabled: false + # -- Additional labels for the ServiceMonitor. + additionalLabels: {} + # -- Configuration on `http-metrics` endpoint for the ServiceMonitor. + # Not to be used to add additional endpoints. + # See the Prometheus operator documentation for configurable fields https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint + endpointConfig: {} +# -- Number of replicas. +replicas: 2 +# -- The number of old ReplicaSets to retain to allow rollback. +revisionHistoryLimit: 10 +# -- Strategy for updating the pod. +strategy: + rollingUpdate: + maxUnavailable: 1 +# -- Additional labels for the pod. +podLabels: {} +# -- Additional annotations for the pod. +podAnnotations: {} +podDisruptionBudget: + name: karpenter + maxUnavailable: 1 +# -- SecurityContext for the pod. +podSecurityContext: + fsGroup: 65532 +# -- PriorityClass name for the pod. +priorityClassName: system-cluster-critical +# -- Override the default termination grace period for the pod. +terminationGracePeriodSeconds: +# -- Bind the pod to the host network. +# This is required when using a custom CNI. +hostNetwork: false +# -- Specify which Kubernetes scheduler should dispatch the pod. +schedulerName: default-scheduler +# -- Configure the DNS Policy for the pod +dnsPolicy: ClusterFirst +# -- Configure DNS Config for the pod +dnsConfig: {} +# options: +# - name: ndots +# value: "1" +# -- Node selectors to schedule the pod to nodes with labels. +nodeSelector: + kubernetes.io/os: linux +# -- Affinity rules for scheduling the pod. If an explicit label selector is not provided for pod affinity or pod anti-affinity one will be created from the pod selector labels. +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: karpenter.sh/nodepool + operator: DoesNotExist + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" +# -- Topology spread constraints to increase the controller resilience by distributing pods across the cluster zones. If an explicit label selector is not provided one will be created from the pod selector labels. +topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule +# -- Tolerations to allow the pod to be scheduled to nodes with taints. +tolerations: + - key: CriticalAddonsOnly + operator: Exists +# -- Additional volumes for the pod. +extraVolumes: [] +# - name: aws-iam-token +# projected: +# defaultMode: 420 +# sources: +# - serviceAccountToken: +# audience: sts.amazonaws.com +# expirationSeconds: 86400 +# path: token +controller: + # -- Distinguishing container name (containerName: karpenter-controller). + containerName: controller + image: + # -- Repository path to the controller image. + repository: public.ecr.aws/karpenter/controller + # -- Tag of the controller image. + tag: 1.3.0 + # -- SHA256 digest of the controller image. + digest: sha256:23876d27c39f4b99ad41ee245319fc2a2fa499183131e9cfccb550658d003045 + # -- Additional environment variables for the controller pod. + env: [] + # - name: AWS_REGION + # value: eu-west-1 + envFrom: [] + # -- Resources for the controller pod. + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 1 + # memory: 1Gi + + # -- Additional volumeMounts for the controller pod. + extraVolumeMounts: [] + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # -- Additional sidecarContainer config + sidecarContainer: [] + # -- Additional volumeMounts for the sidecar - this will be added to the volume mounts on top of extraVolumeMounts + sidecarVolumeMounts: [] + metrics: + # -- The container port to use for metrics. + port: 8080 + healthProbe: + # -- The container port to use for http health probe. + port: 8081 +# -- Global log level, defaults to 'info' +logLevel: info +# -- Log outputPaths - defaults to stdout only +logOutputPaths: + - stdout +# -- Log errorOutputPaths - defaults to stderr only +logErrorOutputPaths: + - stderr +# -- Global Settings to configure Karpenter +settings: + # -- The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one + # time which usually results in fewer but larger nodes. + batchMaxDuration: 10s + # -- The maximum amount of time with no new ending pods that if exceeded ends the current batching window. If pods arrive + # faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods + # will be batched separately. + batchIdleDuration: 1s + # -- Cluster CA bundle for TLS configuration of provisioned nodes. If not set, this is taken from the controller's TLS configuration for the API server. + clusterCABundle: "" + # -- Cluster name. + clusterName: "" + # -- Cluster endpoint. If not set, will be discovered during startup (EKS only) + clusterEndpoint: "" + # -- If true then assume we can't reach AWS services which don't have a VPC endpoint + # This also has the effect of disabling look-ups to the AWS pricing endpoint + isolatedVPC: false + # Marking this true means that your cluster is running with an EKS control plane and Karpenter should attempt to discover cluster details from the DescribeCluster API + eksControlPlane: false + # -- The VM memory overhead as a percent that will be subtracted from the total memory for all instance types. The value of `0.075` equals to 7.5%. + vmMemoryOverheadPercent: 0.075 + # -- Interruption queue is the name of the SQS queue used for processing interruption events from EC2 + # Interruption handling is disabled if not specified. Enabling interruption handling may + # require additional permissions on the controller service account. Additional permissions are outlined in the docs. + interruptionQueue: "" + # -- Reserved ENIs are not included in the calculations for max-pods or kube-reserved + # This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html + reservedENIs: "0" + # -- Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates + # in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features + featureGates: + # -- nodeRepair is ALPHA and is disabled by default. + # Setting this to true will enable node repair. + nodeRepair: false + # -- reservedCapacity is ALPHA and is disabled by default. + # Setting this will enable native on-demand capacity reservation support. + reservedCapacity: false + # -- spotToSpotConsolidation is ALPHA and is disabled by default. + # Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation. + spotToSpotConsolidation: false diff --git a/karpenter/values.yaml.tpl b/karpenter/values.yaml.tpl new file mode 100644 index 0000000..1c0b489 --- /dev/null +++ b/karpenter/values.yaml.tpl @@ -0,0 +1,52 @@ +settings: + clusterName: ${cluster_name} + clusterEndpoint: ${cluster_endpoint} + interruptionQueue: ${queue_name} + isolatedVPC: true + featureGates: + nodeRepair: true + reservedCapacity: true + spotToSpotConsolidation: true + +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: ${iam_role_arn} + +replicas: 2 + +serviceMonitor: + enabled: false +logLevel: debug + +controller: + resources: + requests: + cpu: 1m + memory: 1Mi + limits: + cpu: 500m + memory: 1Gi + image: + repository: ${repository} + tag: ${tag} + digest: ${digest} + env: + - name: AWS_REGION + value: ${region} + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: karpenter.sh/nodepool + operator: DoesNotExist + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" diff --git a/main.tf b/main.tf index 59c8cd0..638cfdc 100644 --- a/main.tf +++ b/main.tf @@ -1,5 +1,4 @@ locals { - namespace = var.namespace amd_ami_family = "Bottlerocket" amd_ami_alias = "bottlerocket@latest" } @@ -12,7 +11,10 @@ module "karpenter_resources" { cluster_name = var.cluster_name enable_irsa = true irsa_oidc_provider_arn = var.oidc_provider_arn - irsa_namespace_service_accounts = ["${local.namespace}:karpenter"] + irsa_namespace_service_accounts = ["${var.namespace}:karpenter"] + create_pod_identity_association = true + enable_v1_permissions = true + create_instance_profile = true # Attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { @@ -22,20 +24,6 @@ module "karpenter_resources" { tags = var.tags } -# Create IAM Instance Profile for Karpenter Nodes -resource "aws_iam_instance_profile" "karpenter_node" { - name = "${var.cluster_name}-karpenter-node-profile" - role = module.karpenter_resources.node_iam_role_name - - tags = merge( - var.tags, - { - "karpenter.sh/managed-by" = var.cluster_name - "kubernetes.io/cluster/${var.cluster_name}" = "owned" - } - ) -} - # Install Karpenter data "aws_ecr_image" "karpenter_image" { depends_on = [ @@ -52,82 +40,29 @@ resource "helm_release" "karpenter" { name = "karpenter" chart = "karpenter" version = var.karpenter_helm_chart - namespace = local.namespace + namespace = var.namespace description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" repository = var.karpenter_helm_repo create_namespace = true - set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key" - value = "karpenter.sh/nodepool" - } - set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator" - value = "DoesNotExist" - } - set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].key" - value = "eks.amazonaws.com/nodegroup" - } - set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].operator" - value = "In" - } - set { - name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].values[0]" - value = var.karpenter_node_group_name - } - set { - name = "affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey" - value = "kubernetes.io/hostname" - } - set { - name = "settings.clusterName" - value = var.cluster_name - } - set { - name = "settings.clusterEndpoint" - value = var.cluster_endpoint - } - set { - name = "settings.featureGates.spotToSpotConsolidation" - value = "true" - } - set { - name = "settings.interruptionQueue" - value = module.karpenter_resources.queue_name - } - set { - name = "settings.isolatedVPC" - value = "true" - } - set { - name = "controller.image.repository" - value = format("%v/%v", - module.images.images[local.karpenter_key].dest_registry, - module.images.images[local.karpenter_key].dest_repository - ) - } - set { - name = "controller.image.tag" - value = module.images.images[local.karpenter_key].tag - } - set { - name = "controller.image.digest" - value = data.aws_ecr_image.karpenter_image.id - } - set { - name = "controller.env[0].name" - value = "AWS_REGION" - } - set { - name = "controller.env[0].value" - value = var.region - } - set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.karpenter_resources.iam_role_arn - } + values = [ + templatefile("${path.module}/karpenter/values.yaml.tpl", { + cluster_endpoint = var.cluster_endpoint + cluster_name = var.cluster_name + digest = data.aws_ecr_image.karpenter_image.id + iam_role_arn = module.karpenter_resources.iam_role_arn + karpenter_node_group_name = var.karpenter_node_group_name + karpenter_node_profile = module.karpenter_resources.instance_profile_name + queue_name = module.karpenter_resources.queue_name + region = var.region + repository = format("%v/%v", + module.images.images[local.karpenter_key].dest_registry, + module.images.images[local.karpenter_key].dest_repository + ) + tag = module.images.images[local.karpenter_key].tag + }) + ] + timeout = 300 } @@ -139,68 +74,17 @@ resource "helm_release" "nodepool_resources" { description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" chart = "./karpenter-resources" version = local.module_version - namespace = local.namespace + namespace = var.namespace values = [ - file("${path.module}/karpenter-resources/values.yaml") + templatefile("${path.module}/karpenter-resources/values.yaml.tpl", { + amd_ami_alias = local.amd_ami_alias + amd_ami_family = local.amd_ami_family + cluster_name = var.cluster_name + karpenter_node_profile = module.karpenter_resources.instance_profile_name + subnets_name = var.subnets_name + tags = var.tags + vpc_id = var.vpc_id + }) ] - #### Default values overrides - set { - name = "name" - value = var.cluster_name - } - set { - name = "clusterName" - value = var.cluster_name - } - set { - name = "ec2NodeClass.karpenterRole" - value = aws_iam_instance_profile.karpenter_node.name - } - set { - name = "ec2NodeClass.amiFamily" - value = local.amd_ami_family - } - set { - name = "ec2NodeClass.amiSelectorTerms[0].alias" - value = local.amd_ami_alias - } - set { - name = "ec2NodeClass.securityGroupSelectorTerms[0].tags.Name" - value = "${var.cluster_name}-node" - } - set { - name = "ec2NodeClass.subnetSelectorTerms[0].tags.Name" - value = var.subnets_name - } - # set { - # name = "ec2NodeClass.subnetSelectorTerms[0].tags.\"kubernetes.io/cluster/${var.cluster_name}\"" - # value = "shared" - # } - ### Node Class Inputs - set { - name = "nodePool.nodeClassRef.name" - value = "${var.cluster_name}-node-class" - } - set { - name = "nodePool.name" - value = "${var.cluster_name}-karpenter-node-pool" - } - set { - name = "nodeClass.cluster_name" - value = var.cluster_name - } - set { - name = "nodePool.limits.cpu" - value = "1000" - } - set { - name = "nodePool.limits.memory" - value = "1000Gi" - } - set { - name = "ec2NodeClass.tags" - value = jsonencode(var.tags) - } - } diff --git a/variables.tf b/variables.tf index 85e1724..72bd4d8 100644 --- a/variables.tf +++ b/variables.tf @@ -48,12 +48,6 @@ variable "karpenter_tag" { default = "1.1.1" } -variable "namespace" { - description = "Namespace in which to install cert manager" - type = string - default = "kube-system" -} - variable "subnets_name" { description = "Define the name of the subnets to be used by this cluster" type = string @@ -65,3 +59,14 @@ variable "tags" { type = map(string) default = {} } + +variable "namespace" { + description = "Namespace for Karpenter" + type = string + default = "namespace" +} + +variable "vpc_id" { + description = "The ID of the VPC where the resources will be deployed" + type = string +}