diff --git a/.cz.yaml b/.cz.yaml
new file mode 100644
index 0000000..b1981ec
--- /dev/null
+++ b/.cz.yaml
@@ -0,0 +1,8 @@
+---
+commitizen:
+ major_version_zero: true
+ name: cz_gitmoji
+ tag_format: $version
+ update_changelog_on_bump: true
+ version_provider: scm
+ version_scheme: semver2
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..867570d
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+ - package-ecosystem: "terraform" # See documentation for possible values
+ directory: "/" # Location of package manifests
+ schedule:
+ interval: "daily"
diff --git a/.gitignore b/.gitignore
index 7c1cae9..98d790c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
# Local .terraform directories
**/.terraform/*
+**/.terragrunt-cache/*
# terraform lock file.
**/.terraform.lock.hcl
@@ -35,4 +36,3 @@ override.tf.json
# Ignore CLI configuration files
.terraformrc
terraform.rc
-
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000..d6091dd
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,111 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.6.0
+ hooks:
+ # Git style
+ - id: check-added-large-files
+ - id: check-merge-conflict
+ - id: check-vcs-permalinks
+ - id: forbid-new-submodules
+ - id: no-commit-to-branch
+ - id: check-byte-order-marker
+ - id: check-case-conflict
+ - id: check-json
+ - id: check-merge-conflict
+ - id: check-symlinks
+ - id: check-vcs-permalinks
+ - id: check-toml
+ - id: check-xml
+ - id: detect-private-key
+ - id: requirements-txt-fixer
+ - id: sort-simple-yaml
+
+ # Common errors
+ - id: end-of-file-fixer
+ - id: trailing-whitespace
+ args: [--markdown-linebreak-ext=md]
+ exclude: CHANGELOG.md
+ - id: check-yaml
+ - id: check-merge-conflict
+ - id: check-executables-have-shebangs
+
+ # Cross platform
+ - id: check-case-conflict
+ - id: mixed-line-ending
+ args: [--fix=lf]
+
+ # Security
+ - id: detect-aws-credentials
+ args: ['--allow-missing-credentials']
+ - id: detect-private-key
+
+# JSON5 Linter
+- repo: https://github.com/pre-commit/mirrors-prettier
+ rev: v4.0.0-alpha.8
+ hooks:
+ - id: prettier
+ # https://prettier.io/docs/en/options.html#parser
+ files: '.json5$'
+
+# Terraform Hooks
+- repo: https://github.com/antonbabenko/pre-commit-terraform
+ rev: v1.92.1 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases
+ hooks:
+ - id: terraform_fmt
+ args:
+ - --hook-config=--parallelism-ci-cpu-cores=2
+ - id: terraform_docs
+ args:
+ - --hook-config=--parallelism-ci-cpu-cores=2
+ - id: terraform_tflint
+ name: Terraform validate with tflint
+ description: Validates all Terraform configuration files with TFLint.
+ require_serial: true
+ entry: hooks/terraform_tflint.sh
+ language: script
+ files: (\.tf|\.tfvars)$
+ exclude: \.(terraform/.*|terragrunt-cache)$
+ args:
+ - --hook-config=--parallelism-ci-cpu-cores=2
+ - id: terragrunt_fmt
+ name: Terragrunt fmt
+ description: Rewrites all Terragrunt configuration files to a canonical format.
+ entry: hooks/terragrunt_fmt.sh
+ language: script
+ files: (\.hcl)$
+ exclude: \.(terraform/.*|terragrunt-cache)$
+ args:
+ - --hook-config=--parallelism-ci-cpu-cores=2
+ # Will require dependency mocks
+ # - id: terragrunt_validate
+ # name: Terragrunt validate
+ # description: Validates all Terragrunt configuration files.
+ # entry: hooks/terragrunt_validate.sh
+ # language: script
+ # files: (\.hcl)$
+ # exclude: \.(terraform/.*|terragrunt-cache)$
+ # args:
+ # - --hook-config=--parallelism-ci-cpu-cores=2
+ # - id: terragrunt_validate_inputs
+ # name: Terragrunt validate inputs
+ # description: Validates Terragrunt unused and undefined inputs.
+ # entry: hooks/terragrunt_validate_inputs.sh
+ # language: script
+ # files: (\.hcl)$
+ # exclude: \.(terraform/.*|terragrunt-cache)$
+ # args:
+ # - --hook-config=--parallelism-ci-cpu-cores=2
+ # - id: terragrunt_providers_lock
+ # name: Terragrunt providers lock
+ # description: Updates provider signatures in dependency lock files using terragrunt.
+ # entry: hooks/terragrunt_providers_lock.sh
+ # language: script
+ # files: (terragrunt|\.terraform\.lock)\.hcl$
+ # exclude: \.(terraform/.*|terragrunt-cache)$
+ # args:
+ # - --hook-config=--parallelism-ci-cpu-cores=2
+
+- repo: https://github.com/ljnsn/cz-conventional-gitmoji
+ rev: v0.3.2
+ hooks:
+ - id: conventional-gitmoji
diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml
new file mode 100644
index 0000000..266f0c7
--- /dev/null
+++ b/.pre-commit-hooks.yaml
@@ -0,0 +1,170 @@
+#### THESE ARE NOT ENABLED, THEY ARE FOR REFERENCE
+# - id: infracost_breakdown
+# name: Infracost breakdown
+# description: Check terraform infrastructure cost
+# entry: hooks/infracost_breakdown.sh
+# language: script
+# require_serial: true
+# files: \.(tf(vars)?|hcl)$
+# exclude: \.terraform/.*$
+
+# - id: terraform_fmt
+# name: Terraform fmt
+# description: Rewrites all Terraform configuration files to a canonical format.
+# entry: hooks/terraform_fmt.sh
+# language: script
+# files: (\.tf|\.tfvars)$
+# exclude: \.terraform/.*$
+
+# - id: terraform_docs
+# name: Terraform docs
+# description: Inserts input and output documentation into README.md (using terraform-docs).
+# require_serial: true
+# entry: hooks/terraform_docs.sh
+# language: script
+# files: (\.tf|\.terraform\.lock\.hcl)$
+# exclude: \.terraform/.*$
+
+# - id: terraform_docs_without_aggregate_type_defaults
+# name: Terraform docs (without aggregate type defaults)
+# description: Inserts input and output documentation into README.md (using terraform-docs). Identical to terraform_docs.
+# require_serial: true
+# entry: hooks/terraform_docs.sh
+# language: script
+# files: (\.tf)$
+# exclude: \.terraform/.*$
+
+# - id: terraform_docs_replace
+# name: Terraform docs (overwrite README.md)
+# description: Overwrite content of README.md with terraform-docs.
+# require_serial: true
+# entry: terraform_docs_replace
+# language: python
+# files: (\.tf)$
+# exclude: \.terraform/.*$
+
+# - id: terraform_validate
+# name: Terraform validate
+# description: Validates all Terraform configuration files.
+# require_serial: true
+# entry: hooks/terraform_validate.sh
+# language: script
+# files: \.(tf(vars)?|terraform\.lock\.hcl)$
+# exclude: \.terraform/.*$
+
+# - id: terraform_providers_lock
+# name: Lock terraform provider versions
+# description: Updates provider signatures in dependency lock files.
+# require_serial: true
+# entry: hooks/terraform_providers_lock.sh
+# language: script
+# files: (\.terraform\.lock\.hcl)$
+# exclude: \.terraform/.*$
+
+# - id: terraform_tflint
+# name: Terraform validate with tflint
+# description: Validates all Terraform configuration files with TFLint.
+# require_serial: true
+# entry: hooks/terraform_tflint.sh
+# language: script
+# files: (\.tf|\.tfvars)$
+# exclude: \.terraform/.*$
+
+# - id: terragrunt_fmt
+# name: Terragrunt fmt
+# description: Rewrites all Terragrunt configuration files to a canonical format.
+# entry: hooks/terragrunt_fmt.sh
+# language: script
+# files: (\.hcl)$
+# exclude: \.terraform/.*$
+
+# - id: terragrunt_validate
+# name: Terragrunt validate
+# description: Validates all Terragrunt configuration files.
+# entry: hooks/terragrunt_validate.sh
+# language: script
+# files: (\.hcl)$
+# exclude: \.terraform/.*$
+
+# - id: terragrunt_validate_inputs
+# name: Terragrunt validate inputs
+# description: Validates Terragrunt unused and undefined inputs.
+# entry: hooks/terragrunt_validate_inputs.sh
+# language: script
+# files: (\.hcl)$
+# exclude: \.terraform/.*$
+
+# - id: terragrunt_providers_lock
+# name: Terragrunt providers lock
+# description: Updates provider signatures in dependency lock files using terragrunt.
+# entry: hooks/terragrunt_providers_lock.sh
+# language: script
+# files: (terragrunt|\.terraform\.lock)\.hcl$
+# exclude: \.(terraform/.*|terragrunt-cache)$
+
+# - id: terraform_tfsec
+# name: Terraform validate with tfsec (deprecated, use "terraform_trivy")
+# description: Static analysis of Terraform templates to spot potential security issues.
+# require_serial: true
+# entry: hooks/terraform_tfsec.sh
+# files: \.tf(vars)?$
+# language: script
+
+# - id: terraform_trivy
+# name: Terraform validate with trivy
+# description: Static analysis of Terraform templates to spot potential security issues.
+# require_serial: true
+# entry: hooks/terraform_trivy.sh
+# files: \.tf(vars)?$
+# language: script
+
+# - id: checkov
+# name: checkov (deprecated, use "terraform_checkov")
+# description: Runs checkov on Terraform templates.
+# entry: checkov -d .
+# language: python
+# pass_filenames: false
+# always_run: false
+# files: \.tf$
+# exclude: \.terraform/.*$
+# require_serial: true
+
+# - id: terraform_checkov
+# name: Checkov
+# description: Runs checkov on Terraform templates.
+# entry: hooks/terraform_checkov.sh
+# language: script
+# always_run: false
+# files: \.tf$
+# exclude: \.terraform/.*$
+# require_serial: true
+
+# - id: terraform_wrapper_module_for_each
+# name: Terraform wrapper with for_each in module
+# description: Generate Terraform wrappers with for_each in module.
+# entry: hooks/terraform_wrapper_module_for_each.sh
+# language: script
+# pass_filenames: false
+# always_run: false
+# require_serial: true
+# files: \.tf$
+# exclude: \.terraform/.*$
+
+# - id: terrascan
+# name: terrascan
+# description: Runs terrascan on Terraform templates.
+# language: script
+# entry: hooks/terrascan.sh
+# files: \.tf$
+# exclude: \.terraform/.*$
+# require_serial: true
+
+# - id: tfupdate
+# name: tfupdate
+# description: Runs tfupdate on Terraform templates.
+# language: script
+# entry: hooks/tfupdate.sh
+# args:
+# - --args=terraform
+# files: \.tf$
+# require_serial: true
diff --git a/.releaserc.json b/.releaserc.json
new file mode 100644
index 0000000..6e39031
--- /dev/null
+++ b/.releaserc.json
@@ -0,0 +1,36 @@
+{
+ "branches": [
+ "main",
+ "master"
+ ],
+ "ci": false,
+ "plugins": [
+ "@semantic-release/commit-analyzer",
+ "@semantic-release/release-notes-generator",
+ [
+ "@semantic-release/github",
+ {
+ "successComment":
+ "This ${issue.pull_request ? 'PR is included' : 'issue has been resolved'} in version ${nextRelease.version} :tada:",
+ "labels": false,
+ "releasedLabels": false
+ }
+ ],
+ [
+ "@semantic-release/changelog",
+ {
+ "changelogFile": "CHANGELOG.md",
+ "changelogTitle": "# Changelog\n\nAll notable changes to this project will be documented in this file."
+ }
+ ],
+ [
+ "@semantic-release/git",
+ {
+ "assets": [
+ "CHANGELOG.md"
+ ],
+ "message": "chore(release): version ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
+ }
+ ]
+ ]
+}
diff --git a/.terraform-docs.yml b/.terraform-docs.yml
new file mode 100644
index 0000000..fabfb8d
--- /dev/null
+++ b/.terraform-docs.yml
@@ -0,0 +1,44 @@
+formatter: markdown table
+
+header-from: main.tf
+footer-from: ""
+
+sections:
+## hide: []
+ show:
+ - data-sources
+ - header
+ - footer
+ - inputs
+ - modules
+ - outputs
+ - providers
+ - requirements
+ - resources
+
+output:
+ file: README.md
+ mode: inject
+ template: |-
+
+ {{ .Content }}
+
+
+output-values:
+ enabled: false
+ from: ""
+
+sort:
+ enabled: true
+ by: name
+
+settings:
+ anchor: true
+ color: true
+ default: true
+ description: true
+ escape: true
+ indent: 2
+ required: true
+ sensitive: true
+ type: true
diff --git a/.tflint.hcl b/.tflint.hcl
new file mode 100644
index 0000000..684d807
--- /dev/null
+++ b/.tflint.hcl
@@ -0,0 +1,21 @@
+config {
+ module = true
+ force = false
+ disabled_by_default = false
+}
+
+rule "aws_instance_invalid_type" {
+ enabled = true
+}
+
+plugin "aws" {
+ enabled = true
+ version = "0.32.0"
+ source = "github.com/terraform-linters/tflint-ruleset-aws"
+}
+
+plugin "terraform" {
+ enabled = true
+ version = "0.9.0"
+ source = "github.com/terraform-linters/tflint-ruleset-terraform"
+}
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000..ada5fcf
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,19 @@
+## 0.1.0 (2024-08-19)
+
+### ✨ Features
+
+- **karpenter**: update module upstream to 1.0.0
+- **many**: subnet tagging updated nodepoolandnodeclass
+- **prometheus**: enables serviceMonitor to captures karpenter metrics in prometheus
+
+### 🐛🚑️ Fixes
+
+- **kubectl**: fixes for kubectl image
+
+### ci
+
+- **base**: add repo features
+
+### fix
+
+- **main.tf**: correct the value for amd_ami_family
diff --git a/EC2NodeClass.yaml.tpl b/EC2NodeClass.yaml.tpl
deleted file mode 100644
index 4ba530d..0000000
--- a/EC2NodeClass.yaml.tpl
+++ /dev/null
@@ -1,14 +0,0 @@
-apiVersion: karpenter.k8s.aws/v1beta1
-kind: EC2NodeClass
-metadata:
- name: default
-spec:
- amiFamily: AL2
- instanceProfile: "KarpenterNodeInstanceProfile-${cluster_name}"
- subnetSelectorTerms:
- - tags:
- Name: "*-container-*"
- securityGroupSelectorTerms:
- - tags:
- Name: "${cluster_name}-node"
- tags: ${tags}
\ No newline at end of file
diff --git a/EC2NodePool.yaml.tpl b/EC2NodePool.yaml.tpl
deleted file mode 100644
index 538f282..0000000
--- a/EC2NodePool.yaml.tpl
+++ /dev/null
@@ -1,22 +0,0 @@
-apiVersion: karpenter.sh/v1beta1
-kind: NodePool
-metadata:
- name: default
-spec:
- template:
- spec:
- requirements:
- - key: karpenter.sh/capacity-type
- operator: In
- values: ["spot", "on-demand"]
- - key: "node.kubernetes.io/instance-type"
- operator: In
- values: ["t3a.xlarge", "t3.xlarge", "c5.large", "m5.large", "r5.large", "m5.xlarge"]
- nodeClassRef:
- name: default
- limits:
- cpu: 1000
- memory: 1000Gi
- disruption:
- consolidationPolicy: WhenUnderutilized
- expireAfter: 24h # 30 * 24h = 720h
diff --git a/README.md b/README.md
index acf93ce..5fc006a 100644
--- a/README.md
+++ b/README.md
@@ -10,15 +10,7 @@ ref: [Karpenter Best Practices](https://aws.github.io/aws-eks-best-practices/kar
NOTE: Deploying Karpenter after deploying Istio is not recommended because it disrupts the handling of sidecars to pods on Karpenter managed nodes. As long as Karpenter is deployed first, there is no issue.
-The module deploys Karpenter needed AWS resources, namely in IAM. It copies the Karpenter controller from the public registry to local ECR, deploys Karpenter via Helm and deploys a single default Karpenter Nodepool with EC2NodeClass.
-
-# CHANGELOG
-* 0.0.2 -- 2024-07-30
- - update after removing cluster-autoscaler
- - add second yaml template
- - update images module
-* 0.0.1 -- 2024-07-26
- - Genesis/Initial development
+The module deploys Karpenter needed AWS resources, namely in IAM. It copies the Karpenter controller from the public registry to local ECR, deploys Karpenter via Helm and deploys a single default Karpenter Nodepool with EC2NodeClass.
## Requirements
@@ -29,33 +21,38 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the
| [aws](#requirement\_aws) | >= 5.14.0 |
| [helm](#requirement\_helm) | >= 2.11.0 |
| [kubectl](#requirement\_kubectl) | >= 1.14.0 |
-| [kubernetes](#requirement\_kubernetes) | >= 2.23.0 |
## Providers
| Name | Version |
|------|---------|
-| [aws](#provider\_aws) | >= 5.14.0 |
-| [helm](#provider\_helm) | >= 2.11.0 |
-| [kubectl](#provider\_kubectl) | >= 1.14.0 |
+| [aws](#provider\_aws) | 5.61.0 |
+| [helm](#provider\_helm) | 2.14.0 |
+| [kubectl](#provider\_kubectl) | 1.14.0 |
## Modules
| Name | Source | Version |
|------|--------|---------|
-| [images](#module\_images) | git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/ | 2.0.2 |
-| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | v20.8.5 |
+| [images](#module\_images) | git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/ | tf-upgrade |
+| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | v20.23.0 |
## Resources
| Name | Type |
|------|------|
-| [aws_iam_instance_profile.KarpenterNodeInstanceProfile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource |
+| [aws_ec2_tag.container_subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ec2_tag) | resource |
+| [aws_iam_instance_profile.karpenter_node_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource |
| [aws_iam_service_linked_role.ec2_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_service_linked_role) | resource |
| [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
-| [kubectl_manifest.karpenter_EC2NodeClass](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
-| [kubectl_manifest.karpenter_NodePool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [kubectl_manifest.karpenter_ec2_node_class](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
| [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source |
+| [aws_ecr_image.kubectl_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source |
+| [aws_iam_roles.roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_roles) | data source |
+| [aws_subnets.container_subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source |
## Inputs
@@ -63,15 +60,22 @@ The module deploys Karpenter needed AWS resources, namely in IAM. It copies the
|------|-------------|------|---------|:--------:|
| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint for your Kubernetes API server | `string` | n/a | yes |
| [cluster\_name](#input\_cluster\_name) | EKS cluster name name component used through out the EKS cluster describing its purpose (ex: dice-dev) | `string` | n/a | yes |
-| [karpenter\_helm\_chart](#input\_karpenter\_helm\_chart) | Which helm chart of karpenter | `string` | `"0.37.0"` | no |
+| [karpenter\_helm\_chart](#input\_karpenter\_helm\_chart) | Which helm chart of karpenter | `string` | `"1.0.0"` | no |
| [karpenter\_helm\_repo](#input\_karpenter\_helm\_repo) | Helm repo for official karpenter chart | `string` | `"oci://public.ecr.aws/karpenter"` | no |
-| [karpenter\_tag](#input\_karpenter\_tag) | Which tag of karpenter image | `string` | `"0.37.0"` | no |
+| [karpenter\_node\_group\_name](#input\_karpenter\_node\_group\_name) | The cluster node group that will host karpenter, should not be a karpenter managed node group | `string` | n/a | yes |
+| [karpenter\_tag](#input\_karpenter\_tag) | Which tag of karpenter image | `string` | `"1.0.0"` | no |
+| [kubectl\_tag](#input\_kubectl\_tag) | Which tag of kubectl image | `string` | `"1.30.4"` | no |
| [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes |
| [profile](#input\_profile) | AWS config profile | `string` | `""` | no |
| [region](#input\_region) | AWS region | `string` | n/a | yes |
+| [subnets\_name](#input\_subnets\_name) | Define the name of the subnets to be used by this cluster | `string` | `"*-container-*"` | no |
| [tags](#input\_tags) | AWS Tags to apply to appropriate resources | `map(string)` | `{}` | no |
+| [vpc\_id](#input\_vpc\_id) | vpc to use for subnet tags | `string` | `""` | no |
## Outputs
-No outputs.
+| Name | Description |
+|------|-------------|
+| [module\_name](#output\_module\_name) | The name of this module. |
+| [module\_version](#output\_module\_version) | The version of this module. |
diff --git a/aws_data.tf b/aws_data.tf
new file mode 100644
index 0000000..2c6aade
--- /dev/null
+++ b/aws_data.tf
@@ -0,0 +1,5 @@
+data "aws_caller_identity" "current" {}
+
+data "aws_arn" "current" {
+ arn = data.aws_caller_identity.current.arn
+}
diff --git a/copy_images.tf b/copy_images.tf
index 4c05c03..c98c243 100644
--- a/copy_images.tf
+++ b/copy_images.tf
@@ -1,5 +1,6 @@
locals {
karpenter_key = format("%v#%v", "karpenter/controller", var.karpenter_tag)
+ kubectl_key = format("%v#%v", "bitnami/kubectl", var.kubectl_tag)
image_config = [
{
@@ -11,6 +12,15 @@ locals {
source_tag = var.karpenter_tag
tag = var.karpenter_tag
},
+ {
+ enabled = true
+ dest_path = null
+ name = "bitnami/kubectl"
+ source_image = "bitnami/kubectl"
+ source_registry = "docker.io"
+ source_tag = var.kubectl_tag
+ tag = var.kubectl_tag
+ },
]
}
diff --git a/ec2_node_class.yaml.tpl b/ec2_node_class.yaml.tpl
new file mode 100644
index 0000000..4251292
--- /dev/null
+++ b/ec2_node_class.yaml.tpl
@@ -0,0 +1,71 @@
+---
+apiVersion: karpenter.k8s.aws/v1
+kind: EC2NodeClass
+metadata:
+ name: ${cluster_name}-karpenter-node-class
+ annotations:
+ kubernetes.io/description: "EC2NodeClass for running Bottlerocket nodes"
+spec:
+ kubelet:
+ podsPerCore: 2
+ maxPods: 20
+ systemReserved:
+ cpu: 100m
+ memory: 100Mi
+ ephemeral-storage: 1Gi
+ kubeReserved:
+ cpu: 200m
+ memory: 100Mi
+ ephemeral-storage: 3Gi
+ evictionHard:
+ memory.available: 5%
+ nodefs.available: 10%
+ nodefs.inodesFree: 10%
+ evictionSoft:
+ memory.available: 500Mi
+ nodefs.available: 15%
+ nodefs.inodesFree: 15%
+ evictionSoftGracePeriod:
+ memory.available: 1m
+ nodefs.available: 1m30s
+ nodefs.inodesFree: 2m
+ evictionMaxPodGracePeriod: 60
+ imageGCHighThresholdPercent: 85
+ imageGCLowThresholdPercent: 80
+ cpuCFSQuota: true
+
+ # Required, resolves a default ami and userdata
+ amiFamily: ${amd_ami_family}
+ amiSelectorTerms:
+ - alias: bottlerocket@latest # Bottlerocket
+ # Required, discovers subnets to attach to instances
+ # Each term in the array of subnetSelectorTerms is ORed together
+ # Within a single term, all conditions are ANDed
+ subnetSelectorTerms:
+ - tags:
+ Name: "*-container-*"
+ # karpenter.sh/discovery: "${cluster_name}"
+
+ # Required, discovers security groups to attach to instances
+ # Each term in the array of securityGroupSelectorTerms is ORed together
+ # Within a single term, all conditions are ANDed
+ securityGroupSelectorTerms:
+ - tags:
+ Name: "${cluster_name}-node"
+
+ # Optional, IAM instance profile to use for the node identity.
+ # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes
+ instanceProfile: "${cluster_name}-karpenter-instance-profile"
+
+ # Optional, use instance-store volumes for node ephemeral-storage
+ instanceStorePolicy: RAID0
+
+ # Optional, configures IMDS for the instance
+ metadataOptions:
+ httpEndpoint: enabled
+ httpProtocolIPv6: enabled
+ httpPutResponseHopLimit: 2
+ httpTokens: required
+
+ # Optional, configures detailed monitoring for the instance
+ detailedMonitoring: true
diff --git a/ec2_node_pool.yaml.tpl b/ec2_node_pool.yaml.tpl
new file mode 100644
index 0000000..0da0ee8
--- /dev/null
+++ b/ec2_node_pool.yaml.tpl
@@ -0,0 +1,34 @@
+---
+apiVersion: karpenter.sh/v1
+kind: NodePool
+metadata:
+ name: ${cluster_name}-node-pool-general
+ annotations:
+ kubernetes.io/description: "General purpose NodePool for generic workloads"
+spec:
+ template:
+ spec:
+ requirements:
+ - key: kubernetes.io/arch
+ operator: In
+ values: ["amd64"]
+ - key: kubernetes.io/os
+ operator: In
+ values: ["linux"]
+ - key: karpenter.sh/capacity-type
+ operator: In
+ values: ["on-demand", "spot"]
+ - key: karpenter.k8s.aws/instance-category
+ operator: In
+ values: ["c", "m", "r", "t"]
+ - key: karpenter.k8s.aws/instance-generation
+ operator: Gt
+ values: ["2"]
+ nodeClassRef:
+ group: karpenter.k8s.aws
+ kind: EC2NodeClass
+ name: ${cluster_name}-karpenter-node-class
+ disruption:
+ consolidationPolicy: WhenEmptyOrUnderutilized
+ consolidateAfter: 1h
+ expireAfter: 1h
diff --git a/examples/simple/README.md b/examples/simple/README.md
index 0cc9a49..2263b21 100644
--- a/examples/simple/README.md
+++ b/examples/simple/README.md
@@ -1,6 +1,6 @@
# Karpenter Example
-Configuration in this directory creates an AWS EKS cluster with Karpenter provisioned for managing compute resource scaling. In the example, Karpenter is provisioned on top of an EKS Managed Node Group following best practice recommendations.
+Configuration in this directory creates an AWS EKS cluster with Karpenter provisioned for managing compute resource scaling. In the example, Karpenter is provisioned on top of an EKS Managed Node Group following best practice recommendations.
## Usage
@@ -15,15 +15,15 @@ $ terraform apply
Verifying Karpenter is deployed:
```bash
-$ # Verify Karpenter controller pods
+$ # Verify Karpenter controller pods
$ kubectl get pods -n karpenter
-$
+$
$ # Review controller logs with
$ kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller
-$
+$
$ # Identify Karpenter managed nodes
$ kubectl get nodes -L karpenter.sh/registered
-```
+```
### Tear Down & Clean-Up
@@ -83,4 +83,4 @@ terraform destroy
## Outputs
No outputs.
-
\ No newline at end of file
+
diff --git a/examples/simple/karpenter.tf b/examples/simple/karpenter.tf
index ee22cd7..d11d114 100644
--- a/examples/simple/karpenter.tf
+++ b/examples/simple/karpenter.tf
@@ -9,4 +9,4 @@ module "karpenter" {
karpenter_node_group_name = var.karpenter_node_group_name
tags = var.tags
-}
\ No newline at end of file
+}
diff --git a/examples/simple/providers.tf b/examples/simple/providers.tf
index 02b5460..f0663ae 100644
--- a/examples/simple/providers.tf
+++ b/examples/simple/providers.tf
@@ -1,8 +1,21 @@
terraform {
required_version = ">= 1.5.0"
required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = ">= 5.14.0"
+ }
+ helm = {
+ source = "hashicorp/helm"
+ version = ">= 2.11.0"
+ }
kubectl = {
- source = "gavinbunney/kubectl"
+ source = "gavinbunney/kubectl"
+ version = ">= 1.14.0"
+ }
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = ">= 2.23.0"
}
}
}
@@ -41,4 +54,4 @@ provider "kubectl" {
cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
token = data.aws_eks_cluster_auth.cluster.token
load_config_file = false
-}
\ No newline at end of file
+}
diff --git a/examples/simple/variables.tf b/examples/simple/variables.tf
index e049b3f..c13a56c 100644
--- a/examples/simple/variables.tf
+++ b/examples/simple/variables.tf
@@ -33,4 +33,4 @@ variable "tags" {
description = "AWS Tags to apply to appropriate resources"
type = map(string)
default = {}
-}
\ No newline at end of file
+}
diff --git a/examples/terragrunt.hcl b/examples/terragrunt.hcl
new file mode 100644
index 0000000..aa1a6b3
--- /dev/null
+++ b/examples/terragrunt.hcl
@@ -0,0 +1,24 @@
+# include "root" {
+# path = find_in_parent_folders()
+# }
+
+terraform {
+ source = "../../tfmod-karpenter"
+ extra_arguments "retry_lock" {
+ commands = get_terraform_commands_that_need_locking()
+ arguments = ["-lock-timeout=20m"]
+ }
+}
+
+dependency "eks" {
+ config_path = "../../tfmod-eks"
+}
+
+inputs = {
+ cluster_endpoint = dependency.eks.outputs.cluster_endpoint
+ cluster_name = dependency.eks.inputs.cluster_name
+ karpenter_node_group_name = dependency.eks.outputs.node_group_name
+ oidc_provider_arn = dependency.eks.outputs.oidc_provider_arn
+ profile = dependency.eks.inputs.profile
+ region = dependency.eks.inputs.region
+}
diff --git a/main.tf b/main.tf
index 94ff312..4d2d1ab 100644
--- a/main.tf
+++ b/main.tf
@@ -1,24 +1,46 @@
locals {
- namespace = "karpenter"
+ namespace = "karpenter"
+ container_subnets_name = var.subnets_name
+ vpc_id = var.vpc_id
+}
+
+data "aws_subnets" "container_subnets" {
+ filter {
+ name = "tag:Name"
+ values = [local.container_subnets_name]
+ }
+ filter {
+ name = "vpc-id"
+ values = [local.vpc_id]
+ }
}
# Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter
# Karpenter IRSA roles and policies
module "karpenter_resources" {
- source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.20.0"
+ source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.23.0"
- cluster_name = var.cluster_name
- enable_irsa = true
- irsa_oidc_provider_arn = var.oidc_provider_arn
+ cluster_name = var.cluster_name
+ enable_irsa = true
+ irsa_oidc_provider_arn = var.oidc_provider_arn
+ create_pod_identity_association = true
# Attach additional IAM policies to the Karpenter node IAM role
node_iam_role_additional_policies = {
- AmazonSSMManagedInstanceCore = "arn:aws-us-gov:iam::aws:policy/AmazonSSMManagedInstanceCore"
+ AmazonSSMManagedInstanceCore = format("arn:%v:iam::%v:%v", data.aws_arn.current.partition, "aws", "policy/AmazonSSMManagedInstanceCore")
}
tags = var.tags
}
+# Tag subnets for Karpenter Discovery
+resource "aws_ec2_tag" "container_subnets" {
+ for_each = toset(data.aws_subnets.container_subnets.ids)
+ resource_id = each.value
+ key = "karpenter.sh/discovery"
+ value = var.cluster_name
+}
+
# Install Karpenter
data "aws_ecr_image" "karpenter_image" {
depends_on = [
@@ -28,10 +50,18 @@ data "aws_ecr_image" "karpenter_image" {
image_tag = module.images.images[local.karpenter_key].tag
}
-resource "helm_release" "karpenter" {
+data "aws_ecr_image" "kubectl_image" {
depends_on = [
module.images,
]
+ repository_name = module.images.images[local.kubectl_key].dest_repository
+ image_tag = module.images.images[local.kubectl_key].tag
+}
+
+resource "helm_release" "karpenter" {
+ depends_on = [
+ data.aws_ecr_image.karpenter_image
+ ]
name = "karpenter"
chart = "karpenter"
version = var.karpenter_helm_chart
@@ -72,6 +102,10 @@ resource "helm_release" "karpenter" {
name = "settings.clusterEndpoint"
value = var.cluster_endpoint
}
+ set {
+ name = "settings.featureGates.spotToSpotConsolidation"
+ value = "true"
+ }
set {
name = "settings.interruptionQueue"
value = module.karpenter_resources.queue_name
@@ -95,6 +129,21 @@ resource "helm_release" "karpenter" {
name = "controller.image.digest"
value = data.aws_ecr_image.karpenter_image.id
}
+ set {
+ name = "postInstallHook.image.repository"
+ value = format("%v/%v",
+ module.images.images[local.kubectl_key].dest_registry,
+ module.images.images[local.kubectl_key].dest_repository
+ )
+ }
+ set {
+ name = "postInstallHook.image.tag"
+ value = module.images.images[local.kubectl_key].tag
+ }
+ set {
+ name = "postInstallHook.image.digest"
+ value = data.aws_ecr_image.kubectl_image.id
+ }
set {
name = "controller.env[0].name"
value = "AWS_REGION"
@@ -127,43 +176,42 @@ resource "helm_release" "karpenter" {
timeout = 300
}
-
-resource "aws_iam_instance_profile" "KarpenterNodeInstanceProfile" {
- name = "KarpenterNodeInstanceProfile-${var.cluster_name}"
+resource "aws_iam_instance_profile" "karpenter_node_instance_profile" {
+ name = "${var.cluster_name}-karpenter-instance-profile"
role = module.karpenter_resources.node_iam_role_name
}
-# Permission for spot
+# Query to see if service-linked role exists
+data "aws_iam_roles" "roles" {
+ path_prefix = "/aws-reserved/spot.amazonaws.com/${var.cluster_name}"
+}
+
+# Permission for spot
resource "aws_iam_service_linked_role" "ec2_spot" {
+ count = data.aws_iam_roles.roles == null ? 1 : 0
aws_service_name = "spot.amazonaws.com"
+ custom_suffix = var.cluster_name
}
-# Lookup eks amis // didn't use this but would be path to get to mixed architectures/processors
-# data "aws_ssm_parameter" "arm_ami_id" {
-# name = "/aws/service/eks/optimized-ami/1.30/amazon-linux-2-arm64/recommended/image_id"
-# }
-
-# data "aws_ssm_parameter" "amd_ami_id" {
-# name = "/aws/service/eks/optimized-ami/1.30/amazon-linux-2/recommended/image_id"
-# }
-
# SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART
-# Create karpenter default
-resource "kubectl_manifest" "karpenter_EC2NodeClass" {
+# Create karpenter default
+resource "kubectl_manifest" "karpenter_ec2_node_class" {
depends_on = [
helm_release.karpenter,
]
- yaml_body = templatefile("${path.module}/EC2NodeClass.yaml.tpl", {
- cluster_name = var.cluster_name
- tags = jsonencode(var.tags)
+ yaml_body = templatefile("${path.module}/ec2_node_class.yaml.tpl", {
+ cluster_name = var.cluster_name
+ amd_ami_family = "Bottlerocket"
})
}
-# Create karpenter default NodePool
-resource "kubectl_manifest" "karpenter_NodePool" {
+# Create karpenter default NodePool
+resource "kubectl_manifest" "karpenter_node_pool" {
depends_on = [
- kubectl_manifest.karpenter_EC2NodeClass,
+ kubectl_manifest.karpenter_ec2_node_class,
]
- yaml_body = templatefile("${path.module}/EC2NodePool.yaml.tpl", {})
+ yaml_body = templatefile("${path.module}/ec2_node_pool.yaml.tpl", {
+ cluster_name = var.cluster_name
+ })
}
diff --git a/outputs.tf b/outputs.tf
index 0349496..b999ac9 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -4,10 +4,10 @@
output "module_name" {
description = "The name of this module."
- value = local._module_name
+ value = local.module_name
}
output "module_version" {
description = "The version of this module."
- value = local._module_version
+ value = local.module_version
}
diff --git a/requirements.tf b/requirements.tf
index 8c1f05b..4eab89e 100644
--- a/requirements.tf
+++ b/requirements.tf
@@ -14,10 +14,5 @@ terraform {
source = "gavinbunney/kubectl"
version = ">= 1.14.0"
}
- kubernetes = {
- source = "hashicorp/kubernetes"
- version = ">= 2.23.0"
- }
-
}
}
diff --git a/variables.tf b/variables.tf
index 1150802..03fe6f9 100644
--- a/variables.tf
+++ b/variables.tf
@@ -39,13 +39,25 @@ variable "karpenter_helm_repo" {
variable "karpenter_helm_chart" {
description = "Which helm chart of karpenter"
type = string
- default = "0.37.0"
+ default = "1.0.0"
}
variable "karpenter_tag" {
description = "Which tag of karpenter image"
type = string
- default = "0.37.0"
+ default = "1.0.0"
+}
+
+variable "kubectl_tag" {
+ description = "Which tag of kubectl image"
+ type = string
+ default = "1.30.4"
+}
+
+variable "subnets_name" {
+ description = "Define the name of the subnets to be used by this cluster"
+ type = string
+ default = "*-container-*"
}
variable "tags" {
@@ -53,3 +65,9 @@ variable "tags" {
type = map(string)
default = {}
}
+
+variable "vpc_id" {
+ description = "vpc to use for subnet tags"
+ type = string
+ default = ""
+}
diff --git a/version.tf b/version.tf
index 5970ee3..3ae0d94 100644
--- a/version.tf
+++ b/version.tf
@@ -1,4 +1,4 @@
locals {
- _module_name = "tfmod-karpenter"
- _module_version = "0.0.2"
+ module_name = "tfmod-karpenter"
+ module_version = "0.0.2"
}