From 2d2d7897a69344095adc8b60148dac663e2cee46 Mon Sep 17 00:00:00 2001 From: mcgin314 Date: Wed, 24 Jul 2024 13:38:50 -0400 Subject: [PATCH] Genesis/Initial development --- .gitignore | 38 +++++++ EC2NodeClass.yaml.tpl | 14 +++ README.md | 70 +++++++++++++ copy_images.tf | 27 +++++ examples/simple/karpenter.tf | 12 +++ examples/simple/providers.tf | 44 ++++++++ examples/simple/variables.tf | 36 +++++++ main.tf | 194 +++++++++++++++++++++++++++++++++++ requirements.tf | 23 +++++ variables.tf | 55 ++++++++++ 10 files changed, 513 insertions(+) create mode 100644 .gitignore create mode 100644 EC2NodeClass.yaml.tpl create mode 100644 README.md create mode 100644 copy_images.tf create mode 100644 examples/simple/karpenter.tf create mode 100644 examples/simple/providers.tf create mode 100644 examples/simple/variables.tf create mode 100644 main.tf create mode 100644 requirements.tf create mode 100644 variables.tf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7c1cae9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,38 @@ +# Local .terraform directories +**/.terraform/* + +# terraform lock file. +**/.terraform.lock.hcl + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, +# such as password, private keys, and other secrets. These should not be +# part of version control as they are data points which are potentially +# sensitive and subject to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources +# locally and so are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + diff --git a/EC2NodeClass.yaml.tpl b/EC2NodeClass.yaml.tpl new file mode 100644 index 0000000..4ba530d --- /dev/null +++ b/EC2NodeClass.yaml.tpl @@ -0,0 +1,14 @@ +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiFamily: AL2 + instanceProfile: "KarpenterNodeInstanceProfile-${cluster_name}" + subnetSelectorTerms: + - tags: + Name: "*-container-*" + securityGroupSelectorTerms: + - tags: + Name: "${cluster_name}-node" + tags: ${tags} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..40bbb85 --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +# tfmod-karpenter + +Karpenter is an open source tool developed by AWS for cluster autoscaling (Horizontal Cluster Autoscaling - HCA). It automatically provisions and manages compute resources lifecycle by monitoring the scheduling status of pods and dynamically adjusting the cluster size with an efficient approach to utilization and cost. Effectly, Karpenter becomes a replacement to the legacy Cluster Autoscaler and is preferred given it improves flexibility and is more cluster aware. More specifically, Karpenter was designed to overcome some of the challenges presented by Cluster Autoscaler by providing simplified ways to: + +- Provision nodes based on workload requirements. +- Create diverse node configurations by instance type, using flexible workload provisioner options. Instead of managing many specific custom node groups, Karpenter allows diverse workload capacity with a single, flexible provisioner. +- Achieve improved pod scheduling at scale by quickly launching nodes and scheduling pods. + +ref: [Karpenter Best Practices](https://aws.github.io/aws-eks-best-practices/karpenter/) + +# CHANGELOG + +* 0.0.1 -- 2024-07-23 + - Genesis/Initial development + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13 | +| [aws](#requirement\_aws) | >= 5.14.0 | +| [helm](#requirement\_helm) | >= 2.11.0 | +| [kubectl](#requirement\_kubectl) | >= 1.14.0 | +| [kubernetes](#requirement\_kubernetes) | >= 2.23.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 5.14.0 | +| [helm](#provider\_helm) | >= 2.11.0 | +| [kubectl](#provider\_kubectl) | >= 1.14.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [images](#module\_images) | git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/ | 2.0.2 | +| [karpenter\_resources](#module\_karpenter\_resources) | git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter | v20.8.5 | + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_instance_profile.KarpenterNodeInstanceProfile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_service_linked_role.ec2_spot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_service_linked_role) | resource | +| [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubectl_manifest.karpenter_EC2NodeClass](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_NodePool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [aws_ecr_image.karpenter_image](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint for your Kubernetes API server | `string` | n/a | yes | +| [cluster\_name](#input\_cluster\_name) | EKS cluster name name component used through out the EKS cluster describing its purpose (ex: dice-dev) | `string` | n/a | yes | +| [karpenter\_helm\_chart](#input\_karpenter\_helm\_chart) | Which helm chart of karpenter | `string` | `"0.37.0"` | no | +| [karpenter\_helm\_repo](#input\_karpenter\_helm\_repo) | Helm repo for official karpenter chart | `string` | `"oci://public.ecr.aws/karpenter"` | no | +| [karpenter\_tag](#input\_karpenter\_tag) | Which tag of karpenter image | `string` | `"0.37.0"` | no | +| [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | `string` | n/a | yes | +| [profile](#input\_profile) | AWS config profile | `string` | `""` | no | +| [region](#input\_region) | AWS region | `string` | n/a | yes | +| [tags](#input\_tags) | AWS Tags to apply to appropriate resources | `map(string)` | `{}` | no | + +## Outputs + +No outputs. + \ No newline at end of file diff --git a/copy_images.tf b/copy_images.tf new file mode 100644 index 0000000..00f6212 --- /dev/null +++ b/copy_images.tf @@ -0,0 +1,27 @@ +locals { + karpenter_key = format("%v#%v", "karpenter/controller", var.karpenter_tag) + + image_config = [ + { + enabled = true + dest_path = null + name = "karpenter/controller" + source_image = "karpenter/controller" + source_registry = "public.ecr.aws" + source_tag = var.karpenter_tag + tag = var.karpenter_tag + }, + ] +} + +# 224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter:0.37.0 +# map[repository:224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-1/karpenter tag:0.37.0] + +module "images" { + source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=2.0.2" + + profile = var.profile + application_name = var.cluster_name + image_config = local.image_config + tags = {} +} diff --git a/examples/simple/karpenter.tf b/examples/simple/karpenter.tf new file mode 100644 index 0000000..ee22cd7 --- /dev/null +++ b/examples/simple/karpenter.tf @@ -0,0 +1,12 @@ +module "karpenter" { + source = "../.." + + region = var.region + profile = var.profile + cluster_name = var.cluster_name + oidc_provider_arn = var.oidc_provider_arn + cluster_endpoint = var.cluster_endpoint + karpenter_node_group_name = var.karpenter_node_group_name + + tags = var.tags +} \ No newline at end of file diff --git a/examples/simple/providers.tf b/examples/simple/providers.tf new file mode 100644 index 0000000..02b5460 --- /dev/null +++ b/examples/simple/providers.tf @@ -0,0 +1,44 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + kubectl = { + source = "gavinbunney/kubectl" + } + } +} + +provider "aws" { + profile = var.profile + region = var.region +} + +data "aws_eks_cluster" "cluster" { + name = var.cluster_name +} + +data "aws_eks_cluster_auth" "cluster" { + name = var.cluster_name +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.cluster.endpoint + + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token + } +} + +provider "kubectl" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false +} \ No newline at end of file diff --git a/examples/simple/variables.tf b/examples/simple/variables.tf new file mode 100644 index 0000000..e049b3f --- /dev/null +++ b/examples/simple/variables.tf @@ -0,0 +1,36 @@ +variable "region" { + description = "AWS region" + type = string +} + +variable "profile" { + description = "AWS config profile" + type = string + default = "" +} + +variable "cluster_name" { + description = "EKS cluster name name component used through out the EKS cluster describing its purpose (ex: dice-dev)" + type = string +} + +variable "cluster_endpoint" { + description = "Endpoint for your Kubernetes API server" + type = string +} + +variable "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + type = string +} + +variable "karpenter_node_group_name" { + description = "The cluster node group that will host karpenter, should not be a karpenter managed node group" + type = string +} + +variable "tags" { + description = "AWS Tags to apply to appropriate resources" + type = map(string) + default = {} +} \ No newline at end of file diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..cee21c8 --- /dev/null +++ b/main.tf @@ -0,0 +1,194 @@ +locals { + namespace = "karpenter" +} + +# Replicating from here: https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/tree/master/modules/karpenter +# Karpenter IRSA roles and policies +module "karpenter_resources" { + source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git//modules/karpenter?ref=v20.8.5" + + cluster_name = var.cluster_name + enable_irsa = true + irsa_oidc_provider_arn = var.oidc_provider_arn + + # Attach additional IAM policies to the Karpenter node IAM role + node_iam_role_additional_policies = { + AmazonSSMManagedInstanceCore = "arn:aws-us-gov:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + + tags = var.tags +} + +# Install Karpenter +data "aws_ecr_image" "karpenter_image" { + depends_on = [ + module.images, + ] + repository_name = module.images.images[local.karpenter_key].dest_repository + image_tag = module.images.images[local.karpenter_key].tag +} + +resource "helm_release" "karpenter" { + depends_on = [ + module.images, + ] + + name = "karpenter" + chart = "karpenter" + version = var.karpenter_helm_chart + namespace = local.namespace + description = "Kubernetes Node Autoscaling: built for flexibility, performance, and simplicity" + repository = var.karpenter_helm_repo + create_namespace = true + + set { + name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key" + value = "karpenter.sh/nodepool" + } + set { + name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator" + value = "DoesNotExist" + } + set { + name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].key" + value = "eks.amazonaws.com/nodegroup" + } + set { + name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].operator" + value = "In" + } + set { + name = "affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[1].matchExpressions[0].values[0]" + value = var.karpenter_node_group_name + } + set { + name = "affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey" + value = "kubernetes.io/hostname" + } + set { + name = "settings.clusterName" + value = var.cluster_name + } + set { + name = "settings.clusterEndpoint" + value = var.cluster_endpoint + } + set { + name = "settings.interruptionQueue" + value = module.karpenter_resources.queue_name + } + set { + name = "settings.isolatedVPC" + value = "true" + } + set { + name = "controller.image.repository" + value = format("%v/%v", + module.images.images[local.karpenter_key].dest_registry, + module.images.images[local.karpenter_key].dest_repository + ) + } + set { + name = "controller.image.tag" + value = module.images.images[local.karpenter_key].tag + } + set { + name = "controller.image.digest" + value = data.aws_ecr_image.karpenter_image.id + } + set { + name = "controller.env[0].name" + value = "AWS_REGION" + } + set { + name = "controller.env[0].value" + value = var.region + } + set { + name = "resources.requests.cpu" + value = 1 + } + set { + name = "resources.requests.memory" + value = "1Gi" + } + set { + name = "resources.limits.cpu" + value = 1 + } + set { + name = "resources.limits.memory" + value = "1Gi" + } + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.karpenter_resources.iam_role_arn + } + + timeout = 300 +} + + +resource "aws_iam_instance_profile" "KarpenterNodeInstanceProfile" { + name = "KarpenterNodeInstanceProfile-${var.cluster_name}" + role = module.karpenter_resources.node_iam_role_name +} + +# Permission for spot +resource "aws_iam_service_linked_role" "ec2_spot" { + aws_service_name = "spot.amazonaws.com" +} + + +# Lookup eks amis // didn't use this but would be path to get to mixed architectures/processors +# data "aws_ssm_parameter" "arm_ami_id" { +# name = "/aws/service/eks/optimized-ami/1.30/amazon-linux-2-arm64/recommended/image_id" +# } + +# data "aws_ssm_parameter" "amd_ami_id" { +# name = "/aws/service/eks/optimized-ami/1.30/amazon-linux-2/recommended/image_id" +# } + +# SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART +# Create karpenter default +resource "kubectl_manifest" "karpenter_EC2NodeClass" { + depends_on = [ + helm_release.karpenter, + ] + yaml_body = templatefile("${path.module}/EC2NodeClass.yaml.tpl", { + cluster_name = var.cluster_name + tags = jsonencode(var.tags) + }) +} + +# Create karpenter default NodePool +resource "kubectl_manifest" "karpenter_NodePool" { + depends_on = [ + kubectl_manifest.karpenter_EC2NodeClass, + ] + + yaml_body = <