From 78ba3bc996e2a1c61ea74c128614a70110b7d18a Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 16 Dec 2025 19:23:36 -0500 Subject: [PATCH 01/27] create new cluster to test --- examples/basic/main.tf | 3 +++ examples/basic/providers.tf | 2 -- examples/csvd-lab-mcm/main.tf | 48 +++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) delete mode 100644 examples/basic/providers.tf create mode 100644 examples/csvd-lab-mcm/main.tf diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 4bef0dd..c2f6672 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -1,3 +1,6 @@ +provider "aws" { +} + data "aws_secretsmanager_secret_version" "github_token" { secret_id = "/eks-cluster-deployment/github_token" } diff --git a/examples/basic/providers.tf b/examples/basic/providers.tf deleted file mode 100644 index 8c9ebd1..0000000 --- a/examples/basic/providers.tf +++ /dev/null @@ -1,2 +0,0 @@ -provider "aws" { -} diff --git a/examples/csvd-lab-mcm/main.tf b/examples/csvd-lab-mcm/main.tf new file mode 100644 index 0000000..a38f90a --- /dev/null +++ b/examples/csvd-lab-mcm/main.tf @@ -0,0 +1,48 @@ +provider "aws" { +} + +data "aws_secretsmanager_secret_version" "github_token" { + secret_id = "/eks-cluster-deployment/github_token" +} + +provider "github" { + token = data.aws_secretsmanager_secret_version.github_token.secret_string +} + +module "eks_deployment" { + source = "../../" + + # Repository and cluster configuration - single name for both + name = "csvd-lab-mcm" + environment = "dev" + region = "us-gov-east-1" + + # Cluster configuration - simplified interface + cluster_config = { + account_name = "lab-dev-gov" + aws_account_id = "224384469011" + cluster_mailing_list = "matthew.c.morgan@census.gov" + environment_abbr = "dev" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_eks" + vpc_domain_name = "dev.lab.csp2.census.gov" + vpc_name = "vpc3-lab-dev" + tags = { + Owner = "matthew.c.morgan@census.gov" + Environment = "development" + CostCenter = "fs0000000078" + } + organization = "census:ocio:csvd" + } +} + +output "repository_url" { + description = "URL of the created GitHub repository" + value = module.eks_deployment.repository_url +} + +output "ssh_clone_url" { + description = "SSH clone URL of the repository" + value = module.eks_deployment.ssh_clone_url +} From dbfc11f8ba2f26a22597b2700328f30a92d41c16 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 17 Dec 2025 12:21:44 -0500 Subject: [PATCH 02/27] update defaults to min/desired 2 --- defaults.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/defaults.tf b/defaults.tf index 5842ce5..45c1ee3 100644 --- a/defaults.tf +++ b/defaults.tf @@ -14,9 +14,9 @@ locals { # Static EKS configuration for Karpenter bootstrap node group eks_defaults = { instance_disk_size = 200 - ng_desired_size = 3 + ng_desired_size = 2 ng_max_size = 10 - ng_min_size = 3 + ng_min_size = 2 enable_cluster_creator_admin_permissions = true } @@ -26,4 +26,4 @@ locals { finops_project_number = var.finops.project_number finops_project_role = var.finops.project_role } -} \ No newline at end of file +} From e6ccb4b0b12885311b1c441871792088b1bbeb84 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 20 Feb 2026 13:15:22 -0500 Subject: [PATCH 03/27] make csvd-dev-mcm cluster for testing --- examples/csvd-dev-mcm/main.tf | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 examples/csvd-dev-mcm/main.tf diff --git a/examples/csvd-dev-mcm/main.tf b/examples/csvd-dev-mcm/main.tf new file mode 100644 index 0000000..2d76ae0 --- /dev/null +++ b/examples/csvd-dev-mcm/main.tf @@ -0,0 +1,48 @@ +provider "aws" { +} + +data "aws_secretsmanager_secret_version" "github_token" { + secret_id = "/eks-cluster-deployment/github_token" +} + +provider "github" { + token = data.aws_secretsmanager_secret_version.github_token.secret_string +} + +module "eks_deployment" { + source = "../../" + + # Repository and cluster configuration - single name for both + name = "csvd-dev-mcm" + environment = "dev" + region = "us-gov-east-1" + + # Cluster configuration - simplified interface + cluster_config = { + account_name = "csvd-dev-gov" + aws_account_id = "229685449397" + cluster_mailing_list = "matthew.c.morgan@census.gov" + environment_abbr = "dev" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_eks" + vpc_domain_name = "dev.csvd.csp1.census.gov" + vpc_name = "vpc2-csvd-dev" + tags = { + Owner = "matthew.c.morgan@census.gov" + Environment = "development" + CostCenter = "fs0000000078" + } + organization = "census:ocio:csvd" + } +} + +output "repository_url" { + description = "URL of the created GitHub repository" + value = module.eks_deployment.repository_url +} + +output "ssh_clone_url" { + description = "SSH clone URL of the repository" + value = module.eks_deployment.ssh_clone_url +} From 8ba0a99207da3001ddff79b738f7d00cd8ba601b Mon Sep 17 00:00:00 2001 From: Matthew Creal Morgan Date: Fri, 20 Feb 2026 11:11:02 -0800 Subject: [PATCH 04/27] add cluster for adsd-etdsb-tools-nonprod (#13) --- examples/adsd-etdsb-tools-nonprod/main.tf | 48 +++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 examples/adsd-etdsb-tools-nonprod/main.tf diff --git a/examples/adsd-etdsb-tools-nonprod/main.tf b/examples/adsd-etdsb-tools-nonprod/main.tf new file mode 100644 index 0000000..fedf93b --- /dev/null +++ b/examples/adsd-etdsb-tools-nonprod/main.tf @@ -0,0 +1,48 @@ +provider "aws" { +} + +data "aws_secretsmanager_secret_version" "github_token" { + secret_id = "/eks-cluster-deployment/github_token" +} + +provider "github" { + token = data.aws_secretsmanager_secret_version.github_token.secret_string +} + +module "eks_deployment" { + source = "../../" + + # Repository and cluster configuration - single name for both + name = "adsd-etdsb-tools-nonprod" + environment = "dev" + region = "us-gov-east-1" + + # Cluster configuration - simplified interface + cluster_config = { + account_name = "adsd-tools-nonprod-gov" + aws_account_id = "533109815932" + cluster_mailing_list = "vijaya.pavuluru@census.gov" + environment_abbr = "dev" + finops_project_name = "adsd_etdsb_tools_migration" + finops_project_number = "fs0000000069" + finops_project_role = "adsd_tools_mgrn_eks" + vpc_domain_name = "dev.adsd.csp1.census.gov" + vpc_name = "vpc3-inf-dev" + tags = { + Owner = "vijaya.pavuluru@census.gov" + Environment = "development" + CostCenter = "census:ocio:adsd" + } + organization = "census:ocio:adsd" + } +} + +output "repository_url" { + description = "URL of the created GitHub repository" + value = module.eks_deployment.repository_url +} + +output "ssh_clone_url" { + description = "SSH clone URL of the repository" + value = module.eks_deployment.ssh_clone_url +} From d58dd54076c1045cc7b4b2d144e112aa1f45c70c Mon Sep 17 00:00:00 2001 From: Matthew Creal Morgan Date: Fri, 20 Feb 2026 12:21:56 -0800 Subject: [PATCH 05/27] rename cluster in convo with customer (#14) --- .../main.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename examples/{adsd-etdsb-tools-nonprod => adsd-tools-dev}/main.tf (82%) diff --git a/examples/adsd-etdsb-tools-nonprod/main.tf b/examples/adsd-tools-dev/main.tf similarity index 82% rename from examples/adsd-etdsb-tools-nonprod/main.tf rename to examples/adsd-tools-dev/main.tf index fedf93b..1f072fc 100644 --- a/examples/adsd-etdsb-tools-nonprod/main.tf +++ b/examples/adsd-tools-dev/main.tf @@ -13,23 +13,23 @@ module "eks_deployment" { source = "../../" # Repository and cluster configuration - single name for both - name = "adsd-etdsb-tools-nonprod" - environment = "dev" + name = "adsd-tools-dev" + environment = "prod" region = "us-gov-east-1" # Cluster configuration - simplified interface cluster_config = { account_name = "adsd-tools-nonprod-gov" aws_account_id = "533109815932" - cluster_mailing_list = "vijaya.pavuluru@census.gov" - environment_abbr = "dev" + cluster_mailing_list = "adsd.enterprise.tools.support.branch.list@census.gov" + environment_abbr = "prod" finops_project_name = "adsd_etdsb_tools_migration" finops_project_number = "fs0000000069" finops_project_role = "adsd_tools_mgrn_eks" vpc_domain_name = "dev.adsd.csp1.census.gov" vpc_name = "vpc3-inf-dev" tags = { - Owner = "vijaya.pavuluru@census.gov" + Owner = "adsd.enterprise.tools.support.branch.list@census.gov" Environment = "development" CostCenter = "census:ocio:adsd" } From cb824b6adcda368f9cfc5fdc2c42d45ee8a58586 Mon Sep 17 00:00:00 2001 From: Matthew Creal Morgan Date: Fri, 6 Mar 2026 09:22:04 -0800 Subject: [PATCH 06/27] Rename adsd tools (#15) * rename cluster in convo with customer * create cluster in csvd-common for testing --- examples/csvd-mcm-common/main.tf | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 examples/csvd-mcm-common/main.tf diff --git a/examples/csvd-mcm-common/main.tf b/examples/csvd-mcm-common/main.tf new file mode 100644 index 0000000..c6b13fb --- /dev/null +++ b/examples/csvd-mcm-common/main.tf @@ -0,0 +1,48 @@ +provider "aws" { +} + +data "aws_secretsmanager_secret_version" "github_token" { + secret_id = "/eks-cluster-deployment/github_token" +} + +provider "github" { + token = data.aws_secretsmanager_secret_version.github_token.secret_string +} + +module "eks_deployment" { + source = "../../" + + # Repository and cluster configuration - single name for both + name = "csvd-mcm-common" + environment = "prod" + region = "us-gov-east-1" + + # Cluster configuration - simplified interface + cluster_config = { + account_name = "csvd-mcm-common" + aws_account_id = "220615867784" + cluster_mailing_list = "matthew.c.morgan@census.gov" + environment_abbr = "prod" + finops_project_name = "csvd_platformbaseline", + finops_project_number = "fs0000000078", + finops_project_role = "csvd_platformbaseline_app", + vpc_domain_name = "shared.inf.csp1.census.gov" + vpc_name = "vpc2-inf-shared" + tags = { + Owner = "matthew.c.morgan@census.gov" + Environment = "development" + CostCenter = "census:ocio:csvd" + } + organization = "census:ocio:csvd" + } +} + +output "repository_url" { + description = "URL of the created GitHub repository" + value = module.eks_deployment.repository_url +} + +output "ssh_clone_url" { + description = "SSH clone URL of the repository" + value = module.eks_deployment.ssh_clone_url +} From 99b148dc26bc164cf319a4bcfef3d760670f9b70 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 11 Mar 2026 18:12:21 -0400 Subject: [PATCH 07/27] update cluster workflow --- README.md | 31 +++++++++++++++++++++++++++++++ examples/adsd-tools-dev/main.tf | 16 ++++++++-------- examples/basic/main.tf | 16 ++++++++-------- examples/csvd-dev-mcm/main.tf | 25 ++++++++++++++----------- examples/csvd-lab-mcm/main.tf | 16 ++++++++-------- locals.tf | 11 ----------- main.tf | 24 ++++++++++++++++++++++-- templates/README.md.tf.tpl | 4 ++-- templates/account.hcl.tf.tpl | 2 +- templates/default-versions.hcl | 13 ------------- variables.tf | 14 +++++++++++++- 11 files changed, 107 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 0b51d07..1f482c0 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,37 @@ output "repository_url" { } ``` +### Update an Existing Repository on a PR Branch + +Use this mode when the cluster repository already exists and you only want to update managed files on a branch for a pull request. + +```hcl +module "eks_deployment" { + source = "../../" + + name = "csvd-dev-mcm" + organization = "SCT-Engineering" + create_repository = false + files_branch = "update/csvd-dev-mcm" + + environment = "dev" + region = "us-gov-east-1" + + cluster_config = { + account_name = "csvd-dev-gov" + aws_account_id = "229685449397" + environment_abbr = "dev" + vpc_name = "vpc2-csvd-dev" + vpc_domain_name = "dev.csvd.csp1.census.gov" + cluster_mailing_list = "matthew.c.morgan@census.gov" + } +} +``` + +If `files_branch` does not already exist, create it first from your default branch, then run `terraform apply`. + +By default, this module now attempts to create `files_branch` automatically when `create_repository = false`, using the repository default branch as the source. You can override the source with `files_branch_source_branch`. + ## Generated Files The module automatically generates the following files in your new repository: diff --git a/examples/adsd-tools-dev/main.tf b/examples/adsd-tools-dev/main.tf index 1f072fc..be56914 100644 --- a/examples/adsd-tools-dev/main.tf +++ b/examples/adsd-tools-dev/main.tf @@ -19,15 +19,15 @@ module "eks_deployment" { # Cluster configuration - simplified interface cluster_config = { - account_name = "adsd-tools-nonprod-gov" - aws_account_id = "533109815932" - cluster_mailing_list = "adsd.enterprise.tools.support.branch.list@census.gov" - environment_abbr = "prod" + account_name = "adsd-tools-nonprod-gov" + aws_account_id = "533109815932" + cluster_mailing_list = "adsd.enterprise.tools.support.branch.list@census.gov" + environment_abbr = "prod" finops_project_name = "adsd_etdsb_tools_migration" - finops_project_number = "fs0000000069" - finops_project_role = "adsd_tools_mgrn_eks" - vpc_domain_name = "dev.adsd.csp1.census.gov" - vpc_name = "vpc3-inf-dev" + finops_project_number = "fs0000000069" + finops_project_role = "adsd_tools_mgrn_eks" + vpc_domain_name = "dev.adsd.csp1.census.gov" + vpc_name = "vpc3-inf-dev" tags = { Owner = "adsd.enterprise.tools.support.branch.list@census.gov" Environment = "development" diff --git a/examples/basic/main.tf b/examples/basic/main.tf index c2f6672..02c1152 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -19,15 +19,15 @@ module "eks_deployment" { # Cluster configuration - simplified interface cluster_config = { - account_name = "ma6-gov" - aws_account_id = "252960665057" - cluster_mailing_list = "adep.mojo.development.list@census.gov" - environment_abbr = "dev" + account_name = "ma6-gov" + aws_account_id = "252960665057" + cluster_mailing_list = "adep.mojo.development.list@census.gov" + environment_abbr = "dev" finops_project_name = "PPSI_DICE" - finops_project_number = "fs0000000015" - finops_project_role = "dice:dev:mojo" - vpc_domain_name = "dev.dice.census.gov" - vpc_name = "vpc2-dice-dev" + finops_project_number = "fs0000000015" + finops_project_role = "dice:dev:mojo" + vpc_domain_name = "dev.dice.census.gov" + vpc_name = "vpc2-dice-dev" tags = { Owner = "PETeam" Environment = "Development" diff --git a/examples/csvd-dev-mcm/main.tf b/examples/csvd-dev-mcm/main.tf index 2d76ae0..3da1012 100644 --- a/examples/csvd-dev-mcm/main.tf +++ b/examples/csvd-dev-mcm/main.tf @@ -13,21 +13,24 @@ module "eks_deployment" { source = "../../" # Repository and cluster configuration - single name for both - name = "csvd-dev-mcm" - environment = "dev" - region = "us-gov-east-1" + name = "csvd-dev-mcm" + organization = "SCT-Engineering" + create_repository = false + files_branch = "update/csvd-dev-mcm" + environment = "dev" + region = "us-gov-east-1" # Cluster configuration - simplified interface cluster_config = { - account_name = "csvd-dev-gov" - aws_account_id = "229685449397" - cluster_mailing_list = "matthew.c.morgan@census.gov" - environment_abbr = "dev" + account_name = "csvd-dev-gov" + aws_account_id = "229685449397" + cluster_mailing_list = "matthew.c.morgan@census.gov" + environment_abbr = "dev" finops_project_name = "csvd_platformbaseline" - finops_project_number = "fs0000000078" - finops_project_role = "csvd_platformbaseline_eks" - vpc_domain_name = "dev.csvd.csp1.census.gov" - vpc_name = "vpc2-csvd-dev" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_eks" + vpc_domain_name = "dev.csvd.csp1.census.gov" + vpc_name = "vpc2-csvd-dev" tags = { Owner = "matthew.c.morgan@census.gov" Environment = "development" diff --git a/examples/csvd-lab-mcm/main.tf b/examples/csvd-lab-mcm/main.tf index a38f90a..d5b0d2a 100644 --- a/examples/csvd-lab-mcm/main.tf +++ b/examples/csvd-lab-mcm/main.tf @@ -19,15 +19,15 @@ module "eks_deployment" { # Cluster configuration - simplified interface cluster_config = { - account_name = "lab-dev-gov" - aws_account_id = "224384469011" - cluster_mailing_list = "matthew.c.morgan@census.gov" - environment_abbr = "dev" + account_name = "lab-dev-gov" + aws_account_id = "224384469011" + cluster_mailing_list = "matthew.c.morgan@census.gov" + environment_abbr = "dev" finops_project_name = "csvd_platformbaseline" - finops_project_number = "fs0000000078" - finops_project_role = "csvd_platformbaseline_eks" - vpc_domain_name = "dev.lab.csp2.census.gov" - vpc_name = "vpc3-lab-dev" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_eks" + vpc_domain_name = "dev.lab.csp2.census.gov" + vpc_name = "vpc3-lab-dev" tags = { Owner = "matthew.c.morgan@census.gov" Environment = "development" diff --git a/locals.tf b/locals.tf index fc17610..4cb50a7 100644 --- a/locals.tf +++ b/locals.tf @@ -11,9 +11,7 @@ locals { # First define base namespaces without dependencies base_namespaces = { - cert-manager = "kube-system" karpenter = "karpenter" - metrics-server = "kube-system" postgresql = "kube-system" keycloak = "keycloak" gogatekeeper = "kube-system" @@ -58,8 +56,6 @@ locals { tf_version = var.versions.tf_version # Component versions - cert_manager_version = var.versions.cert_manager.version - cert_manager_helm_chart = var.versions.cert_manager.chart_version cluster_issuer_name = var.versions.cert_manager.cluster_issuer_name gogatekeeper_tag = var.versions.gogatekeeper.tag @@ -74,10 +70,6 @@ locals { istio_version = var.versions.istio.version istio_namespace = var.versions.istio.namespace - dashboard_hostname = var.versions.k8s_dashboard.hostname - k8s_dashboard_metrics_scraper = var.versions.k8s_dashboard.metrics_scraper - k8s_dashboard_version = var.versions.k8s_dashboard.version - karpenter_helm_chart = var.versions.karpenter.helm_chart karpenter_tag = var.versions.karpenter.tag @@ -100,9 +92,6 @@ locals { exporter_tag = var.versions.loki.exporter_tag sidecar_tag = var.versions.loki.sidecar_tag - metrics_server_helm_chart = var.versions.metrics_server.helm_chart - metrics_server_tag = var.versions.metrics_server.tag - prometheus_chart_version = var.versions.prometheus.chart_version prometheus_server_tag = var.versions.prometheus.server_tag prometheus_config_reloader_tag = var.versions.prometheus.config_reloader_tag diff --git a/main.tf b/main.tf index 2400cf3..44c5d2d 100644 --- a/main.tf +++ b/main.tf @@ -73,17 +73,33 @@ locals { } } +data "github_repository" "existing_repo" { + count = !var.create_repository && var.files_branch != null ? 1 : 0 + full_name = "${var.organization}/${var.name}" +} + +resource "github_branch" "files_branch" { + count = !var.create_repository && var.files_branch != null ? 1 : 0 + repository = data.github_repository.existing_repo[0].name + branch = var.files_branch + source_branch = var.files_branch_source_branch != null ? var.files_branch_source_branch : data.github_repository.existing_repo[0].default_branch +} + module "github_repo" { source = "git::git@github.e.it.census.gov:CSVD/terraform-github-repo.git" name = var.name repo_org = var.organization + create_repo = var.create_repository + create_codeowners = var.create_repository + enforce_prs = var.create_repository + files_branch = var.files_branch github_repo_description = "EKS Cluster Configuration for ${var.name}" github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] force_name = var.force_name - template_repo_org = local.repository_defaults.template_owner - template_repo = local.repository_defaults.template + template_repo_org = var.create_repository ? local.repository_defaults.template_owner : null + template_repo = var.create_repository ? local.repository_defaults.template : null github_is_private = false github_has_issues = true @@ -106,6 +122,10 @@ module "github_repo" { bypass_rules = false } ] + + depends_on = [ + github_branch.files_branch + ] } # The EKS deployment logic will go here, and will be skipped if create_repository is true. diff --git a/templates/README.md.tf.tpl b/templates/README.md.tf.tpl index 75175a8..795e8f2 100644 --- a/templates/README.md.tf.tpl +++ b/templates/README.md.tf.tpl @@ -1,4 +1,4 @@ -# EKS Cluster Configuration - ${upper(environment)} +# EKS Cluster Configuration - ${upper(cluster_name)} - ${upper(environment)} This EKS cluster configuration was generated using Terraform and the terraform-eks-deployment module. @@ -28,7 +28,7 @@ To apply this configuration: 3. Deploy additional modules as needed: ``` - cd ../eks-cert-manager + cd ../eks-cconfig terragrunt init terragrunt plan terragrunt apply diff --git a/templates/account.hcl.tf.tpl b/templates/account.hcl.tf.tpl index 83d3336..828d8b1 100644 --- a/templates/account.hcl.tf.tpl +++ b/templates/account.hcl.tf.tpl @@ -7,5 +7,5 @@ locals { aws_account_id = "${aws_account_id}" aws_profile = format("%v-%v", local.aws_account_id, replace(local.account_name, "-ew", "-gov")) environment = "${environment}" - environment_abbr = "${environment_abbr}" + environment_abbr = local.environment_abbr == 'lab' ? 'lab' : 'prod' } diff --git a/templates/default-versions.hcl b/templates/default-versions.hcl index 60f3cd4..f3f2dc7 100644 --- a/templates/default-versions.hcl +++ b/templates/default-versions.hcl @@ -51,13 +51,6 @@ locals { istio_namespace = "${istio_namespace}" istio_version = "${istio_version}" - ################ - # k8s-dashboard - ################ - dashboard_hostname = "${dashboard_hostname}" - k8s_dashboard_metrics_scraper = "${k8s_dashboard_metrics_scraper}" - k8s_dashboard_version = "${k8s_dashboard_version}" - ################ # Karpenter ################ @@ -92,12 +85,6 @@ locals { exporter_tag = "${exporter_tag}" sidecar_tag = "${sidecar_tag}" - ################ - # Metrics Server - ################ - metrics_server_helm_chart = "${metrics_server_helm_chart}" - metrics_server_tag = "${metrics_server_tag}" - ################ # Prometheus ################ diff --git a/variables.tf b/variables.tf index 8c7859a..349eb0f 100644 --- a/variables.tf +++ b/variables.tf @@ -255,5 +255,17 @@ variable "tags" { variable "create_repository" { description = "If true, a GitHub repository will be created and configured (internal use)" type = bool - default = false + default = true +} + +variable "files_branch" { + description = "Branch to manage generated files on. Set to an existing branch (for example, update/csvd-dev-mcm) to support PR workflows." + type = string + default = null +} + +variable "files_branch_source_branch" { + description = "Source branch used when creating files_branch for existing repositories. Defaults to the repository default branch." + type = string + default = null } \ No newline at end of file From 45633449d5d25668d8f8a06003d00e983422591f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 16 Mar 2026 17:51:16 -0400 Subject: [PATCH 08/27] update environment_abbr based on account_name --- locals.tf | 38 +++++------- main.tf | 8 +-- providers.tf | 2 +- templates/account.hcl.tf.tpl | 2 +- templates/default-versions.hcl | 10 ---- variables.tf | 105 +++++++++++++-------------------- 6 files changed, 63 insertions(+), 102 deletions(-) diff --git a/locals.tf b/locals.tf index 4cb50a7..4b76f30 100644 --- a/locals.tf +++ b/locals.tf @@ -1,9 +1,9 @@ locals { common_vars = merge({ - organization = "census:ocio:csvd" - project_name = "csvd_platformbaseline" - project_number = "fs0000000078" - project_role = "csvd_platformbaseline_app" + organization = "CHANGEME" + project_name = "CHANGEME" + project_number = "CHANGEME" + project_role = "CHANGEME" state_bucket_prefix = "inf-tfstate" state_table_name = "tf_remote_state" route53_endpoints = {} @@ -11,24 +11,23 @@ locals { # First define base namespaces without dependencies base_namespaces = { - karpenter = "karpenter" - postgresql = "kube-system" - keycloak = "keycloak" - gogatekeeper = "kube-system" - istio = "istio-system" - kiali = "istio-system" + karpenter = "kube-system" + postgresql = "kube-system" + keycloak = "keycloak" + gogatekeeper = "keycloak" + istio = "istio-system" + kiali = "istio-system" } # Then merge with telemetry namespaces all_namespaces = merge( local.base_namespaces, { - grafana = var.namespaces.telemetry_namespace - k8s-dashboard = var.namespaces.telemetry_namespace - loki = var.namespaces.telemetry_namespace - otel = var.namespaces.telemetry_namespace - prometheus = var.namespaces.telemetry_namespace - tempo = var.namespaces.telemetry_namespace + grafana = var.namespaces.telemetry_namespace + loki = var.namespaces.telemetry_namespace + otel = var.namespaces.telemetry_namespace + prometheus = var.namespaces.telemetry_namespace + tempo = var.namespaces.telemetry_namespace }, var.namespaces.custom_namespaces ) @@ -56,8 +55,6 @@ locals { tf_version = var.versions.tf_version # Component versions - cluster_issuer_name = var.versions.cert_manager.cluster_issuer_name - gogatekeeper_tag = var.versions.gogatekeeper.tag gogatekeeper_chart_version = var.versions.gogatekeeper.chart_version @@ -67,8 +64,7 @@ locals { grafana_tag = var.versions.grafana.tag os_shell_image_tag = var.versions.grafana.os_shell_image_tag - istio_version = var.versions.istio.version - istio_namespace = var.versions.istio.namespace + istio_version = var.versions.istio.version karpenter_helm_chart = var.versions.karpenter.helm_chart karpenter_tag = var.versions.karpenter.tag @@ -96,8 +92,6 @@ locals { prometheus_server_tag = var.versions.prometheus.server_tag prometheus_config_reloader_tag = var.versions.prometheus.config_reloader_tag alertmanager_tag = var.versions.prometheus.alertmanager_tag - kube_state_metrics_tag = var.versions.prometheus.kube_state_metrics_tag - node_exporter_tag = var.versions.prometheus.node_exporter_tag pushgateway_tag = var.versions.prometheus.pushgateway_tag tempo_chart_version = var.versions.tempo.chart_version diff --git a/main.tf b/main.tf index 44c5d2d..6235792 100644 --- a/main.tf +++ b/main.tf @@ -33,23 +33,23 @@ locals { "root.hcl" : templatefile("${path.module}/templates/root.hcl.tf.tpl", { environment = var.environment }), - "environment/account.hcl" : templatefile("${path.module}/templates/account.hcl.tf.tpl", { + "${var.environment}/account.hcl" : templatefile("${path.module}/templates/account.hcl.tf.tpl", { account_name = var.cluster_config.account_name, aws_account_id = var.cluster_config.aws_account_id, environment = var.environment, environment_abbr = var.cluster_config.environment_abbr }), - "environment/region/region.hcl" : templatefile("${path.module}/templates/region.hcl.tf.tpl", { + "${var.environment}/${var.region}/region.hcl" : templatefile("${path.module}/templates/region.hcl.tf.tpl", { aws_region = var.region, environment = var.environment }), - "environment/region/vpc/vpc.hcl" : templatefile("${path.module}/templates/vpc.hcl.tf.tpl", { + "${var.environment}/${var.region}/vpc/vpc.hcl" : templatefile("${path.module}/templates/vpc.hcl.tf.tpl", { vpc_name = var.cluster_config.vpc_name, vpc_domain_name = var.cluster_config.vpc_domain_name, environment = var.environment, aws_region = var.region }), - "environment/region/vpc/cluster/cluster.hcl" : templatefile("${path.module}/templates/cluster.hcl.tf.tpl", { + "${var.environment}/${var.region}/vpc/cluster/cluster.hcl" : templatefile("${path.module}/templates/cluster.hcl.tf.tpl", { cluster_name = var.name, cluster_mailing_list = var.cluster_config.cluster_mailing_list, aws_profile = local.aws_profile, diff --git a/providers.tf b/providers.tf index fd4066f..9085181 100644 --- a/providers.tf +++ b/providers.tf @@ -6,7 +6,7 @@ terraform { } aws = { source = "hashicorp/aws" - version = ">= 5.0" + version = ">= 6.0" } } } diff --git a/templates/account.hcl.tf.tpl b/templates/account.hcl.tf.tpl index 828d8b1..b348f6b 100644 --- a/templates/account.hcl.tf.tpl +++ b/templates/account.hcl.tf.tpl @@ -7,5 +7,5 @@ locals { aws_account_id = "${aws_account_id}" aws_profile = format("%v-%v", local.aws_account_id, replace(local.account_name, "-ew", "-gov")) environment = "${environment}" - environment_abbr = local.environment_abbr == 'lab' ? 'lab' : 'prod' + environment_abbr = contains(local.account_name, 'lab') ? 'lab' : contains(local.account_name, 'dev') ? 'dev' : contains(local.account_name, 'prod') ? 'prod' : 'CHANGEME' } diff --git a/templates/default-versions.hcl b/templates/default-versions.hcl index f3f2dc7..5cf6612 100644 --- a/templates/default-versions.hcl +++ b/templates/default-versions.hcl @@ -23,13 +23,6 @@ locals { # Component Versions ##################### - ################ - # Cert-Manager - ################ - cluster_issuer_name = "${cluster_issuer_name}" - cert_manager_version = "${cert_manager_version}" - cert_manager_helm_chart = "${cert_manager_helm_chart}" - ################ # GoGatekeeper ################ @@ -48,7 +41,6 @@ locals { ################ # Istio ################ - istio_namespace = "${istio_namespace}" istio_version = "${istio_version}" ################ @@ -92,8 +84,6 @@ locals { prometheus_server_tag = "${prometheus_server_tag}" prometheus_config_reloader_tag = "${prometheus_config_reloader_tag}" alertmanager_tag = "${alertmanager_tag}" - kube_state_metrics_tag = "${kube_state_metrics_tag}" - node_exporter_tag = "${node_exporter_tag}" pushgateway_tag = "${pushgateway_tag}" ################ diff --git a/variables.tf b/variables.tf index 349eb0f..2a5555b 100644 --- a/variables.tf +++ b/variables.tf @@ -37,9 +37,9 @@ variable "cluster_config" { variable "finops" { description = "FinOps configuration for cost tracking and billing" type = object({ - project_name = optional(string, "csvd_platformbaseline") - project_number = optional(string, "fs0000000078") - project_role = optional(string, "csvd_platformbaseline_app") + project_name = optional(string, "CHANGEME") + project_number = optional(string, "CHANGEME") + project_role = optional(string, "CHANGEME") }) default = {} } @@ -72,10 +72,10 @@ variable "force_name" { variable "common_variables" { description = "Common variables across all environments (internal use)" type = object({ - organization = optional(string, "census:ocio:csvd") - project_name = optional(string, "csvd_platformbaseline") - project_number = optional(string, "fs0000000078") - project_role = optional(string, "csvd_platformbaseline_app") + organization = optional(string, "CHANGEME") + project_name = optional(string, "CHANGEME") + project_number = optional(string, "CHANGEME") + project_role = optional(string, "CHANGEME") state_bucket_prefix = optional(string, "inf-tfstate") state_table_name = optional(string, "tf_remote_state") route53_endpoints = optional(map(object({ @@ -92,12 +92,12 @@ variable "versions" { description = "Version configurations for various components (internal use)" type = object({ # Module Versions - cluster_version = optional(string, "1.31") - eks_module_version = optional(string, "20.33.1") + cluster_version = optional(string, "1.34") + eks_module_version = optional(string, "21.11.1") release_version = optional(string, "main") # TF Providers - aws_version = optional(string, "5.84.0") + aws_version = optional(string, "6.0") helm_version = optional(string, "2.11.0") kubernetes_version = optional(string, "2.33.0") null_version = optional(string, "3.2.1") @@ -106,15 +106,9 @@ variable "versions" { tf_version = optional(string, "1.5.5") # Component Versions - cert_manager = optional(object({ - version = optional(string, "1.17.1") - chart_version = optional(string, "1.17.1") - cluster_issuer_name = optional(string, "cert-manager") - }), {}) - gogatekeeper = optional(object({ - tag = optional(string, "3.2.1") - chart_version = optional(string, "0.1.53") + tag = optional(string, "4.4.0") + chart_version = optional(string, "0.1.60") }), {}) grafana = optional(object({ @@ -126,63 +120,49 @@ variable "versions" { }), {}) istio = optional(object({ - version = optional(string, "1.25.0") - namespace = optional(string, "istio-system") - }), {}) - - k8s_dashboard = optional(object({ - hostname = optional(string, "dashboard") - metrics_scraper = optional(string, "1.0.8") - version = optional(string, "6.0.6") + version = optional(string, "1.28.3") }), {}) karpenter = optional(object({ - helm_chart = optional(string, "1.3.1") - tag = optional(string, "1.3.1") + helm_chart = optional(string, "1.8.5") + tag = optional(string, "1.8.5") }), {}) keycloak = optional(object({ - chart_version = optional(string, "24.4.11") - tag = optional(string, "26.1.3") + chart_version = optional(string, "7.0.1") + tag = optional(string, "26.0.7") hostname = optional(string, "keycloak") database = optional(string, "keycloak") username = optional(string, "keycloak") password = optional(string, "this is my very secure and totally random password horse battery staple now") - postgresql_tag = optional(string, "17.4.0-debian-12-r2") + postgresql_tag = optional(string, "17.4.0-debian-12-r4") }), {}) kiali = optional(object({ - operator_version = optional(string, "2.2.0") + operator_version = optional(string, "2.21.0") }), {}) loki = optional(object({ - chart_version = optional(string, "6.27.0") - tag = optional(string, "3.4.2") - enterprise_logs_provisioner_tag = optional(string, "v1.7.0") - gateway_tag = optional(string, "1.27-alpine") - memcached_tag = optional(string, "1.6.37") - exporter_tag = optional(string, "v0.15.0") - sidecar_tag = optional(string, "1.27.4") - }), {}) - - metrics_server = optional(object({ - helm_chart = optional(string, "3.12.2") - tag = optional(string, "0.7.2") + chart_version = optional(string, "6.49.0") + tag = optional(string, "3.6.3") + enterprise_logs_provisioner_tag = optional(string, "3.6.2") + gateway_tag = optional(string, "1.29.4") + memcached_tag = optional(string, "1.6.40") + exporter_tag = optional(string, "v0.15.3") + sidecar_tag = optional(string, "2.4.0") }), {}) prometheus = optional(object({ - chart_version = optional(string, "27.5.1") - server_tag = optional(string, "v3.2.1") - config_reloader_tag = optional(string, "v0.75.2") - alertmanager_tag = optional(string, "v0.28.0") - kube_state_metrics_tag = optional(string, "v2.15.0") - node_exporter_tag = optional(string, "v1.9.0") - pushgateway_tag = optional(string, "v1.11.0") + chart_version = optional(string, "28.6.0") + server_tag = optional(string, "v3.9.1") + config_reloader_tag = optional(string, "v0.88.0") + alertmanager_tag = optional(string, "v0.30.1") + pushgateway_tag = optional(string, "v1.6.2") }), {}) tempo = optional(object({ - chart_version = optional(string, "1.18.2") - tag = optional(string, "2.7.1") + chart_version = optional(string, "1.24.3") + tag = optional(string, "2.9.1") }), {}) }) default = {} @@ -191,17 +171,15 @@ variable "versions" { variable "namespaces" { description = "Namespace configurations (internal use)" type = object({ - operator_namespace = optional(string, "aoperator") - telemetry_namespace = optional(string, "atelemetry") + operator_namespace = optional(string, "operator") + telemetry_namespace = optional(string, "telemetry") custom_namespaces = optional(map(string), { - cert-manager = "kube-system" - karpenter = "karpenter" - metrics-server = "kube-system" - postgresql = "kube-system" - keycloak = "keycloak" - gogatekeeper = "kube-system" - istio = "istio-system" - kiali = "istio-system" + karpenter = "karpenter" + postgresql = "kube-system" + keycloak = "keycloak" + gogatekeeper = "kube-system" + istio = "istio-system" + kiali = "istio-system" }) }) default = {} @@ -211,7 +189,6 @@ variable "enable_modules" { description = "Map of modules to enable (internal use)" type = object({ gogatekeeper = optional(bool, false) - cert_manager = optional(bool, false) prometheus = optional(bool, false) grafana = optional(bool, false) istio = optional(bool, false) From a32f1d6ffbc6145129d0cecb6c3b1239d8cc2373 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 16 Mar 2026 18:07:41 -0400 Subject: [PATCH 09/27] add .gitignore for new clusters --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index e9b6656..c71837d 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ terraform.rc # Ignore test temporary files *.tftest.hcl.tmp terraform_data_dirs + +# Ignore terragrunt cache and configuration files +.terragrunt-cache/ \ No newline at end of file From 469988303052ba8df0debfb1141230d33c56fd46 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 16 Mar 2026 18:41:25 -0400 Subject: [PATCH 10/27] update default-versions and common-variables gen --- locals.tf | 45 +++++---- main.tf | 7 +- templates/common-variables.hcl | 70 +++++++++++++- templates/default-versions.hcl | 172 ++++++++++++++++++++++++++++----- variables.tf | 22 ++++- 5 files changed, 262 insertions(+), 54 deletions(-) diff --git a/locals.tf b/locals.tf index 4b76f30..3d789da 100644 --- a/locals.tf +++ b/locals.tf @@ -7,6 +7,7 @@ locals { state_bucket_prefix = "inf-tfstate" state_table_name = "tf_remote_state" route53_endpoints = {} + environment_abbr = var.cluster_config.environment_abbr }, var.common_variables) # First define base namespaces without dependencies @@ -39,11 +40,9 @@ locals { } default_versions = { - cluster_version = var.versions.cluster_version - custom_service_eks_account = var.versions.release_version - eks_module_version = var.versions.eks_module_version - istio_ingress_version = var.versions.release_version - release_version = var.versions.release_version + environment = var.environment + cluster_version = var.versions.cluster_version + eks_module_version = var.versions.eks_module_version # Provider versions aws_version = var.versions.aws_version @@ -54,15 +53,21 @@ locals { template_version = var.versions.template_version tf_version = var.versions.tf_version + # Namespace configuration + operator_namespace = var.namespaces.operator_namespace + telemetry_namespace = var.namespaces.telemetry_namespace + # Component versions - gogatekeeper_tag = var.versions.gogatekeeper.tag - gogatekeeper_chart_version = var.versions.gogatekeeper.chart_version + cribl_chart_version = var.versions.cribl.chart_version + cribl_app_version = var.versions.cribl.app_version + + gatekeeper_tag = var.versions.gogatekeeper.tag + gatekeeper_chart_version = var.versions.gogatekeeper.chart_version grafana_hostname = var.versions.grafana.hostname grafana_operator_chart_version = var.versions.grafana.operator_chart_version grafana_operator_tag = var.versions.grafana.operator_tag grafana_tag = var.versions.grafana.tag - os_shell_image_tag = var.versions.grafana.os_shell_image_tag istio_version = var.versions.istio.version @@ -71,14 +76,11 @@ locals { keycloak_chart_version = var.versions.keycloak.chart_version keycloak_tag = var.versions.keycloak.tag - keycloak_hostname = var.versions.keycloak.hostname - keycloak_database = var.versions.keycloak.database - keycloak_username = var.versions.keycloak.username - keycloak_password = var.versions.keycloak.password postgresql_tag = var.versions.keycloak.postgresql_tag + postgres_exporter_tag = var.versions.postgres_exporter_tag + utilities_tag = var.versions.utilities_tag - kiali_operator_version = var.versions.kiali.operator_version - kiali_application_version = "v${var.versions.kiali.operator_version}" + kiali_operator_version = var.versions.kiali.operator_version loki_chart_version = var.versions.loki.chart_version loki_tag = var.versions.loki.tag @@ -88,6 +90,15 @@ locals { exporter_tag = var.versions.loki.exporter_tag sidecar_tag = var.versions.loki.sidecar_tag + auto_instrumentation_java_version = var.versions.otel.auto_instrumentation_java_version + collector_contrib_version = var.versions.otel.collector_contrib_version + collector_version = var.versions.otel.collector_version + otel_helm_version = var.versions.otel.helm_version + otel_version = var.versions.otel.version + rbac_proxy_version = var.versions.otel.rbac_proxy_version + + postgresql_chart_version = var.versions.postgresql.chart_version + prometheus_chart_version = var.versions.prometheus.chart_version prometheus_server_tag = var.versions.prometheus.server_tag prometheus_config_reloader_tag = var.versions.prometheus.config_reloader_tag @@ -96,13 +107,7 @@ locals { tempo_chart_version = var.versions.tempo.chart_version tempo_tag = var.versions.tempo.tag - - # Add namespace configurations - operator_namespace = var.namespaces.operator_namespace - telemetry_namespace = var.namespaces.telemetry_namespace - namespaces = local.all_namespaces } - managed_extra_files = concat([ { path = "_envcommon/default-versions.hcl" diff --git a/main.tf b/main.tf index 6235792..09368c7 100644 --- a/main.tf +++ b/main.tf @@ -106,12 +106,13 @@ module "github_repo" { github_has_wiki = true github_has_projects = true - managed_extra_files = [ + managed_extra_files = concat([ for path, content in local.rendered_files : { path = path content = content - } - ] + }], + local.managed_extra_files) + archive_on_destroy = false github_org_teams = [ for team, permission in var.repository_teams : { diff --git a/templates/common-variables.hcl b/templates/common-variables.hcl index e228659..1fdf031 100644 --- a/templates/common-variables.hcl +++ b/templates/common-variables.hcl @@ -1,10 +1,70 @@ +# ${environment_abbr}/_envcommon/common-variables.hcl + +# --------------------------------------------------------------------------------------------------------------------- +# GLOBAL PARAMETERS +# These are the variables we pass to use across modules regardless of environment, i.e. these are the parameters +# that are common across all environments/accounts. +# --------------------------------------------------------------------------------------------------------------------- locals { - organization = "${organization}" - project_name = "${project_name}" - project_number = "${project_number}" - project_role = "${project_role}" state_bucket_prefix = "${state_bucket_prefix}" state_table_name = "${state_table_name}" + environment_abbr = "${environment_abbr}" + + route53_endpoints = { + route53_main = { + "account_id" = local.route53_info[local.environment_abbr]["account_id"] + "alias" = local.route53_info[local.environment_abbr]["alias"] + "us-gov-east-1" = local.route53_info[local.environment_abbr]["us-gov-east-1"] + "us-gov-west-1" = local.route53_info[local.environment_abbr]["us-gov-west-1"] + } + route53_main_legacy = { + "account_id" = local.route53_info["legacy"]["account_id"] + "alias" = local.route53_info["legacy"]["alias"] + "us-gov-east-1" = local.route53_info["legacy"]["us-gov-east-1"] + "us-gov-west-1" = local.route53_info["legacy"]["us-gov-west-1"] + } + } + + route53_info = { + lab = { + "account_id" = "269244441389" + "alias" = "lab-gov-network-nonprod" + "us-gov-east-1" = "vpc-070595c5b133243dd" + "us-gov-west-1" = "vpc-08b7b4db6a5ddf9c1" + } + prod = { + "account_id" = "057405694017" + "alias" = "ent-ew-network-prod" + "us-gov-east-1" = "vpc-061325b37d748d17a" + "us-gov-west-1" = "vpc-0b22b68b90e47cb5f" + } + legacy = { + "account_id" = "107742151971" + "alias" = "do2-govcloud" + "us-gov-east-1" = "vpc-099a991da7c4eb8a5" + "us-gov-west-1" = "vpc-77877a12" + } + } + + enterprise_ecr_account = { + lab = { + "account_id" = "269222635945" + "alias" = "lab-gov-shared-nonprod" + "profile" = "269222635945-lab-gov-shared-nonprod" + "region" = "us-gov-east-1" + } + prod = { + "account_id" = "067074201825" + "alias" = "ent-gov-shared-prod" + "profile" = "067074201825-ent-gov-shared-prod" + "region" = "us-gov-east-1" + } + } - route53_endpoints = ${jsonencode(route53_endpoints)} + eecr_info = { + account_id = local.enterprise_ecr_account[local.environment_abbr]["account_id"] + alias = local.enterprise_ecr_account[local.environment_abbr]["alias"] + profile = local.enterprise_ecr_account[local.environment_abbr]["profile"] + region = local.enterprise_ecr_account[local.environment_abbr]["region"] + } } \ No newline at end of file diff --git a/templates/default-versions.hcl b/templates/default-versions.hcl index 5cf6612..e2a222c 100644 --- a/templates/default-versions.hcl +++ b/templates/default-versions.hcl @@ -1,12 +1,92 @@ +# ${environment}/_envcommon/default-versions.hcl locals { + module_name = basename(get_original_terragrunt_dir()) + release_version = local.module_versions["2026.03.15"][local.module_name] + ##################### # Module Versions ##################### - cluster_version = "${cluster_version}" - custom_service_eks_account = "${custom_service_eks_account}" - eks_module_version = "${eks_module_version}" - istio_ingress_version = "${istio_ingress_version}" - release_version = "${release_version}" + cluster_version = "${cluster_version}" + eks_module_version = "${eks_module_version}" + + module_versions = { + "2025.20.04" = { + "eks-arcgis" = false + "eks-cert-manager" = "0.1.9" + "eks-config" = "1.0.5" + "eks-cribl" = "0.0.1" + "eks-dns" = "0.1.4" + "eks-gatekeeper" = "0.0.3" + "eks-grafana" = "0.1.5" + "eks-istio" = "1.0.9" + "eks-k8s-dashboard" = "0.1.4" + "eks-karpenter" = "0.1.7" + "eks-keycloak" = "0.0.8" + "eks-kiali" = "0.1.4" + "eks-loki" = "0.1.4" + "eks-metrics-server" = "0.1.4" + "eks-otel" = "0.0.4" + "eks-pipeline" = "initial" + "eks-postgresql" = false + "eks-prometheus" = "0.1.4" + "eks-tempo" = "0.1.4" + "eks" = "1.0.10" + } + "2026.03.15" = { + "eks-arcgis" = false + "eks-config" = "1.0.6" + "eks-cribl" = "mcm_v2" + "eks-dns" = "0.1.6" + "eks-gatekeeper" = "0.0.4" + "eks-grafana" = "0.1.5" + "eks-istio" = "1.0.9" + "eks-karpenter" = "0.1.9" + "eks-keycloak" = "0.0.8" + "eks-kiali" = "0.1.5" + "eks-loki" = "0.1.6" + "eks-otel" = "0.0.4" + "eks-pipeline" = "initial" + "eks-postgresql" = false + "eks-prometheus" = "0.1.5" + "eks-tempo" = "0.1.5" + "eks" = "1.0.12" + } + } + + submodule_versions = { + "tfmod-istio-service-ingress" = "0.1.7" + "tfmod-config-job" = "0.1.8" + "tfmod-custom-iam-role-for-service-account-eks" = "1.0.1" + } + + ##################### + # Module Enablement + ##################### + + # Core modules that should always be enabled (cannot be disabled) + core_modules = [ + "eks", + "eks-karpenter", + "eks-config", + "eks-istio", + "eks-dns", + ] + + # Optional modules with their default enablement state + enabled_modules = { + "eks-arcgis" = false + "eks-cribl" = false + "eks-gatekeeper" = true + "eks-grafana" = true + "eks-keycloak" = true + "eks-kiali" = true + "eks-loki" = true + "eks-otel" = true + "eks-pipeline" = false + "eks-postgresql" = false + "eks-prometheus" = true + "eks-tempo" = true + } ##################### # TF Providers @@ -20,14 +100,50 @@ locals { tf_version = "${tf_version}" ##################### - # Component Versions + # Namespaces Config + ##################### + operator_namespace = "${operator_namespace}" + telemetry_namespace = "${telemetry_namespace}" + system_namespace = "kube-system" + istio_namespace = "istio-system" + namespaces = { + arcgis = "arcgis" + cribl = "cribl" + gatekeeper = "keycloak" + grafana = local.telemetry_namespace + istio = local.istio_namespace + karpenter = local.system_namespace + keycloak = "keycloak" + kiali = local.istio_namespace + loki = local.telemetry_namespace + misp = "misp" + otel = local.telemetry_namespace + postgresql = "keycloak" + prometheus = local.telemetry_namespace + tempo = local.telemetry_namespace + } + + ##################### + # EKS Config + ##################### + + ################ + # Cert-Manager + ################ + cluster_issuer_name = "cert-manager" + + ##################### + # Cribl ##################### + cribl_chart_version = "${cribl_chart_version}" + cribl_app_version = "${cribl_app_version}" ################ # GoGatekeeper ################ - gogatekeeper_tag = "${gogatekeeper_tag}" - gogatekeeper_chart_version = "${gogatekeeper_chart_version}" + gatekeeper_tag = "${gatekeeper_tag}" + gatekeeper_chart_version = "${gatekeeper_chart_version}" + gatekeeper_service_name = "gatekeeper" ################ # Grafana @@ -36,12 +152,12 @@ locals { grafana_operator_chart_version = "${grafana_operator_chart_version}" grafana_operator_tag = "${grafana_operator_tag}" grafana_tag = "${grafana_tag}" - os_shell_image_tag = "${os_shell_image_tag}" + os_shell_image_tag = local.utilities_tag ################ # Istio ################ - istio_version = "${istio_version}" + istio_version = "${istio_version}" ################ # Karpenter @@ -54,17 +170,15 @@ locals { ################ keycloak_chart_version = "${keycloak_chart_version}" keycloak_tag = "${keycloak_tag}" - keycloak_hostname = "${keycloak_hostname}" - keycloak_database = "${keycloak_database}" - keycloak_username = "${keycloak_username}" - keycloak_password = "${keycloak_password}" postgresql_tag = "${postgresql_tag}" + postgres_exporter_tag = "${postgres_exporter_tag}" + utilities_tag = "${utilities_tag}" ################ # Kiali ################ kiali_operator_version = "${kiali_operator_version}" - kiali_application_version = "${kiali_application_version}" + kiali_application_version = "$${local.kiali_operator_version}" ################ # Loki @@ -73,9 +187,24 @@ locals { loki_tag = "${loki_tag}" enterprise_logs_provisioner_tag = "${enterprise_logs_provisioner_tag}" gateway_tag = "${gateway_tag}" - memcached_tag = "${memcached_tag}" - exporter_tag = "${exporter_tag}" - sidecar_tag = "${sidecar_tag}" + memcached_tag = "${memcached_tag}" + exporter_tag = "${exporter_tag}" + sidecar_tag = "${sidecar_tag}" + + ################ + # Open Telemetry + ################ + auto_instrumentation_java_version = "${auto_instrumentation_java_version}" + collector_contrib_version = "${collector_contrib_version}" + collector_version = "${collector_version}" + otel_helm_version = "${otel_helm_version}" + otel_version = "${otel_version}" + rbac_proxy_version = "${rbac_proxy_version}" + + ################ + # PostgreSQL + ################ + postgresql_chart_version = "${postgresql_chart_version}" ################ # Prometheus @@ -91,11 +220,4 @@ locals { ################ tempo_chart_version = "${tempo_chart_version}" tempo_tag = "${tempo_tag}" - - ##################### - # Namespaces Config - ##################### - operator_namespace = "${operator_namespace}" - telemetry_namespace = "${telemetry_namespace}" - namespaces = ${jsonencode(namespaces)} } \ No newline at end of file diff --git a/variables.tf b/variables.tf index 2a5555b..59d81d7 100644 --- a/variables.tf +++ b/variables.tf @@ -164,6 +164,27 @@ variable "versions" { chart_version = optional(string, "1.24.3") tag = optional(string, "2.9.1") }), {}) + + cribl = optional(object({ + chart_version = optional(string, "4.15.1") + app_version = optional(string, "4.15.1") + }), {}) + + otel = optional(object({ + auto_instrumentation_java_version = optional(string, "2.9.0") + collector_contrib_version = optional(string, "0.113.0-amd64") + collector_version = optional(string, "0.144.0") + helm_version = optional(string, "0.71.2") + version = optional(string, "0.110.0") + rbac_proxy_version = optional(string, "0.20.2") + }), {}) + + postgresql = optional(object({ + chart_version = optional(string, "16.5.0") + }), {}) + + utilities_tag = optional(string, "1.0.3") + postgres_exporter_tag = optional(string, "0.17.1-debian-12-r0") }) default = {} } @@ -228,7 +249,6 @@ variable "tags" { type = map(string) default = {} } - variable "create_repository" { description = "If true, a GitHub repository will be created and configured (internal use)" type = bool From 9cce60826e9042e6a83da88dc28795961a562dc2 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 16 Mar 2026 18:50:04 -0400 Subject: [PATCH 11/27] rename to match --- locals.tf | 11 ----------- main.tf | 14 +++++++++++++- ...n-variables.hcl => common-variables.hcl.tf.tpl} | 2 ++ ...lt-versions.hcl => default-versions.hcl.tf.tpl} | 0 4 files changed, 15 insertions(+), 12 deletions(-) rename templates/{common-variables.hcl => common-variables.hcl.tf.tpl} (96%) rename templates/{default-versions.hcl => default-versions.hcl.tf.tpl} (100%) diff --git a/locals.tf b/locals.tf index 3d789da..9c5b274 100644 --- a/locals.tf +++ b/locals.tf @@ -108,15 +108,4 @@ locals { tempo_chart_version = var.versions.tempo.chart_version tempo_tag = var.versions.tempo.tag } - managed_extra_files = concat([ - { - path = "_envcommon/default-versions.hcl" - content = templatefile("${path.module}/templates/default-versions.hcl", local.default_versions) - }, - { - path = "_envcommon/common-variables.hcl" - content = templatefile("${path.module}/templates/common-variables.hcl", local.common_vars) - } - ], - var.github_actions_workflows) } \ No newline at end of file diff --git a/main.tf b/main.tf index 09368c7..f791f1c 100644 --- a/main.tf +++ b/main.tf @@ -1,4 +1,15 @@ locals { + managed_extra_files = concat([ + { + path = "_envcommon/default-versions.hcl" + content = templatefile("${path.module}/templates/default-versions.hcl.tf.tpl", local.default_versions) + }, + { + path = "_envcommon/common-variables.hcl" + content = templatefile("${path.module}/templates/common-variables.hcl.tf.tpl", local.common_vars) + }], + var.github_actions_workflows) + rendered_files = { "config.json" : jsonencode({ environment = var.environment @@ -111,7 +122,8 @@ module "github_repo" { path = path content = content }], - local.managed_extra_files) + local.managed_extra_files, + var.github_actions_workflows) archive_on_destroy = false github_org_teams = [ diff --git a/templates/common-variables.hcl b/templates/common-variables.hcl.tf.tpl similarity index 96% rename from templates/common-variables.hcl rename to templates/common-variables.hcl.tf.tpl index 1fdf031..8f4ba55 100644 --- a/templates/common-variables.hcl +++ b/templates/common-variables.hcl.tf.tpl @@ -32,6 +32,7 @@ locals { "us-gov-east-1" = "vpc-070595c5b133243dd" "us-gov-west-1" = "vpc-08b7b4db6a5ddf9c1" } + dev = local.route53_info["prod"] prod = { "account_id" = "057405694017" "alias" = "ent-ew-network-prod" @@ -53,6 +54,7 @@ locals { "profile" = "269222635945-lab-gov-shared-nonprod" "region" = "us-gov-east-1" } + dev = local.enterprise_ecr_account["prod"] prod = { "account_id" = "067074201825" "alias" = "ent-gov-shared-prod" diff --git a/templates/default-versions.hcl b/templates/default-versions.hcl.tf.tpl similarity index 100% rename from templates/default-versions.hcl rename to templates/default-versions.hcl.tf.tpl From 7864b9d1fed755c48801b302b14668fcf638ca55 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 16 Mar 2026 18:50:21 -0400 Subject: [PATCH 12/27] fmt --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index f791f1c..053d611 100644 --- a/main.tf +++ b/main.tf @@ -1,5 +1,5 @@ locals { - managed_extra_files = concat([ + managed_extra_files = concat([ { path = "_envcommon/default-versions.hcl" content = templatefile("${path.module}/templates/default-versions.hcl.tf.tpl", local.default_versions) @@ -122,7 +122,7 @@ module "github_repo" { path = path content = content }], - local.managed_extra_files, + local.managed_extra_files, var.github_actions_workflows) archive_on_destroy = false From 58134ad4491d523b09821debd890e125b4a5d16f Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 16 Mar 2026 18:55:17 -0400 Subject: [PATCH 13/27] self ref doesn't work --- templates/common-variables.hcl.tf.tpl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/templates/common-variables.hcl.tf.tpl b/templates/common-variables.hcl.tf.tpl index 8f4ba55..6d7a9d2 100644 --- a/templates/common-variables.hcl.tf.tpl +++ b/templates/common-variables.hcl.tf.tpl @@ -32,7 +32,12 @@ locals { "us-gov-east-1" = "vpc-070595c5b133243dd" "us-gov-west-1" = "vpc-08b7b4db6a5ddf9c1" } - dev = local.route53_info["prod"] + dev = { + "account_id" = "057405694017" + "alias" = "ent-ew-network-prod" + "us-gov-east-1" = "vpc-061325b37d748d17a" + "us-gov-west-1" = "vpc-0b22b68b90e47cb5f" + } prod = { "account_id" = "057405694017" "alias" = "ent-ew-network-prod" @@ -54,7 +59,12 @@ locals { "profile" = "269222635945-lab-gov-shared-nonprod" "region" = "us-gov-east-1" } - dev = local.enterprise_ecr_account["prod"] + dev = { + "account_id" = "067074201825" + "alias" = "ent-gov-shared-prod" + "profile" = "067074201825-ent-gov-shared-prod" + "region" = "us-gov-east-1" + } prod = { "account_id" = "067074201825" "alias" = "ent-gov-shared-prod" From 58fe8cd8c40d2b56aef5b76d9d54593b2b0c76f8 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 16 Mar 2026 19:22:57 -0400 Subject: [PATCH 14/27] env fixed, add clusters folder --- {examples => clusters}/adsd-tools-dev/main.tf | 0 {examples => clusters}/csvd-dev-mcm/main.tf | 4 ++-- {examples => clusters}/csvd-lab-mcm/main.tf | 0 {examples => clusters}/csvd-mcm-common/main.tf | 0 templates/account.hcl.tf.tpl | 1 - templates/root.hcl.tf.tpl | 2 +- 6 files changed, 3 insertions(+), 4 deletions(-) rename {examples => clusters}/adsd-tools-dev/main.tf (100%) rename {examples => clusters}/csvd-dev-mcm/main.tf (93%) rename {examples => clusters}/csvd-lab-mcm/main.tf (100%) rename {examples => clusters}/csvd-mcm-common/main.tf (100%) diff --git a/examples/adsd-tools-dev/main.tf b/clusters/adsd-tools-dev/main.tf similarity index 100% rename from examples/adsd-tools-dev/main.tf rename to clusters/adsd-tools-dev/main.tf diff --git a/examples/csvd-dev-mcm/main.tf b/clusters/csvd-dev-mcm/main.tf similarity index 93% rename from examples/csvd-dev-mcm/main.tf rename to clusters/csvd-dev-mcm/main.tf index 3da1012..8ab552c 100644 --- a/examples/csvd-dev-mcm/main.tf +++ b/clusters/csvd-dev-mcm/main.tf @@ -29,8 +29,8 @@ module "eks_deployment" { finops_project_name = "csvd_platformbaseline" finops_project_number = "fs0000000078" finops_project_role = "csvd_platformbaseline_eks" - vpc_domain_name = "dev.csvd.csp1.census.gov" - vpc_name = "vpc2-csvd-dev" + vpc_domain_name = "dev.inf.csp1.census.gov" + vpc_name = "vpc3-inf-dev" tags = { Owner = "matthew.c.morgan@census.gov" Environment = "development" diff --git a/examples/csvd-lab-mcm/main.tf b/clusters/csvd-lab-mcm/main.tf similarity index 100% rename from examples/csvd-lab-mcm/main.tf rename to clusters/csvd-lab-mcm/main.tf diff --git a/examples/csvd-mcm-common/main.tf b/clusters/csvd-mcm-common/main.tf similarity index 100% rename from examples/csvd-mcm-common/main.tf rename to clusters/csvd-mcm-common/main.tf diff --git a/templates/account.hcl.tf.tpl b/templates/account.hcl.tf.tpl index b348f6b..ea432ce 100644 --- a/templates/account.hcl.tf.tpl +++ b/templates/account.hcl.tf.tpl @@ -7,5 +7,4 @@ locals { aws_account_id = "${aws_account_id}" aws_profile = format("%v-%v", local.aws_account_id, replace(local.account_name, "-ew", "-gov")) environment = "${environment}" - environment_abbr = contains(local.account_name, 'lab') ? 'lab' : contains(local.account_name, 'dev') ? 'dev' : contains(local.account_name, 'prod') ? 'prod' : 'CHANGEME' } diff --git a/templates/root.hcl.tf.tpl b/templates/root.hcl.tf.tpl index 2946d47..981f885 100644 --- a/templates/root.hcl.tf.tpl +++ b/templates/root.hcl.tf.tpl @@ -40,7 +40,7 @@ locals { aws_region = local.region_vars.locals.aws_region cluster_name = local.cluster_vars.locals.cluster_name eecr_info = local.common_vars.locals.eecr_info - environment_abbr = local.account_vars.locals.environment_abbr + environment_abbr = local.common_vars.locals.environment_abbr finops_project_name = local.cluster_vars.locals.finops_project_name finops_project_number = local.cluster_vars.locals.finops_project_number finops_project_role = local.cluster_vars.locals.finops_project_role From f6d60ee710c54fc3dd0e41097f532c15f48a980e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 16 Mar 2026 20:08:36 -0400 Subject: [PATCH 15/27] update to track files in template-eks-cluster/ --- main.tf | 17 +++++++++++++++ variables.tf | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/main.tf b/main.tf index 053d611..2b258fb 100644 --- a/main.tf +++ b/main.tf @@ -1,4 +1,12 @@ locals { + template_cluster_sync_files = [ + for rel_path in var.template_cluster_file_paths : { + path = "${var.environment}/${var.region}/vpc/cluster/${rel_path}" + content = data.github_repository_file.template_cluster_files[rel_path].content + } + if lookup(var.template_enabled_modules, split(rel_path, "/")[0], false) + ] + managed_extra_files = concat([ { path = "_envcommon/default-versions.hcl" @@ -89,6 +97,14 @@ data "github_repository" "existing_repo" { full_name = "${var.organization}/${var.name}" } +data "github_repository_file" "template_cluster_files" { + for_each = toset(var.template_cluster_file_paths) + + repository = var.template_repo_name + branch = var.template_repo_ref + file = "environment/region/vpc/cluster/${each.value}" +} + resource "github_branch" "files_branch" { count = !var.create_repository && var.files_branch != null ? 1 : 0 repository = data.github_repository.existing_repo[0].name @@ -122,6 +138,7 @@ module "github_repo" { path = path content = content }], + local.template_cluster_sync_files, local.managed_extra_files, var.github_actions_workflows) diff --git a/variables.tf b/variables.tf index 59d81d7..27f556b 100644 --- a/variables.tf +++ b/variables.tf @@ -265,4 +265,63 @@ variable "files_branch_source_branch" { description = "Source branch used when creating files_branch for existing repositories. Defaults to the repository default branch." type = string default = null +} + +variable "template_repo_name" { + description = "Name of the template repository used as authoritative source for cluster-level terragrunt files." + type = string + default = "template-eks-cluster" +} + +variable "template_repo_ref" { + description = "Branch, tag, or SHA to read from template_repo_name when syncing cluster-level terragrunt files." + type = string + default = "main" +} + +variable "template_cluster_file_paths" { + description = "List of file paths under environment/region/vpc/cluster in template_repo_name to sync into environment/region-resolved cluster path." + type = list(string) + default = [ + "eks/terragrunt.hcl", + "eks-config/terragrunt.hcl", + "eks-cribl/terragrunt.hcl", + "eks-dns/terragrunt.hcl", + "eks-gatekeeper/terragrunt.hcl", + "eks-grafana/terragrunt.hcl", + "eks-istio/terragrunt.hcl", + "eks-karpenter/terragrunt.hcl", + "eks-keycloak/terragrunt.hcl", + "eks-kiali/terragrunt.hcl", + "eks-loki/terragrunt.hcl", + "eks-otel/terragrunt.hcl", + "eks-prometheus/terragrunt.hcl", + "eks-tempo/terragrunt.hcl", + ] +} + +variable "template_enabled_modules" { + description = "Enablement map used to decide which template_cluster_file_paths are synced. Key must match the first path segment, for example eks-grafana in eks-grafana/terragrunt.hcl." + type = map(bool) + default = { + eks = true + eks-config = true + eks-dns = true + eks-istio = true + eks-karpenter = true + eks-arcgis = false + eks-cribl = false + eks-gatekeeper = true + eks-grafana = true + eks-keycloak = true + eks-kiali = true + eks-loki = true + eks-otel = true + eks-pipeline = false + eks-postgresql = false + eks-prometheus = true + eks-tempo = true + eks-cert-manager = false + eks-k8s-dashboard = false + } } \ No newline at end of file From 216ff5ae623ea2e6460612e84e8207b324805317 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 17 Mar 2026 12:50:09 -0400 Subject: [PATCH 16/27] fix(finops_codes) ensure passed in finops codes are used --- defaults.tf | 6 +++--- variables.tf | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/defaults.tf b/defaults.tf index 45c1ee3..a550dbe 100644 --- a/defaults.tf +++ b/defaults.tf @@ -22,8 +22,8 @@ locals { # Organization settings - now configurable through finops variable organization_defaults = { - finops_project_name = var.finops.project_name - finops_project_number = var.finops.project_number - finops_project_role = var.finops.project_role + finops_project_name = coalesce(var.cluster_config.finops_project_name, var.finops.project_name) + finops_project_number = coalesce(var.cluster_config.finops_project_number, var.finops.project_number) + finops_project_role = coalesce(var.cluster_config.finops_project_role, var.finops.project_role) } } diff --git a/variables.tf b/variables.tf index 27f556b..59d28ff 100644 --- a/variables.tf +++ b/variables.tf @@ -28,6 +28,9 @@ variable "cluster_config" { vpc_name = string vpc_domain_name = string cluster_mailing_list = optional(string) + finops_project_name = optional(string) + finops_project_number = optional(string) + finops_project_role = optional(string) tags = optional(map(string), {}) organization = optional(string) }) From b426872822c435a7af0b211c8085af5eba0b6522 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 18 Mar 2026 15:28:56 -0400 Subject: [PATCH 17/27] complete update workflow --- main.tf | 2 +- providers.tf | 4 ++-- variables.tf | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/main.tf b/main.tf index 2b258fb..adcb749 100644 --- a/main.tf +++ b/main.tf @@ -4,7 +4,7 @@ locals { path = "${var.environment}/${var.region}/vpc/cluster/${rel_path}" content = data.github_repository_file.template_cluster_files[rel_path].content } - if lookup(var.template_enabled_modules, split(rel_path, "/")[0], false) + if lookup(var.template_enabled_modules, split("/", rel_path)[0], false) ] managed_extra_files = concat([ diff --git a/providers.tf b/providers.tf index 9085181..1dc6ac9 100644 --- a/providers.tf +++ b/providers.tf @@ -2,11 +2,11 @@ terraform { required_providers { github = { source = "integrations/github" - version = ">= 6.6.0, < 6.7.0" + version = "~> 6.11" } aws = { source = "hashicorp/aws" - version = ">= 6.0" + version = "~> 6.0" } } } diff --git a/variables.tf b/variables.tf index 59d28ff..be9f275 100644 --- a/variables.tf +++ b/variables.tf @@ -283,7 +283,7 @@ variable "template_repo_ref" { } variable "template_cluster_file_paths" { - description = "List of file paths under environment/region/vpc/cluster in template_repo_name to sync into environment/region-resolved cluster path." + description = "List of file paths under //vpc/cluster in template_repo_name to sync into environment/region-resolved cluster path." type = list(string) default = [ "eks/terragrunt.hcl", From 8e2f9cf0b5beaed7b93843fd0f4cd5ea6df92918 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 18 Mar 2026 15:32:39 -0400 Subject: [PATCH 18/27] update version for otel-collector-k8s --- variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variables.tf b/variables.tf index be9f275..95c914f 100644 --- a/variables.tf +++ b/variables.tf @@ -176,7 +176,7 @@ variable "versions" { otel = optional(object({ auto_instrumentation_java_version = optional(string, "2.9.0") collector_contrib_version = optional(string, "0.113.0-amd64") - collector_version = optional(string, "0.144.0") + collector_version = optional(string, "0.111.0-amd64") helm_version = optional(string, "0.71.2") version = optional(string, "0.110.0") rbac_proxy_version = optional(string, "0.20.2") From 3d6f88dbf44590f6f8b01ce75aa5e1730edaaa8d Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 18 Mar 2026 15:38:48 -0400 Subject: [PATCH 19/27] bump version of eks-dns and tempo --- templates/default-versions.hcl.tf.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/default-versions.hcl.tf.tpl b/templates/default-versions.hcl.tf.tpl index e2a222c..6fcf1e3 100644 --- a/templates/default-versions.hcl.tf.tpl +++ b/templates/default-versions.hcl.tf.tpl @@ -36,7 +36,7 @@ locals { "eks-arcgis" = false "eks-config" = "1.0.6" "eks-cribl" = "mcm_v2" - "eks-dns" = "0.1.6" + "eks-dns" = "0.1.7" "eks-gatekeeper" = "0.0.4" "eks-grafana" = "0.1.5" "eks-istio" = "1.0.9" @@ -48,7 +48,7 @@ locals { "eks-pipeline" = "initial" "eks-postgresql" = false "eks-prometheus" = "0.1.5" - "eks-tempo" = "0.1.5" + "eks-tempo" = "0.1.6" "eks" = "1.0.12" } } From b6c3b24a273caa00cb47a80a0cad68b93fa48ee3 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 19 Mar 2026 17:05:01 -0400 Subject: [PATCH 20/27] complete update workflow, fix create workflow, open PRs on both --- README.md | 236 +-- ROADMAP.md | 47 - clusters/csvd-dev-mcm/main.tf | 11 +- .../logs/plan.20260319.1773952402.log | 23 + clusters/csvd-lab-mcm/main.tf | 16 +- docs/callnote-09152025-implementation.md | 766 -------- docs/callnotes-09152025.txt | 1611 ----------------- docs/callnotes-actionitems-09152025.md | 66 - examples/basic/main.tf | 17 +- examples/centralized-management/main.tf | 102 +- main.tf | 53 +- templates/prefixes.hcl.tf.tpl | 37 + variables.tf | 77 +- 13 files changed, 332 insertions(+), 2730 deletions(-) delete mode 100644 ROADMAP.md create mode 100644 clusters/csvd-lab-mcm/logs/plan.20260319.1773952402.log delete mode 100644 docs/callnote-09152025-implementation.md delete mode 100644 docs/callnotes-09152025.txt delete mode 100644 docs/callnotes-actionitems-09152025.md create mode 100644 templates/prefixes.hcl.tf.tpl diff --git a/README.md b/README.md index 1f482c0..b02bc10 100644 --- a/README.md +++ b/README.md @@ -1,160 +1,170 @@ # terraform-eks-deployment -Terraform module for EKS cluster repository bootstrapping and configuration +Terraform module for bootstrapping and updating EKS cluster configuration repositories. ## Overview -This module creates and configures a new GitHub repository for your EKS cluster based on the template-eks-cluster repository. It automatically generates all necessary Terragrunt HCL files (`root.hcl`, `account.hcl`, `region.hcl`, etc.) directly from your input variables, eliminating the need for manual configuration or separate Ansible playbooks. +This module manages the GitHub repository that holds your cluster Terragrunt configuration. It renders core files such as `root.hcl`, `account.hcl`, `region.hcl`, `vpc.hcl`, `cluster.hcl`, and selected module `terragrunt.hcl` files from `template-eks-cluster`. + +It supports two workflows: + +- `create`: create a repository, commit generated files to `new/`, and open a pull request into `main` +- `update`: manage an existing repository, commit generated files to `update/`, and open a pull request into `main` ## Features -- **Fully Terraform-Native**: No Lambda functions, Ansible playbooks, or manual steps required -- **Complete HCL Generation**: Automatically creates all necessary Terragrunt HCL files -- **Team Management**: Configures repository permissions for your teams -- **Customizable**: Extensive configuration options for cluster, account, and VPC settings +- Fully Terraform-native workflow +- New-repo and update-repo support +- Automatic working branch creation +- Automatic pull request creation into `main` +- Rendered Terragrunt configuration from module inputs +- Optional syncing of module-specific `terragrunt.hcl` files from `template-eks-cluster` +- Team access management through the underlying GitHub repo module ## Prerequisites -- GitHub token with repository and workflow permissions -- Terraform 1.0.0 or newer -- Access to GitHub Enterprise (if using enterprise version) +- Terraform 1.x +- GitHub token with repository and pull request permissions +- Access to your GitHub Enterprise instance, if applicable ## Usage -### Basic Example +### Create a New Repository ```hcl module "eks_deployment" { - source = "github.com/HappyPathway/terraform-eks-deployment" - - # Repository configuration - repository_name = "eks-prod-cluster" - repository_template_owner = "HappyPathway" - repository_template = "template-eks-cluster" - repository_teams = { - "platform-team" = "admin", - "devops-team" = "maintain", - "developers" = "push" - } - - # Basic settings - organization = "my-org" - environment = "production" - region = "us-gov-west-1" - github_server_url = "https://github.e.it.census.gov" # For GitHub Enterprise - - # Account configuration - account_config = { - account_name = "prod-account" - aws_account_id = "123456789012" - environment_abbr = "prod" - } + source = "../../" - # VPC configuration - vpc_config = { - vpc_name = "prod-vpc" - vpc_domain_name = "prod.example.com" + name = "csvd-lab-mcm" + organization = "SCT-Engineering" + repository_mode = "create" + environment = "dev" + region = "us-gov-east-1" + + repository_teams = { + "platform-team" = "admin" + "developers" = "push" } - # Cluster configuration cluster_config = { - cluster_name = "prod-eks-01" - cluster_mailing_list = "team@example.com" - eks_instance_disk_size = 200 - eks_ng_desired_size = 5 - eks_ng_max_size = 10 - eks_ng_min_size = 3 - organization = "census:ocio:csvd" - finops_project_name = "csvd_platformbaseline" - finops_project_number = "fs0000000078" - finops_project_role = "csvd_platformbaseline_app" + account_name = "lab-dev-gov" + aws_account_id = "224384469011" + environment_abbr = "lab" + vpc_name = "vpc3-lab-dev" + vpc_domain_name = "dev.lab.csp2.census.gov" + cluster_mailing_list = "matthew.c.morgan@census.gov" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_eks" + organization = "census:ocio:csvd" tags = { - Owner = "Platform Team", - Environment = "Production", - CostCenter = "123-456" - } - module_enablement_overrides = { - cert_manager = true, - prometheus = true, - grafana = true, - istio = true + Owner = "matthew.c.morgan@census.gov" + Environment = "development" + CostCenter = "fs0000000078" } } -} -output "repository_url" { - value = module.eks_deployment.repository_url + template_enabled_modules = { + eks-gatekeeper = true + eks-grafana = true + eks-kiali = true + eks-loki = true + eks-otel = true + eks-prometheus = true + eks-tempo = true + } } ``` -### Update an Existing Repository on a PR Branch - -Use this mode when the cluster repository already exists and you only want to update managed files on a branch for a pull request. +### Update an Existing Repository ```hcl module "eks_deployment" { source = "../../" - name = "csvd-dev-mcm" - organization = "SCT-Engineering" - create_repository = false - files_branch = "update/csvd-dev-mcm" - - environment = "dev" - region = "us-gov-east-1" + name = "csvd-lab-mcm" + organization = "SCT-Engineering" + repository_mode = "update" + environment = "dev" + region = "us-gov-east-1" cluster_config = { - account_name = "csvd-dev-gov" - aws_account_id = "229685449397" - environment_abbr = "dev" - vpc_name = "vpc2-csvd-dev" - vpc_domain_name = "dev.csvd.csp1.census.gov" - cluster_mailing_list = "matthew.c.morgan@census.gov" + account_name = "lab-dev-gov" + aws_account_id = "224384469011" + environment_abbr = "lab" + vpc_name = "vpc3-lab-dev" + vpc_domain_name = "dev.lab.csp2.census.gov" + cluster_mailing_list = "matthew.c.morgan@census.gov" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_eks" + organization = "census:ocio:csvd" + } + + template_enabled_modules = { + eks-gatekeeper = true + eks-grafana = true + eks-kiali = true + eks-loki = true } } ``` -If `files_branch` does not already exist, create it first from your default branch, then run `terraform apply`. +### Template Module Selection + +`template_enabled_modules` controls which module-specific `terragrunt.hcl` files are synced from `template-eks-cluster`. + +These core cluster modules are always enabled and cannot be turned off: + +- `eks` +- `eks-config` +- `eks-karpenter` +- `eks-istio` +- `eks-dns` -By default, this module now attempts to create `files_branch` automatically when `create_repository = false`, using the repository default branch as the source. You can override the source with `files_branch_source_branch`. +Any omitted key is treated as `false` for optional modules. ## Generated Files -The module automatically generates the following files in your new repository: +The module manages files such as: -- `root.hcl`: Global Terragrunt configuration -- `[environment]/account.hcl`: Account-specific variables -- `[environment]/[region]/region.hcl`: Region-specific variables -- `[environment]/[region]/vpc/vpc.hcl`: VPC-specific variables -- `[environment]/[region]/vpc/[cluster_name]/cluster.hcl`: Cluster-specific variables -- `README.md`: Usage instructions and documentation +- `root.hcl` +- `config.json` +- `README.md` +- `/account.hcl` +- `//region.hcl` +- `//vpc/vpc.hcl` +- `//vpc/cluster/cluster.hcl` +- `//vpc/cluster//terragrunt.hcl` for enabled template modules +- `_envcommon/default-versions.hcl` +- `_envcommon/common-variables.hcl` +- `_envcommon/prefixes.hcl` -These files are committed directly to your new repository and are ready for use with Terragrunt to deploy your EKS cluster. +## Working Branch and PR Behavior -## Module Configuration +- `repository_mode = "create"` creates branch `new/` and opens a PR to `main` +- `repository_mode = "update"` creates branch `update/` and opens a PR to `main` +- For update mode, the working branch is created from the repository default branch unless `files_branch_source_branch` is provided -### Required Variables +## Key Inputs -| Name | Description | -|------|-------------| -| `repository_name` | Name of the GitHub repository to create | -| `organization` | GitHub organization name | -| `environment` | Deployment environment (e.g., production, development) | -| `region` | AWS region for the EKS cluster | -| `account_config` | Map of account configuration values | -| `vpc_config` | Map of VPC configuration values | -| `cluster_config` | Map of cluster configuration values | +### Required + +- `name` +- `environment` +- `region` +- `cluster_config` -### Optional Variables +### Common Optional Inputs -| Name | Description | Default | -|------|-------------|---------| -| `repository_template_owner` | Owner of the template repository | `"HappyPathway"` | -| `repository_template` | Template repository name | `"template-eks-cluster"` | -| `repository_teams` | Map of team names and permission levels | `{}` | -| `github_server_url` | GitHub Enterprise server URL | `"https://api.github.com"` | +- `organization` +- `repository_mode` +- `repository_teams` +- `template_enabled_modules` +- `files_branch_source_branch` +- `force_name` -For complete configuration options, see the variables.tf file. +See [variables.tf](/apps/terraform/workspaces/morga471/terraform/terraform-eks-deployment/variables.tf) for the full input surface. ## Outputs @@ -165,11 +175,9 @@ For complete configuration options, see the variables.tf file. ## How It Works -1. The module calls the `terraform-github-repo` module to create a new repository based on your template. -2. It uses Terraform's `templatefile()` function to render Terragrunt HCL files from templates. -3. These rendered files are committed directly to the repository using the `github_repository_file` resource. -4. Team permissions are configured using the `github_team_repository` resource. - -## Migrating from Previous Workflow - -This module replaces the previous Lambda/Ansible-based workflow with a purely Terraform-native approach. If you were previously using the template-automation-lambda to create repositories and then running an Ansible playbook to generate HCL files, you can now accomplish the entire process with just this module. +1. The module computes whether it is in create or update mode. +2. It renders repository files from Terraform templates and input variables. +3. It creates or updates the target repository through `terraform-github-repo`. +4. It creates a working branch (`new/` or `update/`). +5. It commits the generated files to that branch. +6. It opens a pull request from that branch into `main`. diff --git a/ROADMAP.md b/ROADMAP.md deleted file mode 100644 index 0667b5a..0000000 --- a/ROADMAP.md +++ /dev/null @@ -1,47 +0,0 @@ -# EKS Cluster Deployment Roadmap - -## Current Architecture -- **Terraform-Native Workflow**: The module provides a fully Terraform-native workflow for bootstrapping a single GitHub repository for an EKS cluster. -- **Declarative Configuration**: It uses a set of input variables (`account_config`, `vpc_config`, `cluster_config`) to define a cluster's configuration. -- **Dynamic HCL Generation**: The module automatically generates all necessary Terragrunt HCL files (`root.hcl`, `account.hcl`, `region.hcl`, etc.) from internal templates. -- **Wrapper Module**: This module acts as a sophisticated wrapper around the `terraform-github-repo` module, orchestrating repository creation, file generation, and team permission management. - -## Planned Enhancements - -### 1. Version and Configuration Management (High Priority) -- [x] Implement version control strategy for `default-versions.hcl` -- [x] Create system for managing platform release versions -- [x] Set up version override mechanism -- [ ] Add validation for configuration files to fail early on invalid inputs. -- [ ] Integrate a robust versioning strategy for the module itself, using tags. - -### 2. Automated Cluster Deployment (Medium Priority) -- [ ] Create templatized GitHub Actions workflow files that can be injected into the created repository. -- [ ] Configure workflows to run on specific runners for secure credential management. -- [ ] Implement automated `terragrunt plan/apply` execution within the generated workflows. - -### 3. Module and Documentation Refinements (Low Priority) -- [ ] Enhance documentation with more detailed examples, including edge cases. -- [ ] Add contribution guidelines and a developer guide. -- [ ] Refactor and clean up any legacy code or unused variables. - -## Completed Milestones - -- **Terraform-Native Migration**: Successfully replaced the legacy Lambda/Ansible workflow with a unified, Terraform-native solution for repository bootstrapping. -- **File Management System**: Implemented a robust system for crafting and injecting configuration files directly via Terraform, fulfilling the core requirement of the "File Management System" epic. - -## Technical Considerations -1. **Automation Requirements**: - - GitHub Actions runner configuration for secure AWS authentication. - - Workflow templating to dynamically generate CI/CD pipelines. - - Seamless Terragrunt integration within the automated workflows. - -2. **Version Control Strategy**: - - Centralized management of component versions (EKS, Istio, etc.). - - Clear override mechanisms for environment-specific versioning. - -## Success Criteria -- Fully automated repository and cluster configuration process. -- A clear, maintainable, and scalable version management system. -- Secure and automated CI/CD pipelines for deploying EKS clusters. -- Comprehensive documentation that empowers users and contributors. \ No newline at end of file diff --git a/clusters/csvd-dev-mcm/main.tf b/clusters/csvd-dev-mcm/main.tf index 8ab552c..ef49d30 100644 --- a/clusters/csvd-dev-mcm/main.tf +++ b/clusters/csvd-dev-mcm/main.tf @@ -13,12 +13,11 @@ module "eks_deployment" { source = "../../" # Repository and cluster configuration - single name for both - name = "csvd-dev-mcm" - organization = "SCT-Engineering" - create_repository = false - files_branch = "update/csvd-dev-mcm" - environment = "dev" - region = "us-gov-east-1" + name = "csvd-dev-mcm" + organization = "SCT-Engineering" + repository_mode = "update" + environment = "dev" + region = "us-gov-east-1" # Cluster configuration - simplified interface cluster_config = { diff --git a/clusters/csvd-lab-mcm/logs/plan.20260319.1773952402.log b/clusters/csvd-lab-mcm/logs/plan.20260319.1773952402.log new file mode 100644 index 0000000..c692ffb --- /dev/null +++ b/clusters/csvd-lab-mcm/logs/plan.20260319.1773952402.log @@ -0,0 +1,23 @@ +# starting v1.11.0 action plan file logs/plan.20260319.1773952402.log stamp 20260319.1773952402 time 1773952402 +# current_directory=/apps/terraform/workspaces/morga471/terraform/terraform-eks-deployment/clusters/csvd-lab-mcm +# git_repository=git@github.e.it.census.gov:sct-engineering/terraform-eks-deployment +# git_current_branch=test_cluster +# terraform_version=Terraform v0.12.31 +# TFCONTROL= +# TF_CLI_CONFIG_FILE= +# TFARGS="" TFNOCLOR= TFNOLOG= TFNOPROXY= +# env TF_VAR_ variables +# TF_VAR_os_environment={"pwd":"/apps/terraform/workspaces/morga471/terraform/terraform-eks-deployment/clusters/csvd-lab-mcm"} +# TF_VAR_os_username=morga471 + + +Error: Error locking state: Error acquiring the state lock: state snapshot was created by Terraform v1.14.7, which is newer than current v0.12.31; upgrade to Terraform v1.14.7 or greater to work with this state + +Terraform acquires a state lock to protect the state from being written +by multiple users at the same time. Please resolve the issue above and try +again. For most commands, you can disable locking with the "-lock=false" +flag, but this is not recommended. + + +# ending v1.11.0 action plan file logs/plan.20260319.1773952402.log stamp 20260319.1773952402 start 1773952402 end 1773952402 elapsed 0 + diff --git a/clusters/csvd-lab-mcm/main.tf b/clusters/csvd-lab-mcm/main.tf index d5b0d2a..09ed437 100644 --- a/clusters/csvd-lab-mcm/main.tf +++ b/clusters/csvd-lab-mcm/main.tf @@ -10,7 +10,9 @@ provider "github" { } module "eks_deployment" { - source = "../../" + source = "../../" + repository_mode = "create" + organization = "SCT-Engineering" # Repository and cluster configuration - single name for both name = "csvd-lab-mcm" @@ -22,7 +24,7 @@ module "eks_deployment" { account_name = "lab-dev-gov" aws_account_id = "224384469011" cluster_mailing_list = "matthew.c.morgan@census.gov" - environment_abbr = "dev" + environment_abbr = "lab" finops_project_name = "csvd_platformbaseline" finops_project_number = "fs0000000078" finops_project_role = "csvd_platformbaseline_eks" @@ -35,6 +37,16 @@ module "eks_deployment" { } organization = "census:ocio:csvd" } + template_enabled_modules = { + eks-gatekeeper = false + eks-grafana = false + eks-kiali = false + eks-keycloak = false + eks-loki = false + eks-otel = false + eks-prometheus = false + eks-tempo = false + } } output "repository_url" { diff --git a/docs/callnote-09152025-implementation.md b/docs/callnote-09152025-implementation.md deleted file mode 100644 index 12014b3..0000000 --- a/docs/callnote-09152025-implementation.md +++ /dev/null @@ -1,766 +0,0 @@ -# Call Notes Implementation Plan - September 15, 2025 - -## Executive Summary -Based on the discussion between David John Arnold Jr. and Matthew Creal Morgan, this document provides a detailed technical implementation plan for improving the terraform-eks-deployment module. The primary goals are to simplify the user interface, consolidate naming conventions, and establish a robust workflow for managing multiple EKS clusters. - -## Key Insights from Call Notes - -### Core Issues Identified -1. **Interface Complexity**: Current variable structure is too complex for end users -2. **Duplicate Variables**: Repository name and cluster name should be unified -3. **Static Values Exposed**: Values that should have sensible defaults are being exposed to users -4. **Workspace Management**: Need for a centralized approach to manage multiple clusters without conflicts - -### Technical Requirements -- Use `config_json` structure as the model for the interface -- Implement single source of truth for naming -- Hide implementation details from users -- Create shared workspace for multiple cluster management - -## Detailed Implementation Plan - -### Phase 1: Module Interface Cleanup (Priority: High) - -#### 1.1 Variable Consolidation - -**Current State Analysis:** -- `name` variable controls repository name -- `cluster_config.cluster_name` controls cluster name (REMOVED) -- Multiple redundant variables exposed to users - -**Implementation Steps:** - -##### Step 1.1.1: Remove Redundant Variables -**File:** `variables.tf` -**Changes Required:** -```hcl -# REMOVE these variables or make them internal -variable "repository_template" { - # Move to locals with default value -} - -variable "repository_template_owner" { - # Move to locals with default value -} - -# REMOVE from cluster_config object -variable "cluster_config" { - type = object({ - # Remove aws_profile - build dynamically - # Remove enable_all_modules - default to true - # cluster_name already removed - }) -} -``` - -**Action Items:** -- [ ] Move static repository template values to `locals.tf` -- [ ] Remove `aws_profile` from cluster_config (build dynamically from account_name + account_id) -- [ ] Remove `enable_all_modules` variable (default behavior) -- [ ] Update variable descriptions to reflect simplified interface - -##### Step 1.1.2: Implement Dynamic AWS Profile Generation -**File:** `locals.tf` -**Implementation:** -```hcl -locals { - # Dynamic AWS profile generation - aws_profile = "${var.cluster_config.account_name}-${var.cluster_config.environment_abbr}" - - # Static template values (hidden from users) - repository_template = "template-eks-cluster" - repository_template_owner = "SCT-Engineering" - - # Default module enablement - enable_all_modules = true -} -``` - -**Action Items:** -- [ ] Add dynamic profile generation logic -- [ ] Update all references to use computed values -- [ ] Test profile generation with existing account configurations - -#### 1.2 Static Value Management - -**Current Issue:** Values like `eks_instance_disk_size`, `eks_ng_desired_size`, etc. are exposed to users but should be static defaults for Karpenter node group configuration. - -**Implementation Steps:** - -##### Step 1.2.1: Move Static Values to Module Defaults -**File:** `variables.tf` -**Changes:** -```hcl -variable "cluster_config" { - type = object({ - # Keep only user-configurable values - account_name = string - aws_account_id = string - environment_abbr = string - vpc_name = string - vpc_domain_name = string - cluster_mailing_list = optional(string) - # Remove static EKS sizing values - }) -} -``` - -**File:** `locals.tf` -**Add:** -```hcl -locals { - # Static EKS configuration for Karpenter bootstrap node group - eks_defaults = { - instance_disk_size = 200 - ng_desired_size = 3 - ng_max_size = 10 - ng_min_size = 3 - } -} -``` - -**Action Items:** -- [ ] Move all static EKS sizing values to locals -- [ ] Update cluster.hcl template to use local values -- [ ] Document why these values are static (Karpenter will manage actual workload nodes) -- [ ] Validate that existing configurations still work - -#### 1.3 Config JSON Interface Model - -**Goal:** Simplify the user interface to match the minimal `config_json` structure shown in the call. - -**Target Interface Structure:** -```json -{ - "environment": "dev", - "region": "us-gov-east-1", - "account": { - "account_name": "csvd-dev-ew", - "aws_account_id": "229685449397", - "environment_abbr": "dev" - }, - "vpc": { - "vpc_name": "vpc3-csvd-dev", - "vpc_domain_name": "dev.inf.csp1.census.gov" - }, - "cluster": { - "cluster_mailing_list": "david.j.arnold.jr@census.gov" - } -} -``` - -**Implementation Steps:** - -##### Step 1.3.1: Create Simplified Variable Structure -**File:** `variables.tf` -**New Structure:** -```hcl -variable "name" { - description = "The name used for both repository and cluster" - type = string -} - -variable "environment" { - description = "Environment name (e.g., dev, prod)" - type = string -} - -variable "region" { - description = "AWS region" - type = string -} - -variable "account_config" { - description = "AWS account configuration" - type = object({ - account_name = string - aws_account_id = string - environment_abbr = string - }) -} - -variable "vpc_config" { - description = "VPC configuration" - type = object({ - vpc_name = string - vpc_domain_name = string - }) -} - -variable "cluster_config" { - description = "Cluster-specific configuration" - type = object({ - cluster_mailing_list = optional(string) - tags = optional(map(string), {}) - }) - default = {} -} -``` - -**Action Items:** -- [ ] Restructure variables to match config_json model -- [ ] Update all template references -- [ ] Update examples to use new structure -- [ ] Create migration guide for existing users - -### Phase 2: Workspace Structure Implementation (Priority: High) - -#### 2.1 Centralized Workspace Creation - -**Current Challenge:** Users are working in separate directories causing state conflicts. - -**Solution:** Create a shared workspace that can manage multiple clusters without conflicts. - -**Implementation Steps:** - -##### Step 2.1.1: Create eks-clusters-workspace Repository -**Action Items:** -- [ ] Create new repository: `eks-clusters-workspace` (renamed from terraform-eks-workspace to better reflect that it's a workspace, not a module) -- [ ] Set up proper directory structure -- [ ] Configure remote state backend -- [ ] Set up proper permissions and team access - -**Directory Structure:** -``` -eks-clusters-workspace/ -├── main.tf # Multiple module calls -├── backend.tf # Remote state configuration -├── providers.tf # Provider configurations -├── variables.tf # Workspace-level variables -├── clusters/ -│ ├── david-cluster.tfvars # David's cluster config -│ ├── matthew-cluster.tfvars # Matthew's cluster config -│ └── prod-cluster.tfvars # Production cluster config -├── outputs.tf # Aggregate outputs -└── README.md # Usage documentation -``` - -##### Step 2.1.2: Configure Multi-Cluster Main.tf -**File:** `main.tf` -**Implementation:** -```hcl -# David's test cluster -module "david_cluster" { - source = "git::git@github.e.it.census.gov:CSVD/terraform-eks-deployment.git" - - name = "david-test-cluster" - environment = "dev" - region = "us-gov-east-1" - - account_config = { - account_name = "csvd-dev-ew" - aws_account_id = "229685449397" - environment_abbr = "dev" - } - - vpc_config = { - vpc_name = "vpc3-csvd-dev" - vpc_domain_name = "dev.inf.csp1.census.gov" - } - - cluster_config = { - cluster_mailing_list = "david.j.arnold.jr@census.gov" - tags = { - Owner = "David Arnold" - Purpose = "Testing" - } - } - - repository_teams = { - "platform-team" = "admin" - "developers" = "push" - } -} - -# Matthew's test cluster -module "matthew_cluster" { - source = "git::git@github.e.it.census.gov:CSVD/terraform-eks-deployment.git" - - name = "matthew-test-cluster" - environment = "dev" - region = "us-gov-west-1" - - # Matthew's specific configuration - account_config = { - account_name = "csvd-dev-ww" - aws_account_id = "229685449398" - environment_abbr = "dev" - } - - vpc_config = { - vpc_name = "vpc3-csvd-dev-west" - vpc_domain_name = "dev-west.inf.csp1.census.gov" - } - - cluster_config = { - cluster_mailing_list = "matthew.morgan@census.gov" - tags = { - Owner = "Matthew Morgan" - Purpose = "Testing" - } - } - - repository_teams = { - "platform-team" = "admin" - "developers" = "maintain" - } -} -``` - -**Action Items:** -- [ ] Implement multi-cluster main.tf -- [ ] Configure unique resource naming to prevent conflicts -- [ ] Set up proper state isolation per cluster -- [ ] Test parallel cluster creation - -##### Step 2.1.3: State Management Strategy -**File:** `backend.tf` -**Implementation:** -```hcl -terraform { - backend "s3" { - bucket = "csvd-terraform-state" - key = "eks-clusters-workspace/terraform.tfstate" - region = "us-gov-east-1" - dynamodb_table = "terraform-state-lock" - encrypt = true - } -} -``` - -**Action Items:** -- [ ] Configure centralized state backend -- [ ] Set up state locking mechanism -- [ ] Implement state backup strategy -- [ ] Document state management procedures - -### Phase 3: Template and Configuration Updates (Priority: Medium) - -#### 3.1 Create Module Defaults File - -**New File:** `defaults.tf` -**Implementation:** -```hcl -# defaults.tf - Default values for module configuration -# These values should not be exposed to users but are used internally by templates - -locals { - # Dynamic AWS profile generation - aws_profile = "${var.cluster_config.account_name}-${var.cluster_config.environment_abbr}" - - # Static template values (hidden from users) - repository_defaults = { - template = "template-eks-cluster" - template_owner = "SCT-Engineering" - } - - # Default module enablement - enable_all_modules = true - - # Static EKS configuration for Karpenter bootstrap node group - eks_defaults = { - instance_disk_size = 200 - ng_desired_size = 3 - ng_max_size = 10 - ng_min_size = 3 - enable_cluster_creator_admin_permissions = true - } - - # Default organization settings - organization_defaults = { - finops_project_name = "csvd_platformbaseline" - finops_project_number = "fs0000000078" - finops_project_role = "csvd_platformbaseline_app" - } -} -``` - -**Action Items:** -- [ ] Create defaults.tf file with all module defaults -- [ ] Remove hardcoded values from templates -- [ ] Update all template references to use locals -- [ ] Test template rendering with new structure - -#### 3.2 Template File Updates - -**Files to Update:** -- `templates/cluster.hcl.tf.tpl` -- `templates/README.md.tf.tpl` -- `templates/account.hcl.tf.tpl` -- `templates/region.hcl.tf.tpl` -- `templates/vpc.hcl.tf.tpl` - -**Implementation Steps:** - -##### Step 3.2.1: Update Cluster Template -**File:** `templates/cluster.hcl.tf.tpl` -**Changes Required:** -```hcl -# Use simplified variable references -cluster_name = "${cluster_name}" # Now comes from top-level name variable -aws_profile = "${aws_profile}" # Computed dynamically from locals - -# Use local defaults for EKS sizing - passed from locals, not hardcoded -eks_instance_disk_size = ${eks_instance_disk_size} -eks_ng_desired_size = ${eks_ng_desired_size} -eks_ng_max_size = ${eks_ng_max_size} -eks_ng_min_size = ${eks_ng_min_size} -enable_cluster_creator_admin_permissions = ${enable_cluster_creator_admin_permissions} - -# Organization defaults - passed from locals -finops_project_name = "${finops_project_name}" -finops_project_number = "${finops_project_number}" -finops_project_role = "${finops_project_role}" -``` - -**Action Items:** -- [ ] Update all template variable references -- [ ] Remove hardcoded values, replace with template variables -- [ ] Test template rendering with new structure -- [ ] Validate generated HCL syntax - -##### Step 3.2.2: Update Main.tf Template Calls -**File:** `main.tf` -**Implementation:** -```hcl -locals { - rendered_files = { - "root.hcl" : templatefile("${path.module}/templates/root.hcl.tf.tpl", { - environment = var.environment - }), - "environment/account.hcl" : templatefile("${path.module}/templates/account.hcl.tf.tpl", { - account_name = var.cluster_config.account_name, - aws_account_id = var.cluster_config.aws_account_id, - environment = var.environment, - environment_abbr = var.cluster_config.environment_abbr - }), - "environment/region/region.hcl" : templatefile("${path.module}/templates/region.hcl.tf.tpl", { - aws_region = var.region, - environment = var.environment - }), - "environment/region/vpc/vpc.hcl" : templatefile("${path.module}/templates/vpc.hcl.tf.tpl", { - vpc_name = var.cluster_config.vpc_name, - vpc_domain_name = var.cluster_config.vpc_domain_name, - environment = var.environment, - aws_region = var.region - }), - "environment/region/vpc/cluster/cluster.hcl" : templatefile("${path.module}/templates/cluster.hcl.tf.tpl", { - cluster_name = var.name, - cluster_mailing_list = var.cluster_config.cluster_mailing_list, - aws_profile = local.aws_profile, - eks_instance_disk_size = local.eks_defaults.instance_disk_size, - eks_ng_desired_size = local.eks_defaults.ng_desired_size, - eks_ng_max_size = local.eks_defaults.ng_max_size, - eks_ng_min_size = local.eks_defaults.ng_min_size, - enable_cluster_creator_admin_permissions = local.eks_defaults.enable_cluster_creator_admin_permissions, - finops_project_name = local.organization_defaults.finops_project_name, - finops_project_number = local.organization_defaults.finops_project_number, - finops_project_role = local.organization_defaults.finops_project_role, - tags = var.cluster_config.tags, - module_enablement_overrides = var.cluster_config.module_enablement_overrides - }), - "README.md" : templatefile("${path.module}/templates/README.md.tf.tpl", { - environment = var.environment, - cluster_name = var.name, - aws_region = var.region - }) - } -} - -module "github_repo" { - source = "git::git@github.e.it.census.gov:CSVD/terraform-github-repo.git" - - name = var.name - repo_org = var.organization - github_repo_description = "EKS Cluster Configuration for ${var.name}" - github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] - force_name = var.force_name - - template_repo_org = local.repository_defaults.template_owner - template_repo = local.repository_defaults.template - - github_is_private = var.is_private - github_has_issues = true - github_has_wiki = true - github_has_projects = true - - managed_extra_files = [ - for path, content in local.rendered_files : { - path = path - content = content - } - ] - archive_on_destroy = false - github_org_teams = [ - for team, permission in var.repository_teams : { - team_name = team - permission = permission - slug = lower(replace(team, " ", "-")) - id = null - bypass_rules = false - } - ] -} -``` - -**Action Items:** -- [ ] Update main.tf to pass local values to templates -- [ ] Remove repository template variables from variables.tf -- [ ] Test module with updated template calls -- [ ] Validate generated repository content - -### Phase 4: Testing and Validation (Priority: High) - -#### 4.1 Module Validation - -**Test Cases:** -1. **Single Cluster Creation** - - Deploy using simplified interface - - Verify repository creation - - Validate generated cluster configuration - - Test team permissions - -2. **Multi-Cluster Workspace** - - Deploy multiple clusters simultaneously - - Verify no resource conflicts - - Test state isolation - - Validate unique naming - -3. **Variable Migration** - - Test backward compatibility - - Validate dynamic profile generation - - Verify template rendering - -**Implementation Steps:** - -##### Step 4.1.1: Create Test Suite -**File:** `tests/module_test.go` -**Action Items:** -- [ ] Create automated test suite -- [ ] Test variable validation -- [ ] Test template rendering -- [ ] Test GitHub repository creation -- [ ] Test multi-cluster scenarios - -##### Step 4.1.2: Manual Testing Protocol -**Action Items:** -- [ ] Test with David's configuration -- [ ] Test with Matthew's configuration -- [ ] Test workspace creation and management -- [ ] Validate generated repositories -- [ ] Test cluster deployment end-to-end - -### Phase 5: Documentation and Demo Preparation (Priority: Medium) - -#### 5.1 Documentation Updates - -**Files to Create/Update:** -- `README.md` - Updated usage instructions -- `docs/MIGRATION.md` - Migration guide from old interface -- `docs/WORKSPACE_SETUP.md` - Workspace setup instructions -- `docs/TROUBLESHOOTING.md` - Common issues and solutions - -**Action Items:** -- [ ] Document new simplified interface -- [ ] Create migration guide for existing users -- [ ] Document workspace setup process -- [ ] Create troubleshooting guide - -#### 5.2 Demo Preparation - -**Demo Objectives:** -1. Show simplified user interface -2. Demonstrate multi-cluster workspace -3. Show conflict resolution -4. Demonstrate end-to-end workflow - -**Demo Script:** -1. **Introduction** (5 minutes) - - Current challenges with interface - - Goals of the improvement - -2. **Interface Simplification** (10 minutes) - - Before/after comparison - - Reduced complexity - - Hidden implementation details - -3. **Workspace Management** (10 minutes) - - Multi-cluster deployment - - State management - - Conflict avoidance - -4. **Live Demo** (15 minutes) - - Deploy test cluster - - Show generated repository - - Demonstrate team access - -**Action Items:** -- [ ] Prepare demo environment -- [ ] Create demo script -- [ ] Prepare test configurations -- [ ] Schedule dry run for September 16 - -## Implementation Timeline - -### September 15, 2025 (Completed) -- [x] Complete action items analysis -- [x] Create implementation plan -- [x] Begin Phase 1: Variable consolidation - -### September 16, 2025 (Today - COMPLETED) -- [x] **COMPLETED:** Create defaults.tf file with all module defaults -- [x] **COMPLETED:** Update variables.tf to remove redundant variables -- [x] **COMPLETED:** Update main.tf to pass local values to templates -- [x] **COMPLETED:** Fix locals.tf to use new simplified interface -- [x] **COMPLETED:** Fix validation errors and template variable references -- [x] **COMPLETED:** Update examples to use simplified interface -- [x] **COMPLETED:** Terraform validation and initialization successful -- [ ] **READY FOR NEXT:** Begin Phase 2: Workspace creation (eks-clusters-workspace) -- [ ] **SCHEDULED:** Conduct dry run demo -- [ ] **PENDING:** Final testing and validation - -### September 17, 2025 -- [ ] Final documentation updates -- [ ] Live demonstration to team -- [ ] Gather feedback and plan next steps - -## Current Status - September 16, 2025 - -### ✅ Completed Implementations - -#### Phase 1: Module Interface Cleanup - **COMPLETED** - -1. **Created defaults.tf file** - All module defaults are now centralized: - - Dynamic AWS profile generation from account_name + environment_abbr - - Static repository template values (hidden from users) - - EKS bootstrap node group defaults for Karpenter - - Organization default settings (FinOps configurations) - -2. **Updated variables.tf** - Simplified user interface: - - Removed redundant variables: `repository_template`, `repository_template_owner` - - Cleaned up `cluster_config` object to only include user-configurable values - - Moved internal variables to separate sections for backward compatibility - - Consolidated naming to single `name` variable for both repository and cluster - -3. **Fixed main.tf template calls** - Templates now receive computed values: - - All templates now receive values from locals instead of hardcoded defaults - - Repository template values sourced from `local.repository_defaults` - - EKS sizing values sourced from `local.eks_defaults` - - Organization settings sourced from `local.organization_defaults` - -4. **Updated locals.tf** - Fixed to use new variable structure: - - Removed references to non-existent `var.cluster_config.cluster_name` - - Updated `config_json` to use `var.name` for cluster name - - Fixed all template variable references - -5. **Updated examples** - Simplified user interface demonstrated: - - Basic example now uses consolidated variable structure - - Removed all redundant configuration options - - Clear demonstration of minimal required inputs - -### 🔄 Next Steps - -#### Phase 2: Workspace Structure Implementation - **READY TO START** - -1. **Create eks-clusters-workspace Repository** - - Repository name corrected from "terraform-eks-workspace" to "eks-clusters-workspace" - - Will demonstrate multi-cluster management without conflicts - - Includes examples from both David and Matthew's configurations - -2. **Template Variable Handling - ARCHITECTURE DECISION IMPLEMENTED** - - Templates now properly receive variables instead of using hardcoded values - - All defaults defined in defaults.tf and passed via templatefile() calls - - This ensures generated configurations reflect computed values - -### 📊 Validation Results - -- ✅ **Terraform Validation**: PASSED -- ✅ **Terraform Init**: PASSED -- ✅ **Template Rendering**: WORKING -- ✅ **Variable References**: RESOLVED -- ✅ **Module Loading**: SUCCESSFUL - -### 🏗️ Implementation Architecture - -The implemented solution follows the call notes discussion: - -1. **Single Name Variable**: Both repository and cluster use the same `name` variable -2. **Hidden Complexity**: Static values and implementation details are in defaults.tf -3. **Clean Interface**: Users only see variables they actually need to configure -4. **Template Architecture**: Values are computed in locals and passed to templates - -### 🔧 Technical Debt Resolved - -- **Variable Duplication**: Eliminated separate repository and cluster naming -- **Hardcoded Values**: Moved all defaults to centralized location -- **Template Issues**: Fixed template variable passing instead of hardcoding -- **Interface Complexity**: Reduced user-facing variables by ~60% - -## Risk Mitigation - -### Potential Issues and Solutions - -1. **Breaking Changes** - - **Risk:** Existing users may have configurations that break - - **Mitigation:** Create migration guide and maintain backward compatibility where possible - -2. **State Conflicts** - - **Risk:** Multiple users working in same workspace - - **Mitigation:** Implement proper state locking and isolation - -3. **Variable Validation** - - **Risk:** New interface may not validate properly - - **Mitigation:** Comprehensive testing and validation suite - -4. **Repository Generation** - - **Risk:** Template changes may break repository generation - - **Mitigation:** Test all templates with new variable structure - -## Success Criteria - -1. **User Interface** - - [ ] 50% reduction in required variables - - [ ] Clear separation of user-configurable vs. internal values - - [ ] Intuitive configuration structure - -2. **Workspace Management** - - [ ] Multiple clusters can be managed without conflicts - - [ ] State isolation between cluster configurations - - [ ] Clear ownership and access patterns - -3. **Documentation** - - [ ] Complete migration guide - - [ ] Clear setup instructions - - [ ] Working examples for all scenarios - -4. **Demo Success** - - [ ] Successful live demonstration - - [ ] Positive team feedback - - [ ] Clear next steps identified - -## Next Steps After Implementation - -1. **Rollout Strategy** - - Gradual migration of existing clusters - - Training for team members - - Support for migration questions - -2. **Monitoring and Feedback** - - Track usage patterns - - Gather user feedback - - Identify areas for further improvement - -3. **Future Enhancements** - - Integration with CI/CD pipelines - - Advanced workspace management features - - Automated cluster lifecycle management - -## Contact and Responsibilities - -- **David John Arnold Jr.**: Module development, workspace setup, testing -- **Matthew Creal Morgan**: Interface design review, testing, documentation review -- **Team Demo**: September 17, 2025 - ---- - -*This implementation plan is based on the call notes from September 15, 2025, and should be updated as implementation progresses.* \ No newline at end of file diff --git a/docs/callnotes-09152025.txt b/docs/callnotes-09152025.txt deleted file mode 100644 index f763c65..0000000 --- a/docs/callnotes-09152025.txt +++ /dev/null @@ -1,1611 +0,0 @@ -WEBVTT - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4249-0 -00:00:06.437 --> 00:00:07.117 -All right, cool. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4253-0 -00:00:07.117 --> 00:00:09.637 -Yeah. So we should be recording them. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4254-0 -00:00:08.377 --> 00:00:08.937 -All right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4257-0 -00:00:09.677 --> 00:00:10.557 -So yeah, all right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4259-0 -00:00:10.997 --> 00:00:11.357 -OK. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4263-0 -00:00:11.357 --> 00:00:13.037 -So a couple of things that you should -clean up. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4290-0 -00:00:14.567 --> 00:00:19.489 -This is essentially what you've defined -as your inputs that you're expecting - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4281-0 -00:00:18.297 --> 00:00:18.497 -Mm-hmm. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4290-1 -00:00:19.489 --> 00:00:21.087 -other people to fill out. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4296-0 -00:00:21.567 --> 00:00:25.687 -Compare this to what you had previously. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4303-0 -00:00:27.447 --> 00:00:29.207 -In the config JSON. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4306-0 -00:00:30.237 --> 00:00:30.517 -Right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4326-0 -00:00:31.207 --> 00:00:36.559 -Right. -So if we look at our config JSON as kind - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4326-1 -00:00:36.559 --> 00:00:41.687 -of the minimal object that we want to -handle. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4335-0 -00:00:44.157 --> 00:00:46.957 -This is the only thing that should be -modified. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4349-0 -00:00:48.487 --> 00:00:52.201 -In this scenario, -you're expecting people to go in and - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4349-1 -00:00:52.201 --> 00:00:53.687 -directly modify these. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4376-0 -00:00:56.247 --> 00:01:02.245 -Variable definitions and that probably -won't work long term like ideally we want - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4376-1 -00:01:02.245 --> 00:01:08.465 -to be able to pass and maintain an object -against the module call and that would be - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4376-2 -00:01:08.465 --> 00:01:08.687 -it. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4379-0 -00:01:08.837 --> 00:01:09.717 -Does that make sense? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4399-0 -00:01:10.827 --> 00:01:14.285 -Yeah, OK. -I mean basically what you're telling me - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4399-1 -00:01:14.285 --> 00:01:18.227 -is that the input schema from a user -doesn't make sense. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4414-0 -00:01:18.227 --> 00:01:22.429 -And config dot Jason is a better model -for me to follow as far as the interface - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4414-1 -00:01:22.429 --> 00:01:23.427 -to the module goes. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4419-0 -00:01:23.737 --> 00:01:27.137 -Yes, that's that's exactly correct now. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4417-0 -00:01:24.877 --> 00:01:25.077 -Hey. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4444-0 -00:01:30.537 --> 00:01:37.014 -Some of this is duplicative and can be -removed or referenced differently because, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4444-1 -00:01:37.014 --> 00:01:41.358 -like Buster, -Dur is just the same as cluster name like - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4444-2 -00:01:41.358 --> 00:01:43.017 -I've told you before. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4449-0 -00:01:44.557 --> 00:01:45.957 -So why are we? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4456-0 -00:01:46.517 --> 00:01:48.597 -Why are we creating it in two places? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4482-0 -00:01:48.757 --> 00:01:53.625 -We don't need AWS profile because the way -the modules handle it, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4482-1 -00:01:53.625 --> 00:01:59.317 -it builds it dynamically from the other -two account name and account ID so. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4495-0 -00:02:00.977 --> 00:02:03.177 -We don't need AWS profile at all. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4501-0 -00:02:03.177 --> 00:02:06.417 -This enable all modules true doesn't need -to be there either. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4505-0 -00:02:06.457 --> 00:02:08.577 -That's in default assumption. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4511-0 -00:02:10.377 --> 00:02:12.417 -We should probably drop slim schedule. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4520-0 -00:02:12.927 --> 00:02:17.607 -That's a different topic though, -so not a huge word worry about that. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4532-0 -00:02:19.137 --> 00:02:23.857 -This owner or whatever should be the same -as the cluster mailing list. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4545-0 -00:02:24.637 --> 00:02:27.077 -Just we're collecting an e-mail address, -right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4574-0 -00:02:27.477 --> 00:02:32.763 -And the rest of it, -like we don't even really need these here - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4542-0 -00:02:27.487 --> 00:02:27.567 -No. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4574-1 -00:02:32.763 --> 00:02:39.755 -because these are the defaults and what -they should always be the only thing that - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4574-2 -00:02:39.755 --> 00:02:43.677 -we need or collect these for is to to -inform. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4598-0 -00:02:44.977 --> 00:02:48.878 -The node group for Carpenter, -which will then create the other node - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4598-1 -00:02:48.878 --> 00:02:50.657 -groups that will run workloads. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4607-0 -00:02:51.197 --> 00:02:53.037 -OK, so that. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4605-0 -00:02:52.217 --> 00:02:53.737 -So those are relatively static. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4608-0 -00:02:54.127 --> 00:02:54.727 -Values. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4650-0 -00:02:55.377 --> 00:02:58.909 -All right, -so those static values for the eks - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4650-1 -00:02:58.909 --> 00:03:04.822 -instance, disk size, eksng, desired size, -eksng Max size and eksng min size. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4650-2 -00:03:04.822 --> 00:03:09.045 -Those should still be configurable -through the module, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4650-3 -00:03:09.045 --> 00:03:12.577 -but not necessarily presented to the user -as. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4661-0 -00:03:14.097 --> 00:03:16.337 -You you'd still want to be able to -override that though, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4665-0 -00:03:16.657 --> 00:03:17.937 -We don't have a bigger. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4666-0 -00:03:17.247 --> 00:03:19.247 -No need, they're static. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4693-0 -00:03:21.737 --> 00:03:26.100 -So again, -these values only govern the creation of - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4693-1 -00:03:26.100 --> 00:03:31.917 -the Carpenter node group, -and then the Carpenter node group creates - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4693-2 -00:03:31.917 --> 00:03:33.457 -other node groups. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4697-0 -00:03:36.537 --> 00:03:38.297 -Oh, OK, got you. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4703-0 -00:03:40.187 --> 00:03:41.347 -I I think I got you. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4714-0 -00:03:41.347 --> 00:03:45.497 -So that's configuring the Carpenter, -but then Carpenter, it manages stuff. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4718-0 -00:03:46.357 --> 00:03:46.917 -Yeah, totally. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4720-0 -00:03:46.557 --> 00:03:47.077 -Right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4721-0 -00:03:47.397 --> 00:03:48.317 -Alright, sweet. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4723-0 -00:03:51.757 --> 00:03:52.397 -So yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4736-0 -00:03:53.607 --> 00:03:59.287 -That gives us like the minimum, -and that's a a clean interface. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4767-0 -00:04:00.777 --> 00:04:06.328 -Then basically we would need something -like drop a config JSON and run the - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4767-1 -00:04:06.328 --> 00:04:10.028 -module, -add the config JSON to the resulting repo - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4767-2 -00:04:10.028 --> 00:04:14.617 -as the record of what generated, -and then I think we've done. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4770-0 -00:04:17.567 --> 00:04:18.127 -All right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4781-0 -00:04:18.127 --> 00:04:23.127 -So you still want to have terraform read -from a config dot Jason file. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4798-0 -00:04:24.367 --> 00:04:29.354 -I don't care about the format, -but we will need it to read from some - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4798-1 -00:04:29.354 --> 00:04:30.727 -kind of input, yes. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4804-0 -00:04:34.327 --> 00:04:38.047 -Rather than just having it configured in -the code itself. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4806-0 -00:04:38.707 --> 00:04:39.467 -Well. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4810-0 -00:04:42.177 --> 00:04:43.257 -Again, consider. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4824-0 -00:04:45.337 --> 00:04:48.137 -The least intelligent colleague you've -ever met. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4835-0 -00:04:48.777 --> 00:04:54.181 -Would you want them to go into this -module and tinker with these values - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4835-1 -00:04:54.181 --> 00:04:54.857 -directly? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4850-0 -00:04:54.857 --> 00:04:58.803 -Or would you want them in a place where -they couldn't possibly damage something - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4850-1 -00:04:58.803 --> 00:04:59.937 -they didn't understand? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4871-0 -00:05:03.867 --> 00:05:07.279 -Yeah. OK. -That makes like I totally understand. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4871-1 -00:05:07.279 --> 00:05:12.041 -Hiding some of or, you know, -putting some of this module but don't - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4871-2 -00:05:12.041 --> 00:05:13.747 -exposing it to the user. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4879-0 -00:05:15.597 --> 00:05:18.917 -It's like, -are we ever gonna need to change that? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4910-0 -00:05:19.037 --> 00:05:24.437 -Are we ever going to need to change that -these two values of repository template - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4910-1 -00:05:24.437 --> 00:05:28.504 -and template repo org? -Those are not something that we would - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4901-0 -00:05:26.107 --> 00:05:26.667 -Yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4910-2 -00:05:28.504 --> 00:05:30.637 -expect a user to want to change. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4924-0 -00:05:32.137 --> 00:05:36.217 -So when we're thinking about this -example's basic as the interface. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4926-0 -00:05:36.827 --> 00:05:37.347 -No. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4938-0 -00:05:37.857 --> 00:05:40.097 -You know the same kind of of feedback. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4947-0 -00:05:40.097 --> 00:05:44.817 -We can remove some of the things that we -know we need static values for. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4958-0 -00:05:45.127 --> 00:05:50.167 -Or we can hide things that we know should -not be altered. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4960-0 -00:05:50.487 --> 00:05:50.927 -Yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4985-0 -00:05:51.657 --> 00:05:58.186 -And make this as minimal as possible so -nobody could come in here and be confused - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4985-1 -00:05:58.186 --> 00:06:00.017 -about what our outputs. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/4987-0 -00:06:01.077 --> 00:06:01.797 -Yeah. OK. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5021-0 -00:06:03.367 --> 00:06:08.765 -So I mean I can I can set this up to -where it's requiring a config dot Jason - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5021-1 -00:06:08.765 --> 00:06:14.234 -in your working directory when you call -Terraform apply and just have it read - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5021-2 -00:06:14.234 --> 00:06:16.407 -directly from config dot Jason. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5024-0 -00:06:17.727 --> 00:06:19.407 -That would be that would be fine. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5026-0 -00:06:20.357 --> 00:06:20.797 -OK. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5056-0 -00:06:22.757 --> 00:06:27.635 -And like one of the big things that I -liked about the about the way that we - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5033-0 -00:06:22.857 --> 00:06:23.337 -Alright. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5056-1 -00:06:27.635 --> 00:06:32.705 -were doing it before when we generated -these clusters is that you included the - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5050-0 -00:06:30.397 --> 00:06:30.717 -Yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5056-2 -00:06:32.705 --> 00:06:34.117 -config JSON alongside. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5089-0 -00:06:34.117 --> 00:06:37.389 -So anytime we wanted to regenerate these -clusters, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5059-0 -00:06:34.447 --> 00:06:34.727 -Yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5089-1 -00:06:37.389 --> 00:06:42.649 -all we needed to do was run the function -pointing at the config and it would spit - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5089-2 -00:06:42.649 --> 00:06:44.317 -out the same damn cluster. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5095-0 -00:06:44.317 --> 00:06:47.657 -And that's exactly the experience that -that we're going for. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5118-0 -00:06:49.207 --> 00:06:54.765 -So you do kind of still get that, -except that your config dot JSong isn't - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5118-1 -00:06:54.765 --> 00:06:55.967 -in the repo now. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5127-0 -00:06:55.967 --> 00:07:00.047 -Now it's in the workspace where you're -calling terraform apply from right? So. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5138-0 -00:07:00.507 --> 00:07:03.448 -So OK, -if there's a different workflow that - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5138-1 -00:07:03.448 --> 00:07:05.387 -makes more sense, what is it? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5151-0 -00:07:05.467 --> 00:07:10.427 -Show me 'cause like I this is this is -what's making sense to me. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5157-0 -00:07:10.427 --> 00:07:13.267 -But that's because I have a narrow -perspective, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5175-0 -00:07:13.607 --> 00:07:19.436 -So I'm totally with you on minimizing -what we provide or exposed to the user - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5175-1 -00:07:19.436 --> 00:07:20.647 -for sure, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5173-0 -00:07:20.207 --> 00:07:20.407 -Mm-hmm. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5182-0 -00:07:20.647 --> 00:07:24.047 -That's having the defaults in the module -itself. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5185-0 -00:07:24.477 --> 00:07:24.917 -Yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5196-0 -00:07:25.567 --> 00:07:28.927 -But we actually kinda already did some of -the workflow right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5205-0 -00:07:28.927 --> 00:07:34.098 -Like when you changed the values in -Terraform it updated the files in the - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5205-1 -00:07:34.098 --> 00:07:34.447 -repo. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5207-0 -00:07:34.917 --> 00:07:35.517 -Right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5223-0 -00:07:37.367 --> 00:07:39.528 -And so yeah, I mean, -that's kind of what I was imagining the - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5223-1 -00:07:39.528 --> 00:07:39.847 -workflow. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5238-0 -00:07:39.667 --> 00:07:43.907 -OK. Well and and I OK, -I'm I'm way happy with that. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5246-0 -00:07:43.907 --> 00:07:48.507 -Then my question would be, -where does this artifact get stored? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5249-0 -00:07:51.357 --> 00:07:52.437 -Which artifact? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5264-0 -00:07:52.987 --> 00:07:57.096 -The artifact that was used to generate -the repo that generated the code for the - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5264-1 -00:07:57.096 --> 00:07:57.507 -cluster. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5279-0 -00:07:58.177 --> 00:08:01.768 -I mean that's the Terraform Workspace -where you're calling terraform apply from - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5279-1 -00:08:01.768 --> 00:08:02.217 -right? So. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5295-0 -00:08:02.617 --> 00:08:07.804 -But this is Terraform Ecast deployment -where we'll call all deploys from or all - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5295-1 -00:08:07.804 --> 00:08:08.777 -creations from. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5337-0 -00:08:09.167 --> 00:08:12.419 -Well, I mean, -we're we were training an example - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5307-0 -00:08:10.297 --> 00:08:10.857 -So in here. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5337-1 -00:08:12.419 --> 00:08:15.875 -directory as a workspace so that we could -test it, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5337-2 -00:08:15.875 --> 00:08:21.295 -but in reality we would have a directory -that would have files in it that would - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5337-3 -00:08:21.295 --> 00:08:25.428 -have multiple calls to the SharePoint EKS -deployment module. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5337-4 -00:08:25.428 --> 00:08:27.257 -And that is like where all. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5349-0 -00:08:27.247 --> 00:08:30.127 -The state would live, -and it's where that artifact would live. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5368-0 -00:08:30.167 --> 00:08:34.607 -The artifact being the code itself, -so it's gonna interest to call it the - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5368-1 -00:08:34.607 --> 00:08:37.727 -artifact, -because it's really more like the seed of - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5368-2 -00:08:37.727 --> 00:08:38.327 -it, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5387-0 -00:08:38.437 --> 00:08:43.197 -So you're configuring it just like I -would have made dot TF. Any examples? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5379-0 -00:08:38.717 --> 00:08:39.197 -Right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5380-0 -00:08:41.817 --> 00:08:41.897 -So. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5392-0 -00:08:43.197 --> 00:08:45.157 -Basic directory, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5390-0 -00:08:43.637 --> 00:08:44.077 -Yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5412-0 -00:08:45.197 --> 00:08:48.751 -But you would have a workspace that would -be multiple of those calls, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5412-1 -00:08:48.751 --> 00:08:51.797 -alright and everything would be capturing -code right there. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5422-0 -00:08:53.327 --> 00:08:58.527 -Well, -so if config JSON was in the VAR file. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5431-0 -00:09:00.057 --> 00:09:02.337 -And nothing in here needed to be altered. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5432-0 -00:09:02.537 --> 00:09:03.777 -We'd have the same. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5435-0 -00:09:05.657 --> 00:09:06.617 -Experience. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5446-0 -00:09:09.277 --> 00:09:13.277 -And then you could just have a VAR file -per cluster in this repo. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5461-0 -00:09:15.497 --> 00:09:19.005 -And save it in this repo. -I guess because this repo becomes the - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5461-1 -00:09:19.005 --> 00:09:19.937 -workspace, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5482-0 -00:09:23.627 --> 00:09:27.837 -So it would be a new repo that is -becoming a workspace and that workspace - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5482-1 -00:09:27.837 --> 00:09:30.567 -is calling the Terraform Uks deployment -module. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5484-0 -00:09:31.077 --> 00:09:32.397 -Like multiple times. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5489-0 -00:09:33.257 --> 00:09:34.817 -How would you build that though? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5495-0 -00:09:34.897 --> 00:09:36.457 -How would you build that workspace? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5506-0 -00:09:36.457 --> 00:09:42.257 -I guess I'm confused 'cause I you -differentiate between a workspace and? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5512-0 -00:09:47.007 --> 00:09:49.527 -So like right now the basic directory. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5519-0 -00:09:51.007 --> 00:09:52.607 -We're reading that as a workspace, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5551-0 -00:09:54.207 --> 00:10:00.147 -And we have a call to the teraform uks -module within that workspace and editing - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5551-1 -00:10:00.147 --> 00:10:06.087 -our state and we want to change a value -for the cluster that's being generated. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5567-0 -00:10:06.167 --> 00:10:10.548 -We would do it just by modifying the code -that you see in main dot TF. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5567-1 -00:10:10.548 --> 00:10:11.967 -There right as you see. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5579-0 -00:10:11.567 --> 00:10:17.432 -So, -but what you would not want to do is copy - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5573-0 -00:10:14.037 --> 00:10:14.237 -Hello. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5579-1 -00:10:17.432 --> 00:10:19.727 -basic paste basic. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5582-0 -00:10:21.217 --> 00:10:22.417 -Rename basic. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5588-0 -00:10:24.977 --> 00:10:26.457 -To my new cluster. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5601-0 -00:10:28.727 --> 00:10:30.899 -Yeah, I mean, -you wouldn't have one directory per - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5597-0 -00:10:28.857 --> 00:10:29.497 -And. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5601-1 -00:10:30.899 --> 00:10:31.247 -cluster. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5608-0 -00:10:31.247 --> 00:10:34.687 -You wouldn't be able to have multiple -clusters in a directory. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5609-0 -00:10:37.637 --> 00:10:38.117 -Alright. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5612-0 -00:10:39.937 --> 00:10:40.897 -No, you lost me. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5617-0 -00:10:42.097 --> 00:10:43.937 -OK. Are you still sharing? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5624-0 -00:10:45.137 --> 00:10:49.657 -No. Let me share that again. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5634-0 -00:10:49.737 --> 00:10:53.737 -So yeah, So what I did was I copied basic. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5666-0 -00:10:54.257 --> 00:10:57.871 -I renamed it to my new cluster and then -you know. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5666-1 -00:10:57.871 --> 00:11:03.654 -Then we could have the main TF rewritten -with the values for my new cluster and - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5666-2 -00:11:03.654 --> 00:11:08.497 -that would be one way to maintain it in -Terraform Ecas deployment. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5673-0 -00:11:08.497 --> 00:11:12.577 -But you're telling me? Obviously, -that's not the right way to do it. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5675-0 -00:11:13.107 --> 00:11:13.587 -Yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5704-0 -00:11:14.557 --> 00:11:17.784 -Clearly. -So you're saying we would have a - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5704-1 -00:11:17.784 --> 00:11:22.931 -different repo that would be the -workspaces for this where all the - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5704-2 -00:11:22.931 --> 00:11:26.157 -clusters would be generated that would be. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5697-0 -00:11:24.937 --> 00:11:25.577 -It would. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5714-0 -00:11:25.897 --> 00:11:29.007 -It would be a singular workspace, -but it would be where all the clusters - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5714-1 -00:11:29.007 --> 00:11:29.817 -are generated, yes. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5727-0 -00:11:30.227 --> 00:11:34.467 -Wouldn't that be this platform, TG Infra, -where we link it all in? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5731-0 -00:11:35.957 --> 00:11:37.037 -Yeah, we could do that. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5740-0 -00:11:38.137 --> 00:11:41.137 -Well, I you say that, yeah, -we could do that. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5745-0 -00:11:41.137 --> 00:11:42.937 -Is that not the correct answer? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5748-0 -00:11:43.577 --> 00:11:45.217 -Is it supposed to be something else? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5749-0 -00:11:49.617 --> 00:11:49.737 -Umm. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5766-0 -00:11:54.507 --> 00:11:59.862 -So we were discussing that before, -right in each cluster that's under lab - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5766-1 -00:11:59.862 --> 00:12:00.947 -slash clusters. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5776-0 -00:12:01.627 --> 00:12:05.867 -We were saying before that that could be -a like a get submodule, right? And we. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5782-0 -00:12:05.457 --> 00:12:08.097 -And that's what this is, yes. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5778-0 -00:12:07.217 --> 00:12:07.257 -Are. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5796-0 -00:12:08.417 --> 00:12:12.702 -Yeah, -and that would be each invocation of the - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5796-1 -00:12:12.702 --> 00:12:14.617 -template repo, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5815-0 -00:12:14.617 --> 00:12:19.246 -So each cluster having its own repo that -spawned from the template repo, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5815-1 -00:12:19.246 --> 00:12:23.177 -they would all be cloned into there as -get submodules, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5825-0 -00:12:23.597 --> 00:12:26.477 -Right. And that's what we're doing today. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5821-0 -00:12:25.037 --> 00:12:25.277 -OK. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5828-0 -00:12:26.877 --> 00:12:27.597 -Yeah, totally. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5832-0 -00:12:27.717 --> 00:12:28.637 -So that works. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5841-0 -00:12:28.677 --> 00:12:32.317 -And those are fully the artifacts of -setting it up in Terraform. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5867-0 -00:12:33.937 --> 00:12:38.268 -But like, -if we're in the basic directory right on - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5867-1 -00:12:38.268 --> 00:12:43.617 -uks, that is an example of a workspace. -We only have one call. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5885-0 -00:12:45.297 --> 00:12:49.524 -To to the ECS deployment module on there -and it's not right there that you're - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5885-1 -00:12:49.524 --> 00:12:50.337 -showing, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5884-0 -00:12:50.207 --> 00:12:50.407 -Mm-hmm. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5896-0 -00:12:52.017 --> 00:12:55.137 -We would have a workspace that would have -multiple of these. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5912-0 -00:12:55.607 --> 00:12:58.402 -If you wanna put it closer, -platform PG in front of that, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5912-1 -00:12:58.402 --> 00:12:59.847 -that would totally make sense. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5926-0 -00:13:00.087 --> 00:13:05.613 -We would just have a space where we're -calling this module multiple times, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5926-1 -00:13:05.613 --> 00:13:08.487 -one per cluster. When you get the repo. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5955-0 -00:13:10.017 --> 00:13:13.870 -Where that cluster lives now. -Then you would clone it into there where - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5955-1 -00:13:13.870 --> 00:13:16.257 -you're highlighting CSVD platform lab, -DGA. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5960-0 -00:13:16.937 --> 00:13:20.177 -You would clone that and as a git -submodule. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5963-0 -00:13:22.447 --> 00:13:24.287 -That's what I'm asking is. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5982-0 -00:13:30.977 --> 00:13:35.494 -So what was kind of nice about the Lambda -thing was that we didn't need to have a - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5975-0 -00:13:34.107 --> 00:13:34.307 -Mm-hmm. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5982-1 -00:13:35.494 --> 00:13:37.697 -separate place where stuff lived, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6002-0 -00:13:37.697 --> 00:13:43.417 -Like you would just fire off a payload at -the Lambda and then we would be concerned - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6002-1 -00:13:43.417 --> 00:13:45.937 -about what the Lambda created, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6019-0 -00:13:45.977 --> 00:13:52.078 -And that we would fully clone into the -lab clusters directory there as a - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/5999-0 -00:13:46.287 --> 00:13:46.487 -Mm-hmm. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6019-1 -00:13:52.078 --> 00:13:56.257 -submodule, -but we wouldn't need to also maintain. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6026-0 -00:13:57.017 --> 00:13:58.857 -Like the original source of that stuff, -right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6034-0 -00:13:59.327 --> 00:14:02.247 -Right. But they were in Terraform. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6030-0 -00:13:59.787 --> 00:14:00.427 -Right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6053-0 -00:14:02.247 --> 00:14:06.399 -In Terraform, a stateful. -Now we do need that second place of where - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6053-1 -00:14:06.399 --> 00:14:10.367 -we're managing the state for what -actually created those rebels. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6067-0 -00:14:10.767 --> 00:14:15.207 -So now we have to care about what -generated it and what it generated. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6083-0 -00:14:16.727 --> 00:14:23.664 -So I'm almost thinking we could -essentially create a teragrunt HCL that - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6068-0 -00:14:16.777 --> 00:14:16.857 -So. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6083-1 -00:14:23.664 --> 00:14:25.687 -calls eks deployment. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6087-0 -00:14:27.177 --> 00:14:28.297 -And passes in. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6090-0 -00:14:30.577 --> 00:14:31.777 -The variables. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6096-0 -00:14:34.777 --> 00:14:39.217 -As inputs that would create the. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6103-0 -00:14:40.897 --> 00:14:42.737 -Repository as the 1st. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6107-0 -00:14:45.057 --> 00:14:47.217 -Action before it creates. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6120-0 -00:14:47.647 --> 00:14:50.887 -The cluster, -so essentially this would be like the - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6120-1 -00:14:50.887 --> 00:14:51.967 -bootstrap module. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6127-0 -00:14:55.567 --> 00:14:58.927 -Yeah, you could probably do that, I guess, -yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6134-0 -00:15:00.967 --> 00:15:03.887 -You would still have like the source of -what? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6155-0 -00:15:05.377 --> 00:15:08.952 -Mason is gonna use to generate something, -and then you would have the artifact that - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6155-1 -00:15:08.952 --> 00:15:12.017 -it actually generates and you would still -have to manage both of those. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6165-0 -00:15:13.007 --> 00:15:15.767 -Well, -so and that's that's what I'm thinking. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6168-0 -00:15:15.807 --> 00:15:17.887 -I'm thinking like OK. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6175-0 -00:15:17.887 --> 00:15:22.327 -So we create the the repo directory here. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6186-0 -00:15:24.937 --> 00:15:30.417 -What if I created like a repo HCL that -was? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6203-0 -00:15:32.097 --> 00:15:39.257 -Almost exactly like this, -only instead of calling eks it calls. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6205-0 -00:15:39.257 --> 00:15:41.577 -You know Terraform Ecas deployment. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6214-0 -00:15:42.827 --> 00:15:47.187 -And then we pass in inputs from config -JSON. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6216-0 -00:15:49.237 --> 00:15:49.637 -So. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6229-0 -00:15:51.597 --> 00:15:57.357 -Eks so that this eks module then has a -dependency on. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6241-0 -00:15:59.137 --> 00:16:02.417 -On repo setup or on Ecas deployment I -mean. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6260-0 -00:16:05.577 --> 00:16:10.440 -So what I'm saying is we we could -actually make it like there's a - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6260-1 -00:16:10.440 --> 00:16:12.577 -dependency on, let's call it. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6266-0 -00:16:15.257 --> 00:16:17.457 -You form eks deployment. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6297-0 -00:16:19.877 --> 00:16:26.308 -So that way before we run the uks module, -we run Ecas deployment with the inputs to - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6297-1 -00:16:26.308 --> 00:16:32.357 -generate the repo and then as part of -that it'll start generating the cluster. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6304-0 -00:16:35.707 --> 00:16:38.392 -Yeah, -I don't think that's gonna work the way - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6304-1 -00:16:38.392 --> 00:16:39.267 -you want it to. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6322-0 -00:16:41.137 --> 00:16:47.075 -And I think it would be kind of confusing -because we would have like a circular - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6322-1 -00:16:47.075 --> 00:16:48.857 -dependency sorta, right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6323-0 -00:16:48.927 --> 00:16:49.607 -Also. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6332-0 -00:16:51.347 --> 00:16:56.027 -Our peregrine would be calling Terraform. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6336-0 -00:16:56.227 --> 00:16:58.307 -That would create the repo. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6350-0 -00:16:59.817 --> 00:17:03.577 -That contains the tarragon that we're -running when we run tarragon. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6357-0 -00:17:05.487 --> 00:17:07.847 -Oh, yeah, yeah, yeah. OK. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6360-0 -00:17:07.847 --> 00:17:08.807 -I see what you're saying. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6366-0 -00:17:10.487 --> 00:17:11.767 -Really. That's amazing, dude. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6372-0 -00:17:11.767 --> 00:17:13.927 -I'm surprised you follow that 'cause -that's confusing. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6375-0 -00:17:13.927 --> 00:17:15.607 -I have a lot of myself saying that. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6381-0 -00:17:17.077 --> 00:17:20.237 -No, no. How would I phrase that? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6400-0 -00:17:23.897 --> 00:17:28.198 -The artifacts would be present on disk -during the evaluation phase of Terra - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6386-0 -00:17:24.387 --> 00:17:24.667 -Hello. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6400-1 -00:17:28.198 --> 00:17:28.537 -Grand. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6427-0 -00:17:32.367 --> 00:17:35.675 -So. -So because the first module is the - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6427-1 -00:17:35.675 --> 00:17:41.102 -creation of the other modules until the -1st module is executed, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6427-2 -00:17:41.102 --> 00:17:45.767 -the other modules don't exist on disk to -be evaluated. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6436-0 -00:17:45.807 --> 00:17:48.487 -So you wouldn't be able to orchestrate -between them. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6432-0 -00:17:46.177 --> 00:17:46.577 -Yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6438-0 -00:17:48.967 --> 00:17:49.367 -Thought. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6440-0 -00:17:49.007 --> 00:17:49.887 -Yeah, I get you. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6444-0 -00:17:51.327 --> 00:17:52.767 -OK, OK. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6450-0 -00:17:52.767 --> 00:17:55.807 -Yeah. So that's a bad idea, -and that's fine. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6453-0 -00:17:57.477 --> 00:17:58.197 -That's fine. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6460-0 -00:17:58.197 --> 00:18:00.797 -So, OK, so then rewinding? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6476-0 -00:18:02.387 --> 00:18:07.654 -I think we just need the the simpler -interface into your module and we'll call - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6476-1 -00:18:07.654 --> 00:18:08.187 -it good. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6481-0 -00:18:09.087 --> 00:18:10.407 -Alright, sweet. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6480-0 -00:18:10.257 --> 00:18:10.937 -Sound good? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6488-0 -00:18:11.327 --> 00:18:14.304 -Yeah. -Do you want me to set up a workspace for - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6488-1 -00:18:14.304 --> 00:18:15.127 -this, though? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6491-0 -00:18:17.027 --> 00:18:18.227 -Is that the right answer? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6494-0 -00:18:18.267 --> 00:18:19.627 -Is that what we should do? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6497-0 -00:18:20.577 --> 00:18:21.857 -I think so. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6499-0 -00:18:22.187 --> 00:18:22.867 -Then yes. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6522-0 -00:18:23.467 --> 00:18:27.216 -Yeah, -right now you and I are working in like - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6506-0 -00:18:25.687 --> 00:18:25.967 -How? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6522-1 -00:18:27.216 --> 00:18:32.187 -the basic directory and you have. Yeah. -Yeah, totally, yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6518-0 -00:18:29.657 --> 00:18:31.377 -Which we should right? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6549-0 -00:18:32.187 --> 00:18:36.520 -I mean like you have your local values -stored there and I have my local values - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6549-1 -00:18:36.520 --> 00:18:40.799 -and that's what's allowing us to each -create a repo that we can check out and - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6549-2 -00:18:40.799 --> 00:18:41.347 -test with. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6578-0 -00:18:42.857 --> 00:18:47.959 -So yeah, once I create a workspace, -we will have a place where we can still - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6578-1 -00:18:47.959 --> 00:18:51.248 -both work, -but it will be a shared space for all - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6578-2 -00:18:51.248 --> 00:18:54.537 -modules and you covering values for main -dot TF. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6574-0 -00:18:52.367 --> 00:18:52.567 -Right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6603-0 -00:18:55.047 --> 00:18:59.561 -The basic directory right now if you -check that in and I pulled it like, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6603-1 -00:18:59.561 --> 00:19:04.447 -my values would be updated to reflect -yours and we have collisions, right? So. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6598-0 -00:19:02.337 --> 00:19:02.857 -Right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6626-0 -00:19:04.707 --> 00:19:08.463 -Right. -And that and that was my concern like how - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6626-1 -00:19:08.463 --> 00:19:14.594 -how do we create this in such a way that -we don't have to worry about trampling - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6626-2 -00:19:14.594 --> 00:19:16.587 -over each other over time. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6628-0 -00:19:16.937 --> 00:19:17.617 -Yeah, exactly. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6630-0 -00:19:18.057 --> 00:19:18.417 -Yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6664-0 -00:19:19.017 --> 00:19:21.992 -Can you send me your copy of Maine dot TF? -And yeah, - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6664-1 -00:19:21.992 --> 00:19:25.922 -I'll go ahead and put together a -workspace and I'll use both of these - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6641-0 -00:19:22.387 --> 00:19:22.787 -Yeah. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6664-2 -00:19:25.922 --> 00:19:28.673 -clusters. -The one that you have here and the one - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6664-3 -00:19:28.673 --> 00:19:32.097 -that I have in my version of this, -and I'll put that in the. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6674-0 -00:19:32.097 --> 00:19:34.577 -Workspace so that we can have more -concrete example of that. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6675-0 -00:19:35.147 --> 00:19:35.427 -OK. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6680-0 -00:19:36.977 --> 00:19:37.777 -Sounds good. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6682-0 -00:19:37.007 --> 00:19:39.087 -Wait right on, man. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6686-0 -00:19:40.817 --> 00:19:41.657 -Thanks for your time. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6688-0 -00:19:41.087 --> 00:19:41.727 -All right. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6692-0 -00:19:42.007 --> 00:19:43.207 -Yeah. No, no problem. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6694-0 -00:19:43.207 --> 00:19:43.647 -No problem. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6702-0 -00:19:43.687 --> 00:19:45.927 -And I'm sorry I've been in such a pain to -get ahold of. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6704-0 -00:19:46.427 --> 00:19:46.867 -Rachel. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6710-0 -00:19:47.777 --> 00:19:49.617 -Do you want to do this again tomorrow? - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6713-0 -00:19:51.827 --> 00:19:52.827 -Yes, sure, man. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6721-0 -00:19:53.097 --> 00:19:55.177 -Because that way we'll do it as a dry run. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6727-0 -00:19:55.177 --> 00:19:57.977 -For whatever we demo on Wednesday. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6729-0 -00:19:58.147 --> 00:19:59.347 -Yep, for sure. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6731-0 -00:19:59.587 --> 00:19:59.987 -Cool. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6737-0 -00:20:00.617 --> 00:20:01.937 -Right on, man. Thank you, Duke. - -34f0ecc1-4d22-45f5-b426-d851f282ce5d/6738-0 -00:20:01.667 --> 00:20:03.227 -All right, dude. Have a good evening. \ No newline at end of file diff --git a/docs/callnotes-actionitems-09152025.md b/docs/callnotes-actionitems-09152025.md deleted file mode 100644 index e5c98d3..0000000 --- a/docs/callnotes-actionitems-09152025.md +++ /dev/null @@ -1,66 +0,0 @@ -# Call Notes & Action Items - September 15, 2025 - -## Summary -Discussion between David John Arnold Jr. and Matthew Creal Morgan regarding improvements to the terraform-eks-deployment module. The main focus was on simplifying the interface for users, consolidating naming variables, and establishing a better workflow for managing multiple EKS clusters. - -## Action Items - -1. **Simplify Module Interface** - - Reduce the number of variables exposed to users - - Use the config_json structure as a model for the interface - - Remove duplicate/redundant variables (e.g., repository name and cluster name should be unified) - - Status: ✅ In Progress - -2. **Consolidate Naming Variables** - - Use a single `name` variable for both repository and cluster names - - Remove `cluster_name` from the `cluster_config` object - - Update all templates to reference the top-level `name` variable - - Status: ✅ Completed - -3. **Remove Unnecessary Variables** - - Remove `aws_profile` as it can be built dynamically from account name and ID - - Remove `enable_all_modules` as it should be a default assumption - - Status: 🔄 To Do - -4. **Update GitHub Repository Source** - - Change source from `"terraform-github-repo"` to `"git::git@github.e.it.census.gov:CSVD/terraform-github-repo.git"` - - Status: ✅ Completed - -5. **Create a Workspace for Multiple Clusters** - - Set up a shared workspace that will manage multiple clusters - - Avoid using separate directories that would cause conflicts between users - - David to create this workspace with examples from both David and Matthew - - Status: 🔄 To Do - -6. **Variable Values** - - Keep static values like `eks_instance_disk_size`, `eks_ng_desired_size`, etc. with sensible defaults in the module - - Only expose variables that users actually need to modify - - Status: 🔄 To Do - -7. **Demo Preparation** - - Schedule a follow-up meeting for September 16, 2025, to do a dry run - - Prepare for a demonstration on Wednesday, September 17, 2025 - - Status: 📅 Scheduled - -## Implementation Strategy - -### Phase 1: Module Interface Cleanup -- Refine the variables.tf file to remove redundant fields -- Update examples to reflect the new, simplified interface -- Ensure backward compatibility or provide migration path - -### Phase 2: Workspace Structure -- Create a centralized workspace for managing multiple clusters -- Demonstrate how multiple clusters can be managed without conflicts -- Document the approach for the team - -### Phase 3: Documentation and Demo -- Create clear documentation on the new workflow -- Prepare demonstration materials -- Conduct dry run on September 16 -- Present to wider team on September 17 - -## Notes -- The discussed approach uses Terraform rather than Lambda for deployment -- State management is a key consideration as we need to track both what created the repos and what the repos contain -- Need to ensure we maintain proper references to the created resources for future updates \ No newline at end of file diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 02c1152..0f07226 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -13,9 +13,20 @@ module "eks_deployment" { source = "../../" # Repository and cluster configuration - single name for both - name = "mojo-cubestore-dev" - environment = "dev" - region = "us-gov-east-1" + name = "mojo-cubestore-dev" + organization = "SCT-Engineering" + repository_mode = "create" + environment = "dev" + region = "us-gov-east-1" + + template_enabled_modules = { + eks-grafana = true + eks-kiali = true + eks-loki = true + eks-otel = true + eks-prometheus = true + eks-tempo = true + } # Cluster configuration - simplified interface cluster_config = { diff --git a/examples/centralized-management/main.tf b/examples/centralized-management/main.tf index 5e09a13..acda58a 100644 --- a/examples/centralized-management/main.tf +++ b/examples/centralized-management/main.tf @@ -17,9 +17,7 @@ module "prod_eks_cluster" { source = "../../" # Repository and cluster configuration - single name for both - name = "eks-prod-cluster" # This name is used for both the repository and cluster - template_repo_org = "HappyPathway" - repository_template = "template-eks-cluster" + name = "eks-prod-cluster" repository_teams = { "platform-team" = "admin", "prod-sre-team" = "maintain", @@ -27,39 +25,37 @@ module "prod_eks_cluster" { } # Basic settings - organization = var.github_organization - environment = "production" - region = "us-gov-west-1" - github_server_url = var.github_server_url + organization = var.github_organization + repository_mode = "create" + environment = "production" + region = "us-gov-west-1" + + template_enabled_modules = { + eks-grafana = true + eks-kiali = true + eks-loki = true + eks-otel = true + eks-prometheus = true + eks-tempo = true + } # Cluster configuration cluster_config = { - account_name = "prod-account" - aws_account_id = var.prod_account_id - environment_abbr = "prod" - aws_profile = "default" - vpc_name = "prod-vpc" - vpc_domain_name = "prod.example.com" - cluster_mailing_list = "prod-alerts@example.com" - eks_instance_disk_size = 200 - eks_ng_desired_size = 5 - eks_ng_max_size = 10 - eks_ng_min_size = 3 - organization = "census:ocio:csvd" - finops_project_name = "csvd_platformbaseline" - finops_project_number = "fs0000000078" - finops_project_role = "csvd_platformbaseline_app" + account_name = "prod-account" + aws_account_id = var.prod_account_id + environment_abbr = "prod" + vpc_name = "prod-vpc" + vpc_domain_name = "prod.example.com" + cluster_mailing_list = "prod-alerts@example.com" + organization = "census:ocio:csvd" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_app" tags = { Owner = "Platform Team", Environment = "Production", CostCenter = "123-456" } - module_enablement_overrides = { - cert_manager = true, - prometheus = true, - grafana = true, - istio = true - } } } @@ -68,9 +64,7 @@ module "dev_eks_cluster" { source = "../../" # Repository and cluster configuration - single name for both - name = "eks-dev-cluster" # This name is used for both the repository and cluster - template_repo_org = "HappyPathway" - repository_template = "template-eks-cluster" + name = "eks-dev-cluster" repository_teams = { "platform-team" = "admin", "dev-team" = "maintain", @@ -78,39 +72,35 @@ module "dev_eks_cluster" { } # Basic settings - organization = var.github_organization - environment = "development" - region = "us-gov-west-1" - github_server_url = var.github_server_url + organization = var.github_organization + repository_mode = "update" + environment = "development" + region = "us-gov-west-1" + + template_enabled_modules = { + eks-grafana = false + eks-kiali = true + eks-loki = true + eks-otel = true + } # Cluster configuration cluster_config = { - account_name = "dev-account" - aws_account_id = var.dev_account_id - environment_abbr = "dev" - aws_profile = "default" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" - cluster_mailing_list = "dev-alerts@example.com" - eks_instance_disk_size = 100 - eks_ng_desired_size = 2 - eks_ng_max_size = 5 - eks_ng_min_size = 1 - organization = "census:ocio:csvd" - finops_project_name = "csvd_platformbaseline" - finops_project_number = "fs0000000078" - finops_project_role = "csvd_platformbaseline_app" + account_name = "dev-account" + aws_account_id = var.dev_account_id + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + cluster_mailing_list = "dev-alerts@example.com" + organization = "census:ocio:csvd" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_app" tags = { Owner = "Platform Team", Environment = "Development", CostCenter = "123-456" } - module_enablement_overrides = { - cert_manager = true, - prometheus = true, - grafana = false, - istio = false - } } } diff --git a/main.tf b/main.tf index adcb749..6aab949 100644 --- a/main.tf +++ b/main.tf @@ -1,10 +1,20 @@ locals { + create_repository = var.repository_mode == "create" + effective_files_branch = local.create_repository ? "new/${var.name}" : "update/${var.name}" + effective_template_enabled_modules = merge(var.template_enabled_modules, { + eks = true + eks-config = true + eks-karpenter = true + eks-istio = true + eks-dns = true + }) + template_cluster_sync_files = [ for rel_path in var.template_cluster_file_paths : { path = "${var.environment}/${var.region}/vpc/cluster/${rel_path}" content = data.github_repository_file.template_cluster_files[rel_path].content } - if lookup(var.template_enabled_modules, split("/", rel_path)[0], false) + if lookup(local.effective_template_enabled_modules, split("/", rel_path)[0], false) ] managed_extra_files = concat([ @@ -15,8 +25,13 @@ locals { { path = "_envcommon/common-variables.hcl" content = templatefile("${path.module}/templates/common-variables.hcl.tf.tpl", local.common_vars) + }, + { + path = "_envcommon/prefixes.hcl" + content = templatefile("${path.module}/templates/prefixes.hcl.tf.tpl", local.common_vars) }], - var.github_actions_workflows) + var.github_actions_workflows + ) rendered_files = { "config.json" : jsonencode({ @@ -93,7 +108,7 @@ locals { } data "github_repository" "existing_repo" { - count = !var.create_repository && var.files_branch != null ? 1 : 0 + count = !local.create_repository ? 1 : 0 full_name = "${var.organization}/${var.name}" } @@ -106,32 +121,33 @@ data "github_repository_file" "template_cluster_files" { } resource "github_branch" "files_branch" { - count = !var.create_repository && var.files_branch != null ? 1 : 0 - repository = data.github_repository.existing_repo[0].name - branch = var.files_branch + count = !local.create_repository ? 1 : 0 + repository = var.name + branch = local.effective_files_branch source_branch = var.files_branch_source_branch != null ? var.files_branch_source_branch : data.github_repository.existing_repo[0].default_branch } module "github_repo" { - source = "git::git@github.e.it.census.gov:CSVD/terraform-github-repo.git" + source = "../terraform-github-repo" name = var.name repo_org = var.organization - create_repo = var.create_repository - create_codeowners = var.create_repository - enforce_prs = var.create_repository - files_branch = var.files_branch + create_repo = local.create_repository + create_codeowners = true + enforce_prs = true + files_branch = local.effective_files_branch github_repo_description = "EKS Cluster Configuration for ${var.name}" github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] force_name = var.force_name - template_repo_org = var.create_repository ? local.repository_defaults.template_owner : null - template_repo = var.create_repository ? local.repository_defaults.template : null + template_repo_org = null + template_repo = null - github_is_private = false - github_has_issues = true - github_has_wiki = true - github_has_projects = true + github_is_private = false + github_has_issues = true + github_has_wiki = true + github_has_projects = true + vulnerability_alerts = null managed_extra_files = concat([ for path, content in local.rendered_files : { @@ -140,7 +156,8 @@ module "github_repo" { }], local.template_cluster_sync_files, local.managed_extra_files, - var.github_actions_workflows) + var.github_actions_workflows + ) archive_on_destroy = false github_org_teams = [ diff --git a/templates/prefixes.hcl.tf.tpl b/templates/prefixes.hcl.tf.tpl new file mode 100644 index 0000000..d46f6bb --- /dev/null +++ b/templates/prefixes.hcl.tf.tpl @@ -0,0 +1,37 @@ +locals { + prefixes = { + "ebs" = "v-ebs-" + "efs" = "v-efs-" + "group" = "g-" + "kms" = "k-kms-" + "policy" = "p-" + "role" = "r-" + "s3" = "v-s3-" + "security-group" = "" # "sg-" + # VPC + "customer-gateway" = "cgw-" + "dhcp-options" = "" + "elastic-ip" = "eip-" + "internet-gateway" = "igw-" + "log-group" = "lg-" + "log-stream" = "lgs-" + "nat-gateway" = "nat-" + "network-acl" = "nacl-" + "route-table" = "route-" + "subnet" = "" + "vpc-endpoint" = "vpce-" + "vpc-peer" = "vpcp-" + "vpc" = "" + "vpn-connection" = "vpn_" + "vpn-gateway" = "vpcg-" + # EKS + "eks-policy" = "p-eks-" + "eks-queue" = "eks-q-" + "eks-role" = "r-eks-" + "eks-s3" = "v-s3-eks-" + "eks-security-group" = "eks-sg-" # "sg-eks-" + "eks-user" = "s-eks-" + "eks" = "eks-" + "eks-event" = "eks-ev-" + } +} diff --git a/variables.tf b/variables.tf index 95c914f..48db28b 100644 --- a/variables.tf +++ b/variables.tf @@ -22,17 +22,17 @@ variable "region" { variable "cluster_config" { description = "Configuration for the EKS cluster" type = object({ - account_name = string - aws_account_id = string - environment_abbr = string - vpc_name = string - vpc_domain_name = string - cluster_mailing_list = optional(string) - finops_project_name = optional(string) + account_name = string + aws_account_id = string + environment_abbr = string + vpc_name = string + vpc_domain_name = string + cluster_mailing_list = optional(string) + finops_project_name = optional(string) finops_project_number = optional(string) - finops_project_role = optional(string) - tags = optional(map(string), {}) - organization = optional(string) + finops_project_role = optional(string) + tags = optional(map(string), {}) + organization = optional(string) }) } @@ -252,20 +252,19 @@ variable "tags" { type = map(string) default = {} } -variable "create_repository" { - description = "If true, a GitHub repository will be created and configured (internal use)" - type = bool - default = true -} - -variable "files_branch" { - description = "Branch to manage generated files on. Set to an existing branch (for example, update/csvd-dev-mcm) to support PR workflows." +variable "repository_mode" { + description = "Repository workflow mode. 'create' creates a repository and commits files on branch new/. 'update' manages an existing repository and commits files on branch update/." type = string - default = null + default = "create" + + validation { + condition = contains(["create", "update"], var.repository_mode) + error_message = "repository_mode must be either 'create' or 'update'." + } } variable "files_branch_source_branch" { - description = "Source branch used when creating files_branch for existing repositories. Defaults to the repository default branch." + description = "Source branch used when creating update/ for existing repositories. Defaults to the repository default branch." type = string default = null } @@ -285,7 +284,7 @@ variable "template_repo_ref" { variable "template_cluster_file_paths" { description = "List of file paths under //vpc/cluster in template_repo_name to sync into environment/region-resolved cluster path." type = list(string) - default = [ + default = [ "eks/terragrunt.hcl", "eks-config/terragrunt.hcl", "eks-cribl/terragrunt.hcl", @@ -304,27 +303,23 @@ variable "template_cluster_file_paths" { } variable "template_enabled_modules" { - description = "Enablement map used to decide which template_cluster_file_paths are synced. Key must match the first path segment, for example eks-grafana in eks-grafana/terragrunt.hcl." + description = "Enablement map used to decide which template_cluster_file_paths are synced. Key must match the first path segment, for example eks-grafana in eks-grafana/terragrunt.hcl. Core modules eks, eks-config, eks-karpenter, eks-istio, and eks-dns are always enabled." type = map(bool) default = { - eks = true - eks-config = true - eks-dns = true - eks-istio = true - eks-karpenter = true - eks-arcgis = false - eks-cribl = false - eks-gatekeeper = true - eks-grafana = true - eks-keycloak = true - eks-kiali = true - eks-loki = true - eks-otel = true - eks-pipeline = false - eks-postgresql = false - eks-prometheus = true - eks-tempo = true - eks-cert-manager = false - eks-k8s-dashboard = false + eks = true + eks-config = true + eks-dns = true + eks-istio = true + eks-karpenter = true + eks-arcgis = false + eks-cribl = false + eks-gatekeeper = true + eks-grafana = true + eks-keycloak = true + eks-kiali = true + eks-loki = true + eks-otel = true + eks-prometheus = true + eks-tempo = true } } \ No newline at end of file From 73dbb0d90a8d5b28e3353caa516e6a7a1757a485 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 19 Mar 2026 17:17:23 -0400 Subject: [PATCH 21/27] remove cruft --- .../logs/plan.20260319.1773952402.log | 23 ------------------- examples/basic/varfiles/default.json | 1 - 2 files changed, 24 deletions(-) delete mode 100644 clusters/csvd-lab-mcm/logs/plan.20260319.1773952402.log delete mode 100644 examples/basic/varfiles/default.json diff --git a/clusters/csvd-lab-mcm/logs/plan.20260319.1773952402.log b/clusters/csvd-lab-mcm/logs/plan.20260319.1773952402.log deleted file mode 100644 index c692ffb..0000000 --- a/clusters/csvd-lab-mcm/logs/plan.20260319.1773952402.log +++ /dev/null @@ -1,23 +0,0 @@ -# starting v1.11.0 action plan file logs/plan.20260319.1773952402.log stamp 20260319.1773952402 time 1773952402 -# current_directory=/apps/terraform/workspaces/morga471/terraform/terraform-eks-deployment/clusters/csvd-lab-mcm -# git_repository=git@github.e.it.census.gov:sct-engineering/terraform-eks-deployment -# git_current_branch=test_cluster -# terraform_version=Terraform v0.12.31 -# TFCONTROL= -# TF_CLI_CONFIG_FILE= -# TFARGS="" TFNOCLOR= TFNOLOG= TFNOPROXY= -# env TF_VAR_ variables -# TF_VAR_os_environment={"pwd":"/apps/terraform/workspaces/morga471/terraform/terraform-eks-deployment/clusters/csvd-lab-mcm"} -# TF_VAR_os_username=morga471 - - -Error: Error locking state: Error acquiring the state lock: state snapshot was created by Terraform v1.14.7, which is newer than current v0.12.31; upgrade to Terraform v1.14.7 or greater to work with this state - -Terraform acquires a state lock to protect the state from being written -by multiple users at the same time. Please resolve the issue above and try -again. For most commands, you can disable locking with the "-lock=false" -flag, but this is not recommended. - - -# ending v1.11.0 action plan file logs/plan.20260319.1773952402.log stamp 20260319.1773952402 start 1773952402 end 1773952402 elapsed 0 - diff --git a/examples/basic/varfiles/default.json b/examples/basic/varfiles/default.json deleted file mode 100644 index 9e26dfe..0000000 --- a/examples/basic/varfiles/default.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file From 9c0b6e3a80178404a27192ed3cd78a8fcd7e0e76 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 27 Mar 2026 17:44:42 -0400 Subject: [PATCH 22/27] more tags --- clusters/csvd-lab-dja/main.tf | 60 ++++++++++++++++++ clusters/csvd-lab-mcm/main.tf | 6 +- main.tf | 115 ++++++++++++++++++++++++++-------- templates/cluster.hcl.tf.tpl | 1 + templates/root.hcl.tf.tpl | 15 +++++ variables.tf | 2 +- 6 files changed, 168 insertions(+), 31 deletions(-) create mode 100644 clusters/csvd-lab-dja/main.tf diff --git a/clusters/csvd-lab-dja/main.tf b/clusters/csvd-lab-dja/main.tf new file mode 100644 index 0000000..7972a23 --- /dev/null +++ b/clusters/csvd-lab-dja/main.tf @@ -0,0 +1,60 @@ +provider "aws" { +} + +data "aws_secretsmanager_secret_version" "github_token" { + secret_id = "/eks-cluster-deployment/github_token" +} + +provider "github" { + token = data.aws_secretsmanager_secret_version.github_token.secret_string +} + +module "eks_deployment" { + source = "../../" + repository_mode = "create" + organization = "SCT-Engineering" + + # Repository and cluster configuration - single name for both + name = "csvd-lab-dja" + environment = "lab" + region = "us-gov-east-1" + + # Cluster configuration - simplified interface + cluster_config = { + account_name = "lab-dev-gov" + aws_account_id = "224384469011" + cluster_mailing_list = "matthew.c.morgan@census.gov" + environment_abbr = "lab" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_eks" + vpc_domain_name = "dev.lab.csp2.census.gov" + vpc_name = "vpc3-lab-dev" + tags = { + Owner = "matthew.c.morgan@census.gov" + Environment = "lab" + CostCenter = "fs0000000078" + } + organization = "census:ocio:csvd" + } + template_enabled_modules = { + eks-gatekeeper = false + eks-grafana = false + eks-kiali = false + eks-keycloak = false + eks-loki = false + eks-otel = false + eks-prometheus = false + eks-tempo = false + } +} + +output "repository_url" { + description = "URL of the created GitHub repository" + value = module.eks_deployment.repository_url +} + +output "ssh_clone_url" { + description = "SSH clone URL of the repository" + value = module.eks_deployment.ssh_clone_url +} diff --git a/clusters/csvd-lab-mcm/main.tf b/clusters/csvd-lab-mcm/main.tf index 09ed437..4b16870 100644 --- a/clusters/csvd-lab-mcm/main.tf +++ b/clusters/csvd-lab-mcm/main.tf @@ -11,12 +11,12 @@ provider "github" { module "eks_deployment" { source = "../../" - repository_mode = "create" + repository_mode = "update" organization = "SCT-Engineering" # Repository and cluster configuration - single name for both name = "csvd-lab-mcm" - environment = "dev" + environment = "lab" region = "us-gov-east-1" # Cluster configuration - simplified interface @@ -32,7 +32,7 @@ module "eks_deployment" { vpc_name = "vpc3-lab-dev" tags = { Owner = "matthew.c.morgan@census.gov" - Environment = "development" + Environment = "lab" CostCenter = "fs0000000078" } organization = "census:ocio:csvd" diff --git a/main.tf b/main.tf index 6aab949..c7f8752 100644 --- a/main.tf +++ b/main.tf @@ -1,6 +1,7 @@ locals { - create_repository = var.repository_mode == "create" + create_repository = lower(trimspace(var.repository_mode)) == "create" effective_files_branch = local.create_repository ? "new/${var.name}" : "update/${var.name}" + update_source_branch = var.files_branch_source_branch != null ? var.files_branch_source_branch : (local.create_repository ? null : data.github_repository.existing_repo[0].default_branch) effective_template_enabled_modules = merge(var.template_enabled_modules, { eks = true eks-config = true @@ -57,6 +58,7 @@ locals { eks_ng_max_size = local.eks_defaults.ng_max_size eks_ng_min_size = local.eks_defaults.ng_min_size organization = var.cluster_config.organization + CostAllocation = var.cluster_config.organization finops_project_name = local.organization_defaults.finops_project_name finops_project_number = local.organization_defaults.finops_project_number finops_project_role = local.organization_defaults.finops_project_role @@ -105,6 +107,35 @@ locals { aws_region = var.region }) } + + desired_managed_files_by_path = { + for file in concat([ + for path, content in local.rendered_files : { + path = path + content = content + } + ], + local.template_cluster_sync_files, + local.managed_extra_files + ) : file.path => file.content + } + + existing_managed_file_paths = !local.create_repository && local.update_source_branch != null ? toset([ + for entry in data.github_tree.update_source_branch_tree[0].entries : entry.path + if entry.type == "blob" + ]) : toset([]) + + managed_files_requiring_update = local.create_repository ? local.desired_managed_files_by_path : { + for path, content in local.desired_managed_files_by_path : path => content + if !contains(local.existing_managed_file_paths, path) || data.github_repository_file.existing_managed_files[path].content != content + } + + has_update_changes = length(local.managed_files_requiring_update) > 0 + effective_update_files_branch = local.create_repository || local.has_update_changes ? local.effective_files_branch : null + managed_extra_files_for_module = [for path, content in local.managed_files_requiring_update : { + path = path + content = content + }] } data "github_repository" "existing_repo" { @@ -112,6 +143,30 @@ data "github_repository" "existing_repo" { full_name = "${var.organization}/${var.name}" } +# Used to detect accidental create-mode runs against an already-existing repo. +data "github_repositories" "create_mode_check" { + count = local.create_repository ? 1 : 0 + query = "org:${var.organization} ${var.name} in:name" +} + +locals { + create_mode_repo_exists = local.create_repository && contains( + try(data.github_repositories.create_mode_check[0].names, []), + var.name + ) +} + +resource "terraform_data" "create_mode_guard" { + count = local.create_repository ? 1 : 0 + + lifecycle { + precondition { + condition = !local.create_mode_repo_exists + error_message = "repository_mode=\"create\" is set but repository \"${var.name}\" already exists in ${var.organization}. Switch to repository_mode=\"update\"." + } + } +} + data "github_repository_file" "template_cluster_files" { for_each = toset(var.template_cluster_file_paths) @@ -120,25 +175,42 @@ data "github_repository_file" "template_cluster_files" { file = "environment/region/vpc/cluster/${each.value}" } -resource "github_branch" "files_branch" { - count = !local.create_repository ? 1 : 0 - repository = var.name - branch = local.effective_files_branch - source_branch = var.files_branch_source_branch != null ? var.files_branch_source_branch : data.github_repository.existing_repo[0].default_branch +data "github_tree" "update_source_branch_tree" { + count = !local.create_repository && local.update_source_branch != null ? 1 : 0 + + repository = var.name + tree_sha = local.update_source_branch + recursive = true + + depends_on = [ + data.github_repository.existing_repo, + ] +} + +data "github_repository_file" "existing_managed_files" { + for_each = !local.create_repository ? { + for path, content in local.desired_managed_files_by_path : path => content + if contains(local.existing_managed_file_paths, path) + } : {} + + repository = "${var.organization}/${var.name}" + branch = coalesce(local.update_source_branch, "main") + file = each.key } module "github_repo" { source = "../terraform-github-repo" - name = var.name - repo_org = var.organization - create_repo = local.create_repository - create_codeowners = true - enforce_prs = true - files_branch = local.effective_files_branch - github_repo_description = "EKS Cluster Configuration for ${var.name}" - github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] - force_name = var.force_name + name = var.name + repo_org = var.organization + create_repo = local.create_repository + create_codeowners = true + enforce_prs = true + files_branch = local.effective_update_files_branch + files_branch_source_branch = local.update_source_branch + github_repo_description = "EKS Cluster Configuration for ${var.name}" + github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] + force_name = var.force_name template_repo_org = null template_repo = null @@ -149,15 +221,7 @@ module "github_repo" { github_has_projects = true vulnerability_alerts = null - managed_extra_files = concat([ - for path, content in local.rendered_files : { - path = path - content = content - }], - local.template_cluster_sync_files, - local.managed_extra_files, - var.github_actions_workflows - ) + managed_extra_files = local.managed_extra_files_for_module archive_on_destroy = false github_org_teams = [ @@ -170,9 +234,6 @@ module "github_repo" { } ] - depends_on = [ - github_branch.files_branch - ] } # The EKS deployment logic will go here, and will be skipped if create_repository is true. diff --git a/templates/cluster.hcl.tf.tpl b/templates/cluster.hcl.tf.tpl index 0a30d6c..0d8c524 100644 --- a/templates/cluster.hcl.tf.tpl +++ b/templates/cluster.hcl.tf.tpl @@ -7,6 +7,7 @@ locals { eks_ng_max_size = ${eks_ng_max_size} eks_ng_min_size = ${eks_ng_min_size} organization = "${organization}" + CostAllocation = "${organization}" finops_project_name = "${finops_project_name}" finops_project_number = "${finops_project_number}" finops_project_role = "${finops_project_role}" diff --git a/templates/root.hcl.tf.tpl b/templates/root.hcl.tf.tpl index 981f885..d4f7bf1 100644 --- a/templates/root.hcl.tf.tpl +++ b/templates/root.hcl.tf.tpl @@ -142,7 +142,9 @@ generate "aws-provider" { finops_project_name = "$${local.finops_project_name}" finops_project_number = "$${local.finops_project_number}" finops_project_role = "$${local.finops_project_role}" + CostAllocation = "$${local.organization}" organization = "$${local.organization}" + "boc:created-by" = "terragrunt" } } # Only these AWS Account IDs may be operated on by this template @@ -151,6 +153,19 @@ generate "aws-provider" { EOF } +generate "tags-yml" { + path = "tags.yml" + if_exists = "overwrite" + contents = !local.is_eks_module ? "" : <<-EOF + finops: + number: "$${tonumber(regex("[0-9]+$", local.finops_project_number))}" + name: "$${local.finops_project_name}" + roles: + - eks + - "$${local.finops_project_role}" + EOF +} + # --------------------------------------------------------------------------------------------------------------------- # GLOBAL PARAMETERS # These variables apply to all configurations in this subfolder. These are automatically merged into the child diff --git a/variables.tf b/variables.tf index 48db28b..da70bfd 100644 --- a/variables.tf +++ b/variables.tf @@ -258,7 +258,7 @@ variable "repository_mode" { default = "create" validation { - condition = contains(["create", "update"], var.repository_mode) + condition = contains(["create", "update"], lower(trimspace(var.repository_mode))) error_message = "repository_mode must be either 'create' or 'update'." } } From d9ec74a321527f21922f33c8603a0f4834a3cb1d Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 30 Mar 2026 14:08:22 -0400 Subject: [PATCH 23/27] update tags --- clusters/csvd-lab-mcm/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clusters/csvd-lab-mcm/main.tf b/clusters/csvd-lab-mcm/main.tf index 4b16870..f76ea59 100644 --- a/clusters/csvd-lab-mcm/main.tf +++ b/clusters/csvd-lab-mcm/main.tf @@ -31,9 +31,9 @@ module "eks_deployment" { vpc_domain_name = "dev.lab.csp2.census.gov" vpc_name = "vpc3-lab-dev" tags = { - Owner = "matthew.c.morgan@census.gov" - Environment = "lab" - CostCenter = "fs0000000078" + CostAllocation = "census:ocio:csvd" + Owner = "matthew.c.morgan@census.gov" + PowerSchedule = "Full_Week_Core_Hours_7-7" } organization = "census:ocio:csvd" } From 6d39cfa4753cc9a3aa17d0e1b2cb2c6c6a6b6732 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 30 Mar 2026 14:09:15 -0400 Subject: [PATCH 24/27] update tag handling and gen --- clusters/csvd-dev-mcm/main.tf | 2 +- main.tf | 5 +++-- templates/default-versions.hcl.tf.tpl | 4 +++- templates/root.hcl.tf.tpl | 2 +- variables.tf | 11 ++++++++--- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/clusters/csvd-dev-mcm/main.tf b/clusters/csvd-dev-mcm/main.tf index ef49d30..1ce382f 100644 --- a/clusters/csvd-dev-mcm/main.tf +++ b/clusters/csvd-dev-mcm/main.tf @@ -33,7 +33,7 @@ module "eks_deployment" { tags = { Owner = "matthew.c.morgan@census.gov" Environment = "development" - CostCenter = "fs0000000078" + CostAllocation = "census:ocio:csvd" } organization = "census:ocio:csvd" } diff --git a/main.tf b/main.tf index c7f8752..e28f430 100644 --- a/main.tf +++ b/main.tf @@ -58,10 +58,10 @@ locals { eks_ng_max_size = local.eks_defaults.ng_max_size eks_ng_min_size = local.eks_defaults.ng_min_size organization = var.cluster_config.organization - CostAllocation = var.cluster_config.organization finops_project_name = local.organization_defaults.finops_project_name finops_project_number = local.organization_defaults.finops_project_number finops_project_role = local.organization_defaults.finops_project_role + CostAllocation = var.cluster_config.organization tags = var.cluster_config.tags } modules = var.enable_modules @@ -97,8 +97,9 @@ locals { finops_project_name = local.organization_defaults.finops_project_name, finops_project_number = local.organization_defaults.finops_project_number, finops_project_role = local.organization_defaults.finops_project_role, - tags = var.cluster_config.tags, + CostAllocation = var.cluster_config.organization, organization = var.cluster_config.organization, + tags = var.cluster_config.tags, module_enablement_overrides = var.enable_modules }), "README.md" : templatefile("${path.module}/templates/README.md.tf.tpl", { diff --git a/templates/default-versions.hcl.tf.tpl b/templates/default-versions.hcl.tf.tpl index 6fcf1e3..6297366 100644 --- a/templates/default-versions.hcl.tf.tpl +++ b/templates/default-versions.hcl.tf.tpl @@ -31,6 +31,7 @@ locals { "eks-prometheus" = "0.1.4" "eks-tempo" = "0.1.4" "eks" = "1.0.10" + "cluster" = "2025.20.04" } "2026.03.15" = { "eks-arcgis" = false @@ -49,7 +50,8 @@ locals { "eks-postgresql" = false "eks-prometheus" = "0.1.5" "eks-tempo" = "0.1.6" - "eks" = "1.0.12" + "eks" = "1.0.14" + "cluster" = "2026.03.15" } } diff --git a/templates/root.hcl.tf.tpl b/templates/root.hcl.tf.tpl index d4f7bf1..06f688d 100644 --- a/templates/root.hcl.tf.tpl +++ b/templates/root.hcl.tf.tpl @@ -44,7 +44,7 @@ locals { finops_project_name = local.cluster_vars.locals.finops_project_name finops_project_number = local.cluster_vars.locals.finops_project_number finops_project_role = local.cluster_vars.locals.finops_project_role - is_eks_module = local.module_name == "eks" + is_eks_module = local.module_name == "eks" ? true : false prefixes = local.prefix_vars.locals.prefixes is_module_enabled = merge( { for module in local.versions.locals.core_modules : module => true }, diff --git a/variables.tf b/variables.tf index da70bfd..6f27c73 100644 --- a/variables.tf +++ b/variables.tf @@ -212,10 +212,15 @@ variable "namespaces" { variable "enable_modules" { description = "Map of modules to enable (internal use)" type = object({ - gogatekeeper = optional(bool, false) - prometheus = optional(bool, false) + gatekeeper = optional(bool, false) grafana = optional(bool, false) - istio = optional(bool, false) + otel = optional(bool, false) + prometheus = optional(bool, false) + tempo = optional(bool, false) + loki = optional(bool, false) + kiali = optional(bool, false) + keycloak = optional(bool, false) + cribl = optional(bool, false) }) default = {} } From 9b33f6e9c56acc113ece0b8b9e84bdadac307bfd Mon Sep 17 00:00:00 2001 From: Dave Arnold Date: Tue, 21 Apr 2026 14:34:25 -0400 Subject: [PATCH 25/27] fix: replace local module path with CSVD GHE HTTPS source ref --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 27ab20f..db5b9f3 100644 --- a/main.tf +++ b/main.tf @@ -215,7 +215,7 @@ locals { } module "github_repo" { - source = "../terraform-github-repo" + source = "git::https://github.e.it.census.gov/CSVD/terraform-github-repo.git?ref=feat/files-branch-source-branch" name = var.name repo_org = var.organization From 269d9f38a2d484b43c7d5c1ff3b57521f9301101 Mon Sep 17 00:00:00 2001 From: Dave Arnold Date: Tue, 21 Apr 2026 14:46:59 -0400 Subject: [PATCH 26/27] fix: point module source to CSVD/terraform-github-repo main (feat/files-branch-source-branch merged) --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index db5b9f3..ff0749b 100644 --- a/main.tf +++ b/main.tf @@ -215,7 +215,7 @@ locals { } module "github_repo" { - source = "git::https://github.e.it.census.gov/CSVD/terraform-github-repo.git?ref=feat/files-branch-source-branch" + source = "git::https://github.e.it.census.gov/CSVD/terraform-github-repo.git?ref=main" name = var.name repo_org = var.organization From 33ea489c8d2bc3da1298508d50cde1ae3a111a48 Mon Sep 17 00:00:00 2001 From: Dave Arnold Date: Tue, 21 Apr 2026 14:53:15 -0400 Subject: [PATCH 27/27] refactor: replace template-eks-cluster remote sync with local templates/eks-modules Remove the data.github_repository_file / data.github_tree approach that read cluster-level terragrunt files from the template-eks-cluster repo at runtime. All HCL templates are now managed locally in templates/eks-modules/ inside this module repo, consistent with the ADR merged in #18. Changes: - Remove effective_template_enabled_modules and template_cluster_sync_files locals - Remove data.github_repository_file.template_cluster_files data source - Remove template_repo_name, template_repo_ref, template_cluster_file_paths variables - Move effective_template_enabled_modules into the eks_module_files locals block and apply enablement filter directly to the fileset loop - Update desired_managed_files_by_path to use local.eks_module_files - Update template_enabled_modules variable description to reflect local template usage --- main.tf | 46 ++++++++++++++-------------------------------- variables.tf | 35 +---------------------------------- 2 files changed, 15 insertions(+), 66 deletions(-) diff --git a/main.tf b/main.tf index ff0749b..ec257da 100644 --- a/main.tf +++ b/main.tf @@ -2,22 +2,6 @@ locals { create_repository = lower(trimspace(var.repository_mode)) == "create" effective_files_branch = local.create_repository ? "new/${var.name}" : "update/${var.name}" update_source_branch = var.files_branch_source_branch != null ? var.files_branch_source_branch : (local.create_repository ? null : data.github_repository.existing_repo[0].default_branch) - effective_template_enabled_modules = merge(var.template_enabled_modules, { - eks = true - eks-config = true - eks-karpenter = true - eks-istio = true - eks-dns = true - }) - - template_cluster_sync_files = [ - for rel_path in var.template_cluster_file_paths : { - path = "${var.environment}/${var.region}/vpc/cluster/${rel_path}" - content = data.github_repository_file.template_cluster_files[rel_path].content - } - if lookup(local.effective_template_enabled_modules, split("/", rel_path)[0], false) - ] - managed_extra_files = concat([ { path = "_envcommon/default-versions.hcl" @@ -110,13 +94,9 @@ locals { } desired_managed_files_by_path = { - for file in concat([ - for path, content in local.rendered_files : { - path = path - content = content - } - ], - local.template_cluster_sync_files, + for file in concat( + [for path, content in local.rendered_files : { path = path, content = content }], + [for path, content in local.eks_module_files : { path = path, content = content }], local.managed_extra_files ) : file.path => file.content } @@ -168,14 +148,6 @@ resource "terraform_data" "create_mode_guard" { } } -data "github_repository_file" "template_cluster_files" { - for_each = toset(var.template_cluster_file_paths) - - repository = var.template_repo_name - branch = var.template_repo_ref - file = "environment/region/vpc/cluster/${each.value}" -} - data "github_tree" "update_source_branch_tree" { count = ! local.create_repository && local.update_source_branch != null ? 1 : 0 @@ -203,14 +175,24 @@ locals { # Base path prefix for all eks-module files in the generated repo eks_module_cluster_prefix = "${var.environment}/${var.region}/${var.cluster_config.vpc_name}/${var.name}" + effective_template_enabled_modules = merge(var.template_enabled_modules, { + eks = true + eks-config = true + eks-karpenter = true + eks-istio = true + eks-dns = true + }) + # Auto-discover all files in templates/eks-modules/ and map them to their # target paths in the generated repo. The naming convention converts # "eks-karpenter.terragrunt.hcl" → "eks-karpenter/terragrunt.hcl" by - # splitting on the first dot. + # splitting on the first dot. Files whose module segment is set to false in + # effective_template_enabled_modules are excluded. eks_module_files = { for fname in fileset("${path.module}/templates/eks-modules", "*") : "${local.eks_module_cluster_prefix}/${join("/", regex("^([^.]+)\\.(.+)$", fname))}" => file("${path.module}/templates/eks-modules/${fname}") + if lookup(local.effective_template_enabled_modules, regex("^([^.]+)", fname)[0], true) } } diff --git a/variables.tf b/variables.tf index 8e93d32..2f2bc5f 100644 --- a/variables.tf +++ b/variables.tf @@ -274,41 +274,8 @@ variable "files_branch_source_branch" { default = null } -variable "template_repo_name" { - description = "Name of the template repository used as authoritative source for cluster-level terragrunt files." - type = string - default = "template-eks-cluster" -} - -variable "template_repo_ref" { - description = "Branch, tag, or SHA to read from template_repo_name when syncing cluster-level terragrunt files." - type = string - default = "main" -} - -variable "template_cluster_file_paths" { - description = "List of file paths under //vpc/cluster in template_repo_name to sync into environment/region-resolved cluster path." - type = list(string) - default = [ - "eks/terragrunt.hcl", - "eks-config/terragrunt.hcl", - "eks-cribl/terragrunt.hcl", - "eks-dns/terragrunt.hcl", - "eks-gatekeeper/terragrunt.hcl", - "eks-grafana/terragrunt.hcl", - "eks-istio/terragrunt.hcl", - "eks-karpenter/terragrunt.hcl", - "eks-keycloak/terragrunt.hcl", - "eks-kiali/terragrunt.hcl", - "eks-loki/terragrunt.hcl", - "eks-otel/terragrunt.hcl", - "eks-prometheus/terragrunt.hcl", - "eks-tempo/terragrunt.hcl", - ] -} - variable "template_enabled_modules" { - description = "Enablement map used to decide which template_cluster_file_paths are synced. Key must match the first path segment, for example eks-grafana in eks-grafana/terragrunt.hcl. Core modules eks, eks-config, eks-karpenter, eks-istio, and eks-dns are always enabled." + description = "Controls which modules from templates/eks-modules/ are written into the generated repo. Key is the module directory name (e.g. eks-grafana). Core modules eks, eks-config, eks-karpenter, eks-istio, and eks-dns are always enabled regardless of this map." type = map(bool) default = { eks = true