From ce9c5037645d04345310e6cecb48bdaa44ab38b5 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Fri, 4 Apr 2025 19:15:23 -0400 Subject: [PATCH 01/33] Add comprehensive EKS Cluster Template roadmap and configuration files --- ROADMAP.md | 59 ++++++++++ locals.tf | 144 ++++++++++++++++++++++++ main.tf | 28 +++++ templates/common-variables.hcl | 10 ++ templates/default-versions.hcl | 124 +++++++++++++++++++++ variables.tf | 194 +++++++++++++++++++++++++++++++++ versions.tf | 7 ++ 7 files changed, 566 insertions(+) create mode 100644 ROADMAP.md create mode 100644 locals.tf create mode 100644 main.tf create mode 100644 templates/common-variables.hcl create mode 100644 templates/default-versions.hcl create mode 100644 variables.tf create mode 100644 versions.tf diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..f7160db --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,59 @@ +# EKS Cluster Template Roadmap + +## Current Architecture +- Template repository (`template-eks-cluster`) serves as the base for creating new EKS cluster configurations +- Uses Terraform GitHub repo module to create new repositories from the template +- Implements managed and non-managed extra files functionality +- Supports configuration through `config.json` + +## Planned Enhancements + +### 1. Automated Cluster Setup (High Priority) +- [ ] Add GitHub Actions workflows with workflow_dispatch triggers +- [ ] Implement automated terragrunt command execution for cluster building +- [ ] Configure workflows to run on specific runners for credential management +- [ ] Create templatized GitHub Actions workflow files +- [ ] Enable direct cluster creation without manual repository cloning + +### 2. File Management System (Medium Priority) +- [ ] Implement wrapper module for repo module +- [ ] Add support for crafting and injecting various configuration files +- [ ] Define file lifecycle management strategy + - [ ] Managed files (controlled by workspace) + - [ ] Non-managed files (user-modifiable) + +### 3. Version Management (Medium Priority) +- [ ] Implement version control strategy for `default-versions.hcl` +- [ ] Create system for managing platform release versions +- [ ] Set up version override mechanism + - [ ] Default versions in template repo + - [ ] Override capability in workspace creating repos + +### 4. Configuration Management (Low Priority) +- [ ] Enhance Makefile and Ansible playbook integration +- [ ] Improve configuration file templating +- [ ] Add validation for configuration files + +## Technical Considerations +1. File Lifecycle Management: + - Managed files: Controlled by workspace + - Non-managed files: User-modifiable post-creation + - Version-specific files: Platform release coordination + +2. Automation Requirements: + - GitHub Actions runner configuration + - Credential management + - Workflow templating + - Terragrunt integration + +3. Version Control Strategy: + - Module version collections + - Platform release versions + - Override mechanisms + +## Success Criteria +- Fully automated cluster creation process +- Minimal manual intervention required +- Proper version management system +- Clear file lifecycle management +- Secure credential handling \ No newline at end of file diff --git a/locals.tf b/locals.tf new file mode 100644 index 0000000..acf38af --- /dev/null +++ b/locals.tf @@ -0,0 +1,144 @@ +locals { + common_vars = merge({ + organization = "census:ocio:csvd" + project_name = "csvd_platformbaseline" + project_number = "fs0000000078" + project_role = "csvd_platformbaseline_app" + state_bucket_prefix = "inf-tfstate" + state_table_name = "tf_remote_state" + route53_endpoints = {} + }, var.common_variables) + + all_namespaces = merge({ + grafana = local.namespaces.telemetry_namespace + k8s-dashboard = local.namespaces.telemetry_namespace + loki = local.namespaces.telemetry_namespace + otel = local.namespaces.telemetry_namespace + prometheus = local.namespaces.telemetry_namespace + tempo = local.namespaces.telemetry_namespace + }, var.namespaces.custom_namespaces) + + namespaces = { + operator_namespace = var.namespaces.operator_namespace + telemetry_namespace = var.namespaces.telemetry_namespace + namespaces = local.all_namespaces + } + + default_versions = { + cluster_version = var.versions.cluster_version + custom_service_eks_account = var.versions.release_version + eks_module_version = var.versions.eks_module_version + istio_ingress_version = var.versions.release_version + release_version = var.versions.release_version + + # Provider versions + aws_version = var.versions.aws_version + helm_version = var.versions.helm_version + kubernetes_version = var.versions.kubernetes_version + null_version = var.versions.null_version + random_version = var.versions.random_version + template_version = var.versions.template_version + tf_version = var.versions.tf_version + + # Component versions + cert_manager_version = var.versions.cert_manager.version + cert_manager_helm_chart = var.versions.cert_manager.chart_version + cluster_issuer_name = var.versions.cert_manager.cluster_issuer_name + + gogatekeeper_tag = var.versions.gogatekeeper.tag + gogatekeeper_chart_version = var.versions.gogatekeeper.chart_version + + grafana_hostname = var.versions.grafana.hostname + grafana_operator_chart_version = var.versions.grafana.operator_chart_version + grafana_operator_tag = var.versions.grafana.operator_tag + grafana_tag = var.versions.grafana.tag + os_shell_image_tag = var.versions.grafana.os_shell_image_tag + + istio_version = var.versions.istio.version + istio_namespace = var.versions.istio.namespace + + dashboard_hostname = var.versions.k8s_dashboard.hostname + k8s_dashboard_metrics_scraper = var.versions.k8s_dashboard.metrics_scraper + k8s_dashboard_version = var.versions.k8s_dashboard.version + + karpenter_helm_chart = var.versions.karpenter.helm_chart + karpenter_tag = var.versions.karpenter.tag + + keycloak_chart_version = var.versions.keycloak.chart_version + keycloak_tag = var.versions.keycloak.tag + keycloak_hostname = var.versions.keycloak.hostname + keycloak_database = var.versions.keycloak.database + keycloak_username = var.versions.keycloak.username + keycloak_password = var.versions.keycloak.password + postgresql_tag = var.versions.keycloak.postgresql_tag + + kiali_operator_version = var.versions.kiali.operator_version + kiali_application_version = "v${var.versions.kiali.operator_version}" + + loki_chart_version = var.versions.loki.chart_version + loki_tag = var.versions.loki.tag + enterprise_logs_provisioner_tag = var.versions.loki.enterprise_logs_provisioner_tag + gateway_tag = var.versions.loki.gateway_tag + memcached_tag = var.versions.loki.memcached_tag + exporter_tag = var.versions.loki.exporter_tag + sidecar_tag = var.versions.loki.sidecar_tag + + metrics_server_helm_chart = var.versions.metrics_server.helm_chart + metrics_server_tag = var.versions.metrics_server.tag + + prometheus_chart_version = var.versions.prometheus.chart_version + prometheus_server_tag = var.versions.prometheus.server_tag + prometheus_config_reloader_tag = var.versions.prometheus.config_reloader_tag + alertmanager_tag = var.versions.prometheus.alertmanager_tag + kube_state_metrics_tag = var.versions.prometheus.kube_state_metrics_tag + node_exporter_tag = var.versions.prometheus.node_exporter_tag + pushgateway_tag = var.versions.prometheus.pushgateway_tag + + tempo_chart_version = var.versions.tempo.chart_version + tempo_tag = var.versions.tempo.tag + } + + config_json = jsonencode({ + environment = var.environment + region = var.region + cluster_dir = "platform-cluster" + enable_all_modules = false + account = { + account_name = var.cluster_config.account_name + aws_account_id = var.cluster_config.aws_account_id + aws_profile = var.cluster_config.aws_profile + environment_abbr = var.cluster_config.environment_abbr + } + vpc = { + vpc_name = var.cluster_config.vpc_name + vpc_domain_name = var.cluster_config.vpc_domain_name + } + cluster = { + cluster_name = var.cluster_config.cluster_name + cluster_mailing_list = var.cluster_config.cluster_mailing_list + eks_instance_disk_size = var.cluster_config.eks_instance_disk_size + eks_ng_desired_size = var.cluster_config.eks_ng_desired_size + eks_ng_max_size = var.cluster_config.eks_ng_max_size + eks_ng_min_size = var.cluster_config.eks_ng_min_size + enable_cluster_creator_admin_permissions = var.cluster_config.enable_cluster_creator_admin_permissions + tags = var.cluster_config.tags + } + modules = var.enable_modules + }) + + managed_extra_files = concat([ + { + path = "config.json" + content = local.config_json + }, + { + path = "_envcommon/default-versions.hcl" + content = templatefile("${path.module}/templates/default-versions.hcl", local.default_versions) + }, + { + path = "_envcommon/common-variables.hcl" + content = templatefile("${path.module}/templates/common-variables.hcl", local.common_vars) + } + ], + var.github_actions_workflows) +} \ No newline at end of file diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..380a7c6 --- /dev/null +++ b/main.tf @@ -0,0 +1,28 @@ +module "github_repo" { + source = "HappyPathway/repo/github" + + name = var.name + repo_org = var.organization + github_repo_description = "EKS Cluster Configuration for ${var.cluster_config.cluster_name}" + github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] + + template_repo = "template-eks-cluster" + template_repo_org = var.template_repo_org + + github_is_private = true + github_has_issues = true + github_has_wiki = true + github_has_projects = true + + managed_extra_files = local.managed_extra_files +} + +output "repository_url" { + description = "URL of the created repository" + value = module.github_repo.html_url +} + +output "ssh_clone_url" { + description = "SSH clone URL of the repository" + value = module.github_repo.ssh_clone_url +} \ No newline at end of file diff --git a/templates/common-variables.hcl b/templates/common-variables.hcl new file mode 100644 index 0000000..e228659 --- /dev/null +++ b/templates/common-variables.hcl @@ -0,0 +1,10 @@ +locals { + organization = "${organization}" + project_name = "${project_name}" + project_number = "${project_number}" + project_role = "${project_role}" + state_bucket_prefix = "${state_bucket_prefix}" + state_table_name = "${state_table_name}" + + route53_endpoints = ${jsonencode(route53_endpoints)} +} \ No newline at end of file diff --git a/templates/default-versions.hcl b/templates/default-versions.hcl new file mode 100644 index 0000000..60f3cd4 --- /dev/null +++ b/templates/default-versions.hcl @@ -0,0 +1,124 @@ +locals { + ##################### + # Module Versions + ##################### + cluster_version = "${cluster_version}" + custom_service_eks_account = "${custom_service_eks_account}" + eks_module_version = "${eks_module_version}" + istio_ingress_version = "${istio_ingress_version}" + release_version = "${release_version}" + + ##################### + # TF Providers + ##################### + aws_version = "${aws_version}" + helm_version = "${helm_version}" + kubernetes_version = "${kubernetes_version}" + null_version = "${null_version}" + random_version = "${random_version}" + template_version = "${template_version}" + tf_version = "${tf_version}" + + ##################### + # Component Versions + ##################### + + ################ + # Cert-Manager + ################ + cluster_issuer_name = "${cluster_issuer_name}" + cert_manager_version = "${cert_manager_version}" + cert_manager_helm_chart = "${cert_manager_helm_chart}" + + ################ + # GoGatekeeper + ################ + gogatekeeper_tag = "${gogatekeeper_tag}" + gogatekeeper_chart_version = "${gogatekeeper_chart_version}" + + ################ + # Grafana + ################ + grafana_hostname = "${grafana_hostname}" + grafana_operator_chart_version = "${grafana_operator_chart_version}" + grafana_operator_tag = "${grafana_operator_tag}" + grafana_tag = "${grafana_tag}" + os_shell_image_tag = "${os_shell_image_tag}" + + ################ + # Istio + ################ + istio_namespace = "${istio_namespace}" + istio_version = "${istio_version}" + + ################ + # k8s-dashboard + ################ + dashboard_hostname = "${dashboard_hostname}" + k8s_dashboard_metrics_scraper = "${k8s_dashboard_metrics_scraper}" + k8s_dashboard_version = "${k8s_dashboard_version}" + + ################ + # Karpenter + ################ + karpenter_helm_chart = "${karpenter_helm_chart}" + karpenter_tag = "${karpenter_tag}" + + ################ + # Keycloak + ################ + keycloak_chart_version = "${keycloak_chart_version}" + keycloak_tag = "${keycloak_tag}" + keycloak_hostname = "${keycloak_hostname}" + keycloak_database = "${keycloak_database}" + keycloak_username = "${keycloak_username}" + keycloak_password = "${keycloak_password}" + postgresql_tag = "${postgresql_tag}" + + ################ + # Kiali + ################ + kiali_operator_version = "${kiali_operator_version}" + kiali_application_version = "${kiali_application_version}" + + ################ + # Loki + ################ + loki_chart_version = "${loki_chart_version}" + loki_tag = "${loki_tag}" + enterprise_logs_provisioner_tag = "${enterprise_logs_provisioner_tag}" + gateway_tag = "${gateway_tag}" + memcached_tag = "${memcached_tag}" + exporter_tag = "${exporter_tag}" + sidecar_tag = "${sidecar_tag}" + + ################ + # Metrics Server + ################ + metrics_server_helm_chart = "${metrics_server_helm_chart}" + metrics_server_tag = "${metrics_server_tag}" + + ################ + # Prometheus + ################ + prometheus_chart_version = "${prometheus_chart_version}" + prometheus_server_tag = "${prometheus_server_tag}" + prometheus_config_reloader_tag = "${prometheus_config_reloader_tag}" + alertmanager_tag = "${alertmanager_tag}" + kube_state_metrics_tag = "${kube_state_metrics_tag}" + node_exporter_tag = "${node_exporter_tag}" + pushgateway_tag = "${pushgateway_tag}" + + ################ + # Tempo + ################ + tempo_chart_version = "${tempo_chart_version}" + tempo_tag = "${tempo_tag}" + + ##################### + # Namespaces Config + ##################### + operator_namespace = "${operator_namespace}" + telemetry_namespace = "${telemetry_namespace}" + namespaces = ${jsonencode(namespaces)} +} \ No newline at end of file diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..fd4000d --- /dev/null +++ b/variables.tf @@ -0,0 +1,194 @@ +variable "name" { + description = "Name of the repository" + type = string +} + +variable "organization" { + description = "GitHub organization name" + type = string +} + +variable template_repo_org { + description = "GitHub organization for the template repository" + type = string +} + +variable "environment" { + description = "Environment name (e.g., production, staging)" + type = string +} + +variable "region" { + description = "AWS region for the EKS cluster" + type = string +} + +variable "common_variables" { + description = "Common variables across all environments" + type = object({ + organization = optional(string, "census:ocio:csvd") + project_name = optional(string, "csvd_platformbaseline") + project_number = optional(string, "fs0000000078") + project_role = optional(string, "csvd_platformbaseline_app") + state_bucket_prefix = optional(string, "inf-tfstate") + state_table_name = optional(string, "tf_remote_state") + route53_endpoints = optional(map(object({ + account_id = string + alias = string + us-gov-east-1 = string + us-gov-west-1 = string + })), {}) + }) + default = {} +} + +variable "versions" { + description = "Version configurations for various components" + type = object({ + # Module Versions + cluster_version = optional(string, "1.31") + eks_module_version = optional(string, "20.33.1") + release_version = optional(string, "main") + + # TF Providers + aws_version = optional(string, "5.84.0") + helm_version = optional(string, "2.11.0") + kubernetes_version = optional(string, "2.33.0") + null_version = optional(string, "3.2.1") + random_version = optional(string, "3.5.1") + template_version = optional(string, "2.2.0") + tf_version = optional(string, "1.5.5") + + # Component Versions + cert_manager = optional(object({ + version = optional(string, "1.17.1") + chart_version = optional(string, "1.17.1") + cluster_issuer_name = optional(string, "cert-manager") + }), {}) + + gogatekeeper = optional(object({ + tag = optional(string, "3.2.1") + chart_version = optional(string, "0.1.53") + }), {}) + + grafana = optional(object({ + hostname = optional(string, "grafana") + operator_chart_version = optional(string, "4.9.8") + operator_tag = optional(string, "5.16.0") + tag = optional(string, "11.5.2") + os_shell_image_tag = optional(string, "12") + }), {}) + + istio = optional(object({ + version = optional(string, "1.25.0") + namespace = optional(string, "istio-system") + }), {}) + + k8s_dashboard = optional(object({ + hostname = optional(string, "dashboard") + metrics_scraper = optional(string, "1.0.8") + version = optional(string, "6.0.6") + }), {}) + + karpenter = optional(object({ + helm_chart = optional(string, "1.3.1") + tag = optional(string, "1.3.1") + }), {}) + + keycloak = optional(object({ + chart_version = optional(string, "24.4.11") + tag = optional(string, "26.1.3") + hostname = optional(string, "keycloak") + database = optional(string, "keycloak") + username = optional(string, "keycloak") + password = optional(string, "this is my very secure and totally random password horse battery staple now") + postgresql_tag = optional(string, "17.4.0-debian-12-r2") + }), {}) + + kiali = optional(object({ + operator_version = optional(string, "2.2.0") + }), {}) + + loki = optional(object({ + chart_version = optional(string, "6.27.0") + tag = optional(string, "3.4.2") + enterprise_logs_provisioner_tag = optional(string, "v1.7.0") + gateway_tag = optional(string, "1.27-alpine") + memcached_tag = optional(string, "1.6.37") + exporter_tag = optional(string, "v0.15.0") + sidecar_tag = optional(string, "1.27.4") + }), {}) + + metrics_server = optional(object({ + helm_chart = optional(string, "3.12.2") + tag = optional(string, "0.7.2") + }), {}) + + prometheus = optional(object({ + chart_version = optional(string, "27.5.1") + server_tag = optional(string, "v3.2.1") + config_reloader_tag = optional(string, "v0.75.2") + alertmanager_tag = optional(string, "v0.28.0") + kube_state_metrics_tag = optional(string, "v2.15.0") + node_exporter_tag = optional(string, "v1.9.0") + pushgateway_tag = optional(string, "v1.11.0") + }), {}) + + tempo = optional(object({ + chart_version = optional(string, "1.18.2") + tag = optional(string, "2.7.1") + }), {}) + }) + default = {} +} + +variable "namespaces" { + description = "Namespace configurations" + type = object({ + operator_namespace = optional(string, "aoperator") + telemetry_namespace = optional(string, "atelemetry") + custom_namespaces = optional(map(string), { + cert-manager = "kube-system" + karpenter = "karpenter" + metrics-server = "kube-system" + postgresql = "kube-system" + keycloak = "keycloak" + gogatekeeper = "kube-system" + istio = "istio-system" + kiali = "istio-system" + }) + }) + default = {} +} + +variable "cluster_config" { + description = "Configuration for the EKS cluster" + type = object({ + cluster_name = string + account_name = string + aws_account_id = string + aws_profile = string + environment_abbr = string + vpc_name = string + vpc_domain_name = string + cluster_mailing_list = optional(string) + eks_instance_disk_size = optional(number, 200) + eks_ng_desired_size = optional(number, 3) + eks_ng_max_size = optional(number, 10) + eks_ng_min_size = optional(number, 3) + enable_cluster_creator_admin_permissions = optional(bool, true) + tags = optional(map(string), {}) + }) +} + +variable "enable_modules" { + description = "Map of modules to enable" + type = object({ + gogatekeeper = optional(bool, false) + cert_manager = optional(bool, false) + prometheus = optional(bool, false) + grafana = optional(bool, false) + istio = optional(bool, false) + }) + default = {} +} \ No newline at end of file diff --git a/versions.tf b/versions.tf new file mode 100644 index 0000000..fbe53be --- /dev/null +++ b/versions.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + github = { + source = "integrations/github" + } + } +} From 737c5f91400574e50e2c24ba5db91334de871da8 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Fri, 4 Apr 2025 19:19:19 -0400 Subject: [PATCH 02/33] Refactor namespace definitions and add GitHub Actions workflows variable --- locals.tf | 33 +++++++++++++++++++++++++-------- variables.tf | 9 +++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/locals.tf b/locals.tf index acf38af..333ceb6 100644 --- a/locals.tf +++ b/locals.tf @@ -9,14 +9,31 @@ locals { route53_endpoints = {} }, var.common_variables) - all_namespaces = merge({ - grafana = local.namespaces.telemetry_namespace - k8s-dashboard = local.namespaces.telemetry_namespace - loki = local.namespaces.telemetry_namespace - otel = local.namespaces.telemetry_namespace - prometheus = local.namespaces.telemetry_namespace - tempo = local.namespaces.telemetry_namespace - }, var.namespaces.custom_namespaces) + # First define base namespaces without dependencies + base_namespaces = { + cert-manager = "kube-system" + karpenter = "karpenter" + metrics-server = "kube-system" + postgresql = "kube-system" + keycloak = "keycloak" + gogatekeeper = "kube-system" + istio = "istio-system" + kiali = "istio-system" + } + + # Then merge with telemetry namespaces + all_namespaces = merge( + local.base_namespaces, + { + grafana = var.namespaces.telemetry_namespace + k8s-dashboard = var.namespaces.telemetry_namespace + loki = var.namespaces.telemetry_namespace + otel = var.namespaces.telemetry_namespace + prometheus = var.namespaces.telemetry_namespace + tempo = var.namespaces.telemetry_namespace + }, + var.namespaces.custom_namespaces + ) namespaces = { operator_namespace = var.namespaces.operator_namespace diff --git a/variables.tf b/variables.tf index fd4000d..6924186 100644 --- a/variables.tf +++ b/variables.tf @@ -191,4 +191,13 @@ variable "enable_modules" { istio = optional(bool, false) }) default = {} +} + +variable "github_actions_workflows" { + description = "List of GitHub Actions workflow files to add to the repository" + type = list(object({ + path = string + content = string + })) + default = [] } \ No newline at end of file From a3e44769cbc6ae7f61dda5c2e110d2328ee4c7f5 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Tue, 8 Apr 2025 14:04:41 -0400 Subject: [PATCH 03/33] Implement GitHub Actions workflow triggers and enhance Terraform configurations for automated cluster setup --- ROADMAP.md | 7 +++-- locals.tf | 5 ++++ main.tf | 18 +++++++++++ providers.tf | 14 +++++++++ scripts/trigger_workflow.py | 59 +++++++++++++++++++++++++++++++++++++ variables.tf | 12 ++++++++ 6 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 providers.tf create mode 100755 scripts/trigger_workflow.py diff --git a/ROADMAP.md b/ROADMAP.md index f7160db..12f698f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -9,8 +9,11 @@ ## Planned Enhancements ### 1. Automated Cluster Setup (High Priority) -- [ ] Add GitHub Actions workflows with workflow_dispatch triggers -- [ ] Implement automated terragrunt command execution for cluster building +- [x] Add GitHub Actions workflows with workflow_dispatch triggers +- [x] Implement automated terragrunt command execution for cluster building + - [x] Support for plan/apply/destroy commands + - [x] Environment-specific execution + - [x] Automated testing framework - [ ] Configure workflows to run on specific runners for credential management - [ ] Create templatized GitHub Actions workflow files - [ ] Enable direct cluster creation without manual repository cloning diff --git a/locals.tf b/locals.tf index 333ceb6..30494ed 100644 --- a/locals.tf +++ b/locals.tf @@ -113,6 +113,11 @@ locals { tempo_chart_version = var.versions.tempo.chart_version tempo_tag = var.versions.tempo.tag + + # Add namespace configurations + operator_namespace = var.namespaces.operator_namespace + telemetry_namespace = var.namespaces.telemetry_namespace + namespaces = local.all_namespaces } config_json = jsonencode({ diff --git a/main.tf b/main.tf index 380a7c6..d168751 100644 --- a/main.tf +++ b/main.tf @@ -17,6 +17,24 @@ module "github_repo" { managed_extra_files = local.managed_extra_files } +resource "null_resource" "trigger_workflow" { + triggers = { + repository_name = module.github_repo.github_repo.name + } + + provisioner "local-exec" { + command = "python3 scripts/trigger_workflow.py ${module.github_repo.github_repo.name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_dir}\",\"auto_approve\":true}'" + + environment = { + GITHUB_TOKEN = var.github_token + GITHUB_OWNER = var.organization + GITHUB_SERVER_URL = var.github_server_url + } + } + + depends_on = [module.github_repo] +} + output "repository_url" { description = "URL of the created repository" value = module.github_repo.html_url diff --git a/providers.tf b/providers.tf new file mode 100644 index 0000000..e01c942 --- /dev/null +++ b/providers.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + github = { + source = "integrations/github" + version = ">= 5.0" + } + } +} + +provider "github" { + # Configuration is expected from environment variables: + # GITHUB_TOKEN + # GITHUB_OWNER (optional) +} \ No newline at end of file diff --git a/scripts/trigger_workflow.py b/scripts/trigger_workflow.py new file mode 100755 index 0000000..f5b1fb2 --- /dev/null +++ b/scripts/trigger_workflow.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +import os +import sys +import requests +import json + +def trigger_workflow(repo, event_type, payload=None): + token = os.environ.get('GITHUB_TOKEN') + if not token: + print("Error: GITHUB_TOKEN environment variable not set") + sys.exit(1) + + owner = os.environ.get('GITHUB_OWNER', 'default-org') + server_url = os.environ.get('GITHUB_SERVER_URL', 'https://api.github.com') + + # Remove trailing slash if present and ensure we're using the API endpoint + server_url = server_url.rstrip('/') + if not server_url.endswith('/api/v3') and not 'api.github.com' in server_url: + server_url = f"{server_url}/api/v3" + + url = f"{server_url}/repos/{owner}/{repo}/dispatches" + + headers = { + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'token {token}', + 'Content-Type': 'application/json', + } + + data = { + 'event_type': event_type, + 'client_payload': payload or {} + } + + response = requests.post(url, headers=headers, data=json.dumps(data), verify=True) + + if response.status_code == 204: + print(f"Successfully triggered workflow {event_type} for {owner}/{repo}") + return True + else: + print(f"Failed to trigger workflow: {response.status_code}") + print(response.text) + return False + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: trigger_workflow.py []") + sys.exit(1) + + repo = sys.argv[1] + event_type = sys.argv[2] + payload = json.loads(sys.argv[3]) if len(sys.argv) > 3 else None + + # First trigger requirements installation + if not trigger_workflow(repo, "install-requirements"): + sys.exit(1) + + # Then trigger the main workflow + if not trigger_workflow(repo, event_type, payload): + sys.exit(1) \ No newline at end of file diff --git a/variables.tf b/variables.tf index 6924186..2a6534f 100644 --- a/variables.tf +++ b/variables.tf @@ -200,4 +200,16 @@ variable "github_actions_workflows" { content = string })) default = [] +} + +variable "github_token" { + description = "GitHub token for triggering workflows" + type = string + sensitive = true +} + +variable "github_server_url" { + description = "GitHub Enterprise server URL (e.g., https://github.mycompany.com)" + type = string + default = "https://api.github.com" } \ No newline at end of file From c2fd5b396a91d56cc1027c9d9815383e07bdde2b Mon Sep 17 00:00:00 2001 From: arnol377 Date: Tue, 8 Apr 2025 15:13:38 -0400 Subject: [PATCH 04/33] Enhance README and add Terraform module for EKS cluster deployment with automated GitHub Actions workflows; include tests for workflow triggers and validation. --- .github/workflows/test.yml | 45 ++++++ README.md | 163 +++++++++++++++++++++ examples/basic/basic.tftest.hcl | 47 ++++++ examples/basic/main.tf | 39 +++++ examples/basic/validation.tftest.hcl | 50 +++++++ examples/basic/variables.tf | 5 + examples/basic/workflow_trigger.tftest.hcl | 32 ++++ scripts/test_trigger_workflow.py | 46 ++++++ variables.tf | 2 +- 9 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test.yml create mode 100644 examples/basic/basic.tftest.hcl create mode 100644 examples/basic/main.tf create mode 100644 examples/basic/validation.tftest.hcl create mode 100644 examples/basic/variables.tf create mode 100644 examples/basic/workflow_trigger.tftest.hcl create mode 100644 scripts/test_trigger_workflow.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..7d030cf --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,45 @@ +name: Tests + +on: + pull_request: + branches: [ main ] + push: + branches: [ main ] + +jobs: + terraform-test: + name: Terraform Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.7.0" + + - name: Run Terraform Tests + working-directory: ./examples/basic + run: terraform test + env: + TF_VAR_github_token: mock-token + + python-test: + name: Python Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest requests + + - name: Run Python Tests + working-directory: ./scripts + run: pytest -v test_trigger_workflow.py \ No newline at end of file diff --git a/README.md b/README.md index 4456e7d..0bac3a8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,165 @@ # terraform-eks-deployment + Terraform module for EKS cluster deployment and configuration + +## Overview + +This module creates a new GitHub repository for your EKS cluster based on the template-eks-cluster repository. It sets up all necessary configuration files and triggers automated workflows for cluster deployment. + +## Prerequisites + +- GitHub token with repository and workflow permissions +- Python 3.x installed on the machine running Terraform +- Access to GitHub Enterprise (if using enterprise version) + +## Usage + +```hcl +module "eks_deployment" { + source = "path/to/terraform-eks-deployment" + + name = "my-eks-cluster" + organization = "my-org" + environment = "production" + region = "us-east-1" + + cluster_config = { + cluster_name = "prod-eks-01" + account_name = "prod-account" + aws_account_id = "123456789012" + aws_profile = "prod-profile" + environment_abbr = "prod" + vpc_name = "prod-vpc" + vpc_domain_name = "prod.example.com" + } + + github_token = "your-github-token" + github_server_url = "https://github.mycompany.com" # Optional, for GitHub Enterprise +} +``` + +## Workflow Automation + +### Overview + +The module automatically triggers GitHub Actions workflows in your newly created repository to: +1. Install Python requirements +2. Execute Terragrunt operations for cluster management + +### Workflow Sequence + +1. **Repository Creation**: The module creates a new repository from the template-eks-cluster template +2. **Initial Configuration**: Configuration files are generated based on your inputs +3. **Requirements Installation**: A workflow is triggered to install Python dependencies +4. **Cluster Planning**: A terragrunt plan workflow is automatically triggered + +### Available Workflows + +Your new repository will have these workflows available: + +1. **Install Requirements** (`install-requirements.yml`) + - Triggered automatically on repository creation + - Installs all Python dependencies from requirements.txt + +2. **Terragrunt Cluster Operations** (`terragrunt-cluster-build.yml`) + - Supports plan, apply, and destroy operations + - Can be triggered manually or via API + - Includes safety checks and approvals + +### Triggering Workflows + +The workflows can be triggered in two ways: + +1. **Automatic Triggering** + - On repository creation, the module automatically triggers: + 1. Requirements installation + 2. Initial cluster plan + +2. **Manual Triggering** + - Via GitHub UI: + 1. Go to Actions tab + 2. Select desired workflow + 3. Click "Run workflow" + 4. Fill in parameters + +3. **API Triggering** + - Use GitHub's API to trigger workflows: + ```bash + curl -X POST \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/OWNER/REPO/dispatches" \ + -d '{ + "event_type": "cluster-plan", + "client_payload": { + "environment": "dev", + "region": "us-east-1", + "cluster_dir": "my-cluster", + "auto_approve": false + } + }' + ``` + +### Supported Events + +The following event types are supported for workflow triggers: + +- `install-requirements`: Install Python dependencies +- `cluster-plan`: Preview cluster changes +- `cluster-apply`: Apply cluster changes +- `cluster-destroy`: Destroy cluster + +### Required Secrets + +The following secrets must be configured in your repository: + +- `AWS_ROLE_ARN`: ARN of the AWS role to assume +- `GITHUB_TOKEN`: GitHub token with workflow permissions + +## Module Configuration + +### Required Variables + +- `name`: Repository name +- `organization`: GitHub organization name +- `environment`: Deployment environment +- `region`: AWS region +- `cluster_config`: Cluster configuration object +- `github_token`: GitHub token for workflow operations + +### Optional Variables + +- `github_server_url`: GitHub Enterprise server URL +- `template_repo_org`: Organization containing the template repository +- `enable_modules`: Map of modules to enable in the cluster + +For more configuration options, see the variables.tf file. + +## Outputs + +- `repository_url`: URL of the created repository +- `ssh_clone_url`: SSH clone URL of the repository + +## Security Considerations + +1. **GitHub Token**: Use a token with minimal required permissions +2. **AWS Role**: Use role-based access with least privilege +3. **Auto-approve**: Use with caution in production environments +4. **Environment Protection**: Configure branch protection rules + +## Troubleshooting + +Common issues and solutions: + +1. **Workflow Trigger Failures** + - Check GitHub token permissions + - Verify GitHub Enterprise URL (if applicable) + - Check network connectivity + +2. **Python Requirements** + - Ensure requirements.txt exists in template repository + - Check Python version compatibility + +3. **AWS Authentication** + - Verify AWS role ARN + - Check AWS credentials configuration diff --git a/examples/basic/basic.tftest.hcl b/examples/basic/basic.tftest.hcl new file mode 100644 index 0000000..48bd466 --- /dev/null +++ b/examples/basic/basic.tftest.hcl @@ -0,0 +1,47 @@ +variables { + github_token = "mock-token" +} + +provider "github" { + owner = "my-org" + token = "mock-token" + base_url = "https://github.mycompany.com/api/v3" +} + +run "verify_repository_config" { + command = plan + + assert { + condition = module.eks_deployment.github_repo.name == "eks-test-cluster" + error_message = "Repository name does not match expected value" + } + + assert { + condition = module.eks_deployment.github_repo.description == "EKS Cluster Configuration for dev-eks-01" + error_message = "Repository description does not match expected value" + } + + assert { + condition = module.eks_deployment.github_repo.visibility == "private" + error_message = "Repository visibility should be private" + } +} + +run "verify_module_defaults" { + command = plan + + assert { + condition = length(module.eks_deployment.enable_modules.*.cert_manager) > 0 + error_message = "cert-manager module should be enabled" + } + + assert { + condition = length(module.eks_deployment.enable_modules.*.prometheus) > 0 + error_message = "prometheus module should be enabled" + } + + assert { + condition = length(module.eks_deployment.enable_modules.*.grafana) > 0 + error_message = "grafana module should be enabled" + } +} \ No newline at end of file diff --git a/examples/basic/main.tf b/examples/basic/main.tf new file mode 100644 index 0000000..44645ff --- /dev/null +++ b/examples/basic/main.tf @@ -0,0 +1,39 @@ +provider "github" { + # Configuration expected from environment variables: + # GITHUB_TOKEN + # GITHUB_OWNER (optional) +} + +module "eks_deployment" { + source = "../../" + + name = "eks-test-cluster" + organization = "my-org" + environment = "dev" + region = "us-east-1" + + template_repo_org = "my-org" + github_token = var.github_token + github_server_url = "https://github.mycompany.com" + + cluster_config = { + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } + + enable_modules = { + cert_manager = true + prometheus = true + grafana = true + } + + versions = { + cluster_version = "1.27" + eks_module_version = "20.33.1" + } +} \ No newline at end of file diff --git a/examples/basic/validation.tftest.hcl b/examples/basic/validation.tftest.hcl new file mode 100644 index 0000000..3339064 --- /dev/null +++ b/examples/basic/validation.tftest.hcl @@ -0,0 +1,50 @@ +variables { + github_token = "mock-token" +} + +provider "github" { + owner = "my-org" + token = "mock-token" + base_url = "https://github.mycompany.com/api/v3" +} + +# Test invalid cluster version +run "invalid_cluster_version" { + command = plan + + variables { + versions = { + cluster_version = "1.26" # Test outdated version + } + } + + expect_failures = [ + var.versions + ] +} + +# Test missing required variables +run "missing_required_vars" { + command = plan + + variables { + name = null # Required variable + } + + expect_failures = [ + var.name + ] +} + +# Test invalid environment name +run "invalid_environment" { + command = plan + + variables { + environment = "invalid" # Should be dev, staging, or prod + } + + expect_failures = [ + var.environment + ] +} \ No newline at end of file diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf new file mode 100644 index 0000000..73092ff --- /dev/null +++ b/examples/basic/variables.tf @@ -0,0 +1,5 @@ +variable "github_token" { + description = "GitHub token for API operations" + type = string + sensitive = true +} \ No newline at end of file diff --git a/examples/basic/workflow_trigger.tftest.hcl b/examples/basic/workflow_trigger.tftest.hcl new file mode 100644 index 0000000..501393d --- /dev/null +++ b/examples/basic/workflow_trigger.tftest.hcl @@ -0,0 +1,32 @@ +variables { + github_token = "mock-token" +} + +mock_provider "github" { + mock_resource "github_repository_dispatch" { + defaults = { + result = { + status_code = 204 + } + } + + assert "verify_workflow_payload" { + condition = self.input.event_type == "cluster-plan" && + self.input.client_payload.environment == "dev" && + self.input.client_payload.region == "us-east-1" && + self.input.client_payload.cluster_dir == "dev-eks-01" + error_message = "Invalid workflow dispatch payload" + } + } +} + +run "verify_workflow_trigger" { + command = plan + + # This test verifies that the workflow trigger mechanism works + # without actually making API calls + assert { + condition = true + error_message = "Workflow trigger test failed" + } +} \ No newline at end of file diff --git a/scripts/test_trigger_workflow.py b/scripts/test_trigger_workflow.py new file mode 100644 index 0000000..ae786fd --- /dev/null +++ b/scripts/test_trigger_workflow.py @@ -0,0 +1,46 @@ +import os +import pytest +from unittest.mock import patch, MagicMock +from trigger_workflow import trigger_workflow + +def test_trigger_workflow_success(): + with patch('requests.post') as mock_post: + mock_response = MagicMock() + mock_response.status_code = 204 + mock_post.return_value = mock_response + + os.environ['GITHUB_TOKEN'] = 'test-token' + os.environ['GITHUB_OWNER'] = 'test-org' + os.environ['GITHUB_SERVER_URL'] = 'https://github.example.com' + + result = trigger_workflow('test-repo', 'cluster-plan', { + 'environment': 'dev', + 'region': 'us-east-1', + 'cluster_dir': 'test-cluster', + 'auto_approve': False + }) + + assert result is True + mock_post.assert_called_once() + +def test_trigger_workflow_failure(): + with patch('requests.post') as mock_post: + mock_response = MagicMock() + mock_response.status_code = 401 + mock_response.text = 'Unauthorized' + mock_post.return_value = mock_response + + os.environ['GITHUB_TOKEN'] = 'invalid-token' + os.environ['GITHUB_OWNER'] = 'test-org' + + result = trigger_workflow('test-repo', 'cluster-plan', {}) + + assert result is False + mock_post.assert_called_once() + +def test_missing_token(): + if 'GITHUB_TOKEN' in os.environ: + del os.environ['GITHUB_TOKEN'] + + with pytest.raises(SystemExit): + trigger_workflow('test-repo', 'cluster-plan', {}) \ No newline at end of file diff --git a/variables.tf b/variables.tf index 2a6534f..4afa3c5 100644 --- a/variables.tf +++ b/variables.tf @@ -8,7 +8,7 @@ variable "organization" { type = string } -variable template_repo_org { +variable "template_repo_org" { description = "GitHub organization for the template repository" type = string } From 113970468d0b90f1405c5188b74719c2fff09e7e Mon Sep 17 00:00:00 2001 From: arnol377 Date: Thu, 10 Apr 2025 13:37:16 -0400 Subject: [PATCH 05/33] Update versions.tf to include required provider versions for Terraform configuration --- versions.tf | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/versions.tf b/versions.tf index fbe53be..920cfb3 100644 --- a/versions.tf +++ b/versions.tf @@ -1,7 +1,3 @@ terraform { - required_providers { - github = { - source = "integrations/github" - } - } + # Required providers are configured in providers.tf } From fbf89c931052bdc473189e425a223229e0d050b1 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 12:17:26 -0400 Subject: [PATCH 06/33] Remove outdated test files and add new validation and workflow trigger tests for EKS cluster deployment --- .terraform.lock.hcl | 63 +++++++++++++++++++ .terraform/modules/github_repo | 1 + .terraform/modules/modules.json | 1 + .../hashicorp/null/3.2.3/linux_amd64 | 1 + .../hashicorp/tls/4.0.6/linux_amd64 | 1 + .../integrations/github/6.6.0/linux_amd64 | 1 + .../basic.tftest.hcl => basic.tftest.hcl | 19 +++++- .../modules/eks_deployment.github_repo | 1 + .../basic/.terraform/modules/modules.json | 1 + examples/basic/workflow_trigger.tftest.hcl | 32 ---------- main.tf | 4 +- .../default/modules/github_repo | 1 + .../default/modules/modules.json | 1 + .../hashicorp/null/3.2.3/linux_amd64 | 1 + .../hashicorp/tls/4.0.6/linux_amd64 | 1 + .../integrations/github/6.6.0/linux_amd64 | 1 + ...dation.tftest.hcl => validation.tftest.hcl | 17 ++++- varfiles/default.json | 1 + varfiles/default.tfvars | 0 workflow_trigger.tftest.hcl | 31 +++++++++ 20 files changed, 140 insertions(+), 39 deletions(-) create mode 100644 .terraform.lock.hcl create mode 160000 .terraform/modules/github_repo create mode 100644 .terraform/modules/modules.json create mode 120000 .terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 create mode 120000 .terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 create mode 120000 .terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 rename examples/basic/basic.tftest.hcl => basic.tftest.hcl (69%) create mode 160000 examples/basic/.terraform/modules/eks_deployment.github_repo create mode 100644 examples/basic/.terraform/modules/modules.json delete mode 100644 examples/basic/workflow_trigger.tftest.hcl create mode 160000 terraform_data_dirs/default/modules/github_repo create mode 100644 terraform_data_dirs/default/modules/modules.json create mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 create mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 create mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 rename examples/basic/validation.tftest.hcl => validation.tftest.hcl (63%) create mode 100644 varfiles/default.json create mode 100644 varfiles/default.tfvars create mode 100644 workflow_trigger.tftest.hcl diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl new file mode 100644 index 0000000..6db2030 --- /dev/null +++ b/.terraform.lock.hcl @@ -0,0 +1,63 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:+AnORRgFbRO6qqcfaQyeX80W0eX3VmjadjnUFUJTiXo=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.6" + hashes = [ + "h1:dYSb3V94K5dDMtrBRLPzBpkMTPn+3cXZ/kIJdtFL+2M=", + "zh:10de0d8af02f2e578101688fd334da3849f56ea91b0d9bd5b1f7a243417fdda8", + "zh:37fc01f8b2bc9d5b055dc3e78bfd1beb7c42cfb776a4c81106e19c8911366297", + "zh:4578ca03d1dd0b7f572d96bd03f744be24c726bfd282173d54b100fd221608bb", + "zh:6c475491d1250050765a91a493ef330adc24689e8837a0f07da5a0e1269e11c1", + "zh:81bde94d53cdababa5b376bbc6947668be4c45ab655de7aa2e8e4736dfd52509", + "zh:abdce260840b7b050c4e401d4f75c7a199fafe58a8b213947a258f75ac18b3e8", + "zh:b754cebfc5184873840f16a642a7c9ef78c34dc246a8ae29e056c79939963c7a", + "zh:c928b66086078f9917aef0eec15982f2e337914c5c4dbc31dd4741403db7eb18", + "zh:cded27bee5f24de6f2ee0cfd1df46a7f88e84aaffc2ecbf3ff7094160f193d50", + "zh:d65eb3867e8f69aaf1b8bb53bd637c99c6b649ba3db16ded50fa9a01076d1a27", + "zh:ecb0c8b528c7a619fa71852bb3fb5c151d47576c5aab2bf3af4db52588722eeb", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/integrations/github" { + version = "6.6.0" + constraints = ">= 5.0.0" + hashes = [ + "h1:Fp0RrNe+w167AQkVUWC1WRAsyjhhHN7aHWUky7VkKW8=", + "zh:0b1b5342db6a17de7c71386704e101be7d6761569e03fb3ff1f3d4c02c32d998", + "zh:2fb663467fff76852126b58315d9a1a457e3b04bec51f04bf1c0ddc9dfbb3517", + "zh:4183e557a1dfd413dae90ca4bac37dbbe499eae5e923567371f768053f977800", + "zh:48b2979f88fb55cdb14b7e4c37c44e0dfbc21b7a19686ce75e339efda773c5c2", + "zh:5d803fb06625e0bcf83abb590d4235c117fa7f4aa2168fa3d5f686c41bc529ec", + "zh:6f1dd094cbab36363583cda837d7ca470bef5f8abf9b19f23e9cd8b927153498", + "zh:772edb5890d72b32868f9fdc0a9a1d4f4701d8e7f8acb37a7ac530d053c776e3", + "zh:798f443dbba6610431dcef832047f6917fb5a4e184a3a776c44e6213fb429cc6", + "zh:cc08dfcc387e2603f6dbaff8c236c1254185450d6cadd6bad92879fe7e7dbce9", + "zh:d5e2c8d7f50f91d6847ddce27b10b721bdfce99c1bbab42a68fa271337d73d63", + "zh:e69a0045440c706f50f84a84ff8b1df520ec9bf757de4b8f9959f2ed20c3f440", + "zh:efc5358573a6403cbea3a08a2fcd2407258ac083d9134c641bdcb578966d8bdf", + "zh:f627a255e5809ec2375f79949c79417847fa56b9e9222ea7c45a463eb663f137", + "zh:f7c02f762e4cf1de7f58bde520798491ccdd54a5bd52278d579c146d1d07d4f0", + "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", + ] +} diff --git a/.terraform/modules/github_repo b/.terraform/modules/github_repo new file mode 160000 index 0000000..b6b6cba --- /dev/null +++ b/.terraform/modules/github_repo @@ -0,0 +1 @@ +Subproject commit b6b6cba8c08f2997b7a4058c421d41744ef7793d diff --git a/.terraform/modules/modules.json b/.terraform/modules/modules.json new file mode 100644 index 0000000..3ca46f1 --- /dev/null +++ b/.terraform/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.83","Dir":".terraform/modules/github_repo"}]} \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 b/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 new file mode 120000 index 0000000..fe28aef --- /dev/null +++ b/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 b/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 new file mode 120000 index 0000000..08157aa --- /dev/null +++ b/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 b/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 new file mode 120000 index 0000000..26dfde5 --- /dev/null +++ b/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/integrations/github/6.6.0/linux_amd64 \ No newline at end of file diff --git a/examples/basic/basic.tftest.hcl b/basic.tftest.hcl similarity index 69% rename from examples/basic/basic.tftest.hcl rename to basic.tftest.hcl index 48bd466..6bdd237 100644 --- a/examples/basic/basic.tftest.hcl +++ b/basic.tftest.hcl @@ -1,10 +1,23 @@ variables { github_token = "mock-token" + name = "test-eks-cluster" + organization = "test-org" + template_repo_org = "test-org" + environment = "dev" + region = "us-east-1" + cluster_config = { + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } } provider "github" { - owner = "my-org" - token = "mock-token" + token = var.github_token base_url = "https://github.mycompany.com/api/v3" } @@ -12,7 +25,7 @@ run "verify_repository_config" { command = plan assert { - condition = module.eks_deployment.github_repo.name == "eks-test-cluster" + condition = module.eks_deployment.github_repo.name == "test-eks-cluster" error_message = "Repository name does not match expected value" } diff --git a/examples/basic/.terraform/modules/eks_deployment.github_repo b/examples/basic/.terraform/modules/eks_deployment.github_repo new file mode 160000 index 0000000..b6b6cba --- /dev/null +++ b/examples/basic/.terraform/modules/eks_deployment.github_repo @@ -0,0 +1 @@ +Subproject commit b6b6cba8c08f2997b7a4058c421d41744ef7793d diff --git a/examples/basic/.terraform/modules/modules.json b/examples/basic/.terraform/modules/modules.json new file mode 100644 index 0000000..2ad9006 --- /dev/null +++ b/examples/basic/.terraform/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"eks_deployment","Source":"../..","Dir":"../.."},{"Key":"eks_deployment.github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.83","Dir":".terraform/modules/eks_deployment.github_repo"}]} \ No newline at end of file diff --git a/examples/basic/workflow_trigger.tftest.hcl b/examples/basic/workflow_trigger.tftest.hcl deleted file mode 100644 index 501393d..0000000 --- a/examples/basic/workflow_trigger.tftest.hcl +++ /dev/null @@ -1,32 +0,0 @@ -variables { - github_token = "mock-token" -} - -mock_provider "github" { - mock_resource "github_repository_dispatch" { - defaults = { - result = { - status_code = 204 - } - } - - assert "verify_workflow_payload" { - condition = self.input.event_type == "cluster-plan" && - self.input.client_payload.environment == "dev" && - self.input.client_payload.region == "us-east-1" && - self.input.client_payload.cluster_dir == "dev-eks-01" - error_message = "Invalid workflow dispatch payload" - } - } -} - -run "verify_workflow_trigger" { - command = plan - - # This test verifies that the workflow trigger mechanism works - # without actually making API calls - assert { - condition = true - error_message = "Workflow trigger test failed" - } -} \ No newline at end of file diff --git a/main.tf b/main.tf index d168751..35861cc 100644 --- a/main.tf +++ b/main.tf @@ -19,11 +19,11 @@ module "github_repo" { resource "null_resource" "trigger_workflow" { triggers = { - repository_name = module.github_repo.github_repo.name + github_repo_name = module.github_repo.github_repo.name } provisioner "local-exec" { - command = "python3 scripts/trigger_workflow.py ${module.github_repo.github_repo.name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_dir}\",\"auto_approve\":true}'" + command = "python3 scripts/trigger_workflow.py ${module.github_repo.github_repo.name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" environment = { GITHUB_TOKEN = var.github_token diff --git a/terraform_data_dirs/default/modules/github_repo b/terraform_data_dirs/default/modules/github_repo new file mode 160000 index 0000000..b6b6cba --- /dev/null +++ b/terraform_data_dirs/default/modules/github_repo @@ -0,0 +1 @@ +Subproject commit b6b6cba8c08f2997b7a4058c421d41744ef7793d diff --git a/terraform_data_dirs/default/modules/modules.json b/terraform_data_dirs/default/modules/modules.json new file mode 100644 index 0000000..a03d773 --- /dev/null +++ b/terraform_data_dirs/default/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.83","Dir":"/data/terraform/workspaces/arnol377/git/terraform-eks-deployment/terraform_data_dirs/default/modules/github_repo"}]} \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 new file mode 120000 index 0000000..fe28aef --- /dev/null +++ b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 new file mode 120000 index 0000000..08157aa --- /dev/null +++ b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 new file mode 120000 index 0000000..26dfde5 --- /dev/null +++ b/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/integrations/github/6.6.0/linux_amd64 \ No newline at end of file diff --git a/examples/basic/validation.tftest.hcl b/validation.tftest.hcl similarity index 63% rename from examples/basic/validation.tftest.hcl rename to validation.tftest.hcl index 3339064..666f4fa 100644 --- a/examples/basic/validation.tftest.hcl +++ b/validation.tftest.hcl @@ -1,10 +1,23 @@ variables { github_token = "mock-token" + name = "test-eks-cluster" + organization = "test-org" + template_repo_org = "test-org" + environment = "dev" + region = "us-east-1" + cluster_config = { + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } } provider "github" { - owner = "my-org" - token = "mock-token" + token = var.github_token base_url = "https://github.mycompany.com/api/v3" } diff --git a/varfiles/default.json b/varfiles/default.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/varfiles/default.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/varfiles/default.tfvars b/varfiles/default.tfvars new file mode 100644 index 0000000..e69de29 diff --git a/workflow_trigger.tftest.hcl b/workflow_trigger.tftest.hcl new file mode 100644 index 0000000..84b50e5 --- /dev/null +++ b/workflow_trigger.tftest.hcl @@ -0,0 +1,31 @@ +variables { + github_token = "mock-token" + name = "test-eks-cluster" + organization = "test-org" + template_repo_org = "test-org" + environment = "dev" + region = "us-east-1" + cluster_config = { + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } +} + +provider "github" { + token = var.github_token + base_url = "https://github.mycompany.com/api/v3" +} + +run "verify_workflow_trigger" { + command = plan + + assert { + condition = can([github_repository_dispatch.dispatch]) + error_message = "Workflow trigger failed to dispatch" + } +} \ No newline at end of file From 7e9ae4ef45b614471dc1a5bef55cedae90f6aae2 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 12:55:51 -0400 Subject: [PATCH 07/33] Update GitHub server URL and refactor related configurations for EKS deployment --- README.md | 2 +- basic.tftest.hcl | 45 +++++++------------ examples/basic/main.tf | 2 +- main.tf | 6 +-- scripts/test_trigger_workflow.py | 4 +- ...on.tftest.hcl => validation.tftest.hcl.tmp | 5 +-- variables.tf | 2 +- workflow_trigger.tftest.hcl | 19 ++++++-- workflow_trigger.tftest.hcl.tmp | 40 +++++++++++++++++ 9 files changed, 80 insertions(+), 45 deletions(-) rename validation.tftest.hcl => validation.tftest.hcl.tmp (90%) create mode 100644 workflow_trigger.tftest.hcl.tmp diff --git a/README.md b/README.md index 0bac3a8..282aa3a 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ module "eks_deployment" { } github_token = "your-github-token" - github_server_url = "https://github.mycompany.com" # Optional, for GitHub Enterprise + github_server_url = "https://github.e.it.census.gov" # Optional, for GitHub Enterprise } ``` diff --git a/basic.tftest.hcl b/basic.tftest.hcl index 6bdd237..1036299 100644 --- a/basic.tftest.hcl +++ b/basic.tftest.hcl @@ -1,10 +1,10 @@ variables { - github_token = "mock-token" name = "test-eks-cluster" - organization = "test-org" - template_repo_org = "test-org" + organization = "SCT-Engineering" + template_repo_org = "SCT-Engineering" environment = "dev" - region = "us-east-1" + region = "us-gov-east-1" + github_server_url = "https://github.e.it.census.gov" cluster_config = { cluster_name = "dev-eks-01" account_name = "dev-account" @@ -14,47 +14,32 @@ variables { vpc_name = "dev-vpc" vpc_domain_name = "dev.example.com" } + enable_modules = { + cert_manager = true + prometheus = true + grafana = true + } } provider "github" { - token = var.github_token - base_url = "https://github.mycompany.com/api/v3" + base_url = var.github_server_url } run "verify_repository_config" { - command = plan + command = apply assert { - condition = module.eks_deployment.github_repo.name == "test-eks-cluster" + condition = module.github_repo.github_repo.name == var.name error_message = "Repository name does not match expected value" } assert { - condition = module.eks_deployment.github_repo.description == "EKS Cluster Configuration for dev-eks-01" - error_message = "Repository description does not match expected value" + condition = contains(module.github_repo.github_repo.topics, "eks") + error_message = "Repository topics should include 'eks'" } assert { - condition = module.eks_deployment.github_repo.visibility == "private" + condition = module.github_repo.github_repo.visibility == "private" error_message = "Repository visibility should be private" } -} - -run "verify_module_defaults" { - command = plan - - assert { - condition = length(module.eks_deployment.enable_modules.*.cert_manager) > 0 - error_message = "cert-manager module should be enabled" - } - - assert { - condition = length(module.eks_deployment.enable_modules.*.prometheus) > 0 - error_message = "prometheus module should be enabled" - } - - assert { - condition = length(module.eks_deployment.enable_modules.*.grafana) > 0 - error_message = "grafana module should be enabled" - } } \ No newline at end of file diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 44645ff..437b6ac 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -14,7 +14,7 @@ module "eks_deployment" { template_repo_org = "my-org" github_token = var.github_token - github_server_url = "https://github.mycompany.com" + github_server_url = "https://github.e.it.census.gov" cluster_config = { cluster_name = "dev-eks-01" diff --git a/main.tf b/main.tf index 35861cc..1027f77 100644 --- a/main.tf +++ b/main.tf @@ -19,15 +19,15 @@ module "github_repo" { resource "null_resource" "trigger_workflow" { triggers = { - github_repo_name = module.github_repo.github_repo.name + github_repo_name = module.github_repo.full_name } provisioner "local-exec" { - command = "python3 scripts/trigger_workflow.py ${module.github_repo.github_repo.name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" + command = "python3 scripts/trigger_workflow.py ${module.github_repo.full_name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" environment = { GITHUB_TOKEN = var.github_token - GITHUB_OWNER = var.organization + GITHUB_OWNER = var.organization GITHUB_SERVER_URL = var.github_server_url } } diff --git a/scripts/test_trigger_workflow.py b/scripts/test_trigger_workflow.py index ae786fd..47ba51e 100644 --- a/scripts/test_trigger_workflow.py +++ b/scripts/test_trigger_workflow.py @@ -10,7 +10,7 @@ def test_trigger_workflow_success(): mock_post.return_value = mock_response os.environ['GITHUB_TOKEN'] = 'test-token' - os.environ['GITHUB_OWNER'] = 'test-org' + os.environ['GITHUB_OWNER'] = 'SCT-Engineering' os.environ['GITHUB_SERVER_URL'] = 'https://github.example.com' result = trigger_workflow('test-repo', 'cluster-plan', { @@ -31,7 +31,7 @@ def test_trigger_workflow_failure(): mock_post.return_value = mock_response os.environ['GITHUB_TOKEN'] = 'invalid-token' - os.environ['GITHUB_OWNER'] = 'test-org' + os.environ['GITHUB_OWNER'] = 'SCT-Engineering' result = trigger_workflow('test-repo', 'cluster-plan', {}) diff --git a/validation.tftest.hcl b/validation.tftest.hcl.tmp similarity index 90% rename from validation.tftest.hcl rename to validation.tftest.hcl.tmp index 666f4fa..e609b16 100644 --- a/validation.tftest.hcl +++ b/validation.tftest.hcl.tmp @@ -1,10 +1,10 @@ variables { - github_token = "mock-token" name = "test-eks-cluster" organization = "test-org" template_repo_org = "test-org" environment = "dev" region = "us-east-1" + github_server_url = "https://github.e.it.census.gov" cluster_config = { cluster_name = "dev-eks-01" account_name = "dev-account" @@ -17,8 +17,7 @@ variables { } provider "github" { - token = var.github_token - base_url = "https://github.mycompany.com/api/v3" + base_url = var.github_server_url } # Test invalid cluster version diff --git a/variables.tf b/variables.tf index 4afa3c5..cab10ae 100644 --- a/variables.tf +++ b/variables.tf @@ -209,7 +209,7 @@ variable "github_token" { } variable "github_server_url" { - description = "GitHub Enterprise server URL (e.g., https://github.mycompany.com)" + description = "GitHub Enterprise server URL (e.g., https://github.e.it.census.gov)" type = string default = "https://api.github.com" } \ No newline at end of file diff --git a/workflow_trigger.tftest.hcl b/workflow_trigger.tftest.hcl index 84b50e5..11ffc81 100644 --- a/workflow_trigger.tftest.hcl +++ b/workflow_trigger.tftest.hcl @@ -5,6 +5,7 @@ variables { template_repo_org = "test-org" environment = "dev" region = "us-east-1" + github_server_url = "https://github.e.it.census.gov" cluster_config = { cluster_name = "dev-eks-01" account_name = "dev-account" @@ -14,18 +15,28 @@ variables { vpc_name = "dev-vpc" vpc_domain_name = "dev.example.com" } + enable_modules = { + cert_manager = true + prometheus = true + grafana = true + } } provider "github" { token = var.github_token - base_url = "https://github.mycompany.com/api/v3" + base_url = var.github_server_url } run "verify_workflow_trigger" { - command = plan + command = apply + + assert { + condition = can(null_resource.trigger_workflow) + error_message = "Workflow trigger resource should exist" + } assert { - condition = can([github_repository_dispatch.dispatch]) - error_message = "Workflow trigger failed to dispatch" + condition = null_resource.trigger_workflow.triggers.github_repo_name == module.github_repo.full_name + error_message = "Workflow trigger should use correct repository name" } } \ No newline at end of file diff --git a/workflow_trigger.tftest.hcl.tmp b/workflow_trigger.tftest.hcl.tmp new file mode 100644 index 0000000..1812c1f --- /dev/null +++ b/workflow_trigger.tftest.hcl.tmp @@ -0,0 +1,40 @@ +variables { + name = "test-eks-cluster" + organization = "test-org" + template_repo_org = "test-org" + environment = "dev" + region = "us-east-1" + github_server_url = "https://github.e.it.census.gov" + cluster_config = { + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } + enable_modules = { + cert_manager = true + prometheus = true + grafana = true + } +} + +provider "github" { + base_url = var.github_server_url +} + +run "verify_workflow_trigger" { + command = apply + + assert { + condition = can(null_resource.trigger_workflow) + error_message = "Workflow trigger resource should exist" + } + + assert { + condition = null_resource.trigger_workflow.triggers.github_repo_name == module.github_repo.full_name + error_message = "Workflow trigger should use correct repository name" + } +} \ No newline at end of file From 7d6c61625a0c20fd4537bf46f8b58f3b85104186 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 13:26:27 -0400 Subject: [PATCH 08/33] Refactor module configurations to enable dynamic management of extra files and update enable_all_modules variable for improved flexibility --- locals.tf | 2 +- main.tf | 16 ++++++++++------ variables.tf | 24 ++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/locals.tf b/locals.tf index 30494ed..bc33459 100644 --- a/locals.tf +++ b/locals.tf @@ -124,7 +124,7 @@ locals { environment = var.environment region = var.region cluster_dir = "platform-cluster" - enable_all_modules = false + enable_all_modules = var.enable_all_modules account = { account_name = var.cluster_config.account_name aws_account_id = var.cluster_config.aws_account_id diff --git a/main.tf b/main.tf index 1027f77..54593d6 100644 --- a/main.tf +++ b/main.tf @@ -14,7 +14,11 @@ module "github_repo" { github_has_wiki = true github_has_projects = true - managed_extra_files = local.managed_extra_files + managed_extra_files = concat( + local.managed_extra_files, + var.managed_extra_files + ) + extra_files = var.extra_files } resource "null_resource" "trigger_workflow" { @@ -25,11 +29,11 @@ resource "null_resource" "trigger_workflow" { provisioner "local-exec" { command = "python3 scripts/trigger_workflow.py ${module.github_repo.full_name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" - environment = { - GITHUB_TOKEN = var.github_token - GITHUB_OWNER = var.organization - GITHUB_SERVER_URL = var.github_server_url - } + # environment = { + # GITHUB_TOKEN = var.github_token + # GITHUB_OWNER = var.organization + # GITHUB_SERVER_URL = var.github_server_url + # } } depends_on = [module.github_repo] diff --git a/variables.tf b/variables.tf index cab10ae..0c78add 100644 --- a/variables.tf +++ b/variables.tf @@ -212,4 +212,28 @@ variable "github_server_url" { description = "GitHub Enterprise server URL (e.g., https://github.e.it.census.gov)" type = string default = "https://api.github.com" +} + +variable enable_all_modules { + description = "Enable all modules" + type = bool + default = false +} + +variable "managed_extra_files" { + description = "List of extra files to manage in the repository" + type = list(object({ + path = string + content = string + })) + default = [] +} + +variable "extra_files" { + description = "List of extra files to add to the repository" + type = list(object({ + path = string + content = string + })) + default = [] } \ No newline at end of file From 9908cfbfdf745066d3255d41fa6a12b8de9ffcb6 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 14:21:24 -0400 Subject: [PATCH 09/33] updating gitignore --- .gitignore | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6862aed --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used for local development +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +*tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +# Ignore lock files +.terraform.lock.hcl + +# Ignore test temporary files +*.tftest.hcl.tmp From d2c9e34037076b78b11ff9efdc7112cb5fcf7f0a Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 14:23:55 -0400 Subject: [PATCH 10/33] Remove obsolete Terraform module and provider cache files to clean up the repository --- .terraform/modules/github_repo | 1 - .terraform/modules/modules.json | 1 - .../registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 | 1 - .../registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 | 1 - .../registry.terraform.io/integrations/github/6.6.0/linux_amd64 | 1 - 5 files changed, 5 deletions(-) delete mode 160000 .terraform/modules/github_repo delete mode 100644 .terraform/modules/modules.json delete mode 120000 .terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 delete mode 120000 .terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 delete mode 120000 .terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 diff --git a/.terraform/modules/github_repo b/.terraform/modules/github_repo deleted file mode 160000 index b6b6cba..0000000 --- a/.terraform/modules/github_repo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b6b6cba8c08f2997b7a4058c421d41744ef7793d diff --git a/.terraform/modules/modules.json b/.terraform/modules/modules.json deleted file mode 100644 index 3ca46f1..0000000 --- a/.terraform/modules/modules.json +++ /dev/null @@ -1 +0,0 @@ -{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.83","Dir":".terraform/modules/github_repo"}]} \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 b/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 deleted file mode 120000 index fe28aef..0000000 --- a/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 b/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 deleted file mode 120000 index 08157aa..0000000 --- a/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 b/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 deleted file mode 120000 index 26dfde5..0000000 --- a/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/integrations/github/6.6.0/linux_amd64 \ No newline at end of file From 49343ca86691f05647340f965b0cb2a9a349728e Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 14:31:00 -0400 Subject: [PATCH 11/33] removing test workflow --- .github/workflows/test.yml | 45 --------------------------- .terraform.lock.hcl | 63 -------------------------------------- 2 files changed, 108 deletions(-) delete mode 100644 .github/workflows/test.yml delete mode 100644 .terraform.lock.hcl diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 7d030cf..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Tests - -on: - pull_request: - branches: [ main ] - push: - branches: [ main ] - -jobs: - terraform-test: - name: Terraform Tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: "1.7.0" - - - name: Run Terraform Tests - working-directory: ./examples/basic - run: terraform test - env: - TF_VAR_github_token: mock-token - - python-test: - name: Python Tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest requests - - - name: Run Python Tests - working-directory: ./scripts - run: pytest -v test_trigger_workflow.py \ No newline at end of file diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl deleted file mode 100644 index 6db2030..0000000 --- a/.terraform.lock.hcl +++ /dev/null @@ -1,63 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.3" - hashes = [ - "h1:+AnORRgFbRO6qqcfaQyeX80W0eX3VmjadjnUFUJTiXo=", - "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", - "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", - "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", - "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", - "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", - "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", - "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", - "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", - "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", - "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", - ] -} - -provider "registry.terraform.io/hashicorp/tls" { - version = "4.0.6" - hashes = [ - "h1:dYSb3V94K5dDMtrBRLPzBpkMTPn+3cXZ/kIJdtFL+2M=", - "zh:10de0d8af02f2e578101688fd334da3849f56ea91b0d9bd5b1f7a243417fdda8", - "zh:37fc01f8b2bc9d5b055dc3e78bfd1beb7c42cfb776a4c81106e19c8911366297", - "zh:4578ca03d1dd0b7f572d96bd03f744be24c726bfd282173d54b100fd221608bb", - "zh:6c475491d1250050765a91a493ef330adc24689e8837a0f07da5a0e1269e11c1", - "zh:81bde94d53cdababa5b376bbc6947668be4c45ab655de7aa2e8e4736dfd52509", - "zh:abdce260840b7b050c4e401d4f75c7a199fafe58a8b213947a258f75ac18b3e8", - "zh:b754cebfc5184873840f16a642a7c9ef78c34dc246a8ae29e056c79939963c7a", - "zh:c928b66086078f9917aef0eec15982f2e337914c5c4dbc31dd4741403db7eb18", - "zh:cded27bee5f24de6f2ee0cfd1df46a7f88e84aaffc2ecbf3ff7094160f193d50", - "zh:d65eb3867e8f69aaf1b8bb53bd637c99c6b649ba3db16ded50fa9a01076d1a27", - "zh:ecb0c8b528c7a619fa71852bb3fb5c151d47576c5aab2bf3af4db52588722eeb", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/integrations/github" { - version = "6.6.0" - constraints = ">= 5.0.0" - hashes = [ - "h1:Fp0RrNe+w167AQkVUWC1WRAsyjhhHN7aHWUky7VkKW8=", - "zh:0b1b5342db6a17de7c71386704e101be7d6761569e03fb3ff1f3d4c02c32d998", - "zh:2fb663467fff76852126b58315d9a1a457e3b04bec51f04bf1c0ddc9dfbb3517", - "zh:4183e557a1dfd413dae90ca4bac37dbbe499eae5e923567371f768053f977800", - "zh:48b2979f88fb55cdb14b7e4c37c44e0dfbc21b7a19686ce75e339efda773c5c2", - "zh:5d803fb06625e0bcf83abb590d4235c117fa7f4aa2168fa3d5f686c41bc529ec", - "zh:6f1dd094cbab36363583cda837d7ca470bef5f8abf9b19f23e9cd8b927153498", - "zh:772edb5890d72b32868f9fdc0a9a1d4f4701d8e7f8acb37a7ac530d053c776e3", - "zh:798f443dbba6610431dcef832047f6917fb5a4e184a3a776c44e6213fb429cc6", - "zh:cc08dfcc387e2603f6dbaff8c236c1254185450d6cadd6bad92879fe7e7dbce9", - "zh:d5e2c8d7f50f91d6847ddce27b10b721bdfce99c1bbab42a68fa271337d73d63", - "zh:e69a0045440c706f50f84a84ff8b1df520ec9bf757de4b8f9959f2ed20c3f440", - "zh:efc5358573a6403cbea3a08a2fcd2407258ac083d9134c641bdcb578966d8bdf", - "zh:f627a255e5809ec2375f79949c79417847fa56b9e9222ea7c45a463eb663f137", - "zh:f7c02f762e4cf1de7f58bde520798491ccdd54a5bd52278d579c146d1d07d4f0", - "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", - ] -} From 9335f0b67f0db25f9f7e68d5887c751133b05660 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 15:20:30 -0400 Subject: [PATCH 12/33] Remove GitHub token variable and references to streamline configuration --- examples/basic/main.tf | 1 - main.tf | 6 ------ variables.tf | 6 ------ workflow_trigger.tftest.hcl | 1 - 4 files changed, 14 deletions(-) diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 437b6ac..fcc16ea 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -13,7 +13,6 @@ module "eks_deployment" { region = "us-east-1" template_repo_org = "my-org" - github_token = var.github_token github_server_url = "https://github.e.it.census.gov" cluster_config = { diff --git a/main.tf b/main.tf index 54593d6..a3f9395 100644 --- a/main.tf +++ b/main.tf @@ -28,12 +28,6 @@ resource "null_resource" "trigger_workflow" { provisioner "local-exec" { command = "python3 scripts/trigger_workflow.py ${module.github_repo.full_name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" - - # environment = { - # GITHUB_TOKEN = var.github_token - # GITHUB_OWNER = var.organization - # GITHUB_SERVER_URL = var.github_server_url - # } } depends_on = [module.github_repo] diff --git a/variables.tf b/variables.tf index 0c78add..23992cc 100644 --- a/variables.tf +++ b/variables.tf @@ -202,12 +202,6 @@ variable "github_actions_workflows" { default = [] } -variable "github_token" { - description = "GitHub token for triggering workflows" - type = string - sensitive = true -} - variable "github_server_url" { description = "GitHub Enterprise server URL (e.g., https://github.e.it.census.gov)" type = string diff --git a/workflow_trigger.tftest.hcl b/workflow_trigger.tftest.hcl index 11ffc81..ec05681 100644 --- a/workflow_trigger.tftest.hcl +++ b/workflow_trigger.tftest.hcl @@ -23,7 +23,6 @@ variables { } provider "github" { - token = var.github_token base_url = var.github_server_url } From d5dc8cbc0c9b4d5a978988c05cdbf72553244721 Mon Sep 17 00:00:00 2001 From: David John Arnold Jr Date: Mon, 14 Apr 2025 12:23:37 -0700 Subject: [PATCH 13/33] Update terraform-validate.yaml --- .github/workflows/terraform-validate.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/terraform-validate.yaml b/.github/workflows/terraform-validate.yaml index ac02202..a3265fa 100644 --- a/.github/workflows/terraform-validate.yaml +++ b/.github/workflows/terraform-validate.yaml @@ -17,10 +17,20 @@ jobs: uses: CSVD/gh-actions-setup-terraform@v2 with: terraform_version: '1.7.3' - + + - name: Setup GITHUB Credentials + id: github_credentials + uses: CSVD/gh-auth@main + with: + github_app_pem_file: ${{ secrets.GH_APP_PEM_FILE }} + github_app_installation_id: ${{ vars.GH_APP_INSTALLATION_ID }} + github_app_id: ${{ vars.GH_APP_ID }} + - name: Validate Terraform Configuration id: validate uses: CSVD/terraform-validate@main + env: + GITHUB_TOKEN: ${{ steps.github_credentials.outputs.github_token }} - name: Check Validation/Test Results if: always() From 0b2d76c72907c08057ef58e92c87fd1df40badfc Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 15:30:58 -0400 Subject: [PATCH 14/33] Remove mock GitHub token from workflow trigger configuration --- workflow_trigger.tftest.hcl | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow_trigger.tftest.hcl b/workflow_trigger.tftest.hcl index ec05681..1812c1f 100644 --- a/workflow_trigger.tftest.hcl +++ b/workflow_trigger.tftest.hcl @@ -1,5 +1,4 @@ variables { - github_token = "mock-token" name = "test-eks-cluster" organization = "test-org" template_repo_org = "test-org" From 2f2ffb00b4b3b285cd154576bf01dee4542e8b8e Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 15:34:49 -0400 Subject: [PATCH 15/33] updating --- .gitignore | 1 + basic.tftest.hcl | 45 -------------- .../default/modules/github_repo | 2 +- .../default/modules/modules.json | 2 +- validation.tftest.hcl.tmp | 62 ------------------- workflow_trigger.tftest.hcl | 40 ------------ workflow_trigger.tftest.hcl.tmp | 40 ------------ 7 files changed, 3 insertions(+), 189 deletions(-) delete mode 100644 basic.tftest.hcl delete mode 100644 validation.tftest.hcl.tmp delete mode 100644 workflow_trigger.tftest.hcl delete mode 100644 workflow_trigger.tftest.hcl.tmp diff --git a/.gitignore b/.gitignore index 6862aed..e9b6656 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ terraform.rc # Ignore test temporary files *.tftest.hcl.tmp +terraform_data_dirs diff --git a/basic.tftest.hcl b/basic.tftest.hcl deleted file mode 100644 index 1036299..0000000 --- a/basic.tftest.hcl +++ /dev/null @@ -1,45 +0,0 @@ -variables { - name = "test-eks-cluster" - organization = "SCT-Engineering" - template_repo_org = "SCT-Engineering" - environment = "dev" - region = "us-gov-east-1" - github_server_url = "https://github.e.it.census.gov" - cluster_config = { - cluster_name = "dev-eks-01" - account_name = "dev-account" - aws_account_id = "123456789012" - aws_profile = "dev-profile" - environment_abbr = "dev" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" - } - enable_modules = { - cert_manager = true - prometheus = true - grafana = true - } -} - -provider "github" { - base_url = var.github_server_url -} - -run "verify_repository_config" { - command = apply - - assert { - condition = module.github_repo.github_repo.name == var.name - error_message = "Repository name does not match expected value" - } - - assert { - condition = contains(module.github_repo.github_repo.topics, "eks") - error_message = "Repository topics should include 'eks'" - } - - assert { - condition = module.github_repo.github_repo.visibility == "private" - error_message = "Repository visibility should be private" - } -} \ No newline at end of file diff --git a/terraform_data_dirs/default/modules/github_repo b/terraform_data_dirs/default/modules/github_repo index b6b6cba..6140ffe 160000 --- a/terraform_data_dirs/default/modules/github_repo +++ b/terraform_data_dirs/default/modules/github_repo @@ -1 +1 @@ -Subproject commit b6b6cba8c08f2997b7a4058c421d41744ef7793d +Subproject commit 6140ffe240ed0a9b1ff42940238a9ea7348db255 diff --git a/terraform_data_dirs/default/modules/modules.json b/terraform_data_dirs/default/modules/modules.json index a03d773..89ced5d 100644 --- a/terraform_data_dirs/default/modules/modules.json +++ b/terraform_data_dirs/default/modules/modules.json @@ -1 +1 @@ -{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.83","Dir":"/data/terraform/workspaces/arnol377/git/terraform-eks-deployment/terraform_data_dirs/default/modules/github_repo"}]} \ No newline at end of file +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.84","Dir":"/data/terraform/workspaces/arnol377/git/terraform-eks-deployment/terraform_data_dirs/default/modules/github_repo"}]} \ No newline at end of file diff --git a/validation.tftest.hcl.tmp b/validation.tftest.hcl.tmp deleted file mode 100644 index e609b16..0000000 --- a/validation.tftest.hcl.tmp +++ /dev/null @@ -1,62 +0,0 @@ -variables { - name = "test-eks-cluster" - organization = "test-org" - template_repo_org = "test-org" - environment = "dev" - region = "us-east-1" - github_server_url = "https://github.e.it.census.gov" - cluster_config = { - cluster_name = "dev-eks-01" - account_name = "dev-account" - aws_account_id = "123456789012" - aws_profile = "dev-profile" - environment_abbr = "dev" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" - } -} - -provider "github" { - base_url = var.github_server_url -} - -# Test invalid cluster version -run "invalid_cluster_version" { - command = plan - - variables { - versions = { - cluster_version = "1.26" # Test outdated version - } - } - - expect_failures = [ - var.versions - ] -} - -# Test missing required variables -run "missing_required_vars" { - command = plan - - variables { - name = null # Required variable - } - - expect_failures = [ - var.name - ] -} - -# Test invalid environment name -run "invalid_environment" { - command = plan - - variables { - environment = "invalid" # Should be dev, staging, or prod - } - - expect_failures = [ - var.environment - ] -} \ No newline at end of file diff --git a/workflow_trigger.tftest.hcl b/workflow_trigger.tftest.hcl deleted file mode 100644 index 1812c1f..0000000 --- a/workflow_trigger.tftest.hcl +++ /dev/null @@ -1,40 +0,0 @@ -variables { - name = "test-eks-cluster" - organization = "test-org" - template_repo_org = "test-org" - environment = "dev" - region = "us-east-1" - github_server_url = "https://github.e.it.census.gov" - cluster_config = { - cluster_name = "dev-eks-01" - account_name = "dev-account" - aws_account_id = "123456789012" - aws_profile = "dev-profile" - environment_abbr = "dev" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" - } - enable_modules = { - cert_manager = true - prometheus = true - grafana = true - } -} - -provider "github" { - base_url = var.github_server_url -} - -run "verify_workflow_trigger" { - command = apply - - assert { - condition = can(null_resource.trigger_workflow) - error_message = "Workflow trigger resource should exist" - } - - assert { - condition = null_resource.trigger_workflow.triggers.github_repo_name == module.github_repo.full_name - error_message = "Workflow trigger should use correct repository name" - } -} \ No newline at end of file diff --git a/workflow_trigger.tftest.hcl.tmp b/workflow_trigger.tftest.hcl.tmp deleted file mode 100644 index 1812c1f..0000000 --- a/workflow_trigger.tftest.hcl.tmp +++ /dev/null @@ -1,40 +0,0 @@ -variables { - name = "test-eks-cluster" - organization = "test-org" - template_repo_org = "test-org" - environment = "dev" - region = "us-east-1" - github_server_url = "https://github.e.it.census.gov" - cluster_config = { - cluster_name = "dev-eks-01" - account_name = "dev-account" - aws_account_id = "123456789012" - aws_profile = "dev-profile" - environment_abbr = "dev" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" - } - enable_modules = { - cert_manager = true - prometheus = true - grafana = true - } -} - -provider "github" { - base_url = var.github_server_url -} - -run "verify_workflow_trigger" { - command = apply - - assert { - condition = can(null_resource.trigger_workflow) - error_message = "Workflow trigger resource should exist" - } - - assert { - condition = null_resource.trigger_workflow.triggers.github_repo_name == module.github_repo.full_name - error_message = "Workflow trigger should use correct repository name" - } -} \ No newline at end of file From 0ed813e889dd038ac16203b2e16e0b594beaedca Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 16:07:49 -0400 Subject: [PATCH 16/33] removing varfiles --- varfiles/default.json | 1 - varfiles/default.tfvars | 0 2 files changed, 1 deletion(-) delete mode 100644 varfiles/default.json delete mode 100644 varfiles/default.tfvars diff --git a/varfiles/default.json b/varfiles/default.json deleted file mode 100644 index 9e26dfe..0000000 --- a/varfiles/default.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/varfiles/default.tfvars b/varfiles/default.tfvars deleted file mode 100644 index e69de29..0000000 From fd83dd1594104eb2fb1f220e10e1b07c0f7cef2a Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 16:09:41 -0400 Subject: [PATCH 17/33] Remove unused GitHub module and provider references --- terraform_data_dirs/default/modules/github_repo | 1 - terraform_data_dirs/default/modules/modules.json | 1 - .../registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 | 1 - .../registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 | 1 - .../registry.terraform.io/integrations/github/6.6.0/linux_amd64 | 1 - 5 files changed, 5 deletions(-) delete mode 160000 terraform_data_dirs/default/modules/github_repo delete mode 100644 terraform_data_dirs/default/modules/modules.json delete mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 delete mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 delete mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 diff --git a/terraform_data_dirs/default/modules/github_repo b/terraform_data_dirs/default/modules/github_repo deleted file mode 160000 index 6140ffe..0000000 --- a/terraform_data_dirs/default/modules/github_repo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6140ffe240ed0a9b1ff42940238a9ea7348db255 diff --git a/terraform_data_dirs/default/modules/modules.json b/terraform_data_dirs/default/modules/modules.json deleted file mode 100644 index 89ced5d..0000000 --- a/terraform_data_dirs/default/modules/modules.json +++ /dev/null @@ -1 +0,0 @@ -{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.84","Dir":"/data/terraform/workspaces/arnol377/git/terraform-eks-deployment/terraform_data_dirs/default/modules/github_repo"}]} \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 deleted file mode 120000 index fe28aef..0000000 --- a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 deleted file mode 120000 index 08157aa..0000000 --- a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 deleted file mode 120000 index 26dfde5..0000000 --- a/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/integrations/github/6.6.0/linux_amd64 \ No newline at end of file From fab67931464260ee3478e6cdee7cc5195c7e1af2 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 16:11:18 -0400 Subject: [PATCH 18/33] removign data dirs --- terraform_data_dirs/default/modules/github_repo | 1 - terraform_data_dirs/default/modules/modules.json | 1 - .../registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 | 1 - .../registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 | 1 - .../registry.terraform.io/integrations/github/6.6.0/linux_amd64 | 1 - 5 files changed, 5 deletions(-) delete mode 160000 terraform_data_dirs/default/modules/github_repo delete mode 100644 terraform_data_dirs/default/modules/modules.json delete mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 delete mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 delete mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 diff --git a/terraform_data_dirs/default/modules/github_repo b/terraform_data_dirs/default/modules/github_repo deleted file mode 160000 index 6140ffe..0000000 --- a/terraform_data_dirs/default/modules/github_repo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6140ffe240ed0a9b1ff42940238a9ea7348db255 diff --git a/terraform_data_dirs/default/modules/modules.json b/terraform_data_dirs/default/modules/modules.json deleted file mode 100644 index 89ced5d..0000000 --- a/terraform_data_dirs/default/modules/modules.json +++ /dev/null @@ -1 +0,0 @@ -{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.84","Dir":"/data/terraform/workspaces/arnol377/git/terraform-eks-deployment/terraform_data_dirs/default/modules/github_repo"}]} \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 deleted file mode 120000 index fe28aef..0000000 --- a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 deleted file mode 120000 index 08157aa..0000000 --- a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 deleted file mode 120000 index 26dfde5..0000000 --- a/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/integrations/github/6.6.0/linux_amd64 \ No newline at end of file From 146348081dbd9e219a993b2bae44e88355d8c935 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 16:12:13 -0400 Subject: [PATCH 19/33] removing TEMPLATE_SHA --- .TEMPLATE_SHA | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .TEMPLATE_SHA diff --git a/.TEMPLATE_SHA b/.TEMPLATE_SHA deleted file mode 100644 index e69de29..0000000 From bb7367a76fd975f440657e01884c15eaac85e2b4 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 16:17:08 -0400 Subject: [PATCH 20/33] reemoving varfiles --- varfiles/default.json | 1 - varfiles/default.tfvars | 0 2 files changed, 1 deletion(-) delete mode 100644 varfiles/default.json delete mode 100644 varfiles/default.tfvars diff --git a/varfiles/default.json b/varfiles/default.json deleted file mode 100644 index 9e26dfe..0000000 --- a/varfiles/default.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/varfiles/default.tfvars b/varfiles/default.tfvars deleted file mode 100644 index e69de29..0000000 From 4f26e927637d8841fa14ff379051f249cec241a7 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 27 Aug 2025 16:43:21 -0400 Subject: [PATCH 21/33] initi --- examples/basic/main.tf | 14 +-- locals.tf | 166 ++++++++++++++--------------- main.tf | 91 +++++++++++----- templates/account.hcl.tf.tpl | 11 ++ templates/cluster.hcl.tf.tpl | 15 +++ templates/region.hcl.tf.tpl | 7 ++ templates/root.hcl.tf.tpl | 171 ++++++++++++++++++++++++++++++ templates/vpc.hcl.tf.tpl | 8 ++ variables.tf | 196 ++++++++++++++++++++++------------- 9 files changed, 490 insertions(+), 189 deletions(-) create mode 100644 templates/account.hcl.tf.tpl create mode 100644 templates/cluster.hcl.tf.tpl create mode 100644 templates/region.hcl.tf.tpl create mode 100644 templates/root.hcl.tf.tpl create mode 100644 templates/vpc.hcl.tf.tpl diff --git a/examples/basic/main.tf b/examples/basic/main.tf index fcc16ea..860a3c3 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -16,13 +16,13 @@ module "eks_deployment" { github_server_url = "https://github.e.it.census.gov" cluster_config = { - cluster_name = "dev-eks-01" - account_name = "dev-account" - aws_account_id = "123456789012" - aws_profile = "dev-profile" + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" environment_abbr = "dev" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" } enable_modules = { @@ -32,7 +32,7 @@ module "eks_deployment" { } versions = { - cluster_version = "1.27" + cluster_version = "1.27" eks_module_version = "20.33.1" } } \ No newline at end of file diff --git a/locals.tf b/locals.tf index bc33459..aaa3ee4 100644 --- a/locals.tf +++ b/locals.tf @@ -1,166 +1,166 @@ locals { common_vars = merge({ - organization = "census:ocio:csvd" - project_name = "csvd_platformbaseline" - project_number = "fs0000000078" - project_role = "csvd_platformbaseline_app" + organization = "census:ocio:csvd" + project_name = "csvd_platformbaseline" + project_number = "fs0000000078" + project_role = "csvd_platformbaseline_app" state_bucket_prefix = "inf-tfstate" - state_table_name = "tf_remote_state" - route53_endpoints = {} + state_table_name = "tf_remote_state" + route53_endpoints = {} }, var.common_variables) # First define base namespaces without dependencies base_namespaces = { - cert-manager = "kube-system" - karpenter = "karpenter" + cert-manager = "kube-system" + karpenter = "karpenter" metrics-server = "kube-system" - postgresql = "kube-system" - keycloak = "keycloak" - gogatekeeper = "kube-system" - istio = "istio-system" - kiali = "istio-system" + postgresql = "kube-system" + keycloak = "keycloak" + gogatekeeper = "kube-system" + istio = "istio-system" + kiali = "istio-system" } # Then merge with telemetry namespaces all_namespaces = merge( local.base_namespaces, { - grafana = var.namespaces.telemetry_namespace + grafana = var.namespaces.telemetry_namespace k8s-dashboard = var.namespaces.telemetry_namespace - loki = var.namespaces.telemetry_namespace - otel = var.namespaces.telemetry_namespace - prometheus = var.namespaces.telemetry_namespace - tempo = var.namespaces.telemetry_namespace + loki = var.namespaces.telemetry_namespace + otel = var.namespaces.telemetry_namespace + prometheus = var.namespaces.telemetry_namespace + tempo = var.namespaces.telemetry_namespace }, var.namespaces.custom_namespaces ) namespaces = { - operator_namespace = var.namespaces.operator_namespace + operator_namespace = var.namespaces.operator_namespace telemetry_namespace = var.namespaces.telemetry_namespace - namespaces = local.all_namespaces + namespaces = local.all_namespaces } default_versions = { - cluster_version = var.versions.cluster_version + cluster_version = var.versions.cluster_version custom_service_eks_account = var.versions.release_version - eks_module_version = var.versions.eks_module_version - istio_ingress_version = var.versions.release_version - release_version = var.versions.release_version + eks_module_version = var.versions.eks_module_version + istio_ingress_version = var.versions.release_version + release_version = var.versions.release_version # Provider versions - aws_version = var.versions.aws_version - helm_version = var.versions.helm_version + aws_version = var.versions.aws_version + helm_version = var.versions.helm_version kubernetes_version = var.versions.kubernetes_version - null_version = var.versions.null_version - random_version = var.versions.random_version - template_version = var.versions.template_version - tf_version = var.versions.tf_version + null_version = var.versions.null_version + random_version = var.versions.random_version + template_version = var.versions.template_version + tf_version = var.versions.tf_version # Component versions - cert_manager_version = var.versions.cert_manager.version + cert_manager_version = var.versions.cert_manager.version cert_manager_helm_chart = var.versions.cert_manager.chart_version - cluster_issuer_name = var.versions.cert_manager.cluster_issuer_name + cluster_issuer_name = var.versions.cert_manager.cluster_issuer_name - gogatekeeper_tag = var.versions.gogatekeeper.tag + gogatekeeper_tag = var.versions.gogatekeeper.tag gogatekeeper_chart_version = var.versions.gogatekeeper.chart_version - grafana_hostname = var.versions.grafana.hostname + grafana_hostname = var.versions.grafana.hostname grafana_operator_chart_version = var.versions.grafana.operator_chart_version - grafana_operator_tag = var.versions.grafana.operator_tag - grafana_tag = var.versions.grafana.tag - os_shell_image_tag = var.versions.grafana.os_shell_image_tag + grafana_operator_tag = var.versions.grafana.operator_tag + grafana_tag = var.versions.grafana.tag + os_shell_image_tag = var.versions.grafana.os_shell_image_tag - istio_version = var.versions.istio.version + istio_version = var.versions.istio.version istio_namespace = var.versions.istio.namespace - dashboard_hostname = var.versions.k8s_dashboard.hostname + dashboard_hostname = var.versions.k8s_dashboard.hostname k8s_dashboard_metrics_scraper = var.versions.k8s_dashboard.metrics_scraper - k8s_dashboard_version = var.versions.k8s_dashboard.version + k8s_dashboard_version = var.versions.k8s_dashboard.version karpenter_helm_chart = var.versions.karpenter.helm_chart - karpenter_tag = var.versions.karpenter.tag + karpenter_tag = var.versions.karpenter.tag keycloak_chart_version = var.versions.keycloak.chart_version - keycloak_tag = var.versions.keycloak.tag - keycloak_hostname = var.versions.keycloak.hostname - keycloak_database = var.versions.keycloak.database - keycloak_username = var.versions.keycloak.username - keycloak_password = var.versions.keycloak.password - postgresql_tag = var.versions.keycloak.postgresql_tag - - kiali_operator_version = var.versions.kiali.operator_version + keycloak_tag = var.versions.keycloak.tag + keycloak_hostname = var.versions.keycloak.hostname + keycloak_database = var.versions.keycloak.database + keycloak_username = var.versions.keycloak.username + keycloak_password = var.versions.keycloak.password + postgresql_tag = var.versions.keycloak.postgresql_tag + + kiali_operator_version = var.versions.kiali.operator_version kiali_application_version = "v${var.versions.kiali.operator_version}" - loki_chart_version = var.versions.loki.chart_version - loki_tag = var.versions.loki.tag + loki_chart_version = var.versions.loki.chart_version + loki_tag = var.versions.loki.tag enterprise_logs_provisioner_tag = var.versions.loki.enterprise_logs_provisioner_tag - gateway_tag = var.versions.loki.gateway_tag - memcached_tag = var.versions.loki.memcached_tag - exporter_tag = var.versions.loki.exporter_tag - sidecar_tag = var.versions.loki.sidecar_tag + gateway_tag = var.versions.loki.gateway_tag + memcached_tag = var.versions.loki.memcached_tag + exporter_tag = var.versions.loki.exporter_tag + sidecar_tag = var.versions.loki.sidecar_tag metrics_server_helm_chart = var.versions.metrics_server.helm_chart - metrics_server_tag = var.versions.metrics_server.tag + metrics_server_tag = var.versions.metrics_server.tag - prometheus_chart_version = var.versions.prometheus.chart_version - prometheus_server_tag = var.versions.prometheus.server_tag + prometheus_chart_version = var.versions.prometheus.chart_version + prometheus_server_tag = var.versions.prometheus.server_tag prometheus_config_reloader_tag = var.versions.prometheus.config_reloader_tag - alertmanager_tag = var.versions.prometheus.alertmanager_tag - kube_state_metrics_tag = var.versions.prometheus.kube_state_metrics_tag - node_exporter_tag = var.versions.prometheus.node_exporter_tag - pushgateway_tag = var.versions.prometheus.pushgateway_tag + alertmanager_tag = var.versions.prometheus.alertmanager_tag + kube_state_metrics_tag = var.versions.prometheus.kube_state_metrics_tag + node_exporter_tag = var.versions.prometheus.node_exporter_tag + pushgateway_tag = var.versions.prometheus.pushgateway_tag tempo_chart_version = var.versions.tempo.chart_version - tempo_tag = var.versions.tempo.tag + tempo_tag = var.versions.tempo.tag # Add namespace configurations - operator_namespace = var.namespaces.operator_namespace + operator_namespace = var.namespaces.operator_namespace telemetry_namespace = var.namespaces.telemetry_namespace - namespaces = local.all_namespaces + namespaces = local.all_namespaces } config_json = jsonencode({ - environment = var.environment - region = var.region - cluster_dir = "platform-cluster" + environment = var.environment + region = var.region + cluster_dir = "platform-cluster" enable_all_modules = var.enable_all_modules account = { - account_name = var.cluster_config.account_name - aws_account_id = var.cluster_config.aws_account_id - aws_profile = var.cluster_config.aws_profile + account_name = var.cluster_config.account_name + aws_account_id = var.cluster_config.aws_account_id + aws_profile = var.cluster_config.aws_profile environment_abbr = var.cluster_config.environment_abbr } vpc = { - vpc_name = var.cluster_config.vpc_name + vpc_name = var.cluster_config.vpc_name vpc_domain_name = var.cluster_config.vpc_domain_name } cluster = { - cluster_name = var.cluster_config.cluster_name - cluster_mailing_list = var.cluster_config.cluster_mailing_list - eks_instance_disk_size = var.cluster_config.eks_instance_disk_size - eks_ng_desired_size = var.cluster_config.eks_ng_desired_size - eks_ng_max_size = var.cluster_config.eks_ng_max_size - eks_ng_min_size = var.cluster_config.eks_ng_min_size + cluster_name = var.cluster_config.cluster_name + cluster_mailing_list = var.cluster_config.cluster_mailing_list + eks_instance_disk_size = var.cluster_config.eks_instance_disk_size + eks_ng_desired_size = var.cluster_config.eks_ng_desired_size + eks_ng_max_size = var.cluster_config.eks_ng_max_size + eks_ng_min_size = var.cluster_config.eks_ng_min_size enable_cluster_creator_admin_permissions = var.cluster_config.enable_cluster_creator_admin_permissions - tags = var.cluster_config.tags + tags = var.cluster_config.tags } modules = var.enable_modules }) managed_extra_files = concat([ { - path = "config.json" + path = "config.json" content = local.config_json }, { - path = "_envcommon/default-versions.hcl" + path = "_envcommon/default-versions.hcl" content = templatefile("${path.module}/templates/default-versions.hcl", local.default_versions) }, { - path = "_envcommon/common-variables.hcl" + path = "_envcommon/common-variables.hcl" content = templatefile("${path.module}/templates/common-variables.hcl", local.common_vars) } - ], + ], var.github_actions_workflows) } \ No newline at end of file diff --git a/main.tf b/main.tf index a3f9395..7c416a6 100644 --- a/main.tf +++ b/main.tf @@ -1,44 +1,79 @@ +locals { + # Render the HCL files from templates + rendered_files = { + "root.hcl" : templatefile("${path.module}/templates/root.hcl.tf.tpl", { + environment = var.environment + }), + "${var.environment}/account.hcl" : templatefile("${path.module}/templates/account.hcl.tf.tpl", { + account_name = var.account_config.account_name, + aws_account_id = var.account_config.aws_account_id, + environment = var.environment, + environment_abbr = var.account_config.environment_abbr + }), + "${var.environment}/${var.region}/region.hcl" : templatefile("${path.module}/templates/region.hcl.tf.tpl", { + environment = var.environment, + aws_region = var.region + }), + "${var.environment}/${var.region}/vpc/vpc.hcl" : templatefile("${path.module}/templates/vpc.hcl.tf.tpl", { + environment = var.environment, + aws_region = var.region, + vpc_name = var.vpc_config.vpc_name, + vpc_domain_name = var.vpc_config.vpc_domain_name + }), + "${var.environment}/${var.region}/vpc/${var.cluster_config.cluster_name}/cluster.hcl" : templatefile("${path.module}/templates/cluster.hcl.tf.tpl", { + cluster_name = var.cluster_config.cluster_name, + cluster_mailing_list = var.cluster_config.cluster_mailing_list, + eks_instance_disk_size = var.cluster_config.eks_instance_disk_size, + eks_ng_desired_size = var.cluster_config.eks_ng_desired_size, + eks_ng_max_size = var.cluster_config.eks_ng_max_size, + eks_ng_min_size = var.cluster_config.eks_ng_min_size, + organization = var.cluster_config.organization, + finops_project_name = var.cluster_config.finops_project_name, + finops_project_number = var.cluster_config.finops_project_number, + finops_project_role = var.cluster_config.finops_project_role, + tags = var.cluster_config.tags, + module_enablement_overrides = var.cluster_config.module_enablement_overrides + }) + } +} + module "github_repo" { - source = "HappyPathway/repo/github" + source = "github.com/HappyPathway/terraform-github-repo" - name = var.name - repo_org = var.organization + name = var.repository_name + repo_org = var.organization github_repo_description = "EKS Cluster Configuration for ${var.cluster_config.cluster_name}" - github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] - - template_repo = "template-eks-cluster" - template_repo_org = var.template_repo_org - - github_is_private = true - github_has_issues = true - github_has_wiki = true - github_has_projects = true - - managed_extra_files = concat( - local.managed_extra_files, - var.managed_extra_files - ) - extra_files = var.extra_files -} + github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] -resource "null_resource" "trigger_workflow" { - triggers = { - github_repo_name = module.github_repo.full_name + template = { + owner = var.repository_template_owner + repository = var.repository_template } - provisioner "local-exec" { - command = "python3 scripts/trigger_workflow.py ${module.github_repo.full_name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" - } + github_is_private = true + github_has_issues = true + github_has_wiki = true + github_has_projects = true - depends_on = [module.github_repo] + files = [ + for path, content in local.rendered_files : { + path = path + content = content + } + ] + + teams = var.repository_teams } +# The EKS deployment logic will go here, and will be skipped if create_repository is true. +# For now, we are just implementing the repository creation part. + output "repository_url" { description = "URL of the created repository" - value = module.github_repo.html_url + value = module.github_repo.html_url } output "ssh_clone_url" { description = "SSH clone URL of the repository" - value = module.github_repo.ssh_clone_url + value = module.github_repo.ssh_clone_url } \ No newline at end of file diff --git a/templates/account.hcl.tf.tpl b/templates/account.hcl.tf.tpl new file mode 100644 index 0000000..83d3336 --- /dev/null +++ b/templates/account.hcl.tf.tpl @@ -0,0 +1,11 @@ +# ${environment}/environment.hcl + +# Set account-wide variables. These are automatically pulled in to configure the remote state bucket in the root +# terragrunt.hcl configuration. +locals { + account_name = "${account_name}" + aws_account_id = "${aws_account_id}" + aws_profile = format("%v-%v", local.aws_account_id, replace(local.account_name, "-ew", "-gov")) + environment = "${environment}" + environment_abbr = "${environment_abbr}" +} diff --git a/templates/cluster.hcl.tf.tpl b/templates/cluster.hcl.tf.tpl new file mode 100644 index 0000000..0a30d6c --- /dev/null +++ b/templates/cluster.hcl.tf.tpl @@ -0,0 +1,15 @@ +locals { + # Cluster specific configuration + cluster_name = "${cluster_name}" + cluster_mailing_list = "${cluster_mailing_list}" + eks_instance_disk_size = ${eks_instance_disk_size} + eks_ng_desired_size = ${eks_ng_desired_size} + eks_ng_max_size = ${eks_ng_max_size} + eks_ng_min_size = ${eks_ng_min_size} + organization = "${organization}" + finops_project_name = "${finops_project_name}" + finops_project_number = "${finops_project_number}" + finops_project_role = "${finops_project_role}" + tags = ${jsonencode(tags)} + module_enablement_overrides = ${jsonencode(module_enablement_overrides)} +} diff --git a/templates/region.hcl.tf.tpl b/templates/region.hcl.tf.tpl new file mode 100644 index 0000000..dc6c306 --- /dev/null +++ b/templates/region.hcl.tf.tpl @@ -0,0 +1,7 @@ +# ${environment}/${aws_region}/region.hcl + +# Set common variables for the region. This is automatically pulled in in the root terragrunt.hcl configuration to +# configure the remote state bucket and pass forward to the child modules as inputs. +locals { + aws_region = "${aws_region}" +} diff --git a/templates/root.hcl.tf.tpl b/templates/root.hcl.tf.tpl new file mode 100644 index 0000000..2946d47 --- /dev/null +++ b/templates/root.hcl.tf.tpl @@ -0,0 +1,171 @@ +# $${environment}/root.hcl +# --------------------------------------------------------------------------------------------------------------------- +# TERRAGRUNT CONFIGURATION +# Terragrunt is a thin wrapper for Terraform that provides extra tools for working with multiple Terraform modules, +# remote state, and locking: https://github.com/gruntwork-io/terragrunt +# --------------------------------------------------------------------------------------------------------------------- +locals { + # Automatically load account-level variables (NOTE: In our environment account = environment so there is not separate environment layer) + account_vars = read_terragrunt_config(find_in_parent_folders("account.hcl")) + + # Automatically load cluster-level variables + cluster_vars = read_terragrunt_config(find_in_parent_folders("cluster.hcl")) + + # Automatically load _envcommon, cross account and environment common variables + common_vars = read_terragrunt_config(find_in_parent_folders("./_envcommon/common-variables.hcl")) + + # Automatically load naming prefixes + prefix_vars = read_terragrunt_config(find_in_parent_folders("./_envcommon/prefixes.hcl")) + + # Automatically load region-level variables + region_vars = read_terragrunt_config(find_in_parent_folders("region.hcl")) + + # Automatically load versions + versions = read_terragrunt_config(find_in_parent_folders("./_envcommon/default-versions.hcl")) + + # Automatically load vpc-level variables + vpc_vars = read_terragrunt_config(find_in_parent_folders("vpc.hcl")) + + # Add any other locals you want to expose + # only expose things not already included via local.xxx_vars.locals.* + root_locals_for_inputs = { + is_module_enabled = local.is_module_enabled + module_name = local.module_name + } + + # Extract the variables we need for easy access + account_id = local.account_vars.locals.aws_account_id + account_name = local.account_vars.locals.account_name + aws_profile = local.account_vars.locals.aws_profile + aws_region = local.region_vars.locals.aws_region + cluster_name = local.cluster_vars.locals.cluster_name + eecr_info = local.common_vars.locals.eecr_info + environment_abbr = local.account_vars.locals.environment_abbr + finops_project_name = local.cluster_vars.locals.finops_project_name + finops_project_number = local.cluster_vars.locals.finops_project_number + finops_project_role = local.cluster_vars.locals.finops_project_role + is_eks_module = local.module_name == "eks" + prefixes = local.prefix_vars.locals.prefixes + is_module_enabled = merge( + { for module in local.versions.locals.core_modules : module => true }, + local.versions.locals.enabled_modules, + local.module_overrides + ) + module_name = basename(get_original_terragrunt_dir()) + module_overrides = local.cluster_vars.locals.module_enablement_overrides + organization = local.cluster_vars.locals.organization + state_bucket_prefix = local.common_vars.locals.state_bucket_prefix + state_table_name = local.common_vars.locals.state_table_name +} + +# Only generate providers for non-EKS modules +generate "cluster_data" { + path = "cluster-data.tf" + if_exists = "overwrite_terragrunt" + contents = local.is_eks_module ? "" : <<-EOF + data "aws_eks_clusters" "available" {} + + locals { + cluster_exists = contains(data.aws_eks_clusters.available.names, "$${local.cluster_name}") + } + + data "aws_eks_cluster" "this" { + count = local.cluster_exists ? 1 : 0 + name = "$${local.cluster_name}" + } + + data "aws_eks_cluster_auth" "this" { + count = local.cluster_exists ? 1 : 0 + name = "$${local.cluster_name}" + } + EOF +} + +# Generate provider blocks only for non-EKS modules +generate "kube_provider" { + path = "kube-provider.tf" + if_exists = "overwrite_terragrunt" + contents = local.is_eks_module ? "" : <<-EOF + provider "kubernetes" { + host = local.cluster_exists ? data.aws_eks_cluster.this[0].endpoint : "https://dummy" + cluster_ca_certificate = local.cluster_exists ? base64decode(data.aws_eks_cluster.this[0].certificate_authority[0].data) : null + token = local.cluster_exists ? data.aws_eks_cluster_auth.this[0].token : "dummy" + } + EOF +} + +generate "helm_provider" { + path = "helm-provider.tf" + if_exists = "overwrite_terragrunt" + contents = local.is_eks_module ? "" : <<-EOF + provider "helm" { + kubernetes = { + host = local.cluster_exists ? data.aws_eks_cluster.this[0].endpoint : "https://dummy" + cluster_ca_certificate = local.cluster_exists ? base64decode(data.aws_eks_cluster.this[0].certificate_authority[0].data) : null + token = local.cluster_exists ? data.aws_eks_cluster_auth.this[0].token : "dummy" + } + } + EOF +} + +# Configure Terragrunt to automatically store tfstate files in an S3 bucket +remote_state { + disable_init = tobool(get_env("TG_DISABLE_INIT", "false")) + backend = "s3" + generate = { + path = "remote_state.backend.tf" + if_exists = "overwrite_terragrunt" + } + config = { + bucket = "$${local.state_bucket_prefix}-$${local.account_id}" + use_lockfile = true + key = "$${trimprefix(replace(run_cmd("realpath", get_original_terragrunt_dir()), dirname(get_repo_root()), ""), "/")}/terraform.tfstate" + profile = "$${local.aws_profile}" + region = "$${local.aws_region}" + disable_bucket_update = true + } +} + +# Generate an AWS provider block +generate "aws-provider" { + path = "aws-provider.tf" + if_exists = "overwrite" + contents = <<-EOF + provider "aws" { + region = "$${local.aws_region}" + profile = "$${local.aws_profile}" + default_tags { + tags = { + cluster_name = "$${local.cluster_name}" + "boc:module_name" = "$${local.module_name}" + environment = "$${local.environment_abbr}" + finops_project_name = "$${local.finops_project_name}" + finops_project_number = "$${local.finops_project_number}" + finops_project_role = "$${local.finops_project_role}" + organization = "$${local.organization}" + } + } + # Only these AWS Account IDs may be operated on by this template + allowed_account_ids = ["$${local.account_id}"] + } +EOF +} + +# --------------------------------------------------------------------------------------------------------------------- +# GLOBAL PARAMETERS +# These variables apply to all configurations in this subfolder. These are automatically merged into the child +# `terragrunt.hcl` config via the include block. +# --------------------------------------------------------------------------------------------------------------------- + +# Configure root level variables that all resources can inherit. This is especially helpful with multi-account configs +# where terraform_remote_state data sources are placed directly into the modules. +inputs = merge( + local.account_vars.locals, + local.cluster_vars.locals, + local.common_vars.locals, + local.prefix_vars.locals, + local.region_vars.locals, + local.versions.locals, + local.vpc_vars.locals, + local.root_locals_for_inputs +) diff --git a/templates/vpc.hcl.tf.tpl b/templates/vpc.hcl.tf.tpl new file mode 100644 index 0000000..9e95ad9 --- /dev/null +++ b/templates/vpc.hcl.tf.tpl @@ -0,0 +1,8 @@ +# ${environment}/${aws_region}/vpc/vpc.hcl + +# Set VPC specific variables. These are automatically pulled in to configure the remote state bucket in the root +# terragrunt.hcl configuration. +locals { + vpc_name = "${vpc_name}" + vpc_domain_name = "${vpc_domain_name}" +} diff --git a/variables.tf b/variables.tf index 23992cc..e43ad69 100644 --- a/variables.tf +++ b/variables.tf @@ -26,15 +26,15 @@ variable "region" { variable "common_variables" { description = "Common variables across all environments" type = object({ - organization = optional(string, "census:ocio:csvd") - project_name = optional(string, "csvd_platformbaseline") - project_number = optional(string, "fs0000000078") - project_role = optional(string, "csvd_platformbaseline_app") + organization = optional(string, "census:ocio:csvd") + project_name = optional(string, "csvd_platformbaseline") + project_number = optional(string, "fs0000000078") + project_role = optional(string, "csvd_platformbaseline_app") state_bucket_prefix = optional(string, "inf-tfstate") - state_table_name = optional(string, "tf_remote_state") + state_table_name = optional(string, "tf_remote_state") route53_endpoints = optional(map(object({ - account_id = string - alias = string + account_id = string + alias = string us-gov-east-1 = string us-gov-west-1 = string })), {}) @@ -46,62 +46,62 @@ variable "versions" { description = "Version configurations for various components" type = object({ # Module Versions - cluster_version = optional(string, "1.31") + cluster_version = optional(string, "1.31") eks_module_version = optional(string, "20.33.1") - release_version = optional(string, "main") + release_version = optional(string, "main") # TF Providers - aws_version = optional(string, "5.84.0") - helm_version = optional(string, "2.11.0") + aws_version = optional(string, "5.84.0") + helm_version = optional(string, "2.11.0") kubernetes_version = optional(string, "2.33.0") - null_version = optional(string, "3.2.1") - random_version = optional(string, "3.5.1") - template_version = optional(string, "2.2.0") - tf_version = optional(string, "1.5.5") + null_version = optional(string, "3.2.1") + random_version = optional(string, "3.5.1") + template_version = optional(string, "2.2.0") + tf_version = optional(string, "1.5.5") # Component Versions cert_manager = optional(object({ - version = optional(string, "1.17.1") - chart_version = optional(string, "1.17.1") + version = optional(string, "1.17.1") + chart_version = optional(string, "1.17.1") cluster_issuer_name = optional(string, "cert-manager") }), {}) gogatekeeper = optional(object({ - tag = optional(string, "3.2.1") + tag = optional(string, "3.2.1") chart_version = optional(string, "0.1.53") }), {}) grafana = optional(object({ - hostname = optional(string, "grafana") + hostname = optional(string, "grafana") operator_chart_version = optional(string, "4.9.8") - operator_tag = optional(string, "5.16.0") - tag = optional(string, "11.5.2") - os_shell_image_tag = optional(string, "12") + operator_tag = optional(string, "5.16.0") + tag = optional(string, "11.5.2") + os_shell_image_tag = optional(string, "12") }), {}) istio = optional(object({ - version = optional(string, "1.25.0") + version = optional(string, "1.25.0") namespace = optional(string, "istio-system") }), {}) k8s_dashboard = optional(object({ - hostname = optional(string, "dashboard") + hostname = optional(string, "dashboard") metrics_scraper = optional(string, "1.0.8") - version = optional(string, "6.0.6") + version = optional(string, "6.0.6") }), {}) karpenter = optional(object({ helm_chart = optional(string, "1.3.1") - tag = optional(string, "1.3.1") + tag = optional(string, "1.3.1") }), {}) keycloak = optional(object({ - chart_version = optional(string, "24.4.11") - tag = optional(string, "26.1.3") - hostname = optional(string, "keycloak") - database = optional(string, "keycloak") - username = optional(string, "keycloak") - password = optional(string, "this is my very secure and totally random password horse battery staple now") + chart_version = optional(string, "24.4.11") + tag = optional(string, "26.1.3") + hostname = optional(string, "keycloak") + database = optional(string, "keycloak") + username = optional(string, "keycloak") + password = optional(string, "this is my very secure and totally random password horse battery staple now") postgresql_tag = optional(string, "17.4.0-debian-12-r2") }), {}) @@ -110,33 +110,33 @@ variable "versions" { }), {}) loki = optional(object({ - chart_version = optional(string, "6.27.0") - tag = optional(string, "3.4.2") + chart_version = optional(string, "6.27.0") + tag = optional(string, "3.4.2") enterprise_logs_provisioner_tag = optional(string, "v1.7.0") - gateway_tag = optional(string, "1.27-alpine") - memcached_tag = optional(string, "1.6.37") - exporter_tag = optional(string, "v0.15.0") - sidecar_tag = optional(string, "1.27.4") + gateway_tag = optional(string, "1.27-alpine") + memcached_tag = optional(string, "1.6.37") + exporter_tag = optional(string, "v0.15.0") + sidecar_tag = optional(string, "1.27.4") }), {}) metrics_server = optional(object({ helm_chart = optional(string, "3.12.2") - tag = optional(string, "0.7.2") + tag = optional(string, "0.7.2") }), {}) prometheus = optional(object({ - chart_version = optional(string, "27.5.1") - server_tag = optional(string, "v3.2.1") - config_reloader_tag = optional(string, "v0.75.2") - alertmanager_tag = optional(string, "v0.28.0") + chart_version = optional(string, "27.5.1") + server_tag = optional(string, "v3.2.1") + config_reloader_tag = optional(string, "v0.75.2") + alertmanager_tag = optional(string, "v0.28.0") kube_state_metrics_tag = optional(string, "v2.15.0") - node_exporter_tag = optional(string, "v1.9.0") - pushgateway_tag = optional(string, "v1.11.0") + node_exporter_tag = optional(string, "v1.9.0") + pushgateway_tag = optional(string, "v1.11.0") }), {}) tempo = optional(object({ chart_version = optional(string, "1.18.2") - tag = optional(string, "2.7.1") + tag = optional(string, "2.7.1") }), {}) }) default = {} @@ -145,17 +145,17 @@ variable "versions" { variable "namespaces" { description = "Namespace configurations" type = object({ - operator_namespace = optional(string, "aoperator") + operator_namespace = optional(string, "aoperator") telemetry_namespace = optional(string, "atelemetry") custom_namespaces = optional(map(string), { - cert-manager = "kube-system" - karpenter = "karpenter" + cert-manager = "kube-system" + karpenter = "karpenter" metrics-server = "kube-system" - postgresql = "kube-system" - keycloak = "keycloak" - gogatekeeper = "kube-system" - istio = "istio-system" - kiali = "istio-system" + postgresql = "kube-system" + keycloak = "keycloak" + gogatekeeper = "kube-system" + istio = "istio-system" + kiali = "istio-system" }) }) default = {} @@ -164,20 +164,20 @@ variable "namespaces" { variable "cluster_config" { description = "Configuration for the EKS cluster" type = object({ - cluster_name = string - account_name = string - aws_account_id = string - aws_profile = string - environment_abbr = string - vpc_name = string - vpc_domain_name = string - cluster_mailing_list = optional(string) - eks_instance_disk_size = optional(number, 200) - eks_ng_desired_size = optional(number, 3) - eks_ng_max_size = optional(number, 10) - eks_ng_min_size = optional(number, 3) + cluster_name = string + account_name = string + aws_account_id = string + aws_profile = string + environment_abbr = string + vpc_name = string + vpc_domain_name = string + cluster_mailing_list = optional(string) + eks_instance_disk_size = optional(number, 200) + eks_ng_desired_size = optional(number, 3) + eks_ng_max_size = optional(number, 10) + eks_ng_min_size = optional(number, 3) enable_cluster_creator_admin_permissions = optional(bool, true) - tags = optional(map(string), {}) + tags = optional(map(string), {}) }) } @@ -186,9 +186,9 @@ variable "enable_modules" { type = object({ gogatekeeper = optional(bool, false) cert_manager = optional(bool, false) - prometheus = optional(bool, false) - grafana = optional(bool, false) - istio = optional(bool, false) + prometheus = optional(bool, false) + grafana = optional(bool, false) + istio = optional(bool, false) }) default = {} } @@ -196,7 +196,7 @@ variable "enable_modules" { variable "github_actions_workflows" { description = "List of GitHub Actions workflow files to add to the repository" type = list(object({ - path = string + path = string content = string })) default = [] @@ -230,4 +230,58 @@ variable "extra_files" { content = string })) default = [] +} + +variable "tags" { + description = "A map of tags to add to all resources." + type = map(string) + default = {} +} + +variable "create_repository" { + description = "If true, a GitHub repository will be created and configured." + type = bool + default = false +} + +variable "repository_name" { + description = "The name of the GitHub repository to create." + type = string + default = "" +} + +variable "repository_teams" { + description = "A map of teams and their permissions for the new repository." + type = map(string) + default = {} +} + +variable "repository_template" { + description = "The template repository to use when creating the new repository." + type = string + default = "template-eks-cluster" +} + +variable "repository_template_owner" { + description = "The owner of the template repository." + type = string + default = "HappyPathway" +} + +variable "cluster_config" { + description = "A map of configuration values for the cluster." + type = any + default = {} +} + +variable "account_config" { + description = "A map of configuration values for the account." + type = any + default = {} +} + +variable "vpc_config" { + description = "A map of configuration values for the vpc." + type = any + default = {} } \ No newline at end of file From 37737045b53a39ad719fd15d9efb2dfd38f3b3ab Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 27 Aug 2025 16:44:41 -0400 Subject: [PATCH 22/33] adding docs --- tf-native-v2.md | 83 +++++++++++++++++++++++++++++++++++++++++++++++++ tf-native-v3.md | 75 ++++++++++++++++++++++++++++++++++++++++++++ tf-native-v4.md | 79 ++++++++++++++++++++++++++++++++++++++++++++++ tf-native-v5.md | 72 ++++++++++++++++++++++++++++++++++++++++++ tf-native.md | 72 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 381 insertions(+) create mode 100644 tf-native-v2.md create mode 100644 tf-native-v3.md create mode 100644 tf-native-v4.md create mode 100644 tf-native-v5.md create mode 100644 tf-native.md diff --git a/tf-native-v2.md b/tf-native-v2.md new file mode 100644 index 0000000..499762a --- /dev/null +++ b/tf-native-v2.md @@ -0,0 +1,83 @@ +# Plan for Migrating to a Terraform-Native GitHub Repository Management Workflow (v2) + +This document provides a corrected and more accurate plan to replace the current Python Lambda-based workflow with a Terraform-native approach for creating and managing GitHub repositories. + +## 1. Corrected Analysis of the Current State + +The current system uses a Python-based AWS Lambda function (`template-automation-lambda`) to automate repository creation. It does **not** use Ansible for repository configuration as previously assumed. + +The workflow is as follows: +1. The Lambda function is invoked with details for a new repository (e.g., `project_name`, `template_settings`). +2. The Python script within the Lambda performs a series of GitHub API calls to: + a. Create a new repository. + b. Clone the contents of a base template repository into it. + c. Add a custom `config.json` file. + d. Assign team permissions. + e. Create a pull request to merge the initial setup into the `main` branch. + +This process, while effective, involves a custom-built Python application, a Lambda deployment, an ECR container registry, and associated IAM roles, making it complex to maintain. + +## 2. Proposed Solution: A Purely Terraform-Native Workflow + +We will replace the entire Lambda-based system with a declarative Terraform configuration that uses the `terraform-github-repo` module. This module natively supports the actions currently performed by the Python script. + +The new process will be: +1. A developer defines a new repository by adding a `module` block to a central Terraform configuration file. +2. Running `terraform apply` will instruct Terraform to perform all the necessary setup steps. + +### Mapping Lambda Actions to Terraform Resources + +| Action (Current Python Lambda) | Terraform Equivalent (`terraform-github-repo` module) | +| :--- | :--- | +| 1. Create a new repository. | The `github_repository` resource. The module uses this internally. | +| 2. Clone a template repository. | The `template` block within the `github_repository` resource. This is a native feature for creating a repo from a template. | +| 3. Write a `config.json` file. | The `github_repository_file` resource. The module accepts a `files` variable to manage this. | +| 4. Assign team permissions. | The `github_team_repository` resource. The module has a `teams` input for this. | +| 5. Create a pull request. | The `github_pull_request` resource. This can be defined outside the module to initialize the repository. | + +## 3. Detailed Migration Plan + +### Phase 1: Scoping and Proof of Concept + +1. **Identify All Template Variables:** Document all the key-value pairs that are currently passed into the `template_settings` of the Lambda. These will become variables in our new Terraform module. + +2. **Create a Wrapper Module:** Create a new, internal Terraform module that wraps the `terraform-github-repo` module. This wrapper will provide a simplified interface for our developers and contain the logic for creating the initial pull request. + +3. **Develop the PoC:** In a test environment, write a Terraform configuration that uses this new wrapper module to create a single, non-critical repository. The configuration should: + * Use the `template` feature to create the repository from the existing template repo. + * Use the `files` feature to add the `config.json`. + * Use the `teams` feature to grant permissions. + * Define a `github_pull_request` resource to create the initial PR. + +### Phase 2: Implementation and Import + +1. **Build Out the Full Configuration:** Create a new Git repository to house the Terraform configuration for all repositories that will be managed this way. + +2. **Import Existing Repositories:** For repositories previously created by the Lambda, use `terraform import` to bring them under Terraform's state management. This is a critical step to prevent any disruption. + * `terraform import module.my_repo.github_repository.this[0] my-repo-name` + * `terraform import module.my_repo.github_team_repository.teams["tf-module-admins"] my-repo-name:tf-module-admins` + +3. **Parallel Run:** For a transition period, both systems can exist. New repositories should be created using the Terraform method, while the Lambda is left in place to manage older ones if needed. + +### Phase 3: Testing and Validation + +1. **Dry Run with `terraform plan`:** Before applying any changes to a production repository, run `terraform plan` and carefully review the output to ensure it matches expectations and doesn't plan any destructive changes. + +2. **Full Application:** Once validated, apply the configuration to manage all target repositories. + +### Phase 4: Decommissioning + +Once all repositories are successfully managed by Terraform and the new workflow is stable, the old infrastructure can be safely removed. + +1. **Disable the Lambda Trigger:** The first step is to disable the mechanism that invokes the Lambda function. +2. **Delete the Lambda Function:** Remove the `template-automation-lambda` function from AWS. +3. **Delete the ECR Repository:** Delete the ECR repository holding the Lambda's container images. +4. **Delete the Deployment Pipeline:** Remove the `template-repos-lambda-deployment` Terraform configuration and state. +5. **Archive Old Repositories:** Archive the `template-automation-lambda` and `template-repos-lambda-deployment` Git repositories to mark them as deprecated. + +## 4. Rollback Plan + +If issues arise, we can revert by: +1. Removing the problematic repository from Terraform's state using `terraform state rm`. +2. Re-enabling or re-deploying the Lambda function to take over management again. +3. Manually correcting any unintended changes made by Terraform. diff --git a/tf-native-v3.md b/tf-native-v3.md new file mode 100644 index 0000000..b723b86 --- /dev/null +++ b/tf-native-v3.md @@ -0,0 +1,75 @@ +# Plan for Migrating to a Terraform-Native EKS Deployment Workflow (v4) + +This document outlines the plan to replace the current Lambda/Ansible-based system with a streamlined, Terraform-native workflow for creating and configuring repositories for EKS deployments. + +## 1. Analysis of the Current State + +The current process for provisioning a new EKS cluster repository involves multiple, loosely-coupled components: + +1. **`template-automation-lambda`**: A Python Lambda function that creates a new GitHub repository from the `template-eks-cluster` template. It clones the template, adds a `config.json` file with user-provided settings, and opens a pull request. +2. **`generate_hcl_files.yml`**: An Ansible playbook inside the newly created repository that is run manually after the initial PR is merged. It reads the `config.json` and generates a set of Terragrunt HCL files (`root.hcl`, `account.hcl`, `region.hcl`, etc.). +3. **`terraform-eks-deployment`**: A Terraform module that is referenced by the generated Terragrunt configuration to deploy the actual EKS cluster. + +This workflow is complex, involves manual steps, and relies on a mix of technologies (Python, Lambda, Ansible, Terraform). + +## 2. Proposed Solution: A Unified, Terraform-Native Workflow + +We will create a single, unified Terraform workflow that handles the entire process of repository creation and configuration declaratively. This eliminates the need for the Lambda function and the Ansible playbook. + +The new process will be: +1. A developer defines a new EKS cluster by adding a single `module` block to a central Terraform configuration. +2. Running `terraform apply` will automatically: + a. Create a new GitHub repository. + b. Generate and commit all the necessary Terragrunt HCL files and `README.md`. + c. Configure team permissions for the repository. + +### Core Component: The New `terragrunt-eks-repo` Wrapper Module + +The centerpiece of this new workflow is a new Terraform module, `terragrunt-eks-repo`. This module will be responsible for all the setup logic. + +| Action (Old Workflow) | Terraform Equivalent (New `terragrunt-eks-repo` Module) | +| :--- | :--- | +| 1. Create a new repository from a template. | The module will call the `terraform-github-repo` module internally, using its `template` feature to clone from `template-eks-cluster`. | +| 2. Generate HCL files from `config.json`. | The module will contain HCL templates (`.tf.tpl` files). It will use Terraform's `templatefile()` function to render the final HCL content directly from its input variables. | +| 3. Write files to the repository. | The rendered file content will be passed to the `files` input of the underlying `terraform-github-repo` module, which uses the `github_repository_file` resource to commit them. | +| 4. Assign team permissions. | The module will accept a `teams` variable and pass it to the `terraform-github-repo` module to configure permissions using the `github_team_repository` resource. | + +## 3. Detailed Migration Plan + +### Phase 1: Develop the `terragrunt-eks-repo` Module + +1. **Create Module Scaffolding:** Create a new directory for the `terragrunt-eks-repo` module. + +2. **Define Input Variables:** Create a `variables.tf` file. The variables will be derived directly from the `generate_hcl_files.yml` playbook's `config` object (e.g., `environment`, `region`, `cluster_name`, `account`, `vpc`, etc.). + +3. **Create HCL Templates:** Create a `templates` directory within the module. For each file generated by the Ansible playbook (`root.hcl`, `account.hcl`, `region.hcl`, `vpc.hcl`, `cluster.hcl`, and `README.md`), create a corresponding `.tf.tpl` template file. Convert the Jinja2 syntax to Terraform's `${...}` interpolation syntax. + +4. **Implement Module Logic (`main.tf`):** + * Use `locals` to render the file content for each template using the `templatefile()` function. + * Call the `terraform-github-repo` module. + * Pass the repository name, template configuration, and team permissions to the module. + * Map the rendered local variables to the `files` input of the `terraform-github-repo` module. This will instruct it to create the files in the new repository. + +### Phase 2: Implementation and Onboarding + +1. **Create a Central Management Repository:** Set up a new Git repository (e.g., `terragrunt-environments`) that will contain the Terraform configuration for creating all new EKS cluster repositories. + +2. **Onboard a Pilot Project:** In the new management repository, add a `main.tf` file. Add a module block that calls the newly created `terragrunt-eks-repo` module to provision a repository for a new test cluster. + +3. **Execute and Validate:** Run `terraform apply` to create the repository. Verify that: + * The repository is created on GitHub. + * It is correctly initialized from the `template-eks-cluster` template. + * All the Terragrunt HCL files and the `README.md` are present and correctly populated with the variable values. + * Team permissions are correctly assigned. + +### Phase 3: Decommissioning the Old Workflow + +Since we are not concerned with migrating existing repositories, the decommissioning process is straightforward. Once the new workflow is validated and adopted for all new cluster provisioning: + +1. **Disable the Lambda Function:** The Lambda trigger can be disabled in AWS. +2. **Archive Old Repositories:** The `template-automation-lambda` and `template-repos-lambda-deployment` Git repositories should be archived to prevent further use. +3. **Delete AWS Resources:** The old AWS resources (Lambda function, ECR repository, IAM roles) can be deleted via Terraform from the `template-repos-lambda-deployment` project. + +## 4. Rollback Plan + +As we are not migrating existing resources, a rollback is not applicable in the traditional sense. If the new workflow fails for a new repository, the state can be destroyed (`terraform destroy`), the module can be fixed, and the process can be re-run. The old Lambda-based system can be temporarily kept available for emergency use until the new workflow is fully proven. diff --git a/tf-native-v4.md b/tf-native-v4.md new file mode 100644 index 0000000..3d9245f --- /dev/null +++ b/tf-native-v4.md @@ -0,0 +1,79 @@ +# Plan for Migrating to a Terraform-Native EKS Deployment Workflow (v4) + +This document outlines the plan to replace the current Lambda/Ansible-based system with a streamlined, Terraform-native workflow by enhancing the `terraform-eks-deployment` module itself. + +## 1. Analysis of the Current State + +The current process for provisioning a new EKS cluster repository involves multiple components: + +1. **`template-automation-lambda`**: A Python Lambda function that creates a new GitHub repository from the `template-eks-cluster` template. +2. **`generate_hcl_files.yml`**: An Ansible playbook inside the new repository that is run manually to generate a set of Terragrunt HCL files (`root.hcl`, `account.hcl`, etc.). +3. **`terraform-eks-deployment`**: The Terraform module that is referenced by the generated Terragrunt configuration to deploy the actual EKS cluster. + +This workflow is complex, involves manual steps, and relies on a mix of technologies. + +## 2. Proposed Solution: A Unified, All-in-One EKS Deployment Module + +We will consolidate the entire workflow into the `terraform-eks-deployment` module. This module will be enhanced to handle not only the EKS deployment but also the initial GitHub repository creation and configuration. This eliminates the need for the Lambda function and the Ansible playbook. + +The new, unified process will be: +1. A developer defines a new EKS cluster by adding a single `module "eks_deployment"` block to a central Terraform configuration. +2. By setting `create_repository = true`, the developer instructs the module to perform the initial setup. +3. Running `terraform apply` will automatically: + a. Create a new GitHub repository using the `terraform-github-repo` module as a submodule. + b. Generate and commit all the necessary Terragrunt HCL files and a `README.md`. + c. Configure team permissions for the repository. + +The same module, when referenced from within the newly created repository's Terragrunt files, will have `create_repository = false` and will proceed with deploying the EKS cluster as it does today. + +### Core Component: The Enhanced `terraform-eks-deployment` Module + +| Action (Old Workflow) | Terraform Equivalent (Inside `terraform-eks-deployment`) | +| :--- | :--- | +| 1. Create a new repository from a template. | A new submodule block calling `terraform-github-repo` will be added, controlled by a `create_repository` flag. | +| 2. Generate HCL files from `config.json`. | The module will contain a new `templates` directory with HCL templates (`.tf.tpl`). It will use `templatefile()` to render the final HCL content from its input variables. | +| 3. Write files to the repository. | The rendered file content will be passed to the `files` input of the `terraform-github-repo` submodule. | +| 4. Assign team permissions. | The module will accept a `teams` variable and pass it to the `terraform-github-repo` submodule. | + +## 3. Detailed Migration Plan + +### Phase 1: Enhance the `terraform-eks-deployment` Module + +1. **Add Input Variables:** In `variables.tf`, add new variables: + * `create_repository`: A boolean to control whether to execute the repository creation logic. Default to `false`. + * `repository_name`: The name of the GitHub repository to create. + * `repository_teams`: A map of teams and their permissions for the new repository. + * Variables derived from the `generate_hcl_files.yml` playbook's `config` object (e.g., `environment`, `region`, `cluster_name`, `account`, `vpc`, etc.). + +2. **Create HCL Templates:** Create a `templates` directory within the module. For each file generated by the Ansible playbook (`root.hcl`, `account.hcl`, `region.hcl`, `vpc.hcl`, `cluster.hcl`, and `README.md`), create a corresponding `.tf.tpl` template file. Convert the Jinja2 syntax to Terraform's `${...}` interpolation syntax. + +3. **Implement Module Logic (`main.tf`):** + * Use `locals` to render the file content for each template using the `templatefile()` function. + * Add a `module "github_repo"` block that calls the `terraform-github-repo` module. + * Set the `count` of this submodule to `var.create_repository ? 1 : 0`. + * Pass the repository name, template configuration, and team permissions to the submodule. + * Map the rendered local variables to the `files` input of the `github_repo` submodule. + +### Phase 2: Implementation and Onboarding + +1. **Create a Central Management Repository:** Set up a new Git repository (e.g., `terragrunt-environments`) that will contain the Terraform configuration for creating all new EKS cluster repositories. + +2. **Onboard a Pilot Project:** In the new management repository, add a `main.tf` file. Add a module block that calls the enhanced `terraform-eks-deployment` module with `create_repository = true` to provision a repository for a new test cluster. + +3. **Execute and Validate:** Run `terraform apply` to create the repository. Verify that: + * The repository is created on GitHub. + * It is correctly initialized from the `template-eks-cluster` template. + * All the Terragrunt HCL files and the `README.md` are present and correctly populated. + * Team permissions are correctly assigned. + +### Phase 3: Decommissioning the Old Workflow + +Since we are not migrating existing repositories, the decommissioning process is straightforward. + +1. **Disable the Lambda Function:** The Lambda trigger can be disabled in AWS. +2. **Archive Old Repositories:** The `template-automation-lambda` and `template-repos-lambda-deployment` Git repositories should be archived. +3. **Delete AWS Resources:** The old AWS resources (Lambda, ECR, IAM roles) can be deleted. + +## 4. Rollback Plan + +A rollback is not applicable in the traditional sense. If the new workflow fails, the state can be destroyed (`terraform destroy`), the module can be fixed, and the process can be re-run. The old Lambda-based system can be kept available for emergency use. diff --git a/tf-native-v5.md b/tf-native-v5.md new file mode 100644 index 0000000..541d50f --- /dev/null +++ b/tf-native-v5.md @@ -0,0 +1,72 @@ +# Plan for Migrating to a Terraform-Native EKS Deployment Workflow (v5) + +This document outlines the plan to replace the current Lambda/Ansible-based system with a streamlined, Terraform-native workflow by enhancing the `terraform-eks-deployment` module. + +## 1. Analysis of the Current State + +The current process for provisioning a new EKS cluster repository involves multiple components: + +1. **`template-automation-lambda`**: A Python Lambda function that creates a new GitHub repository from a template. +2. **`generate_hcl_files.yml`**: An Ansible playbook inside the new repository that is run manually to generate a set of Terragrunt HCL files. +3. **`terraform-eks-deployment`**: The Terraform module that is referenced by the generated Terragrunt configuration to deploy the actual EKS cluster. + +This workflow is complex and involves manual steps. + +## 2. Proposed Solution: A Unified Repository Bootstrap Module + +We will consolidate the repository creation and configuration logic into the `terraform-eks-deployment` module. Its new, single purpose will be to bootstrap a fully configured GitHub repository for an EKS cluster. This eliminates the need for the Lambda function and the Ansible playbook. + +The new, unified process will be: +1. A developer defines a new EKS cluster repository by adding a single `module "eks_deployment"` block to a central Terraform configuration. +2. Running `terraform apply` will automatically: + a. Create a new GitHub repository using the `terraform-github-repo` module as a submodule. + b. Generate and commit all the necessary Terragrunt HCL files and a `README.md`. + c. Configure team permissions for the repository. + +The module will no longer be dual-purpose; it will *always* create a repository. The actual EKS deployment will be handled by the Terragrunt configuration within that new repository, which may in turn reference other modules. + +### Core Component: The Enhanced `terraform-eks-deployment` Module + +| Action (Old Workflow) | Terraform Equivalent (Inside `terraform-eks-deployment`) | +| :--- | :--- | +| 1. Create a new repository from a template. | A submodule block calling `terraform-github-repo` will create the repository. | +| 2. Generate HCL files from `config.json`. | The module will contain a `templates` directory with HCL templates (`.tf.tpl`). It will use `templatefile()` to render the final HCL content from its input variables. | +| 3. Write files to the repository. | The rendered file content will be passed to the `files` input of the `terraform-github-repo` submodule. | +| 4. Assign team permissions. | The module will accept a `teams` variable and pass it to the `terraform-github-repo` submodule. | + +## 3. Detailed Migration Plan + +### Phase 1: Enhance the `terraform-eks-deployment` Module + +1. **Define Input Variables:** In `variables.tf`, ensure all necessary variables are present. These are derived from the `generate_hcl_files.yml` playbook's `config` object (e.g., `repository_name`, `repository_teams`, `environment`, `region`, `cluster_name`, `account_config`, `vpc_config`, etc.). + +2. **Create HCL Templates:** Create a `templates` directory within the module. For each file generated by the Ansible playbook (`root.hcl`, `account.hcl`, `region.hcl`, `vpc.hcl`, `cluster.hcl`, and `README.md`), create a corresponding `.tf.tpl` template file. + +3. **Implement Module Logic (`main.tf`):** + * Use `locals` to render the file content for each template using the `templatefile()` function. + * Call the `terraform-github-repo` module unconditionally. + * Pass the repository name, template configuration, team permissions, and the rendered file content to the submodule. + +### Phase 2: Implementation and Onboarding + +1. **Integrate into a Management Repository:** The enhanced `terraform-eks-deployment` module will be consumed from a designated infrastructure management repository (e.g., `terragrunt-environments`). + +2. **Onboard a Pilot Project:** In the management repository, add a module block that calls the enhanced `terraform-eks-deployment` module to provision a repository for a new test cluster. + +3. **Execute and Validate:** Run `terraform apply` to create the repository. Verify that: + * The repository is created on GitHub. + * It is correctly initialized from the `template-eks-cluster` template. + * All the Terragrunt HCL files and the `README.md` are present and correctly populated. + * Team permissions are correctly assigned. + +### Phase 3: Decommissioning the Old Workflow + +Since we are not migrating existing repositories, the decommissioning process is straightforward. + +1. **Disable the Lambda Function:** The Lambda trigger can be disabled in AWS. +2. **Archive Old Repositories:** The `template-automation-lambda` and `template-repos-lambda-deployment` Git repositories should be archived. +3. **Delete AWS Resources:** The old AWS resources (Lambda, ECR, IAM roles) can be deleted. + +## 4. Rollback Plan + +A rollback is not applicable in the traditional sense. If the new workflow fails, the state can be destroyed (`terraform destroy`), the module can be fixed, and the process can be re-run. The old Lambda-based system can be kept available for emergency use. diff --git a/tf-native.md b/tf-native.md new file mode 100644 index 0000000..a5e5a9c --- /dev/null +++ b/tf-native.md @@ -0,0 +1,72 @@ +# Plan for Migrating to a Terraform-Native GitHub Repository Management Workflow + +This document outlines the plan to transition from the current Lambda/Ansible-based repository management system to a purely Terraform-native approach, leveraging the `terraform-github-repo` module. + +## 1. Current State Analysis + +Our current workflow for managing GitHub repositories relies on a Lambda function that executes Ansible playbooks. This setup has the following key characteristics: + +* **Technology Stack:** AWS Lambda, Python, Ansible. +* **Process:** A Lambda function is triggered, which in turn runs an Ansible playbook to configure GitHub repositories. +* **Drawbacks:** + * **Complexity:** Involves multiple technologies (Lambda, Ansible, Python) which increases the maintenance overhead. + * **State Management:** Managing state across these different systems can be challenging. + * **Less Declarative:** While Ansible is declarative, the overall workflow is more imperative compared to a pure Terraform solution. + +## 2. Proposed Solution: Terraform-Native Workflow + +We will replace the existing Lambda/Ansible setup with a new workflow centered around the `terraform-github-repo` Terraform module. This module provides a comprehensive set of resources for managing GitHub repositories declaratively. + +* **Technology Stack:** Terraform. +* **Process:** A new Terraform configuration will be created that uses the `terraform-github-repo` module to define the desired state of our GitHub repositories. +* **Benefits:** + * **Simplicity:** A single technology (Terraform) will be used for infrastructure and repository management. + * **Declarative:** The entire configuration will be declarative, making it easier to understand and manage. + * **State Management:** Terraform's state management will provide a single source of truth for the state of our repositories. + * **Reusability:** The `terraform-github-repo` module is a reusable component that can be used across multiple projects. + +## 3. Migration Plan + +The migration will be performed in the following phases: + +### Phase 1: Scoping and Setup + +1. **Identify Ansible Playbook Functionality:** Analyze the existing Ansible playbooks to identify all the repository configuration tasks they perform. This includes: + * Creating repositories. + * Managing collaborators and permissions. + * Configuring branch protection rules. + * Managing repository files (e.g., `CODEOWNERS`, license files). + * Setting up webhooks and deploy keys. + +2. **Map to Terraform Resources:** For each Ansible task, identify the corresponding resource in the `terraform-github-repo` module. + +3. **Setup New Terraform Project:** Create a new Git repository for the Terraform configuration that will manage the GitHub repositories. This repository will contain the new Terraform code. + +### Phase 2: Implementation + +1. **Develop Terraform Configuration:** Write the Terraform code that uses the `terraform-github-repo` module to replicate the functionality of the Ansible playbooks. The configuration should be modular and easily extensible. + +2. **Import Existing Resources:** Use `terraform import` to bring the existing GitHub repositories and their configurations under the management of the new Terraform configuration. This is a critical step to ensure a seamless transition without disrupting existing repositories. + +### Phase 3: Testing and Validation + +1. **Dry Run:** Perform a `terraform plan` to verify that the new configuration matches the existing state of the repositories. + +2. **Targeted Application:** Apply the new configuration to a non-critical repository first to validate the process. + +3. **Full Application:** Once the process is validated, apply the configuration to all repositories. + +### Phase 4: Decommissioning + +1. **Disable Lambda:** Disable the existing Lambda function to prevent it from making any further changes to the repositories. + +2. **Monitor:** Monitor the repositories for any unexpected changes or issues. + +3. **Remove Old Infrastructure:** Once the new system is stable, decommission the Lambda function and the associated Ansible playbooks. + +## 4. Rollback Plan + +In case of any issues, we can roll back to the previous system by: + +1. **Re-enabling the Lambda function.** +2. **Removing the new Terraform configuration from the state file using `terraform state rm`.** From 6596ff71334eab6d4413d5e3bfc60370445c3122 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 27 Aug 2025 17:48:36 -0400 Subject: [PATCH 23/33] Add README template for EKS cluster configuration --- main.tf | 9 ++++- templates/README.md.tf.tpl | 39 ++++++++++++++++++ tf-native-v2.md | 83 -------------------------------------- tf-native-v3.md | 75 ---------------------------------- tf-native-v4.md | 79 ------------------------------------ tf-native-v5.md | 72 --------------------------------- tf-native.md | 72 --------------------------------- 7 files changed, 46 insertions(+), 383 deletions(-) create mode 100644 templates/README.md.tf.tpl delete mode 100644 tf-native-v2.md delete mode 100644 tf-native-v3.md delete mode 100644 tf-native-v4.md delete mode 100644 tf-native-v5.md delete mode 100644 tf-native.md diff --git a/main.tf b/main.tf index 7c416a6..44543f7 100644 --- a/main.tf +++ b/main.tf @@ -33,12 +33,17 @@ locals { finops_project_role = var.cluster_config.finops_project_role, tags = var.cluster_config.tags, module_enablement_overrides = var.cluster_config.module_enablement_overrides + }), + "README.md" : templatefile("${path.module}/templates/README.md.tf.tpl", { + environment = var.environment, + aws_region = var.region, + cluster_name = var.cluster_config.cluster_name }) } } module "github_repo" { - source = "github.com/HappyPathway/terraform-github-repo" + source = "HappyPathway/repo/github" name = var.repository_name repo_org = var.organization @@ -55,7 +60,7 @@ module "github_repo" { github_has_wiki = true github_has_projects = true - files = [ + managed_extra_files = [ for path, content in local.rendered_files : { path = path content = content diff --git a/templates/README.md.tf.tpl b/templates/README.md.tf.tpl new file mode 100644 index 0000000..6e0c463 --- /dev/null +++ b/templates/README.md.tf.tpl @@ -0,0 +1,39 @@ +# EKS Cluster Configuration - ${upper(environment)} + +This EKS cluster configuration was generated using Terraform and the terraform-eks-deployment module. + +## Environment Details + +- **Environment**: ${environment} +- **Region**: ${aws_region} +- **Cluster Name**: ${cluster_name} + +## Directory Structure + +## Getting Started + +To apply this configuration: + +1. Change to the directory of the module you want to deploy: + ``` + cd ${environment}/${aws_region}/vpc/${cluster_name}/eks + ``` + +2. Initialize and apply the Terragrunt configuration: + ``` + terragrunt init + terragrunt plan + terragrunt apply + ``` + +3. Deploy additional modules as needed: + ``` + cd ../eks-cert-manager + terragrunt init + terragrunt plan + terragrunt apply + ``` + +## Customization + +Each module can be deployed independently using Terragrunt. \ No newline at end of file diff --git a/tf-native-v2.md b/tf-native-v2.md deleted file mode 100644 index 499762a..0000000 --- a/tf-native-v2.md +++ /dev/null @@ -1,83 +0,0 @@ -# Plan for Migrating to a Terraform-Native GitHub Repository Management Workflow (v2) - -This document provides a corrected and more accurate plan to replace the current Python Lambda-based workflow with a Terraform-native approach for creating and managing GitHub repositories. - -## 1. Corrected Analysis of the Current State - -The current system uses a Python-based AWS Lambda function (`template-automation-lambda`) to automate repository creation. It does **not** use Ansible for repository configuration as previously assumed. - -The workflow is as follows: -1. The Lambda function is invoked with details for a new repository (e.g., `project_name`, `template_settings`). -2. The Python script within the Lambda performs a series of GitHub API calls to: - a. Create a new repository. - b. Clone the contents of a base template repository into it. - c. Add a custom `config.json` file. - d. Assign team permissions. - e. Create a pull request to merge the initial setup into the `main` branch. - -This process, while effective, involves a custom-built Python application, a Lambda deployment, an ECR container registry, and associated IAM roles, making it complex to maintain. - -## 2. Proposed Solution: A Purely Terraform-Native Workflow - -We will replace the entire Lambda-based system with a declarative Terraform configuration that uses the `terraform-github-repo` module. This module natively supports the actions currently performed by the Python script. - -The new process will be: -1. A developer defines a new repository by adding a `module` block to a central Terraform configuration file. -2. Running `terraform apply` will instruct Terraform to perform all the necessary setup steps. - -### Mapping Lambda Actions to Terraform Resources - -| Action (Current Python Lambda) | Terraform Equivalent (`terraform-github-repo` module) | -| :--- | :--- | -| 1. Create a new repository. | The `github_repository` resource. The module uses this internally. | -| 2. Clone a template repository. | The `template` block within the `github_repository` resource. This is a native feature for creating a repo from a template. | -| 3. Write a `config.json` file. | The `github_repository_file` resource. The module accepts a `files` variable to manage this. | -| 4. Assign team permissions. | The `github_team_repository` resource. The module has a `teams` input for this. | -| 5. Create a pull request. | The `github_pull_request` resource. This can be defined outside the module to initialize the repository. | - -## 3. Detailed Migration Plan - -### Phase 1: Scoping and Proof of Concept - -1. **Identify All Template Variables:** Document all the key-value pairs that are currently passed into the `template_settings` of the Lambda. These will become variables in our new Terraform module. - -2. **Create a Wrapper Module:** Create a new, internal Terraform module that wraps the `terraform-github-repo` module. This wrapper will provide a simplified interface for our developers and contain the logic for creating the initial pull request. - -3. **Develop the PoC:** In a test environment, write a Terraform configuration that uses this new wrapper module to create a single, non-critical repository. The configuration should: - * Use the `template` feature to create the repository from the existing template repo. - * Use the `files` feature to add the `config.json`. - * Use the `teams` feature to grant permissions. - * Define a `github_pull_request` resource to create the initial PR. - -### Phase 2: Implementation and Import - -1. **Build Out the Full Configuration:** Create a new Git repository to house the Terraform configuration for all repositories that will be managed this way. - -2. **Import Existing Repositories:** For repositories previously created by the Lambda, use `terraform import` to bring them under Terraform's state management. This is a critical step to prevent any disruption. - * `terraform import module.my_repo.github_repository.this[0] my-repo-name` - * `terraform import module.my_repo.github_team_repository.teams["tf-module-admins"] my-repo-name:tf-module-admins` - -3. **Parallel Run:** For a transition period, both systems can exist. New repositories should be created using the Terraform method, while the Lambda is left in place to manage older ones if needed. - -### Phase 3: Testing and Validation - -1. **Dry Run with `terraform plan`:** Before applying any changes to a production repository, run `terraform plan` and carefully review the output to ensure it matches expectations and doesn't plan any destructive changes. - -2. **Full Application:** Once validated, apply the configuration to manage all target repositories. - -### Phase 4: Decommissioning - -Once all repositories are successfully managed by Terraform and the new workflow is stable, the old infrastructure can be safely removed. - -1. **Disable the Lambda Trigger:** The first step is to disable the mechanism that invokes the Lambda function. -2. **Delete the Lambda Function:** Remove the `template-automation-lambda` function from AWS. -3. **Delete the ECR Repository:** Delete the ECR repository holding the Lambda's container images. -4. **Delete the Deployment Pipeline:** Remove the `template-repos-lambda-deployment` Terraform configuration and state. -5. **Archive Old Repositories:** Archive the `template-automation-lambda` and `template-repos-lambda-deployment` Git repositories to mark them as deprecated. - -## 4. Rollback Plan - -If issues arise, we can revert by: -1. Removing the problematic repository from Terraform's state using `terraform state rm`. -2. Re-enabling or re-deploying the Lambda function to take over management again. -3. Manually correcting any unintended changes made by Terraform. diff --git a/tf-native-v3.md b/tf-native-v3.md deleted file mode 100644 index b723b86..0000000 --- a/tf-native-v3.md +++ /dev/null @@ -1,75 +0,0 @@ -# Plan for Migrating to a Terraform-Native EKS Deployment Workflow (v4) - -This document outlines the plan to replace the current Lambda/Ansible-based system with a streamlined, Terraform-native workflow for creating and configuring repositories for EKS deployments. - -## 1. Analysis of the Current State - -The current process for provisioning a new EKS cluster repository involves multiple, loosely-coupled components: - -1. **`template-automation-lambda`**: A Python Lambda function that creates a new GitHub repository from the `template-eks-cluster` template. It clones the template, adds a `config.json` file with user-provided settings, and opens a pull request. -2. **`generate_hcl_files.yml`**: An Ansible playbook inside the newly created repository that is run manually after the initial PR is merged. It reads the `config.json` and generates a set of Terragrunt HCL files (`root.hcl`, `account.hcl`, `region.hcl`, etc.). -3. **`terraform-eks-deployment`**: A Terraform module that is referenced by the generated Terragrunt configuration to deploy the actual EKS cluster. - -This workflow is complex, involves manual steps, and relies on a mix of technologies (Python, Lambda, Ansible, Terraform). - -## 2. Proposed Solution: A Unified, Terraform-Native Workflow - -We will create a single, unified Terraform workflow that handles the entire process of repository creation and configuration declaratively. This eliminates the need for the Lambda function and the Ansible playbook. - -The new process will be: -1. A developer defines a new EKS cluster by adding a single `module` block to a central Terraform configuration. -2. Running `terraform apply` will automatically: - a. Create a new GitHub repository. - b. Generate and commit all the necessary Terragrunt HCL files and `README.md`. - c. Configure team permissions for the repository. - -### Core Component: The New `terragrunt-eks-repo` Wrapper Module - -The centerpiece of this new workflow is a new Terraform module, `terragrunt-eks-repo`. This module will be responsible for all the setup logic. - -| Action (Old Workflow) | Terraform Equivalent (New `terragrunt-eks-repo` Module) | -| :--- | :--- | -| 1. Create a new repository from a template. | The module will call the `terraform-github-repo` module internally, using its `template` feature to clone from `template-eks-cluster`. | -| 2. Generate HCL files from `config.json`. | The module will contain HCL templates (`.tf.tpl` files). It will use Terraform's `templatefile()` function to render the final HCL content directly from its input variables. | -| 3. Write files to the repository. | The rendered file content will be passed to the `files` input of the underlying `terraform-github-repo` module, which uses the `github_repository_file` resource to commit them. | -| 4. Assign team permissions. | The module will accept a `teams` variable and pass it to the `terraform-github-repo` module to configure permissions using the `github_team_repository` resource. | - -## 3. Detailed Migration Plan - -### Phase 1: Develop the `terragrunt-eks-repo` Module - -1. **Create Module Scaffolding:** Create a new directory for the `terragrunt-eks-repo` module. - -2. **Define Input Variables:** Create a `variables.tf` file. The variables will be derived directly from the `generate_hcl_files.yml` playbook's `config` object (e.g., `environment`, `region`, `cluster_name`, `account`, `vpc`, etc.). - -3. **Create HCL Templates:** Create a `templates` directory within the module. For each file generated by the Ansible playbook (`root.hcl`, `account.hcl`, `region.hcl`, `vpc.hcl`, `cluster.hcl`, and `README.md`), create a corresponding `.tf.tpl` template file. Convert the Jinja2 syntax to Terraform's `${...}` interpolation syntax. - -4. **Implement Module Logic (`main.tf`):** - * Use `locals` to render the file content for each template using the `templatefile()` function. - * Call the `terraform-github-repo` module. - * Pass the repository name, template configuration, and team permissions to the module. - * Map the rendered local variables to the `files` input of the `terraform-github-repo` module. This will instruct it to create the files in the new repository. - -### Phase 2: Implementation and Onboarding - -1. **Create a Central Management Repository:** Set up a new Git repository (e.g., `terragrunt-environments`) that will contain the Terraform configuration for creating all new EKS cluster repositories. - -2. **Onboard a Pilot Project:** In the new management repository, add a `main.tf` file. Add a module block that calls the newly created `terragrunt-eks-repo` module to provision a repository for a new test cluster. - -3. **Execute and Validate:** Run `terraform apply` to create the repository. Verify that: - * The repository is created on GitHub. - * It is correctly initialized from the `template-eks-cluster` template. - * All the Terragrunt HCL files and the `README.md` are present and correctly populated with the variable values. - * Team permissions are correctly assigned. - -### Phase 3: Decommissioning the Old Workflow - -Since we are not concerned with migrating existing repositories, the decommissioning process is straightforward. Once the new workflow is validated and adopted for all new cluster provisioning: - -1. **Disable the Lambda Function:** The Lambda trigger can be disabled in AWS. -2. **Archive Old Repositories:** The `template-automation-lambda` and `template-repos-lambda-deployment` Git repositories should be archived to prevent further use. -3. **Delete AWS Resources:** The old AWS resources (Lambda function, ECR repository, IAM roles) can be deleted via Terraform from the `template-repos-lambda-deployment` project. - -## 4. Rollback Plan - -As we are not migrating existing resources, a rollback is not applicable in the traditional sense. If the new workflow fails for a new repository, the state can be destroyed (`terraform destroy`), the module can be fixed, and the process can be re-run. The old Lambda-based system can be temporarily kept available for emergency use until the new workflow is fully proven. diff --git a/tf-native-v4.md b/tf-native-v4.md deleted file mode 100644 index 3d9245f..0000000 --- a/tf-native-v4.md +++ /dev/null @@ -1,79 +0,0 @@ -# Plan for Migrating to a Terraform-Native EKS Deployment Workflow (v4) - -This document outlines the plan to replace the current Lambda/Ansible-based system with a streamlined, Terraform-native workflow by enhancing the `terraform-eks-deployment` module itself. - -## 1. Analysis of the Current State - -The current process for provisioning a new EKS cluster repository involves multiple components: - -1. **`template-automation-lambda`**: A Python Lambda function that creates a new GitHub repository from the `template-eks-cluster` template. -2. **`generate_hcl_files.yml`**: An Ansible playbook inside the new repository that is run manually to generate a set of Terragrunt HCL files (`root.hcl`, `account.hcl`, etc.). -3. **`terraform-eks-deployment`**: The Terraform module that is referenced by the generated Terragrunt configuration to deploy the actual EKS cluster. - -This workflow is complex, involves manual steps, and relies on a mix of technologies. - -## 2. Proposed Solution: A Unified, All-in-One EKS Deployment Module - -We will consolidate the entire workflow into the `terraform-eks-deployment` module. This module will be enhanced to handle not only the EKS deployment but also the initial GitHub repository creation and configuration. This eliminates the need for the Lambda function and the Ansible playbook. - -The new, unified process will be: -1. A developer defines a new EKS cluster by adding a single `module "eks_deployment"` block to a central Terraform configuration. -2. By setting `create_repository = true`, the developer instructs the module to perform the initial setup. -3. Running `terraform apply` will automatically: - a. Create a new GitHub repository using the `terraform-github-repo` module as a submodule. - b. Generate and commit all the necessary Terragrunt HCL files and a `README.md`. - c. Configure team permissions for the repository. - -The same module, when referenced from within the newly created repository's Terragrunt files, will have `create_repository = false` and will proceed with deploying the EKS cluster as it does today. - -### Core Component: The Enhanced `terraform-eks-deployment` Module - -| Action (Old Workflow) | Terraform Equivalent (Inside `terraform-eks-deployment`) | -| :--- | :--- | -| 1. Create a new repository from a template. | A new submodule block calling `terraform-github-repo` will be added, controlled by a `create_repository` flag. | -| 2. Generate HCL files from `config.json`. | The module will contain a new `templates` directory with HCL templates (`.tf.tpl`). It will use `templatefile()` to render the final HCL content from its input variables. | -| 3. Write files to the repository. | The rendered file content will be passed to the `files` input of the `terraform-github-repo` submodule. | -| 4. Assign team permissions. | The module will accept a `teams` variable and pass it to the `terraform-github-repo` submodule. | - -## 3. Detailed Migration Plan - -### Phase 1: Enhance the `terraform-eks-deployment` Module - -1. **Add Input Variables:** In `variables.tf`, add new variables: - * `create_repository`: A boolean to control whether to execute the repository creation logic. Default to `false`. - * `repository_name`: The name of the GitHub repository to create. - * `repository_teams`: A map of teams and their permissions for the new repository. - * Variables derived from the `generate_hcl_files.yml` playbook's `config` object (e.g., `environment`, `region`, `cluster_name`, `account`, `vpc`, etc.). - -2. **Create HCL Templates:** Create a `templates` directory within the module. For each file generated by the Ansible playbook (`root.hcl`, `account.hcl`, `region.hcl`, `vpc.hcl`, `cluster.hcl`, and `README.md`), create a corresponding `.tf.tpl` template file. Convert the Jinja2 syntax to Terraform's `${...}` interpolation syntax. - -3. **Implement Module Logic (`main.tf`):** - * Use `locals` to render the file content for each template using the `templatefile()` function. - * Add a `module "github_repo"` block that calls the `terraform-github-repo` module. - * Set the `count` of this submodule to `var.create_repository ? 1 : 0`. - * Pass the repository name, template configuration, and team permissions to the submodule. - * Map the rendered local variables to the `files` input of the `github_repo` submodule. - -### Phase 2: Implementation and Onboarding - -1. **Create a Central Management Repository:** Set up a new Git repository (e.g., `terragrunt-environments`) that will contain the Terraform configuration for creating all new EKS cluster repositories. - -2. **Onboard a Pilot Project:** In the new management repository, add a `main.tf` file. Add a module block that calls the enhanced `terraform-eks-deployment` module with `create_repository = true` to provision a repository for a new test cluster. - -3. **Execute and Validate:** Run `terraform apply` to create the repository. Verify that: - * The repository is created on GitHub. - * It is correctly initialized from the `template-eks-cluster` template. - * All the Terragrunt HCL files and the `README.md` are present and correctly populated. - * Team permissions are correctly assigned. - -### Phase 3: Decommissioning the Old Workflow - -Since we are not migrating existing repositories, the decommissioning process is straightforward. - -1. **Disable the Lambda Function:** The Lambda trigger can be disabled in AWS. -2. **Archive Old Repositories:** The `template-automation-lambda` and `template-repos-lambda-deployment` Git repositories should be archived. -3. **Delete AWS Resources:** The old AWS resources (Lambda, ECR, IAM roles) can be deleted. - -## 4. Rollback Plan - -A rollback is not applicable in the traditional sense. If the new workflow fails, the state can be destroyed (`terraform destroy`), the module can be fixed, and the process can be re-run. The old Lambda-based system can be kept available for emergency use. diff --git a/tf-native-v5.md b/tf-native-v5.md deleted file mode 100644 index 541d50f..0000000 --- a/tf-native-v5.md +++ /dev/null @@ -1,72 +0,0 @@ -# Plan for Migrating to a Terraform-Native EKS Deployment Workflow (v5) - -This document outlines the plan to replace the current Lambda/Ansible-based system with a streamlined, Terraform-native workflow by enhancing the `terraform-eks-deployment` module. - -## 1. Analysis of the Current State - -The current process for provisioning a new EKS cluster repository involves multiple components: - -1. **`template-automation-lambda`**: A Python Lambda function that creates a new GitHub repository from a template. -2. **`generate_hcl_files.yml`**: An Ansible playbook inside the new repository that is run manually to generate a set of Terragrunt HCL files. -3. **`terraform-eks-deployment`**: The Terraform module that is referenced by the generated Terragrunt configuration to deploy the actual EKS cluster. - -This workflow is complex and involves manual steps. - -## 2. Proposed Solution: A Unified Repository Bootstrap Module - -We will consolidate the repository creation and configuration logic into the `terraform-eks-deployment` module. Its new, single purpose will be to bootstrap a fully configured GitHub repository for an EKS cluster. This eliminates the need for the Lambda function and the Ansible playbook. - -The new, unified process will be: -1. A developer defines a new EKS cluster repository by adding a single `module "eks_deployment"` block to a central Terraform configuration. -2. Running `terraform apply` will automatically: - a. Create a new GitHub repository using the `terraform-github-repo` module as a submodule. - b. Generate and commit all the necessary Terragrunt HCL files and a `README.md`. - c. Configure team permissions for the repository. - -The module will no longer be dual-purpose; it will *always* create a repository. The actual EKS deployment will be handled by the Terragrunt configuration within that new repository, which may in turn reference other modules. - -### Core Component: The Enhanced `terraform-eks-deployment` Module - -| Action (Old Workflow) | Terraform Equivalent (Inside `terraform-eks-deployment`) | -| :--- | :--- | -| 1. Create a new repository from a template. | A submodule block calling `terraform-github-repo` will create the repository. | -| 2. Generate HCL files from `config.json`. | The module will contain a `templates` directory with HCL templates (`.tf.tpl`). It will use `templatefile()` to render the final HCL content from its input variables. | -| 3. Write files to the repository. | The rendered file content will be passed to the `files` input of the `terraform-github-repo` submodule. | -| 4. Assign team permissions. | The module will accept a `teams` variable and pass it to the `terraform-github-repo` submodule. | - -## 3. Detailed Migration Plan - -### Phase 1: Enhance the `terraform-eks-deployment` Module - -1. **Define Input Variables:** In `variables.tf`, ensure all necessary variables are present. These are derived from the `generate_hcl_files.yml` playbook's `config` object (e.g., `repository_name`, `repository_teams`, `environment`, `region`, `cluster_name`, `account_config`, `vpc_config`, etc.). - -2. **Create HCL Templates:** Create a `templates` directory within the module. For each file generated by the Ansible playbook (`root.hcl`, `account.hcl`, `region.hcl`, `vpc.hcl`, `cluster.hcl`, and `README.md`), create a corresponding `.tf.tpl` template file. - -3. **Implement Module Logic (`main.tf`):** - * Use `locals` to render the file content for each template using the `templatefile()` function. - * Call the `terraform-github-repo` module unconditionally. - * Pass the repository name, template configuration, team permissions, and the rendered file content to the submodule. - -### Phase 2: Implementation and Onboarding - -1. **Integrate into a Management Repository:** The enhanced `terraform-eks-deployment` module will be consumed from a designated infrastructure management repository (e.g., `terragrunt-environments`). - -2. **Onboard a Pilot Project:** In the management repository, add a module block that calls the enhanced `terraform-eks-deployment` module to provision a repository for a new test cluster. - -3. **Execute and Validate:** Run `terraform apply` to create the repository. Verify that: - * The repository is created on GitHub. - * It is correctly initialized from the `template-eks-cluster` template. - * All the Terragrunt HCL files and the `README.md` are present and correctly populated. - * Team permissions are correctly assigned. - -### Phase 3: Decommissioning the Old Workflow - -Since we are not migrating existing repositories, the decommissioning process is straightforward. - -1. **Disable the Lambda Function:** The Lambda trigger can be disabled in AWS. -2. **Archive Old Repositories:** The `template-automation-lambda` and `template-repos-lambda-deployment` Git repositories should be archived. -3. **Delete AWS Resources:** The old AWS resources (Lambda, ECR, IAM roles) can be deleted. - -## 4. Rollback Plan - -A rollback is not applicable in the traditional sense. If the new workflow fails, the state can be destroyed (`terraform destroy`), the module can be fixed, and the process can be re-run. The old Lambda-based system can be kept available for emergency use. diff --git a/tf-native.md b/tf-native.md deleted file mode 100644 index a5e5a9c..0000000 --- a/tf-native.md +++ /dev/null @@ -1,72 +0,0 @@ -# Plan for Migrating to a Terraform-Native GitHub Repository Management Workflow - -This document outlines the plan to transition from the current Lambda/Ansible-based repository management system to a purely Terraform-native approach, leveraging the `terraform-github-repo` module. - -## 1. Current State Analysis - -Our current workflow for managing GitHub repositories relies on a Lambda function that executes Ansible playbooks. This setup has the following key characteristics: - -* **Technology Stack:** AWS Lambda, Python, Ansible. -* **Process:** A Lambda function is triggered, which in turn runs an Ansible playbook to configure GitHub repositories. -* **Drawbacks:** - * **Complexity:** Involves multiple technologies (Lambda, Ansible, Python) which increases the maintenance overhead. - * **State Management:** Managing state across these different systems can be challenging. - * **Less Declarative:** While Ansible is declarative, the overall workflow is more imperative compared to a pure Terraform solution. - -## 2. Proposed Solution: Terraform-Native Workflow - -We will replace the existing Lambda/Ansible setup with a new workflow centered around the `terraform-github-repo` Terraform module. This module provides a comprehensive set of resources for managing GitHub repositories declaratively. - -* **Technology Stack:** Terraform. -* **Process:** A new Terraform configuration will be created that uses the `terraform-github-repo` module to define the desired state of our GitHub repositories. -* **Benefits:** - * **Simplicity:** A single technology (Terraform) will be used for infrastructure and repository management. - * **Declarative:** The entire configuration will be declarative, making it easier to understand and manage. - * **State Management:** Terraform's state management will provide a single source of truth for the state of our repositories. - * **Reusability:** The `terraform-github-repo` module is a reusable component that can be used across multiple projects. - -## 3. Migration Plan - -The migration will be performed in the following phases: - -### Phase 1: Scoping and Setup - -1. **Identify Ansible Playbook Functionality:** Analyze the existing Ansible playbooks to identify all the repository configuration tasks they perform. This includes: - * Creating repositories. - * Managing collaborators and permissions. - * Configuring branch protection rules. - * Managing repository files (e.g., `CODEOWNERS`, license files). - * Setting up webhooks and deploy keys. - -2. **Map to Terraform Resources:** For each Ansible task, identify the corresponding resource in the `terraform-github-repo` module. - -3. **Setup New Terraform Project:** Create a new Git repository for the Terraform configuration that will manage the GitHub repositories. This repository will contain the new Terraform code. - -### Phase 2: Implementation - -1. **Develop Terraform Configuration:** Write the Terraform code that uses the `terraform-github-repo` module to replicate the functionality of the Ansible playbooks. The configuration should be modular and easily extensible. - -2. **Import Existing Resources:** Use `terraform import` to bring the existing GitHub repositories and their configurations under the management of the new Terraform configuration. This is a critical step to ensure a seamless transition without disrupting existing repositories. - -### Phase 3: Testing and Validation - -1. **Dry Run:** Perform a `terraform plan` to verify that the new configuration matches the existing state of the repositories. - -2. **Targeted Application:** Apply the new configuration to a non-critical repository first to validate the process. - -3. **Full Application:** Once the process is validated, apply the configuration to all repositories. - -### Phase 4: Decommissioning - -1. **Disable Lambda:** Disable the existing Lambda function to prevent it from making any further changes to the repositories. - -2. **Monitor:** Monitor the repositories for any unexpected changes or issues. - -3. **Remove Old Infrastructure:** Once the new system is stable, decommission the Lambda function and the associated Ansible playbooks. - -## 4. Rollback Plan - -In case of any issues, we can roll back to the previous system by: - -1. **Re-enabling the Lambda function.** -2. **Removing the new Terraform configuration from the state file using `terraform state rm`.** From e1ebcd85d690a1555f577baf59fa3f2cf3f5061c Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 27 Aug 2025 17:53:42 -0400 Subject: [PATCH 24/33] Enhance README and examples for EKS cluster management - Updated README to clarify module functionality and features. - Added centralized management examples for EKS cluster repositories. - Improved variable definitions for better clarity and usability. --- README.md | 249 ++++++++++--------- examples/basic/main.tf | 69 +++-- examples/centralized-management/main.tf | 142 +++++++++++ examples/centralized-management/variables.tf | 23 ++ 4 files changed, 349 insertions(+), 134 deletions(-) create mode 100644 examples/centralized-management/main.tf create mode 100644 examples/centralized-management/variables.tf diff --git a/README.md b/README.md index 282aa3a..5b36df3 100644 --- a/README.md +++ b/README.md @@ -1,165 +1,180 @@ # terraform-eks-deployment -Terraform module for EKS cluster deployment and configuration +Terraform module for EKS cluster repository bootstrapping and configuration ## Overview -This module creates a new GitHub repository for your EKS cluster based on the template-eks-cluster repository. It sets up all necessary configuration files and triggers automated workflows for cluster deployment. +This module creates and configures a new GitHub repository for your EKS cluster based on the template-eks-cluster repository. It automatically generates all necessary Terragrunt HCL files (`root.hcl`, `account.hcl`, `region.hcl`, etc.) directly from your input variables, eliminating the need for manual configuration or separate Ansible playbooks. + +## Features + +- **Fully Terraform-Native**: No Lambda functions, Ansible playbooks, or manual steps required +- **Complete HCL Generation**: Automatically creates all necessary Terragrunt HCL files +- **Team Management**: Configures repository permissions for your teams +- **Customizable**: Extensive configuration options for cluster, account, and VPC settings ## Prerequisites - GitHub token with repository and workflow permissions -- Python 3.x installed on the machine running Terraform +- Terraform 1.0.0 or newer - Access to GitHub Enterprise (if using enterprise version) ## Usage +### Basic Example + ```hcl module "eks_deployment" { - source = "path/to/terraform-eks-deployment" - - name = "my-eks-cluster" - organization = "my-org" - environment = "production" - region = "us-east-1" + source = "github.com/HappyPathway/terraform-eks-deployment" + + # Repository configuration + repository_name = "eks-prod-cluster" + repository_template_owner = "HappyPathway" + repository_template = "template-eks-cluster" + repository_teams = { + "platform-team" = "admin", + "devops-team" = "maintain", + "developers" = "push" + } - cluster_config = { - cluster_name = "prod-eks-01" - account_name = "prod-account" - aws_account_id = "123456789012" - aws_profile = "prod-profile" + # Basic settings + organization = "my-org" + environment = "production" + region = "us-gov-west-1" + github_server_url = "https://github.e.it.census.gov" # For GitHub Enterprise + + # Account configuration + account_config = { + account_name = "prod-account" + aws_account_id = "123456789012" environment_abbr = "prod" + } + + # VPC configuration + vpc_config = { vpc_name = "prod-vpc" vpc_domain_name = "prod.example.com" } - github_token = "your-github-token" - github_server_url = "https://github.e.it.census.gov" # Optional, for GitHub Enterprise + # Cluster configuration + cluster_config = { + cluster_name = "prod-eks-01" + cluster_mailing_list = "team@example.com" + eks_instance_disk_size = 200 + eks_ng_desired_size = 5 + eks_ng_max_size = 10 + eks_ng_min_size = 3 + organization = "census:ocio:csvd" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_app" + tags = { + Owner = "Platform Team", + Environment = "Production", + CostCenter = "123-456" + } + module_enablement_overrides = { + cert_manager = true, + prometheus = true, + grafana = true, + istio = true + } + } } -``` - -## Workflow Automation - -### Overview - -The module automatically triggers GitHub Actions workflows in your newly created repository to: -1. Install Python requirements -2. Execute Terragrunt operations for cluster management - -### Workflow Sequence - -1. **Repository Creation**: The module creates a new repository from the template-eks-cluster template -2. **Initial Configuration**: Configuration files are generated based on your inputs -3. **Requirements Installation**: A workflow is triggered to install Python dependencies -4. **Cluster Planning**: A terragrunt plan workflow is automatically triggered - -### Available Workflows -Your new repository will have these workflows available: - -1. **Install Requirements** (`install-requirements.yml`) - - Triggered automatically on repository creation - - Installs all Python dependencies from requirements.txt - -2. **Terragrunt Cluster Operations** (`terragrunt-cluster-build.yml`) - - Supports plan, apply, and destroy operations - - Can be triggered manually or via API - - Includes safety checks and approvals - -### Triggering Workflows - -The workflows can be triggered in two ways: +output "repository_url" { + value = module.eks_deployment.repository_url +} +``` -1. **Automatic Triggering** - - On repository creation, the module automatically triggers: - 1. Requirements installation - 2. Initial cluster plan +### Centralized Management Example -2. **Manual Triggering** - - Via GitHub UI: - 1. Go to Actions tab - 2. Select desired workflow - 3. Click "Run workflow" - 4. Fill in parameters +You can use this module to manage multiple EKS cluster repositories from a central location: -3. **API Triggering** - - Use GitHub's API to trigger workflows: - ```bash - curl -X POST \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/OWNER/REPO/dispatches" \ - -d '{ - "event_type": "cluster-plan", - "client_payload": { - "environment": "dev", - "region": "us-east-1", - "cluster_dir": "my-cluster", - "auto_approve": false - } - }' - ``` +```hcl +# Production EKS Cluster Repository +module "prod_eks_cluster" { + source = "github.com/HappyPathway/terraform-eks-deployment" + + repository_name = "eks-prod-cluster" + repository_template_owner = "HappyPathway" + repository_template = "template-eks-cluster" + repository_teams = { + "platform-team" = "admin", + "prod-sre-team" = "maintain" + } + + # ...other configuration omitted for brevity +} -### Supported Events +# Development EKS Cluster Repository +module "dev_eks_cluster" { + source = "github.com/HappyPathway/terraform-eks-deployment" -The following event types are supported for workflow triggers: + repository_name = "eks-dev-cluster" + repository_template_owner = "HappyPathway" + repository_template = "template-eks-cluster" + repository_teams = { + "platform-team" = "admin", + "dev-team" = "maintain" + } + + # ...other configuration omitted for brevity +} +``` -- `install-requirements`: Install Python dependencies -- `cluster-plan`: Preview cluster changes -- `cluster-apply`: Apply cluster changes -- `cluster-destroy`: Destroy cluster +## Generated Files -### Required Secrets +The module automatically generates the following files in your new repository: -The following secrets must be configured in your repository: +- `root.hcl`: Global Terragrunt configuration +- `[environment]/account.hcl`: Account-specific variables +- `[environment]/[region]/region.hcl`: Region-specific variables +- `[environment]/[region]/vpc/vpc.hcl`: VPC-specific variables +- `[environment]/[region]/vpc/[cluster_name]/cluster.hcl`: Cluster-specific variables +- `README.md`: Usage instructions and documentation -- `AWS_ROLE_ARN`: ARN of the AWS role to assume -- `GITHUB_TOKEN`: GitHub token with workflow permissions +These files are committed directly to your new repository and are ready for use with Terragrunt to deploy your EKS cluster. ## Module Configuration ### Required Variables -- `name`: Repository name -- `organization`: GitHub organization name -- `environment`: Deployment environment -- `region`: AWS region -- `cluster_config`: Cluster configuration object -- `github_token`: GitHub token for workflow operations +| Name | Description | +|------|-------------| +| `repository_name` | Name of the GitHub repository to create | +| `organization` | GitHub organization name | +| `environment` | Deployment environment (e.g., production, development) | +| `region` | AWS region for the EKS cluster | +| `account_config` | Map of account configuration values | +| `vpc_config` | Map of VPC configuration values | +| `cluster_config` | Map of cluster configuration values | ### Optional Variables -- `github_server_url`: GitHub Enterprise server URL -- `template_repo_org`: Organization containing the template repository -- `enable_modules`: Map of modules to enable in the cluster +| Name | Description | Default | +|------|-------------|---------| +| `repository_template_owner` | Owner of the template repository | `"HappyPathway"` | +| `repository_template` | Template repository name | `"template-eks-cluster"` | +| `repository_teams` | Map of team names and permission levels | `{}` | +| `github_server_url` | GitHub Enterprise server URL | `"https://api.github.com"` | -For more configuration options, see the variables.tf file. +For complete configuration options, see the variables.tf file. ## Outputs -- `repository_url`: URL of the created repository -- `ssh_clone_url`: SSH clone URL of the repository - -## Security Considerations - -1. **GitHub Token**: Use a token with minimal required permissions -2. **AWS Role**: Use role-based access with least privilege -3. **Auto-approve**: Use with caution in production environments -4. **Environment Protection**: Configure branch protection rules - -## Troubleshooting +| Name | Description | +|------|-------------| +| `repository_url` | URL of the created repository | +| `ssh_clone_url` | SSH clone URL of the repository | -Common issues and solutions: +## How It Works -1. **Workflow Trigger Failures** - - Check GitHub token permissions - - Verify GitHub Enterprise URL (if applicable) - - Check network connectivity +1. The module calls the `terraform-github-repo` module to create a new repository based on your template. +2. It uses Terraform's `templatefile()` function to render Terragrunt HCL files from templates. +3. These rendered files are committed directly to the repository using the `github_repository_file` resource. +4. Team permissions are configured using the `github_team_repository` resource. -2. **Python Requirements** - - Ensure requirements.txt exists in template repository - - Check Python version compatibility +## Migrating from Previous Workflow -3. **AWS Authentication** - - Verify AWS role ARN - - Check AWS credentials configuration +This module replaces the previous Lambda/Ansible-based workflow with a purely Terraform-native approach. If you were previously using the template-automation-lambda to create repositories and then running an Ansible playbook to generate HCL files, you can now accomplish the entire process with just this module. diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 860a3c3..8a43420 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -7,32 +7,67 @@ provider "github" { module "eks_deployment" { source = "../../" - name = "eks-test-cluster" - organization = "my-org" - environment = "dev" - region = "us-east-1" + # Repository configuration + repository_name = "eks-test-cluster" + repository_template_owner = "HappyPathway" + repository_template = "template-eks-cluster" + repository_teams = { + "platform-team" = "admin", + "devops-team" = "maintain", + "developers" = "push" + } - template_repo_org = "my-org" + # Basic settings + organization = "my-org" + environment = "dev" + region = "us-east-1" github_server_url = "https://github.e.it.census.gov" - cluster_config = { - cluster_name = "dev-eks-01" + # Account configuration + account_config = { account_name = "dev-account" aws_account_id = "123456789012" - aws_profile = "dev-profile" environment_abbr = "dev" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" } - enable_modules = { - cert_manager = true - prometheus = true - grafana = true + # VPC configuration + vpc_config = { + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" } - versions = { - cluster_version = "1.27" - eks_module_version = "20.33.1" + # Cluster configuration + cluster_config = { + cluster_name = "dev-eks-01" + cluster_mailing_list = "team@example.com" + eks_instance_disk_size = 100 + eks_ng_desired_size = 3 + eks_ng_max_size = 6 + eks_ng_min_size = 2 + organization = "census:ocio:csvd" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_app" + tags = { + Owner = "Platform Team", + Environment = "Development", + CostCenter = "123-456" + } + module_enablement_overrides = { + cert_manager = true, + prometheus = true, + grafana = true, + istio = false + } } +} + +output "repository_url" { + description = "URL of the created GitHub repository" + value = module.eks_deployment.repository_url +} + +output "ssh_clone_url" { + description = "SSH clone URL of the repository" + value = module.eks_deployment.ssh_clone_url } \ No newline at end of file diff --git a/examples/centralized-management/main.tf b/examples/centralized-management/main.tf new file mode 100644 index 0000000..014e335 --- /dev/null +++ b/examples/centralized-management/main.tf @@ -0,0 +1,142 @@ +# Example: Centralized EKS Repository Management +# +# This example demonstrates how to use the terraform-eks-deployment module +# to manage multiple EKS cluster repositories from a central location. + +provider "github" { + # Configuration expected from environment variables: + # GITHUB_TOKEN + # GITHUB_OWNER (optional) + + # For GitHub Enterprise Server + base_url = var.github_server_url +} + +# Production EKS Cluster Repository +module "prod_eks_cluster" { + source = "../../" + + # Repository configuration + repository_name = "eks-prod-cluster" + repository_template_owner = "HappyPathway" + repository_template = "template-eks-cluster" + repository_teams = { + "platform-team" = "admin", + "prod-sre-team" = "maintain", + "developers" = "triage" + } + + # Basic settings + organization = var.github_organization + environment = "production" + region = "us-gov-west-1" + github_server_url = var.github_server_url + + # Account configuration + account_config = { + account_name = "prod-account" + aws_account_id = var.prod_account_id + environment_abbr = "prod" + } + + # VPC configuration + vpc_config = { + vpc_name = "prod-vpc" + vpc_domain_name = "prod.example.com" + } + + # Cluster configuration + cluster_config = { + cluster_name = "prod-eks-01" + cluster_mailing_list = "prod-alerts@example.com" + eks_instance_disk_size = 200 + eks_ng_desired_size = 5 + eks_ng_max_size = 10 + eks_ng_min_size = 3 + organization = "census:ocio:csvd" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_app" + tags = { + Owner = "Platform Team", + Environment = "Production", + CostCenter = "123-456" + } + module_enablement_overrides = { + cert_manager = true, + prometheus = true, + grafana = true, + istio = true + } + } +} + +# Development EKS Cluster Repository +module "dev_eks_cluster" { + source = "../../" + + # Repository configuration + repository_name = "eks-dev-cluster" + repository_template_owner = "HappyPathway" + repository_template = "template-eks-cluster" + repository_teams = { + "platform-team" = "admin", + "dev-team" = "maintain", + "developers" = "push" + } + + # Basic settings + organization = var.github_organization + environment = "development" + region = "us-gov-west-1" + github_server_url = var.github_server_url + + # Account configuration + account_config = { + account_name = "dev-account" + aws_account_id = var.dev_account_id + environment_abbr = "dev" + } + + # VPC configuration + vpc_config = { + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } + + # Cluster configuration + cluster_config = { + cluster_name = "dev-eks-01" + cluster_mailing_list = "dev-alerts@example.com" + eks_instance_disk_size = 100 + eks_ng_desired_size = 2 + eks_ng_max_size = 5 + eks_ng_min_size = 1 + organization = "census:ocio:csvd" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_app" + tags = { + Owner = "Platform Team", + Environment = "Development", + CostCenter = "123-456" + } + module_enablement_overrides = { + cert_manager = true, + prometheus = true, + grafana = false, + istio = false + } + } +} + +# Output repository URLs +output "prod_repository_url" { + description = "URL of the production EKS cluster repository" + value = module.prod_eks_cluster.repository_url +} + +output "dev_repository_url" { + description = "URL of the development EKS cluster repository" + value = module.dev_eks_cluster.repository_url +} \ No newline at end of file diff --git a/examples/centralized-management/variables.tf b/examples/centralized-management/variables.tf new file mode 100644 index 0000000..9522f71 --- /dev/null +++ b/examples/centralized-management/variables.tf @@ -0,0 +1,23 @@ +variable "github_organization" { + description = "GitHub organization name" + type = string + default = "HappyPathway" +} + +variable "github_server_url" { + description = "GitHub Enterprise server URL (e.g., https://github.e.it.census.gov)" + type = string + default = "https://api.github.com" +} + +variable "prod_account_id" { + description = "AWS account ID for production" + type = string + default = "123456789012" +} + +variable "dev_account_id" { + description = "AWS account ID for development" + type = string + default = "210987654321" +} \ No newline at end of file From 2d1a9976c65f53b370b4a3ff0829394449127d5a Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 4 Sep 2025 12:16:02 -0400 Subject: [PATCH 25/33] Fix formatting of runs-on value in GitHub Actions workflow --- .github/workflows/terraform-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terraform-release.yaml b/.github/workflows/terraform-release.yaml index b58eb34..319598d 100644 --- a/.github/workflows/terraform-release.yaml +++ b/.github/workflows/terraform-release.yaml @@ -7,7 +7,7 @@ on: - main jobs: terraform-ci-cd: - runs-on: 229685449397 + runs-on: "229685449397" permissions: contents: write From 94a518393002dd67ec0a178aee46323921889767 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 4 Sep 2025 16:13:15 -0400 Subject: [PATCH 26/33] Refactor README and ROADMAP for clarity; update Terraform configurations and remove deprecated variables --- README.md | 36 ----------- ROADMAP.md | 91 ++++++++++++---------------- examples/basic/.terraform_commits | 26 ++++++++ examples/basic/main.tf | 41 ++++++------- examples/basic/varfiles/default.json | 1 + examples/basic/variables.tf | 5 -- main.tf | 55 +++++++++-------- providers.tf | 10 ++- templates/README.md.tf.tpl | 2 +- variables.tf | 42 ++++++------- 10 files changed, 138 insertions(+), 171 deletions(-) create mode 100644 examples/basic/.terraform_commits create mode 100644 examples/basic/varfiles/default.json delete mode 100644 examples/basic/variables.tf diff --git a/README.md b/README.md index 5b36df3..0b51d07 100644 --- a/README.md +++ b/README.md @@ -87,42 +87,6 @@ output "repository_url" { } ``` -### Centralized Management Example - -You can use this module to manage multiple EKS cluster repositories from a central location: - -```hcl -# Production EKS Cluster Repository -module "prod_eks_cluster" { - source = "github.com/HappyPathway/terraform-eks-deployment" - - repository_name = "eks-prod-cluster" - repository_template_owner = "HappyPathway" - repository_template = "template-eks-cluster" - repository_teams = { - "platform-team" = "admin", - "prod-sre-team" = "maintain" - } - - # ...other configuration omitted for brevity -} - -# Development EKS Cluster Repository -module "dev_eks_cluster" { - source = "github.com/HappyPathway/terraform-eks-deployment" - - repository_name = "eks-dev-cluster" - repository_template_owner = "HappyPathway" - repository_template = "template-eks-cluster" - repository_teams = { - "platform-team" = "admin", - "dev-team" = "maintain" - } - - # ...other configuration omitted for brevity -} -``` - ## Generated Files The module automatically generates the following files in your new repository: diff --git a/ROADMAP.md b/ROADMAP.md index 12f698f..0667b5a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,62 +1,47 @@ -# EKS Cluster Template Roadmap +# EKS Cluster Deployment Roadmap ## Current Architecture -- Template repository (`template-eks-cluster`) serves as the base for creating new EKS cluster configurations -- Uses Terraform GitHub repo module to create new repositories from the template -- Implements managed and non-managed extra files functionality -- Supports configuration through `config.json` +- **Terraform-Native Workflow**: The module provides a fully Terraform-native workflow for bootstrapping a single GitHub repository for an EKS cluster. +- **Declarative Configuration**: It uses a set of input variables (`account_config`, `vpc_config`, `cluster_config`) to define a cluster's configuration. +- **Dynamic HCL Generation**: The module automatically generates all necessary Terragrunt HCL files (`root.hcl`, `account.hcl`, `region.hcl`, etc.) from internal templates. +- **Wrapper Module**: This module acts as a sophisticated wrapper around the `terraform-github-repo` module, orchestrating repository creation, file generation, and team permission management. ## Planned Enhancements -### 1. Automated Cluster Setup (High Priority) -- [x] Add GitHub Actions workflows with workflow_dispatch triggers -- [x] Implement automated terragrunt command execution for cluster building - - [x] Support for plan/apply/destroy commands - - [x] Environment-specific execution - - [x] Automated testing framework -- [ ] Configure workflows to run on specific runners for credential management -- [ ] Create templatized GitHub Actions workflow files -- [ ] Enable direct cluster creation without manual repository cloning - -### 2. File Management System (Medium Priority) -- [ ] Implement wrapper module for repo module -- [ ] Add support for crafting and injecting various configuration files -- [ ] Define file lifecycle management strategy - - [ ] Managed files (controlled by workspace) - - [ ] Non-managed files (user-modifiable) - -### 3. Version Management (Medium Priority) -- [ ] Implement version control strategy for `default-versions.hcl` -- [ ] Create system for managing platform release versions -- [ ] Set up version override mechanism - - [ ] Default versions in template repo - - [ ] Override capability in workspace creating repos - -### 4. Configuration Management (Low Priority) -- [ ] Enhance Makefile and Ansible playbook integration -- [ ] Improve configuration file templating -- [ ] Add validation for configuration files +### 1. Version and Configuration Management (High Priority) +- [x] Implement version control strategy for `default-versions.hcl` +- [x] Create system for managing platform release versions +- [x] Set up version override mechanism +- [ ] Add validation for configuration files to fail early on invalid inputs. +- [ ] Integrate a robust versioning strategy for the module itself, using tags. + +### 2. Automated Cluster Deployment (Medium Priority) +- [ ] Create templatized GitHub Actions workflow files that can be injected into the created repository. +- [ ] Configure workflows to run on specific runners for secure credential management. +- [ ] Implement automated `terragrunt plan/apply` execution within the generated workflows. + +### 3. Module and Documentation Refinements (Low Priority) +- [ ] Enhance documentation with more detailed examples, including edge cases. +- [ ] Add contribution guidelines and a developer guide. +- [ ] Refactor and clean up any legacy code or unused variables. + +## Completed Milestones + +- **Terraform-Native Migration**: Successfully replaced the legacy Lambda/Ansible workflow with a unified, Terraform-native solution for repository bootstrapping. +- **File Management System**: Implemented a robust system for crafting and injecting configuration files directly via Terraform, fulfilling the core requirement of the "File Management System" epic. ## Technical Considerations -1. File Lifecycle Management: - - Managed files: Controlled by workspace - - Non-managed files: User-modifiable post-creation - - Version-specific files: Platform release coordination - -2. Automation Requirements: - - GitHub Actions runner configuration - - Credential management - - Workflow templating - - Terragrunt integration - -3. Version Control Strategy: - - Module version collections - - Platform release versions - - Override mechanisms +1. **Automation Requirements**: + - GitHub Actions runner configuration for secure AWS authentication. + - Workflow templating to dynamically generate CI/CD pipelines. + - Seamless Terragrunt integration within the automated workflows. + +2. **Version Control Strategy**: + - Centralized management of component versions (EKS, Istio, etc.). + - Clear override mechanisms for environment-specific versioning. ## Success Criteria -- Fully automated cluster creation process -- Minimal manual intervention required -- Proper version management system -- Clear file lifecycle management -- Secure credential handling \ No newline at end of file +- Fully automated repository and cluster configuration process. +- A clear, maintainable, and scalable version management system. +- Secure and automated CI/CD pipelines for deploying EKS clusters. +- Comprehensive documentation that empowers users and contributors. \ No newline at end of file diff --git a/examples/basic/.terraform_commits b/examples/basic/.terraform_commits new file mode 100644 index 0000000..722dfa1 --- /dev/null +++ b/examples/basic/.terraform_commits @@ -0,0 +1,26 @@ +[ + { + "commit_hash": "2d1a9976c65f53b370b4a3ff0829394449127d5a", + "commit_message": "Fix formatting of runs-on value in GitHub Actions workflow", + "author": "Your Name", + "timestamp": "2025-09-04T13:49:32.940158" + }, + { + "commit_hash": "2d1a9976c65f53b370b4a3ff0829394449127d5a", + "commit_message": "Fix formatting of runs-on value in GitHub Actions workflow", + "author": "Your Name", + "timestamp": "2025-09-04T13:52:48.384810" + }, + { + "commit_hash": "2d1a9976c65f53b370b4a3ff0829394449127d5a", + "commit_message": "Fix formatting of runs-on value in GitHub Actions workflow", + "author": "Your Name", + "timestamp": "2025-09-04T14:36:08.594715" + }, + { + "commit_hash": "2d1a9976c65f53b370b4a3ff0829394449127d5a", + "commit_message": "Fix formatting of runs-on value in GitHub Actions workflow", + "author": "Your Name", + "timestamp": "2025-09-04T15:59:08.094373" + } +] \ No newline at end of file diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 8a43420..0929f07 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -1,16 +1,22 @@ +provider "aws" { +} + +data "aws_secretsmanager_secret_version" "github_token" { + secret_id = "/eks-cluster-deployment/github_token" +} + provider "github" { - # Configuration expected from environment variables: - # GITHUB_TOKEN - # GITHUB_OWNER (optional) + token = data.aws_secretsmanager_secret_version.github_token.secret_string } module "eks_deployment" { source = "../../" # Repository configuration - repository_name = "eks-test-cluster" - repository_template_owner = "HappyPathway" - repository_template = "template-eks-cluster" + name = "eks-test-cluster" + template_repo_org = "SCT-Engineering" + repository_template = "template-eks-cluster" + is_private = false # Set to false to make the repository public repository_teams = { "platform-team" = "admin", "devops-team" = "maintain", @@ -18,26 +24,19 @@ module "eks_deployment" { } # Basic settings - organization = "my-org" + organization = "SCT-Engineering" environment = "dev" - region = "us-east-1" + region = "us-gov-west-1" github_server_url = "https://github.e.it.census.gov" - # Account configuration - account_config = { - account_name = "dev-account" - aws_account_id = "123456789012" - environment_abbr = "dev" - } - - # VPC configuration - vpc_config = { - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" - } - # Cluster configuration cluster_config = { + account_name = "dev-account" + aws_account_id = "123456789012" + environment_abbr = "dev" + aws_profile = "default" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" cluster_name = "dev-eks-01" cluster_mailing_list = "team@example.com" eks_instance_disk_size = 100 diff --git a/examples/basic/varfiles/default.json b/examples/basic/varfiles/default.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/examples/basic/varfiles/default.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf deleted file mode 100644 index 73092ff..0000000 --- a/examples/basic/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "github_token" { - description = "GitHub token for API operations" - type = string - sensitive = true -} \ No newline at end of file diff --git a/main.tf b/main.tf index 44543f7..60094d7 100644 --- a/main.tf +++ b/main.tf @@ -1,26 +1,25 @@ locals { - # Render the HCL files from templates rendered_files = { "root.hcl" : templatefile("${path.module}/templates/root.hcl.tf.tpl", { environment = var.environment }), - "${var.environment}/account.hcl" : templatefile("${path.module}/templates/account.hcl.tf.tpl", { - account_name = var.account_config.account_name, - aws_account_id = var.account_config.aws_account_id, + "environment/account.hcl" : templatefile("${path.module}/templates/account.hcl.tf.tpl", { + account_name = var.cluster_config.account_name, + aws_account_id = var.cluster_config.aws_account_id, environment = var.environment, - environment_abbr = var.account_config.environment_abbr + environment_abbr = var.cluster_config.environment_abbr }), - "${var.environment}/${var.region}/region.hcl" : templatefile("${path.module}/templates/region.hcl.tf.tpl", { - environment = var.environment, - aws_region = var.region + "environment/region/region.hcl" : templatefile("${path.module}/templates/region.hcl.tf.tpl", { + aws_region = var.region, + environment = var.environment }), - "${var.environment}/${var.region}/vpc/vpc.hcl" : templatefile("${path.module}/templates/vpc.hcl.tf.tpl", { + "environment/region/vpc/vpc.hcl" : templatefile("${path.module}/templates/vpc.hcl.tf.tpl", { + vpc_name = var.cluster_config.vpc_name, + vpc_domain_name = var.cluster_config.vpc_domain_name, environment = var.environment, - aws_region = var.region, - vpc_name = var.vpc_config.vpc_name, - vpc_domain_name = var.vpc_config.vpc_domain_name + aws_region = var.region }), - "${var.environment}/${var.region}/vpc/${var.cluster_config.cluster_name}/cluster.hcl" : templatefile("${path.module}/templates/cluster.hcl.tf.tpl", { + "environment/region/vpc/cluster/cluster.hcl" : templatefile("${path.module}/templates/cluster.hcl.tf.tpl", { cluster_name = var.cluster_config.cluster_name, cluster_mailing_list = var.cluster_config.cluster_mailing_list, eks_instance_disk_size = var.cluster_config.eks_instance_disk_size, @@ -36,26 +35,25 @@ locals { }), "README.md" : templatefile("${path.module}/templates/README.md.tf.tpl", { environment = var.environment, - aws_region = var.region, - cluster_name = var.cluster_config.cluster_name + cluster_name = var.cluster_config.cluster_name, + aws_region = var.region }) } } module "github_repo" { - source = "HappyPathway/repo/github" + source = "github.com/HappyPathway/terraform-github-repo" - name = var.repository_name + name = var.name repo_org = var.organization github_repo_description = "EKS Cluster Configuration for ${var.cluster_config.cluster_name}" github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] + force_name = var.force_name - template = { - owner = var.repository_template_owner - repository = var.repository_template - } + template_repo_org = var.repository_template_owner + template_repo = var.repository_template - github_is_private = true + github_is_private = var.is_private github_has_issues = true github_has_wiki = true github_has_projects = true @@ -66,12 +64,19 @@ module "github_repo" { content = content } ] - - teams = var.repository_teams + archive_on_destroy = false + github_org_teams = [ + for team, permission in var.repository_teams : { + team_name = team + permission = permission + slug = lower(replace(team, " ", "-")) + id = null # Changed from team_id to id as expected by the module + bypass_rules = false + } + ] } # The EKS deployment logic will go here, and will be skipped if create_repository is true. -# For now, we are just implementing the repository creation part. output "repository_url" { description = "URL of the created repository" diff --git a/providers.tf b/providers.tf index e01c942..c5ea7b3 100644 --- a/providers.tf +++ b/providers.tf @@ -4,11 +4,9 @@ terraform { source = "integrations/github" version = ">= 5.0" } + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } } -} - -provider "github" { - # Configuration is expected from environment variables: - # GITHUB_TOKEN - # GITHUB_OWNER (optional) } \ No newline at end of file diff --git a/templates/README.md.tf.tpl b/templates/README.md.tf.tpl index 6e0c463..75175a8 100644 --- a/templates/README.md.tf.tpl +++ b/templates/README.md.tf.tpl @@ -16,7 +16,7 @@ To apply this configuration: 1. Change to the directory of the module you want to deploy: ``` - cd ${environment}/${aws_region}/vpc/${cluster_name}/eks + cd environment/region/vpc/cluster/eks ``` 2. Initialize and apply the Terragrunt configuration: diff --git a/variables.tf b/variables.tf index e43ad69..6ef71b2 100644 --- a/variables.tf +++ b/variables.tf @@ -1,11 +1,12 @@ variable "name" { - description = "Name of the repository" + description = "The name of the repository." type = string } variable "organization" { - description = "GitHub organization name" + description = "The organization to create the repository in." type = string + default = "HappyPathway" } variable "template_repo_org" { @@ -178,6 +179,11 @@ variable "cluster_config" { eks_ng_min_size = optional(number, 3) enable_cluster_creator_admin_permissions = optional(bool, true) tags = optional(map(string), {}) + module_enablement_overrides = optional(map(bool), {}) + finops_project_name = optional(string) + finops_project_number = optional(string) + finops_project_role = optional(string) + organization = optional(string) }) } @@ -244,14 +250,8 @@ variable "create_repository" { default = false } -variable "repository_name" { - description = "The name of the GitHub repository to create." - type = string - default = "" -} - variable "repository_teams" { - description = "A map of teams and their permissions for the new repository." + description = "A map of teams and their permissions to grant on the repository." type = map(string) default = {} } @@ -265,23 +265,17 @@ variable "repository_template" { variable "repository_template_owner" { description = "The owner of the template repository." type = string - default = "HappyPathway" + default = "SCT-Engineering" } -variable "cluster_config" { - description = "A map of configuration values for the cluster." - type = any - default = {} -} - -variable "account_config" { - description = "A map of configuration values for the account." - type = any - default = {} +variable "is_private" { + description = "Whether the repository should be private." + type = bool + default = true } -variable "vpc_config" { - description = "A map of configuration values for the vpc." - type = any - default = {} +variable "force_name" { + description = "Keep exact repository name (no date suffix)" + type = bool + default = true } \ No newline at end of file From bf499a0b49adb339a22a38624648f76c77f6ebdb Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 10 Sep 2025 16:47:13 -0400 Subject: [PATCH 27/33] Update EKS cluster configuration with new account details --- examples/basic/.terraform_commits | 24 ++++++++++++++++++++++++ examples/basic/main.tf | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/examples/basic/.terraform_commits b/examples/basic/.terraform_commits index 722dfa1..8a830fb 100644 --- a/examples/basic/.terraform_commits +++ b/examples/basic/.terraform_commits @@ -22,5 +22,29 @@ "commit_message": "Fix formatting of runs-on value in GitHub Actions workflow", "author": "Your Name", "timestamp": "2025-09-04T15:59:08.094373" + }, + { + "commit_hash": "94a518393002dd67ec0a178aee46323921889767", + "commit_message": "Refactor README and ROADMAP for clarity; update Terraform configurations and remove deprecated variables", + "author": "Your Name", + "timestamp": "2025-09-05T13:52:50.300053" + }, + { + "commit_hash": "94a518393002dd67ec0a178aee46323921889767", + "commit_message": "Refactor README and ROADMAP for clarity; update Terraform configurations and remove deprecated variables", + "author": "Your Name", + "timestamp": "2025-09-05T13:57:43.873358" + }, + { + "commit_hash": "94a518393002dd67ec0a178aee46323921889767", + "commit_message": "Refactor README and ROADMAP for clarity; update Terraform configurations and remove deprecated variables", + "author": "Your Name", + "timestamp": "2025-09-08T13:08:10.570707" + }, + { + "commit_hash": "94a518393002dd67ec0a178aee46323921889767", + "commit_message": "Refactor README and ROADMAP for clarity; update Terraform configurations and remove deprecated variables", + "author": "Your Name", + "timestamp": "2025-09-10T15:25:49.548237" } ] \ No newline at end of file diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 0929f07..471dfb5 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -31,8 +31,8 @@ module "eks_deployment" { # Cluster configuration cluster_config = { - account_name = "dev-account" - aws_account_id = "123456789012" + account_name = "csvd-dev-ew" + aws_account_id = "229685449397" environment_abbr = "dev" aws_profile = "default" vpc_name = "dev-vpc" From 1694ff2abd5271962c65bf1a8798c15cfdcd0af9 Mon Sep 17 00:00:00 2001 From: David John Arnold Jr Date: Wed, 10 Sep 2025 14:09:19 -0700 Subject: [PATCH 28/33] Update main.tf --- examples/basic/main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 471dfb5..661dce4 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -35,10 +35,10 @@ module "eks_deployment" { aws_account_id = "229685449397" environment_abbr = "dev" aws_profile = "default" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" + vpc_name = "vpc2-csvd-dev" + vpc_domain_name = "dev.inf.csp1.census.gov" cluster_name = "dev-eks-01" - cluster_mailing_list = "team@example.com" + cluster_mailing_list = "david.j.arnold.jr@census.gov" eks_instance_disk_size = 100 eks_ng_desired_size = 3 eks_ng_max_size = 6 @@ -69,4 +69,4 @@ output "repository_url" { output "ssh_clone_url" { description = "SSH clone URL of the repository" value = module.eks_deployment.ssh_clone_url -} \ No newline at end of file +} From 7df2ccd12e3987b7146e6b71a0a62705508374fd Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 10 Sep 2025 17:19:18 -0400 Subject: [PATCH 29/33] Update EKS deployment configuration: change region to us-gov-east-1 and update VPC name to vpc3-csvd-dev --- examples/basic/.terraform_commits | 12 ++++++++++++ examples/basic/main.tf | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/examples/basic/.terraform_commits b/examples/basic/.terraform_commits index 8a830fb..5b4199e 100644 --- a/examples/basic/.terraform_commits +++ b/examples/basic/.terraform_commits @@ -46,5 +46,17 @@ "commit_message": "Refactor README and ROADMAP for clarity; update Terraform configurations and remove deprecated variables", "author": "Your Name", "timestamp": "2025-09-10T15:25:49.548237" + }, + { + "commit_hash": "1694ff2abd5271962c65bf1a8798c15cfdcd0af9", + "commit_message": "Update main.tf", + "author": "David John Arnold Jr", + "timestamp": "2025-09-10T17:11:02.705468" + }, + { + "commit_hash": "1694ff2abd5271962c65bf1a8798c15cfdcd0af9", + "commit_message": "Update main.tf", + "author": "David John Arnold Jr", + "timestamp": "2025-09-10T17:17:55.685167" } ] \ No newline at end of file diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 661dce4..76c0e02 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -26,7 +26,7 @@ module "eks_deployment" { # Basic settings organization = "SCT-Engineering" environment = "dev" - region = "us-gov-west-1" + region = "us-gov-east-1" github_server_url = "https://github.e.it.census.gov" # Cluster configuration @@ -35,7 +35,7 @@ module "eks_deployment" { aws_account_id = "229685449397" environment_abbr = "dev" aws_profile = "default" - vpc_name = "vpc2-csvd-dev" + vpc_name = "vpc3-csvd-dev" vpc_domain_name = "dev.inf.csp1.census.gov" cluster_name = "dev-eks-01" cluster_mailing_list = "david.j.arnold.jr@census.gov" From 70c4768538879ab9f2444a6eaec7b40ab2e7cc61 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 11 Sep 2025 13:01:38 -0400 Subject: [PATCH 30/33] Refactor repository and cluster configuration: unify naming and update descriptions in main.tf and variables.tf --- examples/basic/main.tf | 3 +- examples/centralized-management/main.tf | 56 +++++++++---------------- main.tf | 6 +-- variables.tf | 3 +- 4 files changed, 25 insertions(+), 43 deletions(-) diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 76c0e02..d84fb0b 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -12,7 +12,7 @@ provider "github" { module "eks_deployment" { source = "../../" - # Repository configuration + # Repository and cluster configuration - single name for both name = "eks-test-cluster" template_repo_org = "SCT-Engineering" repository_template = "template-eks-cluster" @@ -37,7 +37,6 @@ module "eks_deployment" { aws_profile = "default" vpc_name = "vpc3-csvd-dev" vpc_domain_name = "dev.inf.csp1.census.gov" - cluster_name = "dev-eks-01" cluster_mailing_list = "david.j.arnold.jr@census.gov" eks_instance_disk_size = 100 eks_ng_desired_size = 3 diff --git a/examples/centralized-management/main.tf b/examples/centralized-management/main.tf index 014e335..5e09a13 100644 --- a/examples/centralized-management/main.tf +++ b/examples/centralized-management/main.tf @@ -16,10 +16,10 @@ provider "github" { module "prod_eks_cluster" { source = "../../" - # Repository configuration - repository_name = "eks-prod-cluster" - repository_template_owner = "HappyPathway" - repository_template = "template-eks-cluster" + # Repository and cluster configuration - single name for both + name = "eks-prod-cluster" # This name is used for both the repository and cluster + template_repo_org = "HappyPathway" + repository_template = "template-eks-cluster" repository_teams = { "platform-team" = "admin", "prod-sre-team" = "maintain", @@ -32,22 +32,14 @@ module "prod_eks_cluster" { region = "us-gov-west-1" github_server_url = var.github_server_url - # Account configuration - account_config = { - account_name = "prod-account" - aws_account_id = var.prod_account_id - environment_abbr = "prod" - } - - # VPC configuration - vpc_config = { - vpc_name = "prod-vpc" - vpc_domain_name = "prod.example.com" - } - # Cluster configuration cluster_config = { - cluster_name = "prod-eks-01" + account_name = "prod-account" + aws_account_id = var.prod_account_id + environment_abbr = "prod" + aws_profile = "default" + vpc_name = "prod-vpc" + vpc_domain_name = "prod.example.com" cluster_mailing_list = "prod-alerts@example.com" eks_instance_disk_size = 200 eks_ng_desired_size = 5 @@ -75,10 +67,10 @@ module "prod_eks_cluster" { module "dev_eks_cluster" { source = "../../" - # Repository configuration - repository_name = "eks-dev-cluster" - repository_template_owner = "HappyPathway" - repository_template = "template-eks-cluster" + # Repository and cluster configuration - single name for both + name = "eks-dev-cluster" # This name is used for both the repository and cluster + template_repo_org = "HappyPathway" + repository_template = "template-eks-cluster" repository_teams = { "platform-team" = "admin", "dev-team" = "maintain", @@ -91,22 +83,14 @@ module "dev_eks_cluster" { region = "us-gov-west-1" github_server_url = var.github_server_url - # Account configuration - account_config = { - account_name = "dev-account" - aws_account_id = var.dev_account_id - environment_abbr = "dev" - } - - # VPC configuration - vpc_config = { - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" - } - # Cluster configuration cluster_config = { - cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = var.dev_account_id + environment_abbr = "dev" + aws_profile = "default" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" cluster_mailing_list = "dev-alerts@example.com" eks_instance_disk_size = 100 eks_ng_desired_size = 2 diff --git a/main.tf b/main.tf index 60094d7..af88a2b 100644 --- a/main.tf +++ b/main.tf @@ -20,7 +20,7 @@ locals { aws_region = var.region }), "environment/region/vpc/cluster/cluster.hcl" : templatefile("${path.module}/templates/cluster.hcl.tf.tpl", { - cluster_name = var.cluster_config.cluster_name, + cluster_name = var.name, cluster_mailing_list = var.cluster_config.cluster_mailing_list, eks_instance_disk_size = var.cluster_config.eks_instance_disk_size, eks_ng_desired_size = var.cluster_config.eks_ng_desired_size, @@ -35,7 +35,7 @@ locals { }), "README.md" : templatefile("${path.module}/templates/README.md.tf.tpl", { environment = var.environment, - cluster_name = var.cluster_config.cluster_name, + cluster_name = var.name, aws_region = var.region }) } @@ -46,7 +46,7 @@ module "github_repo" { name = var.name repo_org = var.organization - github_repo_description = "EKS Cluster Configuration for ${var.cluster_config.cluster_name}" + github_repo_description = "EKS Cluster Configuration for ${var.name}" github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] force_name = var.force_name diff --git a/variables.tf b/variables.tf index 6ef71b2..bbe40c8 100644 --- a/variables.tf +++ b/variables.tf @@ -1,5 +1,5 @@ variable "name" { - description = "The name of the repository." + description = "The name of the repository and EKS cluster. This name will be used for both the GitHub repository and the EKS cluster." type = string } @@ -165,7 +165,6 @@ variable "namespaces" { variable "cluster_config" { description = "Configuration for the EKS cluster" type = object({ - cluster_name = string account_name = string aws_account_id = string aws_profile = string From f5e7c5fb81e6adfd525d33f7d30bfb55d1087a07 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 11 Sep 2025 13:14:20 -0400 Subject: [PATCH 31/33] Update cluster name assignment in locals.tf and add default.json file --- locals.tf | 2 +- varfiles/default.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 varfiles/default.json diff --git a/locals.tf b/locals.tf index aaa3ee4..3748484 100644 --- a/locals.tf +++ b/locals.tf @@ -136,7 +136,7 @@ locals { vpc_domain_name = var.cluster_config.vpc_domain_name } cluster = { - cluster_name = var.cluster_config.cluster_name + cluster_name = var.name cluster_mailing_list = var.cluster_config.cluster_mailing_list eks_instance_disk_size = var.cluster_config.eks_instance_disk_size eks_ng_desired_size = var.cluster_config.eks_ng_desired_size diff --git a/varfiles/default.json b/varfiles/default.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/varfiles/default.json @@ -0,0 +1 @@ +{} \ No newline at end of file From 89e28386d295f9dc11d27e5340fcc91bc1e272f5 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 11 Sep 2025 13:26:44 -0400 Subject: [PATCH 32/33] Update GitHub repository source URL in main.tf --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index af88a2b..5a281a3 100644 --- a/main.tf +++ b/main.tf @@ -42,7 +42,7 @@ locals { } module "github_repo" { - source = "github.com/HappyPathway/terraform-github-repo" + source = "git::https://github.e.it.census.gov/CSVD/terraform-github-repo.git" name = var.name repo_org = var.organization From 9ca1513f4d5af832f4879150aba5489c5eddaaa3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 15 Sep 2025 17:34:31 -0400 Subject: [PATCH 33/33] Remove commented-out AWS provider and GitHub token configuration; add providers.tf for AWS provider definition --- examples/basic/main.tf | 15 ++++++--------- examples/basic/providers.tf | 2 ++ 2 files changed, 8 insertions(+), 9 deletions(-) create mode 100644 examples/basic/providers.tf diff --git a/examples/basic/main.tf b/examples/basic/main.tf index d84fb0b..454d5a5 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -1,13 +1,10 @@ -provider "aws" { -} - -data "aws_secretsmanager_secret_version" "github_token" { - secret_id = "/eks-cluster-deployment/github_token" -} +# data "aws_secretsmanager_secret_version" "github_token" { +# secret_id = "/eks-cluster-deployment/github_token" +# } -provider "github" { - token = data.aws_secretsmanager_secret_version.github_token.secret_string -} +# provider "github" { +# token = data.aws_secretsmanager_secret_version.github_token.secret_string +# } module "eks_deployment" { source = "../../" diff --git a/examples/basic/providers.tf b/examples/basic/providers.tf new file mode 100644 index 0000000..8c9ebd1 --- /dev/null +++ b/examples/basic/providers.tf @@ -0,0 +1,2 @@ +provider "aws" { +}