From ce9c5037645d04345310e6cecb48bdaa44ab38b5 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Fri, 4 Apr 2025 19:15:23 -0400 Subject: [PATCH 01/16] Add comprehensive EKS Cluster Template roadmap and configuration files --- ROADMAP.md | 59 ++++++++++ locals.tf | 144 ++++++++++++++++++++++++ main.tf | 28 +++++ templates/common-variables.hcl | 10 ++ templates/default-versions.hcl | 124 +++++++++++++++++++++ variables.tf | 194 +++++++++++++++++++++++++++++++++ versions.tf | 7 ++ 7 files changed, 566 insertions(+) create mode 100644 ROADMAP.md create mode 100644 locals.tf create mode 100644 main.tf create mode 100644 templates/common-variables.hcl create mode 100644 templates/default-versions.hcl create mode 100644 variables.tf create mode 100644 versions.tf diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..f7160db --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,59 @@ +# EKS Cluster Template Roadmap + +## Current Architecture +- Template repository (`template-eks-cluster`) serves as the base for creating new EKS cluster configurations +- Uses Terraform GitHub repo module to create new repositories from the template +- Implements managed and non-managed extra files functionality +- Supports configuration through `config.json` + +## Planned Enhancements + +### 1. Automated Cluster Setup (High Priority) +- [ ] Add GitHub Actions workflows with workflow_dispatch triggers +- [ ] Implement automated terragrunt command execution for cluster building +- [ ] Configure workflows to run on specific runners for credential management +- [ ] Create templatized GitHub Actions workflow files +- [ ] Enable direct cluster creation without manual repository cloning + +### 2. File Management System (Medium Priority) +- [ ] Implement wrapper module for repo module +- [ ] Add support for crafting and injecting various configuration files +- [ ] Define file lifecycle management strategy + - [ ] Managed files (controlled by workspace) + - [ ] Non-managed files (user-modifiable) + +### 3. Version Management (Medium Priority) +- [ ] Implement version control strategy for `default-versions.hcl` +- [ ] Create system for managing platform release versions +- [ ] Set up version override mechanism + - [ ] Default versions in template repo + - [ ] Override capability in workspace creating repos + +### 4. Configuration Management (Low Priority) +- [ ] Enhance Makefile and Ansible playbook integration +- [ ] Improve configuration file templating +- [ ] Add validation for configuration files + +## Technical Considerations +1. File Lifecycle Management: + - Managed files: Controlled by workspace + - Non-managed files: User-modifiable post-creation + - Version-specific files: Platform release coordination + +2. Automation Requirements: + - GitHub Actions runner configuration + - Credential management + - Workflow templating + - Terragrunt integration + +3. Version Control Strategy: + - Module version collections + - Platform release versions + - Override mechanisms + +## Success Criteria +- Fully automated cluster creation process +- Minimal manual intervention required +- Proper version management system +- Clear file lifecycle management +- Secure credential handling \ No newline at end of file diff --git a/locals.tf b/locals.tf new file mode 100644 index 0000000..acf38af --- /dev/null +++ b/locals.tf @@ -0,0 +1,144 @@ +locals { + common_vars = merge({ + organization = "census:ocio:csvd" + project_name = "csvd_platformbaseline" + project_number = "fs0000000078" + project_role = "csvd_platformbaseline_app" + state_bucket_prefix = "inf-tfstate" + state_table_name = "tf_remote_state" + route53_endpoints = {} + }, var.common_variables) + + all_namespaces = merge({ + grafana = local.namespaces.telemetry_namespace + k8s-dashboard = local.namespaces.telemetry_namespace + loki = local.namespaces.telemetry_namespace + otel = local.namespaces.telemetry_namespace + prometheus = local.namespaces.telemetry_namespace + tempo = local.namespaces.telemetry_namespace + }, var.namespaces.custom_namespaces) + + namespaces = { + operator_namespace = var.namespaces.operator_namespace + telemetry_namespace = var.namespaces.telemetry_namespace + namespaces = local.all_namespaces + } + + default_versions = { + cluster_version = var.versions.cluster_version + custom_service_eks_account = var.versions.release_version + eks_module_version = var.versions.eks_module_version + istio_ingress_version = var.versions.release_version + release_version = var.versions.release_version + + # Provider versions + aws_version = var.versions.aws_version + helm_version = var.versions.helm_version + kubernetes_version = var.versions.kubernetes_version + null_version = var.versions.null_version + random_version = var.versions.random_version + template_version = var.versions.template_version + tf_version = var.versions.tf_version + + # Component versions + cert_manager_version = var.versions.cert_manager.version + cert_manager_helm_chart = var.versions.cert_manager.chart_version + cluster_issuer_name = var.versions.cert_manager.cluster_issuer_name + + gogatekeeper_tag = var.versions.gogatekeeper.tag + gogatekeeper_chart_version = var.versions.gogatekeeper.chart_version + + grafana_hostname = var.versions.grafana.hostname + grafana_operator_chart_version = var.versions.grafana.operator_chart_version + grafana_operator_tag = var.versions.grafana.operator_tag + grafana_tag = var.versions.grafana.tag + os_shell_image_tag = var.versions.grafana.os_shell_image_tag + + istio_version = var.versions.istio.version + istio_namespace = var.versions.istio.namespace + + dashboard_hostname = var.versions.k8s_dashboard.hostname + k8s_dashboard_metrics_scraper = var.versions.k8s_dashboard.metrics_scraper + k8s_dashboard_version = var.versions.k8s_dashboard.version + + karpenter_helm_chart = var.versions.karpenter.helm_chart + karpenter_tag = var.versions.karpenter.tag + + keycloak_chart_version = var.versions.keycloak.chart_version + keycloak_tag = var.versions.keycloak.tag + keycloak_hostname = var.versions.keycloak.hostname + keycloak_database = var.versions.keycloak.database + keycloak_username = var.versions.keycloak.username + keycloak_password = var.versions.keycloak.password + postgresql_tag = var.versions.keycloak.postgresql_tag + + kiali_operator_version = var.versions.kiali.operator_version + kiali_application_version = "v${var.versions.kiali.operator_version}" + + loki_chart_version = var.versions.loki.chart_version + loki_tag = var.versions.loki.tag + enterprise_logs_provisioner_tag = var.versions.loki.enterprise_logs_provisioner_tag + gateway_tag = var.versions.loki.gateway_tag + memcached_tag = var.versions.loki.memcached_tag + exporter_tag = var.versions.loki.exporter_tag + sidecar_tag = var.versions.loki.sidecar_tag + + metrics_server_helm_chart = var.versions.metrics_server.helm_chart + metrics_server_tag = var.versions.metrics_server.tag + + prometheus_chart_version = var.versions.prometheus.chart_version + prometheus_server_tag = var.versions.prometheus.server_tag + prometheus_config_reloader_tag = var.versions.prometheus.config_reloader_tag + alertmanager_tag = var.versions.prometheus.alertmanager_tag + kube_state_metrics_tag = var.versions.prometheus.kube_state_metrics_tag + node_exporter_tag = var.versions.prometheus.node_exporter_tag + pushgateway_tag = var.versions.prometheus.pushgateway_tag + + tempo_chart_version = var.versions.tempo.chart_version + tempo_tag = var.versions.tempo.tag + } + + config_json = jsonencode({ + environment = var.environment + region = var.region + cluster_dir = "platform-cluster" + enable_all_modules = false + account = { + account_name = var.cluster_config.account_name + aws_account_id = var.cluster_config.aws_account_id + aws_profile = var.cluster_config.aws_profile + environment_abbr = var.cluster_config.environment_abbr + } + vpc = { + vpc_name = var.cluster_config.vpc_name + vpc_domain_name = var.cluster_config.vpc_domain_name + } + cluster = { + cluster_name = var.cluster_config.cluster_name + cluster_mailing_list = var.cluster_config.cluster_mailing_list + eks_instance_disk_size = var.cluster_config.eks_instance_disk_size + eks_ng_desired_size = var.cluster_config.eks_ng_desired_size + eks_ng_max_size = var.cluster_config.eks_ng_max_size + eks_ng_min_size = var.cluster_config.eks_ng_min_size + enable_cluster_creator_admin_permissions = var.cluster_config.enable_cluster_creator_admin_permissions + tags = var.cluster_config.tags + } + modules = var.enable_modules + }) + + managed_extra_files = concat([ + { + path = "config.json" + content = local.config_json + }, + { + path = "_envcommon/default-versions.hcl" + content = templatefile("${path.module}/templates/default-versions.hcl", local.default_versions) + }, + { + path = "_envcommon/common-variables.hcl" + content = templatefile("${path.module}/templates/common-variables.hcl", local.common_vars) + } + ], + var.github_actions_workflows) +} \ No newline at end of file diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..380a7c6 --- /dev/null +++ b/main.tf @@ -0,0 +1,28 @@ +module "github_repo" { + source = "HappyPathway/repo/github" + + name = var.name + repo_org = var.organization + github_repo_description = "EKS Cluster Configuration for ${var.cluster_config.cluster_name}" + github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] + + template_repo = "template-eks-cluster" + template_repo_org = var.template_repo_org + + github_is_private = true + github_has_issues = true + github_has_wiki = true + github_has_projects = true + + managed_extra_files = local.managed_extra_files +} + +output "repository_url" { + description = "URL of the created repository" + value = module.github_repo.html_url +} + +output "ssh_clone_url" { + description = "SSH clone URL of the repository" + value = module.github_repo.ssh_clone_url +} \ No newline at end of file diff --git a/templates/common-variables.hcl b/templates/common-variables.hcl new file mode 100644 index 0000000..e228659 --- /dev/null +++ b/templates/common-variables.hcl @@ -0,0 +1,10 @@ +locals { + organization = "${organization}" + project_name = "${project_name}" + project_number = "${project_number}" + project_role = "${project_role}" + state_bucket_prefix = "${state_bucket_prefix}" + state_table_name = "${state_table_name}" + + route53_endpoints = ${jsonencode(route53_endpoints)} +} \ No newline at end of file diff --git a/templates/default-versions.hcl b/templates/default-versions.hcl new file mode 100644 index 0000000..60f3cd4 --- /dev/null +++ b/templates/default-versions.hcl @@ -0,0 +1,124 @@ +locals { + ##################### + # Module Versions + ##################### + cluster_version = "${cluster_version}" + custom_service_eks_account = "${custom_service_eks_account}" + eks_module_version = "${eks_module_version}" + istio_ingress_version = "${istio_ingress_version}" + release_version = "${release_version}" + + ##################### + # TF Providers + ##################### + aws_version = "${aws_version}" + helm_version = "${helm_version}" + kubernetes_version = "${kubernetes_version}" + null_version = "${null_version}" + random_version = "${random_version}" + template_version = "${template_version}" + tf_version = "${tf_version}" + + ##################### + # Component Versions + ##################### + + ################ + # Cert-Manager + ################ + cluster_issuer_name = "${cluster_issuer_name}" + cert_manager_version = "${cert_manager_version}" + cert_manager_helm_chart = "${cert_manager_helm_chart}" + + ################ + # GoGatekeeper + ################ + gogatekeeper_tag = "${gogatekeeper_tag}" + gogatekeeper_chart_version = "${gogatekeeper_chart_version}" + + ################ + # Grafana + ################ + grafana_hostname = "${grafana_hostname}" + grafana_operator_chart_version = "${grafana_operator_chart_version}" + grafana_operator_tag = "${grafana_operator_tag}" + grafana_tag = "${grafana_tag}" + os_shell_image_tag = "${os_shell_image_tag}" + + ################ + # Istio + ################ + istio_namespace = "${istio_namespace}" + istio_version = "${istio_version}" + + ################ + # k8s-dashboard + ################ + dashboard_hostname = "${dashboard_hostname}" + k8s_dashboard_metrics_scraper = "${k8s_dashboard_metrics_scraper}" + k8s_dashboard_version = "${k8s_dashboard_version}" + + ################ + # Karpenter + ################ + karpenter_helm_chart = "${karpenter_helm_chart}" + karpenter_tag = "${karpenter_tag}" + + ################ + # Keycloak + ################ + keycloak_chart_version = "${keycloak_chart_version}" + keycloak_tag = "${keycloak_tag}" + keycloak_hostname = "${keycloak_hostname}" + keycloak_database = "${keycloak_database}" + keycloak_username = "${keycloak_username}" + keycloak_password = "${keycloak_password}" + postgresql_tag = "${postgresql_tag}" + + ################ + # Kiali + ################ + kiali_operator_version = "${kiali_operator_version}" + kiali_application_version = "${kiali_application_version}" + + ################ + # Loki + ################ + loki_chart_version = "${loki_chart_version}" + loki_tag = "${loki_tag}" + enterprise_logs_provisioner_tag = "${enterprise_logs_provisioner_tag}" + gateway_tag = "${gateway_tag}" + memcached_tag = "${memcached_tag}" + exporter_tag = "${exporter_tag}" + sidecar_tag = "${sidecar_tag}" + + ################ + # Metrics Server + ################ + metrics_server_helm_chart = "${metrics_server_helm_chart}" + metrics_server_tag = "${metrics_server_tag}" + + ################ + # Prometheus + ################ + prometheus_chart_version = "${prometheus_chart_version}" + prometheus_server_tag = "${prometheus_server_tag}" + prometheus_config_reloader_tag = "${prometheus_config_reloader_tag}" + alertmanager_tag = "${alertmanager_tag}" + kube_state_metrics_tag = "${kube_state_metrics_tag}" + node_exporter_tag = "${node_exporter_tag}" + pushgateway_tag = "${pushgateway_tag}" + + ################ + # Tempo + ################ + tempo_chart_version = "${tempo_chart_version}" + tempo_tag = "${tempo_tag}" + + ##################### + # Namespaces Config + ##################### + operator_namespace = "${operator_namespace}" + telemetry_namespace = "${telemetry_namespace}" + namespaces = ${jsonencode(namespaces)} +} \ No newline at end of file diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..fd4000d --- /dev/null +++ b/variables.tf @@ -0,0 +1,194 @@ +variable "name" { + description = "Name of the repository" + type = string +} + +variable "organization" { + description = "GitHub organization name" + type = string +} + +variable template_repo_org { + description = "GitHub organization for the template repository" + type = string +} + +variable "environment" { + description = "Environment name (e.g., production, staging)" + type = string +} + +variable "region" { + description = "AWS region for the EKS cluster" + type = string +} + +variable "common_variables" { + description = "Common variables across all environments" + type = object({ + organization = optional(string, "census:ocio:csvd") + project_name = optional(string, "csvd_platformbaseline") + project_number = optional(string, "fs0000000078") + project_role = optional(string, "csvd_platformbaseline_app") + state_bucket_prefix = optional(string, "inf-tfstate") + state_table_name = optional(string, "tf_remote_state") + route53_endpoints = optional(map(object({ + account_id = string + alias = string + us-gov-east-1 = string + us-gov-west-1 = string + })), {}) + }) + default = {} +} + +variable "versions" { + description = "Version configurations for various components" + type = object({ + # Module Versions + cluster_version = optional(string, "1.31") + eks_module_version = optional(string, "20.33.1") + release_version = optional(string, "main") + + # TF Providers + aws_version = optional(string, "5.84.0") + helm_version = optional(string, "2.11.0") + kubernetes_version = optional(string, "2.33.0") + null_version = optional(string, "3.2.1") + random_version = optional(string, "3.5.1") + template_version = optional(string, "2.2.0") + tf_version = optional(string, "1.5.5") + + # Component Versions + cert_manager = optional(object({ + version = optional(string, "1.17.1") + chart_version = optional(string, "1.17.1") + cluster_issuer_name = optional(string, "cert-manager") + }), {}) + + gogatekeeper = optional(object({ + tag = optional(string, "3.2.1") + chart_version = optional(string, "0.1.53") + }), {}) + + grafana = optional(object({ + hostname = optional(string, "grafana") + operator_chart_version = optional(string, "4.9.8") + operator_tag = optional(string, "5.16.0") + tag = optional(string, "11.5.2") + os_shell_image_tag = optional(string, "12") + }), {}) + + istio = optional(object({ + version = optional(string, "1.25.0") + namespace = optional(string, "istio-system") + }), {}) + + k8s_dashboard = optional(object({ + hostname = optional(string, "dashboard") + metrics_scraper = optional(string, "1.0.8") + version = optional(string, "6.0.6") + }), {}) + + karpenter = optional(object({ + helm_chart = optional(string, "1.3.1") + tag = optional(string, "1.3.1") + }), {}) + + keycloak = optional(object({ + chart_version = optional(string, "24.4.11") + tag = optional(string, "26.1.3") + hostname = optional(string, "keycloak") + database = optional(string, "keycloak") + username = optional(string, "keycloak") + password = optional(string, "this is my very secure and totally random password horse battery staple now") + postgresql_tag = optional(string, "17.4.0-debian-12-r2") + }), {}) + + kiali = optional(object({ + operator_version = optional(string, "2.2.0") + }), {}) + + loki = optional(object({ + chart_version = optional(string, "6.27.0") + tag = optional(string, "3.4.2") + enterprise_logs_provisioner_tag = optional(string, "v1.7.0") + gateway_tag = optional(string, "1.27-alpine") + memcached_tag = optional(string, "1.6.37") + exporter_tag = optional(string, "v0.15.0") + sidecar_tag = optional(string, "1.27.4") + }), {}) + + metrics_server = optional(object({ + helm_chart = optional(string, "3.12.2") + tag = optional(string, "0.7.2") + }), {}) + + prometheus = optional(object({ + chart_version = optional(string, "27.5.1") + server_tag = optional(string, "v3.2.1") + config_reloader_tag = optional(string, "v0.75.2") + alertmanager_tag = optional(string, "v0.28.0") + kube_state_metrics_tag = optional(string, "v2.15.0") + node_exporter_tag = optional(string, "v1.9.0") + pushgateway_tag = optional(string, "v1.11.0") + }), {}) + + tempo = optional(object({ + chart_version = optional(string, "1.18.2") + tag = optional(string, "2.7.1") + }), {}) + }) + default = {} +} + +variable "namespaces" { + description = "Namespace configurations" + type = object({ + operator_namespace = optional(string, "aoperator") + telemetry_namespace = optional(string, "atelemetry") + custom_namespaces = optional(map(string), { + cert-manager = "kube-system" + karpenter = "karpenter" + metrics-server = "kube-system" + postgresql = "kube-system" + keycloak = "keycloak" + gogatekeeper = "kube-system" + istio = "istio-system" + kiali = "istio-system" + }) + }) + default = {} +} + +variable "cluster_config" { + description = "Configuration for the EKS cluster" + type = object({ + cluster_name = string + account_name = string + aws_account_id = string + aws_profile = string + environment_abbr = string + vpc_name = string + vpc_domain_name = string + cluster_mailing_list = optional(string) + eks_instance_disk_size = optional(number, 200) + eks_ng_desired_size = optional(number, 3) + eks_ng_max_size = optional(number, 10) + eks_ng_min_size = optional(number, 3) + enable_cluster_creator_admin_permissions = optional(bool, true) + tags = optional(map(string), {}) + }) +} + +variable "enable_modules" { + description = "Map of modules to enable" + type = object({ + gogatekeeper = optional(bool, false) + cert_manager = optional(bool, false) + prometheus = optional(bool, false) + grafana = optional(bool, false) + istio = optional(bool, false) + }) + default = {} +} \ No newline at end of file diff --git a/versions.tf b/versions.tf new file mode 100644 index 0000000..fbe53be --- /dev/null +++ b/versions.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + github = { + source = "integrations/github" + } + } +} From 737c5f91400574e50e2c24ba5db91334de871da8 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Fri, 4 Apr 2025 19:19:19 -0400 Subject: [PATCH 02/16] Refactor namespace definitions and add GitHub Actions workflows variable --- locals.tf | 33 +++++++++++++++++++++++++-------- variables.tf | 9 +++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/locals.tf b/locals.tf index acf38af..333ceb6 100644 --- a/locals.tf +++ b/locals.tf @@ -9,14 +9,31 @@ locals { route53_endpoints = {} }, var.common_variables) - all_namespaces = merge({ - grafana = local.namespaces.telemetry_namespace - k8s-dashboard = local.namespaces.telemetry_namespace - loki = local.namespaces.telemetry_namespace - otel = local.namespaces.telemetry_namespace - prometheus = local.namespaces.telemetry_namespace - tempo = local.namespaces.telemetry_namespace - }, var.namespaces.custom_namespaces) + # First define base namespaces without dependencies + base_namespaces = { + cert-manager = "kube-system" + karpenter = "karpenter" + metrics-server = "kube-system" + postgresql = "kube-system" + keycloak = "keycloak" + gogatekeeper = "kube-system" + istio = "istio-system" + kiali = "istio-system" + } + + # Then merge with telemetry namespaces + all_namespaces = merge( + local.base_namespaces, + { + grafana = var.namespaces.telemetry_namespace + k8s-dashboard = var.namespaces.telemetry_namespace + loki = var.namespaces.telemetry_namespace + otel = var.namespaces.telemetry_namespace + prometheus = var.namespaces.telemetry_namespace + tempo = var.namespaces.telemetry_namespace + }, + var.namespaces.custom_namespaces + ) namespaces = { operator_namespace = var.namespaces.operator_namespace diff --git a/variables.tf b/variables.tf index fd4000d..6924186 100644 --- a/variables.tf +++ b/variables.tf @@ -191,4 +191,13 @@ variable "enable_modules" { istio = optional(bool, false) }) default = {} +} + +variable "github_actions_workflows" { + description = "List of GitHub Actions workflow files to add to the repository" + type = list(object({ + path = string + content = string + })) + default = [] } \ No newline at end of file From a3e44769cbc6ae7f61dda5c2e110d2328ee4c7f5 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Tue, 8 Apr 2025 14:04:41 -0400 Subject: [PATCH 03/16] Implement GitHub Actions workflow triggers and enhance Terraform configurations for automated cluster setup --- ROADMAP.md | 7 +++-- locals.tf | 5 ++++ main.tf | 18 +++++++++++ providers.tf | 14 +++++++++ scripts/trigger_workflow.py | 59 +++++++++++++++++++++++++++++++++++++ variables.tf | 12 ++++++++ 6 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 providers.tf create mode 100755 scripts/trigger_workflow.py diff --git a/ROADMAP.md b/ROADMAP.md index f7160db..12f698f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -9,8 +9,11 @@ ## Planned Enhancements ### 1. Automated Cluster Setup (High Priority) -- [ ] Add GitHub Actions workflows with workflow_dispatch triggers -- [ ] Implement automated terragrunt command execution for cluster building +- [x] Add GitHub Actions workflows with workflow_dispatch triggers +- [x] Implement automated terragrunt command execution for cluster building + - [x] Support for plan/apply/destroy commands + - [x] Environment-specific execution + - [x] Automated testing framework - [ ] Configure workflows to run on specific runners for credential management - [ ] Create templatized GitHub Actions workflow files - [ ] Enable direct cluster creation without manual repository cloning diff --git a/locals.tf b/locals.tf index 333ceb6..30494ed 100644 --- a/locals.tf +++ b/locals.tf @@ -113,6 +113,11 @@ locals { tempo_chart_version = var.versions.tempo.chart_version tempo_tag = var.versions.tempo.tag + + # Add namespace configurations + operator_namespace = var.namespaces.operator_namespace + telemetry_namespace = var.namespaces.telemetry_namespace + namespaces = local.all_namespaces } config_json = jsonencode({ diff --git a/main.tf b/main.tf index 380a7c6..d168751 100644 --- a/main.tf +++ b/main.tf @@ -17,6 +17,24 @@ module "github_repo" { managed_extra_files = local.managed_extra_files } +resource "null_resource" "trigger_workflow" { + triggers = { + repository_name = module.github_repo.github_repo.name + } + + provisioner "local-exec" { + command = "python3 scripts/trigger_workflow.py ${module.github_repo.github_repo.name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_dir}\",\"auto_approve\":true}'" + + environment = { + GITHUB_TOKEN = var.github_token + GITHUB_OWNER = var.organization + GITHUB_SERVER_URL = var.github_server_url + } + } + + depends_on = [module.github_repo] +} + output "repository_url" { description = "URL of the created repository" value = module.github_repo.html_url diff --git a/providers.tf b/providers.tf new file mode 100644 index 0000000..e01c942 --- /dev/null +++ b/providers.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + github = { + source = "integrations/github" + version = ">= 5.0" + } + } +} + +provider "github" { + # Configuration is expected from environment variables: + # GITHUB_TOKEN + # GITHUB_OWNER (optional) +} \ No newline at end of file diff --git a/scripts/trigger_workflow.py b/scripts/trigger_workflow.py new file mode 100755 index 0000000..f5b1fb2 --- /dev/null +++ b/scripts/trigger_workflow.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +import os +import sys +import requests +import json + +def trigger_workflow(repo, event_type, payload=None): + token = os.environ.get('GITHUB_TOKEN') + if not token: + print("Error: GITHUB_TOKEN environment variable not set") + sys.exit(1) + + owner = os.environ.get('GITHUB_OWNER', 'default-org') + server_url = os.environ.get('GITHUB_SERVER_URL', 'https://api.github.com') + + # Remove trailing slash if present and ensure we're using the API endpoint + server_url = server_url.rstrip('/') + if not server_url.endswith('/api/v3') and not 'api.github.com' in server_url: + server_url = f"{server_url}/api/v3" + + url = f"{server_url}/repos/{owner}/{repo}/dispatches" + + headers = { + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'token {token}', + 'Content-Type': 'application/json', + } + + data = { + 'event_type': event_type, + 'client_payload': payload or {} + } + + response = requests.post(url, headers=headers, data=json.dumps(data), verify=True) + + if response.status_code == 204: + print(f"Successfully triggered workflow {event_type} for {owner}/{repo}") + return True + else: + print(f"Failed to trigger workflow: {response.status_code}") + print(response.text) + return False + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: trigger_workflow.py []") + sys.exit(1) + + repo = sys.argv[1] + event_type = sys.argv[2] + payload = json.loads(sys.argv[3]) if len(sys.argv) > 3 else None + + # First trigger requirements installation + if not trigger_workflow(repo, "install-requirements"): + sys.exit(1) + + # Then trigger the main workflow + if not trigger_workflow(repo, event_type, payload): + sys.exit(1) \ No newline at end of file diff --git a/variables.tf b/variables.tf index 6924186..2a6534f 100644 --- a/variables.tf +++ b/variables.tf @@ -200,4 +200,16 @@ variable "github_actions_workflows" { content = string })) default = [] +} + +variable "github_token" { + description = "GitHub token for triggering workflows" + type = string + sensitive = true +} + +variable "github_server_url" { + description = "GitHub Enterprise server URL (e.g., https://github.mycompany.com)" + type = string + default = "https://api.github.com" } \ No newline at end of file From c2fd5b396a91d56cc1027c9d9815383e07bdde2b Mon Sep 17 00:00:00 2001 From: arnol377 Date: Tue, 8 Apr 2025 15:13:38 -0400 Subject: [PATCH 04/16] Enhance README and add Terraform module for EKS cluster deployment with automated GitHub Actions workflows; include tests for workflow triggers and validation. --- .github/workflows/test.yml | 45 ++++++ README.md | 163 +++++++++++++++++++++ examples/basic/basic.tftest.hcl | 47 ++++++ examples/basic/main.tf | 39 +++++ examples/basic/validation.tftest.hcl | 50 +++++++ examples/basic/variables.tf | 5 + examples/basic/workflow_trigger.tftest.hcl | 32 ++++ scripts/test_trigger_workflow.py | 46 ++++++ variables.tf | 2 +- 9 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test.yml create mode 100644 examples/basic/basic.tftest.hcl create mode 100644 examples/basic/main.tf create mode 100644 examples/basic/validation.tftest.hcl create mode 100644 examples/basic/variables.tf create mode 100644 examples/basic/workflow_trigger.tftest.hcl create mode 100644 scripts/test_trigger_workflow.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..7d030cf --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,45 @@ +name: Tests + +on: + pull_request: + branches: [ main ] + push: + branches: [ main ] + +jobs: + terraform-test: + name: Terraform Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.7.0" + + - name: Run Terraform Tests + working-directory: ./examples/basic + run: terraform test + env: + TF_VAR_github_token: mock-token + + python-test: + name: Python Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest requests + + - name: Run Python Tests + working-directory: ./scripts + run: pytest -v test_trigger_workflow.py \ No newline at end of file diff --git a/README.md b/README.md index 4456e7d..0bac3a8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,165 @@ # terraform-eks-deployment + Terraform module for EKS cluster deployment and configuration + +## Overview + +This module creates a new GitHub repository for your EKS cluster based on the template-eks-cluster repository. It sets up all necessary configuration files and triggers automated workflows for cluster deployment. + +## Prerequisites + +- GitHub token with repository and workflow permissions +- Python 3.x installed on the machine running Terraform +- Access to GitHub Enterprise (if using enterprise version) + +## Usage + +```hcl +module "eks_deployment" { + source = "path/to/terraform-eks-deployment" + + name = "my-eks-cluster" + organization = "my-org" + environment = "production" + region = "us-east-1" + + cluster_config = { + cluster_name = "prod-eks-01" + account_name = "prod-account" + aws_account_id = "123456789012" + aws_profile = "prod-profile" + environment_abbr = "prod" + vpc_name = "prod-vpc" + vpc_domain_name = "prod.example.com" + } + + github_token = "your-github-token" + github_server_url = "https://github.mycompany.com" # Optional, for GitHub Enterprise +} +``` + +## Workflow Automation + +### Overview + +The module automatically triggers GitHub Actions workflows in your newly created repository to: +1. Install Python requirements +2. Execute Terragrunt operations for cluster management + +### Workflow Sequence + +1. **Repository Creation**: The module creates a new repository from the template-eks-cluster template +2. **Initial Configuration**: Configuration files are generated based on your inputs +3. **Requirements Installation**: A workflow is triggered to install Python dependencies +4. **Cluster Planning**: A terragrunt plan workflow is automatically triggered + +### Available Workflows + +Your new repository will have these workflows available: + +1. **Install Requirements** (`install-requirements.yml`) + - Triggered automatically on repository creation + - Installs all Python dependencies from requirements.txt + +2. **Terragrunt Cluster Operations** (`terragrunt-cluster-build.yml`) + - Supports plan, apply, and destroy operations + - Can be triggered manually or via API + - Includes safety checks and approvals + +### Triggering Workflows + +The workflows can be triggered in two ways: + +1. **Automatic Triggering** + - On repository creation, the module automatically triggers: + 1. Requirements installation + 2. Initial cluster plan + +2. **Manual Triggering** + - Via GitHub UI: + 1. Go to Actions tab + 2. Select desired workflow + 3. Click "Run workflow" + 4. Fill in parameters + +3. **API Triggering** + - Use GitHub's API to trigger workflows: + ```bash + curl -X POST \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/OWNER/REPO/dispatches" \ + -d '{ + "event_type": "cluster-plan", + "client_payload": { + "environment": "dev", + "region": "us-east-1", + "cluster_dir": "my-cluster", + "auto_approve": false + } + }' + ``` + +### Supported Events + +The following event types are supported for workflow triggers: + +- `install-requirements`: Install Python dependencies +- `cluster-plan`: Preview cluster changes +- `cluster-apply`: Apply cluster changes +- `cluster-destroy`: Destroy cluster + +### Required Secrets + +The following secrets must be configured in your repository: + +- `AWS_ROLE_ARN`: ARN of the AWS role to assume +- `GITHUB_TOKEN`: GitHub token with workflow permissions + +## Module Configuration + +### Required Variables + +- `name`: Repository name +- `organization`: GitHub organization name +- `environment`: Deployment environment +- `region`: AWS region +- `cluster_config`: Cluster configuration object +- `github_token`: GitHub token for workflow operations + +### Optional Variables + +- `github_server_url`: GitHub Enterprise server URL +- `template_repo_org`: Organization containing the template repository +- `enable_modules`: Map of modules to enable in the cluster + +For more configuration options, see the variables.tf file. + +## Outputs + +- `repository_url`: URL of the created repository +- `ssh_clone_url`: SSH clone URL of the repository + +## Security Considerations + +1. **GitHub Token**: Use a token with minimal required permissions +2. **AWS Role**: Use role-based access with least privilege +3. **Auto-approve**: Use with caution in production environments +4. **Environment Protection**: Configure branch protection rules + +## Troubleshooting + +Common issues and solutions: + +1. **Workflow Trigger Failures** + - Check GitHub token permissions + - Verify GitHub Enterprise URL (if applicable) + - Check network connectivity + +2. **Python Requirements** + - Ensure requirements.txt exists in template repository + - Check Python version compatibility + +3. **AWS Authentication** + - Verify AWS role ARN + - Check AWS credentials configuration diff --git a/examples/basic/basic.tftest.hcl b/examples/basic/basic.tftest.hcl new file mode 100644 index 0000000..48bd466 --- /dev/null +++ b/examples/basic/basic.tftest.hcl @@ -0,0 +1,47 @@ +variables { + github_token = "mock-token" +} + +provider "github" { + owner = "my-org" + token = "mock-token" + base_url = "https://github.mycompany.com/api/v3" +} + +run "verify_repository_config" { + command = plan + + assert { + condition = module.eks_deployment.github_repo.name == "eks-test-cluster" + error_message = "Repository name does not match expected value" + } + + assert { + condition = module.eks_deployment.github_repo.description == "EKS Cluster Configuration for dev-eks-01" + error_message = "Repository description does not match expected value" + } + + assert { + condition = module.eks_deployment.github_repo.visibility == "private" + error_message = "Repository visibility should be private" + } +} + +run "verify_module_defaults" { + command = plan + + assert { + condition = length(module.eks_deployment.enable_modules.*.cert_manager) > 0 + error_message = "cert-manager module should be enabled" + } + + assert { + condition = length(module.eks_deployment.enable_modules.*.prometheus) > 0 + error_message = "prometheus module should be enabled" + } + + assert { + condition = length(module.eks_deployment.enable_modules.*.grafana) > 0 + error_message = "grafana module should be enabled" + } +} \ No newline at end of file diff --git a/examples/basic/main.tf b/examples/basic/main.tf new file mode 100644 index 0000000..44645ff --- /dev/null +++ b/examples/basic/main.tf @@ -0,0 +1,39 @@ +provider "github" { + # Configuration expected from environment variables: + # GITHUB_TOKEN + # GITHUB_OWNER (optional) +} + +module "eks_deployment" { + source = "../../" + + name = "eks-test-cluster" + organization = "my-org" + environment = "dev" + region = "us-east-1" + + template_repo_org = "my-org" + github_token = var.github_token + github_server_url = "https://github.mycompany.com" + + cluster_config = { + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } + + enable_modules = { + cert_manager = true + prometheus = true + grafana = true + } + + versions = { + cluster_version = "1.27" + eks_module_version = "20.33.1" + } +} \ No newline at end of file diff --git a/examples/basic/validation.tftest.hcl b/examples/basic/validation.tftest.hcl new file mode 100644 index 0000000..3339064 --- /dev/null +++ b/examples/basic/validation.tftest.hcl @@ -0,0 +1,50 @@ +variables { + github_token = "mock-token" +} + +provider "github" { + owner = "my-org" + token = "mock-token" + base_url = "https://github.mycompany.com/api/v3" +} + +# Test invalid cluster version +run "invalid_cluster_version" { + command = plan + + variables { + versions = { + cluster_version = "1.26" # Test outdated version + } + } + + expect_failures = [ + var.versions + ] +} + +# Test missing required variables +run "missing_required_vars" { + command = plan + + variables { + name = null # Required variable + } + + expect_failures = [ + var.name + ] +} + +# Test invalid environment name +run "invalid_environment" { + command = plan + + variables { + environment = "invalid" # Should be dev, staging, or prod + } + + expect_failures = [ + var.environment + ] +} \ No newline at end of file diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf new file mode 100644 index 0000000..73092ff --- /dev/null +++ b/examples/basic/variables.tf @@ -0,0 +1,5 @@ +variable "github_token" { + description = "GitHub token for API operations" + type = string + sensitive = true +} \ No newline at end of file diff --git a/examples/basic/workflow_trigger.tftest.hcl b/examples/basic/workflow_trigger.tftest.hcl new file mode 100644 index 0000000..501393d --- /dev/null +++ b/examples/basic/workflow_trigger.tftest.hcl @@ -0,0 +1,32 @@ +variables { + github_token = "mock-token" +} + +mock_provider "github" { + mock_resource "github_repository_dispatch" { + defaults = { + result = { + status_code = 204 + } + } + + assert "verify_workflow_payload" { + condition = self.input.event_type == "cluster-plan" && + self.input.client_payload.environment == "dev" && + self.input.client_payload.region == "us-east-1" && + self.input.client_payload.cluster_dir == "dev-eks-01" + error_message = "Invalid workflow dispatch payload" + } + } +} + +run "verify_workflow_trigger" { + command = plan + + # This test verifies that the workflow trigger mechanism works + # without actually making API calls + assert { + condition = true + error_message = "Workflow trigger test failed" + } +} \ No newline at end of file diff --git a/scripts/test_trigger_workflow.py b/scripts/test_trigger_workflow.py new file mode 100644 index 0000000..ae786fd --- /dev/null +++ b/scripts/test_trigger_workflow.py @@ -0,0 +1,46 @@ +import os +import pytest +from unittest.mock import patch, MagicMock +from trigger_workflow import trigger_workflow + +def test_trigger_workflow_success(): + with patch('requests.post') as mock_post: + mock_response = MagicMock() + mock_response.status_code = 204 + mock_post.return_value = mock_response + + os.environ['GITHUB_TOKEN'] = 'test-token' + os.environ['GITHUB_OWNER'] = 'test-org' + os.environ['GITHUB_SERVER_URL'] = 'https://github.example.com' + + result = trigger_workflow('test-repo', 'cluster-plan', { + 'environment': 'dev', + 'region': 'us-east-1', + 'cluster_dir': 'test-cluster', + 'auto_approve': False + }) + + assert result is True + mock_post.assert_called_once() + +def test_trigger_workflow_failure(): + with patch('requests.post') as mock_post: + mock_response = MagicMock() + mock_response.status_code = 401 + mock_response.text = 'Unauthorized' + mock_post.return_value = mock_response + + os.environ['GITHUB_TOKEN'] = 'invalid-token' + os.environ['GITHUB_OWNER'] = 'test-org' + + result = trigger_workflow('test-repo', 'cluster-plan', {}) + + assert result is False + mock_post.assert_called_once() + +def test_missing_token(): + if 'GITHUB_TOKEN' in os.environ: + del os.environ['GITHUB_TOKEN'] + + with pytest.raises(SystemExit): + trigger_workflow('test-repo', 'cluster-plan', {}) \ No newline at end of file diff --git a/variables.tf b/variables.tf index 2a6534f..4afa3c5 100644 --- a/variables.tf +++ b/variables.tf @@ -8,7 +8,7 @@ variable "organization" { type = string } -variable template_repo_org { +variable "template_repo_org" { description = "GitHub organization for the template repository" type = string } From 113970468d0b90f1405c5188b74719c2fff09e7e Mon Sep 17 00:00:00 2001 From: arnol377 Date: Thu, 10 Apr 2025 13:37:16 -0400 Subject: [PATCH 05/16] Update versions.tf to include required provider versions for Terraform configuration --- versions.tf | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/versions.tf b/versions.tf index fbe53be..920cfb3 100644 --- a/versions.tf +++ b/versions.tf @@ -1,7 +1,3 @@ terraform { - required_providers { - github = { - source = "integrations/github" - } - } + # Required providers are configured in providers.tf } From fbf89c931052bdc473189e425a223229e0d050b1 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 12:17:26 -0400 Subject: [PATCH 06/16] Remove outdated test files and add new validation and workflow trigger tests for EKS cluster deployment --- .terraform.lock.hcl | 63 +++++++++++++++++++ .terraform/modules/github_repo | 1 + .terraform/modules/modules.json | 1 + .../hashicorp/null/3.2.3/linux_amd64 | 1 + .../hashicorp/tls/4.0.6/linux_amd64 | 1 + .../integrations/github/6.6.0/linux_amd64 | 1 + .../basic.tftest.hcl => basic.tftest.hcl | 19 +++++- .../modules/eks_deployment.github_repo | 1 + .../basic/.terraform/modules/modules.json | 1 + examples/basic/workflow_trigger.tftest.hcl | 32 ---------- main.tf | 4 +- .../default/modules/github_repo | 1 + .../default/modules/modules.json | 1 + .../hashicorp/null/3.2.3/linux_amd64 | 1 + .../hashicorp/tls/4.0.6/linux_amd64 | 1 + .../integrations/github/6.6.0/linux_amd64 | 1 + ...dation.tftest.hcl => validation.tftest.hcl | 17 ++++- varfiles/default.json | 1 + varfiles/default.tfvars | 0 workflow_trigger.tftest.hcl | 31 +++++++++ 20 files changed, 140 insertions(+), 39 deletions(-) create mode 100644 .terraform.lock.hcl create mode 160000 .terraform/modules/github_repo create mode 100644 .terraform/modules/modules.json create mode 120000 .terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 create mode 120000 .terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 create mode 120000 .terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 rename examples/basic/basic.tftest.hcl => basic.tftest.hcl (69%) create mode 160000 examples/basic/.terraform/modules/eks_deployment.github_repo create mode 100644 examples/basic/.terraform/modules/modules.json delete mode 100644 examples/basic/workflow_trigger.tftest.hcl create mode 160000 terraform_data_dirs/default/modules/github_repo create mode 100644 terraform_data_dirs/default/modules/modules.json create mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 create mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 create mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 rename examples/basic/validation.tftest.hcl => validation.tftest.hcl (63%) create mode 100644 varfiles/default.json create mode 100644 varfiles/default.tfvars create mode 100644 workflow_trigger.tftest.hcl diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl new file mode 100644 index 0000000..6db2030 --- /dev/null +++ b/.terraform.lock.hcl @@ -0,0 +1,63 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:+AnORRgFbRO6qqcfaQyeX80W0eX3VmjadjnUFUJTiXo=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.6" + hashes = [ + "h1:dYSb3V94K5dDMtrBRLPzBpkMTPn+3cXZ/kIJdtFL+2M=", + "zh:10de0d8af02f2e578101688fd334da3849f56ea91b0d9bd5b1f7a243417fdda8", + "zh:37fc01f8b2bc9d5b055dc3e78bfd1beb7c42cfb776a4c81106e19c8911366297", + "zh:4578ca03d1dd0b7f572d96bd03f744be24c726bfd282173d54b100fd221608bb", + "zh:6c475491d1250050765a91a493ef330adc24689e8837a0f07da5a0e1269e11c1", + "zh:81bde94d53cdababa5b376bbc6947668be4c45ab655de7aa2e8e4736dfd52509", + "zh:abdce260840b7b050c4e401d4f75c7a199fafe58a8b213947a258f75ac18b3e8", + "zh:b754cebfc5184873840f16a642a7c9ef78c34dc246a8ae29e056c79939963c7a", + "zh:c928b66086078f9917aef0eec15982f2e337914c5c4dbc31dd4741403db7eb18", + "zh:cded27bee5f24de6f2ee0cfd1df46a7f88e84aaffc2ecbf3ff7094160f193d50", + "zh:d65eb3867e8f69aaf1b8bb53bd637c99c6b649ba3db16ded50fa9a01076d1a27", + "zh:ecb0c8b528c7a619fa71852bb3fb5c151d47576c5aab2bf3af4db52588722eeb", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/integrations/github" { + version = "6.6.0" + constraints = ">= 5.0.0" + hashes = [ + "h1:Fp0RrNe+w167AQkVUWC1WRAsyjhhHN7aHWUky7VkKW8=", + "zh:0b1b5342db6a17de7c71386704e101be7d6761569e03fb3ff1f3d4c02c32d998", + "zh:2fb663467fff76852126b58315d9a1a457e3b04bec51f04bf1c0ddc9dfbb3517", + "zh:4183e557a1dfd413dae90ca4bac37dbbe499eae5e923567371f768053f977800", + "zh:48b2979f88fb55cdb14b7e4c37c44e0dfbc21b7a19686ce75e339efda773c5c2", + "zh:5d803fb06625e0bcf83abb590d4235c117fa7f4aa2168fa3d5f686c41bc529ec", + "zh:6f1dd094cbab36363583cda837d7ca470bef5f8abf9b19f23e9cd8b927153498", + "zh:772edb5890d72b32868f9fdc0a9a1d4f4701d8e7f8acb37a7ac530d053c776e3", + "zh:798f443dbba6610431dcef832047f6917fb5a4e184a3a776c44e6213fb429cc6", + "zh:cc08dfcc387e2603f6dbaff8c236c1254185450d6cadd6bad92879fe7e7dbce9", + "zh:d5e2c8d7f50f91d6847ddce27b10b721bdfce99c1bbab42a68fa271337d73d63", + "zh:e69a0045440c706f50f84a84ff8b1df520ec9bf757de4b8f9959f2ed20c3f440", + "zh:efc5358573a6403cbea3a08a2fcd2407258ac083d9134c641bdcb578966d8bdf", + "zh:f627a255e5809ec2375f79949c79417847fa56b9e9222ea7c45a463eb663f137", + "zh:f7c02f762e4cf1de7f58bde520798491ccdd54a5bd52278d579c146d1d07d4f0", + "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", + ] +} diff --git a/.terraform/modules/github_repo b/.terraform/modules/github_repo new file mode 160000 index 0000000..b6b6cba --- /dev/null +++ b/.terraform/modules/github_repo @@ -0,0 +1 @@ +Subproject commit b6b6cba8c08f2997b7a4058c421d41744ef7793d diff --git a/.terraform/modules/modules.json b/.terraform/modules/modules.json new file mode 100644 index 0000000..3ca46f1 --- /dev/null +++ b/.terraform/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.83","Dir":".terraform/modules/github_repo"}]} \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 b/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 new file mode 120000 index 0000000..fe28aef --- /dev/null +++ b/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 b/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 new file mode 120000 index 0000000..08157aa --- /dev/null +++ b/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 b/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 new file mode 120000 index 0000000..26dfde5 --- /dev/null +++ b/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/integrations/github/6.6.0/linux_amd64 \ No newline at end of file diff --git a/examples/basic/basic.tftest.hcl b/basic.tftest.hcl similarity index 69% rename from examples/basic/basic.tftest.hcl rename to basic.tftest.hcl index 48bd466..6bdd237 100644 --- a/examples/basic/basic.tftest.hcl +++ b/basic.tftest.hcl @@ -1,10 +1,23 @@ variables { github_token = "mock-token" + name = "test-eks-cluster" + organization = "test-org" + template_repo_org = "test-org" + environment = "dev" + region = "us-east-1" + cluster_config = { + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } } provider "github" { - owner = "my-org" - token = "mock-token" + token = var.github_token base_url = "https://github.mycompany.com/api/v3" } @@ -12,7 +25,7 @@ run "verify_repository_config" { command = plan assert { - condition = module.eks_deployment.github_repo.name == "eks-test-cluster" + condition = module.eks_deployment.github_repo.name == "test-eks-cluster" error_message = "Repository name does not match expected value" } diff --git a/examples/basic/.terraform/modules/eks_deployment.github_repo b/examples/basic/.terraform/modules/eks_deployment.github_repo new file mode 160000 index 0000000..b6b6cba --- /dev/null +++ b/examples/basic/.terraform/modules/eks_deployment.github_repo @@ -0,0 +1 @@ +Subproject commit b6b6cba8c08f2997b7a4058c421d41744ef7793d diff --git a/examples/basic/.terraform/modules/modules.json b/examples/basic/.terraform/modules/modules.json new file mode 100644 index 0000000..2ad9006 --- /dev/null +++ b/examples/basic/.terraform/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"eks_deployment","Source":"../..","Dir":"../.."},{"Key":"eks_deployment.github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.83","Dir":".terraform/modules/eks_deployment.github_repo"}]} \ No newline at end of file diff --git a/examples/basic/workflow_trigger.tftest.hcl b/examples/basic/workflow_trigger.tftest.hcl deleted file mode 100644 index 501393d..0000000 --- a/examples/basic/workflow_trigger.tftest.hcl +++ /dev/null @@ -1,32 +0,0 @@ -variables { - github_token = "mock-token" -} - -mock_provider "github" { - mock_resource "github_repository_dispatch" { - defaults = { - result = { - status_code = 204 - } - } - - assert "verify_workflow_payload" { - condition = self.input.event_type == "cluster-plan" && - self.input.client_payload.environment == "dev" && - self.input.client_payload.region == "us-east-1" && - self.input.client_payload.cluster_dir == "dev-eks-01" - error_message = "Invalid workflow dispatch payload" - } - } -} - -run "verify_workflow_trigger" { - command = plan - - # This test verifies that the workflow trigger mechanism works - # without actually making API calls - assert { - condition = true - error_message = "Workflow trigger test failed" - } -} \ No newline at end of file diff --git a/main.tf b/main.tf index d168751..35861cc 100644 --- a/main.tf +++ b/main.tf @@ -19,11 +19,11 @@ module "github_repo" { resource "null_resource" "trigger_workflow" { triggers = { - repository_name = module.github_repo.github_repo.name + github_repo_name = module.github_repo.github_repo.name } provisioner "local-exec" { - command = "python3 scripts/trigger_workflow.py ${module.github_repo.github_repo.name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_dir}\",\"auto_approve\":true}'" + command = "python3 scripts/trigger_workflow.py ${module.github_repo.github_repo.name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" environment = { GITHUB_TOKEN = var.github_token diff --git a/terraform_data_dirs/default/modules/github_repo b/terraform_data_dirs/default/modules/github_repo new file mode 160000 index 0000000..b6b6cba --- /dev/null +++ b/terraform_data_dirs/default/modules/github_repo @@ -0,0 +1 @@ +Subproject commit b6b6cba8c08f2997b7a4058c421d41744ef7793d diff --git a/terraform_data_dirs/default/modules/modules.json b/terraform_data_dirs/default/modules/modules.json new file mode 100644 index 0000000..a03d773 --- /dev/null +++ b/terraform_data_dirs/default/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.83","Dir":"/data/terraform/workspaces/arnol377/git/terraform-eks-deployment/terraform_data_dirs/default/modules/github_repo"}]} \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 new file mode 120000 index 0000000..fe28aef --- /dev/null +++ b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 new file mode 120000 index 0000000..08157aa --- /dev/null +++ b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 new file mode 120000 index 0000000..26dfde5 --- /dev/null +++ b/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/integrations/github/6.6.0/linux_amd64 \ No newline at end of file diff --git a/examples/basic/validation.tftest.hcl b/validation.tftest.hcl similarity index 63% rename from examples/basic/validation.tftest.hcl rename to validation.tftest.hcl index 3339064..666f4fa 100644 --- a/examples/basic/validation.tftest.hcl +++ b/validation.tftest.hcl @@ -1,10 +1,23 @@ variables { github_token = "mock-token" + name = "test-eks-cluster" + organization = "test-org" + template_repo_org = "test-org" + environment = "dev" + region = "us-east-1" + cluster_config = { + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } } provider "github" { - owner = "my-org" - token = "mock-token" + token = var.github_token base_url = "https://github.mycompany.com/api/v3" } diff --git a/varfiles/default.json b/varfiles/default.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/varfiles/default.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/varfiles/default.tfvars b/varfiles/default.tfvars new file mode 100644 index 0000000..e69de29 diff --git a/workflow_trigger.tftest.hcl b/workflow_trigger.tftest.hcl new file mode 100644 index 0000000..84b50e5 --- /dev/null +++ b/workflow_trigger.tftest.hcl @@ -0,0 +1,31 @@ +variables { + github_token = "mock-token" + name = "test-eks-cluster" + organization = "test-org" + template_repo_org = "test-org" + environment = "dev" + region = "us-east-1" + cluster_config = { + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } +} + +provider "github" { + token = var.github_token + base_url = "https://github.mycompany.com/api/v3" +} + +run "verify_workflow_trigger" { + command = plan + + assert { + condition = can([github_repository_dispatch.dispatch]) + error_message = "Workflow trigger failed to dispatch" + } +} \ No newline at end of file From 7e9ae4ef45b614471dc1a5bef55cedae90f6aae2 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 12:55:51 -0400 Subject: [PATCH 07/16] Update GitHub server URL and refactor related configurations for EKS deployment --- README.md | 2 +- basic.tftest.hcl | 45 +++++++------------ examples/basic/main.tf | 2 +- main.tf | 6 +-- scripts/test_trigger_workflow.py | 4 +- ...on.tftest.hcl => validation.tftest.hcl.tmp | 5 +-- variables.tf | 2 +- workflow_trigger.tftest.hcl | 19 ++++++-- workflow_trigger.tftest.hcl.tmp | 40 +++++++++++++++++ 9 files changed, 80 insertions(+), 45 deletions(-) rename validation.tftest.hcl => validation.tftest.hcl.tmp (90%) create mode 100644 workflow_trigger.tftest.hcl.tmp diff --git a/README.md b/README.md index 0bac3a8..282aa3a 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ module "eks_deployment" { } github_token = "your-github-token" - github_server_url = "https://github.mycompany.com" # Optional, for GitHub Enterprise + github_server_url = "https://github.e.it.census.gov" # Optional, for GitHub Enterprise } ``` diff --git a/basic.tftest.hcl b/basic.tftest.hcl index 6bdd237..1036299 100644 --- a/basic.tftest.hcl +++ b/basic.tftest.hcl @@ -1,10 +1,10 @@ variables { - github_token = "mock-token" name = "test-eks-cluster" - organization = "test-org" - template_repo_org = "test-org" + organization = "SCT-Engineering" + template_repo_org = "SCT-Engineering" environment = "dev" - region = "us-east-1" + region = "us-gov-east-1" + github_server_url = "https://github.e.it.census.gov" cluster_config = { cluster_name = "dev-eks-01" account_name = "dev-account" @@ -14,47 +14,32 @@ variables { vpc_name = "dev-vpc" vpc_domain_name = "dev.example.com" } + enable_modules = { + cert_manager = true + prometheus = true + grafana = true + } } provider "github" { - token = var.github_token - base_url = "https://github.mycompany.com/api/v3" + base_url = var.github_server_url } run "verify_repository_config" { - command = plan + command = apply assert { - condition = module.eks_deployment.github_repo.name == "test-eks-cluster" + condition = module.github_repo.github_repo.name == var.name error_message = "Repository name does not match expected value" } assert { - condition = module.eks_deployment.github_repo.description == "EKS Cluster Configuration for dev-eks-01" - error_message = "Repository description does not match expected value" + condition = contains(module.github_repo.github_repo.topics, "eks") + error_message = "Repository topics should include 'eks'" } assert { - condition = module.eks_deployment.github_repo.visibility == "private" + condition = module.github_repo.github_repo.visibility == "private" error_message = "Repository visibility should be private" } -} - -run "verify_module_defaults" { - command = plan - - assert { - condition = length(module.eks_deployment.enable_modules.*.cert_manager) > 0 - error_message = "cert-manager module should be enabled" - } - - assert { - condition = length(module.eks_deployment.enable_modules.*.prometheus) > 0 - error_message = "prometheus module should be enabled" - } - - assert { - condition = length(module.eks_deployment.enable_modules.*.grafana) > 0 - error_message = "grafana module should be enabled" - } } \ No newline at end of file diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 44645ff..437b6ac 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -14,7 +14,7 @@ module "eks_deployment" { template_repo_org = "my-org" github_token = var.github_token - github_server_url = "https://github.mycompany.com" + github_server_url = "https://github.e.it.census.gov" cluster_config = { cluster_name = "dev-eks-01" diff --git a/main.tf b/main.tf index 35861cc..1027f77 100644 --- a/main.tf +++ b/main.tf @@ -19,15 +19,15 @@ module "github_repo" { resource "null_resource" "trigger_workflow" { triggers = { - github_repo_name = module.github_repo.github_repo.name + github_repo_name = module.github_repo.full_name } provisioner "local-exec" { - command = "python3 scripts/trigger_workflow.py ${module.github_repo.github_repo.name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" + command = "python3 scripts/trigger_workflow.py ${module.github_repo.full_name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" environment = { GITHUB_TOKEN = var.github_token - GITHUB_OWNER = var.organization + GITHUB_OWNER = var.organization GITHUB_SERVER_URL = var.github_server_url } } diff --git a/scripts/test_trigger_workflow.py b/scripts/test_trigger_workflow.py index ae786fd..47ba51e 100644 --- a/scripts/test_trigger_workflow.py +++ b/scripts/test_trigger_workflow.py @@ -10,7 +10,7 @@ def test_trigger_workflow_success(): mock_post.return_value = mock_response os.environ['GITHUB_TOKEN'] = 'test-token' - os.environ['GITHUB_OWNER'] = 'test-org' + os.environ['GITHUB_OWNER'] = 'SCT-Engineering' os.environ['GITHUB_SERVER_URL'] = 'https://github.example.com' result = trigger_workflow('test-repo', 'cluster-plan', { @@ -31,7 +31,7 @@ def test_trigger_workflow_failure(): mock_post.return_value = mock_response os.environ['GITHUB_TOKEN'] = 'invalid-token' - os.environ['GITHUB_OWNER'] = 'test-org' + os.environ['GITHUB_OWNER'] = 'SCT-Engineering' result = trigger_workflow('test-repo', 'cluster-plan', {}) diff --git a/validation.tftest.hcl b/validation.tftest.hcl.tmp similarity index 90% rename from validation.tftest.hcl rename to validation.tftest.hcl.tmp index 666f4fa..e609b16 100644 --- a/validation.tftest.hcl +++ b/validation.tftest.hcl.tmp @@ -1,10 +1,10 @@ variables { - github_token = "mock-token" name = "test-eks-cluster" organization = "test-org" template_repo_org = "test-org" environment = "dev" region = "us-east-1" + github_server_url = "https://github.e.it.census.gov" cluster_config = { cluster_name = "dev-eks-01" account_name = "dev-account" @@ -17,8 +17,7 @@ variables { } provider "github" { - token = var.github_token - base_url = "https://github.mycompany.com/api/v3" + base_url = var.github_server_url } # Test invalid cluster version diff --git a/variables.tf b/variables.tf index 4afa3c5..cab10ae 100644 --- a/variables.tf +++ b/variables.tf @@ -209,7 +209,7 @@ variable "github_token" { } variable "github_server_url" { - description = "GitHub Enterprise server URL (e.g., https://github.mycompany.com)" + description = "GitHub Enterprise server URL (e.g., https://github.e.it.census.gov)" type = string default = "https://api.github.com" } \ No newline at end of file diff --git a/workflow_trigger.tftest.hcl b/workflow_trigger.tftest.hcl index 84b50e5..11ffc81 100644 --- a/workflow_trigger.tftest.hcl +++ b/workflow_trigger.tftest.hcl @@ -5,6 +5,7 @@ variables { template_repo_org = "test-org" environment = "dev" region = "us-east-1" + github_server_url = "https://github.e.it.census.gov" cluster_config = { cluster_name = "dev-eks-01" account_name = "dev-account" @@ -14,18 +15,28 @@ variables { vpc_name = "dev-vpc" vpc_domain_name = "dev.example.com" } + enable_modules = { + cert_manager = true + prometheus = true + grafana = true + } } provider "github" { token = var.github_token - base_url = "https://github.mycompany.com/api/v3" + base_url = var.github_server_url } run "verify_workflow_trigger" { - command = plan + command = apply + + assert { + condition = can(null_resource.trigger_workflow) + error_message = "Workflow trigger resource should exist" + } assert { - condition = can([github_repository_dispatch.dispatch]) - error_message = "Workflow trigger failed to dispatch" + condition = null_resource.trigger_workflow.triggers.github_repo_name == module.github_repo.full_name + error_message = "Workflow trigger should use correct repository name" } } \ No newline at end of file diff --git a/workflow_trigger.tftest.hcl.tmp b/workflow_trigger.tftest.hcl.tmp new file mode 100644 index 0000000..1812c1f --- /dev/null +++ b/workflow_trigger.tftest.hcl.tmp @@ -0,0 +1,40 @@ +variables { + name = "test-eks-cluster" + organization = "test-org" + template_repo_org = "test-org" + environment = "dev" + region = "us-east-1" + github_server_url = "https://github.e.it.census.gov" + cluster_config = { + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } + enable_modules = { + cert_manager = true + prometheus = true + grafana = true + } +} + +provider "github" { + base_url = var.github_server_url +} + +run "verify_workflow_trigger" { + command = apply + + assert { + condition = can(null_resource.trigger_workflow) + error_message = "Workflow trigger resource should exist" + } + + assert { + condition = null_resource.trigger_workflow.triggers.github_repo_name == module.github_repo.full_name + error_message = "Workflow trigger should use correct repository name" + } +} \ No newline at end of file From 7d6c61625a0c20fd4537bf46f8b58f3b85104186 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 13:26:27 -0400 Subject: [PATCH 08/16] Refactor module configurations to enable dynamic management of extra files and update enable_all_modules variable for improved flexibility --- locals.tf | 2 +- main.tf | 16 ++++++++++------ variables.tf | 24 ++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/locals.tf b/locals.tf index 30494ed..bc33459 100644 --- a/locals.tf +++ b/locals.tf @@ -124,7 +124,7 @@ locals { environment = var.environment region = var.region cluster_dir = "platform-cluster" - enable_all_modules = false + enable_all_modules = var.enable_all_modules account = { account_name = var.cluster_config.account_name aws_account_id = var.cluster_config.aws_account_id diff --git a/main.tf b/main.tf index 1027f77..54593d6 100644 --- a/main.tf +++ b/main.tf @@ -14,7 +14,11 @@ module "github_repo" { github_has_wiki = true github_has_projects = true - managed_extra_files = local.managed_extra_files + managed_extra_files = concat( + local.managed_extra_files, + var.managed_extra_files + ) + extra_files = var.extra_files } resource "null_resource" "trigger_workflow" { @@ -25,11 +29,11 @@ resource "null_resource" "trigger_workflow" { provisioner "local-exec" { command = "python3 scripts/trigger_workflow.py ${module.github_repo.full_name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" - environment = { - GITHUB_TOKEN = var.github_token - GITHUB_OWNER = var.organization - GITHUB_SERVER_URL = var.github_server_url - } + # environment = { + # GITHUB_TOKEN = var.github_token + # GITHUB_OWNER = var.organization + # GITHUB_SERVER_URL = var.github_server_url + # } } depends_on = [module.github_repo] diff --git a/variables.tf b/variables.tf index cab10ae..0c78add 100644 --- a/variables.tf +++ b/variables.tf @@ -212,4 +212,28 @@ variable "github_server_url" { description = "GitHub Enterprise server URL (e.g., https://github.e.it.census.gov)" type = string default = "https://api.github.com" +} + +variable enable_all_modules { + description = "Enable all modules" + type = bool + default = false +} + +variable "managed_extra_files" { + description = "List of extra files to manage in the repository" + type = list(object({ + path = string + content = string + })) + default = [] +} + +variable "extra_files" { + description = "List of extra files to add to the repository" + type = list(object({ + path = string + content = string + })) + default = [] } \ No newline at end of file From 9908cfbfdf745066d3255d41fa6a12b8de9ffcb6 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 14:21:24 -0400 Subject: [PATCH 09/16] updating gitignore --- .gitignore | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6862aed --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used for local development +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +*tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +# Ignore lock files +.terraform.lock.hcl + +# Ignore test temporary files +*.tftest.hcl.tmp From d2c9e34037076b78b11ff9efdc7112cb5fcf7f0a Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 14:23:55 -0400 Subject: [PATCH 10/16] Remove obsolete Terraform module and provider cache files to clean up the repository --- .terraform/modules/github_repo | 1 - .terraform/modules/modules.json | 1 - .../registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 | 1 - .../registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 | 1 - .../registry.terraform.io/integrations/github/6.6.0/linux_amd64 | 1 - 5 files changed, 5 deletions(-) delete mode 160000 .terraform/modules/github_repo delete mode 100644 .terraform/modules/modules.json delete mode 120000 .terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 delete mode 120000 .terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 delete mode 120000 .terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 diff --git a/.terraform/modules/github_repo b/.terraform/modules/github_repo deleted file mode 160000 index b6b6cba..0000000 --- a/.terraform/modules/github_repo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b6b6cba8c08f2997b7a4058c421d41744ef7793d diff --git a/.terraform/modules/modules.json b/.terraform/modules/modules.json deleted file mode 100644 index 3ca46f1..0000000 --- a/.terraform/modules/modules.json +++ /dev/null @@ -1 +0,0 @@ -{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.83","Dir":".terraform/modules/github_repo"}]} \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 b/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 deleted file mode 120000 index fe28aef..0000000 --- a/.terraform/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 b/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 deleted file mode 120000 index 08157aa..0000000 --- a/.terraform/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 \ No newline at end of file diff --git a/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 b/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 deleted file mode 120000 index 26dfde5..0000000 --- a/.terraform/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 +++ /dev/null @@ -1 +0,0 @@ -/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/integrations/github/6.6.0/linux_amd64 \ No newline at end of file From 49343ca86691f05647340f965b0cb2a9a349728e Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 14:31:00 -0400 Subject: [PATCH 11/16] removing test workflow --- .github/workflows/test.yml | 45 --------------------------- .terraform.lock.hcl | 63 -------------------------------------- 2 files changed, 108 deletions(-) delete mode 100644 .github/workflows/test.yml delete mode 100644 .terraform.lock.hcl diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 7d030cf..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Tests - -on: - pull_request: - branches: [ main ] - push: - branches: [ main ] - -jobs: - terraform-test: - name: Terraform Tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: "1.7.0" - - - name: Run Terraform Tests - working-directory: ./examples/basic - run: terraform test - env: - TF_VAR_github_token: mock-token - - python-test: - name: Python Tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest requests - - - name: Run Python Tests - working-directory: ./scripts - run: pytest -v test_trigger_workflow.py \ No newline at end of file diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl deleted file mode 100644 index 6db2030..0000000 --- a/.terraform.lock.hcl +++ /dev/null @@ -1,63 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.3" - hashes = [ - "h1:+AnORRgFbRO6qqcfaQyeX80W0eX3VmjadjnUFUJTiXo=", - "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", - "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", - "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", - "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", - "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", - "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", - "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", - "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", - "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", - "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", - ] -} - -provider "registry.terraform.io/hashicorp/tls" { - version = "4.0.6" - hashes = [ - "h1:dYSb3V94K5dDMtrBRLPzBpkMTPn+3cXZ/kIJdtFL+2M=", - "zh:10de0d8af02f2e578101688fd334da3849f56ea91b0d9bd5b1f7a243417fdda8", - "zh:37fc01f8b2bc9d5b055dc3e78bfd1beb7c42cfb776a4c81106e19c8911366297", - "zh:4578ca03d1dd0b7f572d96bd03f744be24c726bfd282173d54b100fd221608bb", - "zh:6c475491d1250050765a91a493ef330adc24689e8837a0f07da5a0e1269e11c1", - "zh:81bde94d53cdababa5b376bbc6947668be4c45ab655de7aa2e8e4736dfd52509", - "zh:abdce260840b7b050c4e401d4f75c7a199fafe58a8b213947a258f75ac18b3e8", - "zh:b754cebfc5184873840f16a642a7c9ef78c34dc246a8ae29e056c79939963c7a", - "zh:c928b66086078f9917aef0eec15982f2e337914c5c4dbc31dd4741403db7eb18", - "zh:cded27bee5f24de6f2ee0cfd1df46a7f88e84aaffc2ecbf3ff7094160f193d50", - "zh:d65eb3867e8f69aaf1b8bb53bd637c99c6b649ba3db16ded50fa9a01076d1a27", - "zh:ecb0c8b528c7a619fa71852bb3fb5c151d47576c5aab2bf3af4db52588722eeb", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/integrations/github" { - version = "6.6.0" - constraints = ">= 5.0.0" - hashes = [ - "h1:Fp0RrNe+w167AQkVUWC1WRAsyjhhHN7aHWUky7VkKW8=", - "zh:0b1b5342db6a17de7c71386704e101be7d6761569e03fb3ff1f3d4c02c32d998", - "zh:2fb663467fff76852126b58315d9a1a457e3b04bec51f04bf1c0ddc9dfbb3517", - "zh:4183e557a1dfd413dae90ca4bac37dbbe499eae5e923567371f768053f977800", - "zh:48b2979f88fb55cdb14b7e4c37c44e0dfbc21b7a19686ce75e339efda773c5c2", - "zh:5d803fb06625e0bcf83abb590d4235c117fa7f4aa2168fa3d5f686c41bc529ec", - "zh:6f1dd094cbab36363583cda837d7ca470bef5f8abf9b19f23e9cd8b927153498", - "zh:772edb5890d72b32868f9fdc0a9a1d4f4701d8e7f8acb37a7ac530d053c776e3", - "zh:798f443dbba6610431dcef832047f6917fb5a4e184a3a776c44e6213fb429cc6", - "zh:cc08dfcc387e2603f6dbaff8c236c1254185450d6cadd6bad92879fe7e7dbce9", - "zh:d5e2c8d7f50f91d6847ddce27b10b721bdfce99c1bbab42a68fa271337d73d63", - "zh:e69a0045440c706f50f84a84ff8b1df520ec9bf757de4b8f9959f2ed20c3f440", - "zh:efc5358573a6403cbea3a08a2fcd2407258ac083d9134c641bdcb578966d8bdf", - "zh:f627a255e5809ec2375f79949c79417847fa56b9e9222ea7c45a463eb663f137", - "zh:f7c02f762e4cf1de7f58bde520798491ccdd54a5bd52278d579c146d1d07d4f0", - "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", - ] -} From 9335f0b67f0db25f9f7e68d5887c751133b05660 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 15:20:30 -0400 Subject: [PATCH 12/16] Remove GitHub token variable and references to streamline configuration --- examples/basic/main.tf | 1 - main.tf | 6 ------ variables.tf | 6 ------ workflow_trigger.tftest.hcl | 1 - 4 files changed, 14 deletions(-) diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 437b6ac..fcc16ea 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -13,7 +13,6 @@ module "eks_deployment" { region = "us-east-1" template_repo_org = "my-org" - github_token = var.github_token github_server_url = "https://github.e.it.census.gov" cluster_config = { diff --git a/main.tf b/main.tf index 54593d6..a3f9395 100644 --- a/main.tf +++ b/main.tf @@ -28,12 +28,6 @@ resource "null_resource" "trigger_workflow" { provisioner "local-exec" { command = "python3 scripts/trigger_workflow.py ${module.github_repo.full_name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" - - # environment = { - # GITHUB_TOKEN = var.github_token - # GITHUB_OWNER = var.organization - # GITHUB_SERVER_URL = var.github_server_url - # } } depends_on = [module.github_repo] diff --git a/variables.tf b/variables.tf index 0c78add..23992cc 100644 --- a/variables.tf +++ b/variables.tf @@ -202,12 +202,6 @@ variable "github_actions_workflows" { default = [] } -variable "github_token" { - description = "GitHub token for triggering workflows" - type = string - sensitive = true -} - variable "github_server_url" { description = "GitHub Enterprise server URL (e.g., https://github.e.it.census.gov)" type = string diff --git a/workflow_trigger.tftest.hcl b/workflow_trigger.tftest.hcl index 11ffc81..ec05681 100644 --- a/workflow_trigger.tftest.hcl +++ b/workflow_trigger.tftest.hcl @@ -23,7 +23,6 @@ variables { } provider "github" { - token = var.github_token base_url = var.github_server_url } From d5dc8cbc0c9b4d5a978988c05cdbf72553244721 Mon Sep 17 00:00:00 2001 From: David John Arnold Jr Date: Mon, 14 Apr 2025 12:23:37 -0700 Subject: [PATCH 13/16] Update terraform-validate.yaml --- .github/workflows/terraform-validate.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/terraform-validate.yaml b/.github/workflows/terraform-validate.yaml index ac02202..a3265fa 100644 --- a/.github/workflows/terraform-validate.yaml +++ b/.github/workflows/terraform-validate.yaml @@ -17,10 +17,20 @@ jobs: uses: CSVD/gh-actions-setup-terraform@v2 with: terraform_version: '1.7.3' - + + - name: Setup GITHUB Credentials + id: github_credentials + uses: CSVD/gh-auth@main + with: + github_app_pem_file: ${{ secrets.GH_APP_PEM_FILE }} + github_app_installation_id: ${{ vars.GH_APP_INSTALLATION_ID }} + github_app_id: ${{ vars.GH_APP_ID }} + - name: Validate Terraform Configuration id: validate uses: CSVD/terraform-validate@main + env: + GITHUB_TOKEN: ${{ steps.github_credentials.outputs.github_token }} - name: Check Validation/Test Results if: always() From 0b2d76c72907c08057ef58e92c87fd1df40badfc Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 15:30:58 -0400 Subject: [PATCH 14/16] Remove mock GitHub token from workflow trigger configuration --- workflow_trigger.tftest.hcl | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow_trigger.tftest.hcl b/workflow_trigger.tftest.hcl index ec05681..1812c1f 100644 --- a/workflow_trigger.tftest.hcl +++ b/workflow_trigger.tftest.hcl @@ -1,5 +1,4 @@ variables { - github_token = "mock-token" name = "test-eks-cluster" organization = "test-org" template_repo_org = "test-org" From 2f2ffb00b4b3b285cd154576bf01dee4542e8b8e Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 15:34:49 -0400 Subject: [PATCH 15/16] updating --- .gitignore | 1 + basic.tftest.hcl | 45 -------------- .../default/modules/github_repo | 2 +- .../default/modules/modules.json | 2 +- validation.tftest.hcl.tmp | 62 ------------------- workflow_trigger.tftest.hcl | 40 ------------ workflow_trigger.tftest.hcl.tmp | 40 ------------ 7 files changed, 3 insertions(+), 189 deletions(-) delete mode 100644 basic.tftest.hcl delete mode 100644 validation.tftest.hcl.tmp delete mode 100644 workflow_trigger.tftest.hcl delete mode 100644 workflow_trigger.tftest.hcl.tmp diff --git a/.gitignore b/.gitignore index 6862aed..e9b6656 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ terraform.rc # Ignore test temporary files *.tftest.hcl.tmp +terraform_data_dirs diff --git a/basic.tftest.hcl b/basic.tftest.hcl deleted file mode 100644 index 1036299..0000000 --- a/basic.tftest.hcl +++ /dev/null @@ -1,45 +0,0 @@ -variables { - name = "test-eks-cluster" - organization = "SCT-Engineering" - template_repo_org = "SCT-Engineering" - environment = "dev" - region = "us-gov-east-1" - github_server_url = "https://github.e.it.census.gov" - cluster_config = { - cluster_name = "dev-eks-01" - account_name = "dev-account" - aws_account_id = "123456789012" - aws_profile = "dev-profile" - environment_abbr = "dev" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" - } - enable_modules = { - cert_manager = true - prometheus = true - grafana = true - } -} - -provider "github" { - base_url = var.github_server_url -} - -run "verify_repository_config" { - command = apply - - assert { - condition = module.github_repo.github_repo.name == var.name - error_message = "Repository name does not match expected value" - } - - assert { - condition = contains(module.github_repo.github_repo.topics, "eks") - error_message = "Repository topics should include 'eks'" - } - - assert { - condition = module.github_repo.github_repo.visibility == "private" - error_message = "Repository visibility should be private" - } -} \ No newline at end of file diff --git a/terraform_data_dirs/default/modules/github_repo b/terraform_data_dirs/default/modules/github_repo index b6b6cba..6140ffe 160000 --- a/terraform_data_dirs/default/modules/github_repo +++ b/terraform_data_dirs/default/modules/github_repo @@ -1 +1 @@ -Subproject commit b6b6cba8c08f2997b7a4058c421d41744ef7793d +Subproject commit 6140ffe240ed0a9b1ff42940238a9ea7348db255 diff --git a/terraform_data_dirs/default/modules/modules.json b/terraform_data_dirs/default/modules/modules.json index a03d773..89ced5d 100644 --- a/terraform_data_dirs/default/modules/modules.json +++ b/terraform_data_dirs/default/modules/modules.json @@ -1 +1 @@ -{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.83","Dir":"/data/terraform/workspaces/arnol377/git/terraform-eks-deployment/terraform_data_dirs/default/modules/github_repo"}]} \ No newline at end of file +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.84","Dir":"/data/terraform/workspaces/arnol377/git/terraform-eks-deployment/terraform_data_dirs/default/modules/github_repo"}]} \ No newline at end of file diff --git a/validation.tftest.hcl.tmp b/validation.tftest.hcl.tmp deleted file mode 100644 index e609b16..0000000 --- a/validation.tftest.hcl.tmp +++ /dev/null @@ -1,62 +0,0 @@ -variables { - name = "test-eks-cluster" - organization = "test-org" - template_repo_org = "test-org" - environment = "dev" - region = "us-east-1" - github_server_url = "https://github.e.it.census.gov" - cluster_config = { - cluster_name = "dev-eks-01" - account_name = "dev-account" - aws_account_id = "123456789012" - aws_profile = "dev-profile" - environment_abbr = "dev" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" - } -} - -provider "github" { - base_url = var.github_server_url -} - -# Test invalid cluster version -run "invalid_cluster_version" { - command = plan - - variables { - versions = { - cluster_version = "1.26" # Test outdated version - } - } - - expect_failures = [ - var.versions - ] -} - -# Test missing required variables -run "missing_required_vars" { - command = plan - - variables { - name = null # Required variable - } - - expect_failures = [ - var.name - ] -} - -# Test invalid environment name -run "invalid_environment" { - command = plan - - variables { - environment = "invalid" # Should be dev, staging, or prod - } - - expect_failures = [ - var.environment - ] -} \ No newline at end of file diff --git a/workflow_trigger.tftest.hcl b/workflow_trigger.tftest.hcl deleted file mode 100644 index 1812c1f..0000000 --- a/workflow_trigger.tftest.hcl +++ /dev/null @@ -1,40 +0,0 @@ -variables { - name = "test-eks-cluster" - organization = "test-org" - template_repo_org = "test-org" - environment = "dev" - region = "us-east-1" - github_server_url = "https://github.e.it.census.gov" - cluster_config = { - cluster_name = "dev-eks-01" - account_name = "dev-account" - aws_account_id = "123456789012" - aws_profile = "dev-profile" - environment_abbr = "dev" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" - } - enable_modules = { - cert_manager = true - prometheus = true - grafana = true - } -} - -provider "github" { - base_url = var.github_server_url -} - -run "verify_workflow_trigger" { - command = apply - - assert { - condition = can(null_resource.trigger_workflow) - error_message = "Workflow trigger resource should exist" - } - - assert { - condition = null_resource.trigger_workflow.triggers.github_repo_name == module.github_repo.full_name - error_message = "Workflow trigger should use correct repository name" - } -} \ No newline at end of file diff --git a/workflow_trigger.tftest.hcl.tmp b/workflow_trigger.tftest.hcl.tmp deleted file mode 100644 index 1812c1f..0000000 --- a/workflow_trigger.tftest.hcl.tmp +++ /dev/null @@ -1,40 +0,0 @@ -variables { - name = "test-eks-cluster" - organization = "test-org" - template_repo_org = "test-org" - environment = "dev" - region = "us-east-1" - github_server_url = "https://github.e.it.census.gov" - cluster_config = { - cluster_name = "dev-eks-01" - account_name = "dev-account" - aws_account_id = "123456789012" - aws_profile = "dev-profile" - environment_abbr = "dev" - vpc_name = "dev-vpc" - vpc_domain_name = "dev.example.com" - } - enable_modules = { - cert_manager = true - prometheus = true - grafana = true - } -} - -provider "github" { - base_url = var.github_server_url -} - -run "verify_workflow_trigger" { - command = apply - - assert { - condition = can(null_resource.trigger_workflow) - error_message = "Workflow trigger resource should exist" - } - - assert { - condition = null_resource.trigger_workflow.triggers.github_repo_name == module.github_repo.full_name - error_message = "Workflow trigger should use correct repository name" - } -} \ No newline at end of file From 0ed813e889dd038ac16203b2e16e0b594beaedca Mon Sep 17 00:00:00 2001 From: arnol377 Date: Mon, 14 Apr 2025 16:07:49 -0400 Subject: [PATCH 16/16] removing varfiles --- varfiles/default.json | 1 - varfiles/default.tfvars | 0 2 files changed, 1 deletion(-) delete mode 100644 varfiles/default.json delete mode 100644 varfiles/default.tfvars diff --git a/varfiles/default.json b/varfiles/default.json deleted file mode 100644 index 9e26dfe..0000000 --- a/varfiles/default.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/varfiles/default.tfvars b/varfiles/default.tfvars deleted file mode 100644 index e69de29..0000000