From 5895901f0b1ef8bcdb8bd1dbe9bd1cc5ae7f7a17 Mon Sep 17 00:00:00 2001 From: Terraform Date: Mon, 14 Apr 2025 11:56:16 -0700 Subject: [PATCH 1/4] Update .github/workflows/terraform-release.yaml --- .github/workflows/terraform-release.yaml | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/terraform-release.yaml b/.github/workflows/terraform-release.yaml index e69de29..b58eb34 100644 --- a/.github/workflows/terraform-release.yaml +++ b/.github/workflows/terraform-release.yaml @@ -0,0 +1,40 @@ +name: Terraform CI/CD +on: + workflow_dispatch: + pull_request: + types: [closed] + branches: + - main +jobs: + terraform-ci-cd: + runs-on: 229685449397 + permissions: + contents: write + + steps: + - name: Checkout code + uses: CSVD/gh-actions-checkout@v4 + + - name: Setup GITHUB Credentials + id: github_credentials + uses: CSVD/gh-auth@main + with: + github_app_pem_file: ${{ secrets.GH_APP_PEM_FILE }} + github_app_installation_id: ${{ vars.GH_APP_INSTALLATION_ID }} + github_app_id: ${{ vars.GH_APP_ID }} + + - name: Setup GitHub CLI + run: | + # Force manual authentication since setup-git might not work with GitHub Enterprise + echo "${{ steps.github_credentials.outputs.github_token }}" > /tmp/token.txt + gh auth login --with-token --hostname "github.e.it.census.gov" < /tmp/token.txt + rm /tmp/token.txt + + # Test GitHub CLI auth status + gh auth status || echo "GitHub CLI authentication failed" + + - name: Run Release Action + uses: CSVD/releaser@main + with: + github-token: ${{ steps.github_credentials.outputs.github_token }} + working-directory: '.' \ No newline at end of file From e402c9506648a2dbb3189aee6043f082947c6243 Mon Sep 17 00:00:00 2001 From: David John Arnold Jr Date: Mon, 14 Apr 2025 12:40:59 -0700 Subject: [PATCH 2/4] Add comprehensive EKS Cluster Template roadmap and configuration files (#1) * Add comprehensive EKS Cluster Template roadmap and configuration files * Refactor namespace definitions and add GitHub Actions workflows variable * Implement GitHub Actions workflow triggers and enhance Terraform configurations for automated cluster setup * Enhance README and add Terraform module for EKS cluster deployment with automated GitHub Actions workflows; include tests for workflow triggers and validation. * Update versions.tf to include required provider versions for Terraform configuration * Remove outdated test files and add new validation and workflow trigger tests for EKS cluster deployment * Update GitHub server URL and refactor related configurations for EKS deployment * Refactor module configurations to enable dynamic management of extra files and update enable_all_modules variable for improved flexibility * updating gitignore * Remove obsolete Terraform module and provider cache files to clean up the repository * removing test workflow * Remove GitHub token variable and references to streamline configuration * Update terraform-validate.yaml * Remove mock GitHub token from workflow trigger configuration * updating --- .github/workflows/terraform-validate.yaml | 12 +- .gitignore | 34 +++ README.md | 163 ++++++++++++ ROADMAP.md | 62 +++++ .../modules/eks_deployment.github_repo | 1 + .../basic/.terraform/modules/modules.json | 1 + examples/basic/main.tf | 38 +++ examples/basic/variables.tf | 5 + locals.tf | 166 +++++++++++++ main.tf | 44 ++++ providers.tf | 14 ++ scripts/test_trigger_workflow.py | 46 ++++ scripts/trigger_workflow.py | 59 +++++ templates/common-variables.hcl | 10 + templates/default-versions.hcl | 124 ++++++++++ .../default/modules/github_repo | 1 + .../default/modules/modules.json | 1 + .../hashicorp/null/3.2.3/linux_amd64 | 1 + .../hashicorp/tls/4.0.6/linux_amd64 | 1 + .../integrations/github/6.6.0/linux_amd64 | 1 + varfiles/default.json | 1 + varfiles/default.tfvars | 0 variables.tf | 233 ++++++++++++++++++ versions.tf | 3 + 24 files changed, 1020 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 ROADMAP.md create mode 160000 examples/basic/.terraform/modules/eks_deployment.github_repo create mode 100644 examples/basic/.terraform/modules/modules.json create mode 100644 examples/basic/main.tf create mode 100644 examples/basic/variables.tf create mode 100644 locals.tf create mode 100644 main.tf create mode 100644 providers.tf create mode 100644 scripts/test_trigger_workflow.py create mode 100755 scripts/trigger_workflow.py create mode 100644 templates/common-variables.hcl create mode 100644 templates/default-versions.hcl create mode 160000 terraform_data_dirs/default/modules/github_repo create mode 100644 terraform_data_dirs/default/modules/modules.json create mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 create mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 create mode 120000 terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 create mode 100644 varfiles/default.json create mode 100644 varfiles/default.tfvars create mode 100644 variables.tf create mode 100644 versions.tf diff --git a/.github/workflows/terraform-validate.yaml b/.github/workflows/terraform-validate.yaml index ac02202..a3265fa 100644 --- a/.github/workflows/terraform-validate.yaml +++ b/.github/workflows/terraform-validate.yaml @@ -17,10 +17,20 @@ jobs: uses: CSVD/gh-actions-setup-terraform@v2 with: terraform_version: '1.7.3' - + + - name: Setup GITHUB Credentials + id: github_credentials + uses: CSVD/gh-auth@main + with: + github_app_pem_file: ${{ secrets.GH_APP_PEM_FILE }} + github_app_installation_id: ${{ vars.GH_APP_INSTALLATION_ID }} + github_app_id: ${{ vars.GH_APP_ID }} + - name: Validate Terraform Configuration id: validate uses: CSVD/terraform-validate@main + env: + GITHUB_TOKEN: ${{ steps.github_credentials.outputs.github_token }} - name: Check Validation/Test Results if: always() diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e9b6656 --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used for local development +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +*tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +# Ignore lock files +.terraform.lock.hcl + +# Ignore test temporary files +*.tftest.hcl.tmp +terraform_data_dirs diff --git a/README.md b/README.md index 4456e7d..282aa3a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,165 @@ # terraform-eks-deployment + Terraform module for EKS cluster deployment and configuration + +## Overview + +This module creates a new GitHub repository for your EKS cluster based on the template-eks-cluster repository. It sets up all necessary configuration files and triggers automated workflows for cluster deployment. + +## Prerequisites + +- GitHub token with repository and workflow permissions +- Python 3.x installed on the machine running Terraform +- Access to GitHub Enterprise (if using enterprise version) + +## Usage + +```hcl +module "eks_deployment" { + source = "path/to/terraform-eks-deployment" + + name = "my-eks-cluster" + organization = "my-org" + environment = "production" + region = "us-east-1" + + cluster_config = { + cluster_name = "prod-eks-01" + account_name = "prod-account" + aws_account_id = "123456789012" + aws_profile = "prod-profile" + environment_abbr = "prod" + vpc_name = "prod-vpc" + vpc_domain_name = "prod.example.com" + } + + github_token = "your-github-token" + github_server_url = "https://github.e.it.census.gov" # Optional, for GitHub Enterprise +} +``` + +## Workflow Automation + +### Overview + +The module automatically triggers GitHub Actions workflows in your newly created repository to: +1. Install Python requirements +2. Execute Terragrunt operations for cluster management + +### Workflow Sequence + +1. **Repository Creation**: The module creates a new repository from the template-eks-cluster template +2. **Initial Configuration**: Configuration files are generated based on your inputs +3. **Requirements Installation**: A workflow is triggered to install Python dependencies +4. **Cluster Planning**: A terragrunt plan workflow is automatically triggered + +### Available Workflows + +Your new repository will have these workflows available: + +1. **Install Requirements** (`install-requirements.yml`) + - Triggered automatically on repository creation + - Installs all Python dependencies from requirements.txt + +2. **Terragrunt Cluster Operations** (`terragrunt-cluster-build.yml`) + - Supports plan, apply, and destroy operations + - Can be triggered manually or via API + - Includes safety checks and approvals + +### Triggering Workflows + +The workflows can be triggered in two ways: + +1. **Automatic Triggering** + - On repository creation, the module automatically triggers: + 1. Requirements installation + 2. Initial cluster plan + +2. **Manual Triggering** + - Via GitHub UI: + 1. Go to Actions tab + 2. Select desired workflow + 3. Click "Run workflow" + 4. Fill in parameters + +3. **API Triggering** + - Use GitHub's API to trigger workflows: + ```bash + curl -X POST \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/OWNER/REPO/dispatches" \ + -d '{ + "event_type": "cluster-plan", + "client_payload": { + "environment": "dev", + "region": "us-east-1", + "cluster_dir": "my-cluster", + "auto_approve": false + } + }' + ``` + +### Supported Events + +The following event types are supported for workflow triggers: + +- `install-requirements`: Install Python dependencies +- `cluster-plan`: Preview cluster changes +- `cluster-apply`: Apply cluster changes +- `cluster-destroy`: Destroy cluster + +### Required Secrets + +The following secrets must be configured in your repository: + +- `AWS_ROLE_ARN`: ARN of the AWS role to assume +- `GITHUB_TOKEN`: GitHub token with workflow permissions + +## Module Configuration + +### Required Variables + +- `name`: Repository name +- `organization`: GitHub organization name +- `environment`: Deployment environment +- `region`: AWS region +- `cluster_config`: Cluster configuration object +- `github_token`: GitHub token for workflow operations + +### Optional Variables + +- `github_server_url`: GitHub Enterprise server URL +- `template_repo_org`: Organization containing the template repository +- `enable_modules`: Map of modules to enable in the cluster + +For more configuration options, see the variables.tf file. + +## Outputs + +- `repository_url`: URL of the created repository +- `ssh_clone_url`: SSH clone URL of the repository + +## Security Considerations + +1. **GitHub Token**: Use a token with minimal required permissions +2. **AWS Role**: Use role-based access with least privilege +3. **Auto-approve**: Use with caution in production environments +4. **Environment Protection**: Configure branch protection rules + +## Troubleshooting + +Common issues and solutions: + +1. **Workflow Trigger Failures** + - Check GitHub token permissions + - Verify GitHub Enterprise URL (if applicable) + - Check network connectivity + +2. **Python Requirements** + - Ensure requirements.txt exists in template repository + - Check Python version compatibility + +3. **AWS Authentication** + - Verify AWS role ARN + - Check AWS credentials configuration diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..12f698f --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,62 @@ +# EKS Cluster Template Roadmap + +## Current Architecture +- Template repository (`template-eks-cluster`) serves as the base for creating new EKS cluster configurations +- Uses Terraform GitHub repo module to create new repositories from the template +- Implements managed and non-managed extra files functionality +- Supports configuration through `config.json` + +## Planned Enhancements + +### 1. Automated Cluster Setup (High Priority) +- [x] Add GitHub Actions workflows with workflow_dispatch triggers +- [x] Implement automated terragrunt command execution for cluster building + - [x] Support for plan/apply/destroy commands + - [x] Environment-specific execution + - [x] Automated testing framework +- [ ] Configure workflows to run on specific runners for credential management +- [ ] Create templatized GitHub Actions workflow files +- [ ] Enable direct cluster creation without manual repository cloning + +### 2. File Management System (Medium Priority) +- [ ] Implement wrapper module for repo module +- [ ] Add support for crafting and injecting various configuration files +- [ ] Define file lifecycle management strategy + - [ ] Managed files (controlled by workspace) + - [ ] Non-managed files (user-modifiable) + +### 3. Version Management (Medium Priority) +- [ ] Implement version control strategy for `default-versions.hcl` +- [ ] Create system for managing platform release versions +- [ ] Set up version override mechanism + - [ ] Default versions in template repo + - [ ] Override capability in workspace creating repos + +### 4. Configuration Management (Low Priority) +- [ ] Enhance Makefile and Ansible playbook integration +- [ ] Improve configuration file templating +- [ ] Add validation for configuration files + +## Technical Considerations +1. File Lifecycle Management: + - Managed files: Controlled by workspace + - Non-managed files: User-modifiable post-creation + - Version-specific files: Platform release coordination + +2. Automation Requirements: + - GitHub Actions runner configuration + - Credential management + - Workflow templating + - Terragrunt integration + +3. Version Control Strategy: + - Module version collections + - Platform release versions + - Override mechanisms + +## Success Criteria +- Fully automated cluster creation process +- Minimal manual intervention required +- Proper version management system +- Clear file lifecycle management +- Secure credential handling \ No newline at end of file diff --git a/examples/basic/.terraform/modules/eks_deployment.github_repo b/examples/basic/.terraform/modules/eks_deployment.github_repo new file mode 160000 index 0000000..b6b6cba --- /dev/null +++ b/examples/basic/.terraform/modules/eks_deployment.github_repo @@ -0,0 +1 @@ +Subproject commit b6b6cba8c08f2997b7a4058c421d41744ef7793d diff --git a/examples/basic/.terraform/modules/modules.json b/examples/basic/.terraform/modules/modules.json new file mode 100644 index 0000000..2ad9006 --- /dev/null +++ b/examples/basic/.terraform/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"eks_deployment","Source":"../..","Dir":"../.."},{"Key":"eks_deployment.github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.83","Dir":".terraform/modules/eks_deployment.github_repo"}]} \ No newline at end of file diff --git a/examples/basic/main.tf b/examples/basic/main.tf new file mode 100644 index 0000000..fcc16ea --- /dev/null +++ b/examples/basic/main.tf @@ -0,0 +1,38 @@ +provider "github" { + # Configuration expected from environment variables: + # GITHUB_TOKEN + # GITHUB_OWNER (optional) +} + +module "eks_deployment" { + source = "../../" + + name = "eks-test-cluster" + organization = "my-org" + environment = "dev" + region = "us-east-1" + + template_repo_org = "my-org" + github_server_url = "https://github.e.it.census.gov" + + cluster_config = { + cluster_name = "dev-eks-01" + account_name = "dev-account" + aws_account_id = "123456789012" + aws_profile = "dev-profile" + environment_abbr = "dev" + vpc_name = "dev-vpc" + vpc_domain_name = "dev.example.com" + } + + enable_modules = { + cert_manager = true + prometheus = true + grafana = true + } + + versions = { + cluster_version = "1.27" + eks_module_version = "20.33.1" + } +} \ No newline at end of file diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf new file mode 100644 index 0000000..73092ff --- /dev/null +++ b/examples/basic/variables.tf @@ -0,0 +1,5 @@ +variable "github_token" { + description = "GitHub token for API operations" + type = string + sensitive = true +} \ No newline at end of file diff --git a/locals.tf b/locals.tf new file mode 100644 index 0000000..bc33459 --- /dev/null +++ b/locals.tf @@ -0,0 +1,166 @@ +locals { + common_vars = merge({ + organization = "census:ocio:csvd" + project_name = "csvd_platformbaseline" + project_number = "fs0000000078" + project_role = "csvd_platformbaseline_app" + state_bucket_prefix = "inf-tfstate" + state_table_name = "tf_remote_state" + route53_endpoints = {} + }, var.common_variables) + + # First define base namespaces without dependencies + base_namespaces = { + cert-manager = "kube-system" + karpenter = "karpenter" + metrics-server = "kube-system" + postgresql = "kube-system" + keycloak = "keycloak" + gogatekeeper = "kube-system" + istio = "istio-system" + kiali = "istio-system" + } + + # Then merge with telemetry namespaces + all_namespaces = merge( + local.base_namespaces, + { + grafana = var.namespaces.telemetry_namespace + k8s-dashboard = var.namespaces.telemetry_namespace + loki = var.namespaces.telemetry_namespace + otel = var.namespaces.telemetry_namespace + prometheus = var.namespaces.telemetry_namespace + tempo = var.namespaces.telemetry_namespace + }, + var.namespaces.custom_namespaces + ) + + namespaces = { + operator_namespace = var.namespaces.operator_namespace + telemetry_namespace = var.namespaces.telemetry_namespace + namespaces = local.all_namespaces + } + + default_versions = { + cluster_version = var.versions.cluster_version + custom_service_eks_account = var.versions.release_version + eks_module_version = var.versions.eks_module_version + istio_ingress_version = var.versions.release_version + release_version = var.versions.release_version + + # Provider versions + aws_version = var.versions.aws_version + helm_version = var.versions.helm_version + kubernetes_version = var.versions.kubernetes_version + null_version = var.versions.null_version + random_version = var.versions.random_version + template_version = var.versions.template_version + tf_version = var.versions.tf_version + + # Component versions + cert_manager_version = var.versions.cert_manager.version + cert_manager_helm_chart = var.versions.cert_manager.chart_version + cluster_issuer_name = var.versions.cert_manager.cluster_issuer_name + + gogatekeeper_tag = var.versions.gogatekeeper.tag + gogatekeeper_chart_version = var.versions.gogatekeeper.chart_version + + grafana_hostname = var.versions.grafana.hostname + grafana_operator_chart_version = var.versions.grafana.operator_chart_version + grafana_operator_tag = var.versions.grafana.operator_tag + grafana_tag = var.versions.grafana.tag + os_shell_image_tag = var.versions.grafana.os_shell_image_tag + + istio_version = var.versions.istio.version + istio_namespace = var.versions.istio.namespace + + dashboard_hostname = var.versions.k8s_dashboard.hostname + k8s_dashboard_metrics_scraper = var.versions.k8s_dashboard.metrics_scraper + k8s_dashboard_version = var.versions.k8s_dashboard.version + + karpenter_helm_chart = var.versions.karpenter.helm_chart + karpenter_tag = var.versions.karpenter.tag + + keycloak_chart_version = var.versions.keycloak.chart_version + keycloak_tag = var.versions.keycloak.tag + keycloak_hostname = var.versions.keycloak.hostname + keycloak_database = var.versions.keycloak.database + keycloak_username = var.versions.keycloak.username + keycloak_password = var.versions.keycloak.password + postgresql_tag = var.versions.keycloak.postgresql_tag + + kiali_operator_version = var.versions.kiali.operator_version + kiali_application_version = "v${var.versions.kiali.operator_version}" + + loki_chart_version = var.versions.loki.chart_version + loki_tag = var.versions.loki.tag + enterprise_logs_provisioner_tag = var.versions.loki.enterprise_logs_provisioner_tag + gateway_tag = var.versions.loki.gateway_tag + memcached_tag = var.versions.loki.memcached_tag + exporter_tag = var.versions.loki.exporter_tag + sidecar_tag = var.versions.loki.sidecar_tag + + metrics_server_helm_chart = var.versions.metrics_server.helm_chart + metrics_server_tag = var.versions.metrics_server.tag + + prometheus_chart_version = var.versions.prometheus.chart_version + prometheus_server_tag = var.versions.prometheus.server_tag + prometheus_config_reloader_tag = var.versions.prometheus.config_reloader_tag + alertmanager_tag = var.versions.prometheus.alertmanager_tag + kube_state_metrics_tag = var.versions.prometheus.kube_state_metrics_tag + node_exporter_tag = var.versions.prometheus.node_exporter_tag + pushgateway_tag = var.versions.prometheus.pushgateway_tag + + tempo_chart_version = var.versions.tempo.chart_version + tempo_tag = var.versions.tempo.tag + + # Add namespace configurations + operator_namespace = var.namespaces.operator_namespace + telemetry_namespace = var.namespaces.telemetry_namespace + namespaces = local.all_namespaces + } + + config_json = jsonencode({ + environment = var.environment + region = var.region + cluster_dir = "platform-cluster" + enable_all_modules = var.enable_all_modules + account = { + account_name = var.cluster_config.account_name + aws_account_id = var.cluster_config.aws_account_id + aws_profile = var.cluster_config.aws_profile + environment_abbr = var.cluster_config.environment_abbr + } + vpc = { + vpc_name = var.cluster_config.vpc_name + vpc_domain_name = var.cluster_config.vpc_domain_name + } + cluster = { + cluster_name = var.cluster_config.cluster_name + cluster_mailing_list = var.cluster_config.cluster_mailing_list + eks_instance_disk_size = var.cluster_config.eks_instance_disk_size + eks_ng_desired_size = var.cluster_config.eks_ng_desired_size + eks_ng_max_size = var.cluster_config.eks_ng_max_size + eks_ng_min_size = var.cluster_config.eks_ng_min_size + enable_cluster_creator_admin_permissions = var.cluster_config.enable_cluster_creator_admin_permissions + tags = var.cluster_config.tags + } + modules = var.enable_modules + }) + + managed_extra_files = concat([ + { + path = "config.json" + content = local.config_json + }, + { + path = "_envcommon/default-versions.hcl" + content = templatefile("${path.module}/templates/default-versions.hcl", local.default_versions) + }, + { + path = "_envcommon/common-variables.hcl" + content = templatefile("${path.module}/templates/common-variables.hcl", local.common_vars) + } + ], + var.github_actions_workflows) +} \ No newline at end of file diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..a3f9395 --- /dev/null +++ b/main.tf @@ -0,0 +1,44 @@ +module "github_repo" { + source = "HappyPathway/repo/github" + + name = var.name + repo_org = var.organization + github_repo_description = "EKS Cluster Configuration for ${var.cluster_config.cluster_name}" + github_repo_topics = ["eks", "kubernetes", "terraform", "infrastructure"] + + template_repo = "template-eks-cluster" + template_repo_org = var.template_repo_org + + github_is_private = true + github_has_issues = true + github_has_wiki = true + github_has_projects = true + + managed_extra_files = concat( + local.managed_extra_files, + var.managed_extra_files + ) + extra_files = var.extra_files +} + +resource "null_resource" "trigger_workflow" { + triggers = { + github_repo_name = module.github_repo.full_name + } + + provisioner "local-exec" { + command = "python3 scripts/trigger_workflow.py ${module.github_repo.full_name} cluster-plan '{\"environment\":\"${var.environment}\",\"region\":\"${var.region}\",\"cluster_dir\":\"${var.cluster_config.cluster_name}\",\"auto_approve\":true}'" + } + + depends_on = [module.github_repo] +} + +output "repository_url" { + description = "URL of the created repository" + value = module.github_repo.html_url +} + +output "ssh_clone_url" { + description = "SSH clone URL of the repository" + value = module.github_repo.ssh_clone_url +} \ No newline at end of file diff --git a/providers.tf b/providers.tf new file mode 100644 index 0000000..e01c942 --- /dev/null +++ b/providers.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + github = { + source = "integrations/github" + version = ">= 5.0" + } + } +} + +provider "github" { + # Configuration is expected from environment variables: + # GITHUB_TOKEN + # GITHUB_OWNER (optional) +} \ No newline at end of file diff --git a/scripts/test_trigger_workflow.py b/scripts/test_trigger_workflow.py new file mode 100644 index 0000000..47ba51e --- /dev/null +++ b/scripts/test_trigger_workflow.py @@ -0,0 +1,46 @@ +import os +import pytest +from unittest.mock import patch, MagicMock +from trigger_workflow import trigger_workflow + +def test_trigger_workflow_success(): + with patch('requests.post') as mock_post: + mock_response = MagicMock() + mock_response.status_code = 204 + mock_post.return_value = mock_response + + os.environ['GITHUB_TOKEN'] = 'test-token' + os.environ['GITHUB_OWNER'] = 'SCT-Engineering' + os.environ['GITHUB_SERVER_URL'] = 'https://github.example.com' + + result = trigger_workflow('test-repo', 'cluster-plan', { + 'environment': 'dev', + 'region': 'us-east-1', + 'cluster_dir': 'test-cluster', + 'auto_approve': False + }) + + assert result is True + mock_post.assert_called_once() + +def test_trigger_workflow_failure(): + with patch('requests.post') as mock_post: + mock_response = MagicMock() + mock_response.status_code = 401 + mock_response.text = 'Unauthorized' + mock_post.return_value = mock_response + + os.environ['GITHUB_TOKEN'] = 'invalid-token' + os.environ['GITHUB_OWNER'] = 'SCT-Engineering' + + result = trigger_workflow('test-repo', 'cluster-plan', {}) + + assert result is False + mock_post.assert_called_once() + +def test_missing_token(): + if 'GITHUB_TOKEN' in os.environ: + del os.environ['GITHUB_TOKEN'] + + with pytest.raises(SystemExit): + trigger_workflow('test-repo', 'cluster-plan', {}) \ No newline at end of file diff --git a/scripts/trigger_workflow.py b/scripts/trigger_workflow.py new file mode 100755 index 0000000..f5b1fb2 --- /dev/null +++ b/scripts/trigger_workflow.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +import os +import sys +import requests +import json + +def trigger_workflow(repo, event_type, payload=None): + token = os.environ.get('GITHUB_TOKEN') + if not token: + print("Error: GITHUB_TOKEN environment variable not set") + sys.exit(1) + + owner = os.environ.get('GITHUB_OWNER', 'default-org') + server_url = os.environ.get('GITHUB_SERVER_URL', 'https://api.github.com') + + # Remove trailing slash if present and ensure we're using the API endpoint + server_url = server_url.rstrip('/') + if not server_url.endswith('/api/v3') and not 'api.github.com' in server_url: + server_url = f"{server_url}/api/v3" + + url = f"{server_url}/repos/{owner}/{repo}/dispatches" + + headers = { + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'token {token}', + 'Content-Type': 'application/json', + } + + data = { + 'event_type': event_type, + 'client_payload': payload or {} + } + + response = requests.post(url, headers=headers, data=json.dumps(data), verify=True) + + if response.status_code == 204: + print(f"Successfully triggered workflow {event_type} for {owner}/{repo}") + return True + else: + print(f"Failed to trigger workflow: {response.status_code}") + print(response.text) + return False + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: trigger_workflow.py []") + sys.exit(1) + + repo = sys.argv[1] + event_type = sys.argv[2] + payload = json.loads(sys.argv[3]) if len(sys.argv) > 3 else None + + # First trigger requirements installation + if not trigger_workflow(repo, "install-requirements"): + sys.exit(1) + + # Then trigger the main workflow + if not trigger_workflow(repo, event_type, payload): + sys.exit(1) \ No newline at end of file diff --git a/templates/common-variables.hcl b/templates/common-variables.hcl new file mode 100644 index 0000000..e228659 --- /dev/null +++ b/templates/common-variables.hcl @@ -0,0 +1,10 @@ +locals { + organization = "${organization}" + project_name = "${project_name}" + project_number = "${project_number}" + project_role = "${project_role}" + state_bucket_prefix = "${state_bucket_prefix}" + state_table_name = "${state_table_name}" + + route53_endpoints = ${jsonencode(route53_endpoints)} +} \ No newline at end of file diff --git a/templates/default-versions.hcl b/templates/default-versions.hcl new file mode 100644 index 0000000..60f3cd4 --- /dev/null +++ b/templates/default-versions.hcl @@ -0,0 +1,124 @@ +locals { + ##################### + # Module Versions + ##################### + cluster_version = "${cluster_version}" + custom_service_eks_account = "${custom_service_eks_account}" + eks_module_version = "${eks_module_version}" + istio_ingress_version = "${istio_ingress_version}" + release_version = "${release_version}" + + ##################### + # TF Providers + ##################### + aws_version = "${aws_version}" + helm_version = "${helm_version}" + kubernetes_version = "${kubernetes_version}" + null_version = "${null_version}" + random_version = "${random_version}" + template_version = "${template_version}" + tf_version = "${tf_version}" + + ##################### + # Component Versions + ##################### + + ################ + # Cert-Manager + ################ + cluster_issuer_name = "${cluster_issuer_name}" + cert_manager_version = "${cert_manager_version}" + cert_manager_helm_chart = "${cert_manager_helm_chart}" + + ################ + # GoGatekeeper + ################ + gogatekeeper_tag = "${gogatekeeper_tag}" + gogatekeeper_chart_version = "${gogatekeeper_chart_version}" + + ################ + # Grafana + ################ + grafana_hostname = "${grafana_hostname}" + grafana_operator_chart_version = "${grafana_operator_chart_version}" + grafana_operator_tag = "${grafana_operator_tag}" + grafana_tag = "${grafana_tag}" + os_shell_image_tag = "${os_shell_image_tag}" + + ################ + # Istio + ################ + istio_namespace = "${istio_namespace}" + istio_version = "${istio_version}" + + ################ + # k8s-dashboard + ################ + dashboard_hostname = "${dashboard_hostname}" + k8s_dashboard_metrics_scraper = "${k8s_dashboard_metrics_scraper}" + k8s_dashboard_version = "${k8s_dashboard_version}" + + ################ + # Karpenter + ################ + karpenter_helm_chart = "${karpenter_helm_chart}" + karpenter_tag = "${karpenter_tag}" + + ################ + # Keycloak + ################ + keycloak_chart_version = "${keycloak_chart_version}" + keycloak_tag = "${keycloak_tag}" + keycloak_hostname = "${keycloak_hostname}" + keycloak_database = "${keycloak_database}" + keycloak_username = "${keycloak_username}" + keycloak_password = "${keycloak_password}" + postgresql_tag = "${postgresql_tag}" + + ################ + # Kiali + ################ + kiali_operator_version = "${kiali_operator_version}" + kiali_application_version = "${kiali_application_version}" + + ################ + # Loki + ################ + loki_chart_version = "${loki_chart_version}" + loki_tag = "${loki_tag}" + enterprise_logs_provisioner_tag = "${enterprise_logs_provisioner_tag}" + gateway_tag = "${gateway_tag}" + memcached_tag = "${memcached_tag}" + exporter_tag = "${exporter_tag}" + sidecar_tag = "${sidecar_tag}" + + ################ + # Metrics Server + ################ + metrics_server_helm_chart = "${metrics_server_helm_chart}" + metrics_server_tag = "${metrics_server_tag}" + + ################ + # Prometheus + ################ + prometheus_chart_version = "${prometheus_chart_version}" + prometheus_server_tag = "${prometheus_server_tag}" + prometheus_config_reloader_tag = "${prometheus_config_reloader_tag}" + alertmanager_tag = "${alertmanager_tag}" + kube_state_metrics_tag = "${kube_state_metrics_tag}" + node_exporter_tag = "${node_exporter_tag}" + pushgateway_tag = "${pushgateway_tag}" + + ################ + # Tempo + ################ + tempo_chart_version = "${tempo_chart_version}" + tempo_tag = "${tempo_tag}" + + ##################### + # Namespaces Config + ##################### + operator_namespace = "${operator_namespace}" + telemetry_namespace = "${telemetry_namespace}" + namespaces = ${jsonencode(namespaces)} +} \ No newline at end of file diff --git a/terraform_data_dirs/default/modules/github_repo b/terraform_data_dirs/default/modules/github_repo new file mode 160000 index 0000000..6140ffe --- /dev/null +++ b/terraform_data_dirs/default/modules/github_repo @@ -0,0 +1 @@ +Subproject commit 6140ffe240ed0a9b1ff42940238a9ea7348db255 diff --git a/terraform_data_dirs/default/modules/modules.json b/terraform_data_dirs/default/modules/modules.json new file mode 100644 index 0000000..89ced5d --- /dev/null +++ b/terraform_data_dirs/default/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"github_repo","Source":"registry.terraform.io/HappyPathway/repo/github","Version":"1.0.84","Dir":"/data/terraform/workspaces/arnol377/git/terraform-eks-deployment/terraform_data_dirs/default/modules/github_repo"}]} \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 new file mode 120000 index 0000000..fe28aef --- /dev/null +++ b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/null/3.2.3/linux_amd64 \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 new file mode 120000 index 0000000..08157aa --- /dev/null +++ b/terraform_data_dirs/default/providers/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/hashicorp/tls/4.0.6/linux_amd64 \ No newline at end of file diff --git a/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 b/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 new file mode 120000 index 0000000..26dfde5 --- /dev/null +++ b/terraform_data_dirs/default/providers/registry.terraform.io/integrations/github/6.6.0/linux_amd64 @@ -0,0 +1 @@ +/data/terraform/workspaces/arnol377/terraform-plugin-cache/registry.terraform.io/integrations/github/6.6.0/linux_amd64 \ No newline at end of file diff --git a/varfiles/default.json b/varfiles/default.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/varfiles/default.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/varfiles/default.tfvars b/varfiles/default.tfvars new file mode 100644 index 0000000..e69de29 diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..23992cc --- /dev/null +++ b/variables.tf @@ -0,0 +1,233 @@ +variable "name" { + description = "Name of the repository" + type = string +} + +variable "organization" { + description = "GitHub organization name" + type = string +} + +variable "template_repo_org" { + description = "GitHub organization for the template repository" + type = string +} + +variable "environment" { + description = "Environment name (e.g., production, staging)" + type = string +} + +variable "region" { + description = "AWS region for the EKS cluster" + type = string +} + +variable "common_variables" { + description = "Common variables across all environments" + type = object({ + organization = optional(string, "census:ocio:csvd") + project_name = optional(string, "csvd_platformbaseline") + project_number = optional(string, "fs0000000078") + project_role = optional(string, "csvd_platformbaseline_app") + state_bucket_prefix = optional(string, "inf-tfstate") + state_table_name = optional(string, "tf_remote_state") + route53_endpoints = optional(map(object({ + account_id = string + alias = string + us-gov-east-1 = string + us-gov-west-1 = string + })), {}) + }) + default = {} +} + +variable "versions" { + description = "Version configurations for various components" + type = object({ + # Module Versions + cluster_version = optional(string, "1.31") + eks_module_version = optional(string, "20.33.1") + release_version = optional(string, "main") + + # TF Providers + aws_version = optional(string, "5.84.0") + helm_version = optional(string, "2.11.0") + kubernetes_version = optional(string, "2.33.0") + null_version = optional(string, "3.2.1") + random_version = optional(string, "3.5.1") + template_version = optional(string, "2.2.0") + tf_version = optional(string, "1.5.5") + + # Component Versions + cert_manager = optional(object({ + version = optional(string, "1.17.1") + chart_version = optional(string, "1.17.1") + cluster_issuer_name = optional(string, "cert-manager") + }), {}) + + gogatekeeper = optional(object({ + tag = optional(string, "3.2.1") + chart_version = optional(string, "0.1.53") + }), {}) + + grafana = optional(object({ + hostname = optional(string, "grafana") + operator_chart_version = optional(string, "4.9.8") + operator_tag = optional(string, "5.16.0") + tag = optional(string, "11.5.2") + os_shell_image_tag = optional(string, "12") + }), {}) + + istio = optional(object({ + version = optional(string, "1.25.0") + namespace = optional(string, "istio-system") + }), {}) + + k8s_dashboard = optional(object({ + hostname = optional(string, "dashboard") + metrics_scraper = optional(string, "1.0.8") + version = optional(string, "6.0.6") + }), {}) + + karpenter = optional(object({ + helm_chart = optional(string, "1.3.1") + tag = optional(string, "1.3.1") + }), {}) + + keycloak = optional(object({ + chart_version = optional(string, "24.4.11") + tag = optional(string, "26.1.3") + hostname = optional(string, "keycloak") + database = optional(string, "keycloak") + username = optional(string, "keycloak") + password = optional(string, "this is my very secure and totally random password horse battery staple now") + postgresql_tag = optional(string, "17.4.0-debian-12-r2") + }), {}) + + kiali = optional(object({ + operator_version = optional(string, "2.2.0") + }), {}) + + loki = optional(object({ + chart_version = optional(string, "6.27.0") + tag = optional(string, "3.4.2") + enterprise_logs_provisioner_tag = optional(string, "v1.7.0") + gateway_tag = optional(string, "1.27-alpine") + memcached_tag = optional(string, "1.6.37") + exporter_tag = optional(string, "v0.15.0") + sidecar_tag = optional(string, "1.27.4") + }), {}) + + metrics_server = optional(object({ + helm_chart = optional(string, "3.12.2") + tag = optional(string, "0.7.2") + }), {}) + + prometheus = optional(object({ + chart_version = optional(string, "27.5.1") + server_tag = optional(string, "v3.2.1") + config_reloader_tag = optional(string, "v0.75.2") + alertmanager_tag = optional(string, "v0.28.0") + kube_state_metrics_tag = optional(string, "v2.15.0") + node_exporter_tag = optional(string, "v1.9.0") + pushgateway_tag = optional(string, "v1.11.0") + }), {}) + + tempo = optional(object({ + chart_version = optional(string, "1.18.2") + tag = optional(string, "2.7.1") + }), {}) + }) + default = {} +} + +variable "namespaces" { + description = "Namespace configurations" + type = object({ + operator_namespace = optional(string, "aoperator") + telemetry_namespace = optional(string, "atelemetry") + custom_namespaces = optional(map(string), { + cert-manager = "kube-system" + karpenter = "karpenter" + metrics-server = "kube-system" + postgresql = "kube-system" + keycloak = "keycloak" + gogatekeeper = "kube-system" + istio = "istio-system" + kiali = "istio-system" + }) + }) + default = {} +} + +variable "cluster_config" { + description = "Configuration for the EKS cluster" + type = object({ + cluster_name = string + account_name = string + aws_account_id = string + aws_profile = string + environment_abbr = string + vpc_name = string + vpc_domain_name = string + cluster_mailing_list = optional(string) + eks_instance_disk_size = optional(number, 200) + eks_ng_desired_size = optional(number, 3) + eks_ng_max_size = optional(number, 10) + eks_ng_min_size = optional(number, 3) + enable_cluster_creator_admin_permissions = optional(bool, true) + tags = optional(map(string), {}) + }) +} + +variable "enable_modules" { + description = "Map of modules to enable" + type = object({ + gogatekeeper = optional(bool, false) + cert_manager = optional(bool, false) + prometheus = optional(bool, false) + grafana = optional(bool, false) + istio = optional(bool, false) + }) + default = {} +} + +variable "github_actions_workflows" { + description = "List of GitHub Actions workflow files to add to the repository" + type = list(object({ + path = string + content = string + })) + default = [] +} + +variable "github_server_url" { + description = "GitHub Enterprise server URL (e.g., https://github.e.it.census.gov)" + type = string + default = "https://api.github.com" +} + +variable enable_all_modules { + description = "Enable all modules" + type = bool + default = false +} + +variable "managed_extra_files" { + description = "List of extra files to manage in the repository" + type = list(object({ + path = string + content = string + })) + default = [] +} + +variable "extra_files" { + description = "List of extra files to add to the repository" + type = list(object({ + path = string + content = string + })) + default = [] +} \ No newline at end of file diff --git a/versions.tf b/versions.tf new file mode 100644 index 0000000..920cfb3 --- /dev/null +++ b/versions.tf @@ -0,0 +1,3 @@ +terraform { + # Required providers are configured in providers.tf +} From 4a066b9a8d6007a677cd83661ef9dfddb8003507 Mon Sep 17 00:00:00 2001 From: David John Arnold Jr Date: Mon, 14 Apr 2025 13:08:15 -0700 Subject: [PATCH 3/4] Init (#2) * Add comprehensive EKS Cluster Template roadmap and configuration files * Refactor namespace definitions and add GitHub Actions workflows variable * Implement GitHub Actions workflow triggers and enhance Terraform configurations for automated cluster setup * Enhance README and add Terraform module for EKS cluster deployment with automated GitHub Actions workflows; include tests for workflow triggers and validation. * Update versions.tf to include required provider versions for Terraform configuration * Remove outdated test files and add new validation and workflow trigger tests for EKS cluster deployment * Update GitHub server URL and refactor related configurations for EKS deployment * Refactor module configurations to enable dynamic management of extra files and update enable_all_modules variable for improved flexibility * updating gitignore * Remove obsolete Terraform module and provider cache files to clean up the repository * removing test workflow * Remove GitHub token variable and references to streamline configuration * Update terraform-validate.yaml * Remove mock GitHub token from workflow trigger configuration * updating * removing varfiles From 8a6db9844a1f44a47d2d9af9de8de7f013c61169 Mon Sep 17 00:00:00 2001 From: David John Arnold Jr Date: Mon, 14 Apr 2025 13:10:18 -0700 Subject: [PATCH 4/4] Init (#3) * Add comprehensive EKS Cluster Template roadmap and configuration files * Refactor namespace definitions and add GitHub Actions workflows variable * Implement GitHub Actions workflow triggers and enhance Terraform configurations for automated cluster setup * Enhance README and add Terraform module for EKS cluster deployment with automated GitHub Actions workflows; include tests for workflow triggers and validation. * Update versions.tf to include required provider versions for Terraform configuration * Remove outdated test files and add new validation and workflow trigger tests for EKS cluster deployment * Update GitHub server URL and refactor related configurations for EKS deployment * Refactor module configurations to enable dynamic management of extra files and update enable_all_modules variable for improved flexibility * updating gitignore * Remove obsolete Terraform module and provider cache files to clean up the repository * removing test workflow * Remove GitHub token variable and references to streamline configuration * Update terraform-validate.yaml * Remove mock GitHub token from workflow trigger configuration * updating * removing varfiles * Remove unused GitHub module and provider references