Merge branch 'main' of https://github.com/HappyPathway/eks-automation…

…-lambda
SCT-Engineering · Apr 30, 2025 · ec19f23 · ec19f23
2 parents 83809b3 + 4d564ab
commit ec19f23
Show file tree

Hide file tree

Showing 11 changed files with 255 additions and 38 deletions.
diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl
diff --git a/.tflog b/.tflog
@@ -89,3 +89,77 @@
 2025-04-17T01:21:26.795-0700 [TRACE] statemgr.Filesystem: read nil snapshot
 2025-04-17T01:21:26.795-0700 [TRACE] backend/local: populating backendrun.LocalRun for current working directory
 2025-04-17T01:21:26.807-0700 [TRACE] Config.VerifyDependencySelections: provider registry.terraform.io/hashicorp/aws has no lock file entry to satisfy ""
+2025-04-29T09:32:02.115-0700 [INFO]  Terraform version: 1.10.5
+2025-04-29T09:32:02.116-0700 [DEBUG] using github.com/hashicorp/go-tfe v1.70.0
+2025-04-29T09:32:02.116-0700 [DEBUG] using github.com/hashicorp/hcl/v2 v2.23.0
+2025-04-29T09:32:02.116-0700 [DEBUG] using github.com/hashicorp/terraform-svchost v0.1.1
+2025-04-29T09:32:02.116-0700 [DEBUG] using github.com/zclconf/go-cty v1.16.2
+2025-04-29T09:32:02.116-0700 [INFO]  Go runtime version: go1.23.3
+2025-04-29T09:32:02.116-0700 [INFO]  CLI args: []string{"/usr/local/Cellar/tfenv/1.0.2/versions/1.10.5/terraform", "providers", "schema", "-json", "-no-color"}
+2025-04-29T09:32:02.116-0700 [TRACE] Stdout is not a terminal
+2025-04-29T09:32:02.116-0700 [TRACE] Stderr is not a terminal
+2025-04-29T09:32:02.116-0700 [TRACE] Stdin is not a terminal
+2025-04-29T09:32:02.116-0700 [DEBUG] Attempting to open CLI config file: /Users/darnold/.terraformrc
+2025-04-29T09:32:02.116-0700 [INFO]  Loading CLI configuration from /Users/darnold/.terraformrc
+2025-04-29T09:32:02.116-0700 [INFO]  Loading CLI configuration from /Users/darnold/.terraform.d/credentials.tfrc.json
+2025-04-29T09:32:02.116-0700 [DEBUG] checking for credentials in "/Users/darnold/.terraform.d/plugins"
+2025-04-29T09:32:02.116-0700 [DEBUG] checking for credentials in "/Users/darnold/.terraform.d/plugins/darwin_amd64"
+2025-04-29T09:32:02.116-0700 [DEBUG] Using modified User-Agent: HashiCorp Terraform/1.10.5 (+https://www.terraform.io) HashiCorp-terraform-exec/0.21.0
+2025-04-29T09:32:02.116-0700 [DEBUG] ignoring non-existing provider search directory terraform.d/plugins
+2025-04-29T09:32:02.116-0700 [DEBUG] will search for provider plugins in /Users/darnold/.terraform.d/plugins
+2025-04-29T09:32:02.117-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/HappyPathway/openai/0.1.0" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:32:02.117-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/HappyPathway/openai/0.1.0/darwin_amd64" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:32:02.117-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/HappyPathway/openai/0.1.0/darwin_amd64/terraform-provider-openai" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:32:02.117-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/darnold/gigrack/0.1.0" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:32:02.117-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/darnold/gigrack/0.1.0/darwin_amd64" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:32:02.117-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/darnold/gigrack/0.1.0/darwin_amd64/terraform-provider-gigrack" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:32:02.117-0700 [WARN]  Provider plugin search ignored symlink /Users/darnold/.terraform.d/plugins/darwin_amd64/terraform-provider-configstash: only the base directory /Users/darnold/.terraform.d/plugins may be a symlink
+2025-04-29T09:32:02.117-0700 [TRACE] getproviders.SearchLocalDirectory: found hashicorp.com/edu/hashicups v0.3.1 for darwin_amd64 at /Users/darnold/.terraform.d/plugins/hashicorp.com/edu/hashicups/0.3.1/darwin_amd64
+2025-04-29T09:32:02.117-0700 [TRACE] getproviders.SearchLocalDirectory: found registry.terraform.io/darnold/gigrack v0.1.0 for darwin_amd64 at /Users/darnold/.terraform.d/plugins/registry.terraform.io/darnold/gigrack/0.1.0/darwin_amd64
+2025-04-29T09:32:02.118-0700 [WARN]  ignoring local provider path "/Users/darnold/.terraform.d/plugins/registry.terraform.io/darnold/openai/3b6fef8/darwin_amd64" with invalid version "3b6fef8": invalid characters "b6fef8"
+2025-04-29T09:32:02.118-0700 [WARN]  ignoring local provider path "/Users/darnold/.terraform.d/plugins/registry.terraform.io/darnold/openai/70f25a4/darwin_amd64" with invalid version "70f25a4": invalid characters "f25a4"
+2025-04-29T09:32:02.118-0700 [TRACE] getproviders.SearchLocalDirectory: found registry.terraform.io/happypathway/openai v5.0.0 for darwin_amd64 at /Users/darnold/.terraform.d/plugins/registry.terraform.io/happypathway/openai/5.0.0/darwin_amd64
+2025-04-29T09:32:02.118-0700 [DEBUG] ignoring non-existing provider search directory /Users/darnold/Library/Application Support/io.terraform/plugins
+2025-04-29T09:32:02.118-0700 [DEBUG] ignoring non-existing provider search directory /Library/Application Support/io.terraform/plugins
+2025-04-29T09:32:02.120-0700 [INFO]  Checkpoint disabled. Not running.
+2025-04-29T09:32:02.122-0700 [INFO]  CLI command args: []string{"providers", "schema", "-json", "-no-color"}
+2025-04-29T09:32:02.122-0700 [DEBUG] Using modified User-Agent: Terraform/1.10.5 HashiCorp-terraform-exec/0.21.0
+2025-04-29T09:32:02.122-0700 [DEBUG] Using modified User-Agent: HashiCorp Terraform/1.10.5 (+https://www.terraform.io) HashiCorp-terraform-exec/0.21.0
+2025-04-29T09:32:02.123-0700 [TRACE] Meta.Backend: BackendOpts.Config not set, so using settings loaded from backend.tf:2,3-16
+2025-04-29T09:32:02.123-0700 [TRACE] Meta.Backend: built configuration for "gcs" backend with hash value 3602047810
+2025-04-29T09:32:02.123-0700 [TRACE] Meta.Backend: backend has not previously been initialized in this working directory
+2025-04-29T09:32:02.123-0700 [TRACE] Meta.Backend: moving from default local state only to "gcs" backend
+2025-04-29T09:36:48.710-0700 [INFO]  Terraform version: 1.10.5
+2025-04-29T09:36:48.711-0700 [DEBUG] using github.com/hashicorp/go-tfe v1.70.0
+2025-04-29T09:36:48.711-0700 [DEBUG] using github.com/hashicorp/hcl/v2 v2.23.0
+2025-04-29T09:36:48.711-0700 [DEBUG] using github.com/hashicorp/terraform-svchost v0.1.1
+2025-04-29T09:36:48.711-0700 [DEBUG] using github.com/zclconf/go-cty v1.16.2
+2025-04-29T09:36:48.711-0700 [INFO]  Go runtime version: go1.23.3
+2025-04-29T09:36:48.711-0700 [INFO]  CLI args: []string{"/usr/local/Cellar/tfenv/1.0.2/versions/1.10.5/terraform", "version", "-json"}
+2025-04-29T09:36:48.711-0700 [TRACE] Stdout is not a terminal
+2025-04-29T09:36:48.711-0700 [TRACE] Stderr is not a terminal
+2025-04-29T09:36:48.711-0700 [TRACE] Stdin is not a terminal
+2025-04-29T09:36:48.711-0700 [DEBUG] Attempting to open CLI config file: /Users/darnold/.terraformrc
+2025-04-29T09:36:48.711-0700 [INFO]  Loading CLI configuration from /Users/darnold/.terraformrc
+2025-04-29T09:36:48.711-0700 [INFO]  Loading CLI configuration from /Users/darnold/.terraform.d/credentials.tfrc.json
+2025-04-29T09:36:48.711-0700 [DEBUG] checking for credentials in "/Users/darnold/.terraform.d/plugins"
+2025-04-29T09:36:48.711-0700 [DEBUG] checking for credentials in "/Users/darnold/.terraform.d/plugins/darwin_amd64"
+2025-04-29T09:36:48.711-0700 [DEBUG] Using modified User-Agent: HashiCorp Terraform/1.10.5 (+https://www.terraform.io) HashiCorp-terraform-exec/0.21.0
+2025-04-29T09:36:48.711-0700 [DEBUG] ignoring non-existing provider search directory terraform.d/plugins
+2025-04-29T09:36:48.711-0700 [DEBUG] will search for provider plugins in /Users/darnold/.terraform.d/plugins
+2025-04-29T09:36:48.711-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/HappyPathway/openai/0.1.0" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:36:48.711-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/HappyPathway/openai/0.1.0/darwin_amd64" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:36:48.711-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/HappyPathway/openai/0.1.0/darwin_amd64/terraform-provider-openai" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:36:48.711-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/darnold/gigrack/0.1.0" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:36:48.711-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/darnold/gigrack/0.1.0/darwin_amd64" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:36:48.711-0700 [WARN]  local provider path "/Users/darnold/.terraform.d/plugins/darnold/gigrack/0.1.0/darwin_amd64/terraform-provider-gigrack" contains invalid type "0.1.0"; ignoring
+2025-04-29T09:36:48.711-0700 [WARN]  Provider plugin search ignored symlink /Users/darnold/.terraform.d/plugins/darwin_amd64/terraform-provider-configstash: only the base directory /Users/darnold/.terraform.d/plugins may be a symlink
+2025-04-29T09:36:48.712-0700 [TRACE] getproviders.SearchLocalDirectory: found hashicorp.com/edu/hashicups v0.3.1 for darwin_amd64 at /Users/darnold/.terraform.d/plugins/hashicorp.com/edu/hashicups/0.3.1/darwin_amd64
+2025-04-29T09:36:48.712-0700 [TRACE] getproviders.SearchLocalDirectory: found registry.terraform.io/darnold/gigrack v0.1.0 for darwin_amd64 at /Users/darnold/.terraform.d/plugins/registry.terraform.io/darnold/gigrack/0.1.0/darwin_amd64
+2025-04-29T09:36:48.712-0700 [WARN]  ignoring local provider path "/Users/darnold/.terraform.d/plugins/registry.terraform.io/darnold/openai/3b6fef8/darwin_amd64" with invalid version "3b6fef8": invalid characters "b6fef8"
+2025-04-29T09:36:48.712-0700 [WARN]  ignoring local provider path "/Users/darnold/.terraform.d/plugins/registry.terraform.io/darnold/openai/70f25a4/darwin_amd64" with invalid version "70f25a4": invalid characters "f25a4"
+2025-04-29T09:36:48.712-0700 [TRACE] getproviders.SearchLocalDirectory: found registry.terraform.io/happypathway/openai v5.0.0 for darwin_amd64 at /Users/darnold/.terraform.d/plugins/registry.terraform.io/happypathway/openai/5.0.0/darwin_amd64
+2025-04-29T09:36:48.712-0700 [DEBUG] ignoring non-existing provider search directory /Users/darnold/Library/Application Support/io.terraform/plugins
+2025-04-29T09:36:48.712-0700 [DEBUG] ignoring non-existing provider search directory /Library/Application Support/io.terraform/plugins
+2025-04-29T09:36:48.715-0700 [INFO]  Checkpoint disabled. Not running.
+2025-04-29T09:36:48.716-0700 [INFO]  CLI command args: []string{"version", "-json"}
diff --git a/README.md b/README.md
@@ -9,11 +9,26 @@ with custom configurations from your template.
 
 ## Architecture
 
-- AWS Lambda container image built with Packer and stored in ECR
+- AWS Lambda container image built with Packer and stored in ECR 
 - Infrastructure managed with Terraform
 - Automated CI/CD using GitHub Actions
 - Secret management using AWS Systems Manager Parameter Store
 
+## Repository Structure
+
+This project is split into two repositories:
+
+1. **template-automation-lambda** (this repository)
+   - Contains the Lambda function source code
+   - Builds the container image with Packer
+   - Publishes the image to ECR
+
+2. **terraform-aws-template-automation**
+   - Terraform module that deploys the Lambda infrastructure
+   - Creates and configures all required AWS resources
+   - Manages GitHub-specific configuration via SSM parameters
+   - Handles permissions, API Gateway, and other infrastructure
+
 ## Prerequisites
 
 - AWS credentials with appropriate permissions
@@ -23,6 +38,38 @@ with custom configurations from your template.
 - Packer
 - Python 3.11+
 
+## Configuration
+
+### Lambda Configuration
+
+The Lambda function gets its configuration from SSM Parameter Store with the following parameters:
+
+- `/template-automation/GITHUB_API` - GitHub API URL
+- `/template-automation/GITHUB_ORG_NAME` - GitHub organization name
+- `/template-automation/TEMPLATE_REPO_NAME` - Name of the template repository
+- `/template-automation/TEMPLATE_CONFIG_FILE` - Name of the config file (default: config.json)
+- `/template-automation/GITHUB_COMMIT_AUTHOR_NAME` - Name for commit author
+- `/template-automation/GITHUB_COMMIT_AUTHOR_EMAIL` - Email for commit author
+- `/template-automation/TEMPLATE_TOPICS` - Comma-separated list of repository topics
+
+### Terraform Module Configuration
+
+These parameters are managed by the `terraform-aws-template-automation` module. When deploying
+the Lambda function using the Terraform module, configure these variables in the module:
+
+```hcl
+module "template_automation" {
+  source = "github.com/HappyPathway/terraform-aws-template-automation"
+
+  # GitHub configuration
+  github_api_url            = "https://api.github.com"
+  github_org_name           = "your-org"
+  template_repo_name        = "your-template-repo"
+  
+  # Other module configuration...
+}
+```
+
 ## Local Development
 
 1. Clone this repository:
@@ -118,7 +165,7 @@ cd eks_automation
 python -m pytest tests/ -v -m integration
 ```
 
-Note: Integration tests will create temporary repositories in your GitHub organization. These repositories will be archived (not deleted) after the tests complete.
+Note: Integration tests will create temporary repositories in your GitHub organization. These repositories will be archived (not deleted) after the tests complete. To implement full cleanup, the test code would need to be modified to delete repositories instead of archiving them.
 
 ## Resources
 

diff --git a/main.tf b/main.tf
@@ -1,3 +1,8 @@
+# 
+# This Terraform configuration creates only the ECR repository for the container image.
+# The Lambda function, API Gateway, IAM roles, and other infrastructure are managed by 
+# the terraform-aws-template-automation module.
+
 provider "aws" {
   region = var.aws_region
 }

diff --git a/packer.pkr.hcl b/packer.pkr.hcl
@@ -26,14 +26,14 @@ source "docker" "lambda" {
 }
 
 build {
-  name = "eks-automation-lambda"
+  name = "template-automation-lambda"
 
   sources = [
     "source.docker.lambda"
   ]
 
   provisioner "file" {
-    source      = "./eks_automation/"
+    source      = "./template_automation/"
     destination = "/var/task"
   }
 
@@ -55,4 +55,4 @@ build {
       login_server = var.repository_uri
     }
   }
-}
+}
diff --git a/template_automation/tests/conftest.py b/template_automation/tests/conftest.py
@@ -1,6 +1,49 @@
-import pytest
 import os
-import json
+import pytest
+from github import Github
+import time
+
+@pytest.fixture(scope="session")
+def github_client():
+    """Create a GitHub client for integration tests."""
+    token = os.environ.get("GITHUB_TOKEN")
+    if not token:
+        pytest.skip("GITHUB_TOKEN environment variable not set")
+
+    api_url = os.environ.get("GITHUB_API", "https://api.github.com")
+    return Github(base_url=api_url, login_or_token=token)
+
+@pytest.fixture(scope="session")
+def cleanup_mode():
+    """Determine if repositories should be deleted or just archived."""
+    return os.environ.get("INTEGRATION_TEST_DELETE_REPOS", "").lower() in ("true", "1", "yes")
+
+@pytest.fixture
+def test_repo(github_client, cleanup_mode, request):
+    """Create a test repository and clean it up after the test."""
+    org_name = os.environ.get("GITHUB_ORG")
+    if not org_name:
+        pytest.skip("GITHUB_ORG environment variable not set")
+
+    # Create a unique repo name for this test
+    repo_name = f"test-repo-{pytest.config.getoption('--timestamp', default='')}-{id(request)}"
+
+    org = github_client.get_organization(org_name)
+    repo = org.create_repo(
+        repo_name,
+        description="Temporary repository for integration testing",
+        private=True
+    )
+
+    yield repo
+
+    # Clean up after the test
+    if cleanup_mode:
+        # Delete the repository
+        repo.delete()
+    else:
+        # Archive the repository (the original behavior)
+        repo.edit(archived=True)
 
 @pytest.fixture
 def github_client_params():
@@ -71,4 +114,9 @@ def mock_reference_response():
             "sha": "test-commit-sha",
             "type": "commit"
         }
-    }
+    }
+
+def pytest_addoption(parser):
+    """Add custom command line options."""
+    timestamp = int(time.time())
+    parser.addoption("--timestamp", action="store", default=str(timestamp))
diff --git a/template_automation/tests/integration/test_github_operations.py b/template_automation/tests/integration/test_github_operations.py
@@ -0,0 +1,16 @@
+import pytest
+import os
+
+@pytest.mark.integration
+def test_repository_operations(test_repo, cleanup_mode):
+    """Test basic repository operations."""
+    # Your test code here that uses the test_repo
+
+    # This is just an example verification
+    assert test_repo.name.startswith("test-repo-")
+
+    # Log what will happen to this repository
+    if cleanup_mode:
+        print(f"Repository {test_repo.name} will be DELETED after this test")
+    else:
+        print(f"Repository {test_repo.name} will be ARCHIVED after this test")
diff --git a/template_automation/tests/pytest.ini b/template_automation/tests/pytest.ini
@@ -0,0 +1,5 @@
+[pytest]
+markers =
+    integration: marks tests as integration tests (requires GitHub API access)
+addopts = --timestamp=%(timestamp)s
+python_functions = test_*
diff --git a/...aform_data_dirs/default/providers/registry.terraform.io/hashicorp/aws/5.96.0/darwin_amd64 b/...aform_data_dirs/default/providers/registry.terraform.io/hashicorp/aws/5.96.0/darwin_amd64
@@ -0,0 +1 @@
+/Users/darnold/.terraform.d/plugin-cache/registry.terraform.io/hashicorp/aws/5.96.0/darwin_amd64
diff --git a/terraform_data_dirs/default/terraform.tfstate b/terraform_data_dirs/default/terraform.tfstate
@@ -0,0 +1,19 @@
+{
+  "version": 3,
+  "terraform_version": "1.10.5",
+  "backend": {
+    "type": "gcs",
+    "config": {
+      "access_token": null,
+      "bucket": "hpw-terraform-state",
+      "credentials": null,
+      "encryption_key": null,
+      "impersonate_service_account": null,
+      "impersonate_service_account_delegates": null,
+      "kms_encryption_key": null,
+      "prefix": "eks-automation-lambda",
+      "storage_custom_endpoint": null
+    },
+    "hash": 3602047810
+  }
+}
diff --git a/variables.tf b/variables.tf
@@ -1,3 +1,11 @@
+
+# Note: GitHub-specific variables (github_api, github_org_name, template_repo_name, etc.)
+# have been moved to the terraform-aws-template-automation module.
+# They are now configured as SSM parameters in that module.
+#
+# This file contains only variables related to the container image and
+# ECR repository setup.
+
 variable "aws_region" {
   description = "AWS region where resources will be created"
   type        = string
@@ -16,36 +24,6 @@ variable "environment" {
   default     = "production"
 }
 
-variable "github_api" {
-  description = "URL for the GitHub Enterprise API"
-  type        = string
-  default     = "https://github.enterprise.example.com/api/v3"
-}
-
-variable "github_org_name" {
-  description = "GitHub organization name"
-  type        = string
-  default     = "your-org"
-}
-
-variable "github_token_secret_name" {
-  description = "AWS SSM parameter name for the GitHub token"
-  type        = string
-  default     = "/github/token"
-}
-
-variable "template_repo_name" {
-  description = "GitHub repository name for the EKS template"
-  type        = string
-  default     = "template-eks-cluster"
-}
-
-variable "template_file_name" {
-  description = "Template file name for the EKS configuration"
-  type        = string
-  default     = "eks.hcl.j2"
-}
-
 variable "catalog_data" {
   description = "Configuration for the ECR repository catalog data"
   type = object({