diff --git a/Makefile b/Makefile index bca92c5f..cbb129ea 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ PYTHON = python3 PIP = $(PYTHON) -m pip PYTEST = $(PYTHON) -m pytest -REQUIREMENTS = eks_automation/requirements.txt -TEST_DIR = eks_automation/tests -UNIT_TEST_FILE = $(TEST_DIR)/test_github_client.py -INTEGRATION_TEST_FILE = $(TEST_DIR)/test_github_client_integration.py +REQUIREMENTS = requirements.txt +TEST_DIR = tests +UNIT_TEST_FILES = $(TEST_DIR)/test_app.py $(TEST_DIR)/test_github_client.py +INTEGRATION_TEST_DIR = $(TEST_DIR)/integration # Default target all: test @@ -15,6 +15,7 @@ all: test # Install dependencies install: $(PIP) install -r $(REQUIREMENTS) + $(PIP) install -e . # Run all tests test: test-unit test-integration @@ -24,12 +25,12 @@ test: test-unit test-integration # Run unit tests test-unit: @echo "Running unit tests..." - $(PYTEST) $(UNIT_TEST_FILE) + $(PYTEST) $(UNIT_TEST_FILES) -v # Run integration tests test-integration: @echo "Running integration tests..." - $(PYTEST) $(INTEGRATION_TEST_FILE) + $(PYTEST) $(INTEGRATION_TEST_DIR) -v # Clean up Python cache files clean: diff --git a/README.md b/README.md index cc5462a4..8ffe603f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This Lambda function automates the creation and configuration of new repositories from templates using **CloudFormation Custom Resources**. When invoked as a Custom Resource in a CloudFormation stack (typically through AWS Service Catalog), this Lambda function automatically creates a GitHub/GitLab repository with the appropriate configuration. -This implementation uses CloudFormation Custom Resources to enable Service Catalog integration - the Lambda is invoked directly by CloudFormation during stack provisioning. +This implementation uses CloudFormation Custom Resources to enable Service Catalog integration — the Lambda is invoked directly by CloudFormation during stack provisioning. ## Key Features @@ -15,7 +15,114 @@ This implementation uses CloudFormation Custom Resources to enable Service Catal - **Multi-Provider**: Supports both GitHub and GitLab - **Automatic PR Creation**: Creates pull request with configuration -## Architecture +## Repository Structure + +``` +lambda-template-repo-generator/ +├── main.tf # Step 1: Terraform for ECR repository +├── variables.tf # ECR Terraform variables +├── backend.tf # Terraform state backend +├── varfiles/ # Terraform/Packer variable files +│ ├── default.tfvars # ECR Terraform vars +│ └── packer.pkrvars.hcl # Packer variable overrides +├── config_packer.hcl # Step 2: Packer Pipeline configuration +├── packer.pkr.hcl # Packer template (container image build) +├── deploy/ # Step 3: Lambda deployment Terraform +│ ├── main.tf # Uses terraform-aws-template-automation module +│ ├── variables.tf # Deployment variables +│ └── terraform.tfvars.example +├── template_automation/ # Lambda source code +│ ├── app.py # Lambda handler (CloudFormation events) +│ ├── github_provider.py # GitHub API provider +│ ├── gitlab_provider.py # GitLab API provider +│ ├── repository_provider.py # Abstract provider interface +│ ├── models.py # Pydantic models +│ └── template_manager.py # Jinja2 template rendering +├── tests/ # Unit and integration tests +├── cloudformation-template.yaml # Example Service Catalog product +├── requirements.txt # Python dependencies +└── Makefile # Development commands +``` + +## Build & Deploy Workflow + +The deployment is a three-step process: + +``` +Step 1: Create ECR Repository Step 2: Build Container Image Step 3: Deploy Lambda Function +───────────────────────────── ────────────────────────────── ────────────────────────────── + terraform apply packer-pipeline terraform apply + (root main.tf) (config_packer.hcl) (deploy/ directory) + │ │ │ + ▼ ▼ ▼ + ECR Repository Created CodeBuild Project Lambda Function + (stores container images) Builds Docker Image IAM Roles & Policies + Pushes to ECR API Gateway + SSM Parameters + CloudWatch Log Group + CloudFormation Permission +``` + +### Step 1: Create the ECR Repository + +The root-level Terraform configuration creates an ECR public repository to store the Lambda container image. + +```bash +# Initialize and apply the ECR Terraform +cd lambda-template-repo-generator/ +terraform init +terraform apply -var-file=varfiles/default.tfvars +``` + +This creates: +- An ECR public repository (`template-automation-lambda`) +- Outputs the `repository_uri` needed by the Packer build + +### Step 2: Build the Container Image with Packer Pipeline + +The [packer-pipeline](https://github.e.it.census.gov/SCT-Engineering/packer-pipeline) utility builds the Lambda container image via AWS CodeBuild and pushes it to the ECR repository created in Step 1. + +```bash +# Run the packer pipeline to build and push the container image +python -m packer_pipeline --config config_packer.hcl +``` + +This: +1. Zips the repository source code and uploads it to S3 +2. Creates/updates an AWS CodeBuild project +3. CodeBuild runs the Packer template (`packer.pkr.hcl`) which: + - Starts from the `public.ecr.aws/lambda/python:3.11` base image + - Copies application code into `/var/task` + - Installs Python dependencies from `requirements.txt` + - Tags the image and pushes to the ECR repository + +The `config_packer.hcl` file contains all the build configuration including VPC settings, environment variables, and ECR clone configuration for the base image. + +### Step 3: Deploy the Lambda Function + +The `deploy/` directory contains Terraform that uses the [terraform-aws-template-automation](https://github.com/HappyPathway/terraform-aws-template-automation) module to deploy the Lambda function and all supporting infrastructure. + +```bash +# Copy and edit the example tfvars +cd deploy/ +cp terraform.tfvars.example terraform.tfvars +# Edit terraform.tfvars with your values + +# Initialize and apply +terraform init +terraform apply +``` + +This deploys: +- **Lambda Function** — container image from ECR, with configurable memory/timeout/VPC +- **IAM Role & Policies** — permissions for SSM, Secrets Manager, KMS, CloudWatch, VPC +- **API Gateway** — HTTP API with POST /template endpoint (alternative invocation) +- **SSM Parameters** — GitHub API URL, org name, template repo, commit author settings +- **CloudWatch Log Group** — managed with configurable retention +- **Lambda Permission** — allows CloudFormation to invoke the function as a Custom Resource +- **Secrets Manager** — stores the GitHub token (optional creation via Terraform) + +## Runtime Architecture ``` Service Catalog Product (CloudFormation Template) @@ -88,7 +195,7 @@ All other parameters in `ResourceProperties` are collected and stored in the rep - `OwningTeam` → `owning_team` - `AwsRegion` → `aws_region` -## Workflow +## Lambda Workflow (Runtime) 1. **User Provisions**: User provisions Service Catalog product 2. **CloudFormation Runs**: CloudFormation stack is created @@ -114,3 +221,39 @@ The Lambda creates a `config.json` file in the repository with this structure: "tags": {} } ``` + +## Environment Variables + +The Lambda function uses the following environment variables (configured via SSM Parameters and Terraform): + +| Variable | Description | Source | +|----------|-------------|--------| +| `GITHUB_API` | GitHub API URL | SSM Parameter | +| `GITHUB_TOKEN_SECRET_NAME` | Secrets Manager secret name for GitHub token | Lambda env var | +| `GITHUB_ORG_NAME` | GitHub organization name | SSM Parameter | +| `TEMPLATE_REPO_NAME` | Template repository to clone from | SSM Parameter | +| `TEMPLATE_CONFIG_FILE` | Config filename (default: `config.json`) | SSM Parameter | +| `VERIFY_SSL` | Enable/disable SSL verification | Lambda env var | +| `PARAM_STORE_PREFIX` | SSM parameter path prefix | Lambda env var | + +## Development + +### Running Tests + +```bash +# Install dependencies +pip install -r requirements.txt +pip install -e . + +# Run unit tests +python -m pytest tests/test_app.py tests/test_github_client.py -v + +# Run integration tests (requires GitHub token) +python -m pytest tests/integration/ -v +``` + +### Related Repositories + +- [packer-pipeline](https://github.e.it.census.gov/SCT-Engineering/packer-pipeline) — Build tool for container images via CodeBuild +- [terraform-aws-template-automation](https://github.com/HappyPathway/terraform-aws-template-automation) — Terraform module for Lambda deployment +- [template-automation-lambda](https://github.e.it.census.gov/SCT-Engineering/template-automation-lambda) — Related Lambda implementation diff --git a/deploy/main.tf b/deploy/main.tf index 81132c5f..8383e57c 100644 --- a/deploy/main.tf +++ b/deploy/main.tf @@ -60,15 +60,6 @@ module "service_catalog_repo_generator" { tags = var.tags } -# Grant CloudFormation permission to invoke Lambda -# This allows any CloudFormation stack to invoke the Lambda as a Custom Resource -resource "aws_lambda_permission" "cloudformation" { - statement_id = "AllowCloudFormationInvoke" - action = "lambda:InvokeFunction" - function_name = module.service_catalog_repo_generator.lambda_function_name - principal = "cloudformation.amazonaws.com" -} - # Outputs output "lambda_function_arn" { description = "ARN of the deployed Lambda function - use this as ServiceToken in CloudFormation" @@ -80,6 +71,16 @@ output "lambda_function_name" { value = module.service_catalog_repo_generator.lambda_function_name } +output "cloudwatch_log_group" { + description = "CloudWatch Log Group for Lambda function logs" + value = module.service_catalog_repo_generator.cloudwatch_log_group_name +} + +output "api_endpoint" { + description = "API Gateway endpoint (alternative invocation method)" + value = module.service_catalog_repo_generator.api_endpoint +} + output "cloudformation_template_example" { description = "Example CloudFormation Custom Resource definition" value = <<-EOT