From 813e00fdbe62c1cc1fad8f18ef1ba33bbd3295c6 Mon Sep 17 00:00:00 2001 From: David Arnold <10138997+djaboxx@users.noreply.github.com> Date: Fri, 25 Apr 2025 12:22:59 -0700 Subject: [PATCH] Template automation (#11) * Add initial setup for pytest and GitHubClient tests - Created pytest configuration file `pytest.ini` for test discovery and options. - Added `requirements.txt` for project dependencies including testing libraries. - Introduced `test_payload.json` for storing test data related to EKS settings. - Established a test package with `__init__.py` in the `tests` directory. - Implemented fixtures in `conftest.py` for mocking GitHub API responses. - Developed unit tests for `GitHubClient` methods in `test_github_client.py`. - Created integration tests for `GitHubClient` in `test_github_client_integration.py`. - Added environment cleanup fixture to ensure a clean state for tests. - Implemented unit tests for application logic in `test_app.py`, including SSM parameter retrieval and GitHub operations. * Refactor GitHub token handling to use GITHUB_TOKEN_SECRET_NAME * Add team-based admin access feature to GitHubClient and integration tests --------- Co-authored-by: Dave Arnold --- .github/workflows/integration-tests.yml | 10 +- README.md | 27 +- eks_automation/__init__.py | 0 .../github-actions-trust-policy.json | 20 -- infrastructure/main.tf | 99 ------- infrastructure/outputs.tf | 14 - infrastructure/providers.tf | 12 - infrastructure/variables.tf | 15 - main.tf | 43 +-- setup.py | 17 ++ template-automation-lambda.code-workspace | 17 ++ template_automation/__init__.py | 1 + .../app.py | 264 +++++++++++++++--- .../pytest.ini | 0 .../requirements.txt | 0 .../test_payload.json | 0 .../tests/__init__.py | 0 .../tests/conftest.py | 0 .../tests/test_github_client.py | 0 .../tests/test_github_client_integration.py | 0 tests/conftest.py | 15 + tests/test_app.py | 110 ++++++++ tests/test_github_client.py | 71 +++++ tests/test_integration.py | 2 +- varfiles/default.tfvars | 16 ++ variables.tf | 44 +++ 26 files changed, 556 insertions(+), 241 deletions(-) delete mode 100644 eks_automation/__init__.py delete mode 100644 infrastructure/github-actions-trust-policy.json delete mode 100644 infrastructure/main.tf delete mode 100644 infrastructure/outputs.tf delete mode 100644 infrastructure/providers.tf delete mode 100644 infrastructure/variables.tf create mode 100644 setup.py create mode 100644 template-automation-lambda.code-workspace create mode 100644 template_automation/__init__.py rename {eks_automation => template_automation}/app.py (70%) rename {eks_automation => template_automation}/pytest.ini (100%) rename {eks_automation => template_automation}/requirements.txt (100%) rename {eks_automation => template_automation}/test_payload.json (100%) rename {eks_automation => template_automation}/tests/__init__.py (100%) rename {eks_automation => template_automation}/tests/conftest.py (100%) rename {eks_automation => template_automation}/tests/test_github_client.py (100%) rename {eks_automation => template_automation}/tests/test_github_client_integration.py (100%) create mode 100644 tests/conftest.py create mode 100644 tests/test_app.py create mode 100644 tests/test_github_client.py diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 73dfb5b..03b10fc 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -21,20 +21,20 @@ jobs: with: python-version: '3.9' cache: 'pip' - cache-dependency-path: eks_automation/requirements.txt + cache-dependency-path: template_automation/requirements.txt - name: Install dependencies run: | - cd eks_automation + cd template_automation python -m pip install --upgrade pip pip install -r requirements.txt - name: Run integration tests env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + GITHUB_TOKEN_SECRET_NAME: /dev/secret/ssh/dont/tell GITHUB_API: "https://api.github.com" # Can be overridden with vars if needed - GITHUB_ORG: ${{ github.repository_owner }} - SECRET_NAME: /dev/secret/ssh/dont/tell + GITHUB_ORG_NAME: ${{ github.repository_owner }} run: | - cd eks_automation + cd template_automation python -m pytest tests/ -v -m integration \ No newline at end of file diff --git a/README.md b/README.md index 061b2d5..77d33a5 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# EKS Automation Lambda +# Template Automation Lambda ## Description This repository contains source code and supporting files for a serverless Lambda container application. The application uses an AWS Lambda function to process JSON input and write it to a cloned repository. The changes are then committed and pushed to your GitHub Enterprise Server, creating a new repository -for the EKS CI/CD pipeline. +with custom configurations from your template. ## Architecture @@ -27,13 +27,13 @@ for the EKS CI/CD pipeline. 1. Clone this repository: ```sh - git clone /eks-automation-lambda.git - cd eks-automation-lambda + git clone /template-automation-lambda.git + cd template-automation-lambda ``` 2. Install Python dependencies: ```sh - cd eks_automation + cd template_automation pip install -r requirements.txt ``` @@ -76,24 +76,17 @@ The Lambda function accepts JSON input in the following format: ```json { "project_name": "string", - "eks_settings": { + "template_settings": { "attrs": { "account_name": "my-account", "aws_region": "us-east-1", - "cluster_mailing_list": "someone@example.com", - "cluster_name": "my-eks-cluster", - "eks_instance_disk_size": 100, - "eks_ng_desired_size": 2, - "eks_ng_max_size": 10, - "eks_ng_min_size": 2, + "team_contact": "someone@example.com", + "project_name": "my-project", "environment": "development", "environment_abbr": "dev", "organization": "my-org:my-division:my-team", - "finops_project_name": "my_project_baseline", - "finops_project_number": "fp00000001", - "finops_project_role": "my_project_baseline_app", - "vpc_domain_name": "dev.example.com", - "vpc_name": "vpc-dev" + "project_id": "proj_001", + "domain_name": "dev.example.com" }, "tags": { "slim:schedule": "8:00-17:00" diff --git a/eks_automation/__init__.py b/eks_automation/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/infrastructure/github-actions-trust-policy.json b/infrastructure/github-actions-trust-policy.json deleted file mode 100644 index 684318a..0000000 --- a/infrastructure/github-actions-trust-policy.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam::${AWS_ACCOUNT_ID}:oidc-provider/token.actions.githubusercontent.com" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringLike": { - "token.actions.githubusercontent.com:sub": "repo:${YOUR_GITHUB_ORG}/${YOUR_REPO_NAME}:*" - }, - "StringEquals": { - "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" - } - } - } - ] -} diff --git a/infrastructure/main.tf b/infrastructure/main.tf deleted file mode 100644 index 39b850a..0000000 --- a/infrastructure/main.tf +++ /dev/null @@ -1,99 +0,0 @@ -# IAM Role for CodeBuild -resource "aws_iam_role" "codebuild" { - name = "eks-automation-lambda-codebuild-role" - - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Action = "sts:AssumeRole" - Effect = "Allow" - Principal = { - Service = "codebuild.amazonaws.com" - } - } - ] - }) -} - -# IAM Role Policy for CodeBuild -resource "aws_iam_role_policy" "codebuild" { - name = "eks-automation-lambda-codebuild-policy" - role = aws_iam_role.codebuild.id - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Resource = ["*"] - Action = [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ] - }, - { - Effect = "Allow" - Resource = ["*"] - Action = [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:PutImage", - "ecr:InitiateLayerUpload", - "ecr:UploadLayerPart", - "ecr:CompleteLayerUpload", - "codebuild:StartBuild", - "codebuild:BatchGetBuilds", - "codebuild:StopBuild" - ] - }, - "ecr:CompleteLayerUpload" - ] - } - ] - }) -} - -# CodeBuild Project -resource "aws_codebuild_project" "lambda_builder" { - name = "eks-automation-lambda-builder" - service_role = aws_iam_role.codebuild.arn - build_timeout = "30" - - artifacts { - type = "NO_ARTIFACTS" - } - - environment { - compute_type = "BUILD_GENERAL1_SMALL" - image = "aws/codebuild/amazonlinux2-x86_64-standard:4.0" - type = "LINUX_CONTAINER" - image_pull_credentials_type = "CODEBUILD" - privileged_mode = true - - environment_variable { - name = "REPOSITORY_URI" - value = var.repository_uri - } - } - - source { - type = "GITHUB" - location = var.github_repo_url - git_clone_depth = 1 - buildspec = "buildspec.yml" - } - - cache { - type = "NO_CACHE" - } - - logs_config { - cloudwatch_logs { - status = "ENABLED" - } - } -} diff --git a/infrastructure/outputs.tf b/infrastructure/outputs.tf deleted file mode 100644 index 9d417b0..0000000 --- a/infrastructure/outputs.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "codebuild_project_name" { - description = "Name of the CodeBuild project" - value = aws_codebuild_project.lambda_builder.name -} - -output "codebuild_project_arn" { - description = "ARN of the CodeBuild project" - value = aws_codebuild_project.lambda_builder.arn -} - -output "iam_role_arn" { - description = "ARN of the IAM role used by CodeBuild" - value = aws_iam_role.codebuild.arn -} diff --git a/infrastructure/providers.tf b/infrastructure/providers.tf deleted file mode 100644 index 3a69900..0000000 --- a/infrastructure/providers.tf +++ /dev/null @@ -1,12 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -provider "aws" { - region = var.aws_region -} diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf deleted file mode 100644 index ad68078..0000000 --- a/infrastructure/variables.tf +++ /dev/null @@ -1,15 +0,0 @@ -variable "aws_region" { - description = "AWS region where resources will be created" - type = string - default = "us-west-2" -} - -variable "repository_uri" { - description = "The URI of the ECR repository where the Lambda image will be pushed" - type = string -} - -variable "github_repo_url" { - description = "The HTTPS clone URL of the GitHub repository" - type = string -} diff --git a/main.tf b/main.tf index baf5863..4c63845 100644 --- a/main.tf +++ b/main.tf @@ -1,49 +1,52 @@ provider "aws" { - region = "us-east-1" + region = var.aws_region } data "aws_caller_identity" "current" {} -resource "aws_ecrpublic_repository" "eks-automation-lambda" { - repository_name = "eks-automation-lambda" +resource "aws_ecrpublic_repository" "ecr_repo" { + repository_name = var.repository_name catalog_data { - about_text = "EKS Automation Lambda Image" - architectures = ["x86_64"] - description = "Lambda container image for EKS automation" - operating_systems = ["AmazonLinux2"] - usage_text = "Creates an EKS Automation Lambda container image" + about_text = var.catalog_data.about_text + architectures = var.catalog_data.architectures + description = var.catalog_data.description + operating_systems = var.catalog_data.operating_systems + usage_text = var.catalog_data.usage_text } - tags = { - env = "production" - } + tags = var.tags } locals { - repository_uri = aws_ecrpublic_repository.eks-automation-lambda.repository_uri - repository_id = aws_ecrpublic_repository.eks-automation-lambda.id + repository_uri = aws_ecrpublic_repository.ecr_repo.repository_uri + repository_id = aws_ecrpublic_repository.ecr_repo.id aws_account_id = data.aws_caller_identity.current.account_id - region = "us-east-1" - arn = aws_ecrpublic_repository.eks-automation-lambda.arn + region = var.aws_region + arn = aws_ecrpublic_repository.ecr_repo.arn } output "repository_uri" { - value = local.repository_uri + description = "The URI of the ECR repository" + value = local.repository_uri } output "repository_id" { - value = local.repository_id + description = "The ID of the ECR repository" + value = local.repository_id } output "aws_account_id" { - value = local.aws_account_id + description = "The ID of the AWS account" + value = local.aws_account_id } output "region" { - value = local.region + description = "The AWS region where resources are created" + value = local.region } output "arn" { - value = local.arn + description = "The ARN of the ECR repository" + value = local.arn } \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d4492c1 --- /dev/null +++ b/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup, find_packages + +setup( + name="template-automation", + version="0.1.0", + packages=find_packages(), + install_requires=[ + "boto3", + "requests" + ], + extras_require={ + "test": [ + "pytest", + "pytest-mock" + ] + } +) diff --git a/template-automation-lambda.code-workspace b/template-automation-lambda.code-workspace new file mode 100644 index 0000000..e25aa1f --- /dev/null +++ b/template-automation-lambda.code-workspace @@ -0,0 +1,17 @@ +{ + "folders": [ + { + "path": "." + }, + { + "path": "../terraform-aws-template-automation" + }, + { + "path": "../providers/terraform-provider-aws/website/docs/d" + }, + { + "path": "../providers/terraform-provider-aws/website/docs/r" + } + ], + "settings": {} +} \ No newline at end of file diff --git a/template_automation/__init__.py b/template_automation/__init__.py new file mode 100644 index 0000000..8074dd0 --- /dev/null +++ b/template_automation/__init__.py @@ -0,0 +1 @@ +# Package initialization diff --git a/eks_automation/app.py b/template_automation/app.py similarity index 70% rename from eks_automation/app.py rename to template_automation/app.py index 12ea354..5242cc8 100644 --- a/eks_automation/app.py +++ b/template_automation/app.py @@ -1,8 +1,27 @@ #################################################################################### -# This Lambda function takes JSON input and writes it directly to a config.json file -# in a cloned GitHub repository. The changes are then committed and pushed to the -# GitHub API, creating a new repository for the EKS CI/CD pipeline. -# This implementation uses only pure Python with requests library (no Git CLI dependency). +# This Lambda function creates new GitHub repositories from a template repository. +# It takes JSON input and writes it to a configurable config file in the new repo. +# Key features: +# - Template agnostic: Can be used with any type of template repository +# - Team-based admin access: Set owning team with full admin access +# - Configurable settings via Parameter Store (with prefix) or environment variables: +# - PARAM_STORE_PREFIX: Prefix for SSM parameters (default: /template-automation) +# - TEMPLATE_CONFIG_FILE: Name of config file to write (default: config.json) +# - TEMPLATE_TOPICS: Comma-separated list of topics to add (default: infrastructure) +# - TEMPLATE_REPO_NAME: Source template repository name (required) +# - TEMPLATE_SOURCE_VERSION: Version/tag/SHA to use from template (optional) +# - REPO_NAME_PREFIX: Prefix for generated repository names (optional) +# - GITHUB_API: GitHub API URL (required) +# - GITHUB_ORG_NAME: GitHub organization name (required) +# - GITHUB_COMMIT_AUTHOR_NAME: Name for commits (default: Template Automation) +# - GITHUB_COMMIT_AUTHOR_EMAIL: Email for commits (default: automation@example.com) +# - SECRET_NAME: AWS Secrets Manager secret containing GitHub token (required) +# +# Repository naming: +# - If REPO_NAME_PREFIX is set: Creates repos named {prefix}-{random-8-chars} +# - If not set: Uses the provided project name directly +# +# Implementation uses pure Python with requests library (no Git CLI dependency). #################################################################################### import os @@ -17,6 +36,7 @@ from urllib.parse import urlparse from datetime import datetime import traceback +import uuid import boto3 from botocore.exceptions import ClientError @@ -29,7 +49,10 @@ logger.setLevel("INFO") # Set to "ERROR" to reduce logging messages. # Get environment variables -SECRET_NAME = os.environ["SECRET_NAME"] +GITHUB_TOKEN_SECRET_NAME = os.environ.get("GITHUB_TOKEN_SECRET_NAME") +DEFAULT_CONFIG_FILE = os.environ.get("TEMPLATE_CONFIG_FILE", "config.json") +DEFAULT_TOPICS = os.environ.get("TEMPLATE_TOPICS", "infrastructure").split(",") +PARAM_STORE_PREFIX = os.environ.get("PARAM_STORE_PREFIX", "/template-automation") class GitHubClient: """A class to interact with GitHub API without relying on external Git binaries. @@ -73,12 +96,13 @@ def _create_headers(self): "Content-Type": "application/json" } - def get_repository(self, repo_name, create=False): + def get_repository(self, repo_name, create=False, owning_team=None): """Get or create a repository Args: repo_name (str): Name of the repository create (bool, optional): Create the repository if it doesn't exist + owning_team (str, optional): Name of the GitHub team to give admin access Returns: dict: Repository information from GitHub API @@ -87,6 +111,9 @@ def get_repository(self, repo_name, create=False): try: response = requests.get(get_url, headers=self.headers, verify=False) if response.status_code == 200: + # If owning team is specified and repo exists, ensure the team has admin access + if owning_team: + self.set_team_permission(repo_name, owning_team, "admin") return response.json() elif response.status_code == 404 and create: logger.info(f"Creating repository {repo_name}") @@ -118,6 +145,9 @@ def get_repository(self, repo_name, create=False): try: # Try to get the main branch's reference self.get_reference_sha(repo_name, "heads/main") + # If owning team is specified, give them admin access + if owning_team: + self.set_team_permission(repo_name, owning_team, "admin") return repo except Exception: # If reference doesn't exist yet, wait and retry @@ -569,35 +599,177 @@ def commit_repository_contents(self, repo_name, work_dir, commit_message, branch return target_branch -def operate_github(new_repo_name, eks_settings): - """Write EKS settings to config.json and create/update repository using GitHub API + def _get_secret_value(self): + """Retrieve GitHub token from AWS Secrets Manager""" + try: + session = boto3.session.Session() + client = session.client( + service_name='secretsmanager', + region_name=os.environ.get('AWS_REGION', 'us-east-1') + ) + + response = client.get_secret_value(SecretId=GITHUB_TOKEN_SECRET_NAME) + if 'SecretString' in response: + return response['SecretString'] + raise ValueError("Secret value not found in response") + + except ClientError as e: + logger.error(f"Failed to retrieve secret: {str(e)}") + raise Exception("Failed to retrieve GitHub token from Secrets Manager") from e + + def trigger_workflow(self, repo_name, workflow_id="init-repo.yml", ref="main", inputs=None): + """Trigger a GitHub Actions workflow in the repository + + Args: + repo_name (str): Name of the repository + workflow_id (str): The ID or filename of the workflow to trigger + ref (str): The git reference to run the workflow on + inputs (dict, optional): Input parameters for the workflow + + Returns: + dict: Response from the workflow dispatch API + """ + api_url = f"{self.api_base_url}/repos/{self.org_name}/{repo_name}/actions/workflows/{workflow_id}/dispatches" + + data = { + "ref": ref, + } + if inputs: + data["inputs"] = inputs + + response = requests.post(api_url, headers=self.headers, json=data, verify=False) + + if response.status_code == 204: # GitHub returns 204 No Content for successful workflow dispatch + logger.info(f"Successfully triggered workflow {workflow_id} in {repo_name}") + return {"status": "success"} + else: + error_message = f"Failed to trigger workflow {workflow_id} in {repo_name}: {response.status_code} - {response.text}" + logger.error(error_message) + raise Exception(error_message) + + def set_team_permission(self, repo_name, team_name, permission): + """Set a team's permission on a repository + + Args: + repo_name (str): Name of the repository + team_name (str): Name of the team + permission (str): Permission level (pull, push, admin) + """ + # First get the team ID from the team name + team_url = f"{self.api_base_url}/orgs/{self.org_name}/teams/{team_name}" + try: + team_response = requests.get(team_url, headers=self.headers, verify=False) + if team_response.status_code != 200: + error_message = f"Failed to get team {team_name}: {team_response.status_code} - {team_response.text}" + logger.error(error_message) + raise Exception(error_message) + + team_id = team_response.json()["id"] + + # Add team to repository with specified permission + perm_url = f"{self.api_base_url}/orgs/{self.org_name}/teams/{team_name}/repos/{self.org_name}/{repo_name}" + data = { + "permission": permission + } + + response = requests.put(perm_url, headers=self.headers, json=data, verify=False) + + if response.status_code not in (200, 204): + error_message = f"Failed to set team permission: {response.status_code} - {response.text}" + logger.error(error_message) + raise Exception(error_message) + + logger.info(f"Successfully set {permission} permission for team {team_name} on {repo_name}") + + except requests.exceptions.RequestException as e: + error_message = f"Error setting team permission: {str(e)}" + logger.error(error_message) + raise Exception(error_message) + +def generate_repository_name(project_name): + """Generate repository name based on prefix or project name + + Args: + project_name (str): Name of the project + + Returns: + str: Generated repository name + """ + prefix = os.environ.get("REPO_NAME_PREFIX") + if prefix: + # Generate a short random string (first 8 chars of UUID) + random_suffix = str(uuid.uuid4())[:8] + return f"{prefix}-{random_suffix}" + return project_name + +def get_parameter(name, default=None, decrypt=False): + """Get parameter from SSM Parameter Store or environment variable + + Args: + name (str): Parameter name + default (str, optional): Default value if not found + decrypt (bool, optional): Whether to decrypt the value + + Returns: + str: Parameter value + """ + # Try Parameter Store first with configured prefix + ssm = boto3.client('ssm') + param_name = f"{PARAM_STORE_PREFIX}/{name}" + try: + response = ssm.get_parameter(Name=param_name, WithDecryption=decrypt) + return response['Parameter']['Value'] + except ssm.exceptions.ParameterNotFound: + # Fall back to environment variable + return os.environ.get(name, default) + except Exception as e: + logger.warning(f"Error getting parameter {name}: {str(e)}") + return os.environ.get(name, default) + +def operate_github(new_repo_name, template_settings, trigger_init_workflow=False, owning_team=None): + """Write template settings to config file and create/update repository using GitHub API This implementation uses only the requests library and does not rely on git CLI or any external binaries. Args: new_repo_name (str): Name of the new GitHub repo - eks_settings (json): Input JSON data with all the EKS parameter values + template_settings (json): Input JSON data with all parameter values + trigger_init_workflow (bool): Whether to trigger the init-repo workflow after setup + owning_team (str, optional): Name of the GitHub team to give admin access Returns: None """ logger.info("Starting GitHub repository operation") - logger.info(f"Target repository name: {new_repo_name}") - + + # Generate the actual repository name + actual_repo_name = generate_repository_name(new_repo_name) + logger.info(f"Using repository name: {actual_repo_name}") + token = github_token() logger.info("Successfully retrieved GitHub token from Secrets Manager") - github_api = os.environ.get("GITHUB_API") # No default - must be configured - org_name = os.environ.get("GITHUB_ORG_NAME") # No default - must be configured - commit_author_email = os.environ.get("GITHUB_COMMIT_AUTHOR_EMAIL", "eks-automation@example.com") - commit_author_name = os.environ.get("GITHUB_COMMIT_AUTHOR_NAME", "EKS Automation Lambda") - source_version = os.environ.get("TEMPLATE_SOURCE_VERSION") # Optional - template_repo_name = os.environ.get("TEMPLATE_REPO_NAME", "template-eks-cluster") - config_file_name = "config.json" + # Get all configuration from Parameter Store or environment variables + github_api = get_parameter("GITHUB_API") # Required + if not github_api: + raise ValueError("GITHUB_API must be configured in Parameter Store or environment") + org_name = get_parameter("GITHUB_ORG_NAME") # Required + if not org_name: + raise ValueError("GITHUB_ORG_NAME must be configured in Parameter Store or environment") + + commit_author_email = get_parameter("GITHUB_COMMIT_AUTHOR_EMAIL", "automation@example.com") + commit_author_name = get_parameter("GITHUB_COMMIT_AUTHOR_NAME", "Template Automation") + source_version = get_parameter("TEMPLATE_SOURCE_VERSION") + template_repo_name = get_parameter("TEMPLATE_REPO_NAME") # Required + if not template_repo_name: + raise ValueError("TEMPLATE_REPO_NAME must be configured in Parameter Store or environment") + + config_file_name = get_parameter("TEMPLATE_CONFIG_FILE", DEFAULT_CONFIG_FILE) + # Create work directory if it doesn't exist - work_dir = f"/tmp/{new_repo_name}" + work_dir = f"/tmp/{actual_repo_name}" if os.path.exists(work_dir): shutil.rmtree(work_dir, ignore_errors=False, onerror=remove_readonly) os.makedirs(work_dir, exist_ok=True) @@ -641,33 +813,44 @@ def operate_github(new_repo_name, eks_settings): logger.info(f"Using source commit SHA: {source_commit_sha}") # Get or create the new repository - logger.info(f"Getting or creating repository: {new_repo_name}") - new_repo = github.get_repository(new_repo_name, create=True) + logger.info(f"Getting or creating repository: {actual_repo_name}") + new_repo = github.get_repository(actual_repo_name, create=True, owning_team=owning_team) # Clone the original repository contents from specific commit tree = github.get_tree(template_repo_name, source_commit_sha, recursive=True) github.download_repository_files(template_repo_name, tree, work_dir) - # Write EKS settings directly to config.json + # Write template settings to config file output_file_path = os.path.join(work_dir, config_file_name) - logger.info(f"Writing EKS settings to {output_file_path}") + logger.info(f"Writing template settings to {output_file_path}") with open(output_file_path, "w") as file: - json.dump(eks_settings, file, indent=2) + json.dump(template_settings, file, indent=2) + + # Write version information to .source-version file if source_version is specified + if source_version: + version_file_path = os.path.join(work_dir, ".source-version") + logger.info(f"Writing version information to {version_file_path}") + with open(version_file_path, "w") as file: + file.write(source_version) # Commit all files to the new repository's main branch explicitly - commit_message = "Add the EKS configuration file by the Lambda function" - github.commit_repository_contents(new_repo_name, work_dir, commit_message, branch="main") + commit_message = "Add template configuration by automation" + github.commit_repository_contents(actual_repo_name, work_dir, commit_message, branch="main") + + # Add configurable topics to the repository + topics = DEFAULT_TOPICS - # Add relevant topics to the repository including EKS commit SHA - topics = [ - "eks", - "kubernetes", - "infrastructure", - f"eks:{source_commit_sha[:7]}" # Use first 7 chars of SHA - ] - github.update_repository_topics(new_repo_name, topics) + github.update_repository_topics(actual_repo_name, topics) - logger.info(f"Successfully updated {new_repo_name} repository") + logger.info(f"Successfully updated {actual_repo_name} repository") + + # Trigger init workflow if requested + if trigger_init_workflow: + try: + github.trigger_workflow(actual_repo_name) + logger.info("Successfully triggered init-repo workflow") + except Exception as e: + logger.warning(f"Failed to trigger init-repo workflow: {str(e)}") def github_token(): """Retrieve GitHub access token from AWS Secrets Manager @@ -711,9 +894,14 @@ def lambda_handler(event, context): logger.info(f"Extracted input data from event body: {json.dumps(input_data, indent=2)}") project_name = input_data.get("project_name") - eks_settings = input_data.get("eks_settings") + template_settings = input_data.get("template_settings") + trigger_init_workflow = input_data.get("trigger_init_workflow", False) + owning_team = input_data.get("owning_team") # Get owning team from input + logger.info(f"Project name: {project_name}") - logger.info(f"EKS settings to be applied: {json.dumps(eks_settings, indent=2)}") + logger.info(f"Template settings to be applied: {json.dumps(template_settings, indent=2)}") + logger.info(f"Trigger init workflow: {trigger_init_workflow}") + logger.info(f"Owning team: {owning_team}") if not project_name: logger.error("Missing project name in input") @@ -724,7 +912,7 @@ def lambda_handler(event, context): try: logger.info(f"Starting GitHub operations for project: {project_name}") - operate_github(project_name, eks_settings) + operate_github(project_name, template_settings, trigger_init_workflow, owning_team) logger.info("GitHub operations completed successfully") except Exception as e: # pylint: disable=broad-exception-caught logger.error(f"Error in operate_github: {str(e)}") diff --git a/eks_automation/pytest.ini b/template_automation/pytest.ini similarity index 100% rename from eks_automation/pytest.ini rename to template_automation/pytest.ini diff --git a/eks_automation/requirements.txt b/template_automation/requirements.txt similarity index 100% rename from eks_automation/requirements.txt rename to template_automation/requirements.txt diff --git a/eks_automation/test_payload.json b/template_automation/test_payload.json similarity index 100% rename from eks_automation/test_payload.json rename to template_automation/test_payload.json diff --git a/eks_automation/tests/__init__.py b/template_automation/tests/__init__.py similarity index 100% rename from eks_automation/tests/__init__.py rename to template_automation/tests/__init__.py diff --git a/eks_automation/tests/conftest.py b/template_automation/tests/conftest.py similarity index 100% rename from eks_automation/tests/conftest.py rename to template_automation/tests/conftest.py diff --git a/eks_automation/tests/test_github_client.py b/template_automation/tests/test_github_client.py similarity index 100% rename from eks_automation/tests/test_github_client.py rename to template_automation/tests/test_github_client.py diff --git a/eks_automation/tests/test_github_client_integration.py b/template_automation/tests/test_github_client_integration.py similarity index 100% rename from eks_automation/tests/test_github_client_integration.py rename to template_automation/tests/test_github_client_integration.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d4d2672 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,15 @@ +import os +import pytest + +@pytest.fixture(autouse=True) +def clean_environment(): + """Clean environment variables before each test""" + # Save original environment + env_orig = dict(os.environ) + + # Run test + yield + + # Restore original environment + os.environ.clear() + os.environ.update(env_orig) diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..8a32b7f --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,110 @@ +import os +import pytest +import json +from unittest.mock import patch, MagicMock +from botocore.exceptions import ClientError +from eks_automation.app import get_parameter, operate_github + +@pytest.fixture +def mock_ssm(): + with patch('boto3.client') as mock_client: + ssm_client = MagicMock() + mock_client.return_value = ssm_client + yield ssm_client + +@pytest.fixture +def mock_secrets(): + with patch('eks_automation.app.github_token') as mock_token: + mock_token.return_value = 'fake-token' + yield mock_token + +def test_get_parameter_from_ssm(mock_ssm): + # Setup + mock_ssm.get_parameter.return_value = { + 'Parameter': {'Value': 'param-value'} + } + + # Test + result = get_parameter('test-param') + + # Assert + assert result == 'param-value' + mock_ssm.get_parameter.assert_called_once_with( + Name='/template-automation/test-param', + WithDecryption=False + ) + +def test_get_parameter_from_env(mock_ssm): + # Setup + mock_ssm.get_parameter.side_effect = ClientError( + {'Error': {'Code': 'ParameterNotFound'}}, + 'GetParameter' + ) + os.environ['test-param'] = 'env-value' + + # Test + result = get_parameter('test-param') + + # Assert + assert result == 'env-value' + +def test_get_parameter_with_default(mock_ssm): + # Setup + mock_ssm.get_parameter.side_effect = ClientError( + {'Error': {'Code': 'ParameterNotFound'}}, + 'GetParameter' + ) + + # Test + result = get_parameter('missing-param', default='default-value') + + # Assert + assert result == 'default-value' + +@patch('eks_automation.app.GitHubClient') +def test_operate_github_success(mock_github_client, mock_secrets): + # Setup + mock_client = MagicMock() + mock_github_client.return_value = mock_client + + # Set required environment variables + os.environ['GITHUB_API'] = 'https://api.github.com' + os.environ['GITHUB_ORG_NAME'] = 'test-org' + os.environ['TEMPLATE_REPO_NAME'] = 'template-repo' + + # Test data + new_repo_name = 'test-repo' + template_settings = {'key': 'value'} + + # Test + operate_github(new_repo_name, template_settings) + + # Assert + mock_client.get_repository.assert_called() + mock_client.commit_repository_contents.assert_called() + mock_client.update_repository_topics.assert_called() + +@pytest.mark.parametrize('missing_param', ['GITHUB_API', 'GITHUB_ORG_NAME', 'TEMPLATE_REPO_NAME']) +def test_operate_github_missing_required_params(missing_param, mock_secrets): + # Setup + required_params = { + 'GITHUB_API': 'https://api.github.com', + 'GITHUB_ORG_NAME': 'test-org', + 'TEMPLATE_REPO_NAME': 'template-repo' + } + + # Remove one required parameter + test_params = required_params.copy() + del test_params[missing_param] + + # Set environment variables + for key, value in test_params.items(): + os.environ[key] = value + if missing_param in os.environ: + del os.environ[missing_param] + + # Test + with pytest.raises(ValueError) as exc_info: + operate_github('test-repo', {'key': 'value'}) + + assert missing_param in str(exc_info.value) diff --git a/tests/test_github_client.py b/tests/test_github_client.py new file mode 100644 index 0000000..381c640 --- /dev/null +++ b/tests/test_github_client.py @@ -0,0 +1,71 @@ +import os +import pytest +from unittest.mock import patch, MagicMock +from botocore.exceptions import ClientError +from eks_automation.app import GitHubClient + +@pytest.fixture +def mock_secrets_manager(): + with patch('boto3.session.Session') as mock_session: + secrets_client = MagicMock() + mock_session.return_value.client.return_value = secrets_client + yield secrets_client + +@pytest.fixture +def github_client_env(): + os.environ['GITHUB_TOKEN_SECRET_NAME'] = 'test/github-token' + os.environ['GITHUB_ORG_NAME'] = 'test-org' + yield + del os.environ['GITHUB_TOKEN_SECRET_NAME'] + del os.environ['GITHUB_ORG_NAME'] + +def test_github_client_init_success(mock_secrets_manager, github_client_env): + # Setup + mock_secrets_manager.get_secret_value.return_value = { + 'SecretString': 'fake-token' + } + + # Test + client = GitHubClient() + + # Assert + assert client.token == 'fake-token' + assert client.org_name == 'test-org' + assert client.headers['Authorization'] == 'Bearer fake-token' + mock_secrets_manager.get_secret_value.assert_called_once_with( + SecretId='test/github-token' + ) + +def test_github_client_missing_secret_name(): + # Test + with pytest.raises(ValueError, match="GITHUB_TOKEN_SECRET_NAME environment variable is required"): + GitHubClient() + +def test_github_client_secret_not_found(mock_secrets_manager, github_client_env): + # Setup + mock_secrets_manager.get_secret_value.side_effect = ClientError( + {'Error': {'Code': 'ResourceNotFoundException', 'Message': 'Secret not found'}}, + 'GetSecretValue' + ) + + # Test + with pytest.raises(Exception, match="Failed to retrieve GitHub token from Secrets Manager"): + GitHubClient() + +def test_github_client_trigger_workflow_success(mock_secrets_manager, github_client_env): + # Setup + mock_secrets_manager.get_secret_value.return_value = { + 'SecretString': 'fake-token' + } + + with patch('requests.post') as mock_post: + mock_post.return_value.status_code = 204 + client = GitHubClient() + + # Test + result = client.trigger_workflow('test-repo') + + # Assert + assert result == {"status": "success"} + mock_post.assert_called_once() + assert mock_post.call_args[1]['headers']['Authorization'] == 'Bearer fake-token' diff --git a/tests/test_integration.py b/tests/test_integration.py index 5dcce4b..2d554ba 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -5,7 +5,7 @@ from eks_automation.app import lambda_handler # Test environment variables -os.environ["SECRET_NAME"] = "github-token" # Uses AWS Secrets Manager +os.environ["GITHUB_TOKEN_SECRET_NAME"] = "github-token" # Uses AWS Secrets Manager os.environ["GITHUB_API"] = "https://api.github.com" os.environ["GITHUB_ORG_NAME"] = "your-org-name" # Replace with test org os.environ["TEMPLATE_REPO_NAME"] = "template-eks-cluster" diff --git a/varfiles/default.tfvars b/varfiles/default.tfvars index e69de29..a81194e 100644 --- a/varfiles/default.tfvars +++ b/varfiles/default.tfvars @@ -0,0 +1,16 @@ +aws_region = "us-east-1" +repository_name = "template-automation-lambda" + +catalog_data = { + about_text = "Template Automation Lambda Image" + architectures = ["x86_64"] + description = "Lambda container image for template automation" + operating_systems = ["AmazonLinux2"] + usage_text = "Creates a Template Automation Lambda container image" +} + +tags = { + env = "production" + managed_by = "terraform" + project = "template-automation" +} diff --git a/variables.tf b/variables.tf index 7592d02..28acc55 100644 --- a/variables.tf +++ b/variables.tf @@ -1,3 +1,21 @@ +variable "aws_region" { + description = "AWS region where resources will be created" + type = string + default = "us-east-1" +} + +variable "repository_name" { + description = "Name of the ECR public repository" + type = string + default = "template-automation-lambda" +} + +variable "environment" { + description = "Environment tag value" + type = string + default = "production" +} + variable "github_api" { description = "URL for the GitHub Enterprise API" type = string @@ -26,4 +44,30 @@ variable "template_file_name" { description = "Template file name for the EKS configuration" type = string default = "eks.hcl.j2" +} + +variable "catalog_data" { + description = "Configuration for the ECR repository catalog data" + type = object({ + about_text = string + architectures = list(string) + description = string + operating_systems = list(string) + usage_text = string + }) + default = { + about_text = "Template Automation Lambda Image" + architectures = ["x86_64"] + description = "Lambda container image for template automation" + operating_systems = ["AmazonLinux2"] + usage_text = "Creates a Template Automation Lambda container image" + } +} + +variable "tags" { + description = "Tags to apply to all resources" + type = map(string) + default = { + env = "production" + } } \ No newline at end of file