diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b52d6d0..666df82 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: push: branches: [ "main" ] - + permissions: contents: write id-token: write @@ -13,6 +13,7 @@ jobs: build: runs-on: ubuntu-latest env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-1 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 0000000..af85378 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,38 @@ +name: Integration Tests + +on: + pull_request: + branches: [ main ] + workflow_dispatch: + +jobs: + integration-tests: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + cache: 'pip' + cache-dependency-path: eks_automation/requirements.txt + + - name: Install dependencies + run: | + cd eks_automation + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run integration tests + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + GITHUB_API: "https://api.github.com" # Can be overridden with vars if needed + GITHUB_ORG: ${{ github.repository_owner }} + run: | + cd eks_automation + python -m pytest tests/ -v -m integration \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8af78e5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +dist/ +build/ +*.egg-info/ + +# Virtual Environment +venv/ +env/ +.env/ + +# IDE +.idea/ +.vscode/ + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +coverage.xml +*.cover \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..047e645 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +.PHONY: install test test-unit test-integration clean + +# Variables +PYTHON = python3 +PIP = $(PYTHON) -m pip +PYTEST = $(PYTHON) -m pytest +REQUIREMENTS = eks_automation/requirements.txt +TEST_DIR = eks_automation/tests +UNIT_TEST_FILE = $(TEST_DIR)/test_github_client.py +INTEGRATION_TEST_FILE = $(TEST_DIR)/test_github_client_integration.py + +# Default target +all: test + +# Install dependencies +install: + $(PIP) install -r $(REQUIREMENTS) + +# Run all tests +test: test-unit test-integration + @echo "Running all tests..." + $(PYTEST) $(TEST_DIR) + +# Run unit tests +test-unit: + @echo "Running unit tests..." + $(PYTEST) $(UNIT_TEST_FILE) + +# Run integration tests +test-integration: + @echo "Running integration tests..." + $(PYTEST) $(INTEGRATION_TEST_FILE) + +# Clean up Python cache files +clean: + find . -type f -name '*.pyc' -delete + find . -type d -name '__pycache__' -exec rm -rf {} + + rm -rf .pytest_cache + rm -rf .coverage diff --git a/README.md b/README.md index 7ac5932..061b2d5 100644 --- a/README.md +++ b/README.md @@ -2,153 +2,135 @@ ## Description -This repository contains source code and supporting files for a serverless application that you can deploy with the SAM CLI. +This repository contains source code and supporting files for a serverless Lambda container application. The application uses an AWS Lambda function to process JSON input and write it to a cloned repository. -The changes are then committed and pushed to the Census GitHub Enterprise Server, creating a new repository -for the Census EKS CI/CD pipeline. - -## Getting Started - -First of all, you need access to an AWS account with adequate permission to which the resources will be deployed. -You also need to create an [`AWS CLI` profile](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-quickstart.html#getting-started-quickstart-new). - -A [GitHub Personal Access Token (PAT)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) -is required to access the Census GitHub Enterprise Server. -The `PAT` must be securely stored in `AWS Systems Manager Parameter Store`. The parameter name must match the value of the -"SECRET_NAME" constant defined in the `eks_automation/app.py` file. - -To access the Census GitHub Enterprise Server, a VPC with private subnets and a route to the server must be attached. -The VPC configuration is set in the `template.yaml` file. Update the `Subnet IDs` and `Security Group IDs` as needed. - -You may also want to adjust other settings (API Usage Plan, tags, etc.) in the `template.yaml` file. - -## Prerequites - -- git -- python3.11 -- pip -- pre-commit -- AWS CLI -- SAM CLI - -You may need to submit a support ticket to request the installation of these tools on your laptop. - -### Installing - -- Clone this repository: - - ```sh - git clone git@github.e.it.census.gov:SCT-Engineering/eks-automation-lambda.git - ``` - -- After cloning, access the folder and install `pre-commit hooks` listed in the `.pre-commit-config.yaml`: - - ```sh - cd eks-automation-lambda - pre-commit install - ``` - -## Deploy/Test the application - -- Create an `AWS S3 bucket`: - - ```sh - aws s3api create-bucket --bucket eks-automation-lambda-s3-bucket \ - --create-bucket-configuration LocationConstraint=us-gov-east-1 \ - --region us-gov-east-1 \ - --profile 229685449397-csvd-dev-gov - ``` - - The `bucket name` must match the one specified in the `samconfig.toml` file. - Please adjust the profile name and region accordingly. - -- Download [`git-lambda-layer`](https://github.com/lambci/git-lambda-layer/blob/master/lambda2/layer.zip) `zip` file. -- Upload `git-lambda-layer` to the newly created `AWS S3 bucket`: - - ```sh - aws s3 cp {download-folder}/layer.zip s3://eks-automation-lambda-s3-bucket/ --profile 229685449397-csvd-dev-gov - ``` - -- Build the application: - - ```sh - sam build - ``` - -- Deploy the application: - - ```sh - sam deploy --profile 229685449397-csvd-dev-gov - ``` - - Save the `API Gateway endpoint URL` listed in the output. You will need this URL for testing. - -- Test: - - The `JSON` input payload is in the following format: - - ```json - { - "project_name": "string", - "eks_settings": { - "attrs": { - "attribute1": "value1", - "attribute2": "value2", - ... - }, - "tags" : { - "key1": "value1", - "key2": "value2", - ... - } +The changes are then committed and pushed to your GitHub Enterprise Server, creating a new repository +for the EKS CI/CD pipeline. + +## Architecture + +- AWS Lambda container image built with Packer and stored in ECR +- Infrastructure managed with Terraform +- Automated CI/CD using GitHub Actions +- Secret management using AWS Systems Manager Parameter Store + +## Prerequisites + +- AWS credentials with appropriate permissions +- GitHub Personal Access Token (PAT) stored in AWS Systems Manager Parameter Store +- Docker (for local development) +- Terraform +- Packer +- Python 3.11+ + +## Local Development + +1. Clone this repository: + ```sh + git clone /eks-automation-lambda.git + cd eks-automation-lambda + ``` + +2. Install Python dependencies: + ```sh + cd eks_automation + pip install -r requirements.txt + ``` + +3. Configure AWS credentials either through environment variables or AWS CLI profile + +4. Store your GitHub PAT in AWS Systems Manager Parameter Store. The parameter name should match the + value of `GITHUB_TOKEN_SECRET_NAME` in `eks_automation/app.py` + +## Deployment + +The project uses GitHub Actions for automated deployments. On push to main: + +1. Creates/updates ECR repository using Terraform +2. Builds Lambda container image using Packer +3. Pushes image to ECR +4. Tags the release + +For manual deployment: + +1. Initialize Terraform: + ```sh + terraform init + ``` + +2. Apply Terraform configuration: + ```sh + terraform apply + ``` + +3. Build and push container image: + ```sh + packer init packer.pkr.hcl + packer build -var "repository_uri=$(terraform output -raw repository_uri)" -var "tag=latest" packer.pkr.hcl + ``` + +## Testing + +The Lambda function accepts JSON input in the following format: + +```json +{ + "project_name": "string", + "eks_settings": { + "attrs": { + "account_name": "my-account", + "aws_region": "us-east-1", + "cluster_mailing_list": "someone@example.com", + "cluster_name": "my-eks-cluster", + "eks_instance_disk_size": 100, + "eks_ng_desired_size": 2, + "eks_ng_max_size": 10, + "eks_ng_min_size": 2, + "environment": "development", + "environment_abbr": "dev", + "organization": "my-org:my-division:my-team", + "finops_project_name": "my_project_baseline", + "finops_project_number": "fp00000001", + "finops_project_role": "my_project_baseline_app", + "vpc_domain_name": "dev.example.com", + "vpc_name": "vpc-dev" + }, + "tags": { + "slim:schedule": "8:00-17:00" } } - ``` - - Get the `API Key`: - - ```sh - aws apigateway get-api-keys --query 'items[?contains(name, `eks-`)].value' --include-values --output text --profile 229685449397-csvd-dev-gov - ``` - - ```sh - curl -X POST -H "X-API-Key: {API Key}" https://{API Gateway endpoint URL} -d ' - { - "project_name": "eks-automation-lambda-test", - "eks_settings": { - "attrs": { - "account_name": "lab-dev-ew", - "aws_region": "us-gov-east-1", - "cluster_mailing_list": "someone@census.gov", - "cluster_name": "csvd-platform-lab-mcm", - "eks_instance_disk_size": 100, - "eks_ng_desired_size": 2, - "eks_ng_max_size": 10, - "eks_ng_min_size": 2, - "environment": "development", - "environment_abbr": "dev", - "organization": "census:ocio:csvd", - "finops_project_name": "csvd_platformbaseline", - "finops_project_number": "fs0000000078", - "finops_project_role": "csvd_platformbaseline_app", - "vpc_domain_name": "dev.lab.csp2.census.gov", - "vpc_name": "vpc3-lab-dev" - }, - "tags" : { - "slim:schedule": "8:00-17:00" - } - } - } - ' - ``` - - Replace `{API Key}` with the API key we just retrieved, and `{API Gateway endpoint URL}` with the value saved from the `sam deploy` command output. +} +``` + +### Unit Tests +To run the unit tests: +```sh +cd eks_automation +python -m pytest tests/ -v -m "not integration" +``` + +### Integration Tests +The integration tests require real GitHub API access. To run them: + +1. Set up the required environment variables: +```sh +export GITHUB_TOKEN="your-github-token" +export GITHUB_API="https://api.github.com" # or your GitHub Enterprise URL +export GITHUB_ORG="your-org-name" +``` + +2. Run the integration tests: +```sh +cd eks_automation +python -m pytest tests/ -v -m integration +``` + +Note: Integration tests will create temporary repositories in your GitHub organization. These repositories will be archived (not deleted) after the tests complete. ## Resources -- [AWS Serverless Application Model](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/what-is-sam.html) -- [AWS Lambda](https://docs.aws.amazon.com/lambda/latest/dg/welcome.html) -- [Git Lambda Layer](https://github.com/lambci/git-lambda-layer/) -- [AWS API Gateway](https://docs.aws.amazon.com/apigateway/latest/developerguide/welcome.html) -- [PyGithub](https://pygithub.readthedocs.io/en/stable/introduction.html) -- [GitPython](https://gitpython.readthedocs.io/en/stable/) +- [AWS Lambda Container Images](https://docs.aws.amazon.com/lambda/latest/dg/images-create.html) +- [HashiCorp Packer](https://www.packer.io/docs) +- [AWS ECR Public](https://docs.aws.amazon.com/AmazonECR/latest/public/what-is-ecr.html) +- [GitHub Actions](https://docs.github.com/en/actions) +- [Terraform AWS Provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs) diff --git a/eks-automation-lambda.code-workspace b/eks-automation-lambda.code-workspace new file mode 100644 index 0000000..061431d --- /dev/null +++ b/eks-automation-lambda.code-workspace @@ -0,0 +1,20 @@ +{ + "folders": [ + { + "path": "." + }, + { + "path": "../github-repos" + }, + { + "path": "../github-runner-image" + }, + { + "path": "../template-lambda-deployment" + }, + { + "path": "../terraform-github-repo" + } + ], + "settings": {} +} \ No newline at end of file diff --git a/eks_automation/app.py b/eks_automation/app.py index 57eb741..913d25e 100644 --- a/eks_automation/app.py +++ b/eks_automation/app.py @@ -1,8 +1,7 @@ #################################################################################### -# This Lambda function takes JSON input, processes it using a Jinja2 template, -# and writes the output to a file in a cloned GitHub repository. -# The changes are then committed and pushed to the GitHub API, -# creating a new repository for the EKS CI/CD pipeline. +# This Lambda function takes JSON input and writes it directly to a config.json file +# in a cloned GitHub repository. The changes are then committed and pushed to the +# GitHub API, creating a new repository for the EKS CI/CD pipeline. # This implementation uses only pure Python with requests library (no Git CLI dependency). #################################################################################### @@ -14,33 +13,16 @@ import time import requests import json -from jinja2 import Environment, FileSystemLoader from urllib.parse import urlparse from datetime import datetime import boto3 from botocore.exceptions import ClientError -import os - - -# Get configuration from environment variables with defaults -GITHUB_API = os.environ.get("GITHUB_API") # No default - must be configured -ORG_NAME = os.environ.get("GITHUB_ORG_NAME") # No default - must be configured -SECRET_NAME = os.environ.get("GITHUB_TOKEN_SECRET_NAME", "/eks-cluster-deployment/github_token") -COMMIT_AUTHOR_EMAIL = os.environ.get("GITHUB_COMMIT_AUTHOR_EMAIL", "eks-automation@noreply.github.com") -COMMIT_AUTHOR_NAME = os.environ.get("GITHUB_COMMIT_AUTHOR_NAME", "EKS Automation Lambda") -SOURCE_VERSION = os.environ.get("TEMPLATE_SOURCE_VERSION") # Optional - if not set, uses default branch - -ORIG_REPO_NAME = os.environ.get("TEMPLATE_REPO_NAME", "template-eks-cluster") - -TEMPLATE_FILE_NAME = os.environ.get("TEMPLATE_FILE_NAME", "eks.hcl.j2") -HCL_FILE_NAME = os.environ.get("HCL_FILE_NAME", "eks.hcl") # Initialize the logger logger = logging.getLogger() logger.setLevel("INFO") # Set to "ERROR" to reduce logging messages. - class GitHubClient: """A class to interact with GitHub API without relying on external Git binaries. @@ -48,17 +30,27 @@ class GitHubClient: branches, files, commits and other Git operations using only the requests library. """ - def __init__(self, api_base_url, token, org_name): + def __init__(self, api_base_url, token, org_name, commit_author_name, commit_author_email, source_version=None, template_repo_name=None, config_file_name="config.json"): """Initialize the GitHub client Args: api_base_url (str): Base URL for the GitHub API token (str): GitHub access token org_name (str): GitHub organization name + commit_author_name (str): Name of the commit author + commit_author_email (str): Email of the commit author + source_version (str, optional): Version to use from template repo + template_repo_name (str, optional): Name of the template repository + config_file_name (str, optional): Name of the config file to write """ self.api_base_url = api_base_url self.token = token self.org_name = org_name + self.commit_author_name = commit_author_name + self.commit_author_email = commit_author_email + self.source_version = source_version + self.template_repo_name = template_repo_name + self.config_file_name = config_file_name self.headers = self._create_headers() def _create_headers(self): @@ -92,30 +84,44 @@ def get_repository(self, repo_name, create=False): if response.status_code == 200: # Repository exists return response.json() - elif response.status_code == 404 and create: - # Repository doesn't exist, create it - logger.info(f"Creating repository {repo_name}") - create_url = f"{self.api_base_url}/orgs/{self.org_name}/repos" - repo_data = { - "name": repo_name, - "description": "EKS Automation CI/CD Pipeline Repo", - "private": True - } - create_response = requests.post( - create_url, - headers=self.headers, - json=repo_data, - verify=False - ) - - if create_response.status_code in (201, 200): - return create_response.json() + elif response.status_code == 404: + if create: + # Repository doesn't exist, create it + logger.info(f"Creating repository {repo_name}") + create_url = f"{self.api_base_url}/orgs/{self.org_name}/repos" + repo_data = { + "name": repo_name, + "description": "EKS Automation CI/CD Pipeline Repo", + "private": True, + "auto_init": True, # Initialize with README + "default_branch": "main", + "allow_squash_merge": True, + "allow_merge_commit": True, + "allow_rebase_merge": True, + "delete_branch_on_merge": True, + "enable_branch_protection": False # Disable branch protection + } + create_response = requests.post( + create_url, + headers=self.headers, + json=repo_data, + verify=False + ) + + if create_response.status_code in (201, 200): + # Wait briefly for repository initialization + time.sleep(2) + return create_response.json() + else: + error_message = f"Failed to create repository: {create_response.status_code} - {create_response.text}" + logger.error(error_message) + raise Exception(error_message) else: - error_message = f"Failed to create repository: {create_response.status_code} - {create_response.text}" + error_message = f"Repository {repo_name} not found and create=False" logger.error(error_message) raise Exception(error_message) else: - error_message = f"Repository {repo_name} not found and create=False" + error_message = f"Unexpected response when getting repository: {response.status_code} - {response.text}" logger.error(error_message) raise Exception(error_message) @@ -212,6 +218,9 @@ def download_repository_files(self, repo_name, tree, target_dir): tree (dict): Tree information from get_tree() target_dir (str): Directory to download files to """ + # Ensure target directory exists even if there are no files + os.makedirs(target_dir, exist_ok=True) + for item in tree.get("tree", []): if item["type"] == "blob": # Get the blob contents @@ -224,13 +233,18 @@ def download_repository_files(self, repo_name, tree, target_dir): # Ensure the target directory exists file_path = os.path.join(target_dir, item["path"]) - os.makedirs(os.path.dirname(file_path), exist_ok=True) + dir_path = os.path.dirname(file_path) + os.makedirs(dir_path, exist_ok=True) # GitHub API returns base64 encoded content if blob_data.get("encoding") == "base64": content = base64.b64decode(blob_data.get("content", "")) + else: + # Handle non-base64 content if needed + logger.warning(f"Unexpected encoding for blob {item['sha']}: {blob_data.get('encoding')}") if content is not None: + logger.info(f"Writing file to {file_path}") with open(file_path, "wb") as f: f.write(content) @@ -306,20 +320,21 @@ def create_commit(self, repo_name, message, tree_sha, parent_shas): """ api_url = f"{self.api_base_url}/repos/{self.org_name}/{repo_name}/git/commits" - data = { - "message": message, - "tree": tree_sha, - "parents": parent_shas - } - # Add committer/author information current_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - data["author"] = { - "name": COMMIT_AUTHOR_NAME, - "email": COMMIT_AUTHOR_EMAIL, + author_info = { + "name": self.commit_author_name, + "email": self.commit_author_email, "date": current_time } - data["committer"] = data["author"] + + data = { + "message": message, + "tree": tree_sha, + "parents": parent_shas, + "author": author_info, + "committer": author_info + } response = requests.post(api_url, headers=self.headers, json=data, verify=False) @@ -373,64 +388,65 @@ def create_reference(self, repo_name, ref, sha): logger.error(error_message) raise Exception(error_message) - def clone_repository_contents(self, source_repo, target_dir): + def clone_repository_contents(self, source_repo, target_dir, branch=None): """Clone a repository's contents to a local directory using GitHub API Args: source_repo (str): Name of the source repository target_dir (str): Target directory to download files to + branch (str, optional): Branch to clone from. If None, uses default branch. Returns: - str: The default branch name of the repository + str: The branch name that was cloned """ - # Get default branch of original repo for fallback - default_branch = self.get_default_branch(source_repo) - logger.info(f"Default branch for {source_repo}: {default_branch}") - - # Get tree from original repository - logger.info(f"Getting file tree from {source_repo}") - ref = None - - if SOURCE_VERSION: - try: - # Try to get the tag/release reference first - ref = f"tags/{SOURCE_VERSION}" - tree_sha = self.get_reference_sha(source_repo, ref) - logger.info(f"Using source version: {SOURCE_VERSION}") - except Exception as e: - logger.warning(f"Failed to get version {SOURCE_VERSION}, falling back to default branch: {str(e)}") - ref = f"heads/{default_branch}" - tree_sha = self.get_reference_sha(source_repo, ref) - else: - # Use default branch - ref = f"heads/{default_branch}" - tree_sha = self.get_reference_sha(source_repo, ref) + # Create the target directory if it doesn't exist + os.makedirs(target_dir, exist_ok=True) + try: + if branch: + target_branch = branch + # Try to get the branch's reference directly + tree_sha = self.get_reference_sha(source_repo, f"heads/{target_branch}") + else: + # If no branch specified, use default branch + target_branch = self.get_default_branch(source_repo) + tree_sha = self.get_reference_sha(source_repo, f"heads/{target_branch}") + except Exception as e: + logger.warning(f"Failed to get reference for {branch or 'default branch'}: {str(e)}") + target_branch = branch or "main" + # If we can't get the reference, the branch might not exist yet + tree = {"tree": []} + self.download_repository_files(source_repo, tree, target_dir) + return target_branch + + # Get the full tree for the branch + logger.info(f"Getting file tree from {source_repo} for branch {target_branch}") tree = self.get_tree(source_repo, tree_sha, recursive=True) - - # Download all files from original repo to work directory - logger.info(f"Downloading all files from {source_repo} using ref: {ref}") + + # Download all files + logger.info(f"Downloading all files from {source_repo} using ref: heads/{target_branch}") self.download_repository_files(source_repo, tree, target_dir) - - return default_branch + + return target_branch - def commit_repository_contents(self, repo_name, work_dir, commit_message): + def commit_repository_contents(self, repo_name, work_dir, commit_message, branch=None): """Commit all files from a directory to a repository Args: repo_name (str): Name of the repository work_dir (str): Directory containing the files to commit commit_message (str): Commit message + branch (str, optional): Branch to commit to. If None, uses default branch. Returns: - str: The default branch name of the repository + str: The branch name that was committed to """ # First, get the current state of the target repository try: - target_default_branch = self.get_default_branch(repo_name) + target_branch = branch or self.get_default_branch(repo_name) except Exception: # If we can't get the default branch, it might be a new repo - target_default_branch = "main" + target_branch = branch or "main" # Upload all files to the repository tree_items = [] @@ -463,7 +479,7 @@ def commit_repository_contents(self, repo_name, work_dir, commit_message): # Try to get the latest commit SHA for the branch # If it doesn't exist, we'll create it try: - latest_commit_sha = self.get_reference_sha(repo_name, f"heads/{target_default_branch}") + latest_commit_sha = self.get_reference_sha(repo_name, f"heads/{target_branch}") latest_commit = self.get_commit(repo_name, latest_commit_sha) base_tree_sha = latest_commit["tree"]["sha"] except Exception: @@ -495,18 +511,18 @@ def commit_repository_contents(self, repo_name, work_dir, commit_message): try: self.update_reference( repo_name, - f"heads/{target_default_branch}", + f"heads/{target_branch}", new_commit_sha ) except Exception: # If the reference doesn't exist, create it self.create_reference( repo_name, - f"refs/heads/{target_default_branch}", + f"refs/heads/{target_branch}", new_commit_sha ) - return target_default_branch + return target_branch # pylint: disable=unused-argument @@ -521,12 +537,6 @@ def lambda_handler(event, context): Returns: dict: Dict containing status message. """ - - # For test, load input data from a local file. - # input_data = "" - # with open("data.json", "r") as file: - # input_data = json.load(file) - input_data = json.loads(event["body"]) project_name = input_data["project_name"] @@ -538,7 +548,7 @@ def lambda_handler(event, context): } try: - rendered = operate_github(project_name, eks_settings, HCL_FILE_NAME) + operate_github(project_name, eks_settings) except Exception as e: # pylint: disable=broad-exception-caught logger.error(f"Error in operate_github: {str(e)}") return {"statusCode": 400, "body": json.dumps({"error": str(e)})} @@ -546,12 +556,12 @@ def lambda_handler(event, context): return { "statusCode": 200, "headers": {"Access-Control-Allow-Origin": "*"}, - "body": json.dumps({"result": rendered}), + "body": json.dumps({"result": "Success"}), } -def operate_github(new_repo_name, eks_settings, output_hcl): - """Process template and create/update repository using GitHub API +def operate_github(new_repo_name, eks_settings): + """Write EKS settings to config.json and create/update repository using GitHub API This implementation uses only the requests library and does not rely on git CLI or any external binaries. @@ -559,14 +569,19 @@ def operate_github(new_repo_name, eks_settings, output_hcl): Args: new_repo_name (str): Name of the new GitHub repo. eks_settings (json): Input JSON data with all the EKS parameter values. - output_hcl (str): Name of the EKS parameter file in HCL format. Returns: - str: The rendered EKS parameter string. + None """ - - # Get GitHub access token + # Get GitHub access token and environment variables token = github_token() + github_api = os.environ.get("GITHUB_API") # No default - must be configured + org_name = os.environ.get("GITHUB_ORG_NAME") # No default - must be configured + commit_author_email = os.environ.get("GITHUB_COMMIT_AUTHOR_EMAIL", "eks-automation@example.com") + commit_author_name = os.environ.get("GITHUB_COMMIT_AUTHOR_NAME", "EKS Automation Lambda") + source_version = os.environ.get("TEMPLATE_SOURCE_VERSION") # Optional + template_repo_name = os.environ.get("TEMPLATE_REPO_NAME", "template-eks-cluster") + config_file_name = "config.json" # Create work directory if it doesn't exist work_dir = f"/tmp/{new_repo_name}" @@ -574,53 +589,40 @@ def operate_github(new_repo_name, eks_settings, output_hcl): shutil.rmtree(work_dir, ignore_errors=False, onerror=remove_readonly) os.makedirs(work_dir, exist_ok=True) - # Initialize GitHub client - github = GitHubClient(GITHUB_API, token, ORG_NAME) + # Initialize GitHub client with all required parameters + github = GitHubClient( + github_api, + token, + org_name, + commit_author_name, + commit_author_email, + source_version, + template_repo_name, + config_file_name + ) # Get info about original repo - logger.info(f"Fetching original repository information: {ORIG_REPO_NAME}") - orig_repo = github.get_repository(ORIG_REPO_NAME) + logger.info(f"Fetching original repository information: {template_repo_name}") + orig_repo = github.get_repository(template_repo_name) # Get or create the new repository logger.info(f"Getting or creating repository: {new_repo_name}") new_repo = github.get_repository(new_repo_name, create=True) # Clone the original repository contents - github.clone_repository_contents(ORIG_REPO_NAME, work_dir) + github.clone_repository_contents(template_repo_name, work_dir) - # Render the template and write to file - rendered = render_j2_template(eks_settings, TEMPLATE_FILE_NAME) - output_file_path = os.path.join(work_dir, output_hcl) - - logger.info(f"Writing rendered template to {output_file_path}") + # Write EKS settings directly to config.json + output_file_path = os.path.join(work_dir, config_file_name) + logger.info(f"Writing EKS settings to {output_file_path}") with open(output_file_path, "w") as file: - file.write(rendered) + json.dump(eks_settings, file, indent=2) - # Commit all files to the new repository - commit_message = "Add the EKS parameter file by the Lambda function" - github.commit_repository_contents(new_repo_name, work_dir, commit_message) + # Commit all files to the new repository's main branch explicitly + commit_message = "Add the EKS configuration file by the Lambda function" + github.commit_repository_contents(new_repo_name, work_dir, commit_message, branch="main") logger.info(f"Successfully updated {new_repo_name} repository") - return rendered - - -def render_j2_template(eks_settings, j2_template, j2_template_dir="templates/"): - """Render the j2 template with the input JSON data - - Args: - eks_settings (json): input data in JSON format. - j2_template (j2): Name of the template file to generate the output. - j2_template_dir (str, optional): The directory where the templates are stored. Defaults to "templates/". - - Returns: - str: Rendered template string. - """ - - # Render template - jinja_env = Environment(loader=FileSystemLoader(j2_template_dir), trim_blocks=True) - template = jinja_env.get_template(j2_template) - - return template.render(data=eks_settings) def github_token(): diff --git a/eks_automation/pytest.ini b/eks_automation/pytest.ini new file mode 100644 index 0000000..a2bd95b --- /dev/null +++ b/eks_automation/pytest.ini @@ -0,0 +1,11 @@ +[pytest] +markers = + integration: marks tests as integration tests (deselect with '-m "not integration"') +testpaths = + tests +python_files = + test_*.py + *_test.py +addopts = + -v + --strict-markers \ No newline at end of file diff --git a/eks_automation/requirements.txt b/eks_automation/requirements.txt index b7f4404..e06cdc0 100644 --- a/eks_automation/requirements.txt +++ b/eks_automation/requirements.txt @@ -2,8 +2,11 @@ # black # pre-commit -jinja2 boto3 requests -pygithub -gitpython + +# Testing dependencies +pytest>=7.0.0 +pytest-mock>=3.10.0 +requests-mock>=1.11.0 +coverage>=7.2.0 diff --git a/eks_automation/templates/eks.hcl.j2 b/eks_automation/templates/eks.hcl.j2 deleted file mode 100644 index 40c4feb..0000000 --- a/eks_automation/templates/eks.hcl.j2 +++ /dev/null @@ -1,10 +0,0 @@ -locals { - {% for key, value in data['attrs'] | items -%} - {{ key }} = "{{ value }}" - {% endfor -%} - tags = { - {% for key, value in data['tags'] | items -%} - {{ key }} = "{{ value }}" - {% endfor -%} - } -} diff --git a/eks_automation/test_payload.json b/eks_automation/test_payload.json new file mode 100644 index 0000000..01b2e4f --- /dev/null +++ b/eks_automation/test_payload.json @@ -0,0 +1,26 @@ +{ + "project_name": "eks-automation-lambda-test1", + "eks_settings": { + "attrs": { + "account_name": "dev-account", + "aws_region": "us-east-1", + "cluster_mailing_list": "someone@example.com", + "cluster_name": "example-cluster-dev", + "eks_instance_disk_size": 100, + "eks_ng_desired_size": 2, + "eks_ng_max_size": 10, + "eks_ng_min_size": 2, + "environment": "development", + "environment_abbr": "dev", + "organization": "example:dept:team", + "finops_project_name": "example_project", + "finops_project_number": "fp00000001", + "finops_project_role": "example_project_app", + "vpc_domain_name": "dev.example.com", + "vpc_name": "vpc-dev" + }, + "tags": { + "slim:schedule": "8:00-17:00" + } + } +} \ No newline at end of file diff --git a/eks_automation/tests/__init__.py b/eks_automation/tests/__init__.py new file mode 100644 index 0000000..739954c --- /dev/null +++ b/eks_automation/tests/__init__.py @@ -0,0 +1 @@ +# Tests package \ No newline at end of file diff --git a/eks_automation/tests/conftest.py b/eks_automation/tests/conftest.py new file mode 100644 index 0000000..769a94b --- /dev/null +++ b/eks_automation/tests/conftest.py @@ -0,0 +1,74 @@ +import pytest +import os +import json + +@pytest.fixture +def github_client_params(): + """Fixture providing standard GitHubClient parameters""" + return { + "api_base_url": "https://api.github.example.com", + "token": "test-token", + "org_name": "test-org", + "commit_author_name": "Test Author", + "commit_author_email": "test@example.com", + "source_version": "v1.0.0", + "template_repo_name": "template-repo", + "config_file_name": "config.json" + } + +@pytest.fixture +def mock_repository_response(): + """Fixture providing a standard repository API response""" + return { + "id": 1234, + "name": "test-repo", + "default_branch": "main", + "private": True, + "description": "Test repository" + } + +@pytest.fixture +def mock_tree_response(): + """Fixture providing a standard tree API response""" + return { + "sha": "test-tree-sha", + "tree": [ + { + "path": "test.txt", + "mode": "100644", + "type": "blob", + "sha": "test-blob-sha", + "size": 100 + } + ] + } + +@pytest.fixture +def mock_blob_response(): + """Fixture providing a standard blob API response""" + return { + "sha": "test-blob-sha", + "content": "SGVsbG8gV29ybGQh", # Base64 encoded "Hello World!" + "encoding": "base64" + } + +@pytest.fixture +def mock_commit_response(): + """Fixture providing a standard commit API response""" + return { + "sha": "test-commit-sha", + "tree": { + "sha": "test-tree-sha" + } + } + +@pytest.fixture +def mock_reference_response(): + """Fixture providing a standard reference API response""" + return { + "ref": "refs/heads/main", + "object": { + "sha": "test-commit-sha", + "type": "commit" + } + } \ No newline at end of file diff --git a/eks_automation/tests/test_github_client.py b/eks_automation/tests/test_github_client.py new file mode 100644 index 0000000..89bcd65 --- /dev/null +++ b/eks_automation/tests/test_github_client.py @@ -0,0 +1,245 @@ +import os +import pytest +import base64 +import tempfile +import shutil +from datetime import datetime +from urllib.parse import urljoin + +import requests +import requests_mock + +from ..app import GitHubClient + +class TestGitHubClient: + """Test suite for GitHubClient class""" + + def test_init(self, github_client_params): + """Test GitHubClient initialization""" + client = GitHubClient(**github_client_params) + assert client.api_base_url == github_client_params["api_base_url"] + assert client.token == github_client_params["token"] + assert client.org_name == github_client_params["org_name"] + assert client.commit_author_name == github_client_params["commit_author_name"] + assert client.commit_author_email == github_client_params["commit_author_email"] + assert "Authorization" in client.headers + assert client.headers["Authorization"] == f"token {github_client_params['token']}" + + def test_get_repository_existing(self, requests_mock, github_client_params, mock_repository_response): + """Test getting an existing repository""" + client = GitHubClient(**github_client_params) + repo_name = "test-repo" + + # Mock the API response + requests_mock.get( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}", + json=mock_repository_response + ) + + repo = client.get_repository(repo_name) + assert repo["name"] == mock_repository_response["name"] + assert repo["default_branch"] == mock_repository_response["default_branch"] + + def test_get_repository_create_new(self, requests_mock, github_client_params, mock_repository_response): + """Test creating a new repository""" + client = GitHubClient(**github_client_params) + repo_name = "new-test-repo" + + # Mock 404 for get request and success for create + requests_mock.get( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}", + status_code=404 + ) + requests_mock.post( + f"{github_client_params['api_base_url']}/orgs/{github_client_params['org_name']}/repos", + json=mock_repository_response + ) + + repo = client.get_repository(repo_name, create=True) + assert repo["name"] == mock_repository_response["name"] + + def test_get_default_branch(self, requests_mock, github_client_params, mock_repository_response): + """Test getting repository default branch""" + client = GitHubClient(**github_client_params) + repo_name = "test-repo" + + requests_mock.get( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}", + json=mock_repository_response + ) + + branch = client.get_default_branch(repo_name) + assert branch == mock_repository_response["default_branch"] + + def test_create_blob(self, requests_mock, github_client_params, mock_blob_response): + """Test creating a blob""" + client = GitHubClient(**github_client_params) + repo_name = "test-repo" + content = b"Hello World!" + + requests_mock.post( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/blobs", + json=mock_blob_response + ) + + blob_sha = client.create_blob(repo_name, content) + assert blob_sha == mock_blob_response["sha"] + + def test_create_tree(self, requests_mock, github_client_params, mock_tree_response): + """Test creating a tree""" + client = GitHubClient(**github_client_params) + repo_name = "test-repo" + tree_items = [{ + "path": "test.txt", + "mode": "100644", + "type": "blob", + "sha": "test-blob-sha" + }] + + requests_mock.post( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/trees", + json=mock_tree_response + ) + + tree_sha = client.create_tree(repo_name, tree_items) + assert tree_sha == mock_tree_response["sha"] + + def test_create_commit(self, requests_mock, github_client_params, mock_commit_response): + """Test creating a commit""" + client = GitHubClient(**github_client_params) + repo_name = "test-repo" + message = "Test commit" + tree_sha = "test-tree-sha" + parent_shas = ["parent-sha"] + + requests_mock.post( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/commits", + json=mock_commit_response + ) + + commit_sha = client.create_commit(repo_name, message, tree_sha, parent_shas) + assert commit_sha == mock_commit_response["sha"] + + def test_update_reference(self, requests_mock, github_client_params): + """Test updating a reference""" + client = GitHubClient(**github_client_params) + repo_name = "test-repo" + ref = "heads/main" + sha = "test-commit-sha" + + requests_mock.patch( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/refs/{ref}", + status_code=200 + ) + + # Should not raise an exception + client.update_reference(repo_name, ref, sha) + + def test_create_reference(self, requests_mock, github_client_params): + """Test creating a reference""" + client = GitHubClient(**github_client_params) + repo_name = "test-repo" + ref = "refs/heads/main" + sha = "test-commit-sha" + + requests_mock.post( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/refs", + status_code=201 + ) + + # Should not raise an exception + client.create_reference(repo_name, ref, sha) + + def test_clone_repository_contents(self, requests_mock, github_client_params, mock_repository_response, + mock_reference_response, mock_tree_response, mock_blob_response, tmp_path): + """Test cloning repository contents""" + client = GitHubClient(**github_client_params) + repo_name = "test-repo" + target_dir = str(tmp_path) + + # Mock all required API calls + requests_mock.get( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}", + json=mock_repository_response + ) + requests_mock.get( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/refs/heads/main", + json=mock_reference_response + ) + requests_mock.get( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/trees/{mock_reference_response['object']['sha']}?recursive=1", + json=mock_tree_response + ) + requests_mock.get( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/blobs/{mock_tree_response['tree'][0]['sha']}", + json=mock_blob_response + ) + + default_branch = client.clone_repository_contents(repo_name, target_dir) + assert default_branch == mock_repository_response["default_branch"] + assert os.path.exists(os.path.join(target_dir, mock_tree_response["tree"][0]["path"])) + + def test_commit_repository_contents(self, requests_mock, github_client_params, mock_repository_response, + mock_reference_response, mock_tree_response, mock_commit_response, tmp_path): + """Test committing repository contents""" + client = GitHubClient(**github_client_params) + repo_name = "test-repo" + work_dir = str(tmp_path) + + # Create a test file + test_file = os.path.join(work_dir, "test.txt") + with open(test_file, "w") as f: + f.write("test content") + + # Mock all required API calls + requests_mock.get( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}", + json=mock_repository_response + ) + requests_mock.get( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/refs/heads/main", + json=mock_reference_response + ) + requests_mock.get( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/commits/{mock_reference_response['object']['sha']}", + json=mock_commit_response + ) + requests_mock.post( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/blobs", + json={"sha": "new-blob-sha"} + ) + requests_mock.post( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/trees", + json={"sha": "new-tree-sha"} + ) + requests_mock.post( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/commits", + json={"sha": "new-commit-sha"} + ) + requests_mock.patch( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}/git/refs/heads/main", + status_code=200 + ) + + default_branch = client.commit_repository_contents(repo_name, work_dir, "Test commit") + assert default_branch == mock_repository_response["default_branch"] + + def test_error_handling(self, requests_mock, github_client_params): + """Test error handling in GitHubClient methods""" + client = GitHubClient(**github_client_params) + repo_name = "test-repo" + + # Test error on repository creation + requests_mock.get( + f"{github_client_params['api_base_url']}/repos/{github_client_params['org_name']}/{repo_name}", + status_code=404 + ) + requests_mock.post( + f"{github_client_params['api_base_url']}/orgs/{github_client_params['org_name']}/repos", + status_code=500, + text="Internal Server Error" + ) + + with pytest.raises(Exception) as exc_info: + client.get_repository(repo_name, create=True) + assert "Failed to create repository" in str(exc_info.value) \ No newline at end of file diff --git a/eks_automation/tests/test_github_client_integration.py b/eks_automation/tests/test_github_client_integration.py new file mode 100644 index 0000000..1774b61 --- /dev/null +++ b/eks_automation/tests/test_github_client_integration.py @@ -0,0 +1,196 @@ +import os +import json +import pytest +import requests +import tempfile +import shutil +import uuid +import time +from datetime import datetime + +from ..app import GitHubClient + +# Skip all tests if no GitHub token is available +pytestmark = [ + pytest.mark.skipif( + "GITHUB_TOKEN" not in os.environ, + reason="GITHUB_TOKEN environment variable not set" + ), + pytest.mark.integration +] + +@pytest.fixture +def integration_client(): + """Create a GitHubClient instance for integration testing""" + token = os.environ["GITHUB_TOKEN"] + api_url = os.environ.get("GITHUB_API", "https://api.github.com") + org_name = os.environ.get("GITHUB_ORG", "test-org") + + client = GitHubClient( + api_base_url=api_url, + token=token, + org_name=org_name, + commit_author_name="Integration Test", + commit_author_email="test@example.com", + source_version=None, + template_repo_name="template-lambda-deployment", + config_file_name="config.json" + ) + return client + +@pytest.fixture +def temp_repo_name(): + """Generate a unique temporary repository name""" + return f"temp-test-repo-{uuid.uuid4().hex[:8]}" + +@pytest.fixture +def cleanup_repo(integration_client): + """Fixture to clean up test repository after tests""" + repo_names = [] + + def _register_repo(repo_name): + repo_names.append(repo_name) + return repo_name + + yield _register_repo + + # Clean up all registered repos + for repo_name in repo_names: + try: + # Note: Real deletion would require additional API calls + # For safety in testing, we just archive the repo + requests.patch( + f"{integration_client.api_base_url}/repos/{integration_client.org_name}/{repo_name}", + headers=integration_client.headers, + json={"archived": True}, + verify=False + ) + except Exception as e: + print(f"Failed to archive repository {repo_name}: {e}") + +class TestGitHubClientIntegration: + """Integration tests for GitHubClient using real GitHub API""" + + def test_repository_creation(self, integration_client, temp_repo_name, cleanup_repo): + """Test creating a new repository via the API""" + repo_name = cleanup_repo(temp_repo_name) + + # Create new repository + repo = integration_client.get_repository(repo_name, create=True) + + assert repo is not None + assert repo["name"] == repo_name + assert not repo["archived"] + + # Verify we can get the repository + repo = integration_client.get_repository(repo_name) + assert repo["name"] == repo_name + + def test_file_operations(self, integration_client, temp_repo_name, cleanup_repo, tmp_path): + """Test file operations with real repository""" + repo_name = cleanup_repo(temp_repo_name) + + # Create new repository + repo = integration_client.get_repository(repo_name, create=True) + + # Create a test file + test_content = { + "test": True, + "timestamp": datetime.utcnow().isoformat() + } + + # Write test content to work directory + work_dir = str(tmp_path) + os.makedirs(work_dir, exist_ok=True) + test_file = os.path.join(work_dir, "test-config.json") + + with open(test_file, "w") as f: + json.dump(test_content, f, indent=2) + + # Commit the file + integration_client.commit_repository_contents( + repo_name, + work_dir, + "Test commit from integration tests" + ) + + # Add a short delay to allow GitHub API to become consistent + time.sleep(2) + + # Verify the file exists in the repository + # Clone to a new directory and verify contents + clone_dir = os.path.join(str(tmp_path), "clone") + os.makedirs(clone_dir, exist_ok=True) + + integration_client.clone_repository_contents(repo_name, clone_dir) + + cloned_file = os.path.join(clone_dir, "test-config.json") + assert os.path.exists(cloned_file) + + with open(cloned_file, "r") as f: + cloned_content = json.load(f) + + assert cloned_content["test"] == test_content["test"] + assert cloned_content["timestamp"] == test_content["timestamp"] + + def test_branch_operations(self, integration_client, temp_repo_name, cleanup_repo, tmp_path): + """Test branch creation and updates""" + repo_name = cleanup_repo(temp_repo_name) + + # Create new repository + repo = integration_client.get_repository(repo_name, create=True) + + # Create a test file and commit to main + work_dir = str(tmp_path) + os.makedirs(work_dir, exist_ok=True) + + with open(os.path.join(work_dir, "test.txt"), "w") as f: + f.write("main branch content") + + # Commit to main + integration_client.commit_repository_contents( + repo_name, + work_dir, + "Initial commit", + branch="main" + ) + + # Create a new branch + main_sha = integration_client.get_reference_sha(repo_name, "heads/main") + integration_client.create_reference( + repo_name, + "refs/heads/test-branch", + main_sha + ) + + # Update file in new branch + with open(os.path.join(work_dir, "test.txt"), "w") as f: + f.write("test branch content") + + # Commit to test branch + integration_client.commit_repository_contents( + repo_name, + work_dir, + "Update in test branch", + branch="test-branch" + ) + + # Verify the changes + clone_dir = os.path.join(str(tmp_path), "clone") + os.makedirs(clone_dir, exist_ok=True) + + # Clone and verify main branch + main_dir = os.path.join(clone_dir, "main") + integration_client.clone_repository_contents(repo_name, main_dir, branch="main") + + with open(os.path.join(main_dir, "test.txt"), "r") as f: + main_content = f.read() + assert main_content == "main branch content" + + # Clone and verify test branch contents + test_dir = os.path.join(clone_dir, "test") + integration_client.clone_repository_contents(repo_name, test_dir, branch="test-branch") + + with open(os.path.join(test_dir, "test.txt"), "r") as f: + test_content = f.read() + assert test_content == "test branch content" \ No newline at end of file diff --git a/errors.txt b/errors.txt new file mode 100644 index 0000000..53eb104 --- /dev/null +++ b/errors.txt @@ -0,0 +1,103 @@ +=================================== FAILURES =================================== +_______________ TestGitHubClientIntegration.test_file_operations _______________ +self = +integration_client = +temp_repo_name = 'temp-test-repo-fc9ad0f3' +cleanup_repo = ._register_repo at 0x7f6a6137dee0> +tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/test_file_operations0') + def test_file_operations(self, integration_client, temp_repo_name, cleanup_repo, tmp_path): + """Test file operations with real repository""" + repo_name = cleanup_repo(temp_repo_name) + + # Create new repository + repo = integration_client.get_repository(repo_name, create=True) + + # Create a test file + test_content = { + "test": True, + "timestamp": datetime.utcnow().isoformat() + } + + # Write test content to work directory + work_dir = str(tmp_path) + os.makedirs(work_dir, exist_ok=True) + test_file = os.path.join(work_dir, "test-config.json") + + with open(test_file, "w") as f: + json.dump(test_content, f, indent=2) + + # Commit the file +> integration_client.commit_repository_contents( + repo_name, + work_dir, + "Test commit from integration tests" + ) +tests/test_github_client_integration.py:110: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +app.py:446: in commit_repository_contents + blob_sha = self.create_blob(repo_name, file_content) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +self = +repo_name = 'temp-test-repo-fc9ad0f3' +content = b'{\n "test": true,\n "timestamp": "2025-04-17T16:25:48.668975"\n}' + def create_blob(self, repo_name, content): + """Create a blob in the repository + + Args: + repo_name (str): Name of the repository + content (bytes): Content of the blob + + Returns: + str: SHA of the created blob + """ + api_url = f"{self.api_base_url}/repos/{self.org_name}/{repo_name}/git/blobs" + + # Base64 encode the content + content_b64 = base64.b64encode(content).decode('utf-8') + + data = { + "content": content_b64, + "encoding": "base64" + } + + response = requests.post(api_url, headers=self.headers, json=data, verify=False) + + if response.status_code in (201, 200): + return response.json()["sha"] + else: + error_message = f"Failed to create blob for {repo_name}: {response.status_code} - {response.text}" + logger.error(error_message) +> raise Exception(error_message) +E Exception: Failed to create blob for temp-test-repo-fc9ad0f3: 409 - {"message":"Git Repository is empty.","documentation_url":"https://docs.github.com/rest/git/blobs#create-a-blob","status":"409"} +app.py:256: Exception +------------------------------ Captured log call ------------------------------- +INFO root:app.py:81 Checking if repository temp-test-repo-fc9ad0f3 exists +INFO root:app.py:89 Creating repository temp-test-repo-fc9ad0f3 + "content": content_b64, + "encoding": "base64" + } + + response = requests.post(api_url, headers=self.headers, json=data, verify=False) + + if response.status_code in (201, 200): + return response.json()["sha"] + else: + error_message = f"Failed to create blob for {repo_name}: {response.status_code} - {response.text}" + logger.error(error_message) +> raise Exception(error_message) +E Exception: Failed to create blob for temp-test-repo-e0e2ba0e: 409 - {"message":"Git Repository is empty.","documentation_url":"https://docs.github.com/rest/git/blobs#create-a-blob","status":"409"} +app.py:256: Exception +------------------------------ Captured log call ------------------------------- +INFO root:app.py:81 Checking if repository temp-test-repo-e0e2ba0e exists +INFO root:app.py:89 Creating repository temp-test-repo-e0e2ba0e +ERROR root:app.py:255 Failed to create blob for temp-test-repo-e0e2ba0e: 409 - {"message":"Git Repository is empty.","documentation_url":"https://docs.github.com/rest/git/blobs#create-a-blob","status":"409"} +=============================== warnings summary =============================== +tests/test_github_client_integration.py: 14 warnings + /opt/hostedtoolcache/Python/3.9.22/x64/lib/python3.9/site-packages/urllib3/connectionpool.py:1064: InsecureRequestWarning: Unverified HTTPS request is being made to host 'api.github.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings + warnings.warn( +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +=========================== short test summary info ============================ +FAILED tests/test_github_client_integration.py::TestGitHubClientIntegration::test_file_operations - Exception: Failed to create blob for temp-test-repo-fc9ad0f3: 409 - {"message":"Git Repository is empty.","documentation_url":"https://docs.github.com/rest/git/blobs#create-a-blob","status":"409"} +FAILED tests/test_github_client_integration.py::TestGitHubClientIntegration::test_branch_operations - Exception: Failed to create blob for temp-test-repo-e0e2ba0e: 409 - {"message":"Git Repository is empty.","documentation_url":"https://docs.github.com/rest/git/blobs#create-a-blob","status":"409"} +=========== 2 failed, 1 passed, 12 deselected, 14 warnings in 4.33s ============ +Error: Process completed with exit code 1. \ No newline at end of file diff --git a/packer.pkr.hcl b/packer.pkr.hcl index f4fc1d9..f189f11 100644 --- a/packer.pkr.hcl +++ b/packer.pkr.hcl @@ -21,7 +21,7 @@ source "docker" "lambda" { commit = true changes = [ "WORKDIR /var/task", - "CMD [ \"app.handler\" ]" + "CMD [ \"app.lambda_handler\" ]" ] } @@ -33,14 +33,14 @@ build { ] provisioner "file" { - source = "./eks_automation" + source = "./eks_automation/" destination = "/var/task" } provisioner "shell" { inline = [ - "cd /var/task", - "pip3 install -r requirements.txt -t ." + "ls -la /var/task", # Debug: List contents + "pip3 install -r /var/task/requirements.txt -t /var/task" ] } diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..7592d02 --- /dev/null +++ b/variables.tf @@ -0,0 +1,29 @@ +variable "github_api" { + description = "URL for the GitHub Enterprise API" + type = string + default = "https://github.enterprise.example.com/api/v3" +} + +variable "github_org_name" { + description = "GitHub organization name" + type = string + default = "your-org" +} + +variable "github_token_secret_name" { + description = "AWS SSM parameter name for the GitHub token" + type = string + default = "/github/token" +} + +variable "template_repo_name" { + description = "GitHub repository name for the EKS template" + type = string + default = "template-eks-cluster" +} + +variable "template_file_name" { + description = "Template file name for the EKS configuration" + type = string + default = "eks.hcl.j2" +} \ No newline at end of file