diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..047e645 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +.PHONY: install test test-unit test-integration clean + +# Variables +PYTHON = python3 +PIP = $(PYTHON) -m pip +PYTEST = $(PYTHON) -m pytest +REQUIREMENTS = eks_automation/requirements.txt +TEST_DIR = eks_automation/tests +UNIT_TEST_FILE = $(TEST_DIR)/test_github_client.py +INTEGRATION_TEST_FILE = $(TEST_DIR)/test_github_client_integration.py + +# Default target +all: test + +# Install dependencies +install: + $(PIP) install -r $(REQUIREMENTS) + +# Run all tests +test: test-unit test-integration + @echo "Running all tests..." + $(PYTEST) $(TEST_DIR) + +# Run unit tests +test-unit: + @echo "Running unit tests..." + $(PYTEST) $(UNIT_TEST_FILE) + +# Run integration tests +test-integration: + @echo "Running integration tests..." + $(PYTEST) $(INTEGRATION_TEST_FILE) + +# Clean up Python cache files +clean: + find . -type f -name '*.pyc' -delete + find . -type d -name '__pycache__' -exec rm -rf {} + + rm -rf .pytest_cache + rm -rf .coverage diff --git a/eks-automation-lambda.code-workspace b/eks-automation-lambda.code-workspace new file mode 100644 index 0000000..061431d --- /dev/null +++ b/eks-automation-lambda.code-workspace @@ -0,0 +1,20 @@ +{ + "folders": [ + { + "path": "." + }, + { + "path": "../github-repos" + }, + { + "path": "../github-runner-image" + }, + { + "path": "../template-lambda-deployment" + }, + { + "path": "../terraform-github-repo" + } + ], + "settings": {} +} \ No newline at end of file diff --git a/eks_automation/app.py b/eks_automation/app.py index 88734d4..913d25e 100644 --- a/eks_automation/app.py +++ b/eks_automation/app.py @@ -84,30 +84,44 @@ def get_repository(self, repo_name, create=False): if response.status_code == 200: # Repository exists return response.json() - elif response.status_code == 404 and create: - # Repository doesn't exist, create it - logger.info(f"Creating repository {repo_name}") - create_url = f"{self.api_base_url}/orgs/{self.org_name}/repos" - repo_data = { - "name": repo_name, - "description": "EKS Automation CI/CD Pipeline Repo", - "private": True - } - create_response = requests.post( - create_url, - headers=self.headers, - json=repo_data, - verify=False - ) - - if create_response.status_code in (201, 200): - return create_response.json() + elif response.status_code == 404: + if create: + # Repository doesn't exist, create it + logger.info(f"Creating repository {repo_name}") + create_url = f"{self.api_base_url}/orgs/{self.org_name}/repos" + repo_data = { + "name": repo_name, + "description": "EKS Automation CI/CD Pipeline Repo", + "private": True, + "auto_init": True, # Initialize with README + "default_branch": "main", + "allow_squash_merge": True, + "allow_merge_commit": True, + "allow_rebase_merge": True, + "delete_branch_on_merge": True, + "enable_branch_protection": False # Disable branch protection + } + create_response = requests.post( + create_url, + headers=self.headers, + json=repo_data, + verify=False + ) + + if create_response.status_code in (201, 200): + # Wait briefly for repository initialization + time.sleep(2) + return create_response.json() + else: + error_message = f"Failed to create repository: {create_response.status_code} - {create_response.text}" + logger.error(error_message) + raise Exception(error_message) else: - error_message = f"Failed to create repository: {create_response.status_code} - {create_response.text}" + error_message = f"Repository {repo_name} not found and create=False" logger.error(error_message) raise Exception(error_message) else: - error_message = f"Repository {repo_name} not found and create=False" + error_message = f"Unexpected response when getting repository: {response.status_code} - {response.text}" logger.error(error_message) raise Exception(error_message) @@ -204,6 +218,9 @@ def download_repository_files(self, repo_name, tree, target_dir): tree (dict): Tree information from get_tree() target_dir (str): Directory to download files to """ + # Ensure target directory exists even if there are no files + os.makedirs(target_dir, exist_ok=True) + for item in tree.get("tree", []): if item["type"] == "blob": # Get the blob contents @@ -216,13 +233,18 @@ def download_repository_files(self, repo_name, tree, target_dir): # Ensure the target directory exists file_path = os.path.join(target_dir, item["path"]) - os.makedirs(os.path.dirname(file_path), exist_ok=True) + dir_path = os.path.dirname(file_path) + os.makedirs(dir_path, exist_ok=True) # GitHub API returns base64 encoded content if blob_data.get("encoding") == "base64": content = base64.b64decode(blob_data.get("content", "")) + else: + # Handle non-base64 content if needed + logger.warning(f"Unexpected encoding for blob {item['sha']}: {blob_data.get('encoding')}") if content is not None: + logger.info(f"Writing file to {file_path}") with open(file_path, "wb") as f: f.write(content) @@ -366,64 +388,65 @@ def create_reference(self, repo_name, ref, sha): logger.error(error_message) raise Exception(error_message) - def clone_repository_contents(self, source_repo, target_dir): + def clone_repository_contents(self, source_repo, target_dir, branch=None): """Clone a repository's contents to a local directory using GitHub API Args: source_repo (str): Name of the source repository target_dir (str): Target directory to download files to + branch (str, optional): Branch to clone from. If None, uses default branch. Returns: - str: The default branch name of the repository + str: The branch name that was cloned """ - # Get default branch of original repo for fallback - default_branch = self.get_default_branch(source_repo) - logger.info(f"Default branch for {source_repo}: {default_branch}") - - # Get tree from original repository - logger.info(f"Getting file tree from {source_repo}") - ref = None - - if self.source_version: - try: - # Try to get the tag/release reference first - ref = f"tags/{self.source_version}" - tree_sha = self.get_reference_sha(source_repo, ref) - logger.info(f"Using source version: {self.source_version}") - except Exception as e: - logger.warning(f"Failed to get version {self.source_version}, falling back to default branch: {str(e)}") - ref = f"heads/{default_branch}" - tree_sha = self.get_reference_sha(source_repo, ref) - else: - # Use default branch - ref = f"heads/{default_branch}" - tree_sha = self.get_reference_sha(source_repo, ref) + # Create the target directory if it doesn't exist + os.makedirs(target_dir, exist_ok=True) + try: + if branch: + target_branch = branch + # Try to get the branch's reference directly + tree_sha = self.get_reference_sha(source_repo, f"heads/{target_branch}") + else: + # If no branch specified, use default branch + target_branch = self.get_default_branch(source_repo) + tree_sha = self.get_reference_sha(source_repo, f"heads/{target_branch}") + except Exception as e: + logger.warning(f"Failed to get reference for {branch or 'default branch'}: {str(e)}") + target_branch = branch or "main" + # If we can't get the reference, the branch might not exist yet + tree = {"tree": []} + self.download_repository_files(source_repo, tree, target_dir) + return target_branch + + # Get the full tree for the branch + logger.info(f"Getting file tree from {source_repo} for branch {target_branch}") tree = self.get_tree(source_repo, tree_sha, recursive=True) - - # Download all files from original repo to work directory - logger.info(f"Downloading all files from {source_repo} using ref: {ref}") + + # Download all files + logger.info(f"Downloading all files from {source_repo} using ref: heads/{target_branch}") self.download_repository_files(source_repo, tree, target_dir) - - return default_branch + + return target_branch - def commit_repository_contents(self, repo_name, work_dir, commit_message): + def commit_repository_contents(self, repo_name, work_dir, commit_message, branch=None): """Commit all files from a directory to a repository Args: repo_name (str): Name of the repository work_dir (str): Directory containing the files to commit commit_message (str): Commit message + branch (str, optional): Branch to commit to. If None, uses default branch. Returns: - str: The default branch name of the repository + str: The branch name that was committed to """ # First, get the current state of the target repository try: - target_default_branch = self.get_default_branch(repo_name) + target_branch = branch or self.get_default_branch(repo_name) except Exception: # If we can't get the default branch, it might be a new repo - target_default_branch = "main" + target_branch = branch or "main" # Upload all files to the repository tree_items = [] @@ -456,7 +479,7 @@ def commit_repository_contents(self, repo_name, work_dir, commit_message): # Try to get the latest commit SHA for the branch # If it doesn't exist, we'll create it try: - latest_commit_sha = self.get_reference_sha(repo_name, f"heads/{target_default_branch}") + latest_commit_sha = self.get_reference_sha(repo_name, f"heads/{target_branch}") latest_commit = self.get_commit(repo_name, latest_commit_sha) base_tree_sha = latest_commit["tree"]["sha"] except Exception: @@ -488,18 +511,18 @@ def commit_repository_contents(self, repo_name, work_dir, commit_message): try: self.update_reference( repo_name, - f"heads/{target_default_branch}", + f"heads/{target_branch}", new_commit_sha ) except Exception: # If the reference doesn't exist, create it self.create_reference( repo_name, - f"refs/heads/{target_default_branch}", + f"refs/heads/{target_branch}", new_commit_sha ) - return target_default_branch + return target_branch # pylint: disable=unused-argument @@ -595,9 +618,9 @@ def operate_github(new_repo_name, eks_settings): with open(output_file_path, "w") as file: json.dump(eks_settings, file, indent=2) - # Commit all files to the new repository + # Commit all files to the new repository's main branch explicitly commit_message = "Add the EKS configuration file by the Lambda function" - github.commit_repository_contents(new_repo_name, work_dir, commit_message) + github.commit_repository_contents(new_repo_name, work_dir, commit_message, branch="main") logger.info(f"Successfully updated {new_repo_name} repository") diff --git a/eks_automation/tests/test_github_client_integration.py b/eks_automation/tests/test_github_client_integration.py index 7332b83..1890841 100644 --- a/eks_automation/tests/test_github_client_integration.py +++ b/eks_automation/tests/test_github_client_integration.py @@ -5,6 +5,7 @@ import tempfile import shutil import uuid +import time from datetime import datetime from ..app import GitHubClient @@ -113,6 +114,9 @@ def test_file_operations(self, integration_client, temp_repo_name, cleanup_repo, "Test commit from integration tests" ) + # Add a short delay to allow GitHub API to become consistent + time.sleep(2) + # Verify the file exists in the repository # Clone to a new directory and verify contents clone_dir = os.path.join(str(tmp_path), "clone") @@ -147,7 +151,8 @@ def test_branch_operations(self, integration_client, temp_repo_name, cleanup_rep integration_client.commit_repository_contents( repo_name, work_dir, - "Initial commit" + "Initial commit", + branch="main" ) # Create a new branch @@ -166,7 +171,8 @@ def test_branch_operations(self, integration_client, temp_repo_name, cleanup_rep integration_client.commit_repository_contents( repo_name, work_dir, - "Update in test branch" + "Update in test branch", + branch="test-branch" ) # Verify the changes @@ -175,8 +181,16 @@ def test_branch_operations(self, integration_client, temp_repo_name, cleanup_rep # Clone and verify main branch main_dir = os.path.join(clone_dir, "main") - integration_client.clone_repository_contents(repo_name, main_dir) + integration_client.clone_repository_contents(repo_name, main_dir, branch="main") with open(os.path.join(main_dir, "test.txt"), "r") as f: main_content = f.read() - assert main_content == "main branch content" \ No newline at end of file + assert main_content == "main branch content" + + # Clone and verify test branch contents + test_dir = os.path.join(clone_dir, "test") + integration_client.clone_repository_contents(repo_name, test_dir, branch="test-branch") + + with open(os.path.join(test_dir, "test.txt"), "r") as f: + test_content = f.read() + assert test_content == "test branch content" diff --git a/errors.txt b/errors.txt new file mode 100644 index 0000000..53eb104 --- /dev/null +++ b/errors.txt @@ -0,0 +1,103 @@ +=================================== FAILURES =================================== +_______________ TestGitHubClientIntegration.test_file_operations _______________ +self = +integration_client = +temp_repo_name = 'temp-test-repo-fc9ad0f3' +cleanup_repo = ._register_repo at 0x7f6a6137dee0> +tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/test_file_operations0') + def test_file_operations(self, integration_client, temp_repo_name, cleanup_repo, tmp_path): + """Test file operations with real repository""" + repo_name = cleanup_repo(temp_repo_name) + + # Create new repository + repo = integration_client.get_repository(repo_name, create=True) + + # Create a test file + test_content = { + "test": True, + "timestamp": datetime.utcnow().isoformat() + } + + # Write test content to work directory + work_dir = str(tmp_path) + os.makedirs(work_dir, exist_ok=True) + test_file = os.path.join(work_dir, "test-config.json") + + with open(test_file, "w") as f: + json.dump(test_content, f, indent=2) + + # Commit the file +> integration_client.commit_repository_contents( + repo_name, + work_dir, + "Test commit from integration tests" + ) +tests/test_github_client_integration.py:110: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +app.py:446: in commit_repository_contents + blob_sha = self.create_blob(repo_name, file_content) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +self = +repo_name = 'temp-test-repo-fc9ad0f3' +content = b'{\n "test": true,\n "timestamp": "2025-04-17T16:25:48.668975"\n}' + def create_blob(self, repo_name, content): + """Create a blob in the repository + + Args: + repo_name (str): Name of the repository + content (bytes): Content of the blob + + Returns: + str: SHA of the created blob + """ + api_url = f"{self.api_base_url}/repos/{self.org_name}/{repo_name}/git/blobs" + + # Base64 encode the content + content_b64 = base64.b64encode(content).decode('utf-8') + + data = { + "content": content_b64, + "encoding": "base64" + } + + response = requests.post(api_url, headers=self.headers, json=data, verify=False) + + if response.status_code in (201, 200): + return response.json()["sha"] + else: + error_message = f"Failed to create blob for {repo_name}: {response.status_code} - {response.text}" + logger.error(error_message) +> raise Exception(error_message) +E Exception: Failed to create blob for temp-test-repo-fc9ad0f3: 409 - {"message":"Git Repository is empty.","documentation_url":"https://docs.github.com/rest/git/blobs#create-a-blob","status":"409"} +app.py:256: Exception +------------------------------ Captured log call ------------------------------- +INFO root:app.py:81 Checking if repository temp-test-repo-fc9ad0f3 exists +INFO root:app.py:89 Creating repository temp-test-repo-fc9ad0f3 + "content": content_b64, + "encoding": "base64" + } + + response = requests.post(api_url, headers=self.headers, json=data, verify=False) + + if response.status_code in (201, 200): + return response.json()["sha"] + else: + error_message = f"Failed to create blob for {repo_name}: {response.status_code} - {response.text}" + logger.error(error_message) +> raise Exception(error_message) +E Exception: Failed to create blob for temp-test-repo-e0e2ba0e: 409 - {"message":"Git Repository is empty.","documentation_url":"https://docs.github.com/rest/git/blobs#create-a-blob","status":"409"} +app.py:256: Exception +------------------------------ Captured log call ------------------------------- +INFO root:app.py:81 Checking if repository temp-test-repo-e0e2ba0e exists +INFO root:app.py:89 Creating repository temp-test-repo-e0e2ba0e +ERROR root:app.py:255 Failed to create blob for temp-test-repo-e0e2ba0e: 409 - {"message":"Git Repository is empty.","documentation_url":"https://docs.github.com/rest/git/blobs#create-a-blob","status":"409"} +=============================== warnings summary =============================== +tests/test_github_client_integration.py: 14 warnings + /opt/hostedtoolcache/Python/3.9.22/x64/lib/python3.9/site-packages/urllib3/connectionpool.py:1064: InsecureRequestWarning: Unverified HTTPS request is being made to host 'api.github.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings + warnings.warn( +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +=========================== short test summary info ============================ +FAILED tests/test_github_client_integration.py::TestGitHubClientIntegration::test_file_operations - Exception: Failed to create blob for temp-test-repo-fc9ad0f3: 409 - {"message":"Git Repository is empty.","documentation_url":"https://docs.github.com/rest/git/blobs#create-a-blob","status":"409"} +FAILED tests/test_github_client_integration.py::TestGitHubClientIntegration::test_branch_operations - Exception: Failed to create blob for temp-test-repo-e0e2ba0e: 409 - {"message":"Git Repository is empty.","documentation_url":"https://docs.github.com/rest/git/blobs#create-a-blob","status":"409"} +=========== 2 failed, 1 passed, 12 deselected, 14 warnings in 4.33s ============ +Error: Process completed with exit code 1. \ No newline at end of file