Skip to content

Commit

Permalink
updating
Browse files Browse the repository at this point in the history
  • Loading branch information
Dave Arnold committed Apr 17, 2025
2 parents f15cb3c + 53039e3 commit b6c254d
Show file tree
Hide file tree
Showing 5 changed files with 263 additions and 64 deletions.
39 changes: 39 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
.PHONY: install test test-unit test-integration clean

# Variables
PYTHON = python3
PIP = $(PYTHON) -m pip
PYTEST = $(PYTHON) -m pytest
REQUIREMENTS = eks_automation/requirements.txt
TEST_DIR = eks_automation/tests
UNIT_TEST_FILE = $(TEST_DIR)/test_github_client.py
INTEGRATION_TEST_FILE = $(TEST_DIR)/test_github_client_integration.py

# Default target
all: test

# Install dependencies
install:
$(PIP) install -r $(REQUIREMENTS)

# Run all tests
test: test-unit test-integration
@echo "Running all tests..."
$(PYTEST) $(TEST_DIR)

# Run unit tests
test-unit:
@echo "Running unit tests..."
$(PYTEST) $(UNIT_TEST_FILE)

# Run integration tests
test-integration:
@echo "Running integration tests..."
$(PYTEST) $(INTEGRATION_TEST_FILE)

# Clean up Python cache files
clean:
find . -type f -name '*.pyc' -delete
find . -type d -name '__pycache__' -exec rm -rf {} +
rm -rf .pytest_cache
rm -rf .coverage
20 changes: 20 additions & 0 deletions eks-automation-lambda.code-workspace
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"folders": [
{
"path": "."
},
{
"path": "../github-repos"
},
{
"path": "../github-runner-image"
},
{
"path": "../template-lambda-deployment"
},
{
"path": "../terraform-github-repo"
}
],
"settings": {}
}
143 changes: 83 additions & 60 deletions eks_automation/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,30 +84,44 @@ def get_repository(self, repo_name, create=False):
if response.status_code == 200:
# Repository exists
return response.json()
elif response.status_code == 404 and create:
# Repository doesn't exist, create it
logger.info(f"Creating repository {repo_name}")
create_url = f"{self.api_base_url}/orgs/{self.org_name}/repos"
repo_data = {
"name": repo_name,
"description": "EKS Automation CI/CD Pipeline Repo",
"private": True
}
create_response = requests.post(
create_url,
headers=self.headers,
json=repo_data,
verify=False
)

if create_response.status_code in (201, 200):
return create_response.json()
elif response.status_code == 404:
if create:
# Repository doesn't exist, create it
logger.info(f"Creating repository {repo_name}")
create_url = f"{self.api_base_url}/orgs/{self.org_name}/repos"
repo_data = {
"name": repo_name,
"description": "EKS Automation CI/CD Pipeline Repo",
"private": True,
"auto_init": True, # Initialize with README
"default_branch": "main",
"allow_squash_merge": True,
"allow_merge_commit": True,
"allow_rebase_merge": True,
"delete_branch_on_merge": True,
"enable_branch_protection": False # Disable branch protection
}
create_response = requests.post(
create_url,
headers=self.headers,
json=repo_data,
verify=False
)

if create_response.status_code in (201, 200):
# Wait briefly for repository initialization
time.sleep(2)
return create_response.json()
else:
error_message = f"Failed to create repository: {create_response.status_code} - {create_response.text}"
logger.error(error_message)
raise Exception(error_message)
else:
error_message = f"Failed to create repository: {create_response.status_code} - {create_response.text}"
error_message = f"Repository {repo_name} not found and create=False"
logger.error(error_message)
raise Exception(error_message)
else:
error_message = f"Repository {repo_name} not found and create=False"
error_message = f"Unexpected response when getting repository: {response.status_code} - {response.text}"
logger.error(error_message)
raise Exception(error_message)

Expand Down Expand Up @@ -204,6 +218,9 @@ def download_repository_files(self, repo_name, tree, target_dir):
tree (dict): Tree information from get_tree()
target_dir (str): Directory to download files to
"""
# Ensure target directory exists even if there are no files
os.makedirs(target_dir, exist_ok=True)

for item in tree.get("tree", []):
if item["type"] == "blob":
# Get the blob contents
Expand All @@ -216,13 +233,18 @@ def download_repository_files(self, repo_name, tree, target_dir):

# Ensure the target directory exists
file_path = os.path.join(target_dir, item["path"])
os.makedirs(os.path.dirname(file_path), exist_ok=True)
dir_path = os.path.dirname(file_path)
os.makedirs(dir_path, exist_ok=True)

# GitHub API returns base64 encoded content
if blob_data.get("encoding") == "base64":
content = base64.b64decode(blob_data.get("content", ""))
else:
# Handle non-base64 content if needed
logger.warning(f"Unexpected encoding for blob {item['sha']}: {blob_data.get('encoding')}")

if content is not None:
logger.info(f"Writing file to {file_path}")
with open(file_path, "wb") as f:
f.write(content)

Expand Down Expand Up @@ -366,64 +388,65 @@ def create_reference(self, repo_name, ref, sha):
logger.error(error_message)
raise Exception(error_message)

def clone_repository_contents(self, source_repo, target_dir):
def clone_repository_contents(self, source_repo, target_dir, branch=None):
"""Clone a repository's contents to a local directory using GitHub API
Args:
source_repo (str): Name of the source repository
target_dir (str): Target directory to download files to
branch (str, optional): Branch to clone from. If None, uses default branch.
Returns:
str: The default branch name of the repository
str: The branch name that was cloned
"""
# Get default branch of original repo for fallback
default_branch = self.get_default_branch(source_repo)
logger.info(f"Default branch for {source_repo}: {default_branch}")

# Get tree from original repository
logger.info(f"Getting file tree from {source_repo}")
ref = None

if self.source_version:
try:
# Try to get the tag/release reference first
ref = f"tags/{self.source_version}"
tree_sha = self.get_reference_sha(source_repo, ref)
logger.info(f"Using source version: {self.source_version}")
except Exception as e:
logger.warning(f"Failed to get version {self.source_version}, falling back to default branch: {str(e)}")
ref = f"heads/{default_branch}"
tree_sha = self.get_reference_sha(source_repo, ref)
else:
# Use default branch
ref = f"heads/{default_branch}"
tree_sha = self.get_reference_sha(source_repo, ref)
# Create the target directory if it doesn't exist
os.makedirs(target_dir, exist_ok=True)

try:
if branch:
target_branch = branch
# Try to get the branch's reference directly
tree_sha = self.get_reference_sha(source_repo, f"heads/{target_branch}")
else:
# If no branch specified, use default branch
target_branch = self.get_default_branch(source_repo)
tree_sha = self.get_reference_sha(source_repo, f"heads/{target_branch}")
except Exception as e:
logger.warning(f"Failed to get reference for {branch or 'default branch'}: {str(e)}")
target_branch = branch or "main"
# If we can't get the reference, the branch might not exist yet
tree = {"tree": []}
self.download_repository_files(source_repo, tree, target_dir)
return target_branch

# Get the full tree for the branch
logger.info(f"Getting file tree from {source_repo} for branch {target_branch}")
tree = self.get_tree(source_repo, tree_sha, recursive=True)
# Download all files from original repo to work directory
logger.info(f"Downloading all files from {source_repo} using ref: {ref}")

# Download all files
logger.info(f"Downloading all files from {source_repo} using ref: heads/{target_branch}")
self.download_repository_files(source_repo, tree, target_dir)
return default_branch

return target_branch

def commit_repository_contents(self, repo_name, work_dir, commit_message):
def commit_repository_contents(self, repo_name, work_dir, commit_message, branch=None):
"""Commit all files from a directory to a repository
Args:
repo_name (str): Name of the repository
work_dir (str): Directory containing the files to commit
commit_message (str): Commit message
branch (str, optional): Branch to commit to. If None, uses default branch.
Returns:
str: The default branch name of the repository
str: The branch name that was committed to
"""
# First, get the current state of the target repository
try:
target_default_branch = self.get_default_branch(repo_name)
target_branch = branch or self.get_default_branch(repo_name)
except Exception:
# If we can't get the default branch, it might be a new repo
target_default_branch = "main"
target_branch = branch or "main"

# Upload all files to the repository
tree_items = []
Expand Down Expand Up @@ -456,7 +479,7 @@ def commit_repository_contents(self, repo_name, work_dir, commit_message):
# Try to get the latest commit SHA for the branch
# If it doesn't exist, we'll create it
try:
latest_commit_sha = self.get_reference_sha(repo_name, f"heads/{target_default_branch}")
latest_commit_sha = self.get_reference_sha(repo_name, f"heads/{target_branch}")
latest_commit = self.get_commit(repo_name, latest_commit_sha)
base_tree_sha = latest_commit["tree"]["sha"]
except Exception:
Expand Down Expand Up @@ -488,18 +511,18 @@ def commit_repository_contents(self, repo_name, work_dir, commit_message):
try:
self.update_reference(
repo_name,
f"heads/{target_default_branch}",
f"heads/{target_branch}",
new_commit_sha
)
except Exception:
# If the reference doesn't exist, create it
self.create_reference(
repo_name,
f"refs/heads/{target_default_branch}",
f"refs/heads/{target_branch}",
new_commit_sha
)

return target_default_branch
return target_branch


# pylint: disable=unused-argument
Expand Down Expand Up @@ -595,9 +618,9 @@ def operate_github(new_repo_name, eks_settings):
with open(output_file_path, "w") as file:
json.dump(eks_settings, file, indent=2)

# Commit all files to the new repository
# Commit all files to the new repository's main branch explicitly
commit_message = "Add the EKS configuration file by the Lambda function"
github.commit_repository_contents(new_repo_name, work_dir, commit_message)
github.commit_repository_contents(new_repo_name, work_dir, commit_message, branch="main")

logger.info(f"Successfully updated {new_repo_name} repository")

Expand Down
22 changes: 18 additions & 4 deletions eks_automation/tests/test_github_client_integration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import tempfile
import shutil
import uuid
import time
from datetime import datetime

from ..app import GitHubClient
Expand Down Expand Up @@ -113,6 +114,9 @@ def test_file_operations(self, integration_client, temp_repo_name, cleanup_repo,
"Test commit from integration tests"
)

# Add a short delay to allow GitHub API to become consistent
time.sleep(2)

# Verify the file exists in the repository
# Clone to a new directory and verify contents
clone_dir = os.path.join(str(tmp_path), "clone")
Expand Down Expand Up @@ -147,7 +151,8 @@ def test_branch_operations(self, integration_client, temp_repo_name, cleanup_rep
integration_client.commit_repository_contents(
repo_name,
work_dir,
"Initial commit"
"Initial commit",
branch="main"
)

# Create a new branch
Expand All @@ -166,7 +171,8 @@ def test_branch_operations(self, integration_client, temp_repo_name, cleanup_rep
integration_client.commit_repository_contents(
repo_name,
work_dir,
"Update in test branch"
"Update in test branch",
branch="test-branch"
)

# Verify the changes
Expand All @@ -175,8 +181,16 @@ def test_branch_operations(self, integration_client, temp_repo_name, cleanup_rep

# Clone and verify main branch
main_dir = os.path.join(clone_dir, "main")
integration_client.clone_repository_contents(repo_name, main_dir)
integration_client.clone_repository_contents(repo_name, main_dir, branch="main")

with open(os.path.join(main_dir, "test.txt"), "r") as f:
main_content = f.read()
assert main_content == "main branch content"
assert main_content == "main branch content"

# Clone and verify test branch contents
test_dir = os.path.join(clone_dir, "test")
integration_client.clone_repository_contents(repo_name, test_dir, branch="test-branch")

with open(os.path.join(test_dir, "test.txt"), "r") as f:
test_content = f.read()
assert test_content == "test branch content"

0 comments on commit b6c254d

Please sign in to comment.