From 089926f9bbc75767ce005dc3eb7352f1b2c268a1 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 14 May 2025 14:05:51 -0400 Subject: [PATCH] Enhance GitHubClient and add test event: normalize API URLs for GitHub interactions, improve error handling during repository creation, and introduce a new test event JSON for template settings. --- events/test-event.json | 30 +++++++++++ template_automation/app.py | 24 ++++++++- template_automation/github_client.py | 74 ++++++++++------------------ 3 files changed, 79 insertions(+), 49 deletions(-) create mode 100644 events/test-event.json diff --git a/events/test-event.json b/events/test-event.json new file mode 100644 index 0000000..a8c08ec --- /dev/null +++ b/events/test-event.json @@ -0,0 +1,30 @@ +{ + "owning_team": "platform-team", + "project_name": "example-template-repos-cluster", + "template_settings": { + "attrs": { + "account_name": "dev-account", + "aws_region": "us-gov-west-1", + "cluster_mailing_list": "eks-admins@example.com", + "cluster_name": "example-cluster-dev", + "eks_instance_disk_size": "100", + "eks_ng_desired_size": "2", + "eks_ng_max_size": "10", + "eks_ng_min_size": "2", + "environment": "development", + "environment_abbr": "dev", + "finops_project_name": "example_project", + "finops_project_number": "fp00000001", + "finops_project_role": "example_project_app", + "organization": "example:dept:team", + "vpc_domain_name": "dev.example.com", + "vpc_name": "vpc-dev" + }, + "tags": { + "managed_by": "terraform", + "owner": "platform-team", + "slim:schedule": "8:00-17:00" + } + }, + "trigger_init_workflow": true +} \ No newline at end of file diff --git a/template_automation/app.py b/template_automation/app.py index 43ac13c..f333136 100644 --- a/template_automation/app.py +++ b/template_automation/app.py @@ -134,7 +134,7 @@ def lambda_handler(event: dict, context) -> dict: # Get GitHub configuration from environment/parameter store github_config = GitHubConfig( - api_base_url=os.environ["GITHUB_API"], + api_base_url=get_github_base_url(os.environ["GITHUB_API"]), org_name=os.environ["GITHUB_ORG_NAME"], commit_author_name=os.environ.get("GITHUB_COMMIT_AUTHOR_NAME", "Template Automation"), commit_author_email=os.environ.get("GITHUB_COMMIT_AUTHOR_EMAIL", "automation@example.com"), @@ -235,3 +235,25 @@ def get_github_token() -> str: except ClientError as e: logger.error(f"Failed to get GitHub token: {str(e)}") raise + + +def get_github_base_url(api_url: str) -> str: + """Normalize GitHub API URL for GitHub Enterprise Server. + + Args: + api_url: Raw GitHub API URL from environment + + Returns: + Normalized base URL for GitHub API + """ + # Remove trailing slashes and /api/v3 if present + base_url = api_url.rstrip('/') + if base_url.endswith('/api/v3'): + base_url = base_url[:-7] + elif '/api/v3' in base_url: + # In some GitHub Enterprise setups, the URL might be like https://github.e.it.census.gov/api/v3 + # Extract just the server part + base_url = base_url.split('/api/v3')[0] + + logger.info(f"Using GitHub base URL: {base_url}") + return base_url diff --git a/template_automation/github_client.py b/template_automation/github_client.py index 8ccc526..b5e7248 100644 --- a/template_automation/github_client.py +++ b/template_automation/github_client.py @@ -167,7 +167,7 @@ def get_repository( """ try: # Try to get the repository - url = f"/repos/{self.org_name}/{repo_name}" + url = f"/api/v3/repos/{self.org_name}/{repo_name}" repo = self._request("GET", url) logger.info(f"Found existing repository: {repo_name}") @@ -180,46 +180,24 @@ def get_repository( logger.info(f"Creating repository {repo_name}") # Create a new repository with minimal parameters - url = f"/orgs/{self.org_name}/repos" + url = f"/api/v3/orgs/{self.org_name}/repos" try: # Try with minimal parameters first repo = self._request("POST", url, json={ "name": repo_name, - "private": True, - "auto_init": True + "private": True }) except requests.exceptions.HTTPError as create_error: # Safe handling of response parsing error_message = str(create_error) - try: - if create_error.response.text.strip(): - try: - error_response = create_error.response.json() - logger.error(f"GitHub API error details: {json.dumps(error_response)}") - # Check for validation errors - if "message" in error_response and "Validation Failed" in error_response.get("message", ""): - logger.info("Retrying repository creation with minimal parameters") - repo = self._request("POST", url, json={ - "name": repo_name, - "private": True - }) - else: - raise create_error - except json.JSONDecodeError: - # Handle non-JSON responses - logger.error(f"GitHub API returned non-JSON error response: {create_error.response.text}") - # Try with most minimal parameters as a fallback - logger.info("Retrying repository creation with minimal parameters due to non-JSON error") - repo = self._request("POST", url, json={ - "name": repo_name, - "private": True - }) - else: - logger.error(f"Empty error response with status code: {create_error.response.status_code}") - raise create_error - except (AttributeError, ValueError) as parse_error: - logger.error(f"Error parsing response: {str(parse_error)}") - raise create_error + logger.error(f"Failed to create repository with error: {error_message}") + + # If we got an HTML response instead of JSON (likely an error page) + if "" in error_message or " Dict[str, Any]: Returns: Branch data """ - url = f"/repos/{self.org_name}/{repo_name}/branches/{branch_name}" + url = f"/api/v3/repos/{self.org_name}/{repo_name}/branches/{branch_name}" return self._request("GET", url) def get_default_branch(self, repo_name: str) -> str: @@ -292,7 +270,7 @@ def create_branch(self, repo_name: str, branch_name: str, from_ref: str = "main" commit_sha = source_branch["commit"]["sha"] # Create the new branch - url = f"/repos/{self.org_name}/{repo_name}/git/refs" + url = f"/api/v3/repos/{self.org_name}/{repo_name}/git/refs" self._request("POST", url, json={ "ref": f"refs/heads/{branch_name}", "sha": commit_sha @@ -308,7 +286,7 @@ def create_reference(self, repo_name: str, ref: str, sha: str) -> None: ref: The name of the reference sha: The SHA1 value to set this reference to """ - url = f"/repos/{self.org_name}/{repo_name}/git/refs" + url = f"/api/v3/repos/{self.org_name}/{repo_name}/git/refs" self._request("POST", url, json={ "ref": ref, "sha": sha @@ -325,7 +303,7 @@ def update_reference(self, repo_name: str, ref: str, sha: str, force: bool = Fal sha: The SHA1 value to set this reference to force: Force update if not a fast-forward update """ - url = f"/repos/{self.org_name}/{repo_name}/git/refs/{ref}" + url = f"/api/v3/repos/{self.org_name}/{repo_name}/git/refs/{ref}" self._request("PATCH", url, json={ "sha": sha, "force": force @@ -361,7 +339,7 @@ def write_file( try: file = self.get_file_contents(repo_name, path, branch) # Update existing file - url = f"/repos/{self.org_name}/{repo_name}/contents/{path}" + url = f"/api/v3/repos/{self.org_name}/{repo_name}/contents/{path}" result = self._request("PUT", url, json={ "message": commit_message or f"Update {path}", "content": content_base64, @@ -377,7 +355,7 @@ def write_file( except requests.exceptions.HTTPError as e: if e.response.status_code == 404: # Create new file - url = f"/repos/{self.org_name}/{repo_name}/contents/{path}" + url = f"/api/v3/repos/{self.org_name}/{repo_name}/contents/{path}" result = self._request("PUT", url, json={ "message": commit_message or f"Create {path}", "content": content_base64, @@ -402,7 +380,7 @@ def get_file_contents(self, repo_name: str, path: str, ref: str = "main") -> Dic Returns: File data """ - url = f"/repos/{self.org_name}/{repo_name}/contents/{path}" + url = f"/api/v3/repos/{self.org_name}/{repo_name}/contents/{path}" params = {"ref": ref} return self._request("GET", url, params=params) @@ -442,7 +420,7 @@ def create_pull_request( Returns: The created pull request object """ - url = f"/repos/{self.org_name}/{repo_name}/pulls" + url = f"/api/v3/repos/{self.org_name}/{repo_name}/pulls" pr = self._request("POST", url, json={ "title": title, "body": body, @@ -469,7 +447,7 @@ def trigger_workflow( ref: Git reference to run the workflow on inputs: Input parameters for the workflow """ - url = f"/repos/{self.org_name}/{repo_name}/actions/workflows/{workflow_id}/dispatches" + url = f"/api/v3/repos/{self.org_name}/{repo_name}/actions/workflows/{workflow_id}/dispatches" workflow_inputs = inputs if inputs is not None else {} self._request("POST", url, json={ @@ -478,7 +456,7 @@ def trigger_workflow( }) logger.info(f"Triggered workflow {workflow_id} in {repo_name} on {ref}") - + def set_team_permission(self, repo_name: str, team_name: str, permission: str) -> None: """Set a team's permission on a repository. @@ -489,7 +467,7 @@ def set_team_permission(self, repo_name: str, team_name: str, permission: str) - """ # First check if the team exists try: - team_url = f"/orgs/{self.org_name}/teams/{team_name}" + team_url = f"/api/v3/orgs/{self.org_name}/teams/{team_name}" team = self._request("GET", team_url) logger.info(f"Found team: {team_name}") @@ -497,14 +475,14 @@ def set_team_permission(self, repo_name: str, team_name: str, permission: str) - # Different GitHub Enterprise versions might support different API paths try: # First try the standard endpoint - url = f"/orgs/{self.org_name}/teams/{team_name}/repos/{self.org_name}/{repo_name}" + url = f"/api/v3/orgs/{self.org_name}/teams/{team_name}/repos/{self.org_name}/{repo_name}" self._request("PUT", url, json={"permission": permission}) logger.info(f"Set {team_name} permission on {repo_name} to {permission}") except requests.exceptions.HTTPError as e: if e.response.status_code == 422 or e.response.status_code == 404: # Try alternative endpoint format for older GitHub Enterprise versions try: - alt_url = f"/teams/{team['id']}/repos/{self.org_name}/{repo_name}" + alt_url = f"/api/v3/teams/{team['id']}/repos/{self.org_name}/{repo_name}" self._request("PUT", alt_url, json={"permission": permission}) logger.info(f"Set {team_name} permission on {repo_name} to {permission} using alternative endpoint") except requests.exceptions.HTTPError as alt_e: @@ -528,7 +506,7 @@ def update_repository_topics(self, repo_name: str, topics: List[str]) -> None: """ # GitHub API requires a special media type for repository topics headers = {"Accept": "application/vnd.github.mercy-preview+json"} - url = f"/repos/{self.org_name}/{repo_name}/topics" + url = f"/api/v3/repos/{self.org_name}/{repo_name}/topics" self._request("PUT", url, json={"names": topics}, headers=headers) @@ -554,7 +532,7 @@ def create_repository_from_template( Returns: The newly created repository """ - url = f"/repos/{self.org_name}/{template_repo_name}/generate" + url = f"/api/v3/repos/{self.org_name}/{template_repo_name}/generate" # Create repository from template new_repo = self._request("POST", url, json={