From 58f634b731183c501dc34969021e29846511adc5 Mon Sep 17 00:00:00 2001 From: Dave Arnold Date: Mon, 20 Apr 2026 15:26:39 -0400 Subject: [PATCH] feat: rename template placeholder dirs via GitHub API after repo creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add scripts/rename_template_dirs.py (Python, httpx + rich) that calls the GitHub API to delete environment/region/vpc/cluster/ placeholder paths from the repo-init PR branch and re-add the eks-*/terragrunt.hcl files at their correct computed paths: environment/region/vpc/cluster/eks-*/terragrunt.hcl → ${environment}/${region}/${vpc_name}/${cluster_name}/eks-*/terragrunt.hcl Files already rendered by managed_extra_files (account.hcl, region.hcl, vpc.hcl, cluster.hcl) are deleted from the placeholder paths but not re-added — Terraform already wrote them with real values. Controlled by var.run_in_codebuild (default false). buildspec.yml sets TF_VAR_run_in_codebuild=true so the null_resource only runs in CodeBuild. Also adds the null provider to providers.tf and pip3 install of httpx+rich to the buildspec install phase. --- buildspec.yml | 4 + main.tf | 32 ++++ providers.tf | 4 + scripts/rename_template_dirs.py | 267 ++++++++++++++++++++++++++++++++ variables.tf | 6 + 5 files changed, 313 insertions(+) create mode 100644 scripts/rename_template_dirs.py diff --git a/buildspec.yml b/buildspec.yml index 360ffa5..d37f165 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -25,6 +25,7 @@ env: REPO_BRANCH: "fix/eca-copilot-instructions-and-callnotes" # update to main once merged # Disable TLS verification for Census GHE (Census CA cert not trusted by default) GIT_SSL_NO_VERIFY: "true" + TF_VAR_run_in_codebuild: "true" TF_CLI_ARGS: "-no-color" # Census proxy — required for registry.terraform.io provider downloads HTTPS_PROXY: "http://proxy.tco.census.gov:3128" @@ -58,6 +59,9 @@ phases: fi - terraform version + # ── Install Python dependencies for post-apply scripts ─────────────── + - pip3 install --quiet httpx rich + # ── Clone terraform-eks-deployment ─────────────────────────────────── - | git config --global credential.helper \ diff --git a/main.tf b/main.tf index 1670613..502d8a3 100644 --- a/main.tf +++ b/main.tf @@ -108,6 +108,38 @@ module "github_repo" { ] } +# Rename placeholder environment/ dirs to computed paths via GitHub API. +# Only runs in CodeBuild (var.run_in_codebuild = true, set by TF_VAR_run_in_codebuild in buildspec). +resource "null_resource" "rename_template_dirs" { + count = var.run_in_codebuild ? 1 : 0 + + triggers = { + repo_name = var.name + environment = var.environment + region = var.region + vpc_name = var.cluster_config.vpc_name + cluster_name = var.name + } + + provisioner "local-exec" { + interpreter = ["python3"] + command = "${path.module}/scripts/rename_template_dirs.py" + environment = { + GHE_BASE_URL = var.github_server_url + REPO_ORG = var.organization + REPO_NAME = var.name + ENVIRONMENT = var.environment + REGION = var.region + VPC_NAME = var.cluster_config.vpc_name + CLUSTER_NAME = var.name + PR_BRANCH = "repo-init" + # GITHUB_TOKEN is already set in the CodeBuild environment by the Lambda + } + } + + depends_on = [module.github_repo] +} + # The EKS deployment logic will go here, and will be skipped if create_repository is true. output "repository_url" { diff --git a/providers.tf b/providers.tf index 4e4215b..3be4e24 100644 --- a/providers.tf +++ b/providers.tf @@ -8,6 +8,10 @@ terraform { source = "hashicorp/aws" version = ">= 5.0" } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } } } diff --git a/scripts/rename_template_dirs.py b/scripts/rename_template_dirs.py new file mode 100644 index 0000000..d5a6128 --- /dev/null +++ b/scripts/rename_template_dirs.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +""" +rename_template_dirs.py + +After CSVD/terraform-github-repo clones template-eks-cluster and writes +managed_extra_files, the repo contains placeholder paths from the template: + + environment/region/vpc/cluster/eks-*/terragrunt.hcl + environment/region/vpc/cluster/eks/terragrunt.hcl + environment/account.hcl + environment/region/region.hcl + environment/region/vpc/vpc.hcl + environment/region/vpc/cluster/cluster.hcl + +This script uses the GitHub API to: + 1. Delete all files under environment/ from the repo-init PR branch. + 2. Re-add the eks-* files at their correct computed paths: + ${ENVIRONMENT}/${REGION}/${VPC_NAME}/${CLUSTER_NAME}/eks-*/terragrunt.hcl + +The non-eks files (account.hcl, region.hcl, vpc.hcl, cluster.hcl) are already +written by managed_extra_files with real rendered content, so they are not +re-added here. + +Required environment variables: + GITHUB_TOKEN — GitHub PAT (already set by Lambda / buildspec) + GHE_BASE_URL — e.g. https://github.e.it.census.gov + REPO_ORG — GitHub org (e.g. SCT-Engineering) + REPO_NAME — Repository name (e.g. my-eks-cluster) + ENVIRONMENT — e.g. dev + REGION — e.g. us-gov-west-1 + VPC_NAME — e.g. my-vpc + CLUSTER_NAME — e.g. my-eks-cluster + PR_BRANCH — Branch to modify (default: repo-init) +""" + +import os +import sys + +import httpx +from rich.console import Console +from rich.panel import Panel + +console = Console() + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +REQUIRED_VARS = [ + "GITHUB_TOKEN", + "GHE_BASE_URL", + "REPO_ORG", + "REPO_NAME", + "ENVIRONMENT", + "REGION", + "VPC_NAME", + "CLUSTER_NAME", +] + +# Template placeholder prefix that the CSVD module clones verbatim +TEMPLATE_ENV_PREFIX = "environment/region/vpc/cluster/" + +# These files under the cluster dir are rendered by managed_extra_files with +# real values — do NOT re-add them (just delete the placeholder versions). +MANAGED_BY_TERRAFORM = {"cluster.hcl"} + + +def load_env() -> dict: + missing = [v for v in REQUIRED_VARS if not os.environ.get(v)] + if missing: + console.print(f"[bold red]Missing required env vars: {', '.join(missing)}[/]") + sys.exit(1) + + return { + "token": os.environ["GITHUB_TOKEN"], + "base_url": os.environ["GHE_BASE_URL"].rstrip("/"), + "org": os.environ["REPO_ORG"], + "repo": os.environ["REPO_NAME"], + "environment": os.environ["ENVIRONMENT"], + "region": os.environ["REGION"], + "vpc_name": os.environ["VPC_NAME"], + "cluster_name": os.environ["CLUSTER_NAME"], + "pr_branch": os.environ.get("PR_BRANCH", "repo-init"), + } + + +def api_client(cfg: dict) -> httpx.Client: + """Return an httpx client configured for the Census GHE API.""" + return httpx.Client( + base_url=f"{cfg['base_url']}/api/v3", + headers={ + "Authorization": f"token {cfg['token']}", + "Accept": "application/vnd.github.v3+json", + }, + verify=False, # Census CA cert not in CodeBuild trust store + timeout=30, + ) + + +def get_branch_commit(client: httpx.Client, org: str, repo: str, branch: str) -> tuple[str, str]: + """Return (commit_sha, tree_sha) for the tip of branch.""" + r = client.get(f"/repos/{org}/{repo}/git/ref/heads/{branch}") + r.raise_for_status() + commit_sha = r.json()["object"]["sha"] + + r = client.get(f"/repos/{org}/{repo}/git/commits/{commit_sha}") + r.raise_for_status() + tree_sha = r.json()["tree"]["sha"] + + return commit_sha, tree_sha + + +def get_tree(client: httpx.Client, org: str, repo: str, tree_sha: str) -> list[dict]: + """Return the full recursive tree as a list of entry dicts.""" + r = client.get( + f"/repos/{org}/{repo}/git/trees/{tree_sha}", + params={"recursive": "1"}, + ) + r.raise_for_status() + data = r.json() + if data.get("truncated"): + console.print("[yellow]Warning: tree is truncated — repo may have too many files.[/]") + return data["tree"] + + +def build_new_tree(entries: list[dict], cfg: dict) -> list[dict]: + """ + Return the list of tree update objects to pass to the GitHub Create Tree API. + + Strategy (using base_tree, so we only need to express changes): + - For every file under environment/: set sha=null (delete) + - For every file under environment/region/vpc/cluster/ that starts with eks: + also add it at the correct computed path (preserve sha) + """ + env = cfg["environment"] + region = cfg["region"] + vpc = cfg["vpc_name"] + cluster = cfg["cluster_name"] + correct_prefix = f"{env}/{region}/{vpc}/{cluster}/" + + updates: list[dict] = [] + moved: list[str] = [] + deleted: list[str] = [] + skipped: list[str] = [] + + for entry in entries: + path: str = entry["path"] + if entry["type"] != "blob": + continue + + if not path.startswith("environment/"): + continue + + # Delete the placeholder file + updates.append({"path": path, "mode": entry["mode"], "type": "blob", "sha": None}) + deleted.append(path) + + # Is it under environment/region/vpc/cluster/? + if not path.startswith(TEMPLATE_ENV_PREFIX): + continue + + rel = path[len(TEMPLATE_ENV_PREFIX):] # e.g. "eks-config/terragrunt.hcl" + top_dir = rel.split("/")[0] # e.g. "eks-config" + + if top_dir in MANAGED_BY_TERRAFORM or rel in MANAGED_BY_TERRAFORM: + skipped.append(path) + continue + + # Move it: eks-* and eks/ dirs only + if top_dir.startswith("eks"): + new_path = correct_prefix + rel + updates.append({"path": new_path, "mode": entry["mode"], "type": "blob", "sha": entry["sha"]}) + moved.append(f"{path} → {new_path}") + + return updates, moved, deleted, skipped + + +def create_tree(client: httpx.Client, org: str, repo: str, base_tree_sha: str, updates: list[dict]) -> str: + """POST a new tree and return its SHA.""" + r = client.post( + f"/repos/{org}/{repo}/git/trees", + json={"base_tree": base_tree_sha, "tree": updates}, + ) + r.raise_for_status() + return r.json()["sha"] + + +def create_commit(client: httpx.Client, org: str, repo: str, parent_sha: str, tree_sha: str, message: str) -> str: + """POST a new commit and return its SHA.""" + r = client.post( + f"/repos/{org}/{repo}/git/commits", + json={"message": message, "tree": tree_sha, "parents": [parent_sha]}, + ) + r.raise_for_status() + return r.json()["sha"] + + +def update_ref(client: httpx.Client, org: str, repo: str, branch: str, commit_sha: str) -> None: + """Force-update the branch ref to point to commit_sha.""" + r = client.patch( + f"/repos/{org}/{repo}/git/refs/heads/{branch}", + json={"sha": commit_sha, "force": True}, + ) + r.raise_for_status() + + +def main() -> None: + cfg = load_env() + + console.print(Panel( + f"[bold]rename_template_dirs[/]\n" + f"Repo: [cyan]{cfg['org']}/{cfg['repo']}[/]\n" + f"Branch: [cyan]{cfg['pr_branch']}[/]\n" + f"Target: [cyan]{cfg['environment']}/{cfg['region']}/{cfg['vpc_name']}/{cfg['cluster_name']}/[/]", + title="EKS Template Dir Rename", + )) + + with api_client(cfg) as client: + org, repo, branch = cfg["org"], cfg["repo"], cfg["pr_branch"] + + console.print(f"[dim]Fetching branch tip for [bold]{branch}[/]…") + commit_sha, tree_sha = get_branch_commit(client, org, repo, branch) + console.print(f"[dim]Commit: {commit_sha} Tree: {tree_sha}") + + console.print("[dim]Fetching recursive tree…") + entries = get_tree(client, org, repo, tree_sha) + + env_files = [e for e in entries if e["type"] == "blob" and e["path"].startswith("environment/")] + if not env_files: + console.print("[green]No placeholder environment/ files found — nothing to do.[/]") + return + + updates, moved, deleted, skipped = build_new_tree(entries, cfg) + + console.print(f"\n[bold]Changes to apply:[/]") + for m in moved: + console.print(f" [green]MOVE[/] {m}") + for d in deleted: + if d not in [m.split(" → ")[0] for m in moved]: + console.print(f" [red]DELETE[/] {d} (managed by Terraform)") + console.print() + + if not updates: + console.print("[yellow]No changes needed.[/]") + return + + console.print("[dim]Creating new tree…") + new_tree_sha = create_tree(client, org, repo, tree_sha, updates) + + message = ( + f"chore: rename template placeholder dirs to computed paths\n\n" + f"environment/region/vpc/cluster/ → " + f"{cfg['environment']}/{cfg['region']}/{cfg['vpc_name']}/{cfg['cluster_name']}/\n\n" + f"Moved {len(moved)} eks-module file(s). " + f"Deleted {len(deleted) - len(moved)} file(s) already handled by managed_extra_files." + ) + console.print("[dim]Creating commit…") + new_commit_sha = create_commit(client, org, repo, commit_sha, new_tree_sha, message) + + console.print(f"[dim]Updating [bold]{branch}[/] → {new_commit_sha}…") + update_ref(client, org, repo, branch, new_commit_sha) + + console.print(f"\n[bold green]Done.[/] {len(moved)} file(s) moved, {len(skipped)} skipped (managed by Terraform).") + + +if __name__ == "__main__": + main() diff --git a/variables.tf b/variables.tf index 8c7859a..a22e013 100644 --- a/variables.tf +++ b/variables.tf @@ -68,6 +68,12 @@ variable "force_name" { default = true } +variable "run_in_codebuild" { + description = "Set to true when running inside CodeBuild. Enables the post-apply script that renames placeholder template dirs to computed paths via the GitHub API." + type = bool + default = false +} + # Internal variables - these are kept for backward compatibility but should not be exposed to users in examples variable "common_variables" { description = "Common variables across all environments (internal use)"