feat(codebuild): add CodeBuild cronjob for daily tf apply

- Add codebuild/ Terraform module that deploys a scheduled CodeBuild project (ghe-runner-daily-{workspace}) running tf apply daily - Download tf wrapper script from team gist during install phase so TF_DATA_DIR, var-file injection and JSON env loading are handled automatically without manual replication in the buildspec - Invoke Lambda token refresh before ECS force-redeploy so containers always start with a fresh registration token - Add codebuild/backend-configs/ with distinct state keys (ghe-runner-codebuild) to avoid collision with parent state - Auto-inherit TF_WORKSPACE from parent directory via make - Fix buildspec_path bug in cronjob module (was always resolving to null) - Set insecure_ssl=true to handle internal GHE CA in GovCloud - Update README Automation section to reflect gist-download approach - Update README CodeBuild phase table with correct execution order
CSVD · Mar 17, 2026 · d92c36d · d92c36d
1 parent f3e80d9
commit d92c36d
Show file tree

Hide file tree

Showing 24 changed files with 2,321 additions and 1,031 deletions.
diff --git a/README.md b/README.md
diff --git a/codebuild/Makefile b/codebuild/Makefile
@@ -0,0 +1,329 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# Makefile — ghe-runner CodeBuild cronjob
+#
+# Usage:
+#   make <target> [TF_WORKSPACE=<workspace>] [GITHUB_TOKEN_SECRET_ARN=<arn>]
+#
+# TF_WORKSPACE selects which ghe-runner workspace the CodeBuild job will
+# deploy (maps to varfiles/{workspace}.tfvars in the repo root).  It also
+# drives backend-config selection for this Terraform configuration.
+#
+# Prerequisites:
+#   - AWS credentials exported in the current shell (or via awscreds)
+#   - GITHUB_TOKEN environment variable set to a valid PAT
+#   - terraform >= 1.9 on PATH  (or ~/git/tfenv/bin/terraform)
+# ─────────────────────────────────────────────────────────────────────────────
+
+# ── Configurable variables ────────────────────────────────────────────────────
+
+## Which ghe-runner workspace the CodeBuild job will target.
+## Also used to select the backend-config for this Terraform workspace.
+## Defaults to the active workspace of the parent ghe-runner directory so you
+## don't have to set it manually — just run `tf workspace select <ws>` in the
+## repo root and make will pick it up automatically.
+## Override with: make <target> TF_WORKSPACE=csvd
+TF_WORKSPACE ?= $(shell cd .. && tf workspace show 2>/dev/null || echo default)
+
+## AWS region for all operations.
+AWS_REGION ?= us-gov-west-1
+
+## HTTPS clone URL of the ghe-runner repo.  Passed to Terraform as a variable
+## so CodeBuild knows where to check out from.
+SOURCE_LOCATION ?= https://github.e.it.census.gov/CSVD/ghe-runners.git
+
+## Branch/tag CodeBuild will check out.
+SOURCE_VERSION ?= main
+
+## Name of the Secrets Manager secret that holds the GitHub PAT.
+## The secret value must be a JSON object with a "token" key.
+SECRET_NAME ?= ghe-runner/github-token
+
+## ARN of the Secrets Manager secret.  Populated automatically by `make secret`
+## or can be supplied directly:  make apply GITHUB_TOKEN_SECRET_ARN=arn:...
+GITHUB_TOKEN_SECRET_ARN ?=
+
+# ── Internal variables ────────────────────────────────────────────────────────
+
+REPO_ROOT      := $(shell git -C .. rev-parse --show-toplevel 2>/dev/null || echo ..)
+# Use backend-configs/ local to codebuild/ — these have a distinct state key
+# (ghe-runner-codebuild) so codebuild state never collides with the parent
+# ghe-runner state stored under the same bucket.
+BACKEND_CONFIGS := backend-configs
+# Mirror the resolution order used by ~/bin/tf:
+#   1. TERRAFORM_BINARY env var (explicit override)
+#   2. ~/git/tfenv/bin/terraform (project standard)
+#   3. first `terraform` on PATH (last resort)
+TF := $(or \
+  $(and $(TERRAFORM_BINARY),$(wildcard $(TERRAFORM_BINARY)),$(TERRAFORM_BINARY)),\
+  $(wildcard $(HOME)/git/tfenv/bin/terraform),\
+  $(shell command -v terraform 2>/dev/null))
+TF_DATA_DIR    := $(REPO_ROOT)/terraform_data_dirs/codebuild-$(TF_WORKSPACE)
+
+# Pick a backend-config file: prefer workspace-specific, fall back to csvd-dev-ew
+_BACKEND_FILE  := $(BACKEND_CONFIGS)/$(TF_WORKSPACE).tf
+BACKEND_CONFIG := $(shell [ -f "$(_BACKEND_FILE)" ] && echo "$(_BACKEND_FILE)" || echo "$(BACKEND_CONFIGS)/csvd-dev-ew.tf")
+
+# Build common terraform var flags
+_TF_VARS := \
+	-var="tf_workspace=$(TF_WORKSPACE)" \
+	-var="aws_region=$(AWS_REGION)" \
+	-var="source_location=$(SOURCE_LOCATION)" \
+	-var="source_version=$(SOURCE_VERSION)"
+
+# Append secret ARN only when provided — avoids a blank string reaching terraform
+ifneq ($(GITHUB_TOKEN_SECRET_ARN),)
+  _TF_VARS += -var="github_token_secret_arn=$(GITHUB_TOKEN_SECRET_ARN)"
+endif
+
+# Colorized output helpers (degrade gracefully if no tty)
+BOLD  := $(shell tput bold 2>/dev/null)
+RESET := $(shell tput sgr0 2>/dev/null)
+CYAN  := $(shell tput setaf 6 2>/dev/null)
+YELLOW := $(shell tput setaf 3 2>/dev/null)
+RED   := $(shell tput setaf 1 2>/dev/null)
+
+# ── Phony declarations ────────────────────────────────────────────────────────
+
+.PHONY: help init plan apply destroy validate fmt \
+        setup-credentials secret show-outputs trigger logs clean check-env \
+        force-unlock break-lock
+
+# ── Default target ────────────────────────────────────────────────────────────
+
+.DEFAULT_GOAL := help
+
+## help: Show this help message
+help:
+	@echo ""
+	@echo "$(BOLD)$(CYAN)ghe-runner CodeBuild cronjob$(RESET)"
+	@echo ""
+	@echo "$(BOLD)Usage:$(RESET)"
+	@echo "  make <target> [TF_WORKSPACE=<ws>] [GITHUB_TOKEN_SECRET_ARN=<arn>]"
+	@echo ""
+	@echo "$(BOLD)Current settings:$(RESET)"
+	@echo "  TF_WORKSPACE           = $(TF_WORKSPACE)"
+	@echo "  AWS_REGION             = $(AWS_REGION)"
+	@echo "  SECRET_NAME            = $(SECRET_NAME)"
+	@echo "  GITHUB_TOKEN_SECRET_ARN= $(if $(GITHUB_TOKEN_SECRET_ARN),$(GITHUB_TOKEN_SECRET_ARN),$(YELLOW)(not set — run: make secret)$(RESET))"
+	@echo "  BACKEND_CONFIG         = $(BACKEND_CONFIG)"
+	@echo "  TF_DATA_DIR            = $(TF_DATA_DIR)"
+	@echo ""
+	@echo "$(BOLD)Targets:$(RESET)"
+	@grep -E '^## [a-zA-Z_-]+:' $(MAKEFILE_LIST) \
+	  | sed 's/## /  /' \
+	  | awk -F: '{printf "  $(CYAN)%-24s$(RESET) %s\n", $$1, $$2}'
+	@echo ""
+	@echo "$(BOLD)Recommended first-time flow:$(RESET)"
+	@echo "  1. make setup-credentials  # register GHE PAT with CodeBuild (once per region)"
+	@echo "  2. make secret             # store PAT in Secrets Manager"
+	@echo "  3. make init               # initialize Terraform"
+	@echo "  4. make plan               # preview changes"
+	@echo "  5. make apply              # deploy"
+	@echo "  6. make trigger            # run a manual build to verify"
+	@echo ""
+
+# ── Environment guard ─────────────────────────────────────────────────────────
+
+## check-env: Verify required tools and environment variables are present
+check-env:
+	@echo "$(BOLD)Checking environment...$(RESET)"
+	@command -v aws  >/dev/null 2>&1 || (echo "$(RED)ERROR: aws CLI not found$(RESET)"; exit 1)
+	@$(TF) version   >/dev/null 2>&1 || (echo "$(RED)ERROR: terraform not found at $(TF)$(RESET)"; exit 1)
+	@[ -n "$(GITHUB_TOKEN)" ] || \
+	  (echo "$(RED)ERROR: GITHUB_TOKEN is not set. Export your GitHub PAT before running.$(RESET)"; exit 1)
+	@aws sts get-caller-identity --region $(AWS_REGION) >/dev/null 2>&1 || \
+	  (echo "$(RED)ERROR: AWS credentials are not valid or have expired. Run: awscreds$(RESET)"; exit 1)
+	@echo "  terraform : $$($(TF) version 2>&1 | head -1)"
+	@echo "  aws cli   : $$(aws --version 2>&1 | awk '{print $$1}')"
+	@echo "  identity  : $$(aws sts get-caller-identity --region $(AWS_REGION) --query 'Arn' --output text)"
+	@echo "$(BOLD)$(CYAN)OK$(RESET)"
+
+# ── One-time setup ────────────────────────────────────────────────────────────
+
+## setup-credentials: Register the GitHub PAT with CodeBuild (once per region)
+setup-credentials: check-env
+	@echo "$(BOLD)Registering GHE credentials with CodeBuild in $(AWS_REGION)...$(RESET)"
+	@[ -n "$(GITHUB_TOKEN)" ] || (echo "$(RED)ERROR: GITHUB_TOKEN not set$(RESET)"; exit 1)
+	aws codebuild import-source-credentials \
+	  --server-type GITHUB_ENTERPRISE \
+	  --auth-type PERSONAL_ACCESS_TOKEN \
+	  --token "$(GITHUB_TOKEN)" \
+	  --region $(AWS_REGION)
+	@echo "$(BOLD)$(CYAN)Credentials registered.$(RESET)"
+
+## secret: Create or update the GitHub PAT secret in Secrets Manager
+secret: check-env
+	@[ -n "$(GITHUB_TOKEN)" ] || (echo "$(RED)ERROR: GITHUB_TOKEN not set$(RESET)"; exit 1)
+	@echo "$(BOLD)Writing secret: $(SECRET_NAME)$(RESET)"
+	@EXISTING=$$(aws secretsmanager describe-secret \
+	    --secret-id "$(SECRET_NAME)" \
+	    --region $(AWS_REGION) \
+	    --query 'ARN' --output text 2>/dev/null); \
+	if [ -n "$$EXISTING" ]; then \
+	  echo "  Secret exists — updating value"; \
+	  aws secretsmanager put-secret-value \
+	    --secret-id "$(SECRET_NAME)" \
+	    --secret-string "{\"token\":\"$(GITHUB_TOKEN)\"}" \
+	    --region $(AWS_REGION); \
+	  echo "  ARN: $$EXISTING"; \
+	else \
+	  echo "  Secret does not exist — creating"; \
+	  ARN=$$(aws secretsmanager create-secret \
+	    --name "$(SECRET_NAME)" \
+	    --description "GitHub PAT for ghe-runner CodeBuild job" \
+	    --secret-string "{\"token\":\"$(GITHUB_TOKEN)\"}" \
+	    --region $(AWS_REGION) \
+	    --query 'ARN' --output text); \
+	  echo "  ARN: $$ARN"; \
+	fi
+	@echo ""
+	@echo "$(BOLD)$(YELLOW)Set this ARN before running make plan/apply:$(RESET)"
+	@echo "  export GITHUB_TOKEN_SECRET_ARN=$$(aws secretsmanager describe-secret \
+	    --secret-id '$(SECRET_NAME)' \
+	    --region $(AWS_REGION) \
+	    --query 'ARN' --output text)"
+	@echo ""
+
+# ── Terraform lifecycle ───────────────────────────────────────────────────────
+
+## init: Initialize Terraform with the workspace-appropriate backend config
+init: check-env
+	@echo "$(BOLD)Initializing Terraform...$(RESET)"
+	@echo "  workspace     : $(TF_WORKSPACE)"
+	@echo "  backend-config: $(BACKEND_CONFIG)"
+	@mkdir -p $(TF_DATA_DIR)
+	TF_DATA_DIR=$(TF_DATA_DIR) \
+	  $(TF) init \
+	    -input=false \
+	    -backend-config=$(BACKEND_CONFIG)
+
+## validate: Validate Terraform configuration (no backend needed)
+validate:
+	@echo "$(BOLD)Validating...$(RESET)"
+	TF_DATA_DIR=$(TF_DATA_DIR) $(TF) validate
+
+## fmt: Format all Terraform files in this directory
+fmt:
+	$(TF) fmt -recursive .
+
+## plan: Show what Terraform would change
+plan: check-env _require-secret-arn
+	@echo "$(BOLD)Planning for workspace: $(TF_WORKSPACE)$(RESET)"
+	TF_DATA_DIR=$(TF_DATA_DIR) \
+	  $(TF) plan \
+	    -input=false \
+	    $(_TF_VARS)
+
+## apply: Deploy or update the CodeBuild cronjob infrastructure
+apply: check-env _require-secret-arn
+	@echo "$(BOLD)Applying for workspace: $(TF_WORKSPACE)$(RESET)"
+	TF_DATA_DIR=$(TF_DATA_DIR) \
+	  $(TF) apply \
+	    -input=false \
+	    -auto-approve \
+	    $(_TF_VARS)
+
+## destroy: Tear down the CodeBuild infrastructure for this workspace
+destroy: check-env _require-secret-arn
+	@echo "$(BOLD)$(RED)Destroying CodeBuild infrastructure for workspace: $(TF_WORKSPACE)$(RESET)"
+	@echo "$(YELLOW)Press Ctrl-C within 5 seconds to abort...$(RESET)"
+	@sleep 5
+	TF_DATA_DIR=$(TF_DATA_DIR) \
+	  $(TF) destroy \
+	    -input=false \
+	    $(_TF_VARS)
+
+## show-outputs: Print Terraform outputs for the deployed project
+show-outputs:
+	@TF_DATA_DIR=$(TF_DATA_DIR) $(TF) output
+
+# ── CodeBuild operations ──────────────────────────────────────────────────────
+
+## trigger: Manually start a build outside the daily schedule
+trigger: check-env
+	$(eval PROJECT := $(shell TF_DATA_DIR=$(TF_DATA_DIR) $(TF) output -raw codebuild_project_name 2>/dev/null))
+	@[ -n "$(PROJECT)" ] || (echo "$(RED)ERROR: Could not read codebuild_project_name from state. Run make apply first.$(RESET)"; exit 1)
+	@echo "$(BOLD)Starting build: $(PROJECT)$(RESET)"
+	$(eval BUILD_ID := $(shell aws codebuild start-build \
+	    --project-name $(PROJECT) \
+	    --region $(AWS_REGION) \
+	    --query 'build.id' --output text))
+	@echo "  Build ID : $(BUILD_ID)"
+	@echo "  Console  : https://console.amazonaws-us-gov.com/codesuite/codebuild/projects/$(PROJECT)/build/$(BUILD_ID)/log"
+	@echo ""
+	@echo "Run $(BOLD)make logs BUILD_ID=$(BUILD_ID)$(RESET) to stream the output."
+
+## logs: Stream logs for the most recent build (or specify BUILD_ID=...)
+logs: check-env
+	$(eval PROJECT := $(shell TF_DATA_DIR=$(TF_DATA_DIR) $(TF) output -raw codebuild_project_name 2>/dev/null))
+	@[ -n "$(PROJECT)" ] || (echo "$(RED)ERROR: Could not read codebuild_project_name from state.$(RESET)"; exit 1)
+	@if [ -z "$(BUILD_ID)" ]; then \
+	  echo "$(BOLD)Fetching most recent build for $(PROJECT)...$(RESET)"; \
+	  LATEST=$$(aws codebuild list-builds-for-project \
+	    --project-name $(PROJECT) \
+	    --region $(AWS_REGION) \
+	    --query 'ids[0]' --output text); \
+	  echo "  Build ID: $$LATEST"; \
+	  LOG_GROUP=$$(TF_DATA_DIR=$(TF_DATA_DIR) $(TF) output -raw log_group_name 2>/dev/null); \
+	  aws logs tail "$$LOG_GROUP" \
+	    --follow \
+	    --region $(AWS_REGION); \
+	else \
+	  LOG_GROUP=$$(TF_DATA_DIR=$(TF_DATA_DIR) $(TF) output -raw log_group_name 2>/dev/null); \
+	  aws logs tail "$$LOG_GROUP" \
+	    --follow \
+	    --region $(AWS_REGION); \
+	fi
+
+# ── State lock management ────────────────────────────────────────────────────
+
+## force-unlock: Release a state lock by ID (use when lock ID is shown in error output)
+force-unlock: check-env
+	@[ -n "$(LOCK_ID)" ] || { \
+	  echo "$(RED)ERROR: LOCK_ID is required.$(RESET)"; \
+	  echo "  Usage: make force-unlock LOCK_ID=<id from error output>"; \
+	  exit 1; \
+	}
+	TF_DATA_DIR=$(TF_DATA_DIR) $(TF) force-unlock -force $(LOCK_ID)
+
+## break-lock: Delete the DynamoDB lock record directly (use when lock is malformed/stuck)
+# Derives bucket, table, and region by parsing the active backend-config file.
+break-lock: check-env
+	$(eval _BC      := $(BACKEND_CONFIG))
+	$(eval _BUCKET  := $(shell awk -F'"' '/^[[:space:]]*bucket/{print $$2}' $(_BC)))
+	$(eval _TABLE   := $(shell awk -F'"' '/^[[:space:]]*dynamodb_table/{print $$2}' $(_BC)))
+	$(eval _BREGION := $(shell awk -F'"' '/^[[:space:]]*region/{print $$2}' $(_BC)))
+	$(eval _KEY     := csvd-dev-gov/common/apps/ghe-runner-codebuild)
+	$(eval _LOCK_ID := $(_BUCKET)/$(_KEY))
+	@echo "$(BOLD)Breaking state lock:$(RESET)"
+	@echo "  table  : $(_TABLE)"
+	@echo "  region : $(_BREGION)"
+	@echo "  LockID : $(_LOCK_ID)"
+	@echo "$(YELLOW)Press Ctrl-C within 5 seconds to abort...$(RESET)"
+	@sleep 5
+	aws dynamodb delete-item \
+	  --table-name "$(_TABLE)" \
+	  --key "{\"LockID\": {\"S\": \"$(_LOCK_ID)\"}}" \
+	  --region "$(_BREGION)"
+	@echo "$(BOLD)$(CYAN)Lock cleared.$(RESET)"
+
+# ── Housekeeping ──────────────────────────────────────────────────────────────
+
+## clean: Remove local Terraform cache for this workspace
+clean:
+	@echo "$(BOLD)Removing $(TF_DATA_DIR)...$(RESET)"
+	rm -rf $(TF_DATA_DIR)
+	@echo "$(BOLD)Removing .terraform/...$(RESET)"
+	rm -rf .terraform .terraform.lock.hcl
+
+# ── Internal helpers (not shown in help) ─────────────────────────────────────
+
+_require-secret-arn:
+	@[ -n "$(GITHUB_TOKEN_SECRET_ARN)" ] || { \
+	  echo "$(RED)ERROR: GITHUB_TOKEN_SECRET_ARN is required.$(RESET)"; \
+	  echo ""; \
+	  echo "  Run $(BOLD)make secret$(RESET) to create the secret, then:"; \
+	  echo "    export GITHUB_TOKEN_SECRET_ARN=<arn from above>"; \
+	  echo ""; \
+	  exit 1; \
+	}