fix(codebuild): add apply-and-watch target to Makefile; enhance build…

…spec and tf script for improved error handling and plugin cache management

codebuild/Makefile

@@ -38,9 +38,15 @@ SOURCE_VERSION ?= main
 ## The secret value must be a JSON object with a "token" key.
 SECRET_NAME ?= ghe-runner/github-token
 
-## ARN of the Secrets Manager secret.  Populated automatically by `make secret`
-## or can be supplied directly:  make apply GITHUB_TOKEN_SECRET_ARN=arn:...
-GITHUB_TOKEN_SECRET_ARN ?=
+## Seconds to wait after triggering a build before streaming logs.
+SLEEP_SECS ?= 30
+
+## ARN of the Secrets Manager secret.  Auto-resolved from SECRET_NAME if not set.
+## Can be supplied directly:  make apply GITHUB_TOKEN_SECRET_ARN=arn:...
+GITHUB_TOKEN_SECRET_ARN ?= $(shell aws secretsmanager describe-secret \
+    --secret-id "$(SECRET_NAME)" \
+    --region $(AWS_REGION) \
+    --query 'ARN' --output text 2>/dev/null)
 
 # ── Internal variables ────────────────────────────────────────────────────────
 
@@ -85,7 +91,7 @@ RED   := $(shell tput setaf 1 2>/dev/null)
 # ── Phony declarations ────────────────────────────────────────────────────────
 
 .PHONY: help init plan apply destroy validate fmt \
-        setup-credentials secret show-outputs trigger logs clean check-env \
+        setup-credentials secret show-outputs trigger logs apply-and-watch clean check-env \
         force-unlock break-lock sync-gist
 
 # ── Default target ────────────────────────────────────────────────────────────
@@ -276,6 +282,12 @@ logs: check-env
 	    --region $(AWS_REGION); \
 	fi
 
+## apply-and-watch: Apply infrastructure, trigger a build, wait, then stream logs
+apply-and-watch: apply trigger
+	@echo "$(BOLD)Sleeping $(SLEEP_SECS)s before streaming logs...$(RESET)"
+	@sleep $(SLEEP_SECS)
+	@$(MAKE) logs
+
 # ── State lock management ────────────────────────────────────────────────────
 
 ## force-unlock: Release a state lock by ID (use when lock ID is shown in error output)

codebuild/buildspec.yml

@@ -40,19 +40,29 @@ phases:
       - tf --version
       - pip3 install --quiet requests
       - echo "Warming provider cache from S3..."
-      - mkdir -p "$TF_PLUGIN_CACHE_DIR"
+      - mkdir -p "$TF_PLUGIN_CACHE_DIR" /root/.terraform.d/plugin-cache
       - aws s3 sync "${PROVIDER_CACHE_S3}/" "$TF_PLUGIN_CACHE_DIR/" --region "$PROVIDER_CACHE_S3_REGION" --quiet || true
 
   pre_build:
     on-failure: ABORT
     commands:
       - |
+        set -euo pipefail
+
         TF_WORKSPACE="${TF_WORKSPACE:-default}"
         export TF_WORKSPACE
         echo "TF_WORKSPACE=${TF_WORKSPACE}"
-        if [ -z "$TF_WORKSPACE" ]; then echo "ERROR - TF_WORKSPACE is empty"; exit 1; fi
-      - export TF_VAR_github_token="$GITHUB_TOKEN"
-      - |
+
+        # Ensure plugin cache dir exists (covers both env var and any ~/.terraformrc path)
+        mkdir -p "${TF_PLUGIN_CACHE_DIR:-/root/.terraform.d/plugin-cache}" /root/.terraform.d/plugin-cache
+
+        # Pin TF_DATA_DIR so every tf call in this build uses the same initialized state
+        export TF_DATA_DIR="$(pwd)/terraform_data_dirs/${TF_WORKSPACE}"
+        mkdir -p "$TF_DATA_DIR"
+        echo "TF_DATA_DIR=${TF_DATA_DIR}"
+
+        export TF_VAR_github_token="$GITHUB_TOKEN"
+
         BACKEND_CONFIG="backend-configs/${TF_WORKSPACE}.tf"
         if [ -f "$BACKEND_CONFIG" ]; then
           echo "Initializing with backend-config: $BACKEND_CONFIG"
@@ -61,15 +71,23 @@ phases:
           echo "No workspace-specific backend config found; using backend.tf"
           tf init -input=false
         fi
-      - tf workspace select "$TF_WORKSPACE"
-      - echo "Active workspace $(tf workspace show)"
+
+        tf workspace select "$TF_WORKSPACE"
+        echo "Active workspace: $(tf workspace show)"
 
   build:
     on-failure: ABORT
     commands:
-      - echo "Running tf apply for workspace ${TF_WORKSPACE}"
-      - tf apply -auto-approve -input=false
       - |
+        set -euo pipefail
+        TF_WORKSPACE="${TF_WORKSPACE:-default}"
+        export TF_DATA_DIR="$(pwd)/terraform_data_dirs/${TF_WORKSPACE}"
+        export TF_VAR_github_token="$GITHUB_TOKEN"
+        echo "Running tf apply for workspace ${TF_WORKSPACE} (TF_DATA_DIR=${TF_DATA_DIR})"
+        tf apply -auto-approve -input=false
+      - |
+        set -euo pipefail
+        export TF_DATA_DIR="$(pwd)/terraform_data_dirs/${TF_WORKSPACE:-default}"
         LAMBDA_ARN=$(tf output -raw lambda_token_refresh_arn 2>/dev/null || true)
         AWS_REGION=$(aws configure get region || echo "us-gov-west-1")
         if [ -n "$LAMBDA_ARN" ]; then
@@ -85,6 +103,8 @@ phases:
           echo "Lambda not deployed or output not found; skipping token refresh."
         fi
       - |
+        set -euo pipefail
+        export TF_DATA_DIR="$(pwd)/terraform_data_dirs/${TF_WORKSPACE:-default}"
         AWS_REGION=$(aws configure get region || echo "us-gov-west-1")
         CLUSTER=$(tf output -raw ecs_cluster_name 2>/dev/null)
         SERVICE=$(tf output -raw github_runner_service_name 2>/dev/null)
@@ -106,4 +126,5 @@ phases:
       - |
         echo "Build completed at $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
       - |
+        export TF_DATA_DIR="$(pwd)/terraform_data_dirs/${TF_WORKSPACE:-default}"
         echo "Workspace: $(tf workspace show 2>/dev/null || echo unknown)"

codebuild/tf

@@ -235,11 +235,17 @@ def run_terraform(args):
         except (json.JSONDecodeError, Exception):
             pass
 
-    # Set plugin cache directory if specified in .terraformrc
+    # Set plugin cache directory if specified in .terraformrc (overrides env var)
     plugin_cache_dir = get_plugin_cache_dir()
     if plugin_cache_dir:
         env['TF_PLUGIN_CACHE_DIR'] = plugin_cache_dir
-
+
+    # Ensure the plugin cache dir exists so terraform doesn't abort with
+    # "cannot be opened: no such file or directory"
+    cache_dir = env.get('TF_PLUGIN_CACHE_DIR')
+    if cache_dir:
+        os.makedirs(cache_dir, exist_ok=True)
+
     # Handle workspace commands
     if args and args[0] == 'workspace' and len(args) > 1:
         if args[1] == 'new':