From a16101ceaacdbbefbba8aedcbc22fcde61e0b820 Mon Sep 17 00:00:00 2001 From: Dave Arnold Date: Wed, 20 May 2026 14:03:32 -0400 Subject: [PATCH] feat: flat template repos; Proposer injects into LAYER/REGION_DIR Template repos no longer encode layer/workspace as directory nesting. LAYER and REGION_DIR are already known env vars - the Proposer uses them to determine the destination path in the account repo. buildspec-proposer.yml: - Add TEMPLATE_SOURCE_PATH env var (selects subdirectory variant within repo) - Rewrite template rendering: dst_root = LAYER/REGION_DIR/ instead of '.' - Dotfiles at template root (e.g. .sc-automation.yml) go to account repo root - Document flat layout convention in comments docs/template-management.md: - Rewrite What Belongs section: flat structure, show where files land - template-s3-bucket example is now 3 flat files (not nested infrastructure/west/) - TEMPLATE_SOURCE_PATH explained inline with multi-variant example - Remove old Subdirectory Templates section (replaced with inline example) - tf-run.data, .sc-automation.yml.j2, .terraform.lock.hcl notes updated docs/HOW-IT-WORKS.md: - BUILD phase step 3: document flat layout + dotfile root exception --- buildspec-proposer.yml | 36 +++++++++++--- docs/HOW-IT-WORKS.md | 8 ++-- docs/template-management.md | 95 +++++++++++++++++++++---------------- 3 files changed, 89 insertions(+), 50 deletions(-) diff --git a/buildspec-proposer.yml b/buildspec-proposer.yml index 624710b..28e457d 100644 --- a/buildspec-proposer.yml +++ b/buildspec-proposer.yml @@ -15,7 +15,8 @@ version: 0.2 # # Optional env-var overrides: # GIT_BRANCH - branch to commit/PR from (default: propose/sc-automation) -# TEMPLATE_REPO - GHE repo containing Jinja2/.tf template files +# TEMPLATE_REPO - GHE repo containing workload template files (flat layout) +# TEMPLATE_SOURCE_PATH - subdirectory within TEMPLATE_REPO to use as root (empty = whole repo) # TEMPLATE_VARS - JSON map of Jinja2 variables for template rendering # EXTRA_FILES - JSON map {"relative/path": "content"} written after template rendering # --------------------------------------------------------------------------- @@ -32,6 +33,7 @@ env: # Per-build defaults (overridden via environmentVariablesOverride in Lambda) GIT_BRANCH: "propose/sc-automation" TEMPLATE_REPO: "" + TEMPLATE_SOURCE_PATH: "" # subdirectory within TEMPLATE_REPO to use as root (empty = whole repo) TEMPLATE_VARS: "{}" EXTRA_FILES: "{}" @@ -74,8 +76,11 @@ phases: - git checkout -B "${GIT_BRANCH}" # --- Render template repo (if specified) --- - # Clone TEMPLATE_REPO, render .j2 files with TEMPLATE_VARS via Jinja2 StrictUndefined, - # copy non-template files as-is. Results land at the same relative paths in the account repo. + # Clone TEMPLATE_REPO; render .j2 files with TEMPLATE_VARS via Jinja2 StrictUndefined; + # copy non-template files as-is. + # Template files are FLAT (no layer/workspace nesting inside the template repo). + # They are written into ${LAYER}/${REGION_DIR}/ in the account repo, which is + # already known from the env vars supplied by the Lambda. - | if [ -n "${TEMPLATE_REPO}" ]; then git clone "https://${GITHUB_TOKEN}@github.e.it.census.gov/${GITHUB_ORG}/${TEMPLATE_REPO}.git" /tmp/template-repo @@ -84,8 +89,19 @@ phases: from jinja2 import Environment, FileSystemLoader, StrictUndefined template_vars = json.loads(os.environ.get('TEMPLATE_VARS', '{}')) + layer = os.environ['LAYER'] + region_dir = os.environ['REGION_DIR'] + src_root = pathlib.Path('/tmp/template-repo') - dst_root = pathlib.Path('.') # already inside cloned account repo + # Flat template files land at LAYER/REGION_DIR/ in the account repo. + # source_path lets a single template repo hold multiple product variants + # as subdirectories; only that subdirectory is used as the source root. + source_path = os.environ.get('TEMPLATE_SOURCE_PATH', '').strip('/') + if source_path: + src_root = src_root / source_path + + dst_root = pathlib.Path('.') / layer / region_dir + dst_root.mkdir(parents=True, exist_ok=True) rendered = 0 copied = 0 @@ -93,8 +109,14 @@ phases: if src.is_dir() or any(part.startswith('.git') for part in src.parts): continue rel = src.relative_to(src_root) + # Files starting with '.' at the template root are written to the account + # repo root (e.g. .sc-automation.yml), not into LAYER/REGION_DIR/. + if len(rel.parts) == 1 and rel.name.startswith('.'): + dst_base = pathlib.Path('.') + else: + dst_base = dst_root if src.suffix == '.j2': - dst = dst_root / rel.with_suffix('') + dst = dst_base / rel.with_suffix('') dst.parent.mkdir(parents=True, exist_ok=True) env = Environment( loader=FileSystemLoader(str(src.parent)), @@ -105,11 +127,11 @@ phases: dst.write_text(content) rendered += 1 else: - dst = dst_root / rel + dst = dst_base / rel dst.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src, dst) copied += 1 - print(f'Template repo: rendered {rendered} .j2 file(s), copied {copied} file(s)') + print(f'Template repo: rendered {rendered} .j2 file(s), copied {copied} file(s) -> {layer}/{region_dir}/') PYEOF else echo 'No TEMPLATE_REPO specified — skipping template rendering' diff --git a/docs/HOW-IT-WORKS.md b/docs/HOW-IT-WORKS.md index b507b94..e81925c 100644 --- a/docs/HOW-IT-WORKS.md +++ b/docs/HOW-IT-WORKS.md @@ -241,9 +241,11 @@ TEMPLATE_REPO, TEMPLATE_VARS, EXTRA_FILES, GITHUB_TOKEN 1. Rewrite git remote URLs (`ssh://` → `https://`) using the GHE PAT 2. `git clone` the account repo; `git checkout -B ${GIT_BRANCH}` 3. If `TEMPLATE_REPO` is set: - - Clone the template repo - - Walk all files; render `.j2` files with Jinja2 (`StrictUndefined`) - - Copy rendered + non-template files into account repo at same relative paths + - Clone the template repo (at `TEMPLATE_SOURCE_PATH` subdirectory if set) + - Template files are **flat** — no `layer/workspace/` nesting inside the repo + - Render `.j2` files with Jinja2 (`StrictUndefined`); copy non-template files as-is + - All files land in `${LAYER}/${REGION_DIR}/` in the account repo + - Exception: dotfiles (`.sc-automation.yml`, etc.) go to the account repo root 4. If `EXTRA_FILES` is non-empty: - Parse the JSON dict; write each `path → content` entry directly (overrides templates) 5. **Bootstrap workspace state files** (all file generation must be in the Proposer PR): diff --git a/docs/template-management.md b/docs/template-management.md index 4a08803..4213fdd 100644 --- a/docs/template-management.md +++ b/docs/template-management.md @@ -38,32 +38,55 @@ that existing structure. If the template were to include `.tf-control`, ## What Belongs in a Template Repo -A template repo contains only the workload-specific delta: +A template repo is **flat**. It contains only the files that will be written +into `${LAYER}/${REGION_DIR}/` in the target account repo. The Proposer +already knows the destination path from the `LAYER` and `REGION_DIR` env vars +passed by the Lambda — there is no need to encode that in the template structure. ``` template-{product_type}/ -├── {layer}/ -│ └── {workspace}/ -│ ├── {workload}.tf.j2 # workload resources — rendered by Proposer -│ └── tf-run.data # apply step sequence for this workspace -└── .sc-automation.yml.j2 # optional: Proposer writes this if absent +├── {workload}.tf.j2 # workload resources — rendered into LAYER/REGION_DIR/ +├── tf-run.data # apply step sequence — copied into LAYER/REGION_DIR/ +└── .sc-automation.yml.j2 # optional — written to repo root (dotfiles are special-cased) ``` ### Minimal real example — `template-s3-bucket` ``` template-s3-bucket/ -├── infrastructure/ -│ └── west/ -│ ├── INF.s3-standard.tf.j2 # S3 bucket + policy resources -│ └── tf-run.data # REMOTE-STATE + tf-directory-setup + ALL +├── INF.s3-standard.tf.j2 # S3 bucket + policy resources +├── tf-run.data # REMOTE-STATE + tf-directory-setup + ALL └── .sc-automation.yml.j2 ``` -That is the entire template. Nothing else. The account repo already provides -the execution context: Terraform binary version, plugin cache, proxy settings, -provider config, region, credentials, and the layer-level `remote_state.yml` -from which the workspace `remote_state.yml` is derived. +When the Proposer runs with `LAYER=infrastructure REGION_DIR=west`, these three +files land at: +``` +{account-repo}/infrastructure/west/INF.s3-standard.tf +{account-repo}/infrastructure/west/tf-run.data +{account-repo}/.sc-automation.yml ← dotfiles go to repo root +``` + +The same template repo works unchanged for any account, any region, any layer. +No account-specific values. No directory nesting. + +### Multiple variants in one repo (`TEMPLATE_SOURCE_PATH`) + +When a template repo holds more than one product variant, use subdirectories +and set `TEMPLATE_SOURCE_PATH` to select the one to use: + +``` +template-s3/ +├── standard/ +│ ├── INF.s3-standard.tf.j2 +│ └── tf-run.data +└── encrypted/ + ├── INF.s3-encrypted.tf.j2 + └── tf-run.data +``` + +With `TEMPLATE_SOURCE_PATH=encrypted`, the Proposer uses `encrypted/` as the +root and the nesting is stripped — files still land flat in `LAYER/REGION_DIR/`. ### When the target layer does not yet exist @@ -97,10 +120,10 @@ values come from the validated Pydantic model, not from a `.j2` file. ## Template Repository Conventions -### `tf-run.data` — required in every workspace the template touches +### `tf-run.data` — required, placed at the workspace root -Every workspace directory added by the template must include a `tf-run.data` -with at minimum: +The template must include a `tf-run.data` at its root (it lands in +`LAYER/REGION_DIR/` after the Proposer copies it). Minimum content: ``` VERSION 1.0 @@ -118,19 +141,19 @@ ALL - `TAG apply-start` lets an operator re-run from this point without re-running the setup directives. -### `.sc-automation.yml.j2` — optional +### `.sc-automation.yml.j2` — optional, written to repo root -If the template includes `.sc-automation.yml.j2`, the Proposer renders and -commits it. If absent, the Proposer writes a default `.sc-automation.yml` -using the product type and executor project from the Lambda's `TfRunRequest` -model. Either way, the file ends up at the repo root on `main` after merge. +Files whose names start with `.` at the template root are written to the +**account repo root**, not into `LAYER/REGION_DIR/`. This is how +`.sc-automation.yml.j2` ends up at the right place without a separate +mechanism. If absent, the Proposer writes a default `.sc-automation.yml` +from the Lambda's `TfRunRequest` model. ### `.terraform.lock.hcl` — include if possible -If the template is authored for a known provider set (e.g. `hashicorp/aws`), -include a pre-generated `.terraform.lock.hcl` in each workspace directory. -This avoids a from-scratch provider resolution on first `tf-init` and gives -reviewers visibility into the locked provider versions. +Include a pre-generated `.terraform.lock.hcl` at the template root; it lands +in `LAYER/REGION_DIR/`. This avoids a from-scratch provider resolution on +first `tf-init` and gives reviewers visibility into locked provider versions. If omitted, the Executor generates it on first `tf-init` and commits it back to `main` (tagged `[skip ci]`). @@ -164,21 +187,13 @@ environment-specific values. --- -## Subdirectory Templates - -A single template repo can contain multiple product variants as subdirectories. -The Lambda passes `source_path` to the Proposer to clone only the relevant subtree: - -``` -template-s3/ -├── standard/ -│ └── infrastructure/west/INF.s3-standard.tf.j2 -└── encrypted/ - └── infrastructure/west/INF.s3-encrypted.tf.j2 -``` +## Subdirectory Templates (`TEMPLATE_SOURCE_PATH`) -A product that specifies `source_path: encrypted` copies only -`infrastructure/west/INF.s3-encrypted.tf.j2` into the account repo. +See the [Multiple variants](#multiple-variants-in-one-repo-template_source_path) +section above. The `TEMPLATE_SOURCE_PATH` env var tells the Proposer which +subdirectory of the template repo to treat as the root. The selected subtree +is still rendered flat into `LAYER/REGION_DIR/` — the subdirectory path is +stripped entirely. ---