-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1 from SCT-Engineering/feature/template-repo-rend…
…ering docs: generalized architecture, ADR-001 accepted, ADR-002 Vault AWS Secrets Engine
- Loading branch information
Showing
35 changed files
with
6,322 additions
and
1,757 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| # Packer pipeline zip files | ||
| tf-run-executor-builder.zip | ||
|
|
||
| # Terraform state and local overrides | ||
| *.tfstate | ||
| *.tfstate.backup | ||
| *.tfvars | ||
| .terraform/ | ||
| .terraform.lock.hcl | ||
| .terraform_commits | ||
| terraform_data_dirs/ | ||
| varfiles/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,185 @@ | ||
| version: 0.2 | ||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # tf-run-executor buildspec | ||
| # | ||
| # Purpose: clone account repo main branch, optionally assume a cross-account | ||
| # IAM role, and run tf-run apply in the target layer/region directory. | ||
| # This is triggered AFTER a proposer PR has been reviewed and merged. | ||
| # It does not render templates or open a PR. | ||
| # | ||
| # Required env-var overrides per build (supplied by Lambda): | ||
| # ACCOUNT_REPO - account repo name, e.g. 229685449397-csvd-dev-platform-dev-gov | ||
| # LAYER - terraform layer: common | infrastructure | vpc | ||
| # REGION_DIR - region directory: east | west | global | ||
| # GITHUB_TOKEN - GHE PAT (PLAINTEXT, value from Secrets Manager) | ||
| # | ||
| # Optional env-var overrides: | ||
| # TARGET_ACCOUNT_ID - AWS account ID to assume the cross-account role in | ||
| # (default: empty = run with CodeBuild role, csvd-dev only) | ||
| # CROSS_ACCOUNT_ROLE - IAM role name to assume in TARGET_ACCOUNT_ID | ||
| # (default: r-inf-terraform) | ||
| # TF_RUN_START_TAG - tf-run.data TAG label to start from (default: empty = from top) | ||
| # DRY_RUN - "true" = tf-run plan only, no apply (default: "false") | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
| env: | ||
| variables: | ||
| GITHUB_ORG: "SCT-Engineering" | ||
| TF_BINARY_S3_PREFIX: "s3://csvd-packer-pipeline-assets/terraform" | ||
| GH_CLI_S3_PREFIX: "s3://csvd-packer-pipeline-assets/tools" | ||
| CENSUS_CA_S3: "s3://csvd-packer-pipeline-assets/certs/census-ca.pem" | ||
| TERRAFORM_SUPPORT_REPO: "terraform/support" | ||
| HTTPS_PROXY: "http://proxy.tco.census.gov:3128" | ||
| NO_PROXY: "github.e.it.census.gov,169.254.169.254,169.254.170.2" | ||
| # Per-build defaults (overridden via environmentVariablesOverride in Lambda) | ||
| TARGET_ACCOUNT_ID: "" | ||
| CROSS_ACCOUNT_ROLE: "r-inf-terraform" | ||
| TF_RUN_START_TAG: "" | ||
| DRY_RUN: "false" | ||
|
|
||
| phases: | ||
| install: | ||
| commands: | ||
| # --- Version governance: clone terraform/support to read org-canonical versions --- | ||
| - git clone --depth 1 "https://${GITHUB_TOKEN}@github.e.it.census.gov/${TERRAFORM_SUPPORT_REPO}.git" /tmp/tf-support | ||
| - export TF_VERSION=$(cat /tmp/tf-support/terraform/VERSION) | ||
| - export GH_VERSION=$(cat /tmp/tf-support/github-cli-releases/VERSION) | ||
| - echo "Using Terraform ${TF_VERSION}, gh CLI ${GH_VERSION}" | ||
|
|
||
| # --- Terraform binary (registry.terraform.io is blocked on Census network; use S3) --- | ||
| - aws s3 cp "${TF_BINARY_S3_PREFIX}/terraform_${TF_VERSION}_linux_amd64.zip" /tmp/terraform.zip | ||
| - unzip -o /tmp/terraform.zip -d /usr/local/bin/ && chmod +x /usr/local/bin/terraform | ||
| - ln -sf /usr/local/bin/terraform /usr/local/bin/tf | ||
|
|
||
| # --- Census CA certificate (required for TLS to github.e.it.census.gov) --- | ||
| - aws s3 cp "$CENSUS_CA_S3" /etc/pki/ca-trust/source/anchors/census-ca.pem | ||
| - update-ca-trust extract | ||
|
|
||
| # --- tf-run toolchain (sourced from terraform/support, already cloned above) --- | ||
| # Canonical versions live in terraform/support local-app/ — no copies kept in this repo. | ||
| - cp /tmp/tf-support/local-app/tf-run/tf-run.sh /usr/local/bin/tf-run | ||
| - cp /tmp/tf-support/local-app/tf-control/tf-control.sh /usr/local/bin/tf-control.sh | ||
| - cp /tmp/tf-support/local-app/tf-directory-setup/tf-directory-setup.py /usr/local/bin/tf-directory-setup.py | ||
| - chmod +x /usr/local/bin/tf-run /usr/local/bin/tf-control.sh /usr/local/bin/tf-directory-setup.py | ||
| # Create tf-{action} symlinks expected by tf-run and account repo steps | ||
| - > | ||
| for action in init plan apply destroy refresh output validate import state fmt taint console; do | ||
| ln -sf /usr/local/bin/tf-control.sh /usr/local/bin/tf-${action}; | ||
| done | ||
| # Account repo .tf-control files set TFCOMMAND=terraform_latest (the Census workstation alias). | ||
| # In CodeBuild the binary is just 'terraform'; create the alias so tf-control.sh resolves it. | ||
| - ln -sf /usr/local/bin/terraform /usr/local/bin/terraform_latest | ||
|
|
||
| # --- Plugin cache directory (referenced by .tf-control.tfrc in every account repo) --- | ||
| # .tf-control.tfrc sets plugin_cache_dir = "/data/terraform/terraform.d/plugin-cache" | ||
| # and filesystem_mirror path = "/data/terraform/terraform.d/providers". | ||
| # Create both so Terraform does not error on init; the mirror is empty so Terraform | ||
| # falls through to the 'direct' block in the tfrc (via Census proxy to registry.terraform.io). | ||
| - mkdir -p /data/terraform/terraform.d/plugin-cache /data/terraform/terraform.d/providers | ||
|
|
||
| # --- Python deps for tf-directory-setup.py --- | ||
| - pip3 install --quiet python-dateutil pyyaml | ||
|
|
||
| # --- gh CLI (from S3, for any post-apply verification steps) --- | ||
| - aws s3 cp "${GH_CLI_S3_PREFIX}/gh_${GH_VERSION}_linux_amd64.tar.gz" /tmp/gh.tar.gz | ||
| - mkdir -p /tmp/gh-cli && tar -xzf /tmp/gh.tar.gz -C /tmp/gh-cli --strip-components=1 | ||
| - cp /tmp/gh-cli/bin/gh /usr/local/bin/gh && chmod +x /usr/local/bin/gh | ||
|
|
||
| build: | ||
| commands: | ||
| # --- Configure git to rewrite SSH URLs to HTTPS --- | ||
| # Module sources in account repos use ssh://git@github.e.it.census.gov/... or git@... | ||
| # This rewrite transparently redirects them to HTTPS + PAT at the git layer. | ||
| - git config --global url."https://${GITHUB_TOKEN}@github.e.it.census.gov/".insteadOf "ssh://git@github.e.it.census.gov/" | ||
| - git config --global url."https://${GITHUB_TOKEN}@github.e.it.census.gov/".insteadOf "git@github.e.it.census.gov:" | ||
|
|
||
| # --- Clone account repo from main (the reviewed + merged state) --- | ||
| - git clone "https://${GITHUB_TOKEN}@github.e.it.census.gov/${GITHUB_ORG}/${ACCOUNT_REPO}.git" repo | ||
| - cd repo | ||
| # Verify we are on main (not a work branch) | ||
| - git checkout main | ||
| - echo "Applying from $(git rev-parse --short HEAD) on main" | ||
|
|
||
| # --- Assume cross-account role (if TARGET_ACCOUNT_ID is set) --- | ||
| # The role (default: r-inf-terraform) must exist in the target account and | ||
| # trust arn:...:iam::229685449397:role/tf-run-executor-codebuild. | ||
| # Override CROSS_ACCOUNT_ROLE per-build to use a different role name. | ||
| - | | ||
| if [ -n "${TARGET_ACCOUNT_ID}" ]; then | ||
| PARTITION=$(aws sts get-caller-identity --query Arn --output text | cut -d: -f2) | ||
| ROLE_ARN="arn:${PARTITION}:iam::${TARGET_ACCOUNT_ID}:role/${CROSS_ACCOUNT_ROLE}" | ||
| echo "Assuming cross-account role: ${ROLE_ARN}" | ||
| CREDS=$(aws sts assume-role \ | ||
| --role-arn "${ROLE_ARN}" \ | ||
| --role-session-name "sc-automation-${ACCOUNT_REPO}" \ | ||
| --query Credentials \ | ||
| --output json) | ||
| export AWS_ACCESS_KEY_ID=$(echo "$CREDS" | python3 -c "import json,sys; print(json.load(sys.stdin)['AccessKeyId'])") | ||
| export AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | python3 -c "import json,sys; print(json.load(sys.stdin)['SecretAccessKey'])") | ||
| export AWS_SESSION_TOKEN=$(echo "$CREDS" | python3 -c "import json,sys; print(json.load(sys.stdin)['SessionToken'])") | ||
| echo "Assumed role in account ${TARGET_ACCOUNT_ID}" | ||
| else | ||
| echo "No TARGET_ACCOUNT_ID set — running with CodeBuild role (csvd-dev only)" | ||
| fi | ||
| # --- Run Terraform in target layer/region directory --- | ||
| # tf-run auto-proceeds on non-TTY stdin (read -t timeout defaults to "y") | ||
| # | ||
| # NOTE on file-generating tf-run.data directives: | ||
| # REMOTE-STATE — generates workspace remote_state.yml from parent | ||
| # COMMAND tf-directory-setup.py — generates remote_state.backend.tf + variant files | ||
| # The Proposer already ran both of these and committed the results in the PR. | ||
| # When tf-run hits these steps here they are idempotent: they overwrite files | ||
| # that already exist with identical content. No new files are created at apply time. | ||
| # | ||
| # NOTE on logs/: tf-control.sh writes every plan/apply to logs/{action}.{timestamp}.log. | ||
| # This directory is ephemeral (never committed). Ensure logs/ is in .gitignore. | ||
| - cd "${LAYER}/${REGION_DIR}" | ||
| - | | ||
| if [ "${DRY_RUN}" = "true" ]; then | ||
| echo "DRY_RUN=true — running tf-run plan only" | ||
| TFARGS="-no-color" tf-run plan | ||
| elif [ -n "${TF_RUN_START_TAG}" ]; then | ||
| TFARGS="-auto-approve" tf-run apply "tag:${TF_RUN_START_TAG}" | ||
| else | ||
| TFARGS="-auto-approve" tf-run apply | ||
| fi | ||
| # --- Commit post-apply file changes back to main --- | ||
| # After a successful apply tf-run.data typically runs: | ||
| # COMMAND tf-directory-setup.py --link s3 | ||
| # which re-links remote_state.{dir}.tf from .tf.none → .tf.s3. | ||
| # terraform init also generates/updates .terraform.lock.hcl. | ||
| # Both of these changes must be committed back to main so: | ||
| # (a) the repo reflects actual state for future Proposer re-renders | ||
| # (b) subsequent tf-init on main does not re-download all providers | ||
| # [skip ci] prevents the push from re-triggering the webhook executor. | ||
| - cd "${CODEBUILD_SRC_DIR}/repo" | ||
| - | | ||
| git add -A -- "${LAYER}/${REGION_DIR}/remote_state."* \ | ||
| "${LAYER}/${REGION_DIR}/.terraform.lock.hcl" 2>/dev/null || true | ||
| if ! git diff --cached --quiet; then | ||
| git -c user.email="sc-automation@census.gov" \ | ||
| -c user.name="SC Automation" \ | ||
| commit -m "chore: executor post-apply update ${LAYER}/${REGION_DIR} [skip ci]" | ||
| git push \ | ||
| "https://${GITHUB_TOKEN}@github.e.it.census.gov/${GITHUB_ORG}/${ACCOUNT_REPO}.git" \ | ||
| HEAD:main | ||
| echo "Committed and pushed post-apply changes to main" | ||
| else | ||
| echo "No post-apply file changes to commit" | ||
| fi | ||
| post_build: | ||
| commands: | ||
| - echo "BUILD_RESULT=${CODEBUILD_BUILD_SUCCEEDING}" | ||
| - echo "ACCOUNT_REPO=${ACCOUNT_REPO}" | ||
| - echo "LAYER=${LAYER} REGION_DIR=${REGION_DIR}" | ||
|
|
||
| cache: | ||
| paths: | ||
| # Cache the provider plugin cache across builds for faster tf-init. | ||
| # Providers downloaded via Census proxy are stored here; subsequent builds | ||
| # skip re-downloading providers that haven't changed. | ||
| - /data/terraform/terraform.d/plugin-cache/**/* |
Oops, something went wrong.