Add tf-run-executor infrastructure and Lambda trigger for CodeBuild a…

…utomation

- Create .vscode/settings.json for YAML custom tags in CloudFormation templates.
- Add buildspec.yml for CodeBuild project configuration to run Terraform.
- Define CodeBuild project and source credentials in deploy/codebuild.tf.
- Set up IAM roles and policies for Lambda and CodeBuild in deploy/iam.tf.
- Create ECR repository and Lambda function for tf-run executor in deploy/lambda.tf.
- Configure Terraform provider in deploy/provider.tf.
- Implement Service Catalog product and portfolio in deploy/service_catalog.tf.
- Define necessary variables in deploy/variables.tf for configuration.
- Add Dockerfile for Lambda function with dependencies.
- Implement Lambda function logic in lambda/app.py to handle CloudFormation events.
- Specify Python dependencies in lambda/requirements.txt.
- Create Service Catalog product template in service-catalog/product-template.yaml.

.vscode/settings.json

@@ -0,0 +1,26 @@
+{
+  // Tell the YAML language server about CloudFormation intrinsic function tags
+  // so it doesn't report "Unresolved tag" errors in CFN templates.
+  "yaml.customTags": [
+    "!And sequence",
+    "!Base64 scalar",
+    "!Cidr sequence",
+    "!Condition scalar",
+    "!Equals sequence",
+    "!FindInMap sequence",
+    "!GetAtt scalar",
+    "!GetAZs scalar",
+    "!If sequence",
+    "!ImportValue scalar",
+    "!Join sequence",
+    "!Not sequence",
+    "!Or sequence",
+    "!Ref scalar",
+    "!Select sequence",
+    "!Split sequence",
+    "!Sub scalar",
+    "!Sub sequence",
+    "!Transform mapping",
+    "!Value scalar"
+  ]
+}

buildspec.yml

@@ -0,0 +1,127 @@
+version: 0.2
+
+# ---------------------------------------------------------------------------
+# tf-run-executor buildspec
+#
+# Required env-var overrides per build (supplied by Lambda or manual CLI):
+#   ACCOUNT_REPO      - account repo name, e.g. 229685449397-csvd-dev-platform-dev-gov
+#   LAYER             - terraform layer: common | infrastructure | vpc
+#   REGION_DIR        - region directory: east | west
+#   GITHUB_TOKEN      - GHE PAT (type PLAINTEXT, value from Secrets Manager)
+#
+# Optional env-var overrides:
+#   GIT_BRANCH        - branch to commit/PR from (default: repo-init)
+#   TF_RUN_START_TAG  - tf-run.data TAG label to start from (default: empty = from top)
+#   EXTRA_FILES       - JSON map {"relative/path": "content"} written before tf-run
+#   DRY_RUN           - "true" = tf plan only, no apply (default: "false")
+# ---------------------------------------------------------------------------
+
+env:
+  variables:
+    GITHUB_ORG: "SCT-Engineering"
+    TF_BINARY_S3: "s3://csvd-packer-pipeline-assets/terraform/terraform_1.9.1_linux_amd64.zip"
+    CENSUS_CA_S3: "s3://csvd-packer-pipeline-assets/certs/census-ca.pem"
+    GH_CLI_S3: "s3://csvd-packer-pipeline-assets/tools/gh_2.49.0_linux_amd64.tar.gz"
+    HTTPS_PROXY: "http://proxy.tco.census.gov:3128"
+    NO_PROXY: "github.e.it.census.gov,169.254.169.254"
+    # Per-build defaults (overridden via environmentVariablesOverride in Lambda)
+    GIT_BRANCH: "repo-init"
+    DRY_RUN: "false"
+    TF_RUN_START_TAG: ""
+    EXTRA_FILES: "{}"
+
+phases:
+  install:
+    commands:
+      # --- Terraform binary (registry.terraform.io is blocked; pull from S3) ---
+      - aws s3 cp "$TF_BINARY_S3" /tmp/terraform.zip
+      - unzip -o /tmp/terraform.zip -d /usr/local/bin/ && chmod +x /usr/local/bin/terraform
+      - ln -sf /usr/local/bin/terraform /usr/local/bin/tf
+
+      # --- Census CA certificate (GHE TLS) ---
+      - aws s3 cp "$CENSUS_CA_S3" /etc/pki/ca-trust/source/anchors/census-ca.pem
+      - update-ca-trust extract
+
+      # --- tf-run toolchain (sourced from this repo's scripts/) ---
+      - cp "$CODEBUILD_SRC_DIR/scripts/tf-run" /usr/local/bin/tf-run
+      - cp "$CODEBUILD_SRC_DIR/scripts/tf-control.sh" /usr/local/bin/tf-control.sh
+      - cp "$CODEBUILD_SRC_DIR/scripts/tf-directory-setup.py" /usr/local/bin/tf-directory-setup.py
+      - chmod +x /usr/local/bin/tf-run /usr/local/bin/tf-control.sh /usr/local/bin/tf-directory-setup.py
+      # Create tf-{action} symlinks expected by tf-run and account repo steps
+      - >
+        for action in init plan apply destroy refresh output validate import state fmt taint console; do
+          ln -sf /usr/local/bin/tf-control.sh /usr/local/bin/tf-${action};
+        done
+
+      # --- Python deps for tf-directory-setup.py ---
+      - pip3 install --quiet jinja2 python-dateutil pyyaml
+
+      # --- gh CLI ---
+      - aws s3 cp "$GH_CLI_S3" /tmp/gh.tar.gz
+      - mkdir -p /tmp/gh-cli
+      - tar -xzf /tmp/gh.tar.gz -C /tmp/gh-cli --strip-components=1
+      - cp /tmp/gh-cli/bin/gh /usr/local/bin/gh && chmod +x /usr/local/bin/gh
+
+  build:
+    commands:
+      # --- Clone account repo over HTTPS (SSH is blocked by Census proxy) ---
+      - git clone "https://${GITHUB_TOKEN}@github.e.it.census.gov/${GITHUB_ORG}/${ACCOUNT_REPO}.git" repo
+      - cd repo
+      - git checkout -B "${GIT_BRANCH}"
+
+      # --- Write extra config files passed in from Lambda (JSON map path -> content) ---
+      - |
+        python3 -c "
+        import json, os, pathlib
+        files = json.loads(os.environ.get('EXTRA_FILES', '{}'))
+        for path, content in files.items():
+            p = pathlib.Path(path)
+            p.parent.mkdir(parents=True, exist_ok=True)
+            p.write_text(content)
+        print(f'Wrote {len(files)} extra file(s)')
+        "
+
+      # --- Commit and push (--allow-empty handles no-change case) ---
+      - git add -A
+      - |
+        git -c user.email="sc-automation@census.gov" \
+            -c user.name="SC Automation" \
+            commit -m "SC automation: ${LAYER}/${REGION_DIR} [${ACCOUNT_REPO}]" \
+            --allow-empty
+      - git push origin "${GIT_BRANCH}"
+
+      # --- Run Terraform in target layer/region directory ---
+      # tf-run auto-proceeds on non-TTY stdin (read -t timeout defaults to "y")
+      - cd "${LAYER}/${REGION_DIR}"
+      - |
+        if [ "${DRY_RUN}" = "true" ]; then
+          tf-plan -no-color
+        elif [ -n "${TF_RUN_START_TAG}" ]; then
+          TFARGS="-auto-approve" tf-run apply "tag:${TF_RUN_START_TAG}"
+        else
+          TFARGS="-auto-approve" tf-run apply
+        fi
+
+      # --- Open PR (idempotent: skip if PR already exists) ---
+      - |
+        GH_HOST=github.e.it.census.gov \
+        GH_TOKEN="${GITHUB_TOKEN}" \
+        gh pr create \
+          --title "SC automation: ${LAYER}/${REGION_DIR} [${ACCOUNT_REPO}]" \
+          --body "Triggered by Service Catalog provisioning of **${ACCOUNT_REPO}**." \
+          --base main \
+          --head "${GIT_BRANCH}" \
+        || echo "PR already exists or create failed, continuing"
+
+  post_build:
+    commands:
+      - echo "BUILD_RESULT=${CODEBUILD_BUILD_SUCCEEDING}"
+      # Emit PR_URL so Lambda can parse it from the build output
+      - |
+        PR_URL=$(GH_HOST=github.e.it.census.gov \
+          GH_TOKEN="${GITHUB_TOKEN}" \
+          gh pr view \
+            --repo "${GITHUB_ORG}/${ACCOUNT_REPO}" \
+            "${GIT_BRANCH}" \
+            --json url -q .url 2>/dev/null || echo "")
+        echo "PR_URL=${PR_URL}"

deploy/codebuild.tf

@@ -0,0 +1,116 @@
+data "aws_partition" "current" {}
+data "aws_caller_identity" "current" {}
+data "aws_region" "current" {}
+
+# GHE source credential — one per account per server_type per region.
+# If a credential for GITHUB_ENTERPRISE already exists in this account,
+# import it: terraform import aws_codebuild_source_credential.ghe <arn>
+resource "aws_codebuild_source_credential" "ghe" {
+  auth_type   = "PERSONAL_ACCESS_TOKEN"
+  server_type = "GITHUB_ENTERPRISE"
+  token       = data.aws_secretsmanager_secret_version.ghe_token.secret_string
+}
+
+data "aws_secretsmanager_secret_version" "ghe_token" {
+  secret_id = "ghe-runner/github-token"
+}
+
+resource "aws_codebuild_project" "tf_run_executor" {
+  name          = "tf-run-executor"
+  description   = "Clones account repo, writes config files, runs tf-run, opens PR"
+  build_timeout = 60 # minutes
+  service_role  = aws_iam_role.codebuild_exec.arn
+
+  artifacts {
+    type = "NO_ARTIFACTS"
+  }
+
+  environment {
+    compute_type    = "BUILD_GENERAL1_SMALL"
+    image           = "aws/codebuild/amazonlinux2-x86_64-standard:3.0"
+    type            = "LINUX_CONTAINER"
+    privileged_mode = false
+
+    # --- Static defaults (overridden per-build via environmentVariablesOverride) ---
+    environment_variable {
+      name  = "GITHUB_ORG"
+      value = var.github_org
+    }
+    environment_variable {
+      name  = "TF_BINARY_S3"
+      value = var.tf_binary_s3
+    }
+    environment_variable {
+      name  = "CENSUS_CA_S3"
+      value = var.census_ca_s3
+    }
+    environment_variable {
+      name  = "GH_CLI_S3"
+      value = var.gh_cli_s3
+    }
+    environment_variable {
+      name  = "HTTPS_PROXY"
+      value = var.https_proxy
+    }
+    environment_variable {
+      name  = "NO_PROXY"
+      value = "github.e.it.census.gov,169.254.169.254"
+    }
+    # Placeholder values — always overridden by Lambda per-build
+    environment_variable {
+      name  = "ACCOUNT_REPO"
+      value = "OVERRIDE_PER_BUILD"
+    }
+    environment_variable {
+      name  = "LAYER"
+      value = "OVERRIDE_PER_BUILD"
+    }
+    environment_variable {
+      name  = "REGION_DIR"
+      value = "OVERRIDE_PER_BUILD"
+    }
+    environment_variable {
+      name  = "GITHUB_TOKEN"
+      type  = "SECRETS_MANAGER"
+      value = var.github_token_secret_name
+    }
+    environment_variable {
+      name  = "GIT_BRANCH"
+      value = "repo-init"
+    }
+    environment_variable {
+      name  = "DRY_RUN"
+      value = "false"
+    }
+    environment_variable {
+      name  = "TF_RUN_START_TAG"
+      value = ""
+    }
+    environment_variable {
+      name  = "EXTRA_FILES"
+      value = "{}"
+    }
+  }
+
+  source {
+    type            = "GITHUB_ENTERPRISE"
+    location        = var.source_repo_url
+    buildspec       = "buildspec.yml"
+    git_clone_depth = 1
+  }
+
+  logs_config {
+    cloudwatch_logs {
+      group_name  = "/aws/codebuild/tf-run-executor"
+      stream_name = ""
+      status      = "ENABLED"
+    }
+  }
+
+  tags = {
+    Project   = "sc-automation"
+    ManagedBy = "terraform"
+  }
+
+  depends_on = [aws_codebuild_source_credential.ghe]
+}