move buildspecs to buildspecs dir

SCT-Engineering · May 7, 2025 · a1fe4a4 · a1fe4a4
1 parent 8fe8025
commit a1fe4a4
Show file tree

Hide file tree

Showing 10 changed files with 498 additions and 1 deletion.
diff --git a/.github/platform-tg-infra.code-workspace b/.github/platform-tg-infra.code-workspace
@@ -80,6 +80,10 @@
 			"name": "tfmod-open-telemetry",
 			"path": "../../tfmod-open-telemetry"
 		},
+		{
+			"name": "tfmod-pipeline",
+			"path": "../../tfmod-pipeline"
+		},
 		{
 			"name": "tfmod-prometheus",
 			"path": "../../tfmod-prometheus"
@@ -91,6 +95,12 @@
 		{
 			"name": "tfmod-tempo",
 			"path": "../../tfmod-tempo"
+		},
+		{
+			"path": "../../243219719746-lab-gov-management-nonprod"
+		},
+		{
+			"path": "../../../terraform-modules/aws-s3"
 		}
 	]
 }
diff --git a/.github/workflows/package-and-upload.yml b/.github/workflows/package-and-upload.yml
@@ -0,0 +1,120 @@
+name: Package and Upload Terragrunt Configs
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      bootstrap:
+        description: 'Run bootstrap apply'
+        required: false
+        default: 'false'
+        type: choice
+        options:
+          - 'true'
+          - 'false'
+
+env:
+  NODE_TLS_REJECT_UNAUTHORIZED: '0'
+  ACCOUNT_PROFILE_NAME: "lab-dev-gov"
+  CLUSTER_NAME: "csvd-platform-lab-mcm"
+  SOURCE_KEY: "platform-tg-infra.zip"
+  PIPELINE_PATH: "lab/development/us-gov-east-1/vpc/${CLUSTER_NAME}/eks-pipeline"
+
+permissions:
+  actions: read
+  contents: read
+  id-token: write
+
+jobs:
+  package-and-upload:
+    runs-on: [self-hosted, Linux, X64, buildkitsandbox]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Configure AWS credentials
+        uses: etools/configure-aws-credentials@main
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: "arn:aws-us-gov:iam::${{ vars.AWS_ACCOUNT_ID }}:role/r-inf-terraform-eks"
+          role-skip-session-tagging: true
+
+      - name: Add profile credentials to ~/.aws/credentials
+        run: |
+          aws configure set aws_region ${{ vars.AWS_REGION }} --profile "${{ vars.AWS_ACCOUNT_ID }}-${{ env.ACCOUNT_PROFILE_NAME }}"
+          aws configure set aws_access_key_id ${{ env.AWS_ACCESS_KEY_ID }} --profile "${{ vars.AWS_ACCOUNT_ID }}-${{ env.ACCOUNT_PROFILE_NAME }}"
+          aws configure set aws_secret_access_key ${{ env.AWS_SECRET_ACCESS_KEY }} --profile "${{ vars.AWS_ACCOUNT_ID }}-${{ env.ACCOUNT_PROFILE_NAME }}"
+          aws configure set aws_session_token ${{ env.AWS_SESSION_TOKEN }} --profile "${{ vars.AWS_ACCOUNT_ID }}-${{ env.ACCOUNT_PROFILE_NAME }}"
+          aws sts get-caller-identity --profile "${{ vars.AWS_ACCOUNT_ID }}-${{ env.ACCOUNT_PROFILE_NAME }}"
+
+      - name: Package Terragrunt configs
+        run: |
+          # Create a zip file of the repository contents
+          zip -r platform-tg-infra.zip . -x "*.git*" "*.github*" "*.terragrunt-cache*" "*.terraform*"
+          
+          # Calculate bucket name using the same format as in tfmod-pipeline/s3.tf
+          REGION_SHORT=$(echo ${{ vars.AWS_REGION }} | sed 's/\([a-z]\)[a-z]*-/\1/g')
+          SOURCE_BUCKET="inf-s3-${CLUSTER_NAME}-artifacts-${{ vars.AWS_ACCOUNT_ID }}-${REGION_SHORT}"
+          echo "SOURCE_BUCKET=${SOURCE_BUCKET}" >> $GITHUB_ENV
+          
+          # Calculate the object key with the cluster-specific path
+          OBJECT_KEY="clusters/${CLUSTER_NAME}/platform-tg-infra.zip"
+          echo "OBJECT_KEY=${OBJECT_KEY}" >> $GITHUB_ENV
+          
+          # Check if the source bucket exists, create it if it doesn't
+          if ! aws s3api head-bucket --bucket ${SOURCE_BUCKET} --profile "${{ vars.AWS_ACCOUNT_ID }}-${{ env.ACCOUNT_PROFILE_NAME }}" 2>/dev/null; then
+            echo "Creating source bucket ${SOURCE_BUCKET}"
+            aws s3 mb s3://${SOURCE_BUCKET} --profile "${{ vars.AWS_ACCOUNT_ID }}-${{ env.ACCOUNT_PROFILE_NAME }}" --region ${{ vars.AWS_REGION }}
+            
+            # Configure bucket for versioning
+            aws s3api put-bucket-versioning --bucket ${SOURCE_BUCKET} \
+              --versioning-configuration Status=Enabled \
+              --profile "${{ vars.AWS_ACCOUNT_ID }}-${{ env.ACCOUNT_PROFILE_NAME }}"
+              
+            # Block public access
+            aws s3api put-public-access-block --bucket ${SOURCE_BUCKET} \
+              --public-access-block-configuration BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true \
+              --profile "${{ vars.AWS_ACCOUNT_ID }}-${{ env.ACCOUNT_PROFILE_NAME }}"
+            
+            # Create a flag file to indicate the bucket needs to be imported
+            echo "BUCKET_NEEDS_IMPORT=true" >> $GITHUB_ENV
+          else
+            echo "Bucket ${SOURCE_BUCKET} already exists"
+          fi
+          
+          # Upload the zip file to S3
+          aws s3 cp platform-tg-infra.zip s3://${SOURCE_BUCKET}/${OBJECT_KEY} \
+            --profile "${{ vars.AWS_ACCOUNT_ID }}-${{ env.ACCOUNT_PROFILE_NAME }}"
+          
+          echo "Uploaded platform-tg-infra.zip to s3://${SOURCE_BUCKET}/${OBJECT_KEY}"
+
+      - name: Make Import Script Executable
+        if: ${{ github.event.inputs.bootstrap == 'true' && env.BUCKET_NEEDS_IMPORT == 'true' }}
+        run: |
+          chmod +x scripts/import-s3-bucket.sh
+
+      - name: Bootstrap Pipeline (if requested)
+        if: ${{ github.event.inputs.bootstrap == 'true' }}
+        run: |
+          # If the bucket was just created, import it first
+          if [ "$BUCKET_NEEDS_IMPORT" = "true" ]; then
+            echo "Running import for newly created bucket $SOURCE_BUCKET"
+            ./scripts/import-s3-bucket.sh \
+              "$SOURCE_BUCKET" \
+              "${{ vars.AWS_REGION }}" \
+              "${{ vars.AWS_ACCOUNT_ID }}-${{ env.ACCOUNT_PROFILE_NAME }}" \
+              "$PIPELINE_PATH"
+          fi
+          
+          # Now proceed with terragrunt apply
+          cd $PIPELINE_PATH
+          https_proxy=http://proxy.tco.census.gov:3128 \
+          http_proxy=http://proxy.tco.census.gov:3128 \
+          NO_PROXY=.census.gov,169.254.169.254,148.129.0.0/16,10.0.0.0/8,172.16.0/12,.eks.amazonaws.com,.s3.amazonaws.com,.amazonaws.com,.gcr.io,.pkg.dev \
+          TERRAGRUNT_PROVIDER_CACHE=1 \
+          terragrunt apply --terragrunt-non-interactive -auto-approve
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: help init validate plan fmt check clean
+.PHONY: help init validate plan fmt check clean deploy-to-pipeline
 
 help:
 	@echo "Available targets:"
@@ -8,6 +8,7 @@ help:
 	@echo "  fmt       - Format HCL files"
 	@echo "  check     - Run all checks (format, validate, plan)"
 	@echo "  clean     - Clean up Terragrunt cache and temporary files"
+	@echo "  deploy-to-pipeline - Zip and upload to S3 to trigger CodePipeline"
 
 init:
 	@echo "Initializing Terragrunt configurations..."
@@ -33,3 +34,57 @@ clean:
 	find . -type d -name ".terragrunt-cache" -exec rm -rf {} +
 	find . -type f -name ".terraform.lock.hcl" -delete
 	find . -type f -name "terragrunt-debug.tfvars.json" -delete
+
+deploy-to-pipeline:
+	@echo "Preparing to deploy to pipeline..."
+	@echo "Detecting environment configuration..."
+
+	# Set defaults or use provided values
+	$(eval ENV ?= development)
+	$(eval REGION_DIR ?= us-gov-east-1)
+	$(eval CLUSTER_DIR ?= csvd-platform-lab-mcm)
+
+	# Detect account variables
+	$(eval ACCOUNT_HCL=lab/$(ENV)/account.hcl)
+	$(eval REGION_HCL=lab/$(ENV)/$(REGION_DIR)/region.hcl)
+	$(eval CLUSTER_HCL=lab/$(ENV)/$(REGION_DIR)/vpc/$(CLUSTER_DIR)/cluster.hcl)
+
+	@if [ ! -f "$(ACCOUNT_HCL)" ]; then echo "Error: $(ACCOUNT_HCL) not found"; exit 1; fi
+	@if [ ! -f "$(REGION_HCL)" ]; then echo "Error: $(REGION_HCL) not found"; exit 1; fi
+	@if [ ! -f "$(CLUSTER_HCL)" ]; then echo "Error: $(CLUSTER_HCL) not found"; exit 1; fi
+
+	@echo "Extracting configuration values..."
+	# Extract values from HCL files
+	$(eval AWS_ACCOUNT_ID=$(shell grep -oP 'aws_account_id\s*=\s*"\K[^"]+' $(ACCOUNT_HCL)))
+	$(eval ACCOUNT_NAME=$(shell grep -oP 'account_name\s*=\s*"\K[^"]+' $(ACCOUNT_HCL)))
+	$(eval AWS_PROFILE=$(shell echo $(AWS_ACCOUNT_ID)-$(shell echo $(ACCOUNT_NAME) | sed 's/-ew/-gov/')))
+	$(eval AWS_REGION=$(shell grep -oP 'aws_region\s*=\s*"\K[^"]+' $(REGION_HCL)))
+	$(eval CLUSTER_NAME=$(shell grep -oP 'cluster_name\s*=\s*"\K[^"]+' $(CLUSTER_HCL)))
+
+	@echo "Using configuration:"
+	@echo "  AWS_ACCOUNT_ID: $(AWS_ACCOUNT_ID)"
+	@echo "  ACCOUNT_NAME: $(ACCOUNT_NAME)"
+	@echo "  AWS_PROFILE: $(AWS_PROFILE)"
+	@echo "  AWS_REGION: $(AWS_REGION)"
+	@echo "  CLUSTER_NAME: $(CLUSTER_NAME)"
+
+	@if [ -z "$(AWS_ACCOUNT_ID)" ] || [ -z "$(AWS_PROFILE)" ] || [ -z "$(AWS_REGION)" ] || [ -z "$(CLUSTER_NAME)" ]; then \
+		echo "Error: Failed to extract all required variables from HCL files"; \
+		exit 1; \
+	fi
+
+	@echo "Creating zip file..."
+	zip -r platform-tg-infra.zip . -x "*.git*" "*.github*" "*.terragrunt-cache*" "*.terraform*"
+
+	@echo "Calculating S3 bucket name..."
+	$(eval REGION_SHORT=$(shell echo $(AWS_REGION) | sed 's/\([a-z]\)[a-z]*-/\1/g'))
+	$(eval S3_BUCKET=v-s3-eks-$(CLUSTER_NAME)-artifacts-$(AWS_ACCOUNT_ID)-$(REGION_SHORT))
+	$(eval OBJECT_KEY=clusters/$(CLUSTER_NAME)/platform-tg-infra.zip)
+
+	@echo "Uploading to S3 bucket $(S3_BUCKET)..."
+	aws s3 cp platform-tg-infra.zip s3://$(S3_BUCKET)/$(OBJECT_KEY) --profile $(AWS_PROFILE)
+	@echo "Upload complete. Pipeline should trigger automatically."
+	@echo "Check the AWS CodePipeline console for status."
+
+	@echo "Cleaning up local zip file..."
+	rm -f platform-tg-infra.zip
diff --git a/buildspecs/deploy.terragrunt.yml b/buildspecs/deploy.terragrunt.yml
@@ -0,0 +1,72 @@
+version: 0.2
+
+env:
+  variables:
+    BASE_DIR: "lab"
+    TF_VERSION: "1.5.5"
+    TG_VERSION: "0.72.0"
+    TOOLS_DIR: "/tmp/build-tools"
+  exported-variables:
+    - TERRAGRUNT_PATH
+
+cache:
+  paths:
+    - '/tmp/build-tools/**/*'
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.11
+    commands:
+      - echo "Setting up environment and tools"
+      - export http_proxy=$PROXY_CONFIG
+      - export https_proxy=$PROXY_CONFIG
+      - export NO_PROXY=.census.gov,169.254.169.254,148.129.0.0/16,10.0.0.0/8,172.16.0/12,.eks.amazonaws.com,.s3.amazonaws.com,.amazonaws.com,.gcr.io,.pkg.dev
+
+        # Create tools directory if it doesn't exist
+      - mkdir -p $TOOLS_DIR/bin
+
+        # Check if cached Terraform exists and matches required version
+      - |
+        if [ -f "$TOOLS_DIR/bin/terraform" ] && [ "$($TOOLS_DIR/bin/terraform version | head -n1 | grep -o "v$TF_VERSION")" = "v$TF_VERSION" ]; then
+          echo "Using cached Terraform v$TF_VERSION"
+        else
+          echo "Downloading Terraform v$TF_VERSION"
+          curl -Lo /tmp/terraform.zip "https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_linux_amd64.zip"
+          unzip -o /tmp/terraform.zip -d $TOOLS_DIR/bin/
+          chmod +x $TOOLS_DIR/bin/terraform
+        fi
+
+        # Check if cached Terragrunt exists and matches required version
+      - |
+        if [ -f "$TOOLS_DIR/bin/terragrunt" ] && [ "$($TOOLS_DIR/bin/terragrunt --version | grep -o "v$TG_VERSION")" = "v$TG_VERSION" ]; then
+          echo "Using cached Terragrunt v$TG_VERSION"
+        else
+          echo "Downloading Terragrunt v$TG_VERSION"
+          curl -Lo $TOOLS_DIR/bin/terragrunt "https://github.com/gruntwork-io/terragrunt/releases/download/v${TG_VERSION}/terragrunt_linux_amd64"
+          chmod +x $TOOLS_DIR/bin/terragrunt
+        fi
+
+        # Add tools to PATH
+      - export PATH=$TOOLS_DIR/bin:$PATH
+      - terraform --version
+      - terragrunt --version
+      - aws sts get-caller-identity
+
+  build:
+    commands:
+      - echo "Running Terragrunt apply"
+      - cd $TERRAGRUNT_PATH
+      - export http_proxy=$PROXY_CONFIG
+      - export https_proxy=$PROXY_CONFIG
+      - export NO_PROXY=.census.gov,169.254.169.254,148.129.0.0/16,10.0.0.0/8,172.16.0/12,.eks.amazonaws.com,.s3.amazonaws.com,.amazonaws.com,.gcr.io,.pkg.dev
+      - terragrunt run-all apply --terragrunt-non-interactive -auto-approve
+
+  post_build:
+    commands:
+      - echo "Terragrunt apply completed on `date`"
+
+artifacts:
+  files:
+    - '**/*'
+  base-directory: '.'
diff --git a/buildspecs/security.yml b/buildspecs/security.yml
@@ -0,0 +1,40 @@
+version: 0.2
+
+env:
+  variables:
+    TOOLS_DIR: "/tmp/build-tools"
+
+cache:
+  paths:
+    - '/tmp/build-tools/**/*'
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.9
+    commands:
+      - echo "Setting up security scanning tools"
+      - export http_proxy=$PROXY_CONFIG
+      - export https_proxy=$PROXY_CONFIG
+      - export NO_PROXY=.census.gov,169.254.169.254,148.129.0.0/16,10.0.0.0/8,172.16.0/12,.eks.amazonaws.com,.s3.amazonaws.com,.amazonaws.com,.gcr.io,.pkg.dev
+
+      # Install security scanning tools
+      - mkdir -p $TOOLS_DIR/bin
+      - pip install checkov -q
+      - pip install tfsec -q
+
+  build:
+    commands:
+      - echo "Running security scans"
+      - checkov --directory . --framework terraform --quiet --compact
+      - checkov --directory . --framework terragrunt --quiet --compact
+      - tfsec . --no-color
+
+  post_build:
+    commands:
+      - echo "Security scan completed on `date`"
+
+artifacts:
+  files:
+    - '**/*'
+  base-directory: '.'