refactor pipeline stuff

SCT-Engineering · May 10, 2025 · d982195 · d982195
1 parent 1bb45bb
commit d982195
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 48 deletions.
diff --git a/Makefile b/Makefile
@@ -38,53 +38,58 @@ clean:
 deploy-to-pipeline:
 	@echo "Preparing to deploy to pipeline..."
 	@echo "Detecting environment configuration..."
-	
+
 	# Set defaults or use provided values
 	$(eval ENV ?= development)
 	$(eval REGION_DIR ?= us-gov-east-1)
 	$(eval CLUSTER_DIR ?= csvd-platform-lab-mcm)
-	
+
 	# Detect account variables
 	$(eval ACCOUNT_HCL=lab/$(ENV)/account.hcl)
 	$(eval REGION_HCL=lab/$(ENV)/$(REGION_DIR)/region.hcl)
 	$(eval CLUSTER_HCL=lab/$(ENV)/$(REGION_DIR)/vpc/$(CLUSTER_DIR)/cluster.hcl)
-	
+
 	@if [ ! -f "$(ACCOUNT_HCL)" ]; then echo "Error: $(ACCOUNT_HCL) not found"; exit 1; fi
 	@if [ ! -f "$(REGION_HCL)" ]; then echo "Error: $(REGION_HCL) not found"; exit 1; fi
 	@if [ ! -f "$(CLUSTER_HCL)" ]; then echo "Error: $(CLUSTER_HCL) not found"; exit 1; fi
-	
+
 	@echo "Extracting configuration values..."
 	# Extract values from HCL files
 	$(eval AWS_ACCOUNT_ID=$(shell grep -oP 'aws_account_id\s*=\s*"\K[^"]+' $(ACCOUNT_HCL)))
 	$(eval ACCOUNT_NAME=$(shell grep -oP 'account_name\s*=\s*"\K[^"]+' $(ACCOUNT_HCL)))
 	$(eval AWS_PROFILE=$(shell echo $(AWS_ACCOUNT_ID)-$(shell echo $(ACCOUNT_NAME) | sed 's/-ew/-gov/')))
 	$(eval AWS_REGION=$(shell grep -oP 'aws_region\s*=\s*"\K[^"]+' $(REGION_HCL)))
 	$(eval CLUSTER_NAME=$(shell grep -oP 'cluster_name\s*=\s*"\K[^"]+' $(CLUSTER_HCL)))
-	
+
 	@echo "Using configuration:"
 	@echo "  AWS_ACCOUNT_ID: $(AWS_ACCOUNT_ID)"
 	@echo "  ACCOUNT_NAME: $(ACCOUNT_NAME)"
 	@echo "  AWS_PROFILE: $(AWS_PROFILE)"
 	@echo "  AWS_REGION: $(AWS_REGION)"
 	@echo "  CLUSTER_NAME: $(CLUSTER_NAME)"
-	
+
 	@if [ -z "$(AWS_ACCOUNT_ID)" ] || [ -z "$(AWS_PROFILE)" ] || [ -z "$(AWS_REGION)" ] || [ -z "$(CLUSTER_NAME)" ]; then \
 		echo "Error: Failed to extract all required variables from HCL files"; \
 		exit 1; \
 	fi
-	
+
 	@echo "Creating zip file..."
 	zip -r platform-tg-infra.zip . -x "*.git*" "*.github*" "*.terragrunt-cache*" "*.terraform*"
-	
+
 	@echo "Calculating S3 bucket name..."
 	$(eval REGION_SHORT=$(shell echo $(AWS_REGION) | sed 's/\([a-z]\)[a-z]*-/\1/g'))
 	$(eval S3_BUCKET=v-s3-eks-$(CLUSTER_NAME)-artifacts-$(AWS_ACCOUNT_ID)-$(REGION_SHORT))
 	$(eval OBJECT_KEY=clusters/$(CLUSTER_NAME)/platform-tg-infra.zip)
-	
+
 	@echo "Uploading to S3 bucket $(S3_BUCKET)..."
 	aws s3 cp platform-tg-infra.zip s3://$(S3_BUCKET)/$(OBJECT_KEY) --profile $(AWS_PROFILE)
 	@echo "Upload complete. Pipeline should trigger automatically."
-	@echo "Check the AWS CodePipeline console for status."
-
+
+	@echo "Calculating pipeline URL..."
+	$(eval PIPELINE_NAME=eks-$(CLUSTER_NAME)-pipeline)
+	$(eval PIPELINE_URL=https://console.amazonaws-us-gov.com/codesuite/codepipeline/pipelines/$(PIPELINE_NAME)/view?region=$(AWS_REGION))
+	@echo "Pipeline URL: $(PIPELINE_URL)"
+	@echo "You can access the pipeline directly at the URL above."
+
 	@echo "Cleaning up local zip file..."
 	rm -f platform-tg-infra.zip
diff --git a/buildspecs/terragrunt.yml b/buildspecs/terragrunt.yml
@@ -3,9 +3,14 @@ version: 0.2
 env:
   variables:
     BASE_DIR: "lab"
-    TF_VERSION: "1.5.5"
-    TG_VERSION: "0.72.0"
-    TOOLS_DIR: "/tmp/build-tools"
+    TOOLS_DIR: "/tmp/build-tools/"
+    TERRAGRUNT_PATH: "${TERRAGRUNT_PATH}"
+    ARTIFACTS_BUCKET: "${ARTIFACTS_BUCKET}"
+    PROXY_CONFIG: "${PROXY_CONFIG}"
+
+  secrets-manager:
+    GITHUB_TOKEN: ${GITHUB_TOKEN_ARN}
+
   exported-variables:
     - TERRAGRUNT_PATH
 
@@ -23,44 +28,53 @@ phases:
       - export https_proxy=$PROXY_CONFIG
       - export NO_PROXY=.census.gov,169.254.169.254,148.129.0.0/16,10.0.0.0/8,172.16.0/12,.eks.amazonaws.com,.s3.amazonaws.com,.amazonaws.com,.gcr.io,.pkg.dev
 
-        # Create tools directory if it doesn't exist
+      # Configure Git to use the token from Secrets Manager
+      - echo "Configuring git with GitHub authentication"
+      - git config --global url."https://x-access-token:${GITHUB_TOKEN}@github.e.it.census.gov/".insteadOf "https://github.e.it.census.gov/"
+      - echo "Successfully configured git with GitHub token from Secrets Manager"
+
+      # Create tools directory if it doesn't exist
       - mkdir -p $TOOLS_DIR/bin
 
-        # Check if cached Terraform exists and matches required version
+      # Get tools from S3 artifacts bucket instead of downloading from internet
       - |
-        if [ -f "$TOOLS_DIR/bin/terraform" ] && [ "$($TOOLS_DIR/bin/terraform version | head -n1 | grep -o "v$TF_VERSION")" = "v$TF_VERSION" ]; then
-          echo "Using cached Terraform v$TF_VERSION"
-        else
-          echo "Downloading Terraform v$TF_VERSION"
-          curl -Lo /tmp/terraform.zip "https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_linux_amd64.zip"
-          unzip -o /tmp/terraform.zip -d $TOOLS_DIR/bin/
+        # Terraform
+        if [ ! -f "$TOOLS_DIR/bin/terraform" ]; then
+          echo "Copying Terraform from S3 artifacts bucket"
+          if ! aws s3 cp s3://${ARTIFACTS_BUCKET}/tools/terraform.zip $TOOLS_DIR; then
+            echo "Failed to download Terraform"
+            exit 1
+          fi
+          unzip -o $TOOLS_DIR/terraform.zip -d $TOOLS_DIR/bin/
           chmod +x $TOOLS_DIR/bin/terraform
         fi
 
-        # Check if cached Terragrunt exists and matches required version
-      - |
-        if [ -f "$TOOLS_DIR/bin/terragrunt" ] && [ "$($TOOLS_DIR/bin/terragrunt --version | grep -o "v$TG_VERSION")" = "v$TG_VERSION" ]; then
-          echo "Using cached Terragrunt v$TG_VERSION"
-        else
-          echo "Downloading Terragrunt v$TG_VERSION"
-          curl -Lo $TOOLS_DIR/bin/terragrunt "https://github.com/gruntwork-io/terragrunt/releases/download/v${TG_VERSION}/terragrunt_linux_amd64"
+        # Terragrunt
+        if [ ! -f "$TOOLS_DIR/bin/terragrunt" ]; then
+          echo "Copying Terragrunt from S3 artifacts bucket"
+          if ! aws s3 cp s3://${ARTIFACTS_BUCKET}/tools/terragrunt $TOOLS_DIR; then
+            echo "Failed to download Terragrunt"
+            exit 1
+          fi
+          mv $TOOLS_DIR/terragrunt $TOOLS_DIR/bin/
           chmod +x $TOOLS_DIR/bin/terragrunt
         fi
 
         # Add tools to PATH
       - export PATH=$TOOLS_DIR/bin:$PATH
+      - aws sts get-caller-identity
       - terraform --version
       - terragrunt --version
-      - aws sts get-caller-identity
 
   build:
     commands:
-      - echo "Running Terragrunt plan"
+      - echo "Running Terragrunt plan with assumed role profile"
       - cd $TERRAGRUNT_PATH
       - export http_proxy=$PROXY_CONFIG
       - export https_proxy=$PROXY_CONFIG
       - export NO_PROXY=.census.gov,169.254.169.254,148.129.0.0/16,10.0.0.0/8,172.16.0/12,.eks.amazonaws.com,.s3.amazonaws.com,.amazonaws.com,.gcr.io,.pkg.dev
-      - terragrunt run-all plan --terragrunt-non-interactive
+
+      - terragrunt run-all plan --terragrunt-non-interactive --terragrunt-debug --terragrunt-log-level debug
 
   post_build:
     commands:

diff --git a/lab/development/us-gov-east-1/vpc/csvd-platform-lab-mcm/eks-pipeline/terragrunt.hcl b/lab/development/us-gov-east-1/vpc/csvd-platform-lab-mcm/eks-pipeline/terragrunt.hcl
@@ -7,6 +7,10 @@ include "root" {
 locals {
   # Skip this module if disabled
   skip = !lookup(include.root.locals.is_module_enabled, basename(get_terragrunt_dir()), true)
+  artifact_bucket = format("v-s3-eks-%v-artifacts-%v-%v", 
+                         include.root.inputs.cluster_name, 
+                         include.root.inputs.aws_account_id,
+                         join("", [for c in split("-", include.root.inputs.aws_region) : substr(c, 0, 1)]))
 }
 
 exclude {
@@ -34,23 +38,12 @@ inputs = {
   subnet_filter = "*-container-*"  # or any specific pattern you want to use
 
   # Pipeline specific configurations
-  name = format("%v-pipeline", include.root.inputs.cluster_name)
-
-  # The bucket name must match exactly what's created in the GitHub Action
-  source_configuration = {
-    provider = "S3"
-    s3_config = {
-      bucket     = format("v-s3-eks-%v-artifacts-%v-%v", 
-                         include.root.inputs.cluster_name, 
-                         include.root.inputs.aws_account_id,
-                         join("", [for c in split("-", include.root.inputs.aws_region) : substr(c, 0, 1)]))
-      object_key = format("clusters/%v/platform-tg-infra.zip", include.root.inputs.cluster_name)
-    }
-  }
+  s3_trigger_object_prefix = format("clusters/%v/", include.root.inputs.cluster_name)
 
   is_infrastructure_pipeline = true
 
   # Updated to use buildspecs from the platform-tg-infra repository
+  # made deploy-to-pipeline will update them from tfmod-pipeline module
   buildspec_template_path = "buildspecs"
 
   build_configuration = {
@@ -59,11 +52,12 @@ inputs = {
     buildspec_path  = "terragrunt.yml"
     privileged_mode = true
     environment_variables = {
-      TERRAGRUNT_PATH = "lab/development/${include.root.inputs.aws_region}/vpc/${include.root.inputs.cluster_name}"
+      ARTIFACT_BUCKET = local.artifact_bucket
+      TERRAGRUNT_PATH = "lab/${include.root.inputs.environment}/${include.root.inputs.aws_region}/vpc/${include.root.inputs.cluster_name}"
       REGION          = include.root.inputs.aws_region
       ENVIRONMENT     = include.root.inputs.environment_abbr
       AWS_ACCOUNT_ID  = include.root.inputs.aws_account_id
-      PROXY_CONFIG    = "http://proxy.tco.census.gov:3128"
+      PROXY_CONFIG    = "http://vlab-proxy.tco.census.gov:3128"
     }
   }
 
@@ -85,11 +79,12 @@ inputs = {
     image          = "aws/codebuild/amazonlinux2-x86_64-standard:3.0"
     buildspec_path = "deploy.terragrunt.yml"
     environment_variables = {
+      ARTIFACT_BUCKET = local.artifact_bucket
       TERRAGRUNT_PATH = "lab/${include.root.inputs.environment}/${include.root.inputs.aws_region}/vpc/${include.root.inputs.cluster_name}"
       REGION          = include.root.inputs.aws_region
       ENVIRONMENT     = include.root.inputs.environment_abbr
       AWS_ACCOUNT_ID  = include.root.inputs.aws_account_id
-      PROXY_CONFIG    = "http://proxy.tco.census.gov:3128"
+      PROXY_CONFIG    = "http://vlab-proxy.tco.census.gov:3128"
     }
   }
 }