From d982195c3c74a913db1240c536b4697bf7ed293d Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 9 May 2025 22:53:35 -0400 Subject: [PATCH] refactor pipeline stuff --- Makefile | 27 +++++---- buildspecs/terragrunt.yml | 56 ++++++++++++------- .../eks-pipeline/terragrunt.hcl | 27 ++++----- 3 files changed, 62 insertions(+), 48 deletions(-) diff --git a/Makefile b/Makefile index c55b7d5..52d1e72 100644 --- a/Makefile +++ b/Makefile @@ -38,21 +38,21 @@ clean: deploy-to-pipeline: @echo "Preparing to deploy to pipeline..." @echo "Detecting environment configuration..." - + # Set defaults or use provided values $(eval ENV ?= development) $(eval REGION_DIR ?= us-gov-east-1) $(eval CLUSTER_DIR ?= csvd-platform-lab-mcm) - + # Detect account variables $(eval ACCOUNT_HCL=lab/$(ENV)/account.hcl) $(eval REGION_HCL=lab/$(ENV)/$(REGION_DIR)/region.hcl) $(eval CLUSTER_HCL=lab/$(ENV)/$(REGION_DIR)/vpc/$(CLUSTER_DIR)/cluster.hcl) - + @if [ ! -f "$(ACCOUNT_HCL)" ]; then echo "Error: $(ACCOUNT_HCL) not found"; exit 1; fi @if [ ! -f "$(REGION_HCL)" ]; then echo "Error: $(REGION_HCL) not found"; exit 1; fi @if [ ! -f "$(CLUSTER_HCL)" ]; then echo "Error: $(CLUSTER_HCL) not found"; exit 1; fi - + @echo "Extracting configuration values..." # Extract values from HCL files $(eval AWS_ACCOUNT_ID=$(shell grep -oP 'aws_account_id\s*=\s*"\K[^"]+' $(ACCOUNT_HCL))) @@ -60,31 +60,36 @@ deploy-to-pipeline: $(eval AWS_PROFILE=$(shell echo $(AWS_ACCOUNT_ID)-$(shell echo $(ACCOUNT_NAME) | sed 's/-ew/-gov/'))) $(eval AWS_REGION=$(shell grep -oP 'aws_region\s*=\s*"\K[^"]+' $(REGION_HCL))) $(eval CLUSTER_NAME=$(shell grep -oP 'cluster_name\s*=\s*"\K[^"]+' $(CLUSTER_HCL))) - + @echo "Using configuration:" @echo " AWS_ACCOUNT_ID: $(AWS_ACCOUNT_ID)" @echo " ACCOUNT_NAME: $(ACCOUNT_NAME)" @echo " AWS_PROFILE: $(AWS_PROFILE)" @echo " AWS_REGION: $(AWS_REGION)" @echo " CLUSTER_NAME: $(CLUSTER_NAME)" - + @if [ -z "$(AWS_ACCOUNT_ID)" ] || [ -z "$(AWS_PROFILE)" ] || [ -z "$(AWS_REGION)" ] || [ -z "$(CLUSTER_NAME)" ]; then \ echo "Error: Failed to extract all required variables from HCL files"; \ exit 1; \ fi - + @echo "Creating zip file..." zip -r platform-tg-infra.zip . -x "*.git*" "*.github*" "*.terragrunt-cache*" "*.terraform*" - + @echo "Calculating S3 bucket name..." $(eval REGION_SHORT=$(shell echo $(AWS_REGION) | sed 's/\([a-z]\)[a-z]*-/\1/g')) $(eval S3_BUCKET=v-s3-eks-$(CLUSTER_NAME)-artifacts-$(AWS_ACCOUNT_ID)-$(REGION_SHORT)) $(eval OBJECT_KEY=clusters/$(CLUSTER_NAME)/platform-tg-infra.zip) - + @echo "Uploading to S3 bucket $(S3_BUCKET)..." aws s3 cp platform-tg-infra.zip s3://$(S3_BUCKET)/$(OBJECT_KEY) --profile $(AWS_PROFILE) @echo "Upload complete. Pipeline should trigger automatically." - @echo "Check the AWS CodePipeline console for status." - + + @echo "Calculating pipeline URL..." + $(eval PIPELINE_NAME=eks-$(CLUSTER_NAME)-pipeline) + $(eval PIPELINE_URL=https://console.amazonaws-us-gov.com/codesuite/codepipeline/pipelines/$(PIPELINE_NAME)/view?region=$(AWS_REGION)) + @echo "Pipeline URL: $(PIPELINE_URL)" + @echo "You can access the pipeline directly at the URL above." + @echo "Cleaning up local zip file..." rm -f platform-tg-infra.zip diff --git a/buildspecs/terragrunt.yml b/buildspecs/terragrunt.yml index d73d383..57dae77 100644 --- a/buildspecs/terragrunt.yml +++ b/buildspecs/terragrunt.yml @@ -3,9 +3,14 @@ version: 0.2 env: variables: BASE_DIR: "lab" - TF_VERSION: "1.5.5" - TG_VERSION: "0.72.0" - TOOLS_DIR: "/tmp/build-tools" + TOOLS_DIR: "/tmp/build-tools/" + TERRAGRUNT_PATH: "${TERRAGRUNT_PATH}" + ARTIFACTS_BUCKET: "${ARTIFACTS_BUCKET}" + PROXY_CONFIG: "${PROXY_CONFIG}" + + secrets-manager: + GITHUB_TOKEN: ${GITHUB_TOKEN_ARN} + exported-variables: - TERRAGRUNT_PATH @@ -23,44 +28,53 @@ phases: - export https_proxy=$PROXY_CONFIG - export NO_PROXY=.census.gov,169.254.169.254,148.129.0.0/16,10.0.0.0/8,172.16.0/12,.eks.amazonaws.com,.s3.amazonaws.com,.amazonaws.com,.gcr.io,.pkg.dev - # Create tools directory if it doesn't exist + # Configure Git to use the token from Secrets Manager + - echo "Configuring git with GitHub authentication" + - git config --global url."https://x-access-token:${GITHUB_TOKEN}@github.e.it.census.gov/".insteadOf "https://github.e.it.census.gov/" + - echo "Successfully configured git with GitHub token from Secrets Manager" + + # Create tools directory if it doesn't exist - mkdir -p $TOOLS_DIR/bin - # Check if cached Terraform exists and matches required version + # Get tools from S3 artifacts bucket instead of downloading from internet - | - if [ -f "$TOOLS_DIR/bin/terraform" ] && [ "$($TOOLS_DIR/bin/terraform version | head -n1 | grep -o "v$TF_VERSION")" = "v$TF_VERSION" ]; then - echo "Using cached Terraform v$TF_VERSION" - else - echo "Downloading Terraform v$TF_VERSION" - curl -Lo /tmp/terraform.zip "https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_linux_amd64.zip" - unzip -o /tmp/terraform.zip -d $TOOLS_DIR/bin/ + # Terraform + if [ ! -f "$TOOLS_DIR/bin/terraform" ]; then + echo "Copying Terraform from S3 artifacts bucket" + if ! aws s3 cp s3://${ARTIFACTS_BUCKET}/tools/terraform.zip $TOOLS_DIR; then + echo "Failed to download Terraform" + exit 1 + fi + unzip -o $TOOLS_DIR/terraform.zip -d $TOOLS_DIR/bin/ chmod +x $TOOLS_DIR/bin/terraform fi - # Check if cached Terragrunt exists and matches required version - - | - if [ -f "$TOOLS_DIR/bin/terragrunt" ] && [ "$($TOOLS_DIR/bin/terragrunt --version | grep -o "v$TG_VERSION")" = "v$TG_VERSION" ]; then - echo "Using cached Terragrunt v$TG_VERSION" - else - echo "Downloading Terragrunt v$TG_VERSION" - curl -Lo $TOOLS_DIR/bin/terragrunt "https://github.com/gruntwork-io/terragrunt/releases/download/v${TG_VERSION}/terragrunt_linux_amd64" + # Terragrunt + if [ ! -f "$TOOLS_DIR/bin/terragrunt" ]; then + echo "Copying Terragrunt from S3 artifacts bucket" + if ! aws s3 cp s3://${ARTIFACTS_BUCKET}/tools/terragrunt $TOOLS_DIR; then + echo "Failed to download Terragrunt" + exit 1 + fi + mv $TOOLS_DIR/terragrunt $TOOLS_DIR/bin/ chmod +x $TOOLS_DIR/bin/terragrunt fi # Add tools to PATH - export PATH=$TOOLS_DIR/bin:$PATH + - aws sts get-caller-identity - terraform --version - terragrunt --version - - aws sts get-caller-identity build: commands: - - echo "Running Terragrunt plan" + - echo "Running Terragrunt plan with assumed role profile" - cd $TERRAGRUNT_PATH - export http_proxy=$PROXY_CONFIG - export https_proxy=$PROXY_CONFIG - export NO_PROXY=.census.gov,169.254.169.254,148.129.0.0/16,10.0.0.0/8,172.16.0/12,.eks.amazonaws.com,.s3.amazonaws.com,.amazonaws.com,.gcr.io,.pkg.dev - - terragrunt run-all plan --terragrunt-non-interactive + + - terragrunt run-all plan --terragrunt-non-interactive --terragrunt-debug --terragrunt-log-level debug post_build: commands: diff --git a/lab/development/us-gov-east-1/vpc/csvd-platform-lab-mcm/eks-pipeline/terragrunt.hcl b/lab/development/us-gov-east-1/vpc/csvd-platform-lab-mcm/eks-pipeline/terragrunt.hcl index 85a2765..4aef6e1 100644 --- a/lab/development/us-gov-east-1/vpc/csvd-platform-lab-mcm/eks-pipeline/terragrunt.hcl +++ b/lab/development/us-gov-east-1/vpc/csvd-platform-lab-mcm/eks-pipeline/terragrunt.hcl @@ -7,6 +7,10 @@ include "root" { locals { # Skip this module if disabled skip = !lookup(include.root.locals.is_module_enabled, basename(get_terragrunt_dir()), true) + artifact_bucket = format("v-s3-eks-%v-artifacts-%v-%v", + include.root.inputs.cluster_name, + include.root.inputs.aws_account_id, + join("", [for c in split("-", include.root.inputs.aws_region) : substr(c, 0, 1)])) } exclude { @@ -34,23 +38,12 @@ inputs = { subnet_filter = "*-container-*" # or any specific pattern you want to use # Pipeline specific configurations - name = format("%v-pipeline", include.root.inputs.cluster_name) - - # The bucket name must match exactly what's created in the GitHub Action - source_configuration = { - provider = "S3" - s3_config = { - bucket = format("v-s3-eks-%v-artifacts-%v-%v", - include.root.inputs.cluster_name, - include.root.inputs.aws_account_id, - join("", [for c in split("-", include.root.inputs.aws_region) : substr(c, 0, 1)])) - object_key = format("clusters/%v/platform-tg-infra.zip", include.root.inputs.cluster_name) - } - } + s3_trigger_object_prefix = format("clusters/%v/", include.root.inputs.cluster_name) is_infrastructure_pipeline = true # Updated to use buildspecs from the platform-tg-infra repository + # made deploy-to-pipeline will update them from tfmod-pipeline module buildspec_template_path = "buildspecs" build_configuration = { @@ -59,11 +52,12 @@ inputs = { buildspec_path = "terragrunt.yml" privileged_mode = true environment_variables = { - TERRAGRUNT_PATH = "lab/development/${include.root.inputs.aws_region}/vpc/${include.root.inputs.cluster_name}" + ARTIFACT_BUCKET = local.artifact_bucket + TERRAGRUNT_PATH = "lab/${include.root.inputs.environment}/${include.root.inputs.aws_region}/vpc/${include.root.inputs.cluster_name}" REGION = include.root.inputs.aws_region ENVIRONMENT = include.root.inputs.environment_abbr AWS_ACCOUNT_ID = include.root.inputs.aws_account_id - PROXY_CONFIG = "http://proxy.tco.census.gov:3128" + PROXY_CONFIG = "http://vlab-proxy.tco.census.gov:3128" } } @@ -85,11 +79,12 @@ inputs = { image = "aws/codebuild/amazonlinux2-x86_64-standard:3.0" buildspec_path = "deploy.terragrunt.yml" environment_variables = { + ARTIFACT_BUCKET = local.artifact_bucket TERRAGRUNT_PATH = "lab/${include.root.inputs.environment}/${include.root.inputs.aws_region}/vpc/${include.root.inputs.cluster_name}" REGION = include.root.inputs.aws_region ENVIRONMENT = include.root.inputs.environment_abbr AWS_ACCOUNT_ID = include.root.inputs.aws_account_id - PROXY_CONFIG = "http://proxy.tco.census.gov:3128" + PROXY_CONFIG = "http://vlab-proxy.tco.census.gov:3128" } } }