From 972d53f8d592ed671a47c7dab46bc984fc4ecab2 Mon Sep 17 00:00:00 2001 From: arnol377 Date: Tue, 15 Apr 2025 16:05:12 -0400 Subject: [PATCH] updating terraform based deployment --- .github/workflows/gh-token.yaml | 19 +++ .gitignore | 3 + Makefile | 42 ++++++ backend.tf | 0 main.tf | 227 ++++++++++++++++++++++++++++++++ outputs.tf | 10 ++ samconfig.toml | 9 -- scripts/package.sh | 24 ++++ scripts/pip.conf | 10 ++ scripts/requirements.txt | 5 + template.yaml | 115 ---------------- varfiles/default.json | 1 + varfiles/default.tfvars | 0 varfiles/sct-engineering.json | 1 + varfiles/sct-engineering.tfvars | 0 variables.tf | 53 ++++++++ versions.tf | 10 ++ 17 files changed, 405 insertions(+), 124 deletions(-) create mode 100644 .github/workflows/gh-token.yaml create mode 100644 Makefile create mode 100644 backend.tf create mode 100644 main.tf create mode 100644 outputs.tf delete mode 100644 samconfig.toml create mode 100644 scripts/package.sh create mode 100644 scripts/pip.conf create mode 100644 scripts/requirements.txt delete mode 100644 template.yaml create mode 100644 varfiles/default.json create mode 100644 varfiles/default.tfvars create mode 100644 varfiles/sct-engineering.json create mode 100644 varfiles/sct-engineering.tfvars create mode 100644 variables.tf create mode 100644 versions.tf diff --git a/.github/workflows/gh-token.yaml b/.github/workflows/gh-token.yaml new file mode 100644 index 0000000..8f668b8 --- /dev/null +++ b/.github/workflows/gh-token.yaml @@ -0,0 +1,19 @@ +name: GitHub Token Refresh + +on: + schedule: + - cron: '*/5 * * * *' # Runs every 5 minutes + workflow_dispatch: # Allows manual triggering + +jobs: + refresh-token: + name: Refresh GitHub Token + uses: CSVD/centralized-actions/.github/workflows/upload-github-token.yml@main + with: + aws_region: 'us-east-1' + secret_name: '/dev/eks_automation_github_token' # This matches the SECRET_NAME in app.py + github_app_id: ${{ vars.GITHUB_APP_ID }} + github_installation_id: ${{ vars.GITHUB_INSTALLATION_ID }} + use_ecs_credentials: true + secrets: + github_app_private_key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} diff --git a/.gitignore b/.gitignore index 2be8f1d..a7ea870 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,6 @@ cython_debug/ artifacts/ .aws-sam + +terraform_data_dirs +terraform.tfstate.d \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1b8e3b9 --- /dev/null +++ b/Makefile @@ -0,0 +1,42 @@ +.PHONY: init venv install-deps clean terraform-init terraform-plan terraform-apply package all + +PYTHON=python3 +PIP=pip3 +TERRAFORM=terraform +VENV=.venv +VENV_BIN=$(VENV)/bin +VENV_PIP=$(VENV_BIN)/pip + +# Set PIP_CONFIG_FILE to use custom pip.conf +export PIP_CONFIG_FILE=$(CURDIR)/scripts/pip.conf + +all: venv install-deps package terraform-apply + +venv: + test -d $(VENV) || $(PYTHON) -m venv $(VENV) + $(VENV_PIP) install --upgrade pip setuptools wheel + +install-deps: venv + source $(VENV_BIN)/activate && $(VENV_PIP) install -r scripts/requirements.txt + +clean: + rm -rf dist/ + rm -f *.zip + rm -rf __pycache__/ + rm -rf .terraform/ + rm -rf $(VENV) + +package: venv + source $(VENV_BIN)/activate && chmod +x scripts/package.sh && ./scripts/package.sh + +terraform-init: + $(TERRAFORM) init + +terraform-plan: terraform-init + $(TERRAFORM) plan + +terraform-apply: terraform-init package + $(TERRAFORM) apply -auto-approve + +terraform-destroy: + $(TERRAFORM) destroy -auto-approve diff --git a/backend.tf b/backend.tf new file mode 100644 index 0000000..e69de29 diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..2ff4158 --- /dev/null +++ b/main.tf @@ -0,0 +1,227 @@ +provider "aws" { + default_tags = { + organization = "census:ocio:csvd" + finops_project_name = "csvd_platformbaseline" + finops_project_number = "fs0000000078" + finops_project_role = "csvd_platformbaseline_app" + } +} + +locals { + common_tags = { + environment = var.environment + environment_abbr = var.environment_abbr + organization = var.organization + finops_project_name = var.finops_project_name + finops_project_number = var.finops_project_number + finops_project_role = var.finops_project_role + } +} + +# API Gateway +resource "aws_api_gateway_rest_api" "eks_automation" { + name = "eks-automation-api" + tags = local.common_tags +} + +resource "aws_api_gateway_resource" "eks_automation" { + rest_api_id = aws_api_gateway_rest_api.eks_automation.id + parent_id = aws_api_gateway_rest_api.eks_automation.root_resource_id + path_part = "EKSAutomation" +} + +resource "aws_api_gateway_method" "eks_automation" { + rest_api_id = aws_api_gateway_rest_api.eks_automation.id + resource_id = aws_api_gateway_resource.eks_automation.id + http_method = "POST" + authorization = "NONE" + api_key_required = true +} + +resource "aws_api_gateway_integration" "lambda" { + rest_api_id = aws_api_gateway_rest_api.eks_automation.id + resource_id = aws_api_gateway_resource.eks_automation.id + http_method = aws_api_gateway_method.eks_automation.http_method + integration_http_method = "POST" + type = "AWS_PROXY" + uri = aws_lambda_function.eks_automation.invoke_arn +} + +resource "aws_api_gateway_deployment" "eks_automation" { + rest_api_id = aws_api_gateway_rest_api.eks_automation.id + depends_on = [aws_api_gateway_integration.lambda] +} + +resource "aws_api_gateway_stage" "prod" { + deployment_id = aws_api_gateway_deployment.eks_automation.id + rest_api_id = aws_api_gateway_rest_api.eks_automation.id + stage_name = "Prod" +} + +resource "aws_lambda_permission" "apigw" { + statement_id = "AllowAPIGatewayInvoke" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.eks_automation.function_name + principal = "apigateway.amazonaws.com" + source_arn = "${aws_api_gateway_rest_api.eks_automation.execution_arn}/*/*" +} + +resource "aws_api_gateway_method" "options" { + rest_api_id = aws_api_gateway_rest_api.eks_automation.id + resource_id = aws_api_gateway_resource.eks_automation.id + http_method = "OPTIONS" + authorization = "NONE" +} + +resource "aws_api_gateway_integration" "options" { + rest_api_id = aws_api_gateway_rest_api.eks_automation.id + resource_id = aws_api_gateway_resource.eks_automation.id + http_method = aws_api_gateway_method.options.http_method + type = "MOCK" + request_templates = { + "application/json" = "{\"statusCode\": 200}" + } +} + +resource "aws_api_gateway_method_response" "options" { + rest_api_id = aws_api_gateway_rest_api.eks_automation.id + resource_id = aws_api_gateway_resource.eks_automation.id + http_method = aws_api_gateway_method.options.http_method + status_code = "200" + + response_parameters = { + "method.response.header.Access-Control-Allow-Headers" = true, + "method.response.header.Access-Control-Allow-Methods" = true, + "method.response.header.Access-Control-Allow-Origin" = true + } +} + +resource "aws_api_gateway_integration_response" "options" { + rest_api_id = aws_api_gateway_rest_api.eks_automation.id + resource_id = aws_api_gateway_resource.eks_automation.id + http_method = aws_api_gateway_method.options.http_method + status_code = aws_api_gateway_method_response.options.status_code + + response_parameters = { + "method.response.header.Access-Control-Allow-Headers" = "'Content-Type,Authorization'", + "method.response.header.Access-Control-Allow-Methods" = "'POST,OPTIONS'", + "method.response.header.Access-Control-Allow-Origin" = "'*'" + } +} + +resource "aws_api_gateway_usage_plan" "eks_automation" { + name = "eks-automation-usage-plan" + description = "Usage plan for EKS Automation API" + + api_stages { + api_id = aws_api_gateway_rest_api.eks_automation.id + stage = aws_api_gateway_stage.prod.stage_name + } + + quota_settings { + limit = 5000 + period = "MONTH" + } + + throttle_settings { + burst_limit = 500 + rate_limit = 100 + } + + tags = local.common_tags +} + +resource "aws_api_gateway_api_key" "eks_automation" { + name = "eks-automation-api-key" +} + +resource "aws_api_gateway_usage_plan_key" "eks_automation" { + key_id = aws_api_gateway_api_key.eks_automation.id + key_type = "API_KEY" + usage_plan_id = aws_api_gateway_usage_plan.eks_automation.id +} + +# Lambda Layer +resource "aws_lambda_layer_version" "git" { + filename = "layer.zip" # Make sure to create this zip file with Git binaries + layer_name = "git-lambda-layer" + description = "Git Lambda Layer" + compatible_runtimes = ["python3.9", "python3.10", "python3.11"] +} + +# Lambda Function +resource "aws_lambda_function" "eks_automation" { + filename = "eks_automation.zip" # Make sure to create this zip file + function_name = "eks-automation" + role = aws_iam_role.lambda_role.arn + handler = "app.lambda_handler" + runtime = "python3.11" + timeout = var.lambda_timeout + + vpc_config { + subnet_ids = var.vpc_subnet_ids + security_group_ids = var.vpc_security_group_ids + } + + layers = [aws_lambda_layer_version.git.arn] + + environment { + variables = { + ENVIRONMENT = var.environment + } + } + + tags = local.common_tags +} + +# IAM Role for Lambda +resource "aws_iam_role" "lambda_role" { + name = "eks-automation-lambda-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) + + tags = local.common_tags +} + +# IAM Policies +resource "aws_iam_role_policy_attachment" "lambda_vpc_access" { + role = aws_iam_role.lambda_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +resource "aws_iam_role_policy" "lambda_ssm_access" { + name = "eks-automation-ssm-access" + role = aws_iam_role.lambda_role.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "SSMDescribeParametersPolicy" + Effect = "Allow" + Action = ["ssm:DescribeParameters"] + Resource = "*" + }, + { + Sid = "SSMGetParameterPolicy" + Effect = "Allow" + Action = [ + "ssm:GetParameters", + "ssm:GetParameter" + ] + Resource = "*" + } + ] + }) +} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..b5ab3ff --- /dev/null +++ b/outputs.tf @@ -0,0 +1,10 @@ +output "api_endpoint" { + description = "API Gateway endpoint URL" + value = "${aws_api_gateway_stage.prod.invoke_url}${aws_api_gateway_resource.eks_automation.path}" +} + +output "api_key" { + description = "API Key for accessing the endpoint" + value = aws_api_gateway_api_key.eks_automation.value + sensitive = true +} diff --git a/samconfig.toml b/samconfig.toml deleted file mode 100644 index a86dd99..0000000 --- a/samconfig.toml +++ /dev/null @@ -1,9 +0,0 @@ -version = 0.1 -[default.deploy.parameters] -stack_name = "eks-automation-lambda" -s3_bucket = "eks-automation-lambda-s3-bucket" -region = "us-gov-east-1" -profile = "229685449397-csvd-dev-gov" -confirm_changeset = false -capabilities = "CAPABILITY_IAM" -image_repositories = [] diff --git a/scripts/package.sh b/scripts/package.sh new file mode 100644 index 0000000..d0e462b --- /dev/null +++ b/scripts/package.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Create temporary directories +mkdir -p dist/lambda +mkdir -p dist/layer + +# Package Lambda function +cp -r eks_automation/* dist/lambda/ +cd dist/lambda +zip -r ../eks_automation.zip . +cd ../.. + +# Package Lambda layer +mkdir -p dist/layer/python +pip install -r eks_automation/requirements.txt -t dist/layer/python +cd dist/layer +zip -r ../layer.zip . +cd ../.. + +# Move zip files to root +mv dist/*.zip . + +# Cleanup +rm -rf dist diff --git a/scripts/pip.conf b/scripts/pip.conf new file mode 100644 index 0000000..3c3b843 --- /dev/null +++ b/scripts/pip.conf @@ -0,0 +1,10 @@ +[global] +cert = /etc/pki/ca-trust/source/anchors/katello-server-ca.pem +#proxy = http://proxy.tco.census.gov:3128 +index = https://nexus.it.census.gov:8443/repository/DataScience-Group/pypi +index-url = https://nexus.it.census.gov:8443/repository/DataScience-Group/simple +trusted-host = nexus.it.census.gov + pypi.python.org + pypi.org + files.pythonhosted.org + proxy.tco.census.gov \ No newline at end of file diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..9a69f78 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,5 @@ +Jinja2>=3.1.0 +PyGithub>=2.1.1 +GitPython>=3.1.40 +boto3>=1.34.0 +botocore>=1.34.0 diff --git a/template.yaml b/template.yaml deleted file mode 100644 index c022b1d..0000000 --- a/template.yaml +++ /dev/null @@ -1,115 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Transform: AWS::Serverless-2016-10-31 -Description: > - eks-automation-lambda - -Resources: - EKSAutomationApi: - Type: AWS::Serverless::Api - Properties: - StageName: Prod - Auth: - ApiKeyRequired: true - UsagePlan: - UsagePlanName: eks-automation-usage-plan - CreateUsagePlan: PER_API - Description: Usage plan for EKS Automation API - Quota: - Limit: 5000 - Period: MONTH - Throttle: - BurstLimit: 500 - RateLimit: 100 - Tags: - - Key: finops_project_name - Value: csvd_platformbaseline - - Key: finops_project_number - Value: fs0000000078 - - Key: finops_project_role - Value: csvd_platformbaseline_app - - Key: environment - Value: development - - Key: environment_abbr - Value: dev - - Key: organization - Value: census:ocio:csvd - Cors: - AllowMethods: "'POST,OPTIONS'" - AllowHeaders: "'Content-Type,Authorization'" - AllowOrigin: "'*'" - Tags: - environment: development - environment_abbr: dev - organization: census:ocio:csvd - finops_project_name: csvd_platformbaseline - finops_project_number: fs0000000078 - finops_project_role: csvd_platformbaseline_app - GitLambdaLayer: - Type: AWS::Serverless::LayerVersion - Properties: - LayerName: git-lambda-layer - Description: Git Lambda Layer - ContentUri: s3://eks-automation-lambda-s3-bucket/layer.zip - CompatibleRuntimes: - - python3.9 - - python3.10 - - python3.11 - EKSAutomationFunction: - Type: AWS::Serverless::Function - Properties: - CodeUri: eks_automation/ - Handler: app.lambda_handler - Runtime: python3.11 - Timeout: 30 - Events: - EKSAutomation: - Type: Api - Properties: - RestApiId: !Ref EKSAutomationApi - Path: /EKSAutomation - Method: post - Auth: - ApiKeyRequired: true - Policies: - - AWSLambdaVPCAccessExecutionRole - - Statement: - - Sid: SSMDescribeParametersPolicy - Effect: Allow - Action: - - ssm:DescribeParameters - Resource: '*' - - Sid: SSMGetParameterPolicy - Effect: Allow - Action: - - ssm:GetParameters - - ssm:GetParameter - Resource: '*' - VpcConfig: - SecurityGroupIds: - - sg-03cbf2a626ed55c7e - SubnetIds: - - subnet-05192178ac094f639 - - subnet-022370a5a03585376 - PropagateTags: True - Tags: - environment: development - environment_abbr: dev - organization: census:ocio:csvd - finops_project_name: csvd_platformbaseline - finops_project_number: fs0000000078 - finops_project_role: csvd_platformbaseline_app - Layers: - - !Ref GitLambdaLayer - Environment: - Variables: - HOME: /tmp -Outputs: - EKSAutomationApi: - Description: "API Gateway endpoint URL for Prod stage for EKS Automation function" - Value: !Sub "https://${EKSAutomationApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/EKSAutomation/" - EKSAutomationFunction: - Description: "EKS Automation Lambda Function ARN" - Value: !GetAtt EKSAutomationFunction.Arn - EKSAutomationFunctionIamRole: - Description: "Implicit IAM Role created for EKS Automation function" - Value: !GetAtt EKSAutomationFunctionRole.Arn diff --git a/varfiles/default.json b/varfiles/default.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/varfiles/default.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/varfiles/default.tfvars b/varfiles/default.tfvars new file mode 100644 index 0000000..e69de29 diff --git a/varfiles/sct-engineering.json b/varfiles/sct-engineering.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/varfiles/sct-engineering.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/varfiles/sct-engineering.tfvars b/varfiles/sct-engineering.tfvars new file mode 100644 index 0000000..e69de29 diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..14a8a65 --- /dev/null +++ b/variables.tf @@ -0,0 +1,53 @@ +variable "environment" { + description = "Environment name" + type = string + default = "development" +} + +variable "environment_abbr" { + description = "Environment abbreviation" + type = string + default = "dev" +} + +variable "organization" { + description = "Organization name" + type = string + default = "census:ocio:csvd" +} + +variable "finops_project_name" { + description = "FinOps project name" + type = string + default = "csvd_platformbaseline" +} + +variable "finops_project_number" { + description = "FinOps project number" + type = string + default = "fs0000000078" +} + +variable "finops_project_role" { + description = "FinOps project role" + type = string + default = "csvd_platformbaseline_app" +} + +variable "vpc_security_group_ids" { + description = "List of VPC security group IDs" + type = list(string) + default = ["sg-03cbf2a626ed55c7e"] +} + +variable "vpc_subnet_ids" { + description = "List of VPC subnet IDs" + type = list(string) + default = ["subnet-05192178ac094f639", "subnet-022370a5a03585376"] +} + +variable "lambda_timeout" { + description = "Lambda function timeout in seconds" + type = number + default = 30 +} diff --git a/versions.tf b/versions.tf new file mode 100644 index 0000000..b36f3a4 --- /dev/null +++ b/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +}