S3 migration (#1)

* working on migrating repos to s3 objects * Update code to migrate repositories to S3 objects --------- Co-authored-by: Dave Arnold <dave@roknsound.com>
CSVD · Aug 7, 2024 · 39138c5 · 39138c5
1 parent 7e97464
commit 39138c5
Show file tree

Hide file tree

Showing 7 changed files with 207 additions and 184 deletions.
diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl
diff --git a/data.tf b/data.tf
@@ -0,0 +1,15 @@
+data "aws_codecommit_repository" "ansible" {
+  repository_name = "image-pipeline-ansible-playbooks"
+}
+
+data "aws_codecommit_repository" "goss" {
+  repository_name = "image-pipeline-goss-testing"
+}
+
+data "aws_codecommit_repository" "windows" {
+  repository_name = "windows-image-pipeline"
+}
+
+data "aws_codecommit_repository" "linux" {
+  repository_name = "linux-image-pipeline"
+}
diff --git a/linux.tf b/linux.tf
@@ -0,0 +1,42 @@
+
+module "amazon_linux" {
+  source                    = "HappyPathway/image-pipeline/aws"
+  project_name              = "linux-image-pipeline"
+  builder_image             = "aws/codebuild/standard:7.0"
+  create_new_repo           = false
+  create_new_role           = true
+  create_vpc_endpoint       = true
+  ssh_user                  = "ec2-user"
+  playbook                  = "hello-world.yaml"
+  terraform_version         = "1.8.5"
+  build_permissions_iam_doc = data.aws_iam_policy_document.s3_access
+  build_environment_variables = [
+    for proxy_var in keys(local.proxy_env_vars) :
+    {
+      name  = proxy_var,
+      value = lookup(local.proxy_env_vars, proxy_var),
+      type  = "PLAINTEXT"
+    }
+  ]
+  packer_repo   = data.aws_codecommit_repository.linux
+  ansible_repo  = data.aws_codecommit_repository.ansible
+  goss_repo     = data.aws_codecommit_repository.goss
+  goss_profile  = "base-test"
+  state         = local.state_config
+  vpc_config    = local.vpc_config
+  source_ami    = "ami-03fadeeea589a106b"
+  instance_type = "t2.micro"
+}
+
+output "linux_iam_arn" {
+  value = module.amazon_linux.iam_arn
+}
+
+output "linux_codebuild_user" {
+  value = module.amazon_linux.build_user.name
+}
+
+output "linux_parameters" {
+  value     = keys(module.amazon_linux.parameters)
+  sensitive = true
+}
diff --git a/locals.tf b/locals.tf
@@ -0,0 +1,36 @@
+locals {
+  proxy_env_vars = {
+    HTTP_PROXY  = "http://proxy.tco.census.gov:3128"
+    NO_PROXY    = ".census.gov,169.254.169.254,148.129.*,10.*,172.18.*,172.22.*,172.23.*,172.24.*,172.25.*,.eks.amazonaws.com,.s3.amazonaws.com,.amazonaws.com,.gcr.io,.pkg.dev,${local._vpc_config.region}.compute.internal,.${local._vpc_config.region}.compute.internal"
+    HTTPS_PROXY = "http://proxy.tco.census.gov:3128"
+  }
+  source_repo = "linux-image-pipeline"
+  environment = "dev"
+  _vpc_config = {
+    vpc_id = "vpc-00576a396ec570b94"
+    region = "us-gov-west-1"
+    security_group_ids = [
+      "sg-0d828d223df9834a6"
+    ]
+    subnets = [
+      # "subnet-0b1992a84536c581b"
+      "subnet-062189d742937204e"
+    ]
+  }
+  vpc_config = merge(
+    local._vpc_config,
+    {
+      security_group_ids = concat(
+        local._vpc_config.security_group_ids,
+        [
+          aws_security_group.allow_amznlinux_cdn.id
+      ])
+    }
+  )
+  state_config = {
+    bucket         = aws_s3_bucket.state_bucket.bucket
+    key            = "terraform.tfstate"
+    region         = local.vpc_config.region
+    dynamodb_table = "tf_remote_state"
+  }
+}
diff --git a/main.tf b/main.tf
@@ -1,41 +1,3 @@
-locals {
-    proxy_env_vars = {
-        HTTP_PROXY = "http://proxy.tco.census.gov:3128"
-        NO_PROXY = ".census.gov,169.254.169.254,148.129.*,10.*,172.18.*,172.22.*,172.23.*,172.24.*,172.25.*,.eks.amazonaws.com,.s3.amazonaws.com,.amazonaws.com,.gcr.io,.pkg.dev,${local._vpc_config.region}.compute.internal,.${local._vpc_config.region}.compute.internal"
-        HTTPS_PROXY = "http://proxy.tco.census.gov:3128"
-    }
-    source_repo   = "linux-image-pipeline"
-    project_name  = "linux-image-pipeline"
-    environment   = "dev"
-    _vpc_config = {
-        vpc_id = "vpc-00576a396ec570b94"
-        region = "us-gov-west-1"
-        security_group_ids = [
-          "sg-0d828d223df9834a6"
-        ]
-        subnets = [
-          # "subnet-0b1992a84536c581b"
-          "subnet-062189d742937204e"
-        ]
-    }
-    vpc_config = merge(
-        local._vpc_config,
-        {
-          security_group_ids = concat(
-            local._vpc_config.security_group_ids, 
-            [
-              aws_security_group.allow_amznlinux_cdn.id
-            ])
-        }
-    )
-    state_config = {
-        bucket         = aws_s3_bucket.state_bucket.bucket
-        key            = "terraform.tfstate"
-        region         = local.vpc_config.region
-        dynamodb_table = "tf_remote_state"
-    }  
-}
-
 resource "random_uuid" "random" {}
 
 resource "aws_s3_bucket" "state_bucket" {
@@ -76,134 +38,7 @@ resource "aws_vpc_security_group_egress_rule" "allow_all_traffic_ipv4" {
 }
 
 resource "aws_vpc_security_group_ingress_rule" "allow_all_between_self" {
-  security_group_id              = aws_security_group.allow_amznlinux_cdn.id
-  ip_protocol                    = "-1"
-  referenced_security_group_id   = aws_security_group.allow_amznlinux_cdn.id
-}
-
-data "aws_codecommit_repository" "ansible" {
-  repository_name = "image-pipeline-ansible-playbooks"
-}
-
-data "aws_codecommit_repository" "goss" {
-  repository_name = "image-pipeline-goss-testing"
-}
-
-
-module "main" {
-  source                      = "HappyPathway/image-pipeline/aws"
-  project_name                = local.project_name
-  source_repo_name            = local.source_repo
-  source_repo_branch          = "main"
-  builder_image               = "aws/codebuild/standard:7.0"
-  create_new_repo             = false
-  create_new_role             = true
-  create_vpc_endpoint         = true
-  ssh_user                    = "ec2-user"
-  playbook                    = "hello-world.yaml"
-  terraform_version           = "1.8.5"
-  build_permissions_iam_doc   = data.aws_iam_policy_document.s3_access
-  build_environment_variables = [
-   for proxy_var in keys(local.proxy_env_vars) :
-   {
-     name=proxy_var,
-     value=lookup(local.proxy_env_vars, proxy_var),
-     type = "PLAINTEXT"
-   }
-  ]
-  ansible_repo                = data.aws_codecommit_repository.ansible
-  goss_repo                   = data.aws_codecommit_repository.goss
-  goss_profile                = "base-test"
-  state = local.state_config
-  vpc_config = local.vpc_config
-  source_ami = "ami-03fadeeea589a106b"
-  instance_type = "t2.micro"
-}
-
-module "rhel" {
-  source                      = "HappyPathway/image-pipeline/aws"
-  project_name                = "rhel-image-pipeline-demo"
-  source_repo_name            = local.source_repo
-  source_repo_branch          = "main"
-  builder_image               = "aws/codebuild/standard:7.0"
-  create_new_repo             = false
-  create_new_role             = true
-  create_vpc_endpoint         = false
-  ssh_user                    = "ec2-user"
-  playbook                    = "rhel-arm-baseline.yaml"
-  terraform_version           = "1.8.5"
-  troubleshoot                = true
-  build_permissions_iam_doc   = data.aws_iam_policy_document.s3_access
-  build_environment_variables = [
-    for proxy_var in keys(local.proxy_env_vars) : {
-      name  = proxy_var
-      value = lookup(local.proxy_env_vars, proxy_var)
-      type  = "PLAINTEXT"
-    }
-  ]
-  ansible_repo                = data.aws_codecommit_repository.ansible
-  goss_repo                   = data.aws_codecommit_repository.goss
-  goss_profile                = "rhel-base-test"
-  state                       = local.state_config
-  vpc_config                  = local.vpc_config
-  source_ami                  = "ami-03fadeeea589a106b"  # x86_64 compatible AMI
-  instance_type               = "t3.micro"  # x86_64 compatible instance type
-}
-
-resource "random_password" "winrm" {
-  length           = 24
-  override_special = "$*!"
-  special          = true
-}
-
-output "password" {
-  value = nonsensitive(random_password.winrm.result)
-}
-
-
-module "windows" {
-  source                      = "HappyPathway/image-pipeline/aws"
-  project_name                = "windows-image-pipeline-demo"
-  source_repo_name            = "windows-image-pipeline"
-  source_repo_branch          = "main"
-  builder_image               = "aws/codebuild/standard:7.0"
-  create_new_repo             = false
-  create_new_role             = true
-  playbook                    = "windows-baseline.yaml"
-  terraform_version           = "1.8.5"
-  build_permissions_iam_doc   = data.aws_iam_policy_document.s3_access
-  winrm_credentials = {
-    username = "Administrator"
-    password = random_password.winrm.result
-  }
-  userdata = "userdata/winrm.ps1"
-  build_environment_variables = [
-    for proxy_var in keys(local.proxy_env_vars) : {
-      name  = proxy_var
-      value = lookup(local.proxy_env_vars, proxy_var)
-      type  = "PLAINTEXT"
-    }
-  ]
-  ansible_repo                = data.aws_codecommit_repository.ansible
-  goss_repo                   = data.aws_codecommit_repository.goss
-  goss_profile                = "windows-base-test"
-  state                       = local.state_config
-  vpc_config                  = local.vpc_config
-  source_ami                  = "ami-012fffaddacaa52ff"  # x86_64 compatible AMI
-  instance_type               = "t2.xlarge"  # x86_64 compatible instance type
-}
-
-
-
-output iam_arn {
-  value = module.main.iam_arn
-}
-
-output codebuild_user {
-  value =  module.main.build_user.name
-}
-
-output parameters {
-  value = keys(module.main.parameters)
-  sensitive = true
+  security_group_id            = aws_security_group.allow_amznlinux_cdn.id
+  ip_protocol                  = "-1"
+  referenced_security_group_id = aws_security_group.allow_amznlinux_cdn.id
 }
diff --git a/rhel.tf b/rhel.tf
@@ -0,0 +1,42 @@
+
+module "rhel" {
+  source                    = "HappyPathway/image-pipeline/aws"
+  project_name              = "rhel-image-pipeline-demo"
+  builder_image             = "aws/codebuild/standard:7.0"
+  create_new_repo           = false
+  create_new_role           = true
+  create_vpc_endpoint       = false
+  ssh_user                  = "ec2-user"
+  playbook                  = "rhel-arm-baseline.yaml"
+  terraform_version         = "1.8.5"
+  troubleshoot              = true
+  build_permissions_iam_doc = data.aws_iam_policy_document.s3_access
+  build_environment_variables = [
+    for proxy_var in keys(local.proxy_env_vars) : {
+      name  = proxy_var
+      value = lookup(local.proxy_env_vars, proxy_var)
+      type  = "PLAINTEXT"
+    }
+  ]
+  packer_repo   = data.aws_codecommit_repository.linux
+  ansible_repo  = data.aws_codecommit_repository.ansible
+  goss_repo     = data.aws_codecommit_repository.goss
+  goss_profile  = "rhel-base-test"
+  state         = local.state_config
+  vpc_config    = local.vpc_config
+  source_ami    = "ami-03fadeeea589a106b" # x86_64 compatible AMI
+  instance_type = "t3.micro"              # x86_64 compatible instance type
+}
+
+output "rhel_iam_arn" {
+  value = module.amazon_linux.iam_arn
+}
+
+output "rhel_codebuild_user" {
+  value = module.amazon_linux.build_user.name
+}
+
+output "rhel_parameters" {
+  value     = keys(module.amazon_linux.parameters)
+  sensitive = true
+}