From 53b61c5d3c94cdc0f8040434738addd1d1f93d95 Mon Sep 17 00:00:00 2001 From: badra001 Date: Wed, 13 Nov 2024 09:28:51 -0500 Subject: [PATCH] * 2.7.0 -- 2024-11-13 - inventory - add inventory code for use in decommission (not a submodule at this time) --- .tf-control | 32 ++++++++++++++ .tf-control.tfrc | 24 ++++++++++ CHANGELOG.md | 4 ++ common/version.tf | 2 +- inventory/filter-inspect.sh | 19 ++++++++ inventory/get-billing.sh | 36 +++++++++++++++ inventory/inspect.ew.tf | 87 +++++++++++++++++++++++++++++++++++++ inventory/inspect.gov.tf | 51 ++++++++++++++++++++++ inventory/inspect.tf | 18 ++++++++ inventory/list-iam.sh | 43 ++++++++++++++++++ inventory/locals.tf | 5 +++ inventory/region.tf | 3 ++ inventory/tf-run.data | 28 ++++++++++++ inventory/versions.tf | 38 ++++++++++++++++ 14 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 .tf-control create mode 100644 .tf-control.tfrc create mode 100755 inventory/filter-inspect.sh create mode 100755 inventory/get-billing.sh create mode 100644 inventory/inspect.ew.tf create mode 100644 inventory/inspect.gov.tf create mode 100644 inventory/inspect.tf create mode 100755 inventory/list-iam.sh create mode 100644 inventory/locals.tf create mode 100644 inventory/region.tf create mode 100644 inventory/tf-run.data create mode 100644 inventory/versions.tf diff --git a/.tf-control b/.tf-control new file mode 100644 index 0000000..340816b --- /dev/null +++ b/.tf-control @@ -0,0 +1,32 @@ +# .tf-control +# allows for setting a specific command to be used for tf-* commands under this git repo +# see tf-control.sh help for more info + +TFCONTROL_VERSION="1.0.7" +#TFCOMMAND="terraform_latest" +TFCOMMAND="terraform_current" + +# TF_CLI_CONFIG_FILE=PATH-TO-FILE/.tf-control.tfrc +# TFARGS="" +# TFNOLOG="" +# TFNOCOLOR="" + +# from issue: https://github.com/hashicorp/terraform/issues/32901 +# to get to TF 1.4 and beyond in a shared cache environment +# this is currently in the tf-control.sh script explicitly +#TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE=1 + +# use the following to force a specific version. An upgrade of an existing 0.12.31 to 1.x +# needs you to cycle through 0.13.17, 0.14.11, and then latest (0.15.5 not needed). Other +# steps in between. See https://github.e.it.census.gov/terraform/support/tree/master/docs/how-to/terraform-upgrade for details +# +#TFCOMMAND="terraform_0.12.31" +#TFCOMMAND="terraform_0.13.7" +#TFCOMMAND="terraform_0.14.11" +#TFCOMMAND="terraform_0.15.5" +#TFCOMMAND="terraform_1.3.10" +#TFCOMMAND="terraform_1.4.7" +#TFCOMMAND="terraform_1.5.7" +#TFCOMMAND="terraform_1.6.6" +#TFCOMMAND="terraform_1.7.5" +#TFCOMMAND="terraform_1.8.2" diff --git a/.tf-control.tfrc b/.tf-control.tfrc new file mode 100644 index 0000000..7425488 --- /dev/null +++ b/.tf-control.tfrc @@ -0,0 +1,24 @@ +TFCONTROL_VERSION="1.0.5" + +# https://www.terraform.io/docs/cli/config/config-file.html +plugin_cache_dir = "/data/terraform/terraform.d/plugin-cache" +#disable_checkpoint = true + +provider_installation { +# filesystem_mirror { +# path = "/apps/terraform/terraform.d/providers" +# include = [ "*/*/*" ] +# } + filesystem_mirror { + path = "/data/terraform/terraform.d/providers" + include = [ "*/*/*" ] + } +# filesystem_mirror { +# path = "/apps/terraform/terraform.d/providers" +# include = [ "external.terraform.census.gov/*/*" ] +# } + direct { + include = [ "*/*/*" ] + } +} + diff --git a/CHANGELOG.md b/CHANGELOG.md index da8cf7f..e9ba7ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -374,3 +374,7 @@ * 2.6.2 -- 2024-09-13 - terraform-state - add for Terragrunt: dynamodb:DescribeTable permission, s3:GetBucketVersioning + +* 2.7.0 -- 2024-11-13 + - inventory + - add inventory code for use in decommission (not a submodule at this time) diff --git a/common/version.tf b/common/version.tf index 4d32dce..2499cf3 100644 --- a/common/version.tf +++ b/common/version.tf @@ -1,3 +1,3 @@ locals { - _module_version = "2.6.2" + _module_version = "2.7.0" } diff --git a/inventory/filter-inspect.sh b/inventory/filter-inspect.sh new file mode 100755 index 0000000..51b615f --- /dev/null +++ b/inventory/filter-inspect.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +VERSION="1.0.1" +THIS=$(basename $0 .sh) + +FILES=$@ +if [ -z "$FILES" ] +then + FILES=$(ls setup/cloud-nuke*.log) +fi + +for f in $FILES +do + echo "# $f" + cat $f | grep -v securityhub | grep -v OrgConfig > $f.txt + replace-control-characters.sh $f.txt + sed -i -e 's/\r/\n/g' $f.txt + sed -i -e '0,/^# Found/d' $f.txt +done diff --git a/inventory/get-billing.sh b/inventory/get-billing.sh new file mode 100755 index 0000000..4abcc5b --- /dev/null +++ b/inventory/get-billing.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +VERSION="1.0.1" + +caller=$(tf-aws sts get-caller-identity --query 'Arn' 2> /dev/null) +if [ $(echo $caller | grep -c aws-us-gov) != 0 ] +then + echo "# partition is not commercial, exiting" + exit 1 +fi +exit + +lastmonth=$(date --date="-1 month" +%Y-%m-01) +nextmonth=$(date --date="+1 month" +%Y-%m-01) + +test -d setup || mkdir setup + +tf-aws ce get-cost-and-usage \ + --time-period Start=$lastmonth,End=$nextmonth \ + --granularity MONTHLY \ + --metrics "BlendedCost" "UnblendedCost" "UsageQuantity" \ + --group-by Type=DIMENSION,Key=SERVICE \ + > setup/cost-usage.${lastmonth}_${nextmonth}.json + +tf-aws ce get-cost-and-usage \ + --time-period Start=$lastmonth,End=$nextmonth \ + --granularity MONTHLY \ + --metrics "BlendedCost" "UnblendedCost" "UsageQuantity" \ + --group-by Type=DIMENSION,Key=SERVICE \ + --output text \ + > setup/cost-usage.${lastmonth}_${nextmonth}.txt + +grep ^BLEND setup/cost-usage.${lastmonth}_${nextmonth}.txt |\ + awk '{t+=$2} END {print NR,t}' > setup/cost-usage.${lastmonth}_${nextmonth}.summary + +exit 0 diff --git a/inventory/inspect.ew.tf b/inventory/inspect.ew.tf new file mode 100644 index 0000000..1a30bda --- /dev/null +++ b/inventory/inspect.ew.tf @@ -0,0 +1,87 @@ +# 1398 2024-10-10 15:58:38 DISABLE_TELEMETRY=true AWS_PROFILE=198886018595-ma24-gov cloud-nuke inspect-aws --region us-gov-west-1 |& tee nuke.us-gov-west-1.$(date +%s).log +# 1399 2024-10-10 15:59:39 DISABLE_TELEMETRY=true AWS_PROFILE=198886018595-ma24-gov cloud-nuke inspect-aws --region us-gov-east-1 |& tee nuke.us-gov-east-1.$(date +%s).log + +resource "null_resource" "inspect_east-1" { + count = data.aws_arn.current.partition == "aws" ? 1 : 0 + triggers = { + region = "us-east-1" + directory = null_resource.setup_directory.triggers.directory + timestamp = local.timestamp + } + + provisioner "local-exec" { + command = "cloud-nuke inspect-aws --region ${self.triggers.region} > ${self.triggers.directory}/cloud-nuke.${self.triggers.region}.${self.triggers.timestamp}.log 2>&1" + environment = { + DISABLE_TELEMETRY = true + AWS_PROFILE = var.profile + AWS_REGION = self.triggers.region + } + } +} + +resource "null_resource" "inspect_east-2" { + count = data.aws_arn.current.partition == "aws" ? 1 : 0 + triggers = { + region = "us-east-2" + directory = null_resource.setup_directory.triggers.directory + timestamp = local.timestamp + } + + provisioner "local-exec" { + command = "cloud-nuke inspect-aws --region ${self.triggers.region} > ${self.triggers.directory}/cloud-nuke.${self.triggers.region}.${self.triggers.timestamp}.log 2>&1" + environment = { + DISABLE_TELEMETRY = true + AWS_PROFILE = var.profile + AWS_REGION = self.triggers.region + } + } +} + +resource "null_resource" "inspect_west-1" { + count = data.aws_arn.current.partition == "aws" ? 1 : 0 + triggers = { + region = "us-west-1" + directory = null_resource.setup_directory.triggers.directory + timestamp = local.timestamp + } + + provisioner "local-exec" { + command = "cloud-nuke inspect-aws --region ${self.triggers.region} > ${self.triggers.directory}/cloud-nuke.${self.triggers.region}.${self.triggers.timestamp}.log 2>&1" + environment = { + DISABLE_TELEMETRY = true + AWS_PROFILE = var.profile + AWS_REGION = self.triggers.region + } + } +} + +resource "null_resource" "inspect_west-2" { + count = data.aws_arn.current.partition == "aws" ? 1 : 0 + triggers = { + region = "us-west-2" + directory = null_resource.setup_directory.triggers.directory + timestamp = local.timestamp + } + + provisioner "local-exec" { + command = "cloud-nuke inspect-aws --region ${self.triggers.region} > ${self.triggers.directory}/cloud-nuke.${self.triggers.region}.${self.triggers.timestamp}.log 2>&1" + environment = { + DISABLE_TELEMETRY = true + AWS_PROFILE = var.profile + AWS_REGION = self.triggers.region + } + } +} + +resource "null_resource" "filter_ew" { + count = data.aws_arn.current.partition == "aws" ? 1 : 0 + triggers = { + directory = null_resource.setup_directory.triggers.directory + timestamp = local.timestamp + inspect_complete = join(",", [null_resource.inspect_east-1[0].id, null_resource.inspect_east-2[0].id, null_resource.inspect_west-1[0].id, null_resource.inspect_west-2[0].id]) + } + + provisioner "local-exec" { + command = "bash ${path.root}/filter-inspect.sh ${self.triggers.directory}/cloud-nuke.*.${self.triggers.timestamp}.log 2>&1" + } +} diff --git a/inventory/inspect.gov.tf b/inventory/inspect.gov.tf new file mode 100644 index 0000000..3356319 --- /dev/null +++ b/inventory/inspect.gov.tf @@ -0,0 +1,51 @@ +# 1398 2024-10-10 15:58:38 DISABLE_TELEMETRY=true AWS_PROFILE=198886018595-ma24-gov cloud-nuke inspect-aws --region us-gov-west-1 |& tee nuke.us-gov-west-1.$(date +%s).log +# 1399 2024-10-10 15:59:39 DISABLE_TELEMETRY=true AWS_PROFILE=198886018595-ma24-gov cloud-nuke inspect-aws --region us-gov-east-1 |& tee nuke.us-gov-east-1.$(date +%s).log + +resource "null_resource" "inspect_east" { + count = data.aws_arn.current.partition == "aws-us-gov" ? 1 : 0 + triggers = { + region = "us-gov-east-1" + directory = null_resource.setup_directory.triggers.directory + timestamp = local.timestamp + } + + provisioner "local-exec" { + command = "cloud-nuke inspect-aws --region ${self.triggers.region} > ${self.triggers.directory}/cloud-nuke.${self.triggers.region}.${local.timestamp}.log 2>&1" + environment = { + DISABLE_TELEMETRY = true + AWS_PROFILE = var.profile + AWS_REGION = self.triggers.region + } + } +} + +resource "null_resource" "inspect_west" { + count = data.aws_arn.current.partition == "aws-us-gov" ? 1 : 0 + triggers = { + region = "us-gov-west-1" + directory = null_resource.setup_directory.triggers.directory + timestamp = local.timestamp + } + + provisioner "local-exec" { + command = "cloud-nuke inspect-aws --region ${self.triggers.region} > ${self.triggers.directory}/cloud-nuke.${self.triggers.region}.${self.triggers.timestamp}.log 2>&1" + environment = { + DISABLE_TELEMETRY = true + AWS_PROFILE = var.profile + AWS_REGION = self.triggers.region + } + } +} + +resource "null_resource" "filter_gov" { + count = data.aws_arn.current.partition == "aws-us-gov" ? 1 : 0 + triggers = { + directory = null_resource.setup_directory.triggers.directory + timestamp = local.timestamp + inspect_complete = join(",", [null_resource.inspect_east[0].id, null_resource.inspect_west[0].id]) + } + + provisioner "local-exec" { + command = "bash ${path.root}/filter-inspect.sh ${self.triggers.directory}/cloud-nuke.*.${self.triggers.timestamp}.log 2>&1" + } +} diff --git a/inventory/inspect.tf b/inventory/inspect.tf new file mode 100644 index 0000000..677081f --- /dev/null +++ b/inventory/inspect.tf @@ -0,0 +1,18 @@ +# 1398 2024-10-10 15:58:38 DISABLE_TELEMETRY=true AWS_PROFILE=198886018595-ma24-gov cloud-nuke inspect-aws --region us-gov-west-1 |& tee nuke.us-gov-west-1.$(date +%s).log +# 1399 2024-10-10 15:59:39 DISABLE_TELEMETRY=true AWS_PROFILE=198886018595-ma24-gov cloud-nuke inspect-aws --region us-gov-east-1 |& tee nuke.us-gov-east-1.$(date +%s).log + +locals { + timestamp = try((provider::time::rfc3339_parse(time_static.inspect.rfc3339)).unix, time_static.inspect.rfc3339) +} + +resource "time_static" "inspect" {} + +resource "null_resource" "setup_directory" { + triggers = { + directory = format("%v/setup", path.root) + } + + provisioner "local-exec" { + command = "test -d ${self.triggers.directory} || mkdir ${self.triggers.directory}" + } +} diff --git a/inventory/list-iam.sh b/inventory/list-iam.sh new file mode 100755 index 0000000..3345553 --- /dev/null +++ b/inventory/list-iam.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +LIST="" +#LIST+=" list-access-keys" +LIST+=" list-account-aliases" +#LIST+=" list-attached-group-policies" +#LIST+=" list-attached-role-policies" +#LIST+=" list-attached-user-policies" +#LIST+=" list-entities-for-policy" +#LIST+=" list-group-policies" +LIST+=" list-groups" +#LIST+=" list-groups-for-user" +#LIST+=" list-instance-profile-tags" +LIST+=" list-instance-profiles" +#LIST+=" list-instance-profiles-for-role" +#LIST+=" list-mfa-device-tags" +#LIST+=" list-mfa-devices" +#LIST+=" list-open-id-connect-provider-tags" +LIST+=" list-open-id-connect-providers" +LIST+=" list-policies" +#LIST+=" list-policies-granting-service-access" +#LIST+=" list-policy-tags" +#LIST+=" list-policy-versions" +#LIST+=" list-role-policies" +#LIST+=" list-role-tags" +LIST+=" list-roles" +#LIST+=" list-saml-provider-tags" +LIST+=" list-saml-providers" +#LIST+=" list-server-certificate-tags" +LIST+=" list-server-certificates" +#LIST+=" list-service-specific-credentials" +#LIST+=" list-signing-certificates" +#LIST+=" list-ssh-public-keys" +#LIST+=" list-user-policies" +#LIST+=" list-user-tags" +LIST+=" list-users" +LIST+=" list-virtual-mfa-devices" + +for f in $LIST +do + echo "# $f" + tf-aws iam $f > setup/$f.json +done diff --git a/inventory/locals.tf b/inventory/locals.tf new file mode 100644 index 0000000..6e912e2 --- /dev/null +++ b/inventory/locals.tf @@ -0,0 +1,5 @@ +locals { + base_tags = { + "boc:created_by" = "terraform" + } +} diff --git a/inventory/region.tf b/inventory/region.tf new file mode 100644 index 0000000..f617506 --- /dev/null +++ b/inventory/region.tf @@ -0,0 +1,3 @@ +locals { + region = var.region +} diff --git a/inventory/tf-run.data b/inventory/tf-run.data new file mode 100644 index 0000000..b5aa5c4 --- /dev/null +++ b/inventory/tf-run.data @@ -0,0 +1,28 @@ +VERSION 2.1.2 +TAG setup +REMOTE-STATE +COMMAND tf-directory-setup.py -l none -f +COMMAND setup-new-directory.sh +COMMAND rm remote_state.common.tf + +TAG links +LINKTOP includes.d/variables.account_tags.tf +LINKTOP includes.d/variables.account_tags.auto.tfvars +LINKTOP includes.d/variables.infrastructure_tags.tf +LINKTOP includes.d/variables.infrastructure_tags.auto.tfvars +LINKTOP includes.d/variables.application_tags.tf +LINKTOP includes.d/variables.application_tags.auto.tfvars +# LINKTOP provider_configs.d/provider.ldap_new.auto.tfvars +# LINKTOP provider_configs.d/provider.ldap_new.tf +# LINKTOP provider_configs.d/provider.ldap_new.variables.tf +COMMAND rm -f provider.ldap.* + +TAG init +COMMAND tf-init + +TAG start +#POLICY +ALL + +TAG state-link +COMMAND tf-directory-setup.py -l s3 diff --git a/inventory/versions.tf b/inventory/versions.tf new file mode 100644 index 0000000..e2f46d0 --- /dev/null +++ b/inventory/versions.tf @@ -0,0 +1,38 @@ +terraform { + required_version = ">= 1.8.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.0" + } + time = { + source = "hashicorp/time" + version = ">= 0.12" + } + + # ldap = { + # source = "trevex/ldap" + # version = ">= 0.5.4" + # } + # external = { + # source = "hashicorp/external" + # version = ">= 1.0" + # } + null = { + source = "hashicorp/null" + version = ">= 1.0" + } + # random = { + # source = "hashicorp/random" + # version = ">= 1.0" + # } + # template = { + # source = "hashicorp/template" + # version = ">= 1.0" + # } + # infoblox = { + # source = "infobloxopen/infoblox" + # version = ">= 2.1.0" + # } + } +}