From fc6742fe34dfaf861c1353626788c5ada9aa394e Mon Sep 17 00:00:00 2001 From: badra001 Date: Tue, 2 Aug 2022 08:40:01 -0400 Subject: [PATCH] add flowlogs-transit-gateway --- CHANGELOG.md | 3 + common/data.tf | 6 + common/variables.common.tf | 4 +- common/version.tf | 27 +-- flowlogs-transit-gateway/README.md | 94 +++++++++++ flowlogs-transit-gateway/data.tf | 1 + flowlogs-transit-gateway/defaults.tf | 1 + flowlogs-transit-gateway/main.tf | 155 ++++++++++++++++++ flowlogs-transit-gateway/module_name.tf | 3 + flowlogs-transit-gateway/outputs.tf | 4 + flowlogs-transit-gateway/prefixes.tf | 1 + .../templates/aws_kinesis_tasks.conf.tpl | 9 + flowlogs-transit-gateway/variables.common.tf | 1 + flowlogs-transit-gateway/variables.tf | 19 +++ flowlogs-transit-gateway/version.tf | 1 + flowlogs-transit-gateway/versions.tf | 1 + 16 files changed, 315 insertions(+), 15 deletions(-) create mode 100644 flowlogs-transit-gateway/README.md create mode 120000 flowlogs-transit-gateway/data.tf create mode 120000 flowlogs-transit-gateway/defaults.tf create mode 100644 flowlogs-transit-gateway/main.tf create mode 100644 flowlogs-transit-gateway/module_name.tf create mode 100644 flowlogs-transit-gateway/outputs.tf create mode 120000 flowlogs-transit-gateway/prefixes.tf create mode 100644 flowlogs-transit-gateway/templates/aws_kinesis_tasks.conf.tpl create mode 120000 flowlogs-transit-gateway/variables.common.tf create mode 100644 flowlogs-transit-gateway/variables.tf create mode 120000 flowlogs-transit-gateway/version.tf create mode 120000 flowlogs-transit-gateway/versions.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index 3967635..0a3dc66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -159,3 +159,6 @@ - flowlogs - update template to use account-alias and vpc name +* 2.1.0 -- 2022-08-02 + - flowlogs-transit-gateway + - setup for transit gateway diff --git a/common/data.tf b/common/data.tf index c99f19d..f68425f 100644 --- a/common/data.tf +++ b/common/data.tf @@ -6,6 +6,8 @@ data "aws_arn" "current" { data "aws_region" "current" {} +data "aws_iam_account_alias" "current" {} + # output "caller_account_id" { # value = data.aws_caller_identity.current.account_id # } @@ -21,3 +23,7 @@ data "aws_region" "current" {} # output "account_region_name" { # value = data.aws_region.current.name # } +# +# output "account_alias" { +# value = data.aws_iam_account_alias.current.account_alias +# } diff --git a/common/variables.common.tf b/common/variables.common.tf index 1f96a36..2039ab9 100644 --- a/common/variables.common.tf +++ b/common/variables.common.tf @@ -2,13 +2,13 @@ # account info #--- variable "account_id" { - description = "AWS Account ID (default will pull from current user)" + description = "AWS Account ID (default: will pull from current user)" type = string default = "" } variable "account_alias" { - description = "AWS Account Alias" + description = "AWS Account Alias (default: will pull from current account_alias)" type = string default = "" } diff --git a/common/version.tf b/common/version.tf index 3835a24..a7c3c48 100644 --- a/common/version.tf +++ b/common/version.tf @@ -1,19 +1,20 @@ locals { - _module_version = "2.0.6" + _module_version = "2.1.0" _module_names = { "_main_" = "aws-vpc-setup" - "flowlogs" = "aws-vpc-setup/flowlogs" - "flowlogs-role" = "aws-vpc-setup/flowlogs-role" - "nacl-rules" = "aws-vpc-setup/nacl-rules" - "nacls" = "aws-vpc-setup/nacls" - "peer" = "aws-vpc-setup/peer" - "routing" = "aws-vpc-setup/routing" - "security-groups" = "aws-vpc-setup/security-groups" - "subnets" = "aws-vpc-setup/subnets" - "vpc" = "aws-vpc-setup/vpc" - "vpc-interface-endpoint" = "aws-vpc-setup/vpc-interface-endpoint" - "vpn" = "aws-vpc-setup/vpn" - "vpn-transit-gateway" = "aws-vpc-setup/vpn-transit-gateway" + "flowlogs" = "aws-vpc-setup/flowlogs" + "flowlogs-transit-gateway" = "aws-vpc-setup/flowlogs-transit-gateway" + "flowlogs-role" = "aws-vpc-setup/flowlogs-role" + "nacl-rules" = "aws-vpc-setup/nacl-rules" + "nacls" = "aws-vpc-setup/nacls" + "peer" = "aws-vpc-setup/peer" + "routing" = "aws-vpc-setup/routing" + "security-groups" = "aws-vpc-setup/security-groups" + "subnets" = "aws-vpc-setup/subnets" + "vpc" = "aws-vpc-setup/vpc" + "vpc-interface-endpoint" = "aws-vpc-setup/vpc-interface-endpoint" + "vpn" = "aws-vpc-setup/vpn" + "vpn-transit-gateway" = "aws-vpc-setup/vpn-transit-gateway" } } diff --git a/flowlogs-transit-gateway/README.md b/flowlogs-transit-gateway/README.md new file mode 100644 index 0000000..eb98715 --- /dev/null +++ b/flowlogs-transit-gateway/README.md @@ -0,0 +1,94 @@ +# About aws-vpc-setup :: flowlogs-transit-gateway + +This submodule creates VPC flow logs for a transit gateway. This shoudl be used per transit gateway, one in each region +You will need to run the flow logs role once before (aws-vpc-setup/flowlogs-role). + +This also creates cloudwatch logs and cloudwatch streams (Kinesis), and drops a configuration file in `setup/` +to be used in Splunk. Distribute this configuration to the splunk team for flowlog ingestion. + +``` +# setup/aws_kinesis_tasks.lgs-tgw_ent-gov-sa_us-gov-east-1.conf +[ent-gov-network-sa-flowlogs-us-gov-east-1_lg_ent-gov-sa_us-gov-east-1] +account = 123123123123-ent-gov-network-sa +format = CloudWatchLogs +index = aws_vpc_flow_logs +region = us-gov-east-1 +sourcetype = aws:cloudwatchlogs:vpcflow +init_stream_position = LATEST +stream_names = lgs-tgw_ent-gov-sa_us-gov-east-1 +``` + +# Usage + +```hcl +module "flowlogs-transit-gateway" { + source = "git@github.e.it.census.gov:terraform-modules/aws-vpc-setup.git//flowlogs-transit-gateway?ref=tf-upgrade" + label = "ent-gov-sa" + # account_alias = "ent-gov-network-sa" + transit_gateway_id = aws_transit_gateway.gateway.id + flowlog_bucket_arn = data.terraform_remote_state.common.infrastructure_east.flowlogs_arn + flowlog_role_arn = data.terraform_remote_state.common.outputs.role_flowlogs_arn + + tags = {} +} +``` + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13 | +| [aws](#requirement\_aws) | >= 3.66.0 | +| [ldap](#requirement\_ldap) | >= 0.5.4 | +| [null](#requirement\_null) | >= 3.0 | +| [random](#requirement\_random) | >= 3.0 | +| [template](#requirement\_template) | >= 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.66.0 | +| [local](#provider\_local) | n/a | +| [null](#provider\_null) | >= 3.0 | +| [template](#provider\_template) | >= 2.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.flowlog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_cloudwatch_log_subscription_filter.flowlog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_subscription_filter) | resource | +| [aws_flow_log.flowlog_cloudwatch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/flow_log) | resource | +| [aws_flow_log.flowlog_s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/flow_log) | resource | +| [aws_kinesis_stream.flowlog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_stream) | resource | +| [local_file.splunk_flowlog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [null_resource.splunk_flowlog](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_account_alias.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_account_alias) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | +| [template_file.splunk_flowlog](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [account\_alias](#input\_account\_alias) | AWS Account Alias (default: will pull from current account\_alias) | `string` | `""` | no | +| [account\_id](#input\_account\_id) | AWS Account ID (default: will pull from current user) | `string` | `""` | no | +| [flowlog\_bucket\_arn](#input\_flowlog\_bucket\_arn) | S3 Bucket to hold the VPC flowlogs | `string` | n/a | yes | +| [flowlog\_role\_arn](#input\_flowlog\_role\_arn) | IAM Role with proper permissions to allow writing VPC flowlogs to cloudwatch logs and streamss | `string` | n/a | yes | +| [label](#input\_label) | Text label associated with the Transit Gateway | `string` | n/a | yes | +| [override\_prefixes](#input\_override\_prefixes) | Override built-in prefixes by component. This should be used primarily for common infrastructure things | `map(string)` | `{}` | no | +| [tags](#input\_tags) | AWS Tags to apply to appropriate resources (S3, KMS). Do not include safeguard tags here, use the data\_safeguard field for such things. | `map(string)` | `{}` | no | +| [transit\_gateway\_id](#input\_transit\_gateway\_id) | ID of the Transit Gateway | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [kinesis\_flowlog\_arn](#output\_kinesis\_flowlog\_arn) | VPC Flowlog Kinesis stream ARN | diff --git a/flowlogs-transit-gateway/data.tf b/flowlogs-transit-gateway/data.tf new file mode 120000 index 0000000..995624d --- /dev/null +++ b/flowlogs-transit-gateway/data.tf @@ -0,0 +1 @@ +../common/data.tf \ No newline at end of file diff --git a/flowlogs-transit-gateway/defaults.tf b/flowlogs-transit-gateway/defaults.tf new file mode 120000 index 0000000..a5556ac --- /dev/null +++ b/flowlogs-transit-gateway/defaults.tf @@ -0,0 +1 @@ +../common/defaults.tf \ No newline at end of file diff --git a/flowlogs-transit-gateway/main.tf b/flowlogs-transit-gateway/main.tf new file mode 100644 index 0000000..a0e390e --- /dev/null +++ b/flowlogs-transit-gateway/main.tf @@ -0,0 +1,155 @@ +/* +* # About aws-vpc-setup :: flowlogs-transit-gateway +* +* This submodule creates VPC flow logs for a transit gateway. This shoudl be used per transit gateway, one in each region +* You will need to run the flow logs role once before (aws-vpc-setup/flowlogs-role). +* +* This also creates cloudwatch logs and cloudwatch streams (Kinesis), and drops a configuration file in `setup/` +* to be used in Splunk. Distribute this configuration to the splunk team for flowlog ingestion. +* +* ``` +* # setup/aws_kinesis_tasks.lgs-tgw_ent-gov-sa_us-gov-east-1.conf +* [ent-gov-network-sa-flowlogs-us-gov-east-1_lg_ent-gov-sa_us-gov-east-1] +* account = 123123123123-ent-gov-network-sa +* format = CloudWatchLogs +* index = aws_vpc_flow_logs +* region = us-gov-east-1 +* sourcetype = aws:cloudwatchlogs:vpcflow +* init_stream_position = LATEST +* stream_names = lgs-tgw_ent-gov-sa_us-gov-east-1 +* ``` +* +* # Usage +* +* ```hcl +* module "flowlogs-transit-gateway" { +* source = "git@github.e.it.census.gov:terraform-modules/aws-vpc-setup.git//flowlogs-transit-gateway?ref=tf-upgrade" +* label = "ent-gov-sa" +* # account_alias = "ent-gov-network-sa" +* transit_gateway_id = aws_transit_gateway.gateway.id +* flowlog_bucket_arn = data.terraform_remote_state.common.infrastructure_east.flowlogs_arn +* flowlog_role_arn = data.terraform_remote_state.common.outputs.role_flowlogs_arn +* +* tags = {} +* } +* ``` +*/ + +locals { + account_id = var.account_id != "" ? var.account_id : data.aws_caller_identity.current.account_id + account_environment = data.aws_arn.current.partition == "aws-us-gov" ? "gov" : "ew" + region = data.aws_region.current.name + + base_tags = { + "boc:tf_module_version" = local._module_version + "boc:tf_module_name" = lookup(local._module_names, local._module_name, local._module_names["_main_"]) + "boc:created_by" = "terraform" + } + + _account_alias = var.account_alias == null || var.accunt_alias == "" ? data.aws_iam_account_alias.current.account_alias : var.account_alias + account_alias = replace(local._account_alias, "do2", "do1") + flowlog_stream_name = replace(aws_cloudwatch_log_group.flowlog.name, local._prefixes["log-group"], local._prefixes["log-stream"]) +} + +resource "aws_flow_log" "flowlog_s3" { + log_destination = format("%v/%v/%v/", var.flowlog_bucket_arn, "tgw", var.label) + log_destination_type = "s3" + iam_role_arn = var.flowlog_role_arn + traffic_type = "ALL" + transit_gateway_id = var.transit_gateway_id + + tags = merge( + local.base_tags, + var.tags, + { "Name" = format("%v:%v_%v_%v", "s3", "tgw", var.label, local.region) }, + ) +} + +#--- +# flowlog, cloudwatch +#--- +resource "aws_cloudwatch_log_group" "flowlog" { + name = format("%v%v_%v_%v", local._prefixes["log-group"], "tgw", var.label, local.region) + + tags = merge( + local.base_tags, + var.tags, + { "Name" = format("%v-%v:%v_%v_%v", "cloudwatch", "log-group", "tgw", var.label, local.region) }, + ) +} + +resource "aws_flow_log" "flowlog_cloudwatch" { + log_destination = aws_cloudwatch_log_group.flowlog.arn + iam_role_arn = var.flowlog_role_arn + traffic_type = "ALL" + transit_gateway_id = var.transit_gateway_id + + tags = merge( + local.base_tags, + var.tags, + { "Name" = format("%v-%v:%v_%v_%v", "cloudwatch", "log-group", "tgw", var.label, local.region) }, + ) +} + +resource "aws_kinesis_stream" "flowlog" { + name = local.flowlog_stream_name + shard_count = 1 + retention_period = 48 + shard_level_metrics = ["IncomingBytes", "OutgoingBytes", "IncomingRecords", "OutgoingRecords"] + + tags = merge( + local.base_tags, + var.tags, + tomap({ "Name" = local.flowlog_stream_name }), + ) +} + +# have to add the flowlog arn here to the policy used by flowlogs in common/{east,west}/flowlog.tf +resource "aws_cloudwatch_log_subscription_filter" "flowlog" { + name = local.flowlog_stream_name + role_arn = var.flowlog_role_arn + log_group_name = aws_cloudwatch_log_group.flowlog.name + destination_arn = aws_kinesis_stream.flowlog.arn + filter_pattern = "[action=*]" + distribution = "ByLogStream" +} + +#--- +# generate splunk inputs file +#--- +data "template_file" "splunk_flowlog" { + template = file("${path.module}/templates/aws_kinesis_tasks.conf.tpl") + vars = { + account_id = local.account_id + account_alias = local.account_alias + region = local.region + flowlog_name = aws_cloudwatch_log_group.flowlog.name + flowlog_stream_name = local.flowlog_stream_name + label = var.label + } +} + +resource "null_resource" "splunk_flowlog" { + triggers = { + filename = format("aws_kinesis_tasks.%v-%v.%v.%v.conf", local.account_id, local.account_alias, local.region, local.flowlog_stream_name) + directory = format("%v/setup", path.root) + } + provisioner "local-exec" { + command = "test -d ${self.triggers.directory} || mkdir ${self.triggers.directory}" + } + + # provisioner "local-exec" { + # working_dir = path.root + # command = "test -d setup || mkdir setup" + # } + # provisioner "local-exec" { + # working_dir = "${path.root}/setup" + # command = "echo '${data.template_file.splunk_flowlog_tasks_flowlog.rendered}' > aws_kinesis_tasks.${local.flowlog_stream_name}.conf" + # } +} + +resource "local_file" "splunk_flowlog" { + content = data.template_file.splunk_flowlog.rendered + file_permission = "0644" + filename = format("%v/%v", null_resource.splunk_flowlog.triggers.directory, null_resource.splunk_flowlog.triggers.filename) +} diff --git a/flowlogs-transit-gateway/module_name.tf b/flowlogs-transit-gateway/module_name.tf new file mode 100644 index 0000000..5eaa4a8 --- /dev/null +++ b/flowlogs-transit-gateway/module_name.tf @@ -0,0 +1,3 @@ +locals { + _module_name = "flowlogs" +} diff --git a/flowlogs-transit-gateway/outputs.tf b/flowlogs-transit-gateway/outputs.tf new file mode 100644 index 0000000..bffb008 --- /dev/null +++ b/flowlogs-transit-gateway/outputs.tf @@ -0,0 +1,4 @@ +output "kinesis_flowlog_arn" { + description = "VPC Flowlog Kinesis stream ARN" + value = aws_kinesis_stream.flowlog.arn +} diff --git a/flowlogs-transit-gateway/prefixes.tf b/flowlogs-transit-gateway/prefixes.tf new file mode 120000 index 0000000..7e265d5 --- /dev/null +++ b/flowlogs-transit-gateway/prefixes.tf @@ -0,0 +1 @@ +../common/prefixes.tf \ No newline at end of file diff --git a/flowlogs-transit-gateway/templates/aws_kinesis_tasks.conf.tpl b/flowlogs-transit-gateway/templates/aws_kinesis_tasks.conf.tpl new file mode 100644 index 0000000..894104c --- /dev/null +++ b/flowlogs-transit-gateway/templates/aws_kinesis_tasks.conf.tpl @@ -0,0 +1,9 @@ +[${account_id}-${account_alias}-flowlogs-tgw-${label}-${region}] +account = ${account_id}-${account_alias} +format = CloudWatchLogs +index = aws_vpc_flow_logs +region = ${region} +sourcetype = aws:cloudwatchlogs:vpcflow +init_stream_position = LATEST +stream_names = ${flowlog_stream_name} + diff --git a/flowlogs-transit-gateway/variables.common.tf b/flowlogs-transit-gateway/variables.common.tf new file mode 120000 index 0000000..7439ed8 --- /dev/null +++ b/flowlogs-transit-gateway/variables.common.tf @@ -0,0 +1 @@ +../common/variables.common.tf \ No newline at end of file diff --git a/flowlogs-transit-gateway/variables.tf b/flowlogs-transit-gateway/variables.tf new file mode 100644 index 0000000..a7008a7 --- /dev/null +++ b/flowlogs-transit-gateway/variables.tf @@ -0,0 +1,19 @@ +variable "flowlog_bucket_arn" { + description = "S3 Bucket to hold the VPC flowlogs" + type = string +} + +variable "flowlog_role_arn" { + description = "IAM Role with proper permissions to allow writing VPC flowlogs to cloudwatch logs and streamss" + type = string +} + +variable "transit_gateway_id" { + description = "ID of the Transit Gateway" + type = string +} + +variable "label" { + description = "Text label associated with the Transit Gateway" + type = string +} diff --git a/flowlogs-transit-gateway/version.tf b/flowlogs-transit-gateway/version.tf new file mode 120000 index 0000000..b83c5b7 --- /dev/null +++ b/flowlogs-transit-gateway/version.tf @@ -0,0 +1 @@ +../common/version.tf \ No newline at end of file diff --git a/flowlogs-transit-gateway/versions.tf b/flowlogs-transit-gateway/versions.tf new file mode 120000 index 0000000..41bb22f --- /dev/null +++ b/flowlogs-transit-gateway/versions.tf @@ -0,0 +1 @@ +../common/versions.tf \ No newline at end of file