diff --git a/CHANGELOG.md b/CHANGELOG.md index 56a94f7..0db4ec7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -430,3 +430,8 @@ * 2.10.8 -- 2024-11-18 - vpc-interface-endpoint - add zone_name, zone_id to output + +* 2.11.0 -- 2024-12-11 + - flowlog + - added field enable_s3 so we can stop creating S3 flow logs (and go only to data firehose) + - remove splunk resources, as splunk is gone from our environment diff --git a/common/version.tf b/common/version.tf index 25ec117..e7da970 100644 --- a/common/version.tf +++ b/common/version.tf @@ -1,5 +1,5 @@ locals { - _module_version = "2.10.8" + _module_version = "2.11.0" _module_names = { "_main_" = "aws-vpc-setup" diff --git a/flowlogs/README.md b/flowlogs/README.md index c1803c9..a7f70c1 100644 --- a/flowlogs/README.md +++ b/flowlogs/README.md @@ -114,3 +114,119 @@ No modules. | Name | Description | |------|-------------| | [kinesis\_flowlog\_arn](#output\_kinesis\_flowlog\_arn) | VPC Flowlog Kinesis stream ARN | + + +# About aws-vpc-setup :: flowlogs + +This submodule creates VPC flow logs. It creates one for private and one for public subnets. Run this once +per VPC. You will need to run the flow logs role once before (aws-vpc-setup/flowlogs-role). + +This also creates cloudwatch logs and cloudwatch streams (Kinesis). It no longer creates a configuration file in `setup/` +to be used in Splunk. Distribute this configuration to the splunk team for flowlog ingestion, as we have +removed Splunk from our environment. + +``` +# setup/aws_kinesis_tasks.lgs-vpc2-dice-dev.conf +[ma6-gov-flowlogs-us-gov-east-1_lg-vpc2-dice-dev] +account = ma6-gov +format = CloudWatchLogs +index = aws_vpc_flow_logs +region = us-gov-east-1 +sourcetype = aws:cloudwatchlogs:vpcflow +init_stream_position = LATEST +stream_names = lgs-vpc2-dice-dev +``` + +# Usage + +```hcl +module "flowlogs" { + source = "git@github.e.it.census.gov:terraform-modules/aws-vpc-setup.git//flowlogs" + vpc_id = var.vpc_id + vpc_full_name = var.vpc_full_name + account_alias = "ma6-gov" + public_subnets_ids = module.subnets.public_subnets_ids + private_subnets_ids = module.subnets.private_subnets_ids + flowlog_bucket_arn = data.terraform_remote_state.common.infrastructure_east.flowlogs_arn + flowlog_role_arn = data.terraform_remote_state.common.outputs.role_flowlogs_arn + + ## optional + # public_subnet_ids = [ for s in module.subnets.public_subnets_ids : s.id ] + # private_subnet_ids = [ for s in module.subnets.private_subnets_ids : s.id ] + # use_flowlog_custom_format = true + # flowlog_custom_format = "${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status} ${flow-direction} ${traffic-path}" + # enable_s3 = false + + tags = {} +} +``` + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13 | +| [aws](#requirement\_aws) | >= 3.66.0 | +| [ldap](#requirement\_ldap) | >= 0.5.4 | +| [local](#requirement\_local) | >= 1.0.0 | +| [null](#requirement\_null) | >= 3.0 | +| [random](#requirement\_random) | >= 3.0 | +| [template](#requirement\_template) | >= 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.66.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.flowlog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_cloudwatch_log_subscription_filter.flowlog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_subscription_filter) | resource | +| [aws_flow_log.flowlog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/flow_log) | resource | +| [aws_flow_log.flowlog_cloudwatch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/flow_log) | resource | +| [aws_flow_log.flowlog_public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/flow_log) | resource | +| [aws_kinesis_stream.flowlog](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_stream) | resource | +| [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_account_alias.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_account_alias) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [account\_alias](#input\_account\_alias) | AWS Account Alias (default: will pull from current account\_alias) | `string` | `""` | no | +| [account\_id](#input\_account\_id) | AWS Account ID (default: will pull from current user) | `string` | `""` | no | +| [enable\_kinesis\_stream](#input\_enable\_kinesis\_stream) | Flag to enable or disable creation of kineis stream for Splunk | `bool` | `false` | no | +| [enable\_s3](#input\_enable\_s3) | Flag to enable or disable creation of flowlog to S3 | `bool` | `false` | no | +| [flowlog\_bucket\_arn](#input\_flowlog\_bucket\_arn) | S3 Bucket to hold the VPC flowlogs | `string` | n/a | yes | +| [flowlog\_custom\_format](#input\_flowlog\_custom\_format) | If use\_flowlog\_custom\_format defined, use the custom format listed here. See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html | `string` | `"${account-id} ${vpc-id} ${subnet-id} ${interface-id} ${start} ${end} ${pkt-srcaddr} ${srcaddr} ${srcport} ${pkt-dstaddr} ${dstaddr} ${dstport} ${protocol} ${packets} ${bytes} ${tcp-flags} ${log-status} ${action} ${type} ${flow-direction} ${traffic-path}"` | no | +| [flowlog\_role\_arn](#input\_flowlog\_role\_arn) | IAM Role with proper permissions to allow writing VPC flowlogs to cloudwatch logs and streamss | `string` | n/a | yes | +| [override\_prefixes](#input\_override\_prefixes) | Override built-in prefixes by component. This should be used primarily for common infrastructure things | `map(string)` | `{}` | no | +| [private\_subnet\_ids](#input\_private\_subnet\_ids) | List of private subnet IDs (not objects) | `list(string)` | `[]` | no | +| [private\_subnets\_ids](#input\_private\_subnets\_ids) | List of private subnet objects including: subnet, label, availability\_zone, id |
list(object({
subnet = string
label = string
availability_zone = string
id = string
tags = optional(map(string))
}))
| `[]` | no | +| [public\_subnet\_ids](#input\_public\_subnet\_ids) | List of public subnet IDs (not objects) | `list(string)` | `[]` | no | +| [public\_subnets\_ids](#input\_public\_subnets\_ids) | List of public subnet objects including: subnet, label, availability\_zone, id |
list(object({
subnet = string
label = string
availability_zone = string
id = string
tags = optional(map(string))
}))
| `[]` | no | +| [retention\_in\_days](#input\_retention\_in\_days) | Number of days to keep cloudwatch logs (default is 180). See the documentation for available values. | `number` | `180` | no | +| [tags](#input\_tags) | AWS Tags to apply to appropriate resources (S3, KMS). Do not include safeguard tags here, use the data\_safeguard field for such things. | `map(string)` | `{}` | no | +| [use\_flowlog\_custom\_format](#input\_use\_flowlog\_custom\_format) | Flag to control the use of a custom format. See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html | `bool` | `false` | no | +| [vpc\_environment](#input\_vpc\_environment) | VPC environment purpose (infrastructure, common, shared, dev, stage, ite, prod) | `string` | `null` | no | +| [vpc\_full\_name](#input\_vpc\_full\_name) | VPC full name component (vpc{index}-{vpc\_name}) | `string` | `null` | no | +| [vpc\_id](#input\_vpc\_id) | VPC ID | `string` | n/a | yes | +| [vpc\_index](#input\_vpc\_index) | VPC index number (integer starting at 1) | `number` | `null` | no | +| [vpc\_name](#input\_vpc\_name) | VPC name component used through the VPC descrbing its purpose (ex: dice-dev) | `string` | `null` | no | +| [vpc\_short\_name](#input\_vpc\_short\_name) | VPC short name component (vpc{index}) | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [kinesis\_flowlog\_arn](#output\_kinesis\_flowlog\_arn) | VPC Flowlog Kinesis stream ARN | + \ No newline at end of file diff --git a/flowlogs/main.tf b/flowlogs/main.tf index 2d6d8eb..0e3796f 100644 --- a/flowlogs/main.tf +++ b/flowlogs/main.tf @@ -4,8 +4,9 @@ * This submodule creates VPC flow logs. It creates one for private and one for public subnets. Run this once * per VPC. You will need to run the flow logs role once before (aws-vpc-setup/flowlogs-role). * -* This also creates cloudwatch logs and cloudwatch streams (Kinesis), and drops a configuration file in `setup/` -* to be used in Splunk. Distribute this configuration to the splunk team for flowlog ingestion. +* This also creates cloudwatch logs and cloudwatch streams (Kinesis). It no longer creates a configuration file in `setup/` +* to be used in Splunk. Distribute this configuration to the splunk team for flowlog ingestion, as we have +* removed Splunk from our environment. * * ``` * # setup/aws_kinesis_tasks.lgs-vpc2-dice-dev.conf @@ -37,6 +38,7 @@ * # private_subnet_ids = [ for s in module.subnets.private_subnets_ids : s.id ] * # use_flowlog_custom_format = true * # flowlog_custom_format = "${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status} ${flow-direction} ${traffic-path}" +* # enable_s3 = false * * tags = {} * } @@ -65,7 +67,7 @@ locals { # use s3 for flow logs, create two, one for the whole vpc, and one for just public #--- resource "aws_flow_log" "flowlog_public" { - for_each = toset(local.public_ids) + for_each = var.enable_s3 ? toset(local.public_ids) : toset([]) log_destination = format("%v/%v-%v/", var.flowlog_bucket_arn, var.vpc_full_name, "public") log_destination_type = "s3" log_format = var.use_flowlog_custom_format ? var.flowlog_custom_format : null @@ -82,6 +84,7 @@ resource "aws_flow_log" "flowlog_public" { # whole VPC resource "aws_flow_log" "flowlog" { + count = var.enable_s3 ? 1 : 0 log_destination = format("%v/%v/", var.flowlog_bucket_arn, var.vpc_full_name) log_destination_type = "s3" log_format = var.use_flowlog_custom_format ? var.flowlog_custom_format : null @@ -149,43 +152,43 @@ resource "aws_cloudwatch_log_subscription_filter" "flowlog" { distribution = "ByLogStream" } -#--- -# generate splunk inputs file -#--- -data "template_file" "splunk_flowlog" { - template = file("${path.module}/templates/aws_kinesis_tasks.conf.tpl") - vars = { - account_id = local.account_id - account_alias = local.account_alias - region = local.region - flowlog_name = aws_cloudwatch_log_group.flowlog.name - flowlog_stream_name = local.flowlog_stream_name - vpc_full_name = var.vpc_full_name - } -} - -resource "null_resource" "splunk_flowlog" { - triggers = { - filename = format("aws_kinesis_tasks.%v-%v.%v.%v.conf", local.account_id, local.account_alias, local.region, local.flowlog_stream_name) - directory = format("%v/setup", path.root) - } - provisioner "local-exec" { - command = "test -d ${self.triggers.directory} || mkdir ${self.triggers.directory}" - } - - # provisioner "local-exec" { - # working_dir = path.root - # command = "test -d setup || mkdir setup" - # } - # provisioner "local-exec" { - # working_dir = "${path.root}/setup" - # command = "echo '${data.template_file.splunk_flowlog_tasks_flowlog.rendered}' > aws_kinesis_tasks.${local.flowlog_stream_name}.conf" - # } -} - -resource "local_file" "splunk_flowlog" { - count = var.enable_kinesis_stream ? 1 : 0 - content = data.template_file.splunk_flowlog.rendered - file_permission = "0644" - filename = format("%v/%v", null_resource.splunk_flowlog.triggers.directory, null_resource.splunk_flowlog.triggers.filename) -} +## #--- +## # generate splunk inputs file +## #--- +## data "template_file" "splunk_flowlog" { +## template = file("${path.module}/templates/aws_kinesis_tasks.conf.tpl") +## vars = { +## account_id = local.account_id +## account_alias = local.account_alias +## region = local.region +## flowlog_name = aws_cloudwatch_log_group.flowlog.name +## flowlog_stream_name = local.flowlog_stream_name +## vpc_full_name = var.vpc_full_name +## } +## } +## +## resource "null_resource" "splunk_flowlog" { +## triggers = { +## filename = format("aws_kinesis_tasks.%v-%v.%v.%v.conf", local.account_id, local.account_alias, local.region, local.flowlog_stream_name) +## directory = format("%v/setup", path.root) +## } +## provisioner "local-exec" { +## command = "test -d ${self.triggers.directory} || mkdir ${self.triggers.directory}" +## } +## +## # provisioner "local-exec" { +## # working_dir = path.root +## # command = "test -d setup || mkdir setup" +## # } +## # provisioner "local-exec" { +## # working_dir = "${path.root}/setup" +## # command = "echo '${data.template_file.splunk_flowlog_tasks_flowlog.rendered}' > aws_kinesis_tasks.${local.flowlog_stream_name}.conf" +## # } +## } +## +## resource "local_file" "splunk_flowlog" { +## count = var.enable_kinesis_stream ? 1 : 0 +## content = data.template_file.splunk_flowlog.rendered +## file_permission = "0644" +## filename = format("%v/%v", null_resource.splunk_flowlog.triggers.directory, null_resource.splunk_flowlog.triggers.filename) +## } diff --git a/flowlogs/variables.tf b/flowlogs/variables.tf index f13e97a..42eb8a2 100644 --- a/flowlogs/variables.tf +++ b/flowlogs/variables.tf @@ -55,3 +55,9 @@ variable "flowlog_custom_format" { # default = "${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}" default = "$${account-id} $${vpc-id} $${subnet-id} $${interface-id} $${start} $${end} $${pkt-srcaddr} $${srcaddr} $${srcport} $${pkt-dstaddr} $${dstaddr} $${dstport} $${protocol} $${packets} $${bytes} $${tcp-flags} $${log-status} $${action} $${type} $${flow-direction} $${traffic-path}" } + +variable "enable_s3" { + description = "Flag to enable or disable creation of flowlog to S3" + type = bool + default = false +}