flowlogs-transit-gateway: add retention, custom log format

CHANGELOG.md

@@ -354,3 +354,9 @@
   - flowlogs
     - add use_flowlog_custom_format
     - add flowlog_custom_format
+
+* 2.9.10 -- 2023-10-16
+  - flowlogs-transit-gateway
+    - add use_flowlog_custom_format
+    - add flowlog_custom_format
+    - use default retention of 6 months, allow it as a variable

common/version.tf

@@ -1,5 +1,5 @@
 locals {
-  _module_version = "2.9.9"
+  _module_version = "2.9.10"
   _module_names = {
     "_main_" = "aws-vpc-setup"
 

flowlogs-transit-gateway/README.md

@@ -31,6 +31,9 @@ module "flowlogs-transit-gateway" {
   transit_gateway_id   = aws_transit_gateway.gateway.id
   flowlog_bucket_arn   = data.terraform_remote_state.common.infrastructure_east.flowlogs_arn
   flowlog_role_arn     = data.terraform_remote_state.common.outputs.role_flowlogs_arn
+  ## optional
+  # use_flowlog_custom_format = true
+  # flowlog_custom_format = "${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status} ${flow-direction} ${traffic-path}"
 
   tags = {}
 }
@@ -86,11 +89,14 @@ No modules.
 | <a name="input_account_id"></a> [account\_id](#input\_account\_id) | AWS Account ID (default: will pull from current user) | `string` | `""` | no |
 | <a name="input_enable_kinesis"></a> [enable\_kinesis](#input\_enable\_kinesis) | Flag to enable AWS Kinesis streams for flow logs | `bool` | `true` | no |
 | <a name="input_flowlog_bucket_arn"></a> [flowlog\_bucket\_arn](#input\_flowlog\_bucket\_arn) | S3 Bucket to hold the VPC flowlogs | `string` | n/a | yes |
+| <a name="input_flowlog_custom_format"></a> [flowlog\_custom\_format](#input\_flowlog\_custom\_format) | If use\_flowlog\_custom\_format defined, use the custom format listed here. See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html | `string` | `"${account-id} ${vpc-id} ${subnet-id} ${interface-id} ${start} ${end} ${pkt-srcaddr} ${srcaddr} ${srcport} ${pkt-dstaddr} ${dstaddr} ${dstport} ${protocol} ${packets} ${bytes} ${tcp-flags} ${log-status} ${action} ${type} ${flow-direction} ${traffic-path}"` | no |
 | <a name="input_flowlog_role_arn"></a> [flowlog\_role\_arn](#input\_flowlog\_role\_arn) | IAM Role with proper permissions to allow writing VPC flowlogs to cloudwatch logs and streamss | `string` | n/a | yes |
 | <a name="input_label"></a> [label](#input\_label) | Text label associated with the Transit Gateway | `string` | n/a | yes |
 | <a name="input_override_prefixes"></a> [override\_prefixes](#input\_override\_prefixes) | Override built-in prefixes by component. This should be used primarily for common infrastructure things | `map(string)` | `{}` | no |
+| <a name="input_retention_in_days"></a> [retention\_in\_days](#input\_retention\_in\_days) | Number of days to keep cloudwatch logs (default is 180). See the documentation for available values. | `number` | `180` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | AWS Tags to apply to appropriate resources (S3, KMS).  Do not include safeguard tags here, use the data\_safeguard field for such things. | `map(string)` | `{}` | no |
 | <a name="input_transit_gateway_id"></a> [transit\_gateway\_id](#input\_transit\_gateway\_id) | ID of the Transit Gateway | `string` | n/a | yes |
+| <a name="input_use_flowlog_custom_format"></a> [use\_flowlog\_custom\_format](#input\_use\_flowlog\_custom\_format) | Flag to control the use of a custom format. See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html | `bool` | `false` | no |
 
 ## Outputs
 

flowlogs-transit-gateway/main.tf

@@ -32,6 +32,9 @@
 *   transit_gateway_id   = aws_transit_gateway.gateway.id
 *   flowlog_bucket_arn   = data.terraform_remote_state.common.infrastructure_east.flowlogs_arn
 *   flowlog_role_arn     = data.terraform_remote_state.common.outputs.role_flowlogs_arn
+*   ## optional
+*   # use_flowlog_custom_format = true
+*   # flowlog_custom_format = "${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status} ${flow-direction} ${traffic-path}"
 *
 *   tags = {}
 * }
@@ -61,6 +64,7 @@ resource "aws_flow_log" "flowlog_s3" {
   traffic_type             = "ALL"
   transit_gateway_id       = var.transit_gateway_id
   max_aggregation_interval = 60
+  log_format               = var.use_flowlog_custom_format ? var.flowlog_custom_format : null
 
   tags = merge(
     local.base_tags,
@@ -78,7 +82,8 @@ resource "aws_flow_log" "flowlog_s3" {
 # flowlog, cloudwatch
 #---
 resource "aws_cloudwatch_log_group" "flowlog" {
-  name = format("%v%v_%v_%v", local._prefixes["log-group"], "tgw", var.label, local.region)
+  name              = format("%v%v_%v_%v", local._prefixes["log-group"], "tgw", var.label, local.region)
+  retention_in_days = var.retention_in_days
 
   tags = merge(
     local.base_tags,
@@ -93,6 +98,7 @@ resource "aws_flow_log" "flowlog_cloudwatch" {
   traffic_type             = "ALL"
   transit_gateway_id       = var.transit_gateway_id
   max_aggregation_interval = 60
+  log_format               = var.use_flowlog_custom_format ? var.flowlog_custom_format : null
 
   tags = merge(
     local.base_tags,

flowlogs-transit-gateway/variables.tf

@@ -24,3 +24,32 @@ variable "enable_kinesis" {
   default     = true
 }
 
+variable "retention_in_days" {
+  description = "Number of days to keep cloudwatch logs (default is 180). See the documentation for available values."
+  type        = number
+  default     = 180
+
+  # valid and allowed here: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180
+  # valid but excluded here: 365, 400, 545, 731, 1827, 2192, 2557, 2922, 3288, 3653
+  validation {
+    condition     = var.retention_in_days > 0 && var.retention_in_days <= 180 && contains([1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180], var.retention_in_days)
+    error_message = "VPC flowlogs cloudwatch logs must not be 0 (infinite), and be between 1 and 180 days (180 is default)."
+  }
+}
+
+# https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html
+# https://aws.amazon.com/blogs/aws/learn-from-your-vpc-flow-logs-with-additional-meta-data/
+
+variable "use_flowlog_custom_format" {
+  description = "Flag to control the use of a custom format. See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html"
+  type        = bool
+  default     = false
+}
+
+variable "flowlog_custom_format" {
+  description = "If use_flowlog_custom_format defined, use the custom format listed here. See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html"
+  type        = string
+  # flowlog default, not what we will usehere
+  #  default  = "${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}"
+  default = "$${account-id} $${vpc-id} $${subnet-id} $${interface-id} $${start} $${end} $${pkt-srcaddr} $${srcaddr} $${srcport} $${pkt-dstaddr} $${dstaddr} $${dstport} $${protocol} $${packets} $${bytes} $${tcp-flags} $${log-status} $${action} $${type} $${flow-direction} $${traffic-path}"
+}