- flowlogs

  - add use_flowlog_custom_format
  - add flowlog_custom_format

CHANGELOG.md

@@ -349,3 +349,8 @@
 * 2.9.8 -- 2023-09-28
   - vpc-interface-endpoint
     - add lab-gov-network-nonprod to allow to create dns zones
+
+* 2.9.9 -- 2023-10-12
+  - flowlogs
+    - add use_flowlog_custom_format
+    - add flowlog_custom_format

common/version.tf

@@ -1,5 +1,5 @@
 locals {
-  _module_version = "2.9.8"
+  _module_version = "2.9.9"
   _module_names = {
     "_main_" = "aws-vpc-setup"
 

flowlogs/README.md

@@ -34,6 +34,8 @@ module "flowlogs" {
   ## optional
   # public_subnet_ids    = [ for s in module.subnets.public_subnets_ids : s.id ]
   # private_subnet_ids   = [ for s in module.subnets.private_subnets_ids : s.id ]
+  # use_flowlog_custom_format = true
+  # flowlog_custom_format = "${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status} ${flow-direction} ${traffic-path}"
 
   tags = {}
 }
@@ -88,8 +90,9 @@ No modules.
 |------|-------------|------|---------|:--------:|
 | <a name="input_account_alias"></a> [account\_alias](#input\_account\_alias) | AWS Account Alias (default: will pull from current account\_alias) | `string` | `""` | no |
 | <a name="input_account_id"></a> [account\_id](#input\_account\_id) | AWS Account ID (default: will pull from current user) | `string` | `""` | no |
-| <a name="input_enable_kinesis_stream"></a> [enable\_kinesis\_stream](#input\_enable\_kinesis\_stream) | Flag to enable or disable creation of kineis stream for Splunk | `bool` | `true` | no |
+| <a name="input_enable_kinesis_stream"></a> [enable\_kinesis\_stream](#input\_enable\_kinesis\_stream) | Flag to enable or disable creation of kineis stream for Splunk | `bool` | `false` | no |
 | <a name="input_flowlog_bucket_arn"></a> [flowlog\_bucket\_arn](#input\_flowlog\_bucket\_arn) | S3 Bucket to hold the VPC flowlogs | `string` | n/a | yes |
+| <a name="input_flowlog_custom_format"></a> [flowlog\_custom\_format](#input\_flowlog\_custom\_format) | If use\_flowlog\_custom\_format defined, use the custom format listed here. See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html | `string` | `"${account-id} ${vpc-id} ${subnet-id} ${interface-id} ${start} ${end} ${pkt-srcaddr} ${srcaddr} ${srcport} ${pkt-dstaddr} ${dstaddr} ${dstport} ${protocol} ${packets} ${bytes} ${tcp-flagss} ${log-status} ${action} ${type} ${flow-direction} ${traffic-path}"` | no |
 | <a name="input_flowlog_role_arn"></a> [flowlog\_role\_arn](#input\_flowlog\_role\_arn) | IAM Role with proper permissions to allow writing VPC flowlogs to cloudwatch logs and streamss | `string` | n/a | yes |
 | <a name="input_override_prefixes"></a> [override\_prefixes](#input\_override\_prefixes) | Override built-in prefixes by component. This should be used primarily for common infrastructure things | `map(string)` | `{}` | no |
 | <a name="input_private_subnet_ids"></a> [private\_subnet\_ids](#input\_private\_subnet\_ids) | List of private subnet IDs (not objects) | `list(string)` | `[]` | no |
@@ -98,6 +101,7 @@ No modules.
 | <a name="input_public_subnets_ids"></a> [public\_subnets\_ids](#input\_public\_subnets\_ids) | List of public subnet objects including: subnet, label, availability\_zone, id | <pre>list(object({<br>    subnet            = string<br>    label             = string<br>    availability_zone = string<br>    id                = string<br>    tags              = optional(map(string))<br>  }))</pre> | `[]` | no |
 | <a name="input_retention_in_days"></a> [retention\_in\_days](#input\_retention\_in\_days) | Number of days to keep cloudwatch logs (default is 180). See the documentation for available values. | `number` | `180` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | AWS Tags to apply to appropriate resources (S3, KMS).  Do not include safeguard tags here, use the data\_safeguard field for such things. | `map(string)` | `{}` | no |
+| <a name="input_use_flowlog_custom_format"></a> [use\_flowlog\_custom\_format](#input\_use\_flowlog\_custom\_format) | Flag to control the use of a custom format. See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html | `bool` | `false` | no |
 | <a name="input_vpc_environment"></a> [vpc\_environment](#input\_vpc\_environment) | VPC environment purpose (infrastructure, common, shared, dev, stage, ite, prod) | `string` | `null` | no |
 | <a name="input_vpc_full_name"></a> [vpc\_full\_name](#input\_vpc\_full\_name) | VPC full name component (vpc{index}-{vpc\_name}) | `string` | `null` | no |
 | <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | VPC ID | `string` | n/a | yes |

flowlogs/main.tf

@@ -35,6 +35,8 @@
 *   ## optional
 *   # public_subnet_ids    = [ for s in module.subnets.public_subnets_ids : s.id ]
 *   # private_subnet_ids   = [ for s in module.subnets.private_subnets_ids : s.id ]
+*   # use_flowlog_custom_format = true
+*   # flowlog_custom_format = "${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status} ${flow-direction} ${traffic-path}"
 *
 *   tags = {}
 * }
@@ -66,6 +68,7 @@ resource "aws_flow_log" "flowlog_public" {
   for_each             = toset(local.public_ids)
   log_destination      = format("%v/%v-%v/", var.flowlog_bucket_arn, var.vpc_full_name, "public")
   log_destination_type = "s3"
+  log_format           = var.use_flowlog_custom_format ? var.flowlog_custom_format : null
   #  iam_role_arn         = var.flowlog_role_arn
   traffic_type = "ALL"
   subnet_id    = each.key
@@ -81,6 +84,7 @@ resource "aws_flow_log" "flowlog_public" {
 resource "aws_flow_log" "flowlog" {
   log_destination      = format("%v/%v/", var.flowlog_bucket_arn, var.vpc_full_name)
   log_destination_type = "s3"
+  log_format           = var.use_flowlog_custom_format ? var.flowlog_custom_format : null
   #  iam_role_arn         = var.flowlog_role_arn
   traffic_type = "ALL"
   vpc_id       = var.vpc_id
@@ -108,6 +112,7 @@ resource "aws_cloudwatch_log_group" "flowlog" {
 
 resource "aws_flow_log" "flowlog_cloudwatch" {
   log_destination = aws_cloudwatch_log_group.flowlog.arn
+  log_format      = var.use_flowlog_custom_format ? var.flowlog_custom_format : null
   iam_role_arn    = var.flowlog_role_arn
   traffic_type    = "ALL"
   vpc_id          = var.vpc_id

flowlogs/variables.tf

@@ -23,7 +23,7 @@ variable "private_subnet_ids" {
 variable "enable_kinesis_stream" {
   description = "Flag to enable or disable creation of kineis stream for Splunk"
   type        = bool
-  default     = true
+  default     = false
 }
 
 variable "retention_in_days" {
@@ -38,3 +38,20 @@ variable "retention_in_days" {
     error_message = "VPC flowlogs cloudwatch logs must not be 0 (infinite), and be between 1 and 180 days (180 is default)."
   }
 }
+
+# https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html
+# https://aws.amazon.com/blogs/aws/learn-from-your-vpc-flow-logs-with-additional-meta-data/
+
+variable "use_flowlog_custom_format" {
+  description = "Flag to control the use of a custom format. See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html"
+  type        = bool
+  default     = false
+}
+
+variable "flowlog_custom_format" {
+  description = "If use_flowlog_custom_format defined, use the custom format listed here. See https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html"
+  type        = string
+  # flowlog default, not what we will usehere
+  #  default  = "${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}"
+  default = "$${account-id} $${vpc-id} $${subnet-id} $${interface-id} $${start} $${end} $${pkt-srcaddr} $${srcaddr} $${srcport} $${pkt-dstaddr} $${dstaddr} $${dstport} $${protocol} $${packets} $${bytes} $${tcp-flagss} $${log-status} $${action} $${type} $${flow-direction} $${traffic-path}"
+}