From 5e1b429c9a9a62c680c0594bd1db19b019e4c825 Mon Sep 17 00:00:00 2001 From: badra001 Date: Tue, 18 Mar 2025 16:05:08 -0400 Subject: [PATCH] * 2.11.0 -- 2025-03-18 - s3-logs - copy s3-flow-logs to s3-logs to normalize names (to reuse for other stuff) --- CHANGELOG.md | 4 + common/version.tf | 2 +- s3-flow-logs/main.tf | 8 +- s3-logs/README.md | 187 ++++++++++++++++++++++++++++++++++ s3-logs/data.tf | 1 + s3-logs/defaults.tf | 1 + s3-logs/kms.tf.off | 19 ++++ s3-logs/main.tf | 193 ++++++++++++++++++++++++++++++++++++ s3-logs/outputs.tf | 9 ++ s3-logs/policy_data.tf | 49 +++++++++ s3-logs/prefixes.tf | 1 + s3-logs/variables.common.tf | 1 + s3-logs/variables.tf | 23 +++++ s3-logs/version.tf | 1 + 14 files changed, 495 insertions(+), 4 deletions(-) create mode 100644 s3-logs/README.md create mode 120000 s3-logs/data.tf create mode 120000 s3-logs/defaults.tf create mode 100644 s3-logs/kms.tf.off create mode 100644 s3-logs/main.tf create mode 100644 s3-logs/outputs.tf create mode 100644 s3-logs/policy_data.tf create mode 120000 s3-logs/prefixes.tf create mode 120000 s3-logs/variables.common.tf create mode 100644 s3-logs/variables.tf create mode 120000 s3-logs/version.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index f1fff49..3f346a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -399,3 +399,7 @@ * 2.10.1 -- 2025-02-13 - config - add retention_period_in_days with a default of 3 years (down from 7 years) + +* 2.11.0 -- 2025-03-18 + - s3-logs + - copy s3-flow-logs to s3-logs to normalize names (to reuse for other stuff) diff --git a/common/version.tf b/common/version.tf index 80dad4c..ac684e2 100644 --- a/common/version.tf +++ b/common/version.tf @@ -1,3 +1,3 @@ locals { - _module_version = "2.10.1" + _module_version = "2.11.0" } diff --git a/s3-flow-logs/main.tf b/s3-flow-logs/main.tf index 6dcffd1..05551e7 100644 --- a/s3-flow-logs/main.tf +++ b/s3-flow-logs/main.tf @@ -167,10 +167,12 @@ resource "aws_s3_bucket_lifecycle_configuration" "flowlogs" { abort_incomplete_multipart_upload { days_after_initiation = 1 } - filter {} + filter { + prefix = "" + } expiration { - days = 900 - expired_object_delete_marker = false + days = 900 + # expired_object_delete_marker = false } noncurrent_version_expiration { noncurrent_days = 900 diff --git a/s3-logs/README.md b/s3-logs/README.md new file mode 100644 index 0000000..a348763 --- /dev/null +++ b/s3-logs/README.md @@ -0,0 +1,187 @@ +# aws-inf-setup :: s3-flow-logs + +This set up the needed components for S3 VPC flow log bucket. Only one flow log bucket is +needed + +* S3 bucket +* S3 bucket objects (key prefixes, aka "directories") +* S3 bucket policy + +# Usage +Here is a simple example, the one most commonly expected to be used. + +```hcl +module "flowlogs" { + source = "git@github.e.it.census.gov:terraform-modules/aws-inf-setup.git//s3-flow-flowlogs" +} +``` + +This one can be used if you need to customize stuff, though really, the defaults are all built +for a reason, and deployment code (i.e., Ansible) will expect these defaults to be used in +variable file generation. + +```hcl +module "flowlogs_full" { + source = "git@github.e.it.census.gov:terraform-modules/aws-inf-setup.git//s3-flow-flowlogs" + + # optional + account_alias = "do2-govcloud" + bucket_name = "inf-flowlogs-123456789012" + + # flowlogs is generally not needed and not recommended + component_tags = { + "s3" = { + "SpecialTag1" = "something" + "SpecialTag2" = "somethingElse" + } + } +} +``` + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | +| [null](#provider\_null) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_s3_bucket.flowlogs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [aws_s3_bucket_acl.flowlogs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource | +| [aws_s3_bucket_ownership_controls.flowlogs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_ownership_controls) | resource | +| [aws_s3_bucket_policy.flowlogs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource | +| [aws_s3_bucket_public_access_block.flowlogs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | +| [aws_s3_bucket_server_side_encryption_configuration.flowlogs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | +| [aws_s3_bucket_versioning.flowlogs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource | +| [null_resource.policy_delay](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.flowlogs_s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | +| [aws_regions.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/regions) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [account\_alias](#input\_account\_alias) | AWS Account Alias | `string` | `""` | no | +| [account\_id](#input\_account\_id) | AWS Account ID (default will pull from current user) | `string` | `""` | no | +| [bucket\_name](#input\_bucket\_name) | VPC Flow Logs S3 bucket name | `string` | `""` | no | +| [bucket\_name\_prefix](#input\_bucket\_name\_prefix) | VPC Flow Logs S3 bucket prefix, prepended to the AWS account ID to make the bucket name. | `string` | `"inf-flowlogs"` | no | +| [component\_tags](#input\_component\_tags) | Additional tags for Components (s3, kms, ddb) | `map(map(string))` | 
{
  "ddb": {},
  "kms": {},
  "s3": {}
}
| no | +| [override\_prefixes](#input\_override\_prefixes) | Override built-in prefixes by component (efs, s3, ebs, kms, role, policy, security-group). This should be used primarily for common infrastructure things | `map(string)` | `{}` | no | +| [tags](#input\_tags) | AWS Tags to apply to appropriate resources (S3, KMS). Do not include safeguard tags here, use the data\_safeguard field for such things. | `map(string)` | `{}` | no | +| [versioning\_configuration](#input\_versioning\_configuration) | S3 Versioning Configuration (Enabled, Disabled, Suspended). To disable, use Suspended if existing bucket and Disabled if new | `string` | `"Disabled"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [bucket\_arn](#output\_bucket\_arn) | VPC Flow Logs S3 bucket ARN | +| [bucket\_id](#output\_bucket\_id) | VPC Flow Logs S3 bucket ID | + + +# aws-inf-setup :: s3-logs + +This set up the needed components for S3 log bucket. + +* S3 bucket +* S3 bucket objects (key prefixes, aka "directories") +* S3 bucket policy + +# Usage +Here is a simple example, the one most commonly expected to be used. + +```hcl +module "logs" { + source = "git@github.e.it.census.gov:terraform-modules/aws-inf-setup.git//s3-logs" +} +``` + +This one can be used if you need to customize stuff, though really, the defaults are all built +for a reason, and deployment code (i.e., Ansible) will expect these defaults to be used in +variable file generation. + +```hcl +module "logs_full" { + source = "git@github.e.it.census.gov:terraform-modules/aws-inf-setup.git//s3-logs" + + # optional + account_alias = "do2-govcloud" + bucket_name = "inf-flowlogs-123456789012" + + component_tags = { + "s3" = { + "SpecialTag1" = "something" + "SpecialTag2" = "somethingElse" + } + } +} +``` + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | +| [null](#provider\_null) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_s3_bucket.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [aws_s3_bucket_acl.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource | +| [aws_s3_bucket_intelligent_tiering_configuration.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_intelligent_tiering_configuration) | resource | +| [aws_s3_bucket_lifecycle_configuration.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_lifecycle_configuration) | resource | +| [aws_s3_bucket_ownership_controls.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_ownership_controls) | resource | +| [aws_s3_bucket_policy.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource | +| [aws_s3_bucket_public_access_block.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | +| [aws_s3_bucket_server_side_encryption_configuration.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | +| [aws_s3_bucket_versioning.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource | +| [null_resource.policy_delay](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.logs_s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | +| [aws_regions.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/regions) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [account\_alias](#input\_account\_alias) | AWS Account Alias | `string` | `""` | no | +| [account\_id](#input\_account\_id) | AWS Account ID (default will pull from current user) | `string` | `""` | no | +| [bucket\_name](#input\_bucket\_name) | Logs S3 bucket name | `string` | `""` | no | +| [bucket\_name\_prefix](#input\_bucket\_name\_prefix) | Logs S3 bucket prefix, prepended to the AWS account ID to make the bucket name. | `string` | `"inf-logs"` | no | +| [component\_tags](#input\_component\_tags) | Additional tags for Components (s3, kms, ddb) | `map(map(string))` | 
{
  "ddb": {},
  "kms": {},
  "s3": {}
}
| no | +| [override\_prefixes](#input\_override\_prefixes) | Override built-in prefixes by component (efs, s3, ebs, kms, role, policy, security-group). This should be used primarily for common infrastructure things | `map(string)` | `{}` | no | +| [tags](#input\_tags) | AWS Tags to apply to appropriate resources (S3, KMS). Do not include safeguard tags here, use the data\_safeguard field for such things. | `map(string)` | `{}` | no | +| [versioning\_configuration](#input\_versioning\_configuration) | S3 Versioning Configuration (Enabled, Disabled, Suspended). To disable, use Suspended if existing bucket and Disabled if new | `string` | `"Disabled"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [bucket\_arn](#output\_bucket\_arn) | Logs S3 bucket ARN | +| [bucket\_id](#output\_bucket\_id) | Logs S3 bucket ID | + \ No newline at end of file diff --git a/s3-logs/data.tf b/s3-logs/data.tf new file mode 120000 index 0000000..995624d --- /dev/null +++ b/s3-logs/data.tf @@ -0,0 +1 @@ +../common/data.tf \ No newline at end of file diff --git a/s3-logs/defaults.tf b/s3-logs/defaults.tf new file mode 120000 index 0000000..a5556ac --- /dev/null +++ b/s3-logs/defaults.tf @@ -0,0 +1 @@ +../common/defaults.tf \ No newline at end of file diff --git a/s3-logs/kms.tf.off b/s3-logs/kms.tf.off new file mode 100644 index 0000000..89c7831 --- /dev/null +++ b/s3-logs/kms.tf.off @@ -0,0 +1,19 @@ +resource "aws_kms_key" "key" { + description = "KMS CMK for logs" + enable_key_rotation = true + policy = data.aws_iam_policy_document.key_policy_combined.json + + tags = merge( + local.base_tags, + var.tags, + { + "boc:aws:region" = local.region + Name = local.name + }, + ) +} + +resource "aws_kms_alias" "key" { + name = "alias/${local.kms_key_name}" + target_key_id = aws_kms_key.key.key_id +} diff --git a/s3-logs/main.tf b/s3-logs/main.tf new file mode 100644 index 0000000..a721a40 --- /dev/null +++ b/s3-logs/main.tf @@ -0,0 +1,193 @@ +/* +* # aws-inf-setup :: s3-logs +* +* This set up the needed components for S3 log bucket. +* +* * S3 bucket +* * S3 bucket objects (key prefixes, aka "directories") +* * S3 bucket policy +* +* # Usage +* Here is a simple example, the one most commonly expected to be used. +* +* ```hcl +* module "logs" { +* source = "git@github.e.it.census.gov:terraform-modules/aws-inf-setup.git//s3-logs" +* } +* ``` +* +* This one can be used if you need to customize stuff, though really, the defaults are all built +* for a reason, and deployment code (i.e., Ansible) will expect these defaults to be used in +* variable file generation. +* +* ```hcl +* module "logs_full" { +* source = "git@github.e.it.census.gov:terraform-modules/aws-inf-setup.git//s3-logs" +* +* # optional +* account_alias = "do2-govcloud" +* bucket_name = "inf-flowlogs-123456789012" +* +* component_tags = { +* "s3" = { +* "SpecialTag1" = "something" +* "SpecialTag2" = "somethingElse" +* } +* } +* } +* ``` +*/ + +locals { + account_id = var.account_id != "" ? var.account_id : data.aws_caller_identity.current.account_id + regions = [for r in tolist(data.aws_regions.current.names) : r if startswith(r, "us-")] + region = data.aws_region.current.name + account_environment = data.aws_arn.current.partition == "aws-us-gov" ? "gov" : "ew" + + bucket_name = var.bucket_name != "" ? var.bucket_name : format("%v-%v-%v", var.bucket_name_prefix, local.account_id, local.region) + + base_tags = { + "Organization" = "census:aditcio:csvd" + "boc:tf_module_version" = local._module_version + "boc:created_by" = "terraform" + } +} + +#--- +# s3 +#--- +resource "aws_s3_bucket" "logs" { + bucket = local.bucket_name + # acl = "log-delivery-write" + + # need to create the inf_ key used for infrastucture things like + # vpc flow, cloudtrail, config, sns, sqs + + lifecycle { + prevent_destroy = true + ignore_changes = [tags["boc:tf_module_version"]] + } + + # probably want some migration of old data to some other location + # like glacier + + tags = merge( + var.tags, + local.base_tags, + lookup(var.component_tags, "s3", {}), + tomap({ Name = local.bucket_name }), + ) + + provisioner "local-exec" { + command = "sleep 30" + } +} + +resource "aws_s3_bucket_public_access_block" "logs" { + bucket = aws_s3_bucket.logs.id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_policy" "logs" { + bucket = aws_s3_bucket.logs.id + policy = data.aws_iam_policy_document.logs_s3.json + depends_on = [null_resource.policy_delay] +} + +# consider making sleep value a variable +resource "null_resource" "policy_delay" { + triggers = { + bucket = aws_s3_bucket.logs.id + } + provisioner "local-exec" { + when = create + command = "sleep 120" + } +} + +resource "aws_s3_bucket_ownership_controls" "logs" { + bucket = aws_s3_bucket.logs.id + rule { + object_ownership = "BucketOwnerPreferred" + } +} + +resource "aws_s3_bucket_acl" "logs" { + bucket = aws_s3_bucket.logs.id + acl = "log-delivery-write" +} + +## resource "aws_s3_bucket_logging" "logs" { +## bucket = aws_s3_bucket.logs.id +## target_bucket = var.access_log_bucket +## target_prefix = format("%s/%s/", var.access_log_bucket_prefix, local.bucket_name) +## } + +# see docs: https://aws.amazon.com/premiumsupport/knowledge-center/s3-server-access-log-not-delivered/ + +resource "aws_s3_bucket_server_side_encryption_configuration" "logs" { + bucket = aws_s3_bucket.logs.id + rule { + apply_server_side_encryption_by_default { + # sse_algorithm = "aws:kms" + sse_algorithm = "AES256" + } + bucket_key_enabled = true + } +} + +resource "aws_s3_bucket_versioning" "logs" { + bucket = aws_s3_bucket.logs.id + versioning_configuration { + status = var.versioning_configuration + } +} + + +# m-21-31 says 12 months active and 18 months cold +# * https://www.whitehouse.gov/wp-content/uploads/2021/08/M-21-31-Improving-the-Federal-Governments-Investigative-and-Remediation-Capabilities-Related-to-Cybersecurity-Incidents.pdf +# going to use intellegent tiering which is < 365 active, and > 365 deep, along with delete > 30m (900 days) +# * https://docs.aws.amazon.com/AmazonS3/latest/userguide/intelligent-tiering.html +# may need clarification if the 18 months cold is additional +# * 30 days IA +# * 90 days instant archive +# * 180 days archive access (glacier flexible) +# * 365 deep archive (glacier) +resource "aws_s3_bucket_lifecycle_configuration" "logs" { + bucket = aws_s3_bucket.logs.id + + rule { + id = "logs" + status = "Enabled" + abort_incomplete_multipart_upload { + days_after_initiation = 1 + } + filter { + prefix = "" + } + expiration { + days = 900 + # expired_object_delete_marker = false + } + noncurrent_version_expiration { + noncurrent_days = 900 + } + } +} + +resource "aws_s3_bucket_intelligent_tiering_configuration" "logs" { + bucket = aws_s3_bucket.logs.id + name = "log-bucket" + + tiering { + access_tier = "ARCHIVE_ACCESS" + days = 180 + } + tiering { + access_tier = "DEEP_ARCHIVE_ACCESS" + days = 365 + } +} diff --git a/s3-logs/outputs.tf b/s3-logs/outputs.tf new file mode 100644 index 0000000..388e9f9 --- /dev/null +++ b/s3-logs/outputs.tf @@ -0,0 +1,9 @@ +output "bucket_id" { + description = "Logs S3 bucket ID" + value = aws_s3_bucket.logs.id +} + +output "bucket_arn" { + description = "Logs S3 bucket ARN" + value = aws_s3_bucket.logs.arn +} diff --git a/s3-logs/policy_data.tf b/s3-logs/policy_data.tf new file mode 100644 index 0000000..bc12291 --- /dev/null +++ b/s3-logs/policy_data.tf @@ -0,0 +1,49 @@ +data "aws_iam_policy_document" "logs_s3" { + statement { + sid = "AWSLogDeliveryWrite" + effect = "Allow" + actions = ["s3:PutObject"] + principals { + type = "Service" + identifiers = ["delivery.logs.amazonaws.com"] + } + resources = [format("%v/*", aws_s3_bucket.logs.arn)] + condition { + test = "StringEquals" + variable = "s3:x-amz-acl" + values = ["bucket-owner-full-control"] + } + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [local.account_id] + } + condition { + test = "ArnLike" + variable = "aws:SourceArn" + # values = [for r in local.regions : format("arn:%v:logs:%v:%v:*", data.aws_arn.current.partition, r, local.account_id)] + values = [format("arn:%v:logs:%v:%v:*", data.aws_arn.current.partition, local.region, local.account_id)] + } + } + statement { + sid = "AWSLogDeliveryAclCheck" + effect = "Allow" + actions = ["s3:GetBucketAcl"] + principals { + type = "Service" + identifiers = ["delivery.logs.amazonaws.com"] + } + resources = [aws_s3_bucket.logs.arn] + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [local.account_id] + } + condition { + test = "ArnLike" + variable = "aws:SourceArn" + # values = [for r in local.regions : format("arn:%v:logs:%v:%v:*", data.aws_arn.current.partition, r, local.account_id)] + values = [format("arn:%v:logs:%v:%v:*", data.aws_arn.current.partition, local.region, local.account_id)] + } + } +} diff --git a/s3-logs/prefixes.tf b/s3-logs/prefixes.tf new file mode 120000 index 0000000..7e265d5 --- /dev/null +++ b/s3-logs/prefixes.tf @@ -0,0 +1 @@ +../common/prefixes.tf \ No newline at end of file diff --git a/s3-logs/variables.common.tf b/s3-logs/variables.common.tf new file mode 120000 index 0000000..7439ed8 --- /dev/null +++ b/s3-logs/variables.common.tf @@ -0,0 +1 @@ +../common/variables.common.tf \ No newline at end of file diff --git a/s3-logs/variables.tf b/s3-logs/variables.tf new file mode 100644 index 0000000..fc0eac1 --- /dev/null +++ b/s3-logs/variables.tf @@ -0,0 +1,23 @@ +variable "bucket_name" { + description = "Logs S3 bucket name" + type = string + default = "" +} + +variable "bucket_name_prefix" { + description = "Logs S3 bucket prefix, prepended to the AWS account ID to make the bucket name." + type = string + default = "inf-logs" +} + +variable "component_tags" { + description = "Additional tags for Components (s3, kms, ddb)" + type = map(map(string)) + default = { "s3" = {}, "kms" = {}, "ddb" = {} } +} + +variable "versioning_configuration" { + description = "S3 Versioning Configuration (Enabled, Disabled, Suspended). To disable, use Suspended if existing bucket and Disabled if new" + type = string + default = "Disabled" +} diff --git a/s3-logs/version.tf b/s3-logs/version.tf new file mode 120000 index 0000000..b83c5b7 --- /dev/null +++ b/s3-logs/version.tf @@ -0,0 +1 @@ +../common/version.tf \ No newline at end of file