Split into submodule

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
     - id: terraform_fmt
     - id: terraform_docs_replace
       args: ['table']
-      exclude: common/.*
+      exclude: common/*.tf
       exclude: version.tf
 
 - repo: https://github.com/pre-commit/pre-commit-hooks

CHANGELOG.md

@@ -12,3 +12,15 @@
 
 * v1.2 -- 20201104
   - fix s3 bucket permission to not output statement with deny if no IP and VPCE provided
+
+* v2.0 -- 20201104
+  - change to aws-s3
+  - create submodules `standard` and `title26`
+  - standard
+    - no versioning
+    - no explicit boc:authority tag
+    - auto create encryption key if not provided already
+  - title26
+    - versioning
+    - explicit boc:authority tag
+    - auto create encryption key

README.md

@@ -1,46 +1,30 @@
-# About aws-t26-s3
+# About aws-s3
+
+This module allows you to create an S3 bucket, either a standard bucket or a bucket used for FTI (Title26)
+data.
 
 # Usage
 
+* [Standard](standard)
+
 ```hcl
 module "mybucket" {
-  source = "git@github.e.it.census.gov:terraform-modules/aws-t26-s3.git"
+  source = "git@github.e.it.census.gov:terraform-modules/aws-s3.git//standard"
 
-  bucket_name = "myt26bucket"
+  bucket_name = "mynormalbucket"
+  access_log_bucket = "accesslogs"
 }
 ```
 
-## Requirements
-
-No requirements.
-
-## Providers
-
-| Name | Version |
-|------|---------|
-| aws | n/a |
-| null | n/a |
+* [Title 26](title26)
 
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|------|---------|:--------:|
-| \_module\_version | Module version number | `string` | `"1.2"` | no |
-| access\_log\_bucket | Server Access Logging Bucket ID | `string` | n/a | yes |
-| access\_log\_bucket\_prefix | Access log bucket prefix, to which the bucket name will be appended to make the target\_prefix | `string` | `"s3"` | no |
-| allowed\_cidr | List of allowed source IPs (NOT from within the VPC) | `list(string)` | `[]` | no |
-| allowed\_endpoints | List of allowed VPC endpoint IDs | `list(string)` | `[]` | no |
-| bucket\_folders | List of folders (keys) to create after creation of bucket | `list(string)` | `[]` | no |
-| bucket\_name | AWS Bucket Name | `string` | n/a | yes |
-| force\_destroy | Sets force\_destroy to allow the bucket and contents to be deleted.  The deletion may take a very long time | `bool` | `false` | no |
-| kms\_key\_id | AWS KMS Key ID (one per bucket) | `string` | `""` | no |
-| tags | AWS Tags | `map(string)` | `{}` | no |
+```hcl
+module "mybucket" {
+  source = "git@github.e.it.census.gov:terraform-modules/aws-s3.git//title26"
 
-## Outputs
+  bucket_name = "mytitle26bucket"
+  access_log_bucket = "accesslogs"
+}
+```
 
-| Name | Description |
-|------|-------------|
-| kms\_key\_arn | Created KMS Key ARN |
-| kms\_key\_id | Created KMS Key ID |
-| s3\_bucket\_arn | Created S3 Bucket ARN |
-| s3\_bucket\_id | Created S3 Bucket ID |
+See each section for additional information.

common/README.md

@@ -0,0 +1,34 @@
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| aws | n/a |
+| null | n/a |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| access\_log\_bucket | Server Access Logging Bucket ID | `string` | n/a | yes |
+| access\_log\_bucket\_prefix | Access log bucket prefix, to which the bucket name will be appended to make the target\_prefix | `string` | `"s3"` | no |
+| allowed\_cidr | List of allowed source IPs (NOT from within the VPC).  If empty, there will be no restrictions on source IP.  If provided, you must also use allowed\_endpoints for access within a VPC. | `list(string)` | `[]` | no |
+| allowed\_endpoints | List of allowed VPC endpoint IDs.  If used, it will enable access to the bucket from the specific VPC endpoints. | `list(string)` | `[]` | no |
+| bucket\_folders | List of folders (keys) to create after creation of bucket. They will have object metadata provided based on metadata\_tags and data\_safeguard labels. | `list(string)` | `[]` | no |
+| bucket\_name | AWS Bucket Name.  Standard prefix will be applied here, do not include here. | `string` | n/a | yes |
+| force\_destroy | Sets force\_destroy to allow the bucket and contents to be deleted.  The deletion may take a very long time based on the number of objects.  You normally want to update this to true, apply, and then destroy the resource. | `bool` | `false` | no |
+| kms\_key\_id | AWS KMS Key ID (one per bucket).  This is currently ignored. | `string` | `""` | no |
+| metadata\_tags | AWS S3 Custom metadata (prefix x-amzn-meta- automatically included, not needed here).  If data\_safeguard labels are applied, they will be incorporated on any bucket objects created. | `map(string)` | `{}` | no |
+| tags | AWS Tags to apply to appropriate resources (S3, KMS).  Do not include safeguard tags here, use the data\_safeguard field for such things. | `map(string)` | `{}` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| kms\_key\_arn | Created KMS Key ARN |
+| kms\_key\_id | Created KMS Key ID |
+| s3\_bucket\_arn | Created S3 Bucket ARN |
+| s3\_bucket\_id | Created S3 Bucket ID |

common/defaults.tf

@@ -0,0 +1,7 @@
+# local._defaults["data_safeguards"]
+
+locals {
+  _defaults = {
+    data_safeguards = ["title13", "title26", "title42", "pii", "title5"]
+  }
+}

main.tf → common/resources.tf

@@ -1,18 +1,3 @@
-/* 
-* # About aws-t26-s3
-* 
-* # Usage 
-*
-* ```hcl
-* module "mybucket" {
-*   source = "git@github.e.it.census.gov:terraform-modules/aws-t26-s3.git"
-*
-*   bucket_name = "myt26bucket"
-* }
-* ```
-* 
-*/
-
 locals {
   name        = var.bucket_name
   bucket_name = format("%s%s", local._prefixes["s3"], var.bucket_name)
@@ -34,13 +19,30 @@ locals {
   s3_bucket_conditions_list = list(local.condition_allowed_cidr, local.condition_allowed_endpoints)
   s3_bucket_conditions      = [for x in local.s3_bucket_conditions_list : x if length(x.values) > 0]
 
-  enforced_tags = {
-    "boc:safeguard" = "title26"
-  }
+  #  enforced_tags = {
+  #    "boc:safeguard" = "title26"
+  #  }
   base_tags = {
-    "boc:tf_module_version" = var._module_version
+    "boc:tf_module_version" = local._module_version
     "boc:created_by"        = "terraform"
   }
+  # strip spaces, convert to lowercase, make distinct, sort. Remove those not in the _defaults
+  add_safeguard_tags      = local.enable_title26 ? ["title26"] : []
+  _default_safeguard_tags = { for d in local._defaults["data_safeguards"] : d => d }
+  safeguard_tags          = sort(distinct(compact(concat([for t in var.data_safeguards : lookup(local._default_safeguard_tags, lower(replace(t, " ", "")), "")], local.add_safeguard_tags))))
+  add_tags = {
+    safeguard = {
+      "exists"     = { "boc:safeguard" = join(",", local.safeguard_tags) }
+      "not_exists" = {}
+    }
+  }
+  enforced_tags = merge(
+    local.add_tags["safeguard"][length(local.safeguard_tags) > 0 ? "exists" : "not_exists"]
+  )
+  metadata_tags = merge(
+    var.metadata_tags,
+    { for k, v in local.enforced_tags : format("x-amzn-meta-%v", replace(k, "/\\W/", "_")) => v }
+  )
 }
 
 #---
@@ -61,7 +63,7 @@ resource "aws_s3_bucket" "this" {
   }
 
   versioning {
-    enabled = true
+    enabled = local.versioning
   }
 
   logging {
@@ -180,21 +182,23 @@ resource "null_resource" "policy_delay" {
 }
 
 resource "aws_s3_bucket_object" "this_objects" {
-  bucket = aws_s3_bucket.this.id
-  count  = length(var.bucket_folders)
-  key    = format("%s/", element(var.bucket_folders, count.index))
-  source = "/dev/null"
+  for_each = toset(var.bucket_folders)
+  bucket   = aws_s3_bucket.this.id
+  key      = format("%s/", each.key)
+  source   = "/dev/null"
+  metadata = local.metadata_tags
 
   depends_on = [null_resource.policy_delay]
 }
 
 #---
 # create a key and alias if not specified
+# right now, this can't use an external key, it has to create one per bucket
 #---
 resource "aws_kms_key" "key" {
   description         = "KMS CMK for S3 bucket ${local.name}"
   enable_key_rotation = true
-  #policy              = data.aws_iam_policy_document.key.json
+  # policy              = data.aws_iam_policy_document.key.json
 
   tags = merge(
     local.base_tags,

common/variables.tf

@@ -0,0 +1,58 @@
+variable "bucket_name" {
+  description = "AWS Bucket Name.  Standard prefix will be applied here, do not include here."
+  type        = string
+}
+
+variable "bucket_folders" {
+  description = "List of folders (keys) to create after creation of bucket. They will have object metadata provided based on metadata_tags and data_safeguard labels."
+  type        = list(string)
+  default     = []
+}
+
+variable "kms_key_id" {
+  description = "AWS KMS Key ID (one per bucket).  This is currently ignored."
+  type        = string
+  default     = ""
+}
+
+variable "tags" {
+  description = "AWS Tags to apply to appropriate resources (S3, KMS).  Do not include safeguard tags here, use the data_safeguard field for such things."
+  type        = map(string)
+  default     = {}
+}
+
+variable "metadata_tags" {
+  description = "AWS S3 Custom metadata (prefix x-amzn-meta- automatically included, not needed here).  If data_safeguard labels are applied, they will be incorporated on any bucket objects created."
+  type        = map(string)
+  default     = {}
+}
+
+variable "access_log_bucket_prefix" {
+  description = "Access log bucket prefix, to which the bucket name will be appended to make the target_prefix"
+  type        = string
+  default     = "s3"
+}
+
+variable "access_log_bucket" {
+  description = "Server Access Logging Bucket ID"
+  type        = string
+  #  default     = null
+}
+
+variable "allowed_cidr" {
+  description = "List of allowed source IPs (NOT from within the VPC).  If empty, there will be no restrictions on source IP.  If provided, you must also use allowed_endpoints for access within a VPC."
+  type        = list(string)
+  default     = []
+}
+
+variable "allowed_endpoints" {
+  description = "List of allowed VPC endpoint IDs.  If used, it will enable access to the bucket from the specific VPC endpoints."
+  type        = list(string)
+  default     = []
+}
+
+variable "force_destroy" {
+  description = "Sets force_destroy to allow the bucket and contents to be deleted.  The deletion may take a very long time based on the number of objects.  You normally want to update this to true, apply, and then destroy the resource."
+  type        = bool
+  default     = false
+}

common/version.tf

@@ -0,0 +1,3 @@
+locals {
+  _module_version = "2.0"
+}

standard/README.md

@@ -0,0 +1,57 @@
+# About aws-s3 :: standard
+
+This submodule allows you to create an S3 bucket using the standard prefixes and settings required for  
+non-FTI Data.  This includes
+- Server Access Logging
+
+Other configurations such as versioning or data safegurad tagging (only on the bucket/keys) are oiptional
+
+# Usage
+
+```hcl
+module "mybucket" {
+  source = "git@github.e.it.census.gov:terraform-modules/aws-s3.git//standard"
+
+  bucket_name = "mynormalbucket"
+  access_log_bucket = "mylogbucket"
+}
+```
+
+This automaticaly creates an AWS KMS key used just for this bucket.
+
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| aws | n/a |
+| null | n/a |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| access\_log\_bucket | Server Access Logging Bucket ID | `string` | n/a | yes |
+| access\_log\_bucket\_prefix | Access log bucket prefix, to which the bucket name will be appended to make the target\_prefix | `string` | `"s3"` | no |
+| allowed\_cidr | List of allowed source IPs (NOT from within the VPC).  If empty, there will be no restrictions on source IP.  If provided, you must also use allowed\_endpoints for access within a VPC. | `list(string)` | `[]` | no |
+| allowed\_endpoints | List of allowed VPC endpoint IDs.  If used, it will enable access to the bucket from the specific VPC endpoints. | `list(string)` | `[]` | no |
+| bucket\_folders | List of folders (keys) to create after creation of bucket. They will have object metadata provided based on metadata\_tags and data\_safeguard labels. | `list(string)` | `[]` | no |
+| bucket\_name | AWS Bucket Name.  Standard prefix will be applied here, do not include here. | `string` | n/a | yes |
+| data\_safeguards | Selected available safeguards which apply to the data in the bucket | `list(string)` | `[]` | no |
+| enable\_title26 | Flag to enable bucket with Title 26 (FTI) settings | `bool` | `false` | no |
+| force\_destroy | Sets force\_destroy to allow the bucket and contents to be deleted.  The deletion may take a very long time based on the number of objects.  You normally want to update this to true, apply, and then destroy the resource. | `bool` | `false` | no |
+| kms\_key\_id | AWS KMS Key ID (one per bucket).  This is currently ignored. | `string` | `""` | no |
+| metadata\_tags | AWS S3 Custom metadata (prefix x-amzn-meta- automatically included, not needed here).  If data\_safeguard labels are applied, they will be incorporated on any bucket objects created. | `map(string)` | `{}` | no |
+| tags | AWS Tags to apply to appropriate resources (S3, KMS).  Do not include safeguard tags here, use the data\_safeguard field for such things. | `map(string)` | `{}` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| kms\_key\_arn | Created KMS Key ARN |
+| kms\_key\_id | Created KMS Key ID |
+| s3\_bucket\_arn | Created S3 Bucket ARN |
+| s3\_bucket\_id | Created S3 Bucket ID |

standard/defaults.tf

@@ -0,0 +1 @@
+../common/defaults.tf

standard/main.tf

@@ -0,0 +1,27 @@
+/* 
+* # About aws-s3 :: standard
+*
+* This submodule allows you to create an S3 bucket using the standard prefixes and settings required for
+* non-FTI Data.  This includes
+* - Server Access Logging
+*
+* Other configurations such as versioning or data safegurad tagging (only on the bucket/keys) are oiptional
+* 
+* # Usage 
+*
+* ```hcl
+* module "mybucket" {
+*   source = "git@github.e.it.census.gov:terraform-modules/aws-s3.git//standard"
+*
+*   bucket_name = "mynormalbucket"
+*   access_log_bucket = "mylogbucket"
+* }
+* ```
+* 
+* This automaticaly creates an AWS KMS key used just for this bucket.
+*/
+
+locals {
+  enable_title26 = var.enable_title26 ? true : false
+  versioning     = false
+}

standard/outputs.tf

@@ -0,0 +1 @@
+../common/outputs.tf

standard/prefixes.tf

@@ -0,0 +1 @@
+../common/prefixes.tf

standard/resources.tf

@@ -0,0 +1 @@
+../common/resources.tf