add EMR submodule and outputs

terraform-modules · Dec 16, 2024 · 5bfb39e · 5bfb39e
1 parent d2cd633
commit 5bfb39e
Show file tree

Hide file tree

Showing 8 changed files with 97 additions and 98 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -51,6 +51,10 @@
   - rds-postgres
    - update to use prefix list(s) as variable prefix_list_names
 
+* 2.5.0 -- 2024-12-09
+  - emr
+   - add module for EMR ports and several security groups
+
 # OLDER
 
 ## web

diff --git a/common/version.tf b/common/version.tf
@@ -1,3 +1,3 @@
 locals {
-  _module_version = "2.4.0"
+  _module_version = "2.5.0"
 }
diff --git a/emr/README.md b/emr/README.md
@@ -1,6 +1,13 @@
 # About emr
 
-This describes how to use the aws-common-security-groups submodule for emr.
+This describes how to use the aws-common-security-groups submodule for emr.  It will create several
+security groups, with the proper interrelationships, as follows:
+
+  - emr-core-tasks-node
+  - emr-master-node
+  - emr-service-access
+  - emr-studio-engine
+  - emr-studio
 
 ## Usage
 
@@ -11,28 +18,31 @@ module "emr" {
   vpc_id = var.vpc_id
   name_prefix = "edl-dev-124567"
   ## optional
-  # ingress_prefix_list_names = [ "rds-postgres.edl.project" ]
+  # ingress_prefix_list_names = [ "onprem-networks.core" ]
   # egress_prefix_list_names = [ ]
 
   ## tags for Name, CostAllocation, and Environment are pre-set, but they can be overriden
   # tags = { }
 }
 
 ## ingress_networks
-This is the list of network CIDR blocks for inbound access to the ports defined for RDS Postgres.
+This is the list of network CIDR blocks for inbound access to the ports defined for EMR.  By
+default, this list is empty.
+
+## ingress_prefix_list_names
+In order to use a managed prefix list, you may pass a list of names in this field. The prefix lists
+will be looked up and the resultant IDs used in the security group for inbound port access to EM
+This will fail if the prefix list does not exist.
+
 There is a default set of CIDR blocks provided if this field is not populated.  This is comprised of the
-Census networks:
+Census networks from the prefix list `all-networks.core`:
 * 148.129.0.0/16: Census class B
 * 172.16.0.0/12: Census private class B
 * 192.168.0.0/16: Census private class C
-* 10.0.0.0/8: Censsu private class A
+* 10.0.0.0/8: Census private class A
 
 Passing a null or empty list to this field will ignore the ingress setting on these networks.
-
-## ingress_prefix_list_names
-In order to use a managed prefix list, you may pass a list of names in this field. The prefix lists
-will be looked up and the resultant IDs used in the security group for inbound port access to RDS
-Postgres.  This will fail if the prefix list does not exist.
+To use all census networks but exclude cloud networks, you may use the prefix list `onprem-networks.core`.
 ```
 
 ## Requirements
@@ -59,8 +69,10 @@ No modules.
 | [aws_security_group.sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
 | [aws_vpc_security_group_egress_rule.all](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_egress_rule) | resource |
 | [aws_vpc_security_group_ingress_rule.cidr_block](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource |
+| [aws_vpc_security_group_ingress_rule.prefix_lists](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource |
 | [aws_vpc_security_group_ingress_rule.security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource |
 | [aws_vpc_security_group_ingress_rule.self](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource |
+| [aws_ec2_managed_prefix_list.ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_managed_prefix_list) | data source |
 | [aws_vpc.this_vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source |
 
 ## Inputs
@@ -77,4 +89,6 @@ No modules.
 
 ## Outputs
 
-No outputs.
+| Name | Description |
+|------|-------------|
+| <a name="output_security_group_ids"></a> [security\_group\_ids](#output\_security\_group\_ids) | Map of security groups created for AWS for each of the EMR groupings |
diff --git a/emr/data.prefix_lists.tf b/emr/data.prefix_lists.tf
@@ -0,0 +1,15 @@
+data "aws_ec2_managed_prefix_list" "ingress" {
+  for_each = toset(var.ingress_prefix_list_names)
+  filter {
+    name   = "prefix-list-name"
+    values = [each.key]
+  }
+}
+
+# data "aws_ec2_managed_prefix_list" "egress" {
+#   for_each = toset(var.egress_prefix_list_names)
+#   filter {
+#     name   = "prefix-list-name"
+#     values = [each.key]
+#   }
+# }
diff --git a/emr/main.tf b/emr/main.tf
@@ -1,7 +1,14 @@
 /**
 * # About emr
 *
-* This describes how to use the aws-common-security-groups submodule for emr.
+* This describes how to use the aws-common-security-groups submodule for emr.  It will create several
+* security groups, with the proper interrelationships, as follows:
+*
+*   - emr-core-tasks-node
+*   - emr-master-node
+*   - emr-service-access
+*   - emr-studio-engine
+*   - emr-studio
 *
 * ## Usage
 *
@@ -12,28 +19,31 @@
 *   vpc_id = var.vpc_id
 *   name_prefix = "edl-dev-124567"
 *   ## optional
-*   # ingress_prefix_list_names = [ "rds-postgres.edl.project" ]
+*   # ingress_prefix_list_names = [ "onprem-networks.core" ]
 *   # egress_prefix_list_names = [ ]
 *
 *   ## tags for Name, CostAllocation, and Environment are pre-set, but they can be overriden
 *   # tags = { }
 * }
 *
 * ## ingress_networks
-* This is the list of network CIDR blocks for inbound access to the ports defined for RDS Postgres.
+* This is the list of network CIDR blocks for inbound access to the ports defined for EMR.  By 
+* default, this list is empty.
+*
+* ## ingress_prefix_list_names
+* In order to use a managed prefix list, you may pass a list of names in this field. The prefix lists
+* will be looked up and the resultant IDs used in the security group for inbound port access to EM
+* This will fail if the prefix list does not exist.
+*
 * There is a default set of CIDR blocks provided if this field is not populated.  This is comprised of the
-* Census networks:
+* Census networks from the prefix list `all-networks.core`:
 * * 148.129.0.0/16: Census class B
 * * 172.16.0.0/12: Census private class B
 * * 192.168.0.0/16: Census private class C
-* * 10.0.0.0/8: Censsu private class A
+* * 10.0.0.0/8: Census private class A
 *
 * Passing a null or empty list to this field will ignore the ingress setting on these networks.
-*
-* ## ingress_prefix_list_names
-* In order to use a managed prefix list, you may pass a list of names in this field. The prefix lists
-* will be looked up and the resultant IDs used in the security group for inbound port access to RDS
-* Postgres.  This will fail if the prefix list does not exist.
+* To use all census networks but exclude cloud networks, you may use the prefix list `onprem-networks.core`.
 * ```
 */
 
@@ -52,8 +62,7 @@ data "aws_vpc" "this_vpc" {
 ## }
 
 locals {
-  n_all    = ["0.0.0.0/0"]
-  n_census = ["148.129.0.0/16", "192.168.0.0/16", "172.16.0.0/12", "10.0.0.0/8"]
+  n_all = ["0.0.0.0/0"]
 
   ingress_networks = var.ingress_networks == null ? [] : var.ingress_networks
   egress_networks  = var.egress_networks == null ? [] : var.egress_networks
@@ -72,7 +81,7 @@ locals {
   sg_c1 = flatten([for k, v in local.sg : [for i in v.ingress : merge(i, {
     key         = k,
     label       = format("%v:%v:%v", k, i.from, i.proto)
-    cidr_blocks = try(i.cidr_blocks, null) == "incoming" ? local.ingress_networks : []
+    cidr_blocks = try(i.cidr_blocks, null) == "%%INCOMING%%" ? local.ingress_networks : []
     }
   )]])
   sg_cidr = flatten([for sg in local.sg_c1 : [for c in sg.cidr_blocks : merge(sg, {
@@ -86,31 +95,20 @@ locals {
     security_group_name = i
     }
   )]])
+  sg_pl = flatten([for sg in local.sg_c1 : [for plk, plv in data.aws_ec2_managed_prefix_list.ingress : merge(sg, {
+    prefix_list_label = format("%v:%v", sg.label, plk)
+    prefix_list_id    = plv.id
+    }
+  )]])
 }
 
-
 # create group with just egress. Add all ingress via secondary resource
 resource "aws_security_group" "sg" {
   for_each    = local.sg
   name        = format("%v-%v", var.name_prefix, each.key)
   description = trimspace(format("%v %v", var.description_prefix, each.value.description))
   vpc_id      = var.vpc_id
 
-  ##   dynamic "ingress" {
-  ##     for_each = { for i in each.value.ingress : format("%v:%v", i.from, i.proto) => merge({ label = format("%v:%v", i.from, i.proto) }, i) }
-  ##     iterator = p
-  ##     content {
-  ##       description = p.value.short
-  ##       from_port   = p.value.from
-  ##       to_port     = try(p.value.to, p.value.from)
-  ##       protocol    = p.value.proto
-  ##       cidr_blocks = try(p.value.cidr_blocks, null) == "incoming" ? p.value.ingress_networks : []
-  ##       #      prefix_list_ids = 
-  ##       security_groups = length(try(p.value.ingress_security_groups, [])) > 0 ? [for k, v in aws_security_group.sg : v.id if contains(p.value.ingress_security_groups, k)] : []
-  ##       self            = try(p.value.self, false)
-  ##     }
-  ##   }
-
   tags = merge(
     local.base_tags,
     var.tags,
@@ -168,3 +166,14 @@ resource "aws_vpc_security_group_ingress_rule" "cidr_block" {
   cidr_ipv4         = each.value.cidr_block
 }
 
+# ingress: by prefix_list
+resource "aws_vpc_security_group_ingress_rule" "prefix_lists" {
+  for_each = { for x in local.sg_plr : x.prefix_list_label => x }
+
+  security_group_id = aws_security_group.sg[each.value.key].id
+  description       = each.value.short
+  from_port         = each.value.from
+  to_port           = each.value.to
+  ip_protocol       = each.value.proto
+  prefix_list_id    = each.value.prefix_list_id
+}
diff --git a/emr/main.tf.off b/emr/main.tf.off
diff --git a/emr/settings.yml b/emr/settings.yml
@@ -8,12 +8,12 @@ security-groups:
         to: 9864
         proto: tcp
         short: "HDFS Data Node"
-        cidr_blocks: incoming
+        cidr_blocks: "%%INCOMING%%"
       - from: 8042
         to: 8042
         proto: tcp
         short: "Node Manager"
-        cidr_blocks: incoming
+        cidr_blocks: "%%INCOMING%%"
     ingress_security_groups:
       - emr-master-node
       - emr-service-access
@@ -27,32 +27,32 @@ security-groups:
         to: 22
         proto: tcp
         short: "SSH"
-        cidr_blocks: incoming
+        cidr_blocks: "%%INCOMING%%"
       - from: 80
         to: 80
         proto: tcp
         short: "HTTP"
-        cidr_blocks: incoming
+        cidr_blocks: "%%INCOMING%%"
       - from: 443
         to: 443
         proto: tcp
         short: "HTTPS"
-        cidr_blocks: incoming
+        cidr_blocks: "%%INCOMING%%"
       - from: 8088
         to: 8088
         proto: tcp
         short: "Resource Manager"
-        cidr_blocks: incoming
+        cidr_blocks: "%%INCOMING%%"
       - from: 9870
         to: 9870
         proto: tcp
         short: "HDFS Name Node"
-        cidr_blocks: incoming
+        cidr_blocks: "%%INCOMING%%"
       - from: 18080
         to: 18080
         proto: tcp
         short: "Spark History Server"
-        cidr_blocks: incoming
+        cidr_blocks: "%%INCOMING%%"
     ingress_security_groups:
       - emr-core-tasks-node
       - emr-studio

diff --git a/emr/variables.tf b/emr/variables.tf
@@ -62,12 +62,12 @@ variable "tags" {
   }
 }
 
-## variable "ingress_prefix_list_names" {
-##   description = "List of prefix list names for ingress access"
-##   type        = list(string)
-##   default     = []
-## }
-## 
+ariable "ingress_prefix_list_names" {
+  description = "List of prefix list names for ingress access"
+  type        = list(string)
+  default     = ["all-networks.core"]
+}
+
 ## variable "egress_prefix_list_names" {
 ##   description = "List of prefix list names for eggress access"
 ##   type        = list(string)