diff --git a/CHANGELOG.md b/CHANGELOG.md index 0721b23..4a02dbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,10 @@ - rds-postgres - update to use prefix list(s) as variable prefix_list_names +* 2.5.0 -- 2024-12-09 + - emr + - add module for EMR ports and several security groups + # OLDER ## web diff --git a/common/version.tf b/common/version.tf index f403a49..fca0743 100644 --- a/common/version.tf +++ b/common/version.tf @@ -1,3 +1,3 @@ locals { - _module_version = "2.4.0" + _module_version = "2.5.0" } diff --git a/emr/README.md b/emr/README.md index f0ef9cd..f1623ad 100644 --- a/emr/README.md +++ b/emr/README.md @@ -1,6 +1,13 @@ # About emr -This describes how to use the aws-common-security-groups submodule for emr. +This describes how to use the aws-common-security-groups submodule for emr. It will create several +security groups, with the proper interrelationships, as follows: + + - emr-core-tasks-node + - emr-master-node + - emr-service-access + - emr-studio-engine + - emr-studio ## Usage @@ -11,7 +18,7 @@ module "emr" { vpc_id = var.vpc_id name_prefix = "edl-dev-124567" ## optional - # ingress_prefix_list_names = [ "rds-postgres.edl.project" ] + # ingress_prefix_list_names = [ "onprem-networks.core" ] # egress_prefix_list_names = [ ] ## tags for Name, CostAllocation, and Environment are pre-set, but they can be overriden @@ -19,20 +26,23 @@ module "emr" { } ## ingress_networks -This is the list of network CIDR blocks for inbound access to the ports defined for RDS Postgres. +This is the list of network CIDR blocks for inbound access to the ports defined for EMR. By +default, this list is empty. + +## ingress_prefix_list_names +In order to use a managed prefix list, you may pass a list of names in this field. The prefix lists +will be looked up and the resultant IDs used in the security group for inbound port access to EM +This will fail if the prefix list does not exist. + There is a default set of CIDR blocks provided if this field is not populated. This is comprised of the -Census networks: +Census networks from the prefix list `all-networks.core`: * 148.129.0.0/16: Census class B * 172.16.0.0/12: Census private class B * 192.168.0.0/16: Census private class C -* 10.0.0.0/8: Censsu private class A +* 10.0.0.0/8: Census private class A Passing a null or empty list to this field will ignore the ingress setting on these networks. - -## ingress_prefix_list_names -In order to use a managed prefix list, you may pass a list of names in this field. The prefix lists -will be looked up and the resultant IDs used in the security group for inbound port access to RDS -Postgres. This will fail if the prefix list does not exist. +To use all census networks but exclude cloud networks, you may use the prefix list `onprem-networks.core`. ``` ## Requirements @@ -59,8 +69,10 @@ No modules. | [aws_security_group.sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_vpc_security_group_egress_rule.all](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_egress_rule) | resource | | [aws_vpc_security_group_ingress_rule.cidr_block](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource | +| [aws_vpc_security_group_ingress_rule.prefix_lists](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource | | [aws_vpc_security_group_ingress_rule.security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource | | [aws_vpc_security_group_ingress_rule.self](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource | +| [aws_ec2_managed_prefix_list.ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_managed_prefix_list) | data source | | [aws_vpc.this_vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source | ## Inputs @@ -77,4 +89,6 @@ No modules. ## Outputs -No outputs. +| Name | Description | +|------|-------------| +| [security\_group\_ids](#output\_security\_group\_ids) | Map of security groups created for AWS for each of the EMR groupings | diff --git a/emr/data.prefix_lists.tf b/emr/data.prefix_lists.tf new file mode 100644 index 0000000..47385ce --- /dev/null +++ b/emr/data.prefix_lists.tf @@ -0,0 +1,15 @@ +data "aws_ec2_managed_prefix_list" "ingress" { + for_each = toset(var.ingress_prefix_list_names) + filter { + name = "prefix-list-name" + values = [each.key] + } +} + +# data "aws_ec2_managed_prefix_list" "egress" { +# for_each = toset(var.egress_prefix_list_names) +# filter { +# name = "prefix-list-name" +# values = [each.key] +# } +# } diff --git a/emr/main.tf b/emr/main.tf index 1410e21..19d14aa 100644 --- a/emr/main.tf +++ b/emr/main.tf @@ -1,7 +1,14 @@ /** * # About emr * -* This describes how to use the aws-common-security-groups submodule for emr. +* This describes how to use the aws-common-security-groups submodule for emr. It will create several +* security groups, with the proper interrelationships, as follows: +* +* - emr-core-tasks-node +* - emr-master-node +* - emr-service-access +* - emr-studio-engine +* - emr-studio * * ## Usage * @@ -12,7 +19,7 @@ * vpc_id = var.vpc_id * name_prefix = "edl-dev-124567" * ## optional -* # ingress_prefix_list_names = [ "rds-postgres.edl.project" ] +* # ingress_prefix_list_names = [ "onprem-networks.core" ] * # egress_prefix_list_names = [ ] * * ## tags for Name, CostAllocation, and Environment are pre-set, but they can be overriden @@ -20,20 +27,23 @@ * } * * ## ingress_networks -* This is the list of network CIDR blocks for inbound access to the ports defined for RDS Postgres. +* This is the list of network CIDR blocks for inbound access to the ports defined for EMR. By +* default, this list is empty. +* +* ## ingress_prefix_list_names +* In order to use a managed prefix list, you may pass a list of names in this field. The prefix lists +* will be looked up and the resultant IDs used in the security group for inbound port access to EM +* This will fail if the prefix list does not exist. +* * There is a default set of CIDR blocks provided if this field is not populated. This is comprised of the -* Census networks: +* Census networks from the prefix list `all-networks.core`: * * 148.129.0.0/16: Census class B * * 172.16.0.0/12: Census private class B * * 192.168.0.0/16: Census private class C -* * 10.0.0.0/8: Censsu private class A +* * 10.0.0.0/8: Census private class A * * Passing a null or empty list to this field will ignore the ingress setting on these networks. -* -* ## ingress_prefix_list_names -* In order to use a managed prefix list, you may pass a list of names in this field. The prefix lists -* will be looked up and the resultant IDs used in the security group for inbound port access to RDS -* Postgres. This will fail if the prefix list does not exist. +* To use all census networks but exclude cloud networks, you may use the prefix list `onprem-networks.core`. * ``` */ @@ -52,8 +62,7 @@ data "aws_vpc" "this_vpc" { ## } locals { - n_all = ["0.0.0.0/0"] - n_census = ["148.129.0.0/16", "192.168.0.0/16", "172.16.0.0/12", "10.0.0.0/8"] + n_all = ["0.0.0.0/0"] ingress_networks = var.ingress_networks == null ? [] : var.ingress_networks egress_networks = var.egress_networks == null ? [] : var.egress_networks @@ -72,7 +81,7 @@ locals { sg_c1 = flatten([for k, v in local.sg : [for i in v.ingress : merge(i, { key = k, label = format("%v:%v:%v", k, i.from, i.proto) - cidr_blocks = try(i.cidr_blocks, null) == "incoming" ? local.ingress_networks : [] + cidr_blocks = try(i.cidr_blocks, null) == "%%INCOMING%%" ? local.ingress_networks : [] } )]]) sg_cidr = flatten([for sg in local.sg_c1 : [for c in sg.cidr_blocks : merge(sg, { @@ -86,9 +95,13 @@ locals { security_group_name = i } )]]) + sg_pl = flatten([for sg in local.sg_c1 : [for plk, plv in data.aws_ec2_managed_prefix_list.ingress : merge(sg, { + prefix_list_label = format("%v:%v", sg.label, plk) + prefix_list_id = plv.id + } + )]]) } - # create group with just egress. Add all ingress via secondary resource resource "aws_security_group" "sg" { for_each = local.sg @@ -96,21 +109,6 @@ resource "aws_security_group" "sg" { description = trimspace(format("%v %v", var.description_prefix, each.value.description)) vpc_id = var.vpc_id - ## dynamic "ingress" { - ## for_each = { for i in each.value.ingress : format("%v:%v", i.from, i.proto) => merge({ label = format("%v:%v", i.from, i.proto) }, i) } - ## iterator = p - ## content { - ## description = p.value.short - ## from_port = p.value.from - ## to_port = try(p.value.to, p.value.from) - ## protocol = p.value.proto - ## cidr_blocks = try(p.value.cidr_blocks, null) == "incoming" ? p.value.ingress_networks : [] - ## # prefix_list_ids = - ## security_groups = length(try(p.value.ingress_security_groups, [])) > 0 ? [for k, v in aws_security_group.sg : v.id if contains(p.value.ingress_security_groups, k)] : [] - ## self = try(p.value.self, false) - ## } - ## } - tags = merge( local.base_tags, var.tags, @@ -168,3 +166,14 @@ resource "aws_vpc_security_group_ingress_rule" "cidr_block" { cidr_ipv4 = each.value.cidr_block } +# ingress: by prefix_list +resource "aws_vpc_security_group_ingress_rule" "prefix_lists" { + for_each = { for x in local.sg_plr : x.prefix_list_label => x } + + security_group_id = aws_security_group.sg[each.value.key].id + description = each.value.short + from_port = each.value.from + to_port = each.value.to + ip_protocol = each.value.proto + prefix_list_id = each.value.prefix_list_id +} diff --git a/emr/main.tf.off b/emr/main.tf.off deleted file mode 100644 index 49970d4..0000000 --- a/emr/main.tf.off +++ /dev/null @@ -1,43 +0,0 @@ - -locals { - sg_name_emr_master = "edl-prod-7530562-emr-master-node" - sg_description_emr_master = "Security group for EMR Master Node" -} - -data "aws_security_groups" "emr_sg" { - filter { - name = "vpc-id" - values = [local.vpc_id] - } - filter { - name = "tag:Name" - values = ["sg-edl-prod-7530562-emr-core-tasks-node", "sg-edl-prod-7530562-emr-studio", "sg-edl-prod-7530562-emr-service-access"] - } -} - -module "sg_emr_master" { - source = "git@github.e.it.census.gov:terraform-modules/aws-common-security-groups.git//custom?ref=tf-upgrade" - vpc_id = local.vpc_id - name = local.sg_name_emr_master - description = local.sg_description_emr_master - - ingress_security_groups = tolist(data.aws_security_groups.emr_sg.ids) - ingress_port_list = [ - [22, 22, "tcp", "SSH", var.census_private_cidr], - [80, 80, "tcp", "HTTP", var.census_private_cidr], - [443, 443, "tcp", "HTTPS", var.census_private_cidr], - [9870, 9870, "tcp", "HDFS Name Node", var.census_private_cidr], - [18080, 18080, "tcp", "Spark History Server", var.census_private_cidr], - [8088, 8088, "tcp", "Resource Manager", var.census_private_cidr], - ] - use_vpc_cidr = false - enable_self = true - tags = merge( - local.common_tags, - ) -} - -output "sg_emr_master_id" { - description = "Emr Master node security group" - value = module.sg_emr_master.this_security_group_id -} diff --git a/emr/settings.yml b/emr/settings.yml index 9105686..383d5df 100644 --- a/emr/settings.yml +++ b/emr/settings.yml @@ -8,12 +8,12 @@ security-groups: to: 9864 proto: tcp short: "HDFS Data Node" - cidr_blocks: incoming + cidr_blocks: "%%INCOMING%%" - from: 8042 to: 8042 proto: tcp short: "Node Manager" - cidr_blocks: incoming + cidr_blocks: "%%INCOMING%%" ingress_security_groups: - emr-master-node - emr-service-access @@ -27,32 +27,32 @@ security-groups: to: 22 proto: tcp short: "SSH" - cidr_blocks: incoming + cidr_blocks: "%%INCOMING%%" - from: 80 to: 80 proto: tcp short: "HTTP" - cidr_blocks: incoming + cidr_blocks: "%%INCOMING%%" - from: 443 to: 443 proto: tcp short: "HTTPS" - cidr_blocks: incoming + cidr_blocks: "%%INCOMING%%" - from: 8088 to: 8088 proto: tcp short: "Resource Manager" - cidr_blocks: incoming + cidr_blocks: "%%INCOMING%%" - from: 9870 to: 9870 proto: tcp short: "HDFS Name Node" - cidr_blocks: incoming + cidr_blocks: "%%INCOMING%%" - from: 18080 to: 18080 proto: tcp short: "Spark History Server" - cidr_blocks: incoming + cidr_blocks: "%%INCOMING%%" ingress_security_groups: - emr-core-tasks-node - emr-studio diff --git a/emr/variables.tf b/emr/variables.tf index 447f897..4f7cfc9 100644 --- a/emr/variables.tf +++ b/emr/variables.tf @@ -62,12 +62,12 @@ variable "tags" { } } -## variable "ingress_prefix_list_names" { -## description = "List of prefix list names for ingress access" -## type = list(string) -## default = [] -## } -## +ariable "ingress_prefix_list_names" { + description = "List of prefix list names for ingress access" + type = list(string) + default = ["all-networks.core"] +} + ## variable "egress_prefix_list_names" { ## description = "List of prefix list names for eggress access" ## type = list(string)