add emr module (first pass)

emr/.tf-control

@@ -0,0 +1,32 @@
+# .tf-control
+# allows for setting a specific command to be used for tf-* commands under this git repo
+# see tf-control.sh help for more info
+
+TFCONTROL_VERSION="1.0.7"
+#TFCOMMAND="terraform_latest"
+TFCOMMAND="terraform_current"
+
+# TF_CLI_CONFIG_FILE=PATH-TO-FILE/.tf-control.tfrc
+# TFARGS=""
+# TFNOLOG=""
+# TFNOCOLOR=""
+
+# from issue: https://github.com/hashicorp/terraform/issues/32901
+# to get to TF 1.4 and beyond in a shared cache environment
+# this is currently in the tf-control.sh script explicitly
+#TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE=1
+
+# use the following to force a specific version.  An upgrade of an existing 0.12.31 to 1.x
+# needs you to cycle through 0.13.17, 0.14.11, and then latest (0.15.5 not needed).  Other
+# steps in between.  See https://github.e.it.census.gov/terraform/support/tree/master/docs/how-to/terraform-upgrade for details
+#
+#TFCOMMAND="terraform_0.12.31"
+#TFCOMMAND="terraform_0.13.7"
+#TFCOMMAND="terraform_0.14.11"
+#TFCOMMAND="terraform_0.15.5"
+#TFCOMMAND="terraform_1.3.10"
+#TFCOMMAND="terraform_1.4.7"
+#TFCOMMAND="terraform_1.5.7"
+#TFCOMMAND="terraform_1.6.6"
+#TFCOMMAND="terraform_1.7.5"
+#TFCOMMAND="terraform_1.8.2"

emr/.tf-control.tfrc

@@ -0,0 +1,24 @@
+TFCONTROL_VERSION="1.0.5"
+
+# https://www.terraform.io/docs/cli/config/config-file.html
+plugin_cache_dir   = "/data/terraform/terraform.d/plugin-cache"
+#disable_checkpoint = true
+
+provider_installation {
+#  filesystem_mirror {
+#    path    = "/apps/terraform/terraform.d/providers"
+#    include = [ "*/*/*" ]
+#  }
+  filesystem_mirror {
+    path    = "/data/terraform/terraform.d/providers"
+    include = [ "*/*/*" ]
+  }
+#  filesystem_mirror {
+#    path    = "/apps/terraform/terraform.d/providers"
+#    include = [ "external.terraform.census.gov/*/*" ]
+#  }
+  direct {
+    include = [ "*/*/*" ]
+  }
+}
+

emr/_module_name.tf

@@ -0,0 +1,3 @@
+locals {
+  _module_name = "aws-common-security-groups/emr"
+}

emr/locals.tf

@@ -0,0 +1,7 @@
+locals {
+  base_tags = {
+    "boc:created_by"        = "terraform"
+    "boc:tf_module_version" = local._module_version
+    "boc:tf_module_name "   = local._module_name
+  }
+}

emr/main.tf

@@ -0,0 +1,208 @@
+/**
+* # About emr
+*
+* This describes how to use the aws-common-security-groups submodule for emr.
+*
+* ## Usage
+*
+* ```hcl
+* module "emr" {
+*   source = "git@github.e.it.census.gov:terraform-modules/aws-common-security-groups.git//emr"
+*
+*   vpc_id = var.vpc_id
+*   name_prefix = "edl-dev-124567"
+*   ## optional
+*   # ingress_prefix_list_names = [ "rds-postgres.edl.project" ]
+*   # egress_prefix_list_names = [ ]
+*
+*   ## tags for Name, CostAllocation, and Environment are pre-set, but they can be overriden
+*   # tags = { }
+* }
+*
+* ## ingress_networks
+* This is the list of network CIDR blocks for inbound access to the ports defined for RDS Postgres.
+* There is a default set of CIDR blocks provided if this field is not populated.  This is comprised of the
+* Census networks:
+* * 148.129.0.0/16: Census class B
+* * 172.16.0.0/12: Census private class B
+* * 192.168.0.0/16: Census private class C
+* * 10.0.0.0/8: Censsu private class A
+*
+* Passing a null or empty list to this field will ignore the ingress setting on these networks.
+*
+* ## ingress_prefix_list_names
+* In order to use a managed prefix list, you may pass a list of names in this field. The prefix lists
+* will be looked up and the resultant IDs used in the security group for inbound port access to RDS
+* Postgres.  This will fail if the prefix list does not exist.
+* ```
+*/
+
+data "aws_vpc" "this_vpc" {
+  id = var.vpc_id
+}
+
+## data "aws_security_group" "ingress_security_groups" {
+##   count = length(var.ingress_security_groups)
+##   id    = element(var.ingress_security_groups, count.index)
+## }
+## 
+## data "aws_security_group" "egress_security_groups" {
+##   count = length(var.egress_security_groups)
+##   id    = element(var.egress_security_groups, count.index)
+## }
+
+locals {
+  n_all    = ["0.0.0.0/0"]
+  n_census = ["148.129.0.0/16", "192.168.0.0/16", "172.16.0.0/12", "10.0.0.0/8"]
+
+  ingress_networks = var.ingress_networks == null ? [] : var.ingress_networks
+  egress_networks  = var.egress_networks == null ? [] : var.egress_networks
+}
+
+locals {
+  vpc_networks              = var.use_vpc_cidr ? [data.aws_vpc.this_vpc.cidr_block] : []
+  external_ingress_networks = compact(concat(local.vpc_networks, local.ingress_networks))
+  #  ingress_sg_names          = zipmap(var.ingress_security_groups, data.aws_security_group.ingress_security_groups[*].name)
+  #  egress_sg_names           = zipmap(var.egress_security_groups, data.aws_security_group.egress_security_groups[*].name)
+}
+
+locals {
+  _sg = yamldecode(file("${path.module}/settings.yml"))
+  sg  = { for sg in local._sg["security-groups"] : sg.name => merge(sg, { ingress_networks = flatten(distinct(compact(concat(local.ingress_networks, sg.vpc_cidr ? [data.aws_vpc.this_vpc.cidr_block] : [])))) }) }
+}
+
+# create group with just egress. Add all ingress via secondary resource
+resource "aws_security_group" "sg" {
+  for_each    = local.sg
+  name        = format("%v-%v", var.name_prefix, each.key)
+  description = trimspace(format("%v %v", var.description_prefix, each.value.description))
+  vpc_id      = var.vpc_id
+
+  egress {
+    description = "ALL"
+    from_port   = 0
+    to_port     = 0
+    protocol    = -1
+    cidr_blocks = local.egress_networks
+  }
+
+  dynamic "ingress" {
+    for_each = { for i in each.value.ingress : format("%v:%v", i.from, i.proto) => merge({ label = format("%v:%v", i.from, i.proto) }, i) }
+    iterator = p
+    content {
+      description = p.value.short
+      from_port   = p.value.from
+      to_port     = try(p.value.to, p.value.from)
+      protocol    = p.value.proto
+      cidr_blocks = try(p.value.cidr_blocks, null) == "incoming" ? p.value.ingress_networks : []
+      #      prefix_list_ids = 
+      security_groups = length(try(p.value.ingress_security_groups, [])) > 0 ? [for k, v in aws_security_group.sg : v.id if contains(p.value.ingress_security_groups, k)] : []
+      self            = try(p.value.self, false)
+    }
+  }
+
+  tags = merge(
+    local.base_tags,
+    var.tags,
+    { "Name" = format("sg-%v-%v", var.name_prefix, each.key) }
+  )
+}
+
+
+##   # ingress with prefix lists
+##   dynamic "ingress" {
+##     for_each = length(var.ingress_prefix_list_names) > 0 ? local.port_map["external"] : toset([])
+##     iterator = p
+##     content {
+##       description     = "${local.short_description}: ${p.value["description"]}"
+##       from_port       = p.value["from"]
+##       to_port         = p.value["to"]
+##       protocol        = p.value["proto"]
+##       prefix_list_ids = [for pl in data.aws_ec2_managed_prefix_list.ingress : pl.id]
+##     }
+##   }
+## 
+## 
+##   # ingress security group ids (all)
+##   dynamic "ingress" {
+##     for_each = local.ingress_sg
+##     iterator = sg
+##     content {
+##       description     = "${local.short_description}: ${local.ingress_sg_names[sg.value]}"
+##       from_port       = 0
+##       to_port         = 0
+##       protocol        = -1
+##       security_groups = [sg.value]
+##     }
+##   }
+## 
+## 
+##   # ingress self (list with one or zero items)
+##   dynamic "ingress" {
+##     for_each = local.self
+##     iterator = sg
+##     content {
+##       description = "${local.short_description}: from self"
+##       from_port   = 0
+##       to_port     = 0
+##       protocol    = -1
+##       self        = true
+##     }
+##   }
+## 
+## 
+## 
+##   # egress security group ids (all)
+##   dynamic "egress" {
+##     for_each = local.egress_sg
+##     iterator = sg
+##     content {
+##       description     = "${local.short_description}: ${local.egress_sg_names[sg]}"
+##       from_port       = 0
+##       to_port         = 0
+##       protocol        = -1
+##       security_groups = [sg]
+##     }
+##   }
+## 
+##   # egress with prefix lists
+##   dynamic "egress" {
+##     for_each = length(var.egress_prefix_list_names) > 0 ? local.port_map["external"] : toset([])
+##     iterator = p
+##     content {
+##       description     = "${local.short_description}: ${local.egress_sg_names[sg]}"
+##       from_port       = 0
+##       to_port         = 0
+##       protocol        = -1
+##       prefix_list_ids = [for pl in data.aws_ec2_managed_prefix_list.egress : pl.id]
+##     }
+##   }
+
+
+## resource "aws_vpc_security_group_ingress_rule" "example" {
+##   security_group_id = aws_security_group.example.id
+## 
+##   cidr_ipv4   = "10.0.0.0/8"
+##   from_port   = 80
+##   ip_protocol = "tcp"
+##   to_port     = 80
+## }
+## 
+## @@@
+## 
+##   {
+##     "description" = "EMR Service Access"
+##     "ingress" = [
+##       {
+##         "from" = 9443
+##         "proto" = "tcp"
+##         "security_groups" = "emr-master-node"
+##         "short" = "Master Node"
+##         "to" = 9443
+##       },
+##     ]
+##     "name" = "emr-service-access"
+##     "self" = false
+##     "vpc_cidr" = false
+##   },
+## 

emr/main.tf.off

@@ -0,0 +1,43 @@
+
+locals {
+  sg_name_emr_master        = "edl-prod-7530562-emr-master-node"
+  sg_description_emr_master = "Security group for EMR Master Node"
+}
+
+data "aws_security_groups" "emr_sg" {
+  filter {
+    name   = "vpc-id"
+    values = [local.vpc_id]
+  }
+  filter {
+    name   = "tag:Name"
+    values = ["sg-edl-prod-7530562-emr-core-tasks-node", "sg-edl-prod-7530562-emr-studio", "sg-edl-prod-7530562-emr-service-access"]
+  }
+}
+
+module "sg_emr_master" {
+  source      = "git@github.e.it.census.gov:terraform-modules/aws-common-security-groups.git//custom?ref=tf-upgrade"
+  vpc_id      = local.vpc_id
+  name        = local.sg_name_emr_master
+  description = local.sg_description_emr_master
+
+  ingress_security_groups = tolist(data.aws_security_groups.emr_sg.ids)
+  ingress_port_list = [
+    [22, 22, "tcp", "SSH", var.census_private_cidr],
+    [80, 80, "tcp", "HTTP", var.census_private_cidr],
+    [443, 443, "tcp", "HTTPS", var.census_private_cidr],
+    [9870, 9870, "tcp", "HDFS Name Node", var.census_private_cidr],
+    [18080, 18080, "tcp", "Spark History Server", var.census_private_cidr],
+    [8088, 8088, "tcp", "Resource Manager", var.census_private_cidr],
+  ]
+  use_vpc_cidr = false
+  enable_self  = true
+  tags = merge(
+    local.common_tags,
+  )
+}
+
+output "sg_emr_master_id" {
+  description = "Emr Master node security group"
+  value       = module.sg_emr_master.this_security_group_id
+}