add kafka

kafka/CHANGELOG.md

@@ -0,0 +1,2 @@
+# v1.0.0 -- 20210609
+  - create module

kafka/README.md

@@ -0,0 +1,75 @@
+# About kafka
+
+This describes how to use the aws-common-security-groups submodule for kafka.
+
+Default and auxilliary ports are included in this. This includes TCP only for the following ports:
+
+* 22 ssh
+* 443 https
+* 9000 http
+* 9001 http
+* 9092, 8083, 2181, 2188, 2189, 9999
+
+## Usage
+
+```hcl
+module "kafka" {
+  source = "git@github.e.it.census.gov:terraform-modules/aws-common-security-groups.git//kafka"
+
+  vpc_id = var.vpc_id
+  ## optional
+  # name = "m-kafka"
+
+  ## tags for Name, CostAllocation, and Environment are pre-set, but they can be overriden
+  # tags = { }
+}
+```
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.12 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_security_group.this_security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
+| [aws_security_group.egress_security_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/security_group) | data source |
+| [aws_security_group.ingress_security_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/security_group) | data source |
+| [aws_vpc.this_vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_description"></a> [description](#input\_description) | Security Group Description | `string` | `"Web Security Group"` | no |
+| <a name="input_egress_networks"></a> [egress\_networks](#input\_egress\_networks) | List of egress networks (all ports) | `list(string)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |
+| <a name="input_egress_security_groups"></a> [egress\_security\_groups](#input\_egress\_security\_groups) | List of egress security groups (all ports) | `list(string)` | `[]` | no |
+| <a name="input_enable_self"></a> [enable\_self](#input\_enable\_self) | Enable\|Disable self full access | `bool` | `false` | no |
+| <a name="input_ingress_networks"></a> [ingress\_networks](#input\_ingress\_networks) | List of ingress networks for external access (not all ports) | `list(string)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |
+| <a name="input_ingress_security_groups"></a> [ingress\_security\_groups](#input\_ingress\_security\_groups) | List of ingress security groups for all ports | `list(string)` | `[]` | no |
+| <a name="input_name"></a> [name](#input\_name) | Security Group Name | `string` | `"m-web"` | no |
+| <a name="input_short_description"></a> [short\_description](#input\_short\_description) | Security Group Short Description | `string` | `"Web"` | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | Extra security group tags | `map` | <pre>{<br>  "CostAllocation": "csvd:infrastructure",<br>  "Environment": "csvd-infrastructure"<br>}</pre> | no |
+| <a name="input_use_vpc_cidr"></a> [use\_vpc\_cidr](#input\_use\_vpc\_cidr) | Enable\|Disable use of VPC CIDR block in the ingress\_networks | `bool` | `false` | no |
+| <a name="input_vpc_full_name"></a> [vpc\_full\_name](#input\_vpc\_full\_name) | VPC Name | `string` | `""` | no |
+| <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | VPC ID Number | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_this_security_group_arn"></a> [this\_security\_group\_arn](#output\_this\_security\_group\_arn) | Created security group ARN |
+| <a name="output_this_security_group_id"></a> [this\_security\_group\_id](#output\_this\_security\_group\_id) | Created security group ID |

kafka/main.tf

@@ -0,0 +1,140 @@
+/*
+* # About kafka
+*
+* This describes how to use the aws-common-security-groups submodule for kafka.
+*
+* Default and auxilliary ports are included in this. This includes TCP only for the following ports:
+*
+* * 22 ssh
+* * 443 https
+* * 9000 http
+* * 9001 http
+* * 9092, 8083, 2181, 2188, 2189, 9999
+*
+* ## Usage
+*
+* ```hcl
+* module "kafka" {
+*   source = "git@github.e.it.census.gov:terraform-modules/aws-common-security-groups.git//kafka"
+*
+*   vpc_id = var.vpc_id
+*   ## optional
+*   # name = "m-kafka"
+*
+*   ## tags for Name, CostAllocation, and Environment are pre-set, but they can be overriden
+*   # tags = { }
+* }
+* ```
+*/
+
+data "aws_vpc" "this_vpc" {
+  count = var.use_vpc_cidr ? 1 : 0
+  id    = var.vpc_id
+}
+
+data "aws_security_group" "ingress_security_groups" {
+  count = length(var.ingress_security_groups)
+  id    = element(var.ingress_security_groups, count.index)
+}
+
+data "aws_security_group" "egress_security_groups" {
+  count = length(var.egress_security_groups)
+  id    = element(var.egress_security_groups, count.index)
+}
+
+locals {
+  vpc_networks              = var.use_vpc_cidr ? [data.aws_vpc.this_vpc[0].cidr_block] : []
+  external_ingress_networks = compact(concat(local.vpc_networks, local.ingress_networks))
+  ingress_sg_names          = zipmap(var.ingress_security_groups, data.aws_security_group.ingress_security_groups[*].name)
+  egress_sg_names           = zipmap(var.egress_security_groups, data.aws_security_group.egress_security_groups[*].name)
+  self                      = var.enable_self ? [1] : []
+  short_description         = var.short_description == "" ? var.description : var.short_description
+}
+
+resource "aws_security_group" "this_security_group" {
+  name        = local.name
+  description = var.description
+  vpc_id      = var.vpc_id
+
+  # ingresss external port list (list + vpc if enabaled)
+  dynamic "ingress" {
+    for_each = local.port_map["external"]
+    iterator = p
+    content {
+      description = "${local.short_description}: ${p.value["description"]}"
+      from_port   = p.value["from"]
+      to_port     = p.value["to"]
+      protocol    = p.value["proto"]
+      cidr_blocks = length(p.value["cidr"]) == 0 ? local.external_ingress_networks : p.value["cidr"]
+    }
+  }
+
+  # ingresss vpc port list  (ignores port{} network block)
+  dynamic "ingress" {
+    for_each = local.port_map["vpc"]
+    iterator = p
+    content {
+      description = "${local.short_description}: ${p.value["description"]}"
+      from_port   = p.value["from"]
+      to_port     = p.value["to"]
+      protocol    = p.value["proto"]
+      cidr_blocks = [data.aws_vpc.this_vpc[0].cidr_block]
+    }
+  }
+
+  # ingress security group ids (all)
+  dynamic "ingress" {
+    for_each = local.ingress_sg
+    iterator = sg
+    content {
+      description     = "${local.short_description}: ${local.ingress_sg_names[sg.value]}"
+      from_port       = 0
+      to_port         = 0
+      protocol        = -1
+      security_groups = [sg.value]
+    }
+  }
+
+  # ingress self (list with one or zero items)
+  dynamic "ingress" {
+    for_each = local.self
+    iterator = sg
+    content {
+      description = "${local.short_description}: from self"
+      from_port   = 0
+      to_port     = 0
+      protocol    = -1
+      self        = true
+    }
+  }
+
+  # egress all
+  egress {
+    description = "${local.short_description}: All"
+    from_port   = 0
+    to_port     = 0
+    protocol    = -1
+    cidr_blocks = local.egress_networks
+  }
+
+  # egress security group ids (all)
+  dynamic "egress" {
+    for_each = local.egress_sg
+    iterator = sg
+    content {
+      description     = "${local.short_description}: ${local.egress_sg_names[sg]}"
+      from_port       = 0
+      to_port         = 0
+      protocol        = -1
+      security_groups = [sg]
+    }
+  }
+
+  tags = merge(
+    map("Name", "sg-${local.name}"),
+    var.tags,
+    map("boc:created_by", "terraform"),
+    map("boc:tf_module_version", local._module_version),
+    map("boc:vpc:info", join(" ", compact(list(var.vpc_id, var.vpc_full_name)))),
+  )
+}

kafka/output.tf

@@ -0,0 +1,9 @@
+output "this_security_group_id" {
+  description = "Created security group ID"
+  value       = aws_security_group.this_security_group.id
+}
+
+output "this_security_group_arn" {
+  description = "Created security group ARN"
+  value       = aws_security_group.this_security_group.arn
+}

kafka/ports.tf

@@ -0,0 +1,41 @@
+# ports = list of list of
+#   from_port
+#   to_port
+#   proto
+#   description
+#   cidr_block
+#   list of: all, external (more added as needed)
+
+locals {
+  description   = "module: Kafka common ports"
+  n_all         = ["0.0.0.0/0"]
+  n_census      = ["148.129.0.0/16", "192.168.0.0/16", "172.16.0.0/12", "10.0.0.0/8"]
+  source_groups = ["all", "external", "vpc"]
+
+  name = var.name
+  ports = [
+    [22, 22, "tcp", "ssh", local.n_all, ["external"]],
+    [443, 443, "tcp", "https", local.n_all, ["external"]],
+    [2181, 2181, "tcp", "Zookeeper-client", local.n_all, ["vpc"]],
+    [2188, 2189, "tcp", "Zookeeper-follower", local.n_all, ["vpc"]],
+    [3188, 3188, "tcp", "Zookeeper-internode", local.n_all, ["vpc"]],
+    [8083, 8083, "tcp", "Kafka", local.n_all, ["vpc"]],
+    [9000, 9001, "tcp", "Kafka-UI", local.n_all, ["external"]],
+    [9092, 9093, "tcp", "Kafka-listener", local.n_all, ["external"]],
+    [9999, 9999, "tcp", "Kafka", local.n_all, ["vpc"]],
+  ]
+
+  # these are ignored
+  ingress_networks = var.ingress_networks
+  egress_networks  = var.egress_networks
+
+  # these are ignored
+  ingress_sg = var.ingress_security_groups
+  egress_sg  = var.egress_security_groups
+
+  p_fields = ["from", "to", "proto", "description", "cidr", "source_group"]
+  p_map    = [for p in local.ports : zipmap(local.p_fields, p)]
+  port_map = { for s in local.source_groups :
+    s => [for p in local.p_map : p if contains(p["source_group"], s)]
+  }
+}

kafka/variables.tf

@@ -0,0 +1,79 @@
+#---
+# change between different modules as needed
+#---
+variable "name" {
+  description = "Security Group Name"
+  type        = string
+  default     = "m-web"
+}
+
+variable "description" {
+  description = "Security Group Description"
+  type        = string
+  default     = "Web Security Group"
+}
+
+variable "short_description" {
+  description = "Security Group Short Description"
+  type        = string
+  default     = "Web"
+}
+
+variable "enable_self" {
+  description = "Enable|Disable self full access"
+  type        = bool
+  default     = false
+}
+
+variable "use_vpc_cidr" {
+  description = "Enable|Disable use of VPC CIDR block in the ingress_networks"
+  type        = bool
+  default     = false
+}
+
+#---
+# others with defaults
+#---
+variable "vpc_id" {
+  description = "VPC ID Number"
+  type        = string
+}
+
+variable "vpc_full_name" {
+  description = "VPC Name"
+  type        = string
+  default     = ""
+}
+
+variable "ingress_networks" {
+  description = "List of ingress networks for external access (not all ports)"
+  type        = list(string)
+  default     = ["0.0.0.0/0"]
+}
+
+variable "egress_networks" {
+  description = "List of egress networks (all ports)"
+  type        = list(string)
+  default     = ["0.0.0.0/0"]
+}
+
+variable "ingress_security_groups" {
+  description = "List of ingress security groups for all ports"
+  type        = list(string)
+  default     = []
+}
+
+variable "egress_security_groups" {
+  description = "List of egress security groups (all ports)"
+  type        = list(string)
+  default     = []
+}
+
+variable "tags" {
+  description = "Extra security group tags"
+  type        = map
+  default = {
+    "CostAllocation" = "csvd:infrastructure"
+    "Environment"    = "csvd-infrastructure"
+  }
+}

kafka/version.tf

@@ -0,0 +1,3 @@
+locals {
+  _module_version = "1.0.0"
+}

kafka/versions.tf

@@ -0,0 +1,4 @@
+
+terraform {
+  required_version = ">= 0.12"
+}