diff --git a/cluster_admin_group.tf.disabled b/cluster_admin_group.tf.disable similarity index 100% rename from cluster_admin_group.tf.disabled rename to cluster_admin_group.tf.disable diff --git a/cluster_admin_policies.tf.disabled b/cluster_admin_policies.tf.disable similarity index 100% rename from cluster_admin_policies.tf.disabled rename to cluster_admin_policies.tf.disable diff --git a/cluster_admin_roles.tf.disabled b/cluster_admin_roles.tf.disable similarity index 100% rename from cluster_admin_roles.tf.disabled rename to cluster_admin_roles.tf.disable diff --git a/examples/simple/eks.tf b/examples/simple/eks.tf index 3dab16c..5a851d1 100644 --- a/examples/simple/eks.tf +++ b/examples/simple/eks.tf @@ -1,5 +1,5 @@ module "eks" { - source = "git@github.it.census.gov:SOA/tfmod-eks.git//" + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-eks.git//" #source = "git@github.it.census.gov:SOA/tfmod-eks.git//?ref=v1.0.0" vpc_name = var.vpc_name diff --git a/examples/testing/eks.tf b/examples/testing/eks.tf new file mode 100644 index 0000000..5c958ce --- /dev/null +++ b/examples/testing/eks.tf @@ -0,0 +1,18 @@ +module "eks" { + # source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-eks.git//" + source = "../.." + + vpc_name = var.vpc_name + cluster_name = var.cluster_name + cluster_version = var.cluster_version + domain = var.domain + eks_instance_disk_size = var.eks_instance_disk_size + eks_instance_types = var.eks_instance_types + eks_ng_desired_size = var.eks_ng_desired_size + eks_ng_max_size = var.eks_ng_max_size + eks_ng_min_size = var.eks_ng_min_size + cluster_endpoint_public_access = var.cluster_endpoint_public_access + access_entries = var.access_entries + enable_cluster_creator_admin_permissions = var.enable_cluster_creator_admin_permissions + tags = var.tags +} diff --git a/examples/testing/providers.tf b/examples/testing/providers.tf new file mode 100644 index 0000000..ce03e7a --- /dev/null +++ b/examples/testing/providers.tf @@ -0,0 +1,8 @@ +terraform { + required_version = ">= 1.5.0" +} + +provider "aws" { + profile = var.profile + region = var.region +} diff --git a/examples/testing/variables.tf b/examples/testing/variables.tf new file mode 100644 index 0000000..47630bb --- /dev/null +++ b/examples/testing/variables.tf @@ -0,0 +1,81 @@ +variable "region" { + description = "AWS region" + type = string +} + +variable "profile" { + description = "AWS config profile" + type = string +} + +variable "cluster_name" { + description = "EKS cluster name name component used through out the EKS cluster describing its purpose (ex: dice-dev)" + type = string +} + +variable "cluster_version" { + description = "The Kubernetes version number to use for this EKS cluster. See https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html" + type = string + default = "1.27" +} + +variable "vpc_name" { + description = "AWS vpc in which the cluster will reside" + type = string +} + +variable "cluster_endpoint_public_access" { + type = bool +} + +variable "enable_cluster_creator_admin_permissions" { + description = "Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry" + type = bool + default = true +} + +variable "domain" { + description = "The DNS domain name of the cluster." + type = string +} + +variable "eks_instance_disk_size" { + description = "The size of the disk of the worker nodes in gigabytes. 40 is the approximate minimum. Needs to hold the all of the normal operating system files plus every image that will be used in the cluster." + type = number + default = 40 +} + +variable "eks_instance_types" { + description = "EKS worker node instance types" + type = list(string) + default = [ + "t3.xlarge" + ] +} + +variable "access_entries" { + description = "Map of access entries to add to the cluster" + type = any + default = {} +} + +variable "eks_ng_min_size" { + description = "Node Group minimum size" + type = number + default = 4 +} +variable "eks_ng_desired_size" { + description = "Node Group desired size" + type = number + default = 4 +} +variable "eks_ng_max_size" { + description = "Node Group maximum size" + type = number + default = 15 +} +variable "tags" { + description = "AWS Tags to apply to appropriate resources" + type = map(string) + default = {} +} \ No newline at end of file diff --git a/istio_ports.tf b/istio_ports.tf.disable similarity index 100% rename from istio_ports.tf rename to istio_ports.tf.disable diff --git a/main.tf b/main.tf index 437b6dc..f41967a 100644 --- a/main.tf +++ b/main.tf @@ -47,16 +47,39 @@ locals { ng_name = format("%v%v-nodegroup", local._prefixes["eks"], var.cluster_name) tags = merge(local.base_tags, var.tags) + + # Access entries are the latest AWS model for managing cluster access: https://docs.aws.amazon.com/eks/latest/userguide/access-entries.html + # They make reference to depricating the aws-auth ConfigMap, but this baseline enables access management with both EKS API and ConfigMap + # This is done especially since access entries are fairly course grained, especially given the granularity we can achieve via EKS native + # RBAC constructs in Roles and ClusterRoles and bindings. + # This below is just an example, in practice we'd notionally be creating a role (or multiple) specific to the cluster and setting policy + # to allow the cluster users to assume said role; but we need to spend some time parsing what exactly are the permissions we plan to hand + # out to these clusters. + # access_entries = { + # inf-admin-t2 = { + # principal_arn = "arn:aws-us-gov:iam::224384469011:role/aws-reserved/sso.amazonaws.com/us-gov-east-1/AWSReservedSSO_inf-admin-t2_f3912d726991bbfa" + # kubernetes_groups = [] + # policy_associations = { + # admin = { + # policy_arn = "arn:aws-us-gov:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" + # access_scope = { + # type = "cluster" + # } + # } + # } + # } + # } } module "cluster" { - source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git?ref=v20.8.5" + source = "git@github.e.it.census.gov:SCT-Engineering/terraform-aws-eks.git?ref=v20.8.5" #version = "19.16.0" - cluster_name = var.cluster_name - cluster_version = var.cluster_version - cluster_endpoint_public_access = var.cluster_endpoint_public_access + cluster_name = var.cluster_name + cluster_version = var.cluster_version + cluster_endpoint_public_access = var.cluster_endpoint_public_access enable_cluster_creator_admin_permissions = var.enable_cluster_creator_admin_permissions + # access_entries = local.access_entries cluster_enabled_log_types = [ "audit", @@ -71,10 +94,10 @@ module "cluster" { cluster_addons = { coredns = { - most_recent = false + most_recent = true } kube-proxy = { - most_recent = false + most_recent = true } vpc-cni = { most_recent = false @@ -94,10 +117,14 @@ module "cluster" { ami_type = "AL2_x86_64" } - node_security_group_additional_rules = merge( - local.ingress_rules, - local.egress_rules, - ) + node_security_group_enable_recommended_rules = false + + # node_security_group_additional_rules = merge( + # local.ingress_rules, + # local.egress_rules, + # ) + + node_security_group_additional_rules = local.node_security_group_additional_rules eks_managed_node_groups = { node_group = { diff --git a/node_ports.tf.disable b/node_ports.tf.disable new file mode 100644 index 0000000..403a3d6 --- /dev/null +++ b/node_ports.tf.disable @@ -0,0 +1,45 @@ +locals { + ingress_rules { + ingress_nodes_ephemeral = { + "description" = "Node to node ingress on ephemeral ports" + "protocol" = -1 + "from_port" = 0 + "to_port" = 0 + "type" = "ingress" + "self" = true + } + } +# ingress_nodes_ephemeral = { +# "description": "Node to node ingress on ephemeral ports custom", +# "protocol": -1, +# "from_port": 0, +# "to_port": 0, +# "type": "ingress", +# "self": true +# } +} + + + # { + # "cidr_blocks": [], + # "description": "Envoy inbound", + # "from_port": 15006, + # "ipv6_cidr_blocks": [], + # "prefix_list_ids": [], + # "protocol": "tcp", + # "security_groups": [], + # "self": true, + # "to_port": 15006 + # }, + +# ingress_rules = { +# for ikey, ivalue in local.istio_ports : +# "${ikey}_ingress" => { +# description = ivalue.description +# protocol = "tcp" +# from_port = ivalue.from_port +# to_port = ivalue.to_port +# type = "ingress" +# self = true +# } +# } \ No newline at end of file diff --git a/sg_ports.tf b/sg_ports.tf new file mode 100644 index 0000000..567a8db --- /dev/null +++ b/sg_ports.tf @@ -0,0 +1,182 @@ +# locals { +# istio_ports = [ +# { +# description = "Envoy admin port / outbound" +# from_port = 15000 +# to_port = 15001 +# protocol = "tcp" +# }, +# { +# description = "Debug port" +# from_port = 15004 +# to_port = 15004 +# protocol = "tcp" +# }, +# { +# description = "Envoy inbound" +# from_port = 15006 +# to_port = 15006 +# protocol = "tcp" +# }, +# { +# description = "HBONE mTLS tunnel port / secure networks XDS and CA services (Plaintext)" +# from_port = 15008 +# to_port = 15010 +# protocol = "tcp" +# }, +# { +# description = "XDS and CA services (TLS and mTLS)" +# from_port = 15012 +# to_port = 15012 +# protocol = "tcp" +# }, +# { +# description = "Control plane monitoring" +# from_port = 15014 +# to_port = 15014 +# protocol = "tcp" +# }, +# { +# description = "Control plane monitoring" +# from_port = 15017 +# to_port = 15017 +# protocol = "tcp" +# }, +# { +# description = "Merged Prometheus telemetry data from Istio agent, Envoy, and application, Health checks" +# from_port = 15020 +# to_port = 15021 +# protocol = "tcp" +# }, +# { +# description = "DNS port" +# from_port = 15053 +# to_port = 15053 +# protocol = "tcp" +# }, +# { +# description = "Envoy Prometheus telemetry" +# from_port = 15090 +# to_port = 15090 +# protocol = "tcp" +# }, +# { +# description = "aws-load-balancer-controller" +# from_port = 9443 +# to_port = 9443 +# protocol = "tcp" +# }, +# { +# description = "Node to node ingress on ephemeral ports" +# from_port = 80 +# to_port = 65535 +# protocol = "tcp" +# }, +# { +# description = "Cluster API to node 4443/tcp webhook" +# from_port = 4443 +# to_port = 4443 +# protocol = "tcp" +# # source_cluster_security_group = true +# }, +# # prometheus-adapter +# { +# description = "Cluster API to node 6443/tcp webhook" +# from_port = 6443 +# to_port = 6443 +# protocol = "tcp" +# # source_cluster_security_group = true +# }, +# # Karpenter +# { +# description = "Cluster API to node 8443/tcp webhook" +# from_port = 8443 +# to_port = 8443 +# protocol = "tcp" +# # source_cluster_security_group = true +# }, +# ] + +# ingress_rules = { +# for ikey, ivalue in local.istio_ports : +# "${ikey}_ingress" => { +# description = ivalue.description +# protocol = ivalue.protocol +# from_port = ivalue.from_port +# to_port = ivalue.to_port +# type = "ingress" +# self = true +# } +# } + +# egress_rules = { +# for ekey, evalue in local.istio_ports : +# "${ekey}_egress" => { +# description = evalue.description +# protocol = "tcp" +# from_port = evalue.from_port +# to_port = evalue.to_port +# type = "egress" +# self = true +# } +# } + +# } + +locals { +node_security_group_additional_rules = { + "ingress_nodes_ephemeral" = { + "description" = "Node to node ingress on ephemeral ports" + "from_port" = 80 + "protocol" = "tcp" + "self" = true + "to_port" = 65535 + "type" = "ingress" + } + # metrics-server + ingress_cluster_4443_webhook = { + description = "Cluster API to node 4443/tcp webhook" + protocol = "tcp" + from_port = 4443 + to_port = 4443 + type = "ingress" + source_cluster_security_group = true + } + # prometheus-adapter + ingress_cluster_6443_webhook = { + description = "Cluster API to node 6443/tcp webhook" + protocol = "tcp" + from_port = 6443 + to_port = 6443 + type = "ingress" + source_cluster_security_group = true + } + # Karpenter + ingress_cluster_8443_webhook = { + description = "Cluster API to node 8443/tcp webhook" + protocol = "tcp" + from_port = 8443 + to_port = 8443 + type = "ingress" + source_cluster_security_group = true + } + # ALB controller, NGINX + ingress_cluster_9443_webhook = { + description = "Cluster API to node 9443/tcp webhook" + protocol = "tcp" + from_port = 9443 + to_port = 9443 + type = "ingress" + source_cluster_security_group = true + } + egress_all = { + description = "Allow all egress" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + # ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null + } +} +} \ No newline at end of file diff --git a/variables.tf b/variables.tf index aabef96..3310d87 100644 --- a/variables.tf +++ b/variables.tf @@ -73,6 +73,12 @@ variable "operators_ns" { default = "operators" } +variable "access_entries" { + description = "Map of access entries to add to the cluster" + type = any + default = {} +} + ################################################################### # Common variables ###################################################################