From 7596bafa9658ac03dff1944820154c3626d58285 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 13 Aug 2025 21:19:26 -0400 Subject: [PATCH] add roles policies and sg ports --- README.md | 24 +++++- aws_data.tf | 2 + main.tf | 2 +- policy.tf | 167 +++++++++++++++++++++++++++++++++++++++++ requirements.tf | 2 +- role.tf | 155 ++++++++++++++++++++++++++++++++++++++ securitygroup.ports.tf | 2 - 7 files changed, 348 insertions(+), 6 deletions(-) create mode 100644 policy.tf create mode 100644 role.tf diff --git a/README.md b/README.md index 239cd63..927b215 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ efs-csi-controller 0 5m | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13 | -| [aws](#requirement\_aws) | ~> 5.14 | +| [aws](#requirement\_aws) | ~> 5.0 | | [null](#requirement\_null) | ~> 3.2 | ## Providers @@ -113,9 +113,12 @@ efs-csi-controller 0 5m | Name | Source | Version | |------|--------|---------| | [cloudwatch\_observability\_irsa\_role](#module\_cloudwatch\_observability\_irsa\_role) | git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-iam//modules/iam-role-for-service-accounts-eks | n/a | -| [cluster](#module\_cluster) | git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/ | v20.36.0 | +| [cluster](#module\_cluster) | git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/ | v20.37.2 | | [ebs\_csi\_irsa\_role](#module\_ebs\_csi\_irsa\_role) | git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-iam//modules/iam-role-for-service-accounts-eks | n/a | | [efs\_csi\_irsa\_role](#module\_efs\_csi\_irsa\_role) | git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-iam//modules/iam-role-for-service-accounts-eks | n/a | +| [role\_cluster-admin](#module\_role\_cluster-admin) | git@github.e.it.census.gov:terraform-modules/aws-iam-role.git | tf-upgrade | +| [role\_eks-cluster](#module\_role\_eks-cluster) | git@github.e.it.census.gov:terraform-modules/aws-iam-role.git | tf-upgrade | +| [role\_eks-nodegroup](#module\_role\_eks-nodegroup) | git@github.e.it.census.gov:terraform-modules/aws-iam-role.git | tf-upgrade | | [vpc\_cni\_irsa\_role](#module\_vpc\_cni\_irsa\_role) | git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-iam//modules/iam-role-for-service-accounts-eks | n/a | ## Resources @@ -123,6 +126,10 @@ efs-csi-controller 0 5m | Name | Type | |------|------| | [aws_ec2_tag.container_subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ec2_tag) | resource | +| [aws_iam_policy.cloudwatch-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cluster-admin-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cluster-admin_assume_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.nlb-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_security_group.additional_eks_cluster_sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.all_worker_mgmt](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.extra_cluster_sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | @@ -135,11 +142,21 @@ efs-csi-controller 0 5m | [aws_arn.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ebs_default_kms_key.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ebs_default_kms_key) | data source | +| [aws_iam_policy.cluster_managed_policies](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | +| [aws_iam_policy.nodegroup_managed_policies](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | +| [aws_iam_policy_document.allow_sts](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cloudwatch-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cluster-admin-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cluster-admin_assume_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.ec2_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.eks_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.nlb-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_roles.roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_roles) | data source | | [aws_iam_roles.sso_admins](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_roles) | data source | | [aws_iam_roles.sso_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_roles) | data source | | [aws_iam_session_context.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_session_context) | data source | | [aws_kms_key.ebs_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_key) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | | [aws_subnet.subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | | [aws_subnets.subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | | [aws_vpc.eks_vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source | @@ -203,6 +220,9 @@ efs-csi-controller 0 5m | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` | +| [role\_cluster-admin-role\_arn](#output\_role\_cluster-admin-role\_arn) | Role ARN for EKS Cluster Admin Role | +| [role\_eks-cluster\_arn](#output\_role\_eks-cluster\_arn) | Role ARN for EKS Cluster Role | +| [role\_eks-nodegroup-role\_arn](#output\_role\_eks-nodegroup-role\_arn) | Role ARN for EKS Cluster Nodegroup Role | | [security\_group\_all\_worker\_mgmt\_id](#output\_security\_group\_all\_worker\_mgmt\_id) | The security group to manage all of the worker nodes. | | [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created | | [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups | diff --git a/aws_data.tf b/aws_data.tf index 1402bc0..4c35bb2 100644 --- a/aws_data.tf +++ b/aws_data.tf @@ -1,5 +1,7 @@ data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + data "aws_arn" "current" { arn = data.aws_caller_identity.current.arn } diff --git a/main.tf b/main.tf index 0cd49a8..a3b1324 100644 --- a/main.tf +++ b/main.tf @@ -26,7 +26,7 @@ resource "terraform_data" "subnet_validation" { } module "cluster" { - source = "git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/?ref=v20.36.0" + source = "git::https://github.e.it.census.gov/SCT-Engineering/terraform-aws-eks/?ref=v20.37.2" access_entries = local.access_entries cloudwatch_log_group_retention_in_days = var.cloudwatch_retention_days diff --git a/policy.tf b/policy.tf new file mode 100644 index 0000000..bac89b8 --- /dev/null +++ b/policy.tf @@ -0,0 +1,167 @@ +resource "aws_iam_policy" "nlb-policy" { + name = format("%v%v-nlb", local.prefixes["eks-policy"], var.cluster_name) + path = "/" + description = "Allow configuration of the ELB" + policy = data.aws_iam_policy_document.nlb-policy.json + +} + +# Q: why CreateSecurityGroup +# TBD: refine resources to limit only to eks configurations +data "aws_iam_policy_document" "nlb-policy" { + statement { + sid = "EKSNLBConfiguration" + effect = "Allow" + actions = [ + "elasticloadbalancing:*", + "ec2:CreateSecurityGroup", + "ec2:Describe*", + ] + resources = ["*"] + } +} + +resource "aws_iam_policy" "cloudwatch-policy" { + name = format("%v%v-cloudwatch", local.prefixes["eks-policy"], var.cluster_name) + path = "/" + description = "Allow sending metric data to cloudwatch" + policy = data.aws_iam_policy_document.cloudwatch-policy.json + +} + +# TBD: refine resources to limit only to eks configurations +data "aws_iam_policy_document" "cloudwatch-policy" { + statement { + sid = "EKSCloudwatchMetrics" + effect = "Allow" + actions = [ + "cloudwatch:PutMetricData", + ] + resources = ["*"] + } +} + +#--- +# cluster admin policy +#--- +resource "aws_iam_policy" "cluster-admin-policy" { + name = format("%v%v-cluster-admin", local.prefixes["eks-policy"], var.cluster_name) + path = "/" + description = "Allow for administration of the cluster ${var.cluster_name} using AWS resources" + policy = data.aws_iam_policy_document.cluster-admin-policy.json + +} + +data "aws_iam_policy_document" "cluster-admin-policy" { + dynamic "statement" { + for_each = local.admin_policy_statements + iterator = s + content { + sid = format("%v%vAccess", lookup(s.value, "effect", "Allow"), s.key) + effect = lookup(s.value, "effect", "Allow") + actions = lookup(s.value, "actions", []) + resources = lookup(s.value, "resources", []) + } + } +} + +locals { + iam_arn = format("arn:%v:iam::%v:%%v", data.aws_arn.current.partition, data.aws_caller_identity.current.account_id) + common_arn = format("arn:%v:%%v:%v:%v:%%v", data.aws_arn.current.partition, data.aws_region.current.name, data.aws_caller_identity.current.account_id) + eks_resources = ["cluster", "addon", "nodegroup", "identityproviderconfig"] + + admin_policy_statements = { + ECRRead = { + actions = [ + "ecr:Describe*", + "ecr:Get*", + "ecr:ListImages", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + ] + resources = ["*"] + } + ECRWrite = { + actions = [ + "ecr:BatchDeleteImage", + "ecr:CompleteLayerUpload", + "ecr:CreateRepository", + "ecr:DeleteRepository", + "ecr:InitiateLayerUpload", + "ecr:PutImage", + "ecr:UploadLayerPart" + ] + resources = [format(local.common_arn, "ecr", format("repository/eks/%v/*", var.cluster_name))] + } + EKSRead = { + actions = [ + "eks:ListClusters", + "eks:ListAddons", + "eks:ListNodegroups", + "eks:DescribeCluster", + "eks:DescribeAddon*", + "eks:DescribeNodegroup", + ] + resources = [ + format(local.common_arn, "eks", "cluster/*"), + format(local.common_arn, "eks", "addon/*"), + format(local.common_arn, "eks", "addons/*"), + format(local.common_arn, "eks", "/addons/*"), + format(local.common_arn, "eks", "nodegroup/*"), + ] + } + IAMRead = { + actions = [ + "iam:ListRoles", + ] + resources = ["*"] + } + SSMGet = { + actions = [ + "ssm:GetParameter", + ] + resources = [ + format("arn:%v:%v:%v:%v:%v", data.aws_arn.current.partition, "ssm", data.aws_region.current.name, "", "parameter/aws/service/eks/*") + ] + } + EKSReadMyClusters = { + actions = [ + "eks:List*", + "eks:Read*", + "eks:Describe*", + "eks:AccessKubernetesApi", + ] + resources = flatten(concat( + [format(local.common_arn, "eks", format("/clusters/%v/addons", var.cluster_name))], + [for r in local.eks_resources : [format(local.common_arn, "eks", format("%v/%v", r, var.cluster_name)), + format(local.common_arn, "eks", format("%v/%v/*", r, var.cluster_name))]] + )) + } + } +} + + +#--- +# cluster admin assume policy +#--- +resource "aws_iam_policy" "cluster-admin_assume_policy" { + name = format("%v%v-cluster-admin-assume", local.prefixes["eks-policy"], var.cluster_name) + path = "/" + description = "Allow for assume role to the cluster-admin role for ${var.cluster_name}" + policy = data.aws_iam_policy_document.cluster-admin_assume_policy.json + + tags = merge( + local.base_tags, + tomap({ "Name" = format("%v%v-cluster-admin-assume", local.prefixes["eks-policy"], var.cluster_name) }), + ) +} + +data "aws_iam_policy_document" "cluster-admin_assume_policy" { + statement { + sid = "AllowSTSAssumeClusterAdminRole" + effect = "Allow" + actions = ["sts:AssumeRole"] + resources = [module.role_cluster-admin.role_arn] + } +} diff --git a/requirements.tf b/requirements.tf index ea1dad6..970fa07 100644 --- a/requirements.tf +++ b/requirements.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 5.14" + version = "~> 5.0" } null = { source = "hashicorp/null" diff --git a/role.tf b/role.tf new file mode 100644 index 0000000..060a45a --- /dev/null +++ b/role.tf @@ -0,0 +1,155 @@ +#--- +# cluster +#--- +locals { + cluster_managed_policy_list = [ + "AmazonEKSClusterPolicy", + "AmazonEC2FullAccess", + "CloudWatchLogsFullAccess", + ] + cluster_managed_policies = [for p in data.aws_iam_policy.cluster_managed_policies : p.arn] +} + +data "aws_iam_policy" "cluster_managed_policies" { + for_each = toset(local.cluster_managed_policy_list) + name = each.key +} + +# this needs the two policies nlb-policy and cloudwatch-policy, created first + +module "role_eks-cluster" { + source = "git@github.e.it.census.gov:terraform-modules/aws-iam-role.git?ref=tf-upgrade" + # providers = { + # ldap = ldap.bocas + # } + + role_name = format("%v%v-cluster", local.prefixes["eks"], var.cluster_name) + role_description = "EKS Cluster Role for ${var.cluster_name}" + enable_ldap_creation = false + assume_policy_document = data.aws_iam_policy_document.eks_assume.json + attached_policies = concat([aws_iam_policy.nlb-policy.arn, aws_iam_policy.cloudwatch-policy.arn], local.cluster_managed_policies) + + +} + +data "aws_iam_policy_document" "eks_assume" { + statement { + sid = "EKSAssumeRole" + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["eks.amazonaws.com"] + } + } +} + +output "role_eks-cluster_arn" { + description = "Role ARN for EKS Cluster Role" + value = module.role_eks-cluster.role_arn +} + +#--- +# nodegroup +#--- +locals { + nodegroup_managed_policy_list = [ + "AmazonEKSWorkerNodePolicy", + "AmazonEKS_CNI_Policy", + "AmazonEC2ContainerRegistryPowerUser", + "AmazonEC2ContainerRegistryReadOnly", + "CloudWatchLogsFullAccess", + "AmazonS3FullAccess", + "AmazonSSMManagedInstanceCore", + "AmazonEC2RoleforSSM", + ] + nodegroup_managed_policies = [for p in data.aws_iam_policy.nodegroup_managed_policies : p.arn] +} + +data "aws_iam_policy" "nodegroup_managed_policies" { + for_each = toset(local.nodegroup_managed_policy_list) + name = each.key +} + +module "role_eks-nodegroup" { + source = "git@github.e.it.census.gov:terraform-modules/aws-iam-role.git?ref=tf-upgrade" + # providers = { + # ldap = ldap.bocas + # } + + role_name = format("%v%v-nodegroup", local.prefixes["eks"], var.cluster_name) + role_description = "EKS Nodegroup Role for ${var.cluster_name}" + enable_ldap_creation = false + assume_policy_document = data.aws_iam_policy_document.ec2_assume.json + attached_policies = concat(local.nodegroup_managed_policies) + + +} + +#---- +# STS: ec2 assume +#--- +data "aws_iam_policy_document" "ec2_assume" { + statement { + sid = "EKSAssumeRole" + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +output "role_eks-nodegroup-role_arn" { + description = "Role ARN for EKS Cluster Nodegroup Role" + value = module.role_eks-nodegroup.role_arn +} + +#--- +# cluster-admin +#--- +module "role_cluster-admin" { + source = "git@github.e.it.census.gov:terraform-modules/aws-iam-role.git?ref=tf-upgrade" + # providers = { + # ldap = ldap.bocas + # } + + role_name = format("%v%v-cluster-admin", local.prefixes["eks"], var.cluster_name) + role_description = "SAML EKS cluster admin Role for ${var.cluster_name}" + enable_ldap_creation = false + assume_policy_document = data.aws_iam_policy_document.allow_sts.json + # assume_policy_document = data.aws_iam_policy_document.cluster-admin_combined.json + attached_policies = [aws_iam_policy.cluster-admin-policy.arn] + + +} + +output "role_cluster-admin-role_arn" { + description = "Role ARN for EKS Cluster Admin Role" + value = module.role_cluster-admin.role_arn +} + +data "aws_iam_policy_document" "allow_sts" { + statement { + sid = "AllowSTSAssume" + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + type = "AWS" + identifiers = [ + format(local.iam_arn, "root"), + ] + } + } +} + +# data "aws_iam_policy_document" "cluster-admin_combined" +# source_policy_documents = [ +# data.aws_iam_policy_document.allow_sts.json +# data.aws_iam_policy_document.saml_assume.json, +# ] +# } +# diff --git a/securitygroup.ports.tf b/securitygroup.ports.tf index 4c58b3d..0535c58 100644 --- a/securitygroup.ports.tf +++ b/securitygroup.ports.tf @@ -139,7 +139,6 @@ resource "aws_vpc_security_group_ingress_rule" "additional" { to_port = each.value.to_port ip_protocol = each.value.protocol referenced_security_group_id = each.value.self ? aws_security_group.additional_eks_cluster_sg.id : null - # referenced_security_group_id = aws_security_group.all_worker_mgmt.id } resource "aws_vpc_security_group_egress_rule" "additional" { @@ -151,7 +150,6 @@ resource "aws_vpc_security_group_egress_rule" "additional" { to_port = each.value.to_port ip_protocol = each.value.protocol referenced_security_group_id = each.value.self ? aws_security_group.additional_eks_cluster_sg.id : null - # referenced_security_group_id = aws_security_group.all_worker_mgmt.id } resource "aws_vpc_security_group_ingress_rule" "additional_ingress_rules_2" {