diff --git a/examples/full-cluster-tf-upgrade/1.24/dns-zone.tf b/examples/full-cluster-tf-upgrade/1.24/dns-zone.tf index a7afc52..89ab48f 100644 --- a/examples/full-cluster-tf-upgrade/1.24/dns-zone.tf +++ b/examples/full-cluster-tf-upgrade/1.24/dns-zone.tf @@ -31,7 +31,7 @@ provider "aws" { # dummy vpc, so we can associate the zone to this account #--- data "aws_vpc" "dummy_vpc" { - count = var.shared_vpc_label != null ? 1 : 0 + count = ! (var.shared_vpc_label == null || var.shared_vpc_label == "") ? 1 : 0 filter { name = "tag:Name" values = ["vpc0-dummy"] @@ -44,12 +44,16 @@ resource "aws_route53_zone" "cluster_domain" { force_destroy = false vpc { - vpc_id = var.shared_vpc_label != null ? try(data.aws_vpc.dummy_vpc[0].id, null) : data.aws_vpc.eks_vpc.id + vpc_id = ! (var.shared_vpc_label == null || var.shared_vpc_label == "") ? try(data.aws_vpc.dummy_vpc[0].id, null) : data.aws_vpc.eks_vpc.id vpc_region = local.region } lifecycle { ignore_changes = [vpc] + precondition { + condition = (var.shared_vpc_label == null || var.shared_vpc_label == "") || (! (var.shared_vpc_label == null || var.shared_vpc_label == "") && ! (var.domain == null || var.domain == "")) + error_message = "var.domain must be provided when shared VPCs are in use." + } } tags = merge( @@ -65,7 +69,7 @@ resource "aws_route53_zone" "cluster_domain" { # need to also associate with network-prod account and this vpc #--- module "route53_cluster_domain_east" { - count = local.region == "us-gov-east-1" && var.shared_vpc_label != null ? 1 : 0 + count = local.region == "us-gov-east-1" && ! (var.shared_vpc_label == null || var.shared_vpc_label == "") ? 1 : 0 providers = { aws.self = aws aws.peer = aws.route53_main_east @@ -83,7 +87,7 @@ module "route53_cluster_domain_east" { } module "route53_cluster_domain_west" { - count = local.region == "us-gov-west-1" && var.shared_vpc_label != null ? 1 : 0 + count = local.region == "us-gov-west-1" && ! (var.shared_vpc_label == null || var.shared_vpc_label == "") ? 1 : 0 providers = { aws.self = aws aws.peer = aws.route53_main_west diff --git a/examples/full-cluster-tf-upgrade/1.25/addons/README.addons.md b/examples/full-cluster-tf-upgrade/1.25/addons/README.addons.md new file mode 100644 index 0000000..165248e --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.25/addons/README.addons.md @@ -0,0 +1,3 @@ +tf-aws eks describe-addon-versions --kubernetes-version 1.25 + + diff --git a/examples/full-cluster-tf-upgrade/1.25/addons/variables.addons.tf b/examples/full-cluster-tf-upgrade/1.25/addons/variables.addons.tf index fcbe3c2..4853495 100644 --- a/examples/full-cluster-tf-upgrade/1.25/addons/variables.addons.tf +++ b/examples/full-cluster-tf-upgrade/1.25/addons/variables.addons.tf @@ -9,12 +9,12 @@ variable "addon_versions" { "aws-ebs-csi-driver" = "v1.18.0-eksbuild.1" } "1.25" = { - "coredns" = "v1.9.3-eksbuild.2" - "kube-proxy" = "v1.25.6-eksbuild.1" - "vpc-cni" = "v1.12.2-eksbuild.1" - "aws-ebs-csi-driver" = "v1.18.0-eksbuild.1" + "coredns" = "v1.9.3-eksbuild.5" + "kube-proxy" = "v1.25.11-eksbuild.2" + "vpc-cni" = "v1.13.4-eksbuild.1" + "aws-ebs-csi-driver" = "v1.21.0-eksbuild.1" + "aws-efs-csi-driver" = "v1.5.8-eksbuild.1" + "adot" = "v0.78.0-eksbuild.1" } } } - - diff --git a/examples/full-cluster-tf-upgrade/1.25/common-services/images.tf b/examples/full-cluster-tf-upgrade/1.25/common-services/images.tf index 78668f3..925c0d5 100644 --- a/examples/full-cluster-tf-upgrade/1.25/common-services/images.tf +++ b/examples/full-cluster-tf-upgrade/1.25/common-services/images.tf @@ -1,6 +1,9 @@ locals { image_config = [for k, v in var.image_details : v if v.enabled] - image_output = { for k, v in module.images.image_info : v.name => v } + image_output = { for k, v in module.images.images : v.name => v } + + charts = { for k, v in var.chart_details : k => v if try(v.enabled, true) } + images = { for k, v in var.image_details : k => v if v.enabled } } module "images" { @@ -13,7 +16,6 @@ module "images" { tags = merge( local.base_tags, local.common_tags, - var.tags, var.application_tags, ) diff --git a/examples/full-cluster-tf-upgrade/1.25/dns-zone.tf b/examples/full-cluster-tf-upgrade/1.25/dns-zone.tf index a91b668..89ab48f 100644 --- a/examples/full-cluster-tf-upgrade/1.25/dns-zone.tf +++ b/examples/full-cluster-tf-upgrade/1.25/dns-zone.tf @@ -50,6 +50,10 @@ resource "aws_route53_zone" "cluster_domain" { lifecycle { ignore_changes = [vpc] + precondition { + condition = (var.shared_vpc_label == null || var.shared_vpc_label == "") || (! (var.shared_vpc_label == null || var.shared_vpc_label == "") && ! (var.domain == null || var.domain == "")) + error_message = "var.domain must be provided when shared VPCs are in use." + } } tags = merge( diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/addon.tf b/examples/full-cluster-tf-upgrade/1.25/efs/addon.tf new file mode 100644 index 0000000..38dc396 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.25/efs/addon.tf @@ -0,0 +1,15 @@ +# https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html + +resource "aws_eks_addon" "aws-efs-csi-driver" { + count = lookup(lookup(var.addon_versions, var.cluster_version, {}), "aws-efs-csi-driver", null) != null ? 1 : 0 + + cluster_name = var.cluster_name + addon_name = "aws-efs-csi-driver" + addon_version = lookup(lookup(var.addon_versions, var.cluster_version, {}), "aws-efs-csi-driver", null) + # resolve_conflicts = "OVERWRITE" + # note OVERWRITE resets to eks addon defaults, PRESERVE uses any values set here + resolve_conflicts_on_create = "OVERWRITE" + resolve_conflicts_on_update = "OVERWRITE" + service_account_role_arn = module.role_efs-driver.role_arn + configuration_values = null +} diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/copy_image.sh b/examples/full-cluster-tf-upgrade/1.25/efs/copy_image.sh deleted file mode 120000 index 889e269..0000000 --- a/examples/full-cluster-tf-upgrade/1.25/efs/copy_image.sh +++ /dev/null @@ -1 +0,0 @@ -../bin/copy_image.sh \ No newline at end of file diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/copy_images.tf b/examples/full-cluster-tf-upgrade/1.25/efs/copy_images.tf deleted file mode 100644 index f7e13be..0000000 --- a/examples/full-cluster-tf-upgrade/1.25/efs/copy_images.tf +++ /dev/null @@ -1,59 +0,0 @@ - -data "aws_ecr_authorization_token" "token" {} - -locals { - repo_parent_name = format("eks/%v", var.cluster_name) - images = [ - { - image = "external-provisioner" - tag = var.external_provisioner_tag - }, - { - image = "livenessprobe" - tag = var.livenessprobe_tag - }, - { - image = "node-driver-registrar" - tag = var.node_driver_registrar_tag - }, - ] -} - -resource "aws_ecr_repository" "repository" { - for_each = { for image in local.images : image.image => image } - - name = format("%v/%v", local.repo_parent_name, each.value.image) - image_tag_mutability = "IMMUTABLE" - - image_scanning_configuration { - scan_on_push = true - } - - encryption_configuration { - encryption_type = "KMS" - } - - tags = merge( - #local.common_tags, - #local.base_tags, - #var.application_tags, - tomap({ "Name" = format("ecr-eks-%v-%v", var.cluster_name, each.value.image) }), - ) -} - -resource "null_resource" "copy_images" { - for_each = { for image in local.images : image.image => image } - - provisioner "local-exec" { - command = "${path.module}/copy_image.sh" - environment = { - AWS_PROFILE = var.profile - AWS_REGION = local.region - SOURCE_IMAGE = format("%v/%v:%v", local.src_reg, each.value.image, each.value.tag) - DESTINATION_IMAGE = format("%v:%v", aws_ecr_repository.repository[each.key].repository_url, each.value.tag) - DESTINATION_USERNAME = data.aws_ecr_authorization_token.token.user_name - DESTINATION_PASSWORD = data.aws_ecr_authorization_token.token.password - } - } -} - diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/ecr.tf b/examples/full-cluster-tf-upgrade/1.25/efs/ecr.tf index 228a775..d783265 100644 --- a/examples/full-cluster-tf-upgrade/1.25/efs/ecr.tf +++ b/examples/full-cluster-tf-upgrade/1.25/efs/ecr.tf @@ -1,57 +1,70 @@ - -# Populated from: -# https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html - -data "aws_caller_identity" "whoami" {} - -locals { - af_south_1 = (var.region == "af-south-1" ? "877085696533.dkr.ecr.af-south-1.amazonaws.com/" : "") - af = local.af_south_1 - - ap_east_1 = var.region == "ap-east-1" ? "800184023465.dkr.ecr.ap-east-1.amazonaws.com/" : "" - ap_northeast_1 = var.region == "ap-northeast-1" ? "602401143452.dkr.ecr.ap-northeast-1.amazonaws.com/" : "" - ap_northeast_2 = var.region == "ap-northeast-2" ? "602401143452.dkr.ecr.ap-northeast-2.amazonaws.com/" : "" - ap_northeast_3 = var.region == "ap-northeast-3" ? "602401143452.dkr.ecr.ap-northeast-3.amazonaws.com/" : "" - ap_south_1 = var.region == "ap-south-1" ? "602401143452.dkr.ecr.ap-south-1.amazonaws.com/" : "" - ap_southeast_1 = var.region == "ap-southeast-1" ? "602401143452.dkr.ecr.ap-southeast-1.amazonaws.com/" : "" - ap_southeast_2 = var.region == "ap-southeast-2" ? "602401143452.dkr.ecr.ap-southeast-2.amazonaws.com/" : "" - ap_1 = "${local.ap_east_1}${local.ap_northeast_1}${local.ap_northeast_2}${local.ap_northeast_3}${local.ap_south_1}" - ap_2 = "${local.ap_southeast_1}${local.ap_southeast_2}" - ap = "${local.ap_1}${local.ap_2}" - - ca_central_1 = var.region == "ca-central-1" ? "602401143452.dkr.ecr.ca-central-1.amazonaws.com/" : "" - ca = local.ca_central_1 - - cn_north_1 = var.region == "cn-north-1" ? "918309763551.dkr.ecr.cn-north-1.amazonaws.com.cn/" : "" - cn_northwest_1 = var.region == "cn-northwest-1" ? "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/" : "" - cn = "${local.cn_north_1}${local.cn_northwest_1}" - - eu_central_1 = var.region == "eu-central-1" ? "602401143452.dkr.ecr.eu-central-1.amazonaws.com/" : "" - eu_north_1 = var.region == "eu-north-1" ? "602401143452.dkr.ecr.eu-north-1.amazonaws.com/" : "" - eu_south_1 = var.region == "eu-south-1" ? "590381155156.dkr.ecr.eu-south-1.amazonaws.com/" : "" - eu_west_1 = var.region == "eu-west-1" ? "602401143452.dkr.ecr.eu-west-1.amazonaws.com/" : "" - eu_west_2 = var.region == "eu-west-2" ? "602401143452.dkr.ecr.eu-west-2.amazonaws.com/" : "" - eu_west_3 = var.region == "eu-west-3" ? "602401143452.dkr.ecr.eu-west-3.amazonaws.com/" : "" - eu = "${local.eu_central_1}${local.eu_north_1}${local.eu_south_1}${local.eu_west_1}${local.eu_west_2}${local.eu_west_3}" - - me_south_1 = var.region == "me-south-1" ? "558608220178.dkr.ecr.me-south-1.amazonaws.com/" : "" - me = local.me_south_1 - - sa_east_1 = var.region == "sa-east-1" ? "602401143452.dkr.ecr.sa-east-1.amazonaws.com/" : "" - sa = local.sa_east_1 - - us_east_1 = var.region == "us-east-1" ? "602401143452.dkr.ecr.us-east-1.amazonaws.com/" : "" - us_east_2 = var.region == "us-east-2" ? "602401143452.dkr.ecr.us-east-2.amazonaws.com/" : "" - us_gov_east_1 = var.region == "us-gov-east-1" ? "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/" : "" - us_gov_west_1 = var.region == "us-gov-west-1" ? "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/" : "" - us_west_1 = var.region == "us-west-1" ? "602401143452.dkr.ecr.us-west-1.amazonaws.com/" : "" - us_west_2 = var.region == "us-west-2" ? "602401143452.dkr.ecr.us-west-2.amazonaws.com/" : "" - us = "${local.us_east_1}${local.us_east_2}${local.us_gov_east_1}${local.us_gov_west_1}${local.us_west_1}${local.us_west_2}" - - ecr = "${local.af}${local.ap}${local.ca}${local.cn}${local.eu}${local.me}${local.sa}${local.us}" - - - public_reg = "public.ecr.aws" - src_reg = format("%v/eks-distro/kubernetes-csi", local.public_reg) - account_ecr = "${data.aws_caller_identity.whoami.account_id}.dkr.ecr.${var.region}.amazonaws.com/${var.cluster_name}" +local { + ecr_mapping_default = "602401143452" + ecr_mapping = { + "us-gov-east-1" = "151742754352" + "us-gov-west-1" = "013241004608" + "us-east-1" = "602401143452" + "us-west-2" = "602401143452" + "us-east-1" = "602401143452" + "us-west-2" = "602401143452" + } + public_ecr = format("%v.dkr.ecr.%v.amazonaws.com", lookup(local.ecr_mapping, local.region, local.ecr_mapping_default), local.region) } + +## # Populated from: +## # https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html +## +## data "aws_caller_identity" "whoami" {} +## +## locals { +## af_south_1 = (var.region == "af-south-1" ? "877085696533.dkr.ecr.af-south-1.amazonaws.com/" : "") +## af = local.af_south_1 +## +## ap_east_1 = var.region == "ap-east-1" ? "800184023465.dkr.ecr.ap-east-1.amazonaws.com/" : "" +## ap_northeast_1 = var.region == "ap-northeast-1" ? "602401143452.dkr.ecr.ap-northeast-1.amazonaws.com/" : "" +## ap_northeast_2 = var.region == "ap-northeast-2" ? "602401143452.dkr.ecr.ap-northeast-2.amazonaws.com/" : "" +## ap_northeast_3 = var.region == "ap-northeast-3" ? "602401143452.dkr.ecr.ap-northeast-3.amazonaws.com/" : "" +## ap_south_1 = var.region == "ap-south-1" ? "602401143452.dkr.ecr.ap-south-1.amazonaws.com/" : "" +## ap_southeast_1 = var.region == "ap-southeast-1" ? "602401143452.dkr.ecr.ap-southeast-1.amazonaws.com/" : "" +## ap_southeast_2 = var.region == "ap-southeast-2" ? "602401143452.dkr.ecr.ap-southeast-2.amazonaws.com/" : "" +## ap_1 = "${local.ap_east_1}${local.ap_northeast_1}${local.ap_northeast_2}${local.ap_northeast_3}${local.ap_south_1}" +## ap_2 = "${local.ap_southeast_1}${local.ap_southeast_2}" +## ap = "${local.ap_1}${local.ap_2}" +## +## ca_central_1 = var.region == "ca-central-1" ? "602401143452.dkr.ecr.ca-central-1.amazonaws.com/" : "" +## ca = local.ca_central_1 +## +## cn_north_1 = var.region == "cn-north-1" ? "918309763551.dkr.ecr.cn-north-1.amazonaws.com.cn/" : "" +## cn_northwest_1 = var.region == "cn-northwest-1" ? "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/" : "" +## cn = "${local.cn_north_1}${local.cn_northwest_1}" +## +## eu_central_1 = var.region == "eu-central-1" ? "602401143452.dkr.ecr.eu-central-1.amazonaws.com/" : "" +## eu_north_1 = var.region == "eu-north-1" ? "602401143452.dkr.ecr.eu-north-1.amazonaws.com/" : "" +## eu_south_1 = var.region == "eu-south-1" ? "590381155156.dkr.ecr.eu-south-1.amazonaws.com/" : "" +## eu_west_1 = var.region == "eu-west-1" ? "602401143452.dkr.ecr.eu-west-1.amazonaws.com/" : "" +## eu_west_2 = var.region == "eu-west-2" ? "602401143452.dkr.ecr.eu-west-2.amazonaws.com/" : "" +## eu_west_3 = var.region == "eu-west-3" ? "602401143452.dkr.ecr.eu-west-3.amazonaws.com/" : "" +## eu = "${local.eu_central_1}${local.eu_north_1}${local.eu_south_1}${local.eu_west_1}${local.eu_west_2}${local.eu_west_3}" +## +## me_south_1 = var.region == "me-south-1" ? "558608220178.dkr.ecr.me-south-1.amazonaws.com/" : "" +## me = local.me_south_1 +## +## sa_east_1 = var.region == "sa-east-1" ? "602401143452.dkr.ecr.sa-east-1.amazonaws.com/" : "" +## sa = local.sa_east_1 +## +## us_east_1 = var.region == "us-east-1" ? "602401143452.dkr.ecr.us-east-1.amazonaws.com/" : "" +## us_east_2 = var.region == "us-east-2" ? "602401143452.dkr.ecr.us-east-2.amazonaws.com/" : "" +## us_gov_east_1 = var.region == "us-gov-east-1" ? "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/" : "" +## us_gov_west_1 = var.region == "us-gov-west-1" ? "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/" : "" +## us_west_1 = var.region == "us-west-1" ? "602401143452.dkr.ecr.us-west-1.amazonaws.com/" : "" +## us_west_2 = var.region == "us-west-2" ? "602401143452.dkr.ecr.us-west-2.amazonaws.com/" : "" +## us = "${local.us_east_1}${local.us_east_2}${local.us_gov_east_1}${local.us_gov_west_1}${local.us_west_1}${local.us_west_2}" +## +## ecr = "${local.af}${local.ap}${local.ca}${local.cn}${local.eu}${local.me}${local.sa}${local.us}" +## +## +## public_reg = "public.ecr.aws" +## src_reg = format("%v/eks-distro/kubernetes-csi", local.public_reg) +## account_ecr = "${data.aws_caller_identity.whoami.account_id}.dkr.ecr.${var.region}.amazonaws.com/${var.cluster_name}" +## } +## diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/efs.tf b/examples/full-cluster-tf-upgrade/1.25/efs/efs.tf index 9fb5563..4859130 100644 --- a/examples/full-cluster-tf-upgrade/1.25/efs/efs.tf +++ b/examples/full-cluster-tf-upgrade/1.25/efs/efs.tf @@ -4,9 +4,10 @@ module "efs" { source = "git@github.e.it.census.gov:terraform-modules/aws-efs.git" - name = var.cluster_name - vpc_id = local.vpc_id - subnet_ids = local.subnet_ids + name = var.cluster_name + vpc_id = local.vpc_id + subnet_ids = local.subnet_ids + ## consider changing this to the new extra_cluster_sg security_groups = [local.cluster_worker_sg_id] ## subnet_ids = local.cni_subnet_ids ## security_groups = [local.cluster_cni_worker_sg_id] diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/images.tf.obsolete b/examples/full-cluster-tf-upgrade/1.25/efs/images.tf.obsolete new file mode 100644 index 0000000..40bbf6a --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.25/efs/images.tf.obsolete @@ -0,0 +1,93 @@ +locals { + image_config = [for k, v in var.image_details : v if v.enabled] + image_output = { for k, v in module.images.images : v.name => v } + + charts = { for k, v in var.chart_details : k => v if try(v.enabled, true) } + images = { for k, v in var.image_details : k => v if v.enabled } + +} + +module "images" { + source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git?ref=tf-upgrade" + + profile = var.profile + application_list = [] + application_name = format("eks/%v", var.cluster_name) + image_config = local.image_config + tags = merge( + local.base_tags, + local.common_tags, + var.application_tags, + ) + + ### optional + ## account_alias = "" + ## account_id = "" + ## destination_password = "" + ## destination_username = "" + ## override_prefixes = {} + ## region = "" + ## source_password = "" + ## source_username = "" +} + + +## image_info = { +## "elastic/elasticsearch#7.14.0" = { +## "dest_full_path" = "817869416306.dkr.ecr.us-gov-east-1.amazonaws.com/eks/test-cluster-name/elastic/elasticsearch:7.14.0" +## "dest_registry" = "817869416306.dkr.ecr.us-gov-east-1.amazonaws.com" +## "dest_repository" = "eks/test-cluster-name/elastic/elasticsearch" +## "enabled" = true +## "key" = "elastic/elasticsearch#7.14.0" +## "name" = "elastic/elasticsearch" +## "source_full_path" = "docker.elastic.co/elasticsearch/elasticsearch:7.14.0" +## "source_image" = "elasticsearch/elasticsearch" +## "source_registry" = "docker.elastic.co" +## "tag" = "7.14.0" +## } +## "elastic/kibana#7.14.0" = { +## "dest_full_path" = "817869416306.dkr.ecr.us-gov-east-1.amazonaws.com/eks/test-cluster-name/elastic/kibana:7.14.0" +## "dest_registry" = "817869416306.dkr.ecr.us-gov-east-1.amazonaws.com" +## "dest_repository" = "eks/test-cluster-name/elastic/kibana" +## "enabled" = true +## "key" = "elastic/kibana#7.14.0" +## "name" = "elastic/kibana" +## "source_full_path" = "docker.elastic.co/kibana/kibana:7.14.0" +## "source_image" = "kibana/kibana" +## "source_registry" = "docker.elastic.co" +## "tag" = "7.14.0" +## } +## "fluent/fluentd-kubernetes-daemonset#v1.13.3-debian-elasticsearch7-1.2" = { +## "dest_full_path" = "817869416306.dkr.ecr.us-gov-east-1.amazonaws.com/eks/test-cluster-name/fluent/fluentd-kubernetes-daemonset:v1.13.3-debian-elasticsearch7-1.2" +## "dest_registry" = "817869416306.dkr.ecr.us-gov-east-1.amazonaws.com" +## "dest_repository" = "eks/test-cluster-name/fluent/fluentd-kubernetes-daemonset" +## "enabled" = true +## "key" = "fluent/fluentd-kubernetes-daemonset#v1.13.3-debian-elasticsearch7-1.2" +## "name" = "fluent/fluentd-kubernetes-daemonset" +## "source_full_path" = "docker.io/fluent/fluentd-kubernetes-daemonset:v1.13.3-debian-elasticsearch7-1.2" +## "source_image" = "fluent/fluentd-kubernetes-daemonset" +## "source_registry" = "docker.io" +## "tag" = "v1.13.3-debian-elasticsearch7-1.2" +## } +## } + +## image_output = label => thing + + +## # chart +## repository = local.charts["metrics-server"].use_remote ? local.charts["metrics-server"].repository : "${path.module}/charts" +## version = local.charts["metrics-server"].use_remote ? local.charts["metrics-server"].version : null +## +## set { +## name = "image.registry" +## value = local.image_output["metrics-server"].dest_registry +## } +## set { +## name = "image.repository" +## value = local.image_output["metrics-server"].dest_repository +## } +## +## set { +## name = "image.tag" +## value = local.image_output["metrics-server"].tag +## } diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/main.tf b/examples/full-cluster-tf-upgrade/1.25/efs/main.tf.obsolete similarity index 72% rename from examples/full-cluster-tf-upgrade/1.25/efs/main.tf rename to examples/full-cluster-tf-upgrade/1.25/efs/main.tf.obsolete index 446e3b9..956cb44 100644 --- a/examples/full-cluster-tf-upgrade/1.25/efs/main.tf +++ b/examples/full-cluster-tf-upgrade/1.25/efs/main.tf.obsolete @@ -7,16 +7,16 @@ ## url = data.aws_eks_cluster.cluster.identity[0].oidc[0].issuer ## } -locals { - charts = { - "efs-provisioner" = { - name = "aws-efs-csi-driver" - repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver" - version = "2.1.4" - use_remote = true - } - } -} +## locals { +## charts = { +## "efs-provisioner" = { +## name = "aws-efs-csi-driver" +## repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver" +## version = "2.1.4" +## use_remote = true +## } +## } +## } # Create an IAM policy and role # Step 1b: @@ -41,7 +41,7 @@ locals { # Step 3: # See the readme `Updating the aws-efs-csi-driver chart` on updating this chart. resource "helm_release" "efs-provisioner" { - depends_on = [null_resource.copy_images] + ## depends_on = [null_resource.copy_images] chart = "aws-efs-csi-driver" name = "efs-provisioner" @@ -52,36 +52,37 @@ resource "helm_release" "efs-provisioner" { recreate_pods = true timeout = 300 set { - name = "image.repository" - value = "${local.ecr}eks/aws-efs-csi-driver" + name = "image.repository" + # value = "${local.ecr}eks/aws-efs-csi-driver" + # consider copying this image into our eks + value = format("%v/aws-efs-csi-driver", local.public_ecr) } set { name = "sidecars.livenessProbe.image.repository" - value = aws_ecr_repository.repository["livenessprobe"].repository_url + value = format("%v/%v", local.image_output["livenessprobe"].dest_registry, local.image_output["livenessprobe"].dest_repository) } set { name = "sidecars.livenessProbe.image.tag" - value = var.livenessprobe_tag + value = local.image_output["livenessprobe"].tag } set { name = "sidecars.nodeDriverRegistrar.image.repository" - value = aws_ecr_repository.repository["node-driver-registrar"].repository_url + value = format("%v/%v", local.image_output["node-driver-registrar"].dest_registry, local.image_output["node-driver-registrar"].dest_repository) } set { name = "sidecars.nodeDriverRegistrar.image.tag" - value = var.node_driver_registrar_tag + value = local.image_output["node-driver-registrar"].tag } set { name = "sidecars.csiProvisioner.image.repository" - value = aws_ecr_repository.repository["external-provisioner"].repository_url + value = format("%v/%v", local.image_output["external-provisioner"].dest_registry, local.image_output["external-provisioner"].dest_repository) } set { name = "sidecars.csiProvisioner.image.tag" - value = var.external_provisioner_tag + value = local.image_output["external-provisioner"].tag } set { - name = "controller.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - # value = aws_iam_role.cluster_efs_role.arn + name = "controller.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" value = module.role_efs-driver.role_arn } } @@ -122,4 +123,3 @@ resource "kubernetes_storage_class" "efs-sc" { } mount_options = ["tls"] } - diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/policy.tf b/examples/full-cluster-tf-upgrade/1.25/efs/policy.tf index 2693fde..b98f39d 100644 --- a/examples/full-cluster-tf-upgrade/1.25/efs/policy.tf +++ b/examples/full-cluster-tf-upgrade/1.25/efs/policy.tf @@ -1,3 +1,6 @@ +# created from +# arn:aws-us-gov:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy + # apply policy before creating role # tf-apply -target=aws_iam_policy.efs-policy @@ -11,11 +14,10 @@ resource "aws_iam_policy" "efs-policy" { local.base_tags, local.common_tags, var.application_tags, - tomap({ "Name" = format("%v%v-efs-driver", local._prefixes["eks-policy"], var.cluster_name) }), + { "Name" = format("%v%v-efs-driver", local._prefixes["eks-policy"], var.cluster_name) }, ) } -# TBD: refine resources to limit only to eks configurations data "aws_iam_policy_document" "efs-policy" { statement { sid = "EKSEFSDescribe" @@ -24,6 +26,8 @@ data "aws_iam_policy_document" "efs-policy" { actions = [ "elasticfilesystem:DescribeAccessPoints", "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:DescribeMountTargets", + "ec2:DescribeAvailabilityZones", ] } statement { @@ -34,9 +38,37 @@ data "aws_iam_policy_document" "efs-policy" { "elasticfilesystem:CreateAccessPoint" ] condition { - test = "StringLike" + test = "Null" + variable = "aws:RequestTag/efs.csi.aws.com/cluster" + values = ["false"] + } + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = ["efs.csi.aws.com/cluster"] + } + } + statement { + sid = "AllowTagNewAccessPoints" + effect = "Allow" + resources = ["*"] + actions = [ + "elasticfilesystem:TagResource", + ] + condition { + test = "StringEquals" + variable = "elasticfilesystem:CreateAction" + values = ["CreateAccessPoint"] + } + condition { + test = "Null" variable = "aws:RequestTag/efs.csi.aws.com/cluster" - values = ["true"] + values = ["false"] + } + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = ["efs.csi.aws.com/cluster"] } } statement { @@ -47,9 +79,9 @@ data "aws_iam_policy_document" "efs-policy" { "elasticfilesystem:DeleteAccessPoint" ] condition { - test = "StringLike" + test = "Null" variable = "aws:ResourceTag/efs.csi.aws.com/cluster" - values = ["true"] + values = ["false"] } } } diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/role.tf b/examples/full-cluster-tf-upgrade/1.25/efs/role.tf index 3d203d1..0cda603 100644 --- a/examples/full-cluster-tf-upgrade/1.25/efs/role.tf +++ b/examples/full-cluster-tf-upgrade/1.25/efs/role.tf @@ -35,9 +35,14 @@ data "aws_iam_policy_document" "efs_assume_webidentity" { identifiers = [local.principal] } condition { - test = "StringEquals" + test = "StringLike" variable = "${local.oidc_provider_url}:sub" - values = ["system:serviceaccount:kube-system:efs-csi-controller-sa"] + values = ["system:serviceaccount:kube-system:efs-csi-*"] + } + condition { + test = "StringLike" + variable = "${local.oidc_provider_url}:aud" + values = ["sts.amazonaws.com"] } } } diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/storage-class.tf b/examples/full-cluster-tf-upgrade/1.25/efs/storage-class.tf new file mode 100644 index 0000000..ca88c36 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.25/efs/storage-class.tf @@ -0,0 +1,17 @@ +resource "kubernetes_storage_class" "efs-sc" { + depends_on = [ + module.efs, + aws_eks_addon.aws-efs-csi-driver, + ] + + metadata { + name = "efs" + } + storage_provisioner = "efs.csi.aws.com" + parameters = { + provisioningMode = "efs-ap" + fileSystemId = module.efs.id + directoryPerms = "700" + } + mount_options = ["tls"] +} diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/tf-run.data b/examples/full-cluster-tf-upgrade/1.25/efs/tf-run.data index cbeb185..76a449d 100644 --- a/examples/full-cluster-tf-upgrade/1.25/efs/tf-run.data +++ b/examples/full-cluster-tf-upgrade/1.25/efs/tf-run.data @@ -1,8 +1,7 @@ -VERSION 1.2.1 +VERSION 1.2.6 REMOTE-STATE COMMAND tf-directory-setup.py -l none -f COMMAND setup-new-directory.sh -COMMAND tf-init -upgrade LINKTOP init LINKTOP includes.d/variables.account_tags.tf LINKTOP includes.d/variables.account_tags.auto.tfvars @@ -14,6 +13,16 @@ LINKTOP provider_configs.d/provider.ldap_new.auto.tfvars LINKTOP provider_configs.d/provider.ldap_new.tf LINKTOP provider_configs.d/provider.ldap_new.variables.tf LINK settings.auto.tfvars +LINK includes.d/parent_rs.tf +LINK includes.d/data.eks-subdirectory.tf +LINK includes.d/kubeconfig.eks-subdirectory.tf +LINK variables.eks.tf +LINK prefixes.tf +LINK providers.tf +LINK variables.addons.tf +LINK versions.tf +LINK version.tf +COMMAND tf-init POLICY ALL diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/variables.images.auto.tfvars.obsolete b/examples/full-cluster-tf-upgrade/1.25/efs/variables.images.auto.tfvars.obsolete new file mode 100644 index 0000000..58c0a88 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.25/efs/variables.images.auto.tfvars.obsolete @@ -0,0 +1,46 @@ +chart_details = { + "efs-provisioner" = { + name = "efs-provisioner" + repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver" + # version = "2.1.4" + version = "2.4.8" + use_remote = true + enabled = true + } +} + +image_details = { + "external-provisioner" = { + name = "external-provisioner" + image = "public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner" + dest_path = null + source_registry = "public.ecr.aws" + source_image = "eks-distro/kubernetes-csi/external-provisioner" + source_tag = null + tag = "v3.5.0-eks-1-25-latest" + # tag = "v2.1.1-eks-1-18-2" + enabled = true + } + "livenessprobe" = { + name = "livenessprobe" + image = "public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe" + dest_path = null + source_registry = "public.ecr.aws" + source_image = "eks-distro/kubernetes-csi/livenessprobe" + source_tag = null + tag = "v2.10.0-eks-1-25-latest" + # tag = "v2.2.0-eks-1-18-2" + enabled = true + } + "node-driver-registrar" = { + name = "node-driver-registrar" + image = "public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar" + dest_path = null + source_registry = "public.ecr.aws" + source_image = "eks-distro/kubernetes-csi/node-driver-registrar" + source_tag = null + tag = "v2.8.0-eks-1-25-latest" + # tag = "v2.1.0-eks-1-18-2" + enabled = true + } +} diff --git a/examples/full-cluster-tf-upgrade/1.25/efs/variables.images.tf.obsolete b/examples/full-cluster-tf-upgrade/1.25/efs/variables.images.tf.obsolete new file mode 100644 index 0000000..b7426ef --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.25/efs/variables.images.tf.obsolete @@ -0,0 +1,26 @@ +variable "chart_details" { + description = "Map of object with details about remote charts" + type = map(object( + { + name = string + repository = string + version = string + use_remote = bool + })) + default = {} +} + +variable "image_details" { + description = "List of image configuration objects to copy from SOURCE to DESTINATION" + type = map(object({ + name = string, + tag = string, + dest_path = string, + source_registry = string, + source_image = string, + source_tag = string, + enabled = bool, + })) + default = {} +} + diff --git a/examples/full-cluster-tf-upgrade/1.25/variables.addons.tf b/examples/full-cluster-tf-upgrade/1.25/variables.addons.tf new file mode 100644 index 0000000..4853495 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.25/variables.addons.tf @@ -0,0 +1,20 @@ +variable "addon_versions" { + description = "Map of addon versions by Kubernetes version" + type = map(map(string)) + default = { + "1.24" = { + "coredns" = "v1.8.7-eksbuild.3" + "kube-proxy" = "v1.24.7-eksbuild.2" + "vpc-cni" = "v1.11.4-eksbuild.1" + "aws-ebs-csi-driver" = "v1.18.0-eksbuild.1" + } + "1.25" = { + "coredns" = "v1.9.3-eksbuild.5" + "kube-proxy" = "v1.25.11-eksbuild.2" + "vpc-cni" = "v1.13.4-eksbuild.1" + "aws-ebs-csi-driver" = "v1.21.0-eksbuild.1" + "aws-efs-csi-driver" = "v1.5.8-eksbuild.1" + "adot" = "v0.78.0-eksbuild.1" + } + } +} diff --git a/examples/full-cluster-tf-upgrade/1.25/versions.tf b/examples/full-cluster-tf-upgrade/1.25/versions.tf index ced1ff0..adf8e62 100644 --- a/examples/full-cluster-tf-upgrade/1.25/versions.tf +++ b/examples/full-cluster-tf-upgrade/1.25/versions.tf @@ -1,4 +1,5 @@ terraform { + required_version = ">= 1.0.0" required_providers { aws = { source = "hashicorp/aws" @@ -37,5 +38,4 @@ terraform { version = ">= 0.9" } } - required_version = ">= 0.13" }