replace ebs with addons

examples/full-cluster-tf-upgrade/1.25/addons/addon_coredns.tf

@@ -0,0 +1,8 @@
+resource "aws_eks_addon" "coredns" {
+  count = lookup(local.addon_versions, "coredns", null) != null ? 1 : 0
+
+  cluster_name      = var.cluster_name
+  addon_name        = "coredns"
+  addon_version     = lokup(local.addon_versions, "coredns")
+  resolve_conflicts = "OVERWRITE"
+}

examples/full-cluster-tf-upgrade/1.25/addons/addon_ebs-csi.tf

@@ -0,0 +1,119 @@
+## resource "aws_iam_role" "cluster_ebs_role" {
+##   name               = "${var.cluster_name}_ebs_driver_role"
+##   assume_role_policy = <<POLICY
+## 
+## {
+##   "Version": "2012-10-17",
+##   "Statement": [
+##     {
+##       "Effect": "Allow",
+##       "Principal": {
+##         "Federated": "${local.principal}"
+##       },
+##       "Action": "sts:AssumeRoleWithWebIdentity",
+##       "Condition": {
+##         "StringEquals": {
+##           "${local.oidc}:aud": "sts.amazonaws.com",
+##           "${local.oidc}:sub": "system:serviceaccount:kube-system:ebs-csi-controller-sa"
+##         }
+##       }
+##     }
+##   ]
+## }
+## POLICY
+## }
+## 
+## resource "aws_iam_role_policy_attachment" "role-policy-attachment" {
+##   for_each = toset([
+##     "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy",
+##   ])
+## 
+##   role       = aws_iam_role.cluster_ebs_role.name
+##   policy_arn = each.value
+## }
+
+## locals {
+##   ebs_config = <<CONFIG
+## controller:
+##   nodeSelector:
+##     ${var.durable_node_selector_key} : ${var.durable_node_selector_value}
+## CONFIG
+## }
+
+resource "aws_eks_addon" "aws-ebs-csi-driver" {
+  count = lookup(local.addon_versions, "aws-ebs-csi-driver", null) != null ? 1 : 0
+
+  cluster_name             = var.cluster_name
+  addon_name               = "aws-ebs-csi-driver"
+  addon_version            = lokup(local.addon_versions, "aws-ebs-csi-driver")
+  resolve_conflicts        = "OVERWRITE"
+  service_account_role_arn = module.role_ebs-driver.role_arn
+  configuration_values     = null
+}
+
+## # Delete the old gp2 default storage class.
+## resource "null_resource" "kubectl" {
+##   depends_on = [null_resource.download_kubeconfig]
+## 
+##   provisioner "local-exec" {
+##     command     = "kubectl --kubeconfig ${path.module}/kube_config delete sc gp2"
+##     interpreter = ["/bin/bash", "-c"]
+##     environment = {
+##       KUBECONFIG = "${path.module}/../eks/kube_config"
+##     }
+##   }
+## }
+## 
+## # Create a default storage class.
+## resource "kubernetes_storage_class" "ebs_sc" {
+##   depends_on = [aws_eks_addon.ebs_csi]
+## 
+##   metadata {
+##     name = "gp3"
+##     annotations = {
+##       "storageclass.kubernetes.io/is-default-class" = "true"
+##     }
+##   }
+##   storage_provisioner = "ebs.csi.aws.com"
+## }
+## 
+
+
+data "aws_iam_policy" "ebs-provisioner" {
+  name = "AmazonEBSCSIDriverPolicy"
+}
+
+module "role_ebs-driver" {
+  source = "git@github.e.it.census.gov:terraform-modules/aws-iam-role.git?ref=tf-upgrade"
+
+  role_name              = format("%v%v-ebs-driver", local._prefixes["eks"], var.cluster_name)
+  role_description       = "EKS EBS Driver Role for ${var.cluster_name}"
+  enable_ldap_creation   = false
+  assume_policy_document = data.aws_iam_policy_document.ebs_assume_webidentity.json
+  attached_policies      = [data.aws_iam_policy.ebs-provisioner.arn]
+
+  tags = merge(
+    local.base_tags,
+    local.common_tags,
+    var.application_tags,
+    tomap({ "Name" = format("%v%v-ebs-driver", local._prefixes["eks-role"], var.cluster_name) }),
+  )
+}
+
+data "aws_iam_policy_document" "ebs_assume_webidentity" {
+  statement {
+    sid     = "EFSAssumeRoleWebIdentity"
+    effect  = "Allow"
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+    principals {
+      type        = "Federated"
+      identifiers = [local.principal]
+    }
+    condition {
+      test     = "StringEquals"
+      variable = "${local.oidc_provider_url}:sub"
+      values   = ["system:serviceaccount:kube-system:ebs-csi-controller-sa"]
+    }
+  }
+}
+

examples/full-cluster-tf-upgrade/1.25/addons/addon_kube-proxy.tf

@@ -0,0 +1,10 @@
+resource "aws_eks_addon" "kube-proxy" {
+  count = lookup(local.addon_versions, "kube-proxy", null) != null ? 1 : 0
+
+  cluster_name      = var.cluster_name
+  addon_name        = "kube-proxy"
+  addon_version     = lokup(local.addon_versions, "kube-proxy")
+  resolve_conflicts = "OVERWRITE"
+}
+
+

examples/full-cluster-tf-upgrade/1.25/addons/addon_vpc-cni.tf

@@ -0,0 +1,81 @@
+resource "aws_eks_addon" "vpc-cni" {
+  count = lookup(local.addon_versions, "vpc-cni", null) != null ? 1 : 0
+
+  cluster_name             = var.cluster_name
+  addon_name               = "vpc-cni"
+  addon_version            = lokup(local.addon_versions, "vpc-cni")
+  resolve_conflicts        = "OVERWRITE"
+  service_account_role_arn = module.role_vpc-cni.role_arn
+}
+
+
+resource "kubernetes_annotations" "vpc-cni" {
+  kind = "serviceaccount"
+  metadata {
+    name      = "aws-node"
+    namespace = "kube-system"
+  }
+  annotations = {
+    "eks.amazonaws.com/role-arn" = module.role_vpc-cni.role_arn
+  }
+  depends_on = [aws_eks_addon.vpc-cni]
+}
+
+## resource "null_resource" "kubectl" {
+##   depends_on = [
+##     aws_eks_addon.vpc_cni
+##   ]
+## 
+##   provisioner "local-exec" {
+##     command     = "kubectl annotate serviceaccount -n kube-system aws-node eks.amazonaws.com/role-arn=${module.role_vpc-cni.role_arn}"
+##     interpreter = ["/bin/bash", "-c"]
+##     environment = {
+##       KUBECONFIG = "${path.root}/setup/kube.config"
+##     }
+##   }
+## }
+## 
+
+data "aws_iam_policy" "vpc_cni" {
+  name = "AmazonEKS_CNI_Policy"
+}
+
+module "role_vpc-cni" {
+  source = "git@github.e.it.census.gov:terraform-modules/aws-iam-role.git?ref=tf-upgrade"
+
+  role_name              = format("%v%v-vpc-cni", local._prefixes["eks"], var.cluster_name)
+  role_description       = "EKS VPC-CNI Role for ${var.cluster_name}"
+  enable_ldap_creation   = false
+  assume_policy_document = data.aws_iam_policy_document.vpc_cni_assume_webidentity.json
+  attached_policies      = [data.aws_iam_policy.vpc_cni.arn]
+
+  tags = merge(
+    local.base_tags,
+    local.common_tags,
+    var.application_tags,
+    tomap({ "Name" = format("%v%v-ebs-driver", local._prefixes["eks-role"], var.cluster_name) }),
+  )
+}
+
+data "aws_iam_policy_document" "vpc_cni_assume_webidentity" {
+  statement {
+    sid     = "VPCCNIAssumeRoleWebIdentity"
+    effect  = "Allow"
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+    principals {
+      type        = "Federated"
+      identifiers = [local.principal]
+    }
+    condition {
+      test     = "ForAnyValue:StringEquals"
+      variable = "${local.oidc_provider_url}:aud"
+      values   = ["sts.amazonaws.com"]
+    }
+    condition {
+      test     = "ForAnyValue:StringEquals"
+      variable = "${local.oidc_provider_url}:sub"
+      values   = ["system:serviceaccount:kube-system:aws-node"]
+    }
+
+  }
+}

examples/full-cluster-tf-upgrade/1.25/addons/addons.tf

@@ -0,0 +1,18 @@
+locals {
+  addon_versions = lookup(var.addon_version, var.cluster_version, {})
+}
+
+
+variable "addon_versions" {
+  description = "Map of addon versions by Kubernetes version"
+  type        = map(map(string))
+  default = {
+    "1.24" = {}
+    "1.25" = {
+      "coredns"    = "v1.9.3-eksbuild.2"
+      "kube-proxy" = "v1.25.6-eksbuild.1"
+      "vpc-cni"    = "v1.12.2-eksbuild.1"
+    }
+  }
+}
+

examples/full-cluster-tf-upgrade/1.25/ebs/role.tf → examples/full-cluster-tf-upgrade/1.25/addons/role.tf.off

@@ -10,9 +10,14 @@ data "aws_iam_policy" "ebs-provisioner" {
   name = "AmazonEBSCSIDriverPolicy"
 }
 
+data "aws_iam_policy" "vpc_cni" {
+  name = "AmazonEKS_CNI_Policy"
+}
+
+
 # create: aws_iam_policy.ebs-policy first
 module "role_ebs-driver" {
-  source = "git@github.e.it.census.gov:terraform-modules/aws-iam-role.git?ref=tf-upgrade"
+  source = "git@github.e.it.census.gov:terraform-modules/aws-iam-role.git"
 
   role_name              = format("%v%v-ebs-driver", local._prefixes["eks"], var.cluster_name)
   role_description       = "EKS EBS Driver Role for ${var.cluster_name}"
@@ -28,6 +33,23 @@ module "role_ebs-driver" {
   )
 }
 
+module "role_vpc-cni" {
+  source = "git@github.e.it.census.gov:terraform-modules/aws-iam-role.git"
+
+  role_name              = format("%v%v-vpc-cni", local._prefixes["eks"], var.cluster_name)
+  role_description       = "EKS VPC-CNI Role for ${var.cluster_name}"
+  enable_ldap_creation   = false
+  assume_policy_document = data.aws_iam_policy_document.vpc_cni_assume_webidentity.json
+  attached_policies      = [data.aws_iam_policy.vpc_cni.arn]
+
+  tags = merge(
+    local.base_tags,
+    local.common_tags,
+    var.application_tags,
+    tomap({ "Name" = format("%v%v-ebs-driver", local._prefixes["eks-role"], var.cluster_name) }),
+  )
+}
+
 data "aws_iam_policy_document" "ebs_assume_webidentity" {
   statement {
     sid     = "EFSAssumeRoleWebIdentity"
@@ -45,7 +67,35 @@ data "aws_iam_policy_document" "ebs_assume_webidentity" {
   }
 }
 
+data "aws_iam_policy_document" "vpc_cni_assume_webidentity" {
+  statement {
+    sid     = "VPCCNIAssumeRoleWebIdentity"
+    effect  = "Allow"
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+    principals {
+      type        = "Federated"
+      identifiers = [local.principal]
+    }
+    condition {
+      test     = "ForAnyValue:StringEquals"
+      variable = "${local.oidc_provider_url}:aud"
+      values   = ["sts.amazonaws.com"]
+    }
+    condition {
+      test     = "ForAnyValue:StringEquals"
+      variable = "${local.oidc_provider_url}:sub"
+      values   = ["system:serviceaccount:kube-system:aws-node"]
+    }
+
+  }
+}
+
 output "role_ebs-driver_arn" {
   description = "Role ARN for EKS EBS Driver Role"
   value       = module.role_ebs-driver.role_arn
 }
+
+output "role_vpc-cni_arn" {
+  description = "Role ARN for EKS VPC-CNI Addon"
+  value       = module.role_vpc-cni.role_arn
+}

examples/full-cluster-tf-upgrade/1.25/addons/variables.addons.tf

@@ -0,0 +1,19 @@
+variable "addon_versions" {
+  description = "Map of addon versions by Kubernetes version"
+  type        = map(map(string))
+  default = {
+    "1.24" = {
+      "coredns"            = null
+      "kube-proxy"         = null
+      "vpc-cni"            = null
+      "aws-ebs-csi-driver" = "v1.18.0-eksbuild.1"
+    }
+    "1.25" = {
+      "coredns"            = "v1.9.3-eksbuild.2"
+      "kube-proxy"         = "v1.25.6-eksbuild.1"
+      "vpc-cni"            = "v1.12.2-eksbuild.1"
+      "aws-ebs-csi-driver" = "v1.18.0-eksbuild.1"
+    }
+  }
+}
+

examples/full-cluster-tf-upgrade/1.25/tf-run.data

@@ -49,10 +49,10 @@ STOP Once applied in this subdirectory, come back here and continue with step %%
 
 TAG setup-efs
 COMMENT cd efs and tf-run.sh apply
-STOP Once applied in this subdirectory, come back here and continue with step %%NEXT%% (tag:setup-ebs)
+STOP Once applied in this subdirectory, come back here and continue with step %%NEXT%% (tag:setup-addons)
 
-TAG setup-ebs
-COMMENT cd ebs and tf-run.sh apply
+TAG setup-addons
+COMMENT cd addons and tf-run.sh apply
 STOP Once applied in this subdirectory, come back here and continue with step %%NEXT%% (tag:setup-irsa)
 
 TAG setup-irsa