From fcb4aaf3c1eaf50d91e9b110caa8a9f8c9fe734d Mon Sep 17 00:00:00 2001 From: badra001 Date: Tue, 11 Apr 2023 09:52:13 -0400 Subject: [PATCH] update to 1.23 from ditd-gups-prod --- .../1.23.in-progress/.gitignore | 1 + .../1.23.in-progress/.tf-control | 20 +++ .../1.23.in-progress/.tf-control.tfrc | 24 ++++ .../1.23.in-progress/aws-auth/.tf-control | 20 +++ .../aws-auth/.tf-control.tfrc | 24 ++++ .../1.23.in-progress/aws-auth/tf-run.data | 10 +- .../cluster-roles/.tf-control | 20 +++ .../cluster-roles/.tf-control.tfrc | 24 ++++ .../cluster-roles/tf-run.data | 8 +- .../common-services/.tf-control | 20 +++ .../common-services/.tf-control.tfrc | 24 ++++ .../charts/cluster-autoscaler/Chart.yaml | 4 +- .../charts/cluster-autoscaler/README.md | 100 +++++--------- .../cluster-autoscaler/README.md.gotmpl | 80 +++--------- .../cluster-autoscaler/templates/_helpers.tpl | 30 +++++ .../templates/clusterrole.yaml | 17 ++- .../templates/clusterrolebinding.yaml | 2 +- .../templates/deployment.yaml | 55 ++++++-- .../cluster-autoscaler/templates/pdb.yaml | 1 + .../priority-expander-configmap.yaml | 7 +- .../cluster-autoscaler/templates/role.yaml | 38 +++++- .../templates/rolebinding.yaml | 1 + .../cluster-autoscaler/templates/secret.yaml | 1 + .../cluster-autoscaler/templates/service.yaml | 1 + .../templates/servicemonitor.yaml | 6 +- .../charts/cluster-autoscaler/values.yaml | 65 ++++++++-- .../common-services/copy_images.tf | 2 +- .../1.23.in-progress/common-services/main.tf | 11 +- .../common-services/tf-run.data | 13 +- .../variables.common-services.auto.tfvars | 4 +- .../1.23.in-progress/ebs-encryption.tf | 21 ++- .../1.23.in-progress/ebs/.tf-control | 20 +++ .../1.23.in-progress/ebs/.tf-control.tfrc | 24 ++++ .../1.23.in-progress/ebs/README.ebs.md | 75 +++++++++++ .../1.23.in-progress/ebs/README.md | 122 ++++++++++++++++++ .../1.23.in-progress/ebs/copy_image.sh | 1 + .../1.23.in-progress/ebs/copy_images.tf | 77 +++++++++++ .../ebs/data.eks-subdirectory.tf | 1 + .../1.23.in-progress/ebs/ecr.tf | 53 ++++++++ .../ebs/kubeconfig.eks-subdirectory.tf | 1 + .../1.23.in-progress/ebs/locals.tf | 17 +++ .../1.23.in-progress/ebs/main.tf | 105 +++++++++++++++ .../1.23.in-progress/ebs/parent_rs.tf | 1 + .../1.23.in-progress/ebs/prefixes.tf | 1 + .../1.23.in-progress/ebs/providers.tf | 1 + .../1.23.in-progress/ebs/region.tf | 4 + .../1.23.in-progress/ebs/role.tf | 51 ++++++++ .../1.23.in-progress/ebs/tf-run.data | 21 +++ .../1.23.in-progress/ebs/variables.ebs.tf | 61 +++++++++ .../1.23.in-progress/ebs/variables.eks.tf | 1 + .../1.23.in-progress/ebs/version.tf | 1 + .../1.23.in-progress/efs/.tf-control | 20 +++ .../1.23.in-progress/efs/.tf-control.tfrc | 24 ++++ .../1.23.in-progress/efs/tf-run.data | 10 +- .../1.23.in-progress/irsa-roles/.tf-control | 20 +++ .../irsa-roles/.tf-control.tfrc | 24 ++++ .../irsa-roles/cluster-autoscaler/.tf-control | 20 +++ .../cluster-autoscaler/.tf-control.tfrc | 24 ++++ .../1.23.in-progress/irsa-roles/tf-run.data | 10 +- .../1.23.in-progress/tf-run.data | 24 ++-- 60 files changed, 1256 insertions(+), 212 deletions(-) create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/.tf-control create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/.tf-control.tfrc create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/.tf-control create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/.tf-control.tfrc create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/.tf-control create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/.tf-control.tfrc create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/.tf-control create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/.tf-control.tfrc create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/.tf-control create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/.tf-control.tfrc create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/README.ebs.md create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/README.md create mode 120000 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/copy_image.sh create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/copy_images.tf create mode 120000 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/data.eks-subdirectory.tf create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/ecr.tf create mode 120000 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/kubeconfig.eks-subdirectory.tf create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/locals.tf create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/main.tf create mode 120000 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/parent_rs.tf create mode 120000 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/prefixes.tf create mode 120000 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/providers.tf create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/region.tf create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/role.tf create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/tf-run.data create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/variables.ebs.tf create mode 120000 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/variables.eks.tf create mode 120000 examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/version.tf create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/efs/.tf-control create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/efs/.tf-control.tfrc create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/.tf-control create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/.tf-control.tfrc create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/cluster-autoscaler/.tf-control create mode 100644 examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/cluster-autoscaler/.tf-control.tfrc diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/.gitignore b/examples/full-cluster-tf-upgrade/1.23.in-progress/.gitignore index f416fe8..010b80b 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/.gitignore +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/.gitignore @@ -2,3 +2,4 @@ kube.config ecr-login.txt setup/ec2-ssh-eks-* !setup/ec2-ssh-eks-*.pub +logs diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/.tf-control b/examples/full-cluster-tf-upgrade/1.23.in-progress/.tf-control new file mode 100644 index 0000000..280f449 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/.tf-control @@ -0,0 +1,20 @@ +# .tf-control +# allows for setting a specific command to be used for tf-* commands under this git repo +# see tf-control.sh help for more info + +TFCONTROL_VERSION="1.0.5" + +TFCOMMAND="terraform_latest" +# TF_CLI_CONFIG_FILE=PATH-TO-FILE/.tf-control.tfrc +# TFARGS="" +# TFNOLOG="" +# TFNOCOLOR="" + +# use the following to force a specific version. An upgrade of an existing 0.12.31 to 1.x +# needs you to cycle through 0.13.17, 0.14.11, and then latest (0.15.5 not needed). Other +# steps in between. See https://github.e.it.census.gov/terraform/support/tree/master/docs/how-to/terraform-upgrade for details +# +#TFCOMMAND="terraform_0.12.31" +#TFCOMMAND="terraform_0.13.7" +#TFCOMMAND="terraform_0.14.11" +#TFCOMMAND="terraform_0.15.5" diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/.tf-control.tfrc b/examples/full-cluster-tf-upgrade/1.23.in-progress/.tf-control.tfrc new file mode 100644 index 0000000..7425488 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/.tf-control.tfrc @@ -0,0 +1,24 @@ +TFCONTROL_VERSION="1.0.5" + +# https://www.terraform.io/docs/cli/config/config-file.html +plugin_cache_dir = "/data/terraform/terraform.d/plugin-cache" +#disable_checkpoint = true + +provider_installation { +# filesystem_mirror { +# path = "/apps/terraform/terraform.d/providers" +# include = [ "*/*/*" ] +# } + filesystem_mirror { + path = "/data/terraform/terraform.d/providers" + include = [ "*/*/*" ] + } +# filesystem_mirror { +# path = "/apps/terraform/terraform.d/providers" +# include = [ "external.terraform.census.gov/*/*" ] +# } + direct { + include = [ "*/*/*" ] + } +} + diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/.tf-control b/examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/.tf-control new file mode 100644 index 0000000..280f449 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/.tf-control @@ -0,0 +1,20 @@ +# .tf-control +# allows for setting a specific command to be used for tf-* commands under this git repo +# see tf-control.sh help for more info + +TFCONTROL_VERSION="1.0.5" + +TFCOMMAND="terraform_latest" +# TF_CLI_CONFIG_FILE=PATH-TO-FILE/.tf-control.tfrc +# TFARGS="" +# TFNOLOG="" +# TFNOCOLOR="" + +# use the following to force a specific version. An upgrade of an existing 0.12.31 to 1.x +# needs you to cycle through 0.13.17, 0.14.11, and then latest (0.15.5 not needed). Other +# steps in between. See https://github.e.it.census.gov/terraform/support/tree/master/docs/how-to/terraform-upgrade for details +# +#TFCOMMAND="terraform_0.12.31" +#TFCOMMAND="terraform_0.13.7" +#TFCOMMAND="terraform_0.14.11" +#TFCOMMAND="terraform_0.15.5" diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/.tf-control.tfrc b/examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/.tf-control.tfrc new file mode 100644 index 0000000..7425488 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/.tf-control.tfrc @@ -0,0 +1,24 @@ +TFCONTROL_VERSION="1.0.5" + +# https://www.terraform.io/docs/cli/config/config-file.html +plugin_cache_dir = "/data/terraform/terraform.d/plugin-cache" +#disable_checkpoint = true + +provider_installation { +# filesystem_mirror { +# path = "/apps/terraform/terraform.d/providers" +# include = [ "*/*/*" ] +# } + filesystem_mirror { + path = "/data/terraform/terraform.d/providers" + include = [ "*/*/*" ] + } +# filesystem_mirror { +# path = "/apps/terraform/terraform.d/providers" +# include = [ "external.terraform.census.gov/*/*" ] +# } + direct { + include = [ "*/*/*" ] + } +} + diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/tf-run.data b/examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/tf-run.data index 50f6ccb..8afedd9 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/tf-run.data +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/aws-auth/tf-run.data @@ -1,12 +1,14 @@ -VERSION 1.2.3 +VERSION 1.3.0 REMOTE-STATE COMMAND tf-directory-setup.py -l none -f COMMAND setup-new-directory.sh COMMAND tf-init -upgrade -COMMAND ln -sf ../versions.tf -COMMAND ln -sf ../settings.auto.tfvars -COMMAND ln -sf ../variables.application_tags.auto.tfvars + LINKTOP init +LINK versions.tf +LINK settings.auto.tfvars +LINK variables.application_tags.auto.tfvars + ALL COMMAND tf-directory-setup.py -l s3 STOP cd ../efs and tf-run.sh apply diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/.tf-control b/examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/.tf-control new file mode 100644 index 0000000..280f449 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/.tf-control @@ -0,0 +1,20 @@ +# .tf-control +# allows for setting a specific command to be used for tf-* commands under this git repo +# see tf-control.sh help for more info + +TFCONTROL_VERSION="1.0.5" + +TFCOMMAND="terraform_latest" +# TF_CLI_CONFIG_FILE=PATH-TO-FILE/.tf-control.tfrc +# TFARGS="" +# TFNOLOG="" +# TFNOCOLOR="" + +# use the following to force a specific version. An upgrade of an existing 0.12.31 to 1.x +# needs you to cycle through 0.13.17, 0.14.11, and then latest (0.15.5 not needed). Other +# steps in between. See https://github.e.it.census.gov/terraform/support/tree/master/docs/how-to/terraform-upgrade for details +# +#TFCOMMAND="terraform_0.12.31" +#TFCOMMAND="terraform_0.13.7" +#TFCOMMAND="terraform_0.14.11" +#TFCOMMAND="terraform_0.15.5" diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/.tf-control.tfrc b/examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/.tf-control.tfrc new file mode 100644 index 0000000..7425488 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/.tf-control.tfrc @@ -0,0 +1,24 @@ +TFCONTROL_VERSION="1.0.5" + +# https://www.terraform.io/docs/cli/config/config-file.html +plugin_cache_dir = "/data/terraform/terraform.d/plugin-cache" +#disable_checkpoint = true + +provider_installation { +# filesystem_mirror { +# path = "/apps/terraform/terraform.d/providers" +# include = [ "*/*/*" ] +# } + filesystem_mirror { + path = "/data/terraform/terraform.d/providers" + include = [ "*/*/*" ] + } +# filesystem_mirror { +# path = "/apps/terraform/terraform.d/providers" +# include = [ "external.terraform.census.gov/*/*" ] +# } + direct { + include = [ "*/*/*" ] + } +} + diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/tf-run.data b/examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/tf-run.data index 9179181..1d1a079 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/tf-run.data +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/cluster-roles/tf-run.data @@ -1,16 +1,16 @@ -VERSION 1.2.4 +VERSION 1.3.0 REMOTE-STATE STOP only run this after the cluster roles represented here have been setup in K8S COMMAND tf-directory-setup.py -l none -f COMMAND setup-new-directory.sh COMMAND tf-init -upgrade -COMMAND ln -sf ../versions.tf -COMMAND ln -sf ../settings.auto.tfvars . -COMMAND ln -sf ../variables.application_tags.auto.tfvars LINKTOP init LINKTOP provider_configs.d/provider.ldap_new.auto.tfvars LINKTOP provider_configs.d/provider.ldap_new.tf LINKTOP provider_configs.d/provider.ldap_new.variables.tf +LINK versions.tf +LINK settings.auto.tfvars +LINK variables.application_tags.auto.tfvars POLICY ALL COMMAND tf-directory-setup.py -l s3 diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/.tf-control b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/.tf-control new file mode 100644 index 0000000..280f449 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/.tf-control @@ -0,0 +1,20 @@ +# .tf-control +# allows for setting a specific command to be used for tf-* commands under this git repo +# see tf-control.sh help for more info + +TFCONTROL_VERSION="1.0.5" + +TFCOMMAND="terraform_latest" +# TF_CLI_CONFIG_FILE=PATH-TO-FILE/.tf-control.tfrc +# TFARGS="" +# TFNOLOG="" +# TFNOCOLOR="" + +# use the following to force a specific version. An upgrade of an existing 0.12.31 to 1.x +# needs you to cycle through 0.13.17, 0.14.11, and then latest (0.15.5 not needed). Other +# steps in between. See https://github.e.it.census.gov/terraform/support/tree/master/docs/how-to/terraform-upgrade for details +# +#TFCOMMAND="terraform_0.12.31" +#TFCOMMAND="terraform_0.13.7" +#TFCOMMAND="terraform_0.14.11" +#TFCOMMAND="terraform_0.15.5" diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/.tf-control.tfrc b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/.tf-control.tfrc new file mode 100644 index 0000000..7425488 --- /dev/null +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/.tf-control.tfrc @@ -0,0 +1,24 @@ +TFCONTROL_VERSION="1.0.5" + +# https://www.terraform.io/docs/cli/config/config-file.html +plugin_cache_dir = "/data/terraform/terraform.d/plugin-cache" +#disable_checkpoint = true + +provider_installation { +# filesystem_mirror { +# path = "/apps/terraform/terraform.d/providers" +# include = [ "*/*/*" ] +# } + filesystem_mirror { + path = "/data/terraform/terraform.d/providers" + include = [ "*/*/*" ] + } +# filesystem_mirror { +# path = "/apps/terraform/terraform.d/providers" +# include = [ "external.terraform.census.gov/*/*" ] +# } + direct { + include = [ "*/*/*" ] + } +} + diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/Chart.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/Chart.yaml index 0b94b05..f682ee2 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/Chart.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.21.0 +appVersion: 1.24.0 description: Scales Kubernetes worker nodes within autoscaling groups. engine: gotpl home: https://github.com/kubernetes/autoscaler @@ -17,4 +17,4 @@ name: cluster-autoscaler sources: - https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler type: application -version: 9.10.7 +version: 9.18.1 diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/README.md b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/README.md index 43bf4bf..74d7590 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/README.md +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/README.md @@ -209,6 +209,18 @@ Install the chart with ``` $ helm install my-release autoscaler/cluster-autoscaler -f myvalues.yaml ``` +### Cluster-API + +`cloudProvider: clusterapi` must be set, and then one or more of +- `autoDiscovery.clusterName` +- or `autoDiscovery.labels` +See [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery) for more details + +Additional config parameters avaible, see the `values.yaml` for more details +`clusterAPIMode` +`clusterAPIKubeconfigSecret` +`clusterAPIWorkloadKubeconfigPath` +`clusterAPICloudConfigPath` ## Uninstalling the Chart @@ -226,72 +238,9 @@ The command removes all the Kubernetes components associated with the chart and ### AWS - IAM -The worker running the cluster autoscaler will need access to certain resources and actions: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup" - ], - "Resource": "*" - } - ] -} -``` - -- `DescribeTags` is required for autodiscovery. -- `DescribeLaunchConfigurations` is required to scale up an ASG from 0. - -If you would like to limit the scope of the Cluster Autoscaler to ***only*** modify ASGs for a particular cluster, use the following policy instead: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "ec2:DescribeLaunchTemplateVersions" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup" - ], - "Resource": [ - "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-1", - "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-2", - "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-3" - ], - "Condition": { - "StringEquals": { - "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled": "true", - "autoscaling:ResourceTag/kubernetes.io/cluster/": "owned" - } - } - } - ] -} -``` +The worker running the cluster autoscaler will need access to certain resources and actions depending on the version you run and your configuration of it. -Make sure to replace the variables ``, ``, ``, and the ARNs of the ASGs where applicable. +For the up-to-date IAM permissions required, please see the [cluster autoscaler's AWS Cloudprovider Readme](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#iam-policy) and switch to the tag of the cluster autoscaler image you are using. ### AWS - IAM Roles for Service Accounts (IRSA) @@ -338,7 +287,8 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ |-----|------|---------|-------------| | additionalLabels | object | `{}` | Labels to add to each object of the chart. | | affinity | object | `{}` | Affinity for pod assignment | -| autoDiscovery.clusterName | string | `nil` | Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. | +| autoDiscovery.clusterName | string | `nil` | Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. Enable autodiscovery for `cloudProvider=clusterapi`, for groups matching `autoDiscovery.labels`. Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. | +| autoDiscovery.labels | list | `[]` | Cluster-API labels to match https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery | | autoDiscovery.roles | list | `["worker"]` | Magnum node group roles to match. | | autoDiscovery.tags | list | `["k8s.io/cluster-autoscaler/enabled","k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }}"]` | ASG tags to match, run through `tpl`. | | autoscalingGroups | list | `[]` | For AWS, Azure AKS or Magnum. At least one element is required if not using `autoDiscovery`. For example: 
 - name: asg1
   maxSize: 2
   minSize: 1
| @@ -353,15 +303,21 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ | azureResourceGroup | string | `""` | Azure resource group that the cluster is located. Required if `cloudProvider=azure` | | azureSubscriptionID | string | `""` | Azure subscription where the resources are located. Required if `cloudProvider=azure` | | azureTenantID | string | `""` | Azure tenant where the resources are located. Required if `cloudProvider=azure` | -| azureUseManagedIdentityExtension | bool | `false` | Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID and resource group are set. | +| azureUseManagedIdentityExtension | bool | `false` | Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. | | azureVMType | string | `"AKS"` | Azure VM type. | | cloudConfigPath | string | `"/etc/gce.conf"` | Configuration file for cloud provider. | -| cloudProvider | string | `"aws"` | The cloud provider where the autoscaler runs. Currently only `gce`, `aws`, `azure` and `magnum` are supported. `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. `magnum` for OpenStack Magnum. | +| cloudProvider | string | `"aws"` | The cloud provider where the autoscaler runs. Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported. `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. `magnum` for OpenStack Magnum, `clusterapi` for Cluster API. | +| clusterAPICloudConfigPath | string | `"/etc/kubernetes/mgmt-kubeconfig"` | Path to kubeconfig for connecting to Cluster API Management Cluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or incluster-kubeconfig` | +| clusterAPIConfigMapsNamespace | string | `""` | Namespace on the workload cluster to store Leader election and status configmaps | +| clusterAPIKubeconfigSecret | string | `""` | Secret containing kubeconfig for connecting to Cluster API managed workloadcluster Required if `cloudProvider=clusterapi` and `clusterAPIMode=kubeconfig-kubeconfig,kubeconfig-incluster or incluster-kubeconfig` | +| clusterAPIMode | string | `"incluster-incluster"` | Cluster API mode, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#connecting-cluster-autoscaler-to-cluster-api-management-and-workload-clusters Syntax: workloadClusterMode-ManagementClusterMode for `kubeconfig-kubeconfig`, `incluster-kubeconfig` and `single-kubeconfig` you always must mount the external kubeconfig using either `extraVolumeSecrets` or `extraMounts` and `extraVolumes` if you dont set `clusterAPIKubeconfigSecret`and thus use an in-cluster config or want to use a non capi generated kubeconfig you must do so for the workload kubeconfig as well | +| clusterAPIWorkloadKubeconfigPath | string | `"/etc/kubernetes/value"` | Path to kubeconfig for connecting to Cluster API managed workloadcluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or kubeconfig-incluster` | | containerSecurityContext | object | `{}` | [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | +| deployment.annotations | object | `{}` | Annotations to add to the Deployment object. | | dnsPolicy | string | `"ClusterFirst"` | Defaults to `ClusterFirst`. Valid values are: `ClusterFirstWithHostNet`, `ClusterFirst`, `Default` or `None`. If autoscaler does not depend on cluster DNS, recommended to set this to `Default`. | | envFromConfigMap | string | `""` | ConfigMap name to use as envFrom. | | envFromSecret | string | `""` | Secret name to use as envFrom. | -| expanderPriorities | object | `{}` | The expanderPriorities is used if `extraArgs.expander` is set to `priority` and expanderPriorities is also set with the priorities. If `extraArgs.expander` is set to `priority`, then expanderPriorities is used to define cluster-autoscaler-priority-expander priorities. See: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/expander/priority/readme.md | +| expanderPriorities | object | `{}` | The expanderPriorities is used if `extraArgs.expander` contains `priority` and expanderPriorities is also set with the priorities. If `extraArgs.expander` contains `priority`, then expanderPriorities is used to define cluster-autoscaler-priority-expander priorities. See: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/expander/priority/readme.md | | extraArgs | object | `{"logtostderr":true,"stderrthreshold":"info","v":4}` | Additional container arguments. Refer to https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-the-parameters-to-ca for the full list of cluster autoscaler parameters and their default values. Everything after the first _ will be ignored allowing the use of multi-string arguments. | | extraEnv | object | `{}` | Additional container environment variables. | | extraEnvConfigMaps | object | `{}` | Additional container environment variables from ConfigMaps. | @@ -373,7 +329,7 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.pullSecrets | list | `[]` | Image pull secrets | | image.repository | string | `"k8s.gcr.io/autoscaling/cluster-autoscaler"` | Image repository | -| image.tag | string | `"v1.21.0"` | Image tag | +| image.tag | string | `"v1.24.0"` | Image tag | | kubeTargetVersionOverride | string | `""` | Allow overriding the `.Capabilities.KubeVersion.GitVersion` check. Useful for `helm template` commands. | | magnumCABundlePath | string | `"/etc/kubernetes/ca-bundle.crt"` | Path to the host's CA bundle, from `ca-file` in the cloud-config file. | | magnumClusterName | string | `""` | Cluster name or ID in Magnum. Required if `cloudProvider=magnum` and not setting `autoDiscovery.clusterName`. | @@ -382,13 +338,14 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ | podAnnotations | object | `{}` | Annotations to add to each pod. | | podDisruptionBudget | object | `{"maxUnavailable":1}` | Pod disruption budget. | | podLabels | object | `{}` | Labels to add to each pod. | -| priorityClassName | string | `""` | priorityClassName | +| priorityClassName | string | `"system-cluster-critical"` | priorityClassName | | priorityConfigMapAnnotations | object | `{}` | Annotations to add to `cluster-autoscaler-priority-expander` ConfigMap. | | prometheusRule.additionalLabels | object | `{}` | Additional labels to be set in metadata. | | prometheusRule.enabled | bool | `false` | If true, creates a Prometheus Operator PrometheusRule. | | prometheusRule.interval | string | `nil` | How often rules in the group are evaluated (falls back to `global.evaluation_interval` if not set). | | prometheusRule.namespace | string | `"monitoring"` | Namespace which Prometheus is running in. | | prometheusRule.rules | list | `[]` | Rules spec template (see https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule). | +| rbac.clusterScoped | bool | `true` | if set to false will only provision RBAC to alter resources in the current namespace. Most useful for Cluster-API | | rbac.create | bool | `true` | If `true`, create and use RBAC resources. | | rbac.pspEnabled | bool | `false` | If `true`, creates and uses RBAC resources required in the cluster with [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) enabled. Must be used with `rbac.create` set to `true`. | | rbac.serviceAccount.annotations | object | `{}` | Additional Service Account annotations. | @@ -412,4 +369,5 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ | serviceMonitor.path | string | `"/metrics"` | The path to scrape for metrics; autoscaler exposes `/metrics` (this is standard) | | serviceMonitor.selector | object | `{"release":"prometheus-operator"}` | Default to kube-prometheus install (CoreOS recommended), but should be set according to Prometheus install. | | tolerations | list | `[]` | List of node taints to tolerate (requires Kubernetes >= 1.6). | +| topologySpreadConstraints | list | `[]` | You can use topology spread constraints to control how Pods are spread across your cluster among failure-domains such as regions, zones, nodes, and other user-defined topology domains. (requires Kubernetes >= 1.19). | | updateStrategy | object | `{}` | [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) | diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/README.md.gotmpl b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/README.md.gotmpl index dda305c..306fd57 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/README.md.gotmpl +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/README.md.gotmpl @@ -209,6 +209,19 @@ Install the chart with ``` $ helm install my-release autoscaler/cluster-autoscaler -f myvalues.yaml ``` +### Cluster-API + +`cloudProvider: clusterapi` must be set, and then one or more of +- `autoDiscovery.clusterName` +- or `autoDiscovery.labels` +See [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery) for more details + + +Additional config parameters avaible, see the `values.yaml` for more details +`clusterAPIMode` +`clusterAPIKubeconfigSecret` +`clusterAPIWorkloadKubeconfigPath` +`clusterAPICloudConfigPath` ## Uninstalling the Chart @@ -226,72 +239,9 @@ The command removes all the Kubernetes components associated with the chart and ### AWS - IAM -The worker running the cluster autoscaler will need access to certain resources and actions: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup" - ], - "Resource": "*" - } - ] -} -``` - -- `DescribeTags` is required for autodiscovery. -- `DescribeLaunchConfigurations` is required to scale up an ASG from 0. - -If you would like to limit the scope of the Cluster Autoscaler to ***only*** modify ASGs for a particular cluster, use the following policy instead: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "ec2:DescribeLaunchTemplateVersions" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup" - ], - "Resource": [ - "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-1", - "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-2", - "arn:aws:autoscaling:::autoScalingGroup::autoScalingGroupName/node-group-3" - ], - "Condition": { - "StringEquals": { - "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled": "true", - "autoscaling:ResourceTag/kubernetes.io/cluster/": "owned" - } - } - } - ] -} -``` +The worker running the cluster autoscaler will need access to certain resources and actions depending on the version you run and your configuration of it. -Make sure to replace the variables ``, ``, ``, and the ARNs of the ASGs where applicable. +For the up-to-date IAM permissions required, please see the [cluster autoscaler's AWS Cloudprovider Readme](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#iam-policy) and switch to the tag of the cluster autoscaler image you are using. ### AWS - IAM Roles for Service Accounts (IRSA) diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/_helpers.tpl b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/_helpers.tpl index 0723059..726086e 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/_helpers.tpl +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/_helpers.tpl @@ -85,3 +85,33 @@ Return the service account name used by the pod. {{ default "default" .Values.rbac.serviceAccount.name }} {{- end -}} {{- end -}} + +{{/* +Return true if the priority expander is enabled +*/}} +{{- define "cluster-autoscaler.priorityExpanderEnabled" -}} +{{- $expanders := splitList "," (default "" .Values.extraArgs.expander) -}} +{{- if has "priority" $expanders -}} +{{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Return the autodiscoveryparameters for clusterapi. +*/}} +{{- define "cluster-autoscaler.capiAutodiscoveryConfig" -}} +{{- if .Values.autoDiscovery.clusterName -}} +{{- print "clusterName=" -}}{{ .Values.autoDiscovery.clusterName }} +{{- end -}} +{{- if and .Values.autoDiscovery.clusterName .Values.autoDiscovery.labels -}} +{{- print "," -}} +{{- end -}} +{{- if .Values.autoDiscovery.labels -}} +{{- range $i, $el := .Values.autoDiscovery.labels -}} +{{- if $i -}}{{- print "," -}}{{- end -}} +{{- range $key, $val := $el -}} +{{- $key -}}{{- print "=" -}}{{- $val -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/clusterrole.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/clusterrole.yaml index 409fbe2..e3d3655 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/clusterrole.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/clusterrole.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.create -}} +{{- if and .Values.rbac.create .Values.rbac.clusterScoped -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -146,5 +146,18 @@ rules: verbs: - use {{- end -}} - +{{- if and ( and ( eq .Values.cloudProvider "clusterapi" ) ( .Values.rbac.clusterScoped ) ( or ( eq .Values.clusterAPIMode "incluster-incluster" ) ( eq .Values.clusterAPIMode "kubeconfig-incluster" ) ))}} + - apiGroups: + - cluster.x-k8s.io + resources: + - machinedeployments + - machinedeployments/scale + - machines + - machinesets + verbs: + - get + - list + - update + - watch +{{- end }} {{- end -}} diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/clusterrolebinding.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/clusterrolebinding.yaml index d1e8308..d2384dc 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/clusterrolebinding.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/clusterrolebinding.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.create -}} +{{- if and .Values.rbac.create .Values.rbac.clusterScoped -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/deployment.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/deployment.yaml index 46246f2..e8edc7f 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/deployment.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/deployment.yaml @@ -1,11 +1,14 @@ -{{- if or .Values.autoDiscovery.clusterName .Values.autoscalingGroups }} +{{- if or ( or .Values.autoDiscovery.clusterName .Values.autoDiscovery.labels ) .Values.autoscalingGroups }} {{/* one of the above is required */}} apiVersion: {{ template "deployment.apiVersion" . }} kind: Deployment metadata: + annotations: +{{ toYaml .Values.deployment.annotations | indent 4 }} labels: {{ include "cluster-autoscaler.labels" . | indent 4 }} name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} spec: replicas: {{ .Values.replicaCount }} selector: @@ -46,7 +49,11 @@ spec: command: - ./cluster-autoscaler - --cloud-provider={{ .Values.cloudProvider }} + {{- if and (eq .Values.cloudProvider "clusterapi") (eq .Values.clusterAPIMode "kubeconfig-incluster") }} + - --namespace={{ .Values.clusterAPIConfigMapsNamespace | default "kube-system" }} + {{- else }} - --namespace={{ .Release.Namespace }} + {{- end }} {{- if .Values.autoscalingGroups }} {{- range .Values.autoscalingGroups }} - --nodes={{ .minSize }}:{{ .maxSize }}:{{ .name }} @@ -69,6 +76,21 @@ spec: {{- else }} - --cluster-name={{ .Values.magnumClusterName }} {{- end }} + {{- else if eq .Values.cloudProvider "clusterapi" }} + {{- if or .Values.autoDiscovery.clusterName .Values.autoDiscovery.labels }} + - --node-group-auto-discovery=clusterapi:{{ template "cluster-autoscaler.capiAutodiscoveryConfig" . }} + {{- end }} + {{- if eq .Values.clusterAPIMode "incluster-kubeconfig"}} + - --cloud-config={{ .Values.clusterAPICloudConfigPath }} + {{- else if eq .Values.clusterAPIMode "kubeconfig-incluster"}} + - --kubeconfig={{ .Values.clusterAPIWorkloadKubeconfigPath }} + - --clusterapi-cloud-config-authoritative + {{- else if eq .Values.clusterAPIMode "kubeconfig-kubeconfig"}} + - --kubeconfig={{ .Values.clusterAPIWorkloadKubeconfigPath }} + - --cloud-config={{ .Values.clusterAPICloudConfigPath }} + {{- else if eq .Values.clusterAPIMode "single-kubeconfig"}} + - --kubeconfig={{ .Values.clusterAPIWorkloadKubeconfigPath }} + {{- end }} {{- end }} {{- if eq .Values.cloudProvider "magnum" }} - --cloud-config={{ .Values.cloudConfigPath }} @@ -114,6 +136,11 @@ spec: secretKeyRef: key: VMType name: {{ template "cluster-autoscaler.fullname" . }} + - name: AZURE_CLUSTER_NAME + valueFrom: + secretKeyRef: + key: ClusterName + name: {{ template "cluster-autoscaler.fullname" . }} {{- if .Values.azureUseManagedIdentityExtension }} - name: ARM_USE_MANAGED_IDENTITY_EXTENSION value: "true" @@ -133,11 +160,6 @@ spec: secretKeyRef: key: ClientSecret name: {{ template "cluster-autoscaler.fullname" . }} - - name: AZURE_CLUSTER_NAME - valueFrom: - secretKeyRef: - key: ClusterName - name: {{ template "cluster-autoscaler.fullname" . }} - name: AZURE_NODE_RESOURCE_GROUP valueFrom: secretKeyRef: @@ -186,7 +208,7 @@ spec: securityContext: {{ toYaml .Values.containerSecurityContext | nindent 12 | trim }} {{- end }} - {{- if or (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets .Values.extraVolumeMounts }} + {{- if or (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets .Values.extraVolumeMounts .Values.clusterAPIKubeconfigSecret }} volumeMounts: {{- if eq .Values.cloudProvider "magnum" }} - name: cloudconfig @@ -203,8 +225,12 @@ spec: mountPath: {{ required "Must specify mountPath!" $value.mountPath }} readOnly: true {{- end }} + {{- if .Values.clusterAPIKubeconfigSecret }} + - name: cluster-api-kubeconfig + mountPath: {{ .Values.clusterAPIWorkloadKubeconfigPath | trimSuffix "/value" }} + {{- end }} {{- if .Values.extraVolumeMounts }} - {{ toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} {{- end }} {{- end }} {{- if .Values.affinity }} @@ -218,11 +244,15 @@ spec: serviceAccountName: {{ template "cluster-autoscaler.serviceAccountName" . }} tolerations: {{ toYaml .Values.tolerations | indent 8 }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.topologySpreadConstraints | indent 8 }} + {{- end }} {{- if .Values.securityContext }} securityContext: {{ toYaml .Values.securityContext | nindent 8 | trim }} {{- end }} - {{- if or (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets .Values.extraVolumes }} + {{- if or (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets .Values.extraVolumes .Values.clusterAPIKubeconfigSecret }} volumes: {{- if eq .Values.cloudProvider "magnum" }} - name: cloudconfig @@ -244,7 +274,12 @@ spec: {{- end }} {{- end }} {{- if .Values.extraVolumes }} - {{- toYaml .Values.extraVolumes | nindent 10 }} + {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} + {{- if .Values.clusterAPIKubeconfigSecret }} + - name: cluster-api-kubeconfig + secret: + secretName: {{ .Values.clusterAPIKubeconfigSecret }} {{- end }} {{- end }} {{- if .Values.image.pullSecrets }} diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/pdb.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/pdb.yaml index da5bd56..19a7d01 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/pdb.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/pdb.yaml @@ -5,6 +5,7 @@ metadata: labels: {{ include "cluster-autoscaler.labels" . | indent 4 }} name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} spec: selector: matchLabels: diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml index 5bb2024..1e5b895 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/priority-expander-configmap.yaml @@ -1,9 +1,10 @@ {{- if hasKey .Values.extraArgs "expander" }} -{{- if and (.Values.expanderPriorities) (eq .Values.extraArgs.expander "priority") -}} +{{- if and (.Values.expanderPriorities) (include "cluster-autoscaler.priorityExpanderEnabled" .) -}} apiVersion: v1 kind: ConfigMap metadata: name: cluster-autoscaler-priority-expander + namespace: {{ .Release.Namespace }} labels: {{ include "cluster-autoscaler.labels" . | indent 4 }} {{- if .Values.priorityConfigMapAnnotations }} @@ -12,6 +13,10 @@ metadata: {{- end }} data: priorities: |- +{{- if kindIs "string" .Values.expanderPriorities }} {{ .Values.expanderPriorities | indent 4 }} +{{- else }} +{{ toYaml .Values.expanderPriorities | indent 4 }} +{{- end -}} {{- end -}} {{- end -}} diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/role.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/role.yaml index c1f226e..b22fb58 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/role.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/role.yaml @@ -5,6 +5,7 @@ metadata: labels: {{ include "cluster-autoscaler.labels" . | indent 4 }} name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} rules: - apiGroups: - "" @@ -12,7 +13,7 @@ rules: - configmaps verbs: - create -{{- if eq (default "" .Values.extraArgs.expander) "priority" }} +{{- if (include "cluster-autoscaler.priorityExpanderEnabled" .) }} - list - watch {{- end }} @@ -22,14 +23,14 @@ rules: - configmaps resourceNames: - cluster-autoscaler-status -{{- if eq (default "" .Values.extraArgs.expander) "priority" }} +{{- if (include "cluster-autoscaler.priorityExpanderEnabled" .) }} - cluster-autoscaler-priority-expander {{- end }} verbs: - delete - get - update -{{- if eq (default "" .Values.extraArgs.expander) "priority" }} +{{- if (include "cluster-autoscaler.priorityExpanderEnabled" .) }} - watch {{- end }} {{- if eq (default "" (index .Values.extraArgs "leader-elect-resource-lock")) "configmaps" }} @@ -43,4 +44,35 @@ rules: - get - update {{- end }} +{{- if and ( and ( eq .Values.cloudProvider "clusterapi" ) ( not .Values.rbac.clusterScoped ) ( or ( eq .Values.clusterAPIMode "incluster-incluster" ) ( eq .Values.clusterAPIMode "kubeconfig-incluster" ) ))}} + - apiGroups: + - cluster.x-k8s.io + resources: + - machinedeployments + - machinedeployments/scale + - machines + - machinesets + verbs: + - get + - list + - update + - watch +{{- end }} +{{- if ( not .Values.rbac.clusterScoped ) }} + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - apiGroups: + - coordination.k8s.io + resourceNames: + - cluster-autoscaler + resources: + - leases + verbs: + - get + - update +{{- end }} {{- end -}} diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/rolebinding.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/rolebinding.yaml index 938bc03..ba5f037 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/rolebinding.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/rolebinding.yaml @@ -5,6 +5,7 @@ metadata: labels: {{ include "cluster-autoscaler.labels" . | indent 4 }} name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/secret.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/secret.yaml index 3f0ef09..9c58d0f 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/secret.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/secret.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} data: {{- if eq .Values.cloudProvider "azure" }} ClientID: "{{ .Values.azureClientID | b64enc }}" diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/service.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/service.yaml index dd8903d..d630512 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/service.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/service.yaml @@ -11,6 +11,7 @@ metadata: {{ toYaml .Values.service.labels | indent 4 }} {{- end }} name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} spec: {{- if .Values.service.clusterIP }} clusterIP: "{{ .Values.service.clusterIP }}" diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/servicemonitor.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/servicemonitor.yaml index a0b9d25..be37239 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/servicemonitor.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/templates/servicemonitor.yaml @@ -7,9 +7,9 @@ metadata: namespace: {{ .Values.serviceMonitor.namespace }} {{- end }} labels: - {{- range $key, $value := .Values.serviceMonitor.selector }} - {{ $key }}: {{ $value | quote }} - {{- end }} + {{- range $key, $value := .Values.serviceMonitor.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} spec: selector: matchLabels: diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/values.yaml b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/values.yaml index aebcb66..bdaaf63 100644 --- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/values.yaml +++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/charts/cluster-autoscaler/values.yaml @@ -3,24 +3,29 @@ affinity: {} autoDiscovery: - # cloudProviders `aws`, `gce` and `magnum` are supported by auto-discovery at this time + # cloudProviders `aws`, `gce`, `magnum` and `clusterapi` are supported by auto-discovery at this time # AWS: Set tags as described in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. + # Enable autodiscovery for `cloudProvider=clusterapi`, for groups matching `autoDiscovery.labels`. # Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. # Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. clusterName: "adsd-cumulus-dev" # autoDiscovery.tags -- ASG tags to match, run through `tpl`. tags: - - k8s.io/cluster-autoscaler/enabled - - k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }} + - k8s.io/cluster-autoscaler/enabled + - k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }} # - kubernetes.io/cluster/{{ .Values.autoDiscovery.clusterName }} # autoDiscovery.roles -- Magnum node group roles to match. roles: - - worker + - worker + # autoDiscovery.labels -- Cluster-API labels to match https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery + labels: [] + # - color: green + # - shape: circle # autoscalingGroups -- For AWS, Azure AKS or Magnum. At least one element is required if not using `autoDiscovery`. For example: # 
 # - name: asg1

@@ -89,7 +94,7 @@ azureClusterName: ""
 # Required if `cloudProvider=azure`
 azureNodeResourceGroup: ""
 
-# azureUseManagedIdentityExtension -- Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID and resource group are set.
+# azureUseManagedIdentityExtension -- Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set.
 azureUseManagedIdentityExtension: false
 
 # magnumClusterName -- Cluster name or ID in Magnum.
@@ -99,13 +104,32 @@ magnumClusterName: ""
 # magnumCABundlePath -- Path to the host's CA bundle, from `ca-file` in the cloud-config file.
 magnumCABundlePath: "/etc/kubernetes/ca-bundle.crt"
 
+# clusterAPIMode --  Cluster API mode, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#connecting-cluster-autoscaler-to-cluster-api-management-and-workload-clusters
+# Syntax: workloadClusterMode-ManagementClusterMode
+# for `kubeconfig-kubeconfig`, `incluster-kubeconfig` and `single-kubeconfig` you always must mount the external kubeconfig using either `extraVolumeSecrets` or `extraMounts` and `extraVolumes`
+# if you dont set `clusterAPIKubeconfigSecret`and thus use an in-cluster config or want to use a non capi generated kubeconfig you must do so for the workload kubeconfig as well
+clusterAPIMode: incluster-incluster  # incluster-incluster, incluster-kubeconfig, kubeconfig-incluster, kubeconfig-kubeconfig, single-kubeconfig
+
+# clusterAPIKubeconfigSecret -- Secret containing kubeconfig for connecting to Cluster API managed workloadcluster
+# Required if `cloudProvider=clusterapi` and `clusterAPIMode=kubeconfig-kubeconfig,kubeconfig-incluster or incluster-kubeconfig`
+clusterAPIKubeconfigSecret: ""
+
+# clusterAPIWorkloadKubeconfigPath -- Path to kubeconfig for connecting to Cluster API managed workloadcluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or kubeconfig-incluster`
+clusterAPIWorkloadKubeconfigPath: /etc/kubernetes/value
+
+# clusterAPICloudConfigPath -- Path to kubeconfig for connecting to Cluster API Management Cluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or incluster-kubeconfig`
+clusterAPICloudConfigPath: /etc/kubernetes/mgmt-kubeconfig
+
+# clusterAPIConfigMapsNamespace -- Namespace on the workload cluster to store Leader election and status configmaps
+clusterAPIConfigMapsNamespace: ""
+
 # cloudConfigPath -- Configuration file for cloud provider.
 cloudConfigPath: /etc/gce.conf
 
 # cloudProvider -- The cloud provider where the autoscaler runs.
-# Currently only `gce`, `aws`, `azure` and `magnum` are supported.
+# Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported.
 # `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS.
-# `magnum` for OpenStack Magnum.
+# `magnum` for OpenStack Magnum, `clusterapi` for Cluster API.
 cloudProvider: aws
 
 # containerSecurityContext -- [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)
@@ -114,14 +138,18 @@ containerSecurityContext: {}
   #   drop:
   #   - ALL
 
+deployment:
+  # deployment.annotations -- Annotations to add to the Deployment object.
+  annotations: {}
+
 # dnsPolicy -- Defaults to `ClusterFirst`. Valid values are:
 # `ClusterFirstWithHostNet`, `ClusterFirst`, `Default` or `None`.
 # If autoscaler does not depend on cluster DNS, recommended to set this to `Default`.
 dnsPolicy: ClusterFirst
 
 ## Priorities Expander
-# expanderPriorities -- The expanderPriorities is used if `extraArgs.expander` is set to `priority` and expanderPriorities is also set with the priorities.
-# If `extraArgs.expander` is set to `priority`, then expanderPriorities is used to define cluster-autoscaler-priority-expander priorities.
+# expanderPriorities -- The expanderPriorities is used if `extraArgs.expander` contains `priority` and expanderPriorities is also set with the priorities.
+# If `extraArgs.expander` contains `priority`, then expanderPriorities is used to define cluster-autoscaler-priority-expander priorities.
 # See: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/expander/priority/readme.md
 expanderPriorities: {}
 
@@ -205,7 +233,7 @@ image:
   # image.repository -- Image repository
   repository: 252960665057.dkr.ecr.us-gov-east-1.amazonaws.com/eks/adsd-cumulus-dev/cluster-autoscaler
   # image.tag -- Image tag
-  tag: v1.21.0
+  tag: v1.24.0
   # image.pullPolicy -- Image pull policy
   pullPolicy: IfNotPresent
   ## Optionally specify an array of imagePullSecrets.
@@ -241,7 +269,7 @@ podLabels: {}
 additionalLabels: {}
 
 # priorityClassName -- priorityClassName
-priorityClassName: ""
+priorityClassName: "system-cluster-critical"
 
 rbac:
   # rbac.create -- If `true`, create and use RBAC resources.
@@ -249,6 +277,8 @@ rbac:
   # rbac.pspEnabled -- If `true`, creates and uses RBAC resources required in the cluster with [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) enabled.
   # Must be used with `rbac.create` set to `true`.
   pspEnabled: false
+  # rbac.clusterScoped -- if set to false will only provision RBAC to alter resources in the current namespace. Most useful for Cluster-API
+  clusterScoped: true
   serviceAccount:
     # rbac.serviceAccount.annotations -- Additional Service Account annotations.
     annotations:
@@ -267,10 +297,10 @@ replicaCount: 1
 resources:
   limits:
     cpu: 100m
-    memory: 600Mi
+    memory: 300Mi
   requests:
     cpu: 100m
-    memory: 600Mi
+    memory: 300Mi
 
 # securityContext -- [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)
 securityContext: {}
@@ -331,6 +361,15 @@ prometheusRule:
 # tolerations -- List of node taints to tolerate (requires Kubernetes >= 1.6).
 tolerations: []
 
+# topologySpreadConstraints -- You can use topology spread constraints to control how Pods are spread across your cluster among failure-domains such as regions, zones, nodes, and other user-defined topology domains. (requires Kubernetes >= 1.19).
+topologySpreadConstraints: []
+  # - maxSkew: 1
+  #   topologyKey: topology.kubernetes.io/zone
+  #   whenUnsatisfiable: DoNotSchedule
+  #   labelSelector:
+  #     matchLabels:
+  #       app.kubernetes.io/instance: cluster-autoscaler
+
 # updateStrategy -- [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy)
 updateStrategy: {}
   # rollingUpdate:
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/copy_images.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/copy_images.tf
index ddd67aa..f82bfa7 100644
--- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/copy_images.tf
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/copy_images.tf
@@ -17,7 +17,7 @@ locals {
     },
     {
       name    = "cluster-autoscaler"
-      image   = "public.ecr.aws/v0g0y9g5/cluster-autoscaler"
+      image   = "k8s.gcr.io/autoscaling/cluster-autoscaler"
       tag     = var.cluster_autoscaler_tag
       enabled = true
     },
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/main.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/main.tf
index 85c931e..ebc690a 100644
--- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/main.tf
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/main.tf
@@ -10,9 +10,7 @@ locals {
     "metrics-server" = {
       name       = "metrics-server"
       repository = "https://charts.bitnami.com/bitnami"
-      #      version    = "5.10.4"
-      # version    = "5.11.9"
-      version    = "6.2.4"
+      version    = "6.2.6"
       use_remote = true
     }
 
@@ -47,8 +45,7 @@ resource "helm_release" "metrics-server" {
 
   depends_on = [null_resource.copy_images]
   set {
-    name = "extraArgs[0]"
-    #    value = "InternalIP,ExternalIP,Hostname"
+    name  = "extraArgs[0]"
     value = "--kubelet-preferred-address-types=InternalIP"
   }
   set {
@@ -111,6 +108,10 @@ resource "helm_release" "cluster-autoscaler" {
     name  = "awsRegion"
     value = local.region
   }
+  set {
+    name  = "rbac.serviceAccount.create"
+    value = "false"
+  }
 }
 
 # Install cert-manager
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/tf-run.data b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/tf-run.data
index 47d98a5..b0b0c0f 100644
--- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/tf-run.data
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/tf-run.data
@@ -1,14 +1,15 @@
-VERSION 1.2.3
+VERSION 1.3.0
 REMOTE-STATE
 COMMAND tf-directory-setup.py -l none -f
 COMMAND setup-new-directory.sh
 COMMAND tf-init -upgrade
-COMMAND ln -sf ../variables.vpc.auto.tfvars .
-COMMAND ln -sf ../variables.vpc.tf
-COMMAND ln -sf ../versions.tf
-COMMAND ln -sf ../settings.auto.tfvars
-COMMAND ln -sf ../variables.application_tags.auto.tfvars
+
 LINKTOP init
+LINK versions.tf
+LINK settings.auto.tfvars
+LINK variables.application_tags.auto.tfvars
+LINK variables.vpc.tf
+LINK variables.vpc.auto.tfvars
 
 module.cert
 COMMAND tf-directory-setup.py -l s3
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/variables.common-services.auto.tfvars b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/variables.common-services.auto.tfvars
index c6a82a1..8f4ca2e 100644
--- a/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/variables.common-services.auto.tfvars
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/common-services/variables.common-services.auto.tfvars
@@ -1,9 +1,9 @@
 cert_manager_cainjector_tag    = "v1.4.3"
 cert_manager_controller_tag    = "v1.4.3"
 cert_manager_webhook_tag       = "v1.4.3"
-cluster_autoscaler_tag         = "v1.21.0"
+cluster_autoscaler_tag         = "v1.24.0"
 istio_tag                      = "1.10.1"
-metrics_server_tag             = "0.6.2-debian-11-r0"
+metrics_server_tag             = "0.6.2-debian-11-r9"
 tls_crt_b64                    = ""
 tls_crt_contents               = ""
 tls_crt_file                   = ""
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs-encryption.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs-encryption.tf
index c67da2a..c125a89 100644
--- a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs-encryption.tf
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs-encryption.tf
@@ -3,11 +3,30 @@ locals {
   app_tags_sc_parameters  = { for i in range(0, length(local._app_tags_sc_parameters)) : format("tagSpecification_%v", i + 1) => local._app_tags_sc_parameters[i] }
 }
 
+resource "kubernetes_storage_class" "gp3_encrypted" {
+  metadata {
+    name = "gp3-encrypted"
+    annotations = {
+      "storageclass.kubernetes.io/is-default-class" = "true"
+    }
+  }
+  parameters = {
+    fsType    = "ext4"
+    type      = "gp3"
+    encrypted = "true"
+    kmsKeyId  = data.aws_kms_key.ebs_key.arn
+  }
+  storage_provisioner    = "ebs.csi.aws.com"
+  reclaim_policy         = "Delete"
+  volume_binding_mode    = "Immediate"
+  allow_volume_expansion = "true"
+}
+
 resource "kubernetes_storage_class" "ebs_encrypted" {
   metadata {
     name = "gp2-encrypted"
     annotations = {
-      "storageclass.kubernetes.io/is-default-class" = "true"
+      "storageclass.kubernetes.io/is-default-class" = "false"
     }
   }
   parameters = merge(
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/.tf-control b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/.tf-control
new file mode 100644
index 0000000..280f449
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/.tf-control
@@ -0,0 +1,20 @@
+# .tf-control
+# allows for setting a specific command to be used for tf-* commands under this git repo
+# see tf-control.sh help for more info
+
+TFCONTROL_VERSION="1.0.5"
+
+TFCOMMAND="terraform_latest"
+# TF_CLI_CONFIG_FILE=PATH-TO-FILE/.tf-control.tfrc
+# TFARGS=""
+# TFNOLOG=""
+# TFNOCOLOR=""
+
+# use the following to force a specific version.  An upgrade of an existing 0.12.31 to 1.x
+# needs you to cycle through 0.13.17, 0.14.11, and then latest (0.15.5 not needed).  Other
+# steps in between.  See https://github.e.it.census.gov/terraform/support/tree/master/docs/how-to/terraform-upgrade for details
+#
+#TFCOMMAND="terraform_0.12.31"
+#TFCOMMAND="terraform_0.13.7"
+#TFCOMMAND="terraform_0.14.11"
+#TFCOMMAND="terraform_0.15.5"
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/.tf-control.tfrc b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/.tf-control.tfrc
new file mode 100644
index 0000000..7425488
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/.tf-control.tfrc
@@ -0,0 +1,24 @@
+TFCONTROL_VERSION="1.0.5"
+
+# https://www.terraform.io/docs/cli/config/config-file.html
+plugin_cache_dir   = "/data/terraform/terraform.d/plugin-cache"
+#disable_checkpoint = true
+
+provider_installation {
+#  filesystem_mirror {
+#    path    = "/apps/terraform/terraform.d/providers"
+#    include = [ "*/*/*" ]
+#  }
+  filesystem_mirror {
+    path    = "/data/terraform/terraform.d/providers"
+    include = [ "*/*/*" ]
+  }
+#  filesystem_mirror {
+#    path    = "/apps/terraform/terraform.d/providers"
+#    include = [ "external.terraform.census.gov/*/*" ]
+#  }
+  direct {
+    include = [ "*/*/*" ]
+  }
+}
+
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/README.ebs.md b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/README.ebs.md
new file mode 100644
index 0000000..de10f70
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/README.ebs.md
@@ -0,0 +1,75 @@
+# eks-ebs
+
+With EKS based upon Kubernetes 1.23 or higher, the default gp2 storage class will no longer auto-provision persistent volumes.
+While an EFS-based auto-provisioner which supports all types of persistent volumes has been installed, it does not perform as well as a gp2/gp3 based perstent volume.
+The eks-ebs module installs an ebs-provisioner in the cluster with a storage class of `gp3-encrypted`, deletes the pre-existing `gp2` storage class, and makes `gp3-encrypted` the default storage class for the cluster.
+
+## Parameters
+
+| Name | Description |
+| ---- | ----------- |
+| region | The AWS region that EKS cluster is located. |
+| cluster_name | The name of the cluster in which ebs-provisioner will be installed. |
+| aws_ebs_csi_driver_version | Which version of the aws-ebs-csi-driver helm chart to use.  Currently defaults to 2.14.1. |
+
+## Updating the aws-ebs-csi-driver chart
+
+When using a private VPC, the helm chart cannot be downloaded from "https://kubernetes-sigs.github.io/aws-ebs-csi-driver/" during installation.
+A local copy of the chart is maintained within the terraform script.
+The lastest version of the helm chart can be found by looking at https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/charts/aws-ebs-csi-driver/Chart.yaml and checking the `version:` tag (not the `appVersion` tag.)
+To update this helm chart to the latest version, the procedure is to:
+
+```script
+cd charts
+helm repo add aws-ebs-csi-driver https://kubernetes-sigs.github.io/aws-ebs-csi-driver/
+helm repo update
+rm -fr aws-ebs-csi-driver
+helm pull aws-ebs-csi-driver/aws-ebs-csi-driver --untar
+```
+
+After completing these steps, be sure to examine aws-ebs-csi-driver/values.yaml and confirm that the tags listed for the sidecar images match the tags assigned by default in input.tf.
+For example, the values.yaml file:
+
+```json
+sidecars:
+  livenessProbe:
+    image:
+      repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe
+      tag: v2.2.0-eks-1-18-2
+      pullPolicy: IfNotPresent
+    resources: {}
+  nodeDriverRegistrar:
+    image:
+      repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar
+      tag: v2.1.0-eks-1-18-2
+      pullPolicy: IfNotPresent
+    resources: {}
+  csiProvisioner:
+    image:
+      repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner
+      tag: v2.1.1-eks-1-18-2
+      pullPolicy: IfNotPresent
+    resources: {}
+```
+
+Entries in input.tf:
+
+```hcl
+variable "livenessprobe_tag" {
+  description = "Which tag of public.ecr.aws/eks-distro/kubernetes-csi/livenessp
+robe to use."
+  default     = "v2.2.0-eks-1-18-2"
+}
+
+variable "node_driver_registrar_tag" {
+  description = "Which tag of public.ecr.aws/eks-distro/kubernetes-csi/node-driv
+er-registrar to use."
+  default     = "v2.1.0-eks-1-18-2"
+}
+
+variable "external_provisioner_tag" {
+  description = "Which tag of public.ecr.aws/eks-distro/kubernetes-csi/external-
+provisioner to use."
+  default     = "v2.1.1-eks-1-18-2"
+}
+```
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/README.md b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/README.md
new file mode 100644
index 0000000..37bb3c8
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/README.md
@@ -0,0 +1,122 @@
+# EBS
+
+Staring with EKS based upon Kubernetes 1.23, the gp2 storage class no longer supports auto-provisiong.
+This module sets up the needed resources to provision EBS-based gp3 persistent volumes.  See [this](README.efs.md) for more details.
+
+## Links
+
+* https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html
+* https://github.com/kubernetes-sigs/aws-ebs-csi-driver
+* https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/722
+* https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1086
+
+##  Initialize
+
+* Proxy setup
+
+Proxy is needed because system may not have access to the `registry.terraform.io` site directory,
+and if indirectly, it may not be able to handle a proxy redirect.  You may not need to use this, but if you get
+errors from the `tf-init`, this is your first thing to setup.
+
+```shell
+export HTTP_PROXY=http://proxy.tco.census.gov:3128
+export HTTPS_PROXY=http://proxy.tco.census.gov:3128
+```
+
+## Terraform Automated
+
+A `tf-run.data` file exists here, so the simplest way to implemnt is with the `tf-run.sh` script.
+
+* copy the `remote_state.yml` from the parent and update `directory` to be the current directory
+* run the tf-run.sh
+
+```console
+% tf-run.sh apply
+```
+
+* example of the `tf-run.sh` steps
+
+This is part of a larger cluster configuration, so at the end of the run it indicates another directory
+to visit when done.
+
+```console
+<<>>
+```
+
+It is highly recommended to use the `tf-run.sh` approach.
+
+## Terraform Manual
+
+
+```shell
+tf-directory-setup.py -l none
+setup-new-directory.sh
+tf-init
+````
+
+* Apply the rest
+
+This must be done from a system with the skopeo command, so RHEL8+.
+
+To use the local install, The ebs/charts/ directory
+must be populated with the expected code (see [README.md](README.md)) outside of terraform,
+much like the .tf files are created.  Currently, as the box we run this from has internet access,
+we can deploy by pulling the helm stuff from the internet.
+
+```shell
+tf-apply
+tf-directory-setup.py -l s3
+```
+
+## Post Setup Examination
+
+This gives us (look at the ebs-csi-* ones) to see what was setup.  Your `kubectl` configuration file
+needs to be setup (one is extracted in `setup/kube.config` as part of this configuration).
+
+```console
+% kubectl --kubeconfig setup/kube.config get pods -n kube-system
+<<>>
+```
+
+* Create PVC Automated
+
+Use the `persistent-volume.tf`, which is setup by default, and should happen as part of the final apply above.
+
+* Create PVC Manually
+
+```json
+# pvc.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ebs-test3-claim
+spec:
+  accessModes:
+  - ReadWriteOnce
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: gp3-encrypted
+```
+
+* Examinine the PV and PVC
+
+```console
+% kubectl get pv
+No resources found
+% kubectl get pvc
+No resources found in default namespace.
+% kubectl apply -f pvc.yaml
+persistentvolumeclaim/ebs-test3-claim created
+% kubectl get pvc
+NAME             STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS   AGE
+ebs-test3-claim   Pending                                      gp3-encrypted  39s
+```
+
+* Describing the PVC
+
+```shell
+kubectl --kubeconfig setup/kube.config describe pvc ebs-test3-claim
+```
+
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/copy_image.sh b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/copy_image.sh
new file mode 120000
index 0000000..889e269
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/copy_image.sh
@@ -0,0 +1 @@
+../bin/copy_image.sh
\ No newline at end of file
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/copy_images.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/copy_images.tf
new file mode 100644
index 0000000..55f911c
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/copy_images.tf
@@ -0,0 +1,77 @@
+data "aws_ecr_authorization_token" "token" {}
+
+locals {
+  account_id       = data.aws_caller_identity.current.account_id
+  repo_parent_name = format("eks/%v", var.cluster_name)
+
+  account_ecr_registry = format("%v.dkr.ecr.%v.amazonaws.com", local.account_id, var.region)
+  account_ecr          = format("%v/%v", local.account_ecr_registry, local.repo_parent_name)
+
+  images = [
+    {
+      name  = "aws-ebs-csi-driver"
+      image = "public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver"
+      tag   = var.aws_ebs_csi_driver_tag
+    },
+    {
+      name  = "external-provisioner"
+      image = "public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner"
+      tag   = var.external_provisioner_tag
+    },
+    {
+      name  = "external-attacher"
+      image = "public.ecr.aws/eks-distro/kubernetes-csi/external-attacher"
+      tag   = var.external_attacher_tag
+    },
+    {
+      name  = "csi-snapshotter"
+      image = "public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter"
+      tag   = var.csi_snapshotter_tag
+    },
+    {
+      name  = "livenessprobe"
+      image = "public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe"
+      tag   = var.livenessprobe_tag
+    },
+    {
+      name  = "external-resizer"
+      image = "public.ecr.aws/eks-distro/kubernetes-csi/external-resizer"
+      tag   = var.external_resizer_tag
+    },
+    {
+      name  = "node-driver-registrar"
+      image = "public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar"
+      tag   = var.node_driver_registrar_tag
+    },
+  ]
+  image_repos = { for image in local.images : image.name => format("%v/%v", local.account_ecr, image.name) }
+  image_map = { for image in local.images : image.name =>
+    merge(
+      image,
+      tomap(
+        { "full_path" = local.image_repos[image.name],
+          "registry"  = local.account_ecr_registry,
+        "repository" = format("%v/%v", local.repo_parent_name, image.name), }
+  )) }
+}
+
+resource "null_resource" "copy_images" {
+  for_each = { for image in local.images : image.name => image }
+  triggers = {
+    name  = each.key
+    image = format("%v:%v", each.value.image, each.value.tag)
+  }
+
+  provisioner "local-exec" {
+    command = "${path.module}/copy_image.sh"
+    environment = {
+      AWS_PROFILE          = var.profile
+      AWS_REGION           = local.region
+      SOURCE_IMAGE         = format("%v:%v", each.value.image, each.value.tag)
+      DESTINATION_IMAGE    = format("%v/%v:%v", local.account_ecr, each.value.name, each.value.tag)
+      DESTINATION_USERNAME = data.aws_ecr_authorization_token.token.user_name
+      DESTINATION_PASSWORD = data.aws_ecr_authorization_token.token.password
+    }
+  }
+}
+
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/data.eks-subdirectory.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/data.eks-subdirectory.tf
new file mode 120000
index 0000000..43b5430
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/data.eks-subdirectory.tf
@@ -0,0 +1 @@
+../includes.d/data.eks-subdirectory.tf
\ No newline at end of file
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/ecr.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/ecr.tf
new file mode 100644
index 0000000..a8630eb
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/ecr.tf
@@ -0,0 +1,53 @@
+
+# Populated from:
+# https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html
+
+data "aws_caller_identity" "whoami" {}
+
+locals {
+  af_south_1 = (var.region == "af-south-1" ? "877085696533.dkr.ecr.af-south-1.amazonaws.com/" : "")
+  af         = local.af_south_1
+
+  ap_east_1      = var.region == "ap-east-1" ? "800184023465.dkr.ecr.ap-east-1.amazonaws.com/" : ""
+  ap_northeast_1 = var.region == "ap-northeast-1" ? "602401143452.dkr.ecr.ap-northeast-1.amazonaws.com/" : ""
+  ap_northeast_2 = var.region == "ap-northeast-2" ? "602401143452.dkr.ecr.ap-northeast-2.amazonaws.com/" : ""
+  ap_northeast_3 = var.region == "ap-northeast-3" ? "602401143452.dkr.ecr.ap-northeast-3.amazonaws.com/" : ""
+  ap_south_1     = var.region == "ap-south-1" ? "602401143452.dkr.ecr.ap-south-1.amazonaws.com/" : ""
+  ap_southeast_1 = var.region == "ap-southeast-1" ? "602401143452.dkr.ecr.ap-southeast-1.amazonaws.com/" : ""
+  ap_southeast_2 = var.region == "ap-southeast-2" ? "602401143452.dkr.ecr.ap-southeast-2.amazonaws.com/" : ""
+  ap_1           = "${local.ap_east_1}${local.ap_northeast_1}${local.ap_northeast_2}${local.ap_northeast_3}${local.ap_south_1}"
+  ap_2           = "${local.ap_southeast_1}${local.ap_southeast_2}"
+  ap             = "${local.ap_1}${local.ap_2}"
+
+  ca_central_1 = var.region == "ca-central-1" ? "602401143452.dkr.ecr.ca-central-1.amazonaws.com/" : ""
+  ca           = local.ca_central_1
+
+  cn_north_1     = var.region == "cn-north-1" ? "918309763551.dkr.ecr.cn-north-1.amazonaws.com.cn/" : ""
+  cn_northwest_1 = var.region == "cn-northwest-1" ? "961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/" : ""
+  cn             = "${local.cn_north_1}${local.cn_northwest_1}"
+
+  eu_central_1 = var.region == "eu-central-1" ? "602401143452.dkr.ecr.eu-central-1.amazonaws.com/" : ""
+  eu_north_1   = var.region == "eu-north-1" ? "602401143452.dkr.ecr.eu-north-1.amazonaws.com/" : ""
+  eu_south_1   = var.region == "eu-south-1" ? "590381155156.dkr.ecr.eu-south-1.amazonaws.com/" : ""
+  eu_west_1    = var.region == "eu-west-1" ? "602401143452.dkr.ecr.eu-west-1.amazonaws.com/" : ""
+  eu_west_2    = var.region == "eu-west-2" ? "602401143452.dkr.ecr.eu-west-2.amazonaws.com/" : ""
+  eu_west_3    = var.region == "eu-west-3" ? "602401143452.dkr.ecr.eu-west-3.amazonaws.com/" : ""
+  eu           = "${local.eu_central_1}${local.eu_north_1}${local.eu_south_1}${local.eu_west_1}${local.eu_west_2}${local.eu_west_3}"
+
+  me_south_1 = var.region == "me-south-1" ? "558608220178.dkr.ecr.me-south-1.amazonaws.com/" : ""
+  me         = local.me_south_1
+
+  sa_east_1 = var.region == "sa-east-1" ? "602401143452.dkr.ecr.sa-east-1.amazonaws.com/" : ""
+  sa        = local.sa_east_1
+
+  us_east_1     = var.region == "us-east-1" ? "602401143452.dkr.ecr.us-east-1.amazonaws.com/" : ""
+  us_east_2     = var.region == "us-east-2" ? "602401143452.dkr.ecr.us-east-2.amazonaws.com/" : ""
+  us_gov_east_1 = var.region == "us-gov-east-1" ? "151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/" : ""
+  us_gov_west_1 = var.region == "us-gov-west-1" ? "013241004608.dkr.ecr.us-gov-west-1.amazonaws.com/" : ""
+  us_west_1     = var.region == "us-west-1" ? "602401143452.dkr.ecr.us-west-1.amazonaws.com/" : ""
+  us_west_2     = var.region == "us-west-2" ? "602401143452.dkr.ecr.us-west-2.amazonaws.com/" : ""
+  us            = "${local.us_east_1}${local.us_east_2}${local.us_gov_east_1}${local.us_gov_west_1}${local.us_west_1}${local.us_west_2}"
+
+  ecr = "${local.af}${local.ap}${local.ca}${local.cn}${local.eu}${local.me}${local.sa}${local.us}"
+
+}
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/kubeconfig.eks-subdirectory.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/kubeconfig.eks-subdirectory.tf
new file mode 120000
index 0000000..e3750a4
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/kubeconfig.eks-subdirectory.tf
@@ -0,0 +1 @@
+../includes.d/kubeconfig.eks-subdirectory.tf
\ No newline at end of file
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/locals.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/locals.tf
new file mode 100644
index 0000000..4b9ae5a
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/locals.tf
@@ -0,0 +1,17 @@
+locals {
+  base_tags = {
+    "eks-cluster-name"      = var.cluster_name
+    "boc:tf_module_version" = local._module_version
+    "boc:created_by"        = "terraform"
+  }
+}
+
+# replace TF remote state accordingly in parent_rs with that from the parent directory, and be sure to make the link
+locals {
+  vpc_id               = local.parent_rs.cluster_vpc_id
+  subnet_ids           = local.parent_rs.cluster_subnet_ids
+  cluster_worker_sg_id = local.parent_rs.cluster_worker_sg_id
+
+  oidc_provider_url = local.parent_rs.oidc_provider_url
+  oidc_provider_arn = local.parent_rs.oidc_provider_arn
+}
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/main.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/main.tf
new file mode 100644
index 0000000..b95263c
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/main.tf
@@ -0,0 +1,105 @@
+# Most of this file references the AWS documentation to install the
+# Amazon EBS CSI driver.  This documentation is found here:
+# https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html
+
+
+locals {
+  charts = {
+    "ebs-provisioner" = {
+      name       = "aws-ebs-csi-driver"
+      repository = "https://kubernetes-sigs.github.io/aws-ebs-csi-driver"
+      version    = "2.14.1"
+      use_remote = true
+    }
+  }
+}
+
+# Create an IAM policy and role
+# Step 2b:
+#
+# See role.tf
+
+# Install the Amazon EBS driver
+# Step 3:
+# See the readme `Updating the aws-ebs-csi-driver chart` on updating this chart.
+resource "helm_release" "ebs-provisioner" {
+
+  chart         = "aws-ebs-csi-driver"
+  name          = "ebs-provisioner"
+  namespace     = "kube-system"
+  repository    = local.charts["ebs-provisioner"].use_remote ? local.charts["ebs-provisioner"].repository : "${path.module}/charts"
+  version       = local.charts["ebs-provisioner"].use_remote ? local.charts["ebs-provisioner"].version : null
+  recreate_pods = true
+  timeout       = 300
+  depends_on    = [null_resource.copy_images]
+
+  set {
+    name  = "image.repository"
+    value = local.image_repos["aws-ebs-csi-driver"]
+  }
+  set {
+    name  = "image.tag"
+    value = var.aws_ebs_csi_driver_tag
+  }
+  set {
+    name  = "sidecars.provisioner.image.repository"
+    value = local.image_repos["external-provisioner"]
+  }
+  set {
+    name  = "sidecars.provisioner.image.tag"
+    value = var.external_provisioner_tag
+  }
+  set {
+    name  = "sidecars.attacher.image.repository"
+    value = local.image_repos["external-attacher"]
+  }
+  set {
+    name  = "sidecars.attacher.image.tag"
+    value = var.external_attacher_tag
+  }
+  set {
+    name  = "sidecars.snapshotter.image.repository"
+    value = local.image_repos["csi-snapshotter"]
+  }
+  set {
+    name  = "sidecars.snapshotter.image.tag"
+    value = var.csi_snapshotter_tag
+  }
+  set {
+    name  = "sidecars.livenessProbe.image.repository"
+    value = local.image_repos["livenessprobe"]
+  }
+  set {
+    name  = "sidecars.livenessProbe.image.tag"
+    value = var.livenessprobe_tag
+  }
+  set {
+    name  = "sidecars.resizer.image.repository"
+    value = local.image_repos["external-resizer"]
+  }
+  set {
+    name  = "sidecars.resizer.image.tag"
+    value = var.external_resizer_tag
+  }
+  set {
+    name  = "sidecars.nodeDriverRegistrar.image.repository"
+    value = local.image_repos["node-driver-registrar"]
+  }
+  set {
+    name  = "sidecars.nodeDriverRegistrar.image.tag"
+    value = var.node_driver_registrar_tag
+  }
+  set {
+    name  = "controller.env[0].name"
+    value = "AWS_STS_REGIONAL_ENDPOINTS"
+  }
+  set {
+    name  = "controller.env[0].value"
+    value = "regional"
+  }
+  set {
+    name  = "controller.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
+    value = module.role_ebs-driver.role_arn
+  }
+}
+
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/parent_rs.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/parent_rs.tf
new file mode 120000
index 0000000..d85ece6
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/parent_rs.tf
@@ -0,0 +1 @@
+../includes.d/parent_rs.tf
\ No newline at end of file
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/prefixes.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/prefixes.tf
new file mode 120000
index 0000000..e0bf5ad
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/prefixes.tf
@@ -0,0 +1 @@
+../prefixes.tf
\ No newline at end of file
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/providers.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/providers.tf
new file mode 120000
index 0000000..7244d01
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/providers.tf
@@ -0,0 +1 @@
+../providers.tf
\ No newline at end of file
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/region.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/region.tf
new file mode 100644
index 0000000..b7b1696
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/region.tf
@@ -0,0 +1,4 @@
+locals {
+  region = var.region
+}
+
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/role.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/role.tf
new file mode 100644
index 0000000..5ce22c5
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/role.tf
@@ -0,0 +1,51 @@
+#---
+# cluster
+#---
+locals {
+  #  oidc       = replace(data.aws_eks_cluster.cluster.identity[0].oidc[0].issuer, "https://", "")
+  principal = format("arn:%v:iam::%v:oidc-provider/%v", data.aws_arn.current.partition, local.account_id, local.oidc_provider_url)
+}
+
+data "aws_iam_policy" "ebs-provisioner" {
+  name = "AmazonEBSCSIDriverPolicy"
+}
+
+# create: aws_iam_policy.ebs-policy first
+module "role_ebs-driver" {
+  source = "git@github.e.it.census.gov:terraform-modules/aws-iam-role.git"
+
+  role_name              = format("%v%v-ebs-driver", local._prefixes["eks"], var.cluster_name)
+  role_description       = "EKS EBS Driver Role for ${var.cluster_name}"
+  enable_ldap_creation   = false
+  assume_policy_document = data.aws_iam_policy_document.ebs_assume_webidentity.json
+  attached_policies      = [data.aws_iam_policy.ebs-provisioner.arn]
+
+  tags = merge(
+    local.base_tags,
+    local.common_tags,
+    var.application_tags,
+    tomap({ "Name" = format("%v%v-ebs-driver", local._prefixes["eks-role"], var.cluster_name) }),
+  )
+}
+
+data "aws_iam_policy_document" "ebs_assume_webidentity" {
+  statement {
+    sid     = "EFSAssumeRoleWebIdentity"
+    effect  = "Allow"
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+    principals {
+      type        = "Federated"
+      identifiers = [local.principal]
+    }
+    condition {
+      test     = "StringEquals"
+      variable = "${local.oidc_provider_url}:sub"
+      values   = ["system:serviceaccount:kube-system:ebs-csi-controller-sa"]
+    }
+  }
+}
+
+output "role_ebs-driver_arn" {
+  description = "Role ARN for EKS EBS Driver Role"
+  value       = module.role_ebs-driver.role_arn
+}
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/tf-run.data b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/tf-run.data
new file mode 100644
index 0000000..789be44
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/tf-run.data
@@ -0,0 +1,21 @@
+VERSION 2.0.1
+REMOTE-STATE
+COMMAND tf-directory-setup.py -l none -f
+COMMAND setup-new-directory.sh
+LINKTOP init
+LINKTOP includes.d/variables.account_tags.tf
+LINKTOP includes.d/variables.account_tags.auto.tfvars
+LINKTOP includes.d/variables.infrastructure_tags.tf
+LINKTOP includes.d/variables.infrastructure_tags.auto.tfvars
+LINKTOP includes.d/variables.application_tags.tf
+## LINKTOP includes.d/variables.application_tags.auto.tfvars
+LINK versions.tf
+LINK settings.auto.tfvars
+LINK variables.application_tags.auto.tfvars
+
+COMMAND tf-init -upgrade
+
+#POLICY 
+ALL
+COMMAND tf-directory-setup.py -l s3
+STOP cd ../irsa-roles and tf-run.sh apply
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/variables.ebs.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/variables.ebs.tf
new file mode 100644
index 0000000..2f012de
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/variables.ebs.tf
@@ -0,0 +1,61 @@
+# variable "eks_vpc_name" {
+#   description = "Define the VPC name that will be used by this cluster"
+#   type        = string
+#   default     = "*vpc4*"
+# }
+# 
+# variable "subnets_name" {
+#   description = "Define the name of the subnets to be used by this cluster"
+#   type        = string
+#   default     = "*-apps-*"
+# }
+
+variable "cluster_worker_sg_id" {
+  description = "Security group for all worker management."
+  type        = string
+  default     = ""
+}
+
+# See the readme `Updating the aws-ebs-csi-driver chart` to find these values.
+variable "aws_ebs_csi_driver_tag" {
+  description = "Which tag of public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver"
+  type        = string
+  default     = "v1.15.0"
+}
+
+variable "external_provisioner_tag" {
+  description = "Which tag of public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner"
+  type        = string
+  default     = "v3.3.0-eks-1-25-latest"
+}
+
+variable "external_attacher_tag" {
+  description = "Which tag of public.ecr.aws/eks-distro/kubernetes-csi/external-attacher"
+  type        = string
+  default     = "v4.0.0-eks-1-25-latest"
+}
+
+variable "csi_snapshotter_tag" {
+  description = "Which tag of public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter"
+  type        = string
+  default     = "v6.1.0-eks-1-25-latest"
+}
+
+variable "livenessprobe_tag" {
+  description = "Which tag of public.ecr.aws/eks-distro/kubernetes-csi/liveness"
+  type        = string
+  default     = "v2.8.0-eks-1-25-latest"
+}
+
+variable "external_resizer_tag" {
+  description = "Which tag of public.ecr.aws/eks-distro/kubernetes-csi/external-resizer"
+  type        = string
+  default     = "v1.6.0-eks-1-25-latest"
+}
+
+variable "node_driver_registrar_tag" {
+  description = "Which tag of public.ecr.aws/eks-distro/kubernetes-csi/node-dri"
+  type        = string
+  default     = "v2.6.2-eks-1-25-latest"
+}
+
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/variables.eks.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/variables.eks.tf
new file mode 120000
index 0000000..7dd95db
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/variables.eks.tf
@@ -0,0 +1 @@
+../variables.eks.tf
\ No newline at end of file
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/version.tf b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/version.tf
new file mode 120000
index 0000000..061373c
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/ebs/version.tf
@@ -0,0 +1 @@
+../version.tf
\ No newline at end of file
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/efs/.tf-control b/examples/full-cluster-tf-upgrade/1.23.in-progress/efs/.tf-control
new file mode 100644
index 0000000..280f449
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/efs/.tf-control
@@ -0,0 +1,20 @@
+# .tf-control
+# allows for setting a specific command to be used for tf-* commands under this git repo
+# see tf-control.sh help for more info
+
+TFCONTROL_VERSION="1.0.5"
+
+TFCOMMAND="terraform_latest"
+# TF_CLI_CONFIG_FILE=PATH-TO-FILE/.tf-control.tfrc
+# TFARGS=""
+# TFNOLOG=""
+# TFNOCOLOR=""
+
+# use the following to force a specific version.  An upgrade of an existing 0.12.31 to 1.x
+# needs you to cycle through 0.13.17, 0.14.11, and then latest (0.15.5 not needed).  Other
+# steps in between.  See https://github.e.it.census.gov/terraform/support/tree/master/docs/how-to/terraform-upgrade for details
+#
+#TFCOMMAND="terraform_0.12.31"
+#TFCOMMAND="terraform_0.13.7"
+#TFCOMMAND="terraform_0.14.11"
+#TFCOMMAND="terraform_0.15.5"
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/efs/.tf-control.tfrc b/examples/full-cluster-tf-upgrade/1.23.in-progress/efs/.tf-control.tfrc
new file mode 100644
index 0000000..7425488
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/efs/.tf-control.tfrc
@@ -0,0 +1,24 @@
+TFCONTROL_VERSION="1.0.5"
+
+# https://www.terraform.io/docs/cli/config/config-file.html
+plugin_cache_dir   = "/data/terraform/terraform.d/plugin-cache"
+#disable_checkpoint = true
+
+provider_installation {
+#  filesystem_mirror {
+#    path    = "/apps/terraform/terraform.d/providers"
+#    include = [ "*/*/*" ]
+#  }
+  filesystem_mirror {
+    path    = "/data/terraform/terraform.d/providers"
+    include = [ "*/*/*" ]
+  }
+#  filesystem_mirror {
+#    path    = "/apps/terraform/terraform.d/providers"
+#    include = [ "external.terraform.census.gov/*/*" ]
+#  }
+  direct {
+    include = [ "*/*/*" ]
+  }
+}
+
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/efs/tf-run.data b/examples/full-cluster-tf-upgrade/1.23.in-progress/efs/tf-run.data
index 68a4c31..bb04bbd 100644
--- a/examples/full-cluster-tf-upgrade/1.23.in-progress/efs/tf-run.data
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/efs/tf-run.data
@@ -1,9 +1,15 @@
-VERSION 1.1.2
+VERSION 1.2.0
 REMOTE-STATE
 COMMAND tf-directory-setup.py -l none -f
 COMMAND setup-new-directory.sh
 COMMAND tf-init -upgrade
 LINKTOP init
+LINKTOP includes.d/variables.account_tags.tf
+LINKTOP includes.d/variables.account_tags.auto.tfvars
+LINKTOP includes.d/variables.infrastructure_tags.tf
+LINKTOP includes.d/variables.infrastructure_tags.auto.tfvars
+LINKTOP includes.d/variables.application_tags.tf
+LINKTOP includes.d/variables.application_tags.auto.tfvars
 LINKTOP provider_configs.d/provider.ldap_new.auto.tfvars
 LINKTOP provider_configs.d/provider.ldap_new.tf
 LINKTOP provider_configs.d/provider.ldap_new.variables.tf
@@ -11,4 +17,4 @@ LINKTOP provider_configs.d/provider.ldap_new.variables.tf
 POLICY
 ALL
 COMMAND tf-directory-setup.py -l s3
-STOP cd ../irsa-roles and tf-run.sh apply
+STOP cd ../ebs and tf-run.sh apply
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/.tf-control b/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/.tf-control
new file mode 100644
index 0000000..280f449
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/.tf-control
@@ -0,0 +1,20 @@
+# .tf-control
+# allows for setting a specific command to be used for tf-* commands under this git repo
+# see tf-control.sh help for more info
+
+TFCONTROL_VERSION="1.0.5"
+
+TFCOMMAND="terraform_latest"
+# TF_CLI_CONFIG_FILE=PATH-TO-FILE/.tf-control.tfrc
+# TFARGS=""
+# TFNOLOG=""
+# TFNOCOLOR=""
+
+# use the following to force a specific version.  An upgrade of an existing 0.12.31 to 1.x
+# needs you to cycle through 0.13.17, 0.14.11, and then latest (0.15.5 not needed).  Other
+# steps in between.  See https://github.e.it.census.gov/terraform/support/tree/master/docs/how-to/terraform-upgrade for details
+#
+#TFCOMMAND="terraform_0.12.31"
+#TFCOMMAND="terraform_0.13.7"
+#TFCOMMAND="terraform_0.14.11"
+#TFCOMMAND="terraform_0.15.5"
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/.tf-control.tfrc b/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/.tf-control.tfrc
new file mode 100644
index 0000000..7425488
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/.tf-control.tfrc
@@ -0,0 +1,24 @@
+TFCONTROL_VERSION="1.0.5"
+
+# https://www.terraform.io/docs/cli/config/config-file.html
+plugin_cache_dir   = "/data/terraform/terraform.d/plugin-cache"
+#disable_checkpoint = true
+
+provider_installation {
+#  filesystem_mirror {
+#    path    = "/apps/terraform/terraform.d/providers"
+#    include = [ "*/*/*" ]
+#  }
+  filesystem_mirror {
+    path    = "/data/terraform/terraform.d/providers"
+    include = [ "*/*/*" ]
+  }
+#  filesystem_mirror {
+#    path    = "/apps/terraform/terraform.d/providers"
+#    include = [ "external.terraform.census.gov/*/*" ]
+#  }
+  direct {
+    include = [ "*/*/*" ]
+  }
+}
+
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/cluster-autoscaler/.tf-control b/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/cluster-autoscaler/.tf-control
new file mode 100644
index 0000000..280f449
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/cluster-autoscaler/.tf-control
@@ -0,0 +1,20 @@
+# .tf-control
+# allows for setting a specific command to be used for tf-* commands under this git repo
+# see tf-control.sh help for more info
+
+TFCONTROL_VERSION="1.0.5"
+
+TFCOMMAND="terraform_latest"
+# TF_CLI_CONFIG_FILE=PATH-TO-FILE/.tf-control.tfrc
+# TFARGS=""
+# TFNOLOG=""
+# TFNOCOLOR=""
+
+# use the following to force a specific version.  An upgrade of an existing 0.12.31 to 1.x
+# needs you to cycle through 0.13.17, 0.14.11, and then latest (0.15.5 not needed).  Other
+# steps in between.  See https://github.e.it.census.gov/terraform/support/tree/master/docs/how-to/terraform-upgrade for details
+#
+#TFCOMMAND="terraform_0.12.31"
+#TFCOMMAND="terraform_0.13.7"
+#TFCOMMAND="terraform_0.14.11"
+#TFCOMMAND="terraform_0.15.5"
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/cluster-autoscaler/.tf-control.tfrc b/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/cluster-autoscaler/.tf-control.tfrc
new file mode 100644
index 0000000..7425488
--- /dev/null
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/cluster-autoscaler/.tf-control.tfrc
@@ -0,0 +1,24 @@
+TFCONTROL_VERSION="1.0.5"
+
+# https://www.terraform.io/docs/cli/config/config-file.html
+plugin_cache_dir   = "/data/terraform/terraform.d/plugin-cache"
+#disable_checkpoint = true
+
+provider_installation {
+#  filesystem_mirror {
+#    path    = "/apps/terraform/terraform.d/providers"
+#    include = [ "*/*/*" ]
+#  }
+  filesystem_mirror {
+    path    = "/data/terraform/terraform.d/providers"
+    include = [ "*/*/*" ]
+  }
+#  filesystem_mirror {
+#    path    = "/apps/terraform/terraform.d/providers"
+#    include = [ "external.terraform.census.gov/*/*" ]
+#  }
+  direct {
+    include = [ "*/*/*" ]
+  }
+}
+
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/tf-run.data b/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/tf-run.data
index ddd8fff..2eed474 100644
--- a/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/tf-run.data
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/irsa-roles/tf-run.data
@@ -1,12 +1,14 @@
-VERSION 1.2.3
+VERSION 1.3.0
 REMOTE-STATE
 COMMAND tf-directory-setup.py -l none -f
 COMMAND setup-new-directory.sh
 COMMAND tf-init -upgrade
-COMMAND ln -sf ../versions.tf
-COMMAND ln -sf ../settings.auto.tfvars
-COMMAND ln -sf ../variables.application_tags.auto.tfvars
+
 LINKTOP init
+LINK versions.tf
+LINK settings.auto.tfvars
+LINK variables.application_tags.auto.tfvars
+
 ALL
 COMMAND tf-directory-setup.py -l s3
 
diff --git a/examples/full-cluster-tf-upgrade/1.23.in-progress/tf-run.data b/examples/full-cluster-tf-upgrade/1.23.in-progress/tf-run.data
index b8b8fa9..5aa7ded 100644
--- a/examples/full-cluster-tf-upgrade/1.23.in-progress/tf-run.data
+++ b/examples/full-cluster-tf-upgrade/1.23.in-progress/tf-run.data
@@ -1,4 +1,4 @@
-VERSION 1.3.3
+VERSION 1.4.0
 REMOTE-STATE
 COMMENT make sure the private-lb subnet and container subnets are tagged properly (see README.md)
 STOP then continue with at step %%NEXT%% (tag:subnets-verified)
@@ -6,10 +6,11 @@ STOP then continue with at step %%NEXT%% (tag:subnets-verified)
 TAG subnets-verified
 COMMAND tf-directory-setup.py -l none -f
 COMMAND setup-new-directory.sh
+LINKTOP init
+
 LINKTOP provider_configs.d/provider.ldap_new.auto.tfvars
 LINKTOP provider_configs.d/provider.ldap_new.tf
 LINKTOP provider_configs.d/provider.ldap_new.variables.tf
-LINKTOP init
 
 LINKTOP includes.d/variables.account_tags.tf
 LINKTOP includes.d/variables.account_tags.auto.tfvars
@@ -18,27 +19,22 @@ LINKTOP includes.d/variables.infrastructure_tags.auto.tfvars
 LINKTOP includes.d/variables.application_tags.tf
 LINKTOP includes.d/variables.application_tags.auto.tfvars
 
-COMMAND tf-init -upgrade
-
-COMMENT   There are two placeholder files, variables.vpc.auto.tfvars.make-link and variables.vpc.tf.make-link.
-COMMENT   Ensure that variables.vpc.auto.tfvars and variables.vpc.tf are either (a) linked to the files in the parent or vpc/{region}/vpcN/ directory
-COMMENT   or (b) copied from the vpc/{region}/vpcN/ directory if this repo is separate from the main cloud account
-COMMENT You may uncomment the next two lines in tf-run.data and apply them to make the links
-#COMMAND ln -sf ../../variables.vpc.tf
-#COMMAND ln -sf ../../variables.vpc.auto.tfvars
+LINK variables.vpc.tf
+LINK variables.vpc.auto.tfvars
 
-COMMENT Also check that the variables.application_tags .tf and .auto.tfvars files are linked to the proper includes.d/path/file from the root of the
-COMMENT  git repository.  You will get missing definitions on application_tags otherwise.
+COMMAND tf-init -upgrade
 
 STOP    check variables.vpc.* files and then continue with %%NEXT%% (tag:setup-complete)
 
 TAG setup-complete
 POLICY
 
+TAG ec2-key
 COMMENT EC2 key pairs
 null_resource.generate_keypair
 aws_key_pair.cluster_keypair
 COMMAND tf-directory-setup.py -l s3
+
 COMMENT be sure to add the setup/ec2-ssh-eks-{cluster} to git-secret, git-secret hide, add the setup/*secret and setup/*pub got git, and commit the entirety of the change
 
 ALL
@@ -52,6 +48,10 @@ STOP Once applied in this subdirectory, come back here and continue with step %%
 
 TAG setup-efs
 COMMENT cd efs and tf-run.sh apply
+STOP Once applied in this subdirectory, come back here and continue with step %%NEXT%% (tag:setup-ebs)
+
+TAG setup-ebs
+COMMENT cd ebs and tf-run.sh apply
 STOP Once applied in this subdirectory, come back here and continue with step %%NEXT%% (tag:setup-irsa)
 
 TAG setup-irsa