From 4b77f4709845fce364775c59679a23b885f4e547 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 12 Nov 2024 11:48:49 -0500 Subject: [PATCH] add generators --- .../cluster-tg/gen-eks-cert-mgr-tg.hcl | 48 ++++++++++ .../cluster-tg/gen-eks-config-tg.hcl | 49 ++++++++++ lab/_envcommon/cluster-tg/gen-eks-dns-tg.hcl | 51 +++++++++++ .../cluster-tg/gen-eks-grafana-tg.hcl | 49 ++++++++++ .../cluster-tg/gen-eks-istio-tg.hcl | 41 +++++++++ .../cluster-tg/gen-eks-k8s-dash-tg.hcl | 45 ++++++++++ .../cluster-tg/gen-eks-karpenter-tg.hcl | 51 +++++++++++ .../cluster-tg/gen-eks-kiali-tg.hcl | 90 +++++++++++++++++++ lab/_envcommon/cluster-tg/gen-eks-loki-tg.hcl | 55 ++++++++++++ .../cluster-tg/gen-eks-metrics-tg.hcl | 42 +++++++++ lab/_envcommon/cluster-tg/gen-eks-prom-tg.hcl | 47 ++++++++++ lab/_envcommon/cluster-tg/gen-eks-tempo.hcl | 54 +++++++++++ lab/_envcommon/cluster-tg/gen-eks-tg.hcl | 65 ++++++++++++++ lab/_envcommon/gen-account.hcl | 21 +++++ lab/_envcommon/gen-cluster.hcl | 17 ++++ lab/_envcommon/gen-region.hcl | 14 +++ lab/_envcommon/gen-root.hcl | 80 +++++++++++++++++ lab/_envcommon/gen-vpc.hcl | 15 ++++ .../vpc/platform-eng-eks-mcm/cluster.hcl | 22 +---- mcmcluster.hcl | 28 +++--- 20 files changed, 848 insertions(+), 36 deletions(-) create mode 100644 lab/_envcommon/cluster-tg/gen-eks-cert-mgr-tg.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-config-tg.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-dns-tg.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-grafana-tg.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-istio-tg.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-k8s-dash-tg.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-karpenter-tg.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-kiali-tg.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-loki-tg.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-metrics-tg.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-prom-tg.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-tempo.hcl create mode 100644 lab/_envcommon/cluster-tg/gen-eks-tg.hcl create mode 100644 lab/_envcommon/gen-account.hcl create mode 100644 lab/_envcommon/gen-cluster.hcl create mode 100644 lab/_envcommon/gen-region.hcl create mode 100644 lab/_envcommon/gen-root.hcl create mode 100644 lab/_envcommon/gen-vpc.hcl diff --git a/lab/_envcommon/cluster-tg/gen-eks-cert-mgr-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-cert-mgr-tg.hcl new file mode 100644 index 0000000..29e7295 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-cert-mgr-tg.hcl @@ -0,0 +1,48 @@ +# lab/_envcommon/cluster-tg/gen-eks-cert-mgr-tg.hcl + +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-cert-manager/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-cert-mgr.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + cluster_name = "a-cluster-name" + oidc_provider_arn = "arn:aws-us-gov:iam::111111111111:oidc-provider/oidc.eks.us-gov-east-1.amazonaws.com/id/0000000000000000AAAAAAAAAAAAAAAA" + } + } + + dependency "eks_config" { + config_path = "../eks-config" + skip_outputs = true + } + + inputs = { + cluster_name = dependency.eks.outputs.cluster_name + cluster_mailing_list = dependency.eks.inputs.creator + oidc_provider_arn = dependency.eks.outputs.oidc_provider_arn + profile = include.root.inputs.aws_profile + region = include.root.inputs.aws_region + cert_manager_helm_chart = include.root.inputs.cert_manager_helm_chart + cert_manager_cainjector_tag = include.root.inputs.cert_manager_cainjector_tag + cert_manager_controller_tag = include.root.inputs.cert_manager_controller_tag + cert_manager_startupapicheck_tag = include.root.inputs.cert_manager_startupapicheck_tag + cert_manager_webhook_tag = include.root.inputs.cert_manager_webhook_tag + cluster_issuer_name = include.root.inputs.cluster_issuer_name + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-config-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-config-tg.hcl new file mode 100644 index 0000000..7ef692f --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-config-tg.hcl @@ -0,0 +1,49 @@ +# lab/_envcommon/cluster-tg/gen-eks-config-tg.hcl + +# Generate the terragrunt.hcl for eks-configuration +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-configuration/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-eks-configuration.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + cluster_certificate_authority_data = [{ data = "THISISAVERYLONGCERTSTRINGTHATGOESHEREFORSURENODYEP" }] + cluster_endpoint = "https://12345ABCDEE42BF9C24D4C362D1DC.sk1.us-gov-east-1.eks.amazonaws.com" + cluster_name = "a-cluster-name" + eks_managed_node_groups_autoscaling_group_names = ["eks-eks-a-cluster-name-node_group-0000000000000000000000000-5ac8a5e3-14dd-c043-2cc9-f4b6ffb36d32"] + oidc_provider_arn = "arn:aws-us-gov:iam::111111111111:oidc-provider/oidc.eks.us-gov-east-1.amazonaws.com/id/0000000000000000AAAAAAAAAAAAAAAA" + security_group_all_worker_mgmt_id = "sg-00b0000000000000" + subnets = ["subnet-00000000000000001", "subnet-00000000000000002", "subnet-00000000000000003"] + token = [{ token = "THISISTHETOKENTHATDOESNTEXISTTHEREAREMANYLIKEITBUTHISONEISFORACLUSTER" }] + vpc_id = "a-vpc-id" + } + } + + inputs = { + profile = include.root.inputs.aws_profile + region = include.root.inputs.aws_region + vpc_id = dependency.eks.outputs.vpc_id + cluster_name = dependency.eks.outputs.cluster_name + subnets = dependency.eks.outputs.subnets + security_group_all_worker_mgmt_id = dependency.eks.outputs.security_group_all_worker_mgmt_id + eks_managed_node_groups_autoscaling_group_names = dependency.eks.outputs.eks_managed_node_groups_autoscaling_group_names + oidc_provider_arn = dependency.eks.outputs.oidc_provider_arn + kubectl_image_tag = include.root.inputs.kubectl_image_tag + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-dns-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-dns-tg.hcl new file mode 100644 index 0000000..511dcd0 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-dns-tg.hcl @@ -0,0 +1,51 @@ +# lab/_envcommon/cluster-tg/gen-eks-dns-tg.hcl + +# Generate the eks-dns terragrunt.hcl +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-dns/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-eks-dns.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + subnets = ["subnet-abcdefgh", "subnet-12345678", "subnet-ab12cd34"] + } + } + + dependency "istio" { + config_path = "../eks-istio" + mock_outputs = { + istio_ingress_lb = { + dns_name = "a1111111111111111111111111111111-2bbbbbbbbbbbbbbb.elb.us-gov-east-1.amazonaws.com" + zone_id = "ZABC123456DEF" + } + } + } + + inputs = { + cluster_name = dependency.eks.inputs.cluster_name + istio_ingress_lb = dependency.istio.outputs.istio_ingress_lb + profile = include.root.inputs.aws_profile + region = include.root.inputs.aws_region + subnets = dependency.eks.outputs.subnets + tags = dependency.eks.inputs.tags + vpc_domain_name = dependency.eks.inputs.vpc_domain_name + vpc_name = dependency.eks.inputs.vpc_name + route53_endpoints = include.root.inputs.route53_endpoints + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-grafana-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-grafana-tg.hcl new file mode 100644 index 0000000..dc5bb42 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-grafana-tg.hcl @@ -0,0 +1,49 @@ +# lab/_envcommon/cluster-tg/gen-eks-grafana-tg.hcl + +# Generate the eks-grafana terragrunt.hcl +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-grafana/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-grafana.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + cluster_name = "a-cluster-name" + } + } + + dependency "eks-loki" { + config_path = "../eks-loki" + mock_outputs = { + rwo_storage_class = "gp3-encrypted" + } + } + + inputs = { + profile = include.root.inputs.aws_profile + region = include.root.inputs.aws_region + cluster_name = dependency.eks.outputs.cluster_name + cluster_domain = dependency.eks.inputs.vpc_domain_name + public_hostname = include.root.inputs.grafana_hostname + rwo_storage_class = dependency.eks-loki.outputs.rwo_storage_class + grafana_chart_version = include.root.inputs.grafana_chart_version + grafana_tag = include.root.inputs.grafana_tag + download_dashboards_image_tag = include.root.inputs.download_dashboards_image_tag + init_chown_data_image_tag = include.root.inputs.init_chown_data_image_tag + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-istio-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-istio-tg.hcl new file mode 100644 index 0000000..7561561 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-istio-tg.hcl @@ -0,0 +1,41 @@ +# lab/_envcommon/cluster-tg/gen-eks-istio-tg.hcl + +# Generate the eks-istio terragrunt.hcl +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-istio/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-istio.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + cluster_name = "a-cluster-name" + } + } + dependency "eks-karpenter" { + config_path = "../eks-karpenter" + skip_outputs = true + } + + inputs = { + profile = include.root.inputs.aws_profile + region = include.root.inputs.aws_region + cluster_name = dependency.eks.outputs.cluster_name + istio_chart_version = include.root.inputs.istio_version + istio_version = include.root.inputs.istio_version + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-k8s-dash-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-k8s-dash-tg.hcl new file mode 100644 index 0000000..398eba6 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-k8s-dash-tg.hcl @@ -0,0 +1,45 @@ +# lab/_envcommon/cluster-tg/gen-eks-k8s-dash-tg.hcl + +# Generate the eks-k8s-dashboard terragrunt.hcl +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-k8s-dashboard/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-k8s-dashboard.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + cluster_name = "a-cluster-name" + vpc_domain_name = "example.com" + } + } + + dependency "eks-loki" { + config_path = "../eks-loki" + skip_outputs = true + } + + inputs = { + profile = include.root.inputs.aws_profile + region = include.root.inputs.aws_region + cluster_name = dependency.eks.outputs.cluster_name + cluster_domain = dependency.eks.inputs.vpc_domain_name + public_hostname = include.root.inputs.dashboard_hostname + k8s_dashboard_version = include.root.inputs.k8s_dashboard_version + # datasources = dependency.eks-loki.outputs.gateway_internal_endpoint + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-karpenter-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-karpenter-tg.hcl new file mode 100644 index 0000000..864ee01 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-karpenter-tg.hcl @@ -0,0 +1,51 @@ +# lab/_envcommon/cluster-tg/gen-eks-karpenter-tg.hcl + +# Generate the eks-karpenter terragrunt.hcl +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-karpenter/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-karpenter.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + cluster_endpoint = "https://0000000000000000AAAAAAAAAAAAAAAA.sk1.us-gov-east-1.eks.amazonaws.com" + cluster_name = "a-cluster-name" + node_group_name = "node_group_a-cluster-name" + oidc_provider_arn = "arn:aws-us-gov:iam::111111111111:oidc-provider/oidc.eks.us-gov-east-1.amazonaws.com/id/0000000000000000AAAAAAAAAAAAAAAA" + vpc_id = "a-vpc-name" + } + } + + dependency "eks-config" { + config_path = "../eks-config" + skip_outputs = true + } + + inputs = { + profile = include.root.inputs.aws_profile + region = include.root.inputs.aws_region + cluster_endpoint = dependency.eks.outputs.cluster_endpoint + cluster_name = dependency.eks.outputs.cluster_name + karpenter_node_group_name = dependency.eks.outputs.node_group_name + oidc_provider_arn = dependency.eks.outputs.oidc_provider_arn + vpc_id = dependency.eks.outputs.vpc_id + karpenter_helm_chart = include.root.inputs.karpenter_helm_chart + karpenter_tag = include.root.inputs.karpenter_tag + kubectl_tag = include.root.inputs.kubectl_image_tag + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-kiali-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-kiali-tg.hcl new file mode 100644 index 0000000..e0c1fe0 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-kiali-tg.hcl @@ -0,0 +1,90 @@ +# lab/_envcommon/cluster-tg/gen-eks-kiali-tg.hcl + +# Generate the eks-kiali terragrunt.hcl +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-kiali/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-kiali.git?ref=${include.root.inputs.release_version}" + # source = "../../../../../../../tfmod-kiali" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + cluster_name = "a-cluster-name" + } + } + + dependency "eks-cert-manager" { + config_path = "../eks-cert-manager" + mock_outputs = { + cluster_issuer_name = "acmpca-clusterissuer" + } + } + + dependency "eks-prometheus" { + config_path = "../eks-prometheus" + mock_outputs = { + prometheus_server_internal_endpoint = { + hostname = "prometheus-server.prometheus.svc.cluster.local" + port_number = 9090 + url = "http://prometheus-server.prometheus.svc.cluster.local:9090/" + } + } + } + + dependency "eks-grafana" { + config_path = "../eks-grafana" + mock_outputs = { + internal_endpoint = { + hostname = "grafana.grafana.svc.cluster.local" + port_number = "80" + url = "https://grafana.grafana.svc.cluster.local:80/" + } + namespace = "grafana" + public_endpoint = { + hostname = "grafana.dev.lab.csp2.census.gov" + port_number = "80" + url = "https://grafana.dev.lab.csp2.census.gov:80/" + } + secret_name = "grafana" + } + } + + inputs = { + kiali_operator_version = include.root.inputs.kiali_operator_version + kiali_application_version = include.root.inputs.kiali_application_version + profile = include.root.inputs.aws_profile + cluster_domain = dependency.eks.inputs.vpc_domain_name + operators_namespace = "operators" + cluster_name = dependency.eks.outputs.cluster_name + certificate_issuer = dependency.eks-cert-manager.outputs.cluster_issuer_name + prometheus_internal_url = dependency.eks-prometheus.outputs.prometheus_server_internal_endpoint.url + grafana_internal_url = dependency.eks-grafana.outputs.internal_endpoint.url + grafana_namespace = dependency.eks-grafana.outputs.namespace + grafana_public_url = dependency.eks-grafana.outputs.public_endpoint.url + grafana_secret_name = "grafana" + # grafana_secret_name = dependency.eks-grafana.outputs.secret_name + jaeger_internal_url = "" + # client_id = var.sso_client_id + # client_secret = var.sso_client_secret + # keycloak_public_url = var.keycloak_public_url + # gogatekeeper_chart_version = var.gogatekeeper_chart_version + # gogatekeeper_registry = var.gogatekeeper_registry + # gogatekeeper_repository = var.gogatekeeper_repository + # gogatekeeper_tag = var.gogatekeeper_tag + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-loki-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-loki-tg.hcl new file mode 100644 index 0000000..768def4 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-loki-tg.hcl @@ -0,0 +1,55 @@ +# lab/_envcommon/cluster-tg/gen-eks-loki-tg.hcl + +# Generate the eks-loki terragrunt.hcl +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-loki/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-loki.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + cluster_name = "a-cluster-name" + oidc_provider_arn = "arn:aws-us-gov:iam::111111111111:oidc-provider/oidc.eks.us-gov-east-1.amazonaws.com/id/0000000000000000AAAAAAAAAAAAAAAA" + } + } + + dependency "eks-istio" { + config_path = "../eks-istio" + skip_outputs = true + } + + dependency "eks-prometheus" { + config_path = "../eks-prometheus" + skip_outputs = true + } + + inputs = { + profile = include.root.inputs.aws_profile + region = include.root.inputs.aws_region + cluster_name = dependency.eks.outputs.cluster_name + oidc_provider_arn = dependency.eks.outputs.oidc_provider_arn + loki_chart_version = include.root.inputs.loki_chart_version + loki_tag = include.root.inputs.loki_tag + canary_tag = include.root.inputs.canary_tag + enterprise_logs_provisioner_tag = include.root.inputs.enterprise_logs_provisioner_tag + gateway_tag = include.root.inputs.gateway_tag + memcached_tag = include.root.inputs.memcached_tag + exporter_tag = include.root.inputs.exporter_tag + sidecar_tag = include.root.inputs.sidecar_tag + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-metrics-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-metrics-tg.hcl new file mode 100644 index 0000000..eacb3a8 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-metrics-tg.hcl @@ -0,0 +1,42 @@ +# lab/_envcommon/cluster-tg/gen-eks-metrics-tg.hcl + +# Generate the eks-metrics-server terragrunt.hcl +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-metrics-server/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-metrics-server.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + cluster_name = "a-cluster-name" + } + } + + dependency "eks_config" { + config_path = "../eks-config" + skip_outputs = true + } + + inputs = { + profile = include.root.inputs.aws_profile + cluster_name = dependency.eks.outputs.cluster_name + region = include.root.inputs.aws_region + metrics_server_helm_chart = include.root.inputs.metrics_server_helm_chart + metrics_server_tag = include.root.inputs.metrics_server_tag + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-prom-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-prom-tg.hcl new file mode 100644 index 0000000..fb72ba0 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-prom-tg.hcl @@ -0,0 +1,47 @@ +# lab/_envcommon/cluster-tg/gen-eks-prom-tg.hcl + +# Generate the eks-prom terragrunt.hcl +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-prometheus/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-prometheus.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + cluster_name = "a-cluster-name" + } + } + + dependency "eks-dns" { + config_path = "../eks-dns" + skip_outputs = true + } + + inputs = { + profile = include.root.inputs.aws_profile + region = include.root.inputs.aws_region + cluster_name = dependency.eks.outputs.cluster_name + prometheus_chart_version = include.root.inputs.prometheus_chart_version + prometheus_server_tag = include.root.inputs.prometheus_server_tag + prometheus_config_reloader_tag = include.root.inputs.prometheus_config_reloader_tag + alertmanager_tag = include.root.inputs.alertmanager_tag + kube_state_metrics_tag = include.root.inputs.kube_state_metrics_tag + node_exporter_tag = include.root.inputs.node_exporter_tag + pushgateway_tag = include.root.inputs.pushgateway_tag + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-tempo.hcl b/lab/_envcommon/cluster-tg/gen-eks-tempo.hcl new file mode 100644 index 0000000..71f2d73 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-tempo.hcl @@ -0,0 +1,54 @@ +# lab/_envcommon/cluster-tg/gen-eks-tempo-tg.hcl + +# Generate the eks-tempo terragrunt.hcl +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks-tempo/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-tempo.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + dependency "eks" { + config_path = "../eks" + mock_outputs = { + cluster_name = "a-cluster-name" + oidc_provider_arn = "arn:aws-us-gov:iam::111111111111:oidc-provider/oidc.eks.us-gov-east-1.amazonaws.com/id/0000000000000000AAAAAAAAAAAAAAAA" + } + } + + dependency "eks-prometheus" { + config_path = "../eks-prometheus" + mock_outputs = { + prometheus_server_internal_endpoint = { + hostname = "prometheus-server.prometheus.svc.cluster.local" + port_number = 9090 + url = "http://prometheus-server.prometheus.svc.cluster.local:9090/" + } + prometheus_namespace = "prometheus" + } + } + + inputs = { + account_id = include.root.locals.account_id + profile = include.root.locals.aws_profile + region = include.root.locals.aws_region + cluster_name = dependency.eks.outputs.cluster_name + oidc_provider_arn = dependency.eks.outputs.oidc_provider_arn + prometheus_port = dependency.eks-prometheus.outputs.prometheus_server_internal_endpoint.port_number + prometheus_namespace = dependency.eks-prometheus.outputs.prometheus_namespace + tempo_chart_version = include.root.inputs.tempo_chart_version + tempo_tag = include.root.inputs.tempo_tag + } +EOF +} diff --git a/lab/_envcommon/cluster-tg/gen-eks-tg.hcl b/lab/_envcommon/cluster-tg/gen-eks-tg.hcl new file mode 100644 index 0000000..2fcfb57 --- /dev/null +++ b/lab/_envcommon/cluster-tg/gen-eks-tg.hcl @@ -0,0 +1,65 @@ +# lab/_envcommon/cluster-tg/gen-eks-tg.hcl + +# Generate the eks cluster terragrunt.hcl +generate "terragrunt.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/eks/terragrunt.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + include "root" { + path = find_in_parent_folders("root.hcl") + merge_strategy = "deep" + expose = true + } + + locals { + # Set cluster/platform specific variables, or extract from the hierarchy. + account_id = include.root.inputs.aws_account_id + cluster_endpoint_public_access = include.root.inputs.cluster_endpoint_public_access + cluster_name = include.root.inputs.cluster_name + cluster_version = include.root.inputs.cluster_version + creator = include.root.inputs.creator + eks_instance_disk_size = include.root.inputs.eks_instance_disk_size + eks_ng_desired_size = include.root.inputs.eks_ng_desired_size + eks_ng_max_size = include.root.inputs.eks_ng_max_size + eks_ng_min_size = include.root.inputs.eks_ng_min_size + eks_vpc_name = include.root.inputs.vpc_name + enable_cluster_creator_admin_permissions = include.root.inputs.enable_cluster_creator_admin_permissions + environment_abbr = include.root.inputs.environment_abbr + organization = include.root.inputs.organization + profile = include.root.inputs.aws_profile + project_name = include.root.inputs.project_name + project_number = include.root.inputs.project_number + project_role = include.root.inputs.project_role + region = include.root.inputs.aws_region + tags = include.root.inputs.tags + terraform = include.root.inputs.terraform + terragrunt = include.root.inputs.terragrunt + vpc_domain_name = include.root.inputs.vpc_domain_name + } + + terraform { + source = "git@github.e.it.census.gov:SCT-Engineering/tfmod-eks.git?ref=${include.root.inputs.release_version}" + extra_arguments "retry_lock" { + commands = get_terraform_commands_that_need_locking() + arguments = ["-lock-timeout=20m"] + } + } + + inputs = { + aws_account_id = local.account_id + cluster_endpoint_public_access = local.cluster_endpoint_public_access + cluster_name = local.cluster_name + cluster_version = local.cluster_version + creator = local.creator + eks_instance_disk_size = local.eks_instance_disk_size + eks_ng_desired_size = local.eks_ng_desired_size + eks_ng_max_size = local.eks_ng_max_size + eks_ng_min_size = local.eks_ng_min_size + eks_vpc_name = local.eks_vpc_name + enable_cluster_creator_admin_permissions = local.enable_cluster_creator_admin_permissions + os_username = local.creator + shared_vpc_label = local.environment_abbr + tags = local.tags + } +EOF +} diff --git a/lab/_envcommon/gen-account.hcl b/lab/_envcommon/gen-account.hcl new file mode 100644 index 0000000..7793bf7 --- /dev/null +++ b/lab/_envcommon/gen-account.hcl @@ -0,0 +1,21 @@ + +# Generate a helm provider block +generate "account.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/account.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + # lab/development/account.hcl + + # Set account-wide variables. These are automatically pulled in to configure the remote state bucket in the root + # terragrunt.hcl configuration. Terragrunt often segments account and environment, but given our strategy is to + # leverage accounts as environment boundaries, there is an anticipated 1:1 account to environment model that + # combines these here. + locals { + account_name = "${include.root.inputs.account_name}" + aws_account_id = "${include.root.inputs.account_id}" + aws_profile = "${include.root.inputs.profile}" + environment = "${include.root.inputs.environemnt}" + environment_abbr = "${include.root.inputs.environment_abbr}" + } +EOF +} diff --git a/lab/_envcommon/gen-cluster.hcl b/lab/_envcommon/gen-cluster.hcl new file mode 100644 index 0000000..ade909c --- /dev/null +++ b/lab/_envcommon/gen-cluster.hcl @@ -0,0 +1,17 @@ +# Generate a helm provider block +generate "cluster.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/${include.root.inputs.cluster_name}/cluster.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + # lab/development/us-gov-east-1/vpc/platform-eng-eks-mcm/cluster.hcl + + # Set cluster specific variables. These are automatically pulled in to configure the remote state bucket in the root + # terragrunt.hcl configuration. + locals { + cluster_name = "platform-eng-eks-mcm" + creator = "matthew.c.morgan@census.gov" + # "slim:schedule" = "08:00-17:00" + # "cluster:size" = "min:${include.root.inputs.eks_ng_min_size}-max:${local.eks_ng_max_size}-desired:${local.eks_ng_desired_size}" + } +EOF +} diff --git a/lab/_envcommon/gen-region.hcl b/lab/_envcommon/gen-region.hcl new file mode 100644 index 0000000..0459a39 --- /dev/null +++ b/lab/_envcommon/gen-region.hcl @@ -0,0 +1,14 @@ +# Generate a helm provider block +generate "region.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/region.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + # lab/development/us-gov-east-1/region.hcl + + # Set common variables for the region. This is automatically pulled in in the root terragrunt.hcl configuration to + # configure the remote state bucket and pass forward to the child modules as inputs. + locals { + aws_region = "${include.root.inputs.region}" + } +EOF +} diff --git a/lab/_envcommon/gen-root.hcl b/lab/_envcommon/gen-root.hcl new file mode 100644 index 0000000..adb7f05 --- /dev/null +++ b/lab/_envcommon/gen-root.hcl @@ -0,0 +1,80 @@ +# Generate a helm provider block +generate "root.hcl" { + path = "${get_original_terragrunt_dir()}/root.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + # lab/root.hcl + + # --------------------------------------------------------------------------------------------------------------------- + # TERRAGRUNT CONFIGURATION + # Terragrunt is a thin wrapper for Terraform that provides extra tools for working with multiple Terraform modules, + # remote state, and locking: https://github.com/gruntwork-io/terragrunt + # --------------------------------------------------------------------------------------------------------------------- + locals { + # Automatically load account-level variables (NOTE: In our environment account = environment so there is not separate environment layer) + account_vars = read_terragrunt_config(find_in_parent_folders("account.hcl")) + + # Automatically load _envcommon, cross account and environment common variables + common_vars = read_terragrunt_config(find_in_parent_folders("./_envcommon/common-variables.hcl")) + + # Automatically load cluster-level variables + cluster_vars = read_terragrunt_config(find_in_parent_folders("cluster.hcl")) + + # Automatically load region-level variables + region_vars = read_terragrunt_config(find_in_parent_folders("region.hcl")) + + # Automatically load versions + versions = read_terragrunt_config(find_in_parent_folders("./_envcommon/default-versions.hcl")) + + # Automatically load vpc-level variables + vpc_vars = read_terragrunt_config(find_in_parent_folders("vpc.hcl")) + + # Extract the variables we need for easy access + account_id = local.account_vars.locals.aws_account_id + aws_profile = local.account_vars.locals.aws_profile + aws_region = local.region_vars.locals.aws_region + state_bucket_prefix = local.common_vars.locals.state_bucket_prefix + state_table_name = local.common_vars.locals.state_table_name + } + + # Configure Terragrunt to automatically store tfstate files in an S3 bucket + remote_state { + backend = "s3" + generate = { + path = "remote_state.backend.tf" + if_exists = "overwrite_terragrunt" + } + config = { + bucket = ${local.state_bucket_prefix}-${local.account_id} + dynamodb_table = ${local.state_table_name} + key = ${trimprefix(replace(run_cmd("realpath", get_original_terragrunt_dir()), dirname(get_repo_root()), ""), "/")}/terraform.tfstate + profile = ${local.aws_profile} + region = ${local.aws_region} + disable_bucket_update = true + skip_bucket_enforced_tls = true # use only if you need to access the S3 bucket without TLS being enforced + skip_bucket_public_access_blocking = true + skip_bucket_root_access = true # use only if the AWS account root user should not have access to the remote state bucket for some reason + skip_bucket_ssencryption = true # use only if non-encrypted OpenTofu/Terraform State is required and/or the object store does not support server-side encryption + skip_bucket_versioning = false # use only if the object store does not support versioning + enable_lock_table_ssencryption = false # use only if non-encrypted DynamoDB Lock Table for the OpenTofu/Terraform State is required and/or the NoSQL database service does not support server-side encryption + } + } + + # --------------------------------------------------------------------------------------------------------------------- + # GLOBAL PARAMETERS + # These variables apply to all configurations in this subfolder. These are automatically merged into the child + # `terragrunt.hcl` config via the include block. + # --------------------------------------------------------------------------------------------------------------------- + + # Configure root level variables that all resources can inherit. This is especially helpful with multi-account configs + # where terraform_remote_state data sources are placed directly into the modules. + inputs = merge( + local.account_vars.locals, + local.cluster_vars.locals, + local.common_vars.locals, + local.region_vars.locals, + local.versions.locals, + local.vpc_vars.locals + ) +EOF +} diff --git a/lab/_envcommon/gen-vpc.hcl b/lab/_envcommon/gen-vpc.hcl new file mode 100644 index 0000000..d03bd29 --- /dev/null +++ b/lab/_envcommon/gen-vpc.hcl @@ -0,0 +1,15 @@ +# Generate a helm provider block +generate "vpc.hcl" { + path = "${get_original_terragrunt_dir()}/${include.root.inputs.environment}/${include.root.inputs.region}/vpc/vpc.hcl" + if_exists = "overwrite_terragrunt" + contents = <<-EOF + # lab/development/us-gov-east-1/vpc/vpc.hcl + + # Set VPC specific variables. These are automatically pulled in to configure the remote state bucket in the root + # terragrunt.hcl configuration. + locals { + vpc_name = "${include.root.inputs.vpc_name}" + vpc_domain_name = "${include.root.inputs.vpc_domain_name}" + } +EOF +} diff --git a/lab/development/us-gov-east-1/vpc/platform-eng-eks-mcm/cluster.hcl b/lab/development/us-gov-east-1/vpc/platform-eng-eks-mcm/cluster.hcl index 98d12d7..54fb158 100644 --- a/lab/development/us-gov-east-1/vpc/platform-eng-eks-mcm/cluster.hcl +++ b/lab/development/us-gov-east-1/vpc/platform-eng-eks-mcm/cluster.hcl @@ -3,22 +3,8 @@ # Set cluster specific variables. These are automatically pulled in to configure the remote state bucket in the root # terragrunt.hcl configuration. locals { - cluster_endpoint_public_access = true - cluster_name = "platform-eng-eks-mcm" - creator = "matthew.c.morgan@census.gov" - eks_instance_disk_size = 100 - eks_ng_desired_size = 2 - eks_ng_max_size = 10 - eks_ng_min_size = 0 - enable_cluster_creator_admin_permissions = true - terraform = true - terragrunt = true - tags = { - "slim:schedule" = "8:00-17:00" - "cluster:size" = "min:${local.eks_ng_min_size}-max:${local.eks_ng_max_size}-desired:${local.eks_ng_desired_size}" - } - eks_version = "0.1.1" - eks_enabled = true - - + cluster_name = "platform-eng-eks-mcm" + creator = "matthew.c.morgan@census.gov" + # "slim:schedule" = "08:00-17:00" + # "cluster:size" = "min:${include.root.inputs.eks_ng_min_size}-max:${local.eks_ng_max_size}-desired:${local.eks_ng_desired_size}" } diff --git a/mcmcluster.hcl b/mcmcluster.hcl index b443ab7..5cbe085 100644 --- a/mcmcluster.hcl +++ b/mcmcluster.hcl @@ -1,22 +1,14 @@ locals { - account_name = "lab-dev-ew" - aws_account_id = "224384469011" - aws_profile = "224384469011-lab-dev-gov" - aws_region = "us-gov-east-1" - cluster_endpoint_public_access = true - cluster_name = "platform-eng-eks-mcm" - creator = "matthew.c.morgan@census.gov" - eks_instance_disk_size = 100 - eks_ng_desired_size = 2 - eks_ng_max_size = 10 - eks_ng_min_size = 0 - enable_cluster_creator_admin_permissions = true - environment = "development" - environment_abbr = "dev" - terraform = true - terragrunt = true - vpc_domain_name = "dev.lab.csp2.census.gov" - vpc_name = "vpc3-lab-dev" + account_name = "lab-dev-ew" + aws_account_id = "224384469011" + aws_profile = "224384469011-lab-dev-gov" + aws_region = "us-gov-east-1" + cluster_name = "platform-eng-eks-mcm" + creator = "matthew.c.morgan@census.gov" + environment = "development" + environment_abbr = "dev" + vpc_domain_name = "dev.lab.csp2.census.gov" + vpc_name = "vpc3-lab-dev" tags = { "slim:schedule" = "8:00-17:00" "cluster:size" = "min:${local.eks_ng_min_size}-max:${local.eks_ng_max_size}-desired:${local.eks_ng_desired_size}"