From 4f16eff228ed5464b09ae9e8218e0bcca2cc7537 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 3 Jul 2024 20:31:45 -0400 Subject: [PATCH 1/8] upgrade loki to 3.0.0 --- README.md | 1 + copy_images.tf | 41 +++++++++- main.tf | 215 +++++++++++++++++++++++-------------------------- variables.tf | 76 +++++++++++------ 4 files changed, 194 insertions(+), 139 deletions(-) diff --git a/README.md b/README.md index 98d7d5a..1fed409 100644 --- a/README.md +++ b/README.md @@ -3,5 +3,6 @@ Installs the loki as the log aggregation sink, and promtail to forward the logs to loki. +* Requires additional Node HD space - 40GB is not enough. # tfmod-loki diff --git a/copy_images.tf b/copy_images.tf index d26e301..e35264a 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -1,7 +1,11 @@ locals { loki_key = format("%v#%v", "grafana/loki", var.loki_tag) + canary_key = format("%v#%v", "grafana/loki-canary", var.canary_tag) provisioner_key = format("%v#%v", "grafana/enterprise-logs-provisioner", var.enterprise_logs_provisioner_tag) gateway_key = format("%v#%v", "grafana/nginx-unprivileged", var.gateway_tag) + sidecar_key = format("%v#%v", "kiwigrid/k8s-sidecar", var.sidecar_tag) + memcached_key = format("%v#%v", "memcached", var.memcached_tag) + exporter_key = format("%v#%v", "prom/memcached-exporter", var.exporter_tag) image_config = [ { @@ -13,6 +17,42 @@ locals { source_tag = var.loki_tag tag = var.loki_tag }, + { + enabled = true + dest_path = null + name = "grafana/loki-canary" + source_image = "grafana/loki-canary" + source_registry = "docker.io" + source_tag = var.canary_tag + tag = var.canary_tag + }, + { + enabled = true + dest_path = null + name = "memcached" + source_image = "memcached" + source_registry = "docker.io" + source_tag = var.memcached_tag + tag = var.memcached_tag + }, + { + enabled = true + dest_path = null + name = "prom/memcached-exporter" + source_image = "prom/memcached-exporter" + source_registry = "docker.io" + source_tag = var.exporter_tag + tag = var.exporter_tag + }, + { + enabled = true + dest_path = null + name = "kiwigrid/k8s-sidecar" + source_image = "kiwigrid/k8s-sidecar" + source_registry = "quay.io" + source_tag = var.sidecar_tag + tag = var.sidecar_tag + }, { enabled = true dest_path = null @@ -52,4 +92,3 @@ module "images" { ## source_password = "" ## source_username = "" } - diff --git a/main.tf b/main.tf index 2b58b47..5e4f549 100644 --- a/main.tf +++ b/main.tf @@ -68,72 +68,97 @@ resource "helm_release" "loki" { namespace = local.ns repository = "https://grafana.github.io/helm-charts" + ##### Globals ##### set { - name = "kubectlImage.registry" - value = var.kubectl_image_registry + name = "global.image.registry" + value = module.images.images[local.loki_key].dest_registry } + + ##### Loki Image ##### set { - name = "kubectlImage.repository" - value = var.kubectl_image_repository + name = "loki.image.repository" + value = module.images.images[local.loki_key].dest_repository } set { - name = "kubectlImage.tag" - value = var.kubectl_image_tag + name = "loki.image.tag" + value = module.images.images[local.loki_key].tag } + ##### Provisioner (nginx) Image ##### set { - name = "loki.image.registry" - value = module.images.images[local.loki_key].dest_registry + name = "loki.provisioner.image.repository" + value = split(":", module.images.images[local.provisioner_key].dest_full_path)[0] } set { - name = "loki.image.repository" - value = module.images.images[local.loki_key].dest_repository + name = "loki.provisioner.image.tag" + value = module.images.images[local.provisioner_key].tag } + + #### Gateway Image ##### set { - name = "loki.image.tag" - value = module.images.images[local.loki_key].tag + name = "gateway.image.repository" + value = module.images.images[local.provisioner_key].dest_repository + } + set { + name = "gateway.image.tag" + value = module.images.images[local.provisioner_key].tag } - + ##### grafana/loki-canary ##### set { - name = "table_manager.retention_deletes_enabled" - value = var.table_manager_retention_deletes_enabled + name = "lokiCanary.image.repository" + value = module.images.images[local.canary_key].dest_repository } set { - name = "table_manager.retention_period" - value = var.table_manager_retention_period + name = "lokiCanary.image.tag" + value = module.images.images[local.canary_key].tag } + ##### kiwigrid/k8s-sidecar ##### set { - name = "loki.auth_enabled" - value = "true" + name = "sidecar.image.repository" + value = split(":", module.images.images[local.sidecar_key].dest_full_path)[0] + } + set { + name = "sidecar.image.tag" + value = module.images.images[local.sidecar_key].tag } + ##### memcached ##### set { - name = "loki.limits_config.retention_period" - value = var.table_manager_retention_period + name = "memcached.image.repository" + value = split(":", module.images.images[local.memcached_key].dest_full_path)[0] } set { - name = "loki.limits_config.ingestion_rate_strategy" - value = "local" + name = "memcached.image.tag" + value = module.images.images[local.memcached_key].tag } + + ##### memcachedExporter ##### set { - name = "loki.limits_config.max_global_streams_per_user" - value = "5000" + name = "memcachedExporter.image.repository" + value = split(":", module.images.images[local.exporter_key].dest_full_path)[0] } set { - name = "loki.limits_config.max_query_length" - value = var.table_manager_retention_period + name = "memcachedExporter.image.tag" + value = module.images.images[local.exporter_key].tag } + + ##### Auth ##### set { - name = "loki.limits_config.max_query_parallelism" - value = "32" + name = "loki.auth_enabled" + value = "true" } set { - name = "loki.limits_config.max_streams_per_user" - value = "10000" + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.loki-irsa-role.iam_role_arn + } + set { + name = "memberlist.service.publishNotReadyAddresses" + value = "true" } + ##### Storage ##### set { name = "loki.storage.bucketNames.chunks" value = module.loki-s3.s3_requested_bucket_name @@ -158,11 +183,39 @@ resource "helm_release" "loki" { name = "loki.storage.s3.region" value = var.region } + set { + name = "loki.storage_config.aws.s3" + value = format("s3://%v/%v", + var.region, + module.loki-s3.s3_requested_bucket_name + ) + } + set { + name = "write.extraVolumesMounts[0].name" + value = "data" + } + set { + name = "write.extraVolumesMounts[0].mountPath" + value = "/loki" + } + set { + name = "write.extraVolumes[0].name" + value = "loki" + } + set { + name = "write.persistence.storageClass" + value = var.rwo_storage_class + } + set { + name = "backend.persistence.storageClass" + value = var.rwo_storage_class + } + ##### Schema Config ##### set { name = "loki.schemaConfig.configs[0].from" - value = "2023-09-09" + value = "2024-04-01" } set { name = "loki.schemaConfig.configs[0].index.period" @@ -178,41 +231,25 @@ resource "helm_release" "loki" { } set { name = "loki.schemaConfig.configs[0].schema" - value = "v12" + value = "v13" } set { name = "loki.schemaConfig.configs[0].store" - value = "boltdb-shipper" - } - - - set { - name = "loki.storage_config.aws.s3" - value = format("s3://%v/%v", - var.region, - module.loki-s3.s3_requested_bucket_name - ) - } - set { - name = "loki.storage_config.boltdb_shipper.active_index_directory" - value = "/loki/index" - } - set { - name = "loki.storage_config.boltdb_shipper.shared_store" - value = "s3" + value = "tsdb" } set { - name = "loki.storage_config.boltdb_shipper.cache_location" - value = "/loki/boltdb-cache" + name = "test.enabled" + value = "false" } + ##### Compactor Config ##### set { name = "loki.compactor.working_directory" value = "/loki/compactor" } set { name = "loki.compactor.shared_store" - value = "aws" + value = "s3" } set { name = "loki.compactor.compaction_interval" @@ -231,33 +268,30 @@ resource "helm_release" "loki" { value = "150" } + ##### Limits ##### set { - name = "loki.analytics.reporting_enabled" - value = "false" - } - - set { - name = "loki.provisioner.image.registry" - value = module.images.images[local.provisioner_key].dest_registry + name = "loki.limits_config.ingestion_rate_strategy" + value = "local" } set { - name = "loki.provisioner.image.repository" - value = module.images.images[local.provisioner_key].dest_repository + name = "loki.limits_config.max_global_streams_per_user" + value = "5000" } set { - name = "loki.provisioner.image.tag" - value = module.images.images[local.provisioner_key].tag + name = "loki.limits_config.max_query_parallelism" + value = "32" } - set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.loki-irsa-role.iam_role_arn + name = "loki.limits_config.max_streams_per_user" + value = "10000" } + ##### Monitoring ##### set { - name = "test.enabled" + name = "loki.analytics.reporting_enabled" value = "false" } + set { name = "monitoring.dashboards.enabled" value = "false" @@ -279,47 +313,4 @@ resource "helm_release" "loki" { value = "false" } - set { - name = "write.extraVolumesMounts[0].name" - value = "data" - } - set { - name = "write.extraVolumesMounts[0].mountPath" - value = "/loki" - } - set { - name = "write.extraVolumes[0].name" - value = "loki" - } - set { - name = "write.extraVolumes[0].emptyDir" - value = "{}" - } - - set { - name = "write.persistence.storageClass" - value = var.rwo_storage_class - } - set { - name = "backend.persistence.storageClass" - value = var.rwo_storage_class - } - - set { - name = "memberlist.service.publishNotReadyAddresses" - value = "true" - } - - set { - name = "gateway.image.registry" - value = module.images.images[local.gateway_key].dest_registry - } - set { - name = "gateway.image.repository" - value = module.images.images[local.provisioner_key].dest_repository - } - set { - name = "gateway.image.tag" - value = module.images.images[local.provisioner_key].tag - } } diff --git a/variables.tf b/variables.tf index 47e03e5..19947a4 100644 --- a/variables.tf +++ b/variables.tf @@ -46,35 +46,35 @@ variable "oidc_provider_arn" { variable "rwo_storage_class" { description = "Specify the storage class for read/write/once persistent volumes." type = string - default = "gp3" + default = "gp3-encrypted" } -variable "table_manager_retention_deletes_enabled" { - description = "" - type = string - default = "false" -} +# variable "table_manager_retention_deletes_enabled" { +# description = "" +# type = string +# default = "false" +# } -variable "table_manager_retention_period" { - description = "Loki defaults to 0" - type = string - default = "2160h" -} +# variable "table_manager_retention_period" { +# description = "Loki defaults to 0" +# type = string +# default = "2160h" +# } -variable "kubectl_image_registry" { - description = "The registry holding the kubectl docker image" - type = string -} +# variable "kubectl_image_registry" { +# description = "The registry holding the kubectl docker image" +# type = string +# } -variable "kubectl_image_repository" { - description = "The image repository holding the kubectl docker image" - type = string -} +# variable "kubectl_image_repository" { +# description = "The image repository holding the kubectl docker image" +# type = string +# } -variable "kubectl_image_tag" { - description = "The image tag to use to access the kubectl docker image" - type = string -} +# variable "kubectl_image_tag" { +# description = "The image tag to use to access the kubectl docker image" +# type = string +# } # helm add repo grafana "https://grafana.github.io/helm-charts" @@ -82,14 +82,20 @@ variable "kubectl_image_tag" { variable "loki_chart_version" { description = "Which version of the grafana/loki helm chart to use." type = string - default = "5.15.0" + default = "6.6.5" } # The [APP VERSION] associated with the helm chart. variable "loki_tag" { description = "The tag of the loki image to use." type = string - default = "2.8.4" + default = "3.0.0" +} + +variable "canary_tag" { + description = "The tag of the grafana/loki-canary image to use." + type = string + default = "3.0.0" } variable "enterprise_logs_provisioner_tag" { @@ -101,5 +107,23 @@ variable "enterprise_logs_provisioner_tag" { variable "gateway_tag" { description = "The version of nginxinc/nginx-unprivileged to use for the gateway." type = string - default = "1.19-alpine" + default = "1.25.2-alpine" +} + +variable "memcached_tag" { + description = "The version of memcached to use for the gateway." + type = string + default = "1.6.23-alpine" +} + +variable "exporter_tag" { + description = "The version of prom/memcached-exporter to use for the gateway." + type = string + default = "v0.14.2" +} + +variable "sidecar_tag" { + description = "The version of kiwigrid/k8s-sidecar to use for the gateway." + type = string + default = "1.24.3" } From 4826eb7e9324ae818b2b641be570140a3a7001c0 Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 3 Jul 2024 20:53:07 -0400 Subject: [PATCH 2/8] cleanup --- copy_images.tf | 20 +++++++++++++++++++- variables.tf | 28 ---------------------------- 2 files changed, 19 insertions(+), 29 deletions(-) diff --git a/copy_images.tf b/copy_images.tf index e35264a..7a7801c 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -75,7 +75,7 @@ locals { } module "images" { - source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=2.0.2" + source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" profile = var.profile application_name = var.cluster_name @@ -91,4 +91,22 @@ module "images" { ## region = "" ## source_password = "" ## source_username = "" + + enable_lifecycle_policy = true + lifecycle_policy_all = true + + data "aws_ecr_lifecycle_policy_document" "pushed" { + rule { + priority = 1 + description = "keep images tagged test, last push 28 days ago" + + selection { + tag_status = "tagged" + # tag_pattern_list = ["*test*"] + count_type = "sinceImagePushed" + count_number = 28 + count_unit = days + } + } + } } diff --git a/variables.tf b/variables.tf index 19947a4..b488133 100644 --- a/variables.tf +++ b/variables.tf @@ -49,34 +49,6 @@ variable "rwo_storage_class" { default = "gp3-encrypted" } -# variable "table_manager_retention_deletes_enabled" { -# description = "" -# type = string -# default = "false" -# } - -# variable "table_manager_retention_period" { -# description = "Loki defaults to 0" -# type = string -# default = "2160h" -# } - -# variable "kubectl_image_registry" { -# description = "The registry holding the kubectl docker image" -# type = string -# } - -# variable "kubectl_image_repository" { -# description = "The image repository holding the kubectl docker image" -# type = string -# } - -# variable "kubectl_image_tag" { -# description = "The image tag to use to access the kubectl docker image" -# type = string -# } - - # helm add repo grafana "https://grafana.github.io/helm-charts" # helm search repo grafana/loki variable "loki_chart_version" { From 711e8b1b5ef7354068b359cbad4a4df8544b13ea Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Wed, 3 Jul 2024 21:37:36 -0400 Subject: [PATCH 3/8] update lcp --- copy_images.tf | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/copy_images.tf b/copy_images.tf index 7a7801c..8a9fe95 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -94,19 +94,4 @@ module "images" { enable_lifecycle_policy = true lifecycle_policy_all = true - - data "aws_ecr_lifecycle_policy_document" "pushed" { - rule { - priority = 1 - description = "keep images tagged test, last push 28 days ago" - - selection { - tag_status = "tagged" - # tag_pattern_list = ["*test*"] - count_type = "sinceImagePushed" - count_number = 28 - count_unit = days - } - } - } } From 65a4c5ce83c64071facd9a377e78e0ea43ff69be Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Mon, 8 Jul 2024 18:42:24 -0400 Subject: [PATCH 4/8] fix gateway and compactor --- main.tf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/main.tf b/main.tf index 5e4f549..17d1276 100644 --- a/main.tf +++ b/main.tf @@ -97,11 +97,11 @@ resource "helm_release" "loki" { #### Gateway Image ##### set { name = "gateway.image.repository" - value = module.images.images[local.provisioner_key].dest_repository + value = module.images.images[local.gateway_key].dest_repository } set { name = "gateway.image.tag" - value = module.images.images[local.provisioner_key].tag + value = module.images.images[local.gateway_key].tag } ##### grafana/loki-canary ##### @@ -244,27 +244,27 @@ resource "helm_release" "loki" { ##### Compactor Config ##### set { - name = "loki.compactor.working_directory" + name = "compactor.working_directory" value = "/loki/compactor" } set { - name = "loki.compactor.shared_store" + name = "compactor.shared_store" value = "s3" } set { - name = "loki.compactor.compaction_interval" + name = "compactor.compaction_interval" value = "10m" } set { - name = "loki.compactor.retention_enabled" + name = "compactor.retention_enabled" value = "true" } set { - name = "loki.compactor.retention_delete_delay" + name = "compactor.retention_delete_delay" value = "2h" } set { - name = "loki.compactor.retention_delete_worker_count" + name = "compactor.retention_delete_worker_count" value = "150" } From ac3a403e4e97a727344c589a0619e70ddaf2d49b Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Tue, 16 Jul 2024 20:01:00 -0400 Subject: [PATCH 5/8] use fuzzy operator --- requirements.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.tf b/requirements.tf index 1b7a5da..a19e4ab 100644 --- a/requirements.tf +++ b/requirements.tf @@ -4,23 +4,23 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.14.0" + version = "~>= 5.14.0" } helm = { source = "hashicorp/helm" - version = ">= 2.11.0" + version = "~>= 2.11.0" } kubernetes = { source = "hashicorp/kubernetes" - version = ">= 2.23.0" + version = "~>= 2.23.0" } null = { source = "hashicorp/null" - version = ">= 3.2.1" + version = "~>= 3.2.1" } template = { source = "hashicorp/template" - version = ">= 2.2.0" + version = "~>= 2.2.0" } } } From 1c84417e565327fe5dd609580fdc87c917f6defb Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 18 Jul 2024 21:57:31 -0400 Subject: [PATCH 6/8] no fuzzy --- main.tf | 10 +++++----- requirements.tf | 10 +++++----- version.tf | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/main.tf b/main.tf index 17d1276..c2aabd7 100644 --- a/main.tf +++ b/main.tf @@ -289,12 +289,12 @@ resource "helm_release" "loki" { ##### Monitoring ##### set { name = "loki.analytics.reporting_enabled" - value = "false" + value = "true" } set { name = "monitoring.dashboards.enabled" - value = "false" + value = "true" } set { name = "monitoring.rules.enabled" @@ -302,15 +302,15 @@ resource "helm_release" "loki" { } set { name = "monitoring.serviceMonitor.enabled" - value = "false" + value = "true" } set { name = "monitoring.selfMonitoring.enabled" - value = "false" + value = "true" } set { name = "monitoring.lokiCanary.enabled" - value = "false" + value = "true" } } diff --git a/requirements.tf b/requirements.tf index a19e4ab..1b7a5da 100644 --- a/requirements.tf +++ b/requirements.tf @@ -4,23 +4,23 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~>= 5.14.0" + version = ">= 5.14.0" } helm = { source = "hashicorp/helm" - version = "~>= 2.11.0" + version = ">= 2.11.0" } kubernetes = { source = "hashicorp/kubernetes" - version = "~>= 2.23.0" + version = ">= 2.23.0" } null = { source = "hashicorp/null" - version = "~>= 3.2.1" + version = ">= 3.2.1" } template = { source = "hashicorp/template" - version = "~>= 2.2.0" + version = ">= 2.2.0" } } } diff --git a/version.tf b/version.tf index 94d9790..3a981dc 100644 --- a/version.tf +++ b/version.tf @@ -1,4 +1,4 @@ locals { _module_name = "tfmod-loki" - _module_version = "unknown" + _module_version = "0.0.1" } From fa50bf8c696e03633960e2a4614a6eafc0f52f6e Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Thu, 18 Jul 2024 22:17:04 -0400 Subject: [PATCH 7/8] disable monitoring --- main.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/main.tf b/main.tf index c2aabd7..17d1276 100644 --- a/main.tf +++ b/main.tf @@ -289,12 +289,12 @@ resource "helm_release" "loki" { ##### Monitoring ##### set { name = "loki.analytics.reporting_enabled" - value = "true" + value = "false" } set { name = "monitoring.dashboards.enabled" - value = "true" + value = "false" } set { name = "monitoring.rules.enabled" @@ -302,15 +302,15 @@ resource "helm_release" "loki" { } set { name = "monitoring.serviceMonitor.enabled" - value = "true" + value = "false" } set { name = "monitoring.selfMonitoring.enabled" - value = "true" + value = "false" } set { name = "monitoring.lokiCanary.enabled" - value = "true" + value = "false" } } From 42f89121ea3df8f3399854b9316be17454a98adc Mon Sep 17 00:00:00 2001 From: "Matthew C. Morgan" Date: Fri, 19 Jul 2024 01:14:37 -0400 Subject: [PATCH 8/8] add timeout --- main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.tf b/main.tf index 17d1276..e39b7d6 100644 --- a/main.tf +++ b/main.tf @@ -67,6 +67,8 @@ resource "helm_release" "loki" { name = "loki" namespace = local.ns repository = "https://grafana.github.io/helm-charts" + timeout = 900 + wait = true ##### Globals ##### set {