Skip to content

Commit

Permalink
upgrade loki to 3.0.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@morga471
morga471 committed Jul 4, 2024
1 parent c8932fd commit 4f16eff
Show file tree
Hide file tree
Showing 4 changed files with 194 additions and 139 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@
Installs the loki as the log aggregation sink, and promtail to forward the logs
to loki.

* Requires additional Node HD space - 40GB is not enough.

# tfmod-loki
41 changes: 40 additions & 1 deletion copy_images.tf
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
locals {
loki_key = format("%v#%v", "grafana/loki", var.loki_tag)
canary_key = format("%v#%v", "grafana/loki-canary", var.canary_tag)
provisioner_key = format("%v#%v", "grafana/enterprise-logs-provisioner", var.enterprise_logs_provisioner_tag)
gateway_key = format("%v#%v", "grafana/nginx-unprivileged", var.gateway_tag)
sidecar_key = format("%v#%v", "kiwigrid/k8s-sidecar", var.sidecar_tag)
memcached_key = format("%v#%v", "memcached", var.memcached_tag)
exporter_key = format("%v#%v", "prom/memcached-exporter", var.exporter_tag)

image_config = [
{
Expand All @@ -13,6 +17,42 @@ locals {
source_tag = var.loki_tag
tag = var.loki_tag
},
{
enabled = true
dest_path = null
name = "grafana/loki-canary"
source_image = "grafana/loki-canary"
source_registry = "docker.io"
source_tag = var.canary_tag
tag = var.canary_tag
},
{
enabled = true
dest_path = null
name = "memcached"
source_image = "memcached"
source_registry = "docker.io"
source_tag = var.memcached_tag
tag = var.memcached_tag
},
{
enabled = true
dest_path = null
name = "prom/memcached-exporter"
source_image = "prom/memcached-exporter"
source_registry = "docker.io"
source_tag = var.exporter_tag
tag = var.exporter_tag
},
{
enabled = true
dest_path = null
name = "kiwigrid/k8s-sidecar"
source_image = "kiwigrid/k8s-sidecar"
source_registry = "quay.io"
source_tag = var.sidecar_tag
tag = var.sidecar_tag
},
{
enabled = true
dest_path = null
Expand Down Expand Up @@ -52,4 +92,3 @@ module "images" {
## source_password = ""
## source_username = ""
}

215 changes: 103 additions & 112 deletions main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,72 +68,97 @@ resource "helm_release" "loki" {
namespace = local.ns
repository = "https://grafana.github.io/helm-charts"

##### Globals #####
set {
name = "kubectlImage.registry"
value = var.kubectl_image_registry
name = "global.image.registry"
value = module.images.images[local.loki_key].dest_registry
}

##### Loki Image #####
set {
name = "kubectlImage.repository"
value = var.kubectl_image_repository
name = "loki.image.repository"
value = module.images.images[local.loki_key].dest_repository
}
set {
name = "kubectlImage.tag"
value = var.kubectl_image_tag
name = "loki.image.tag"
value = module.images.images[local.loki_key].tag
}

##### Provisioner (nginx) Image #####
set {
name = "loki.image.registry"
value = module.images.images[local.loki_key].dest_registry
name = "loki.provisioner.image.repository"
value = split(":", module.images.images[local.provisioner_key].dest_full_path)[0]
}
set {
name = "loki.image.repository"
value = module.images.images[local.loki_key].dest_repository
name = "loki.provisioner.image.tag"
value = module.images.images[local.provisioner_key].tag
}

#### Gateway Image #####
set {
name = "loki.image.tag"
value = module.images.images[local.loki_key].tag
name = "gateway.image.repository"
value = module.images.images[local.provisioner_key].dest_repository
}
set {
name = "gateway.image.tag"
value = module.images.images[local.provisioner_key].tag
}


##### grafana/loki-canary #####
set {
name = "table_manager.retention_deletes_enabled"
value = var.table_manager_retention_deletes_enabled
name = "lokiCanary.image.repository"
value = module.images.images[local.canary_key].dest_repository
}
set {
name = "table_manager.retention_period"
value = var.table_manager_retention_period
name = "lokiCanary.image.tag"
value = module.images.images[local.canary_key].tag
}

##### kiwigrid/k8s-sidecar #####
set {
name = "loki.auth_enabled"
value = "true"
name = "sidecar.image.repository"
value = split(":", module.images.images[local.sidecar_key].dest_full_path)[0]
}
set {
name = "sidecar.image.tag"
value = module.images.images[local.sidecar_key].tag
}

##### memcached #####
set {
name = "loki.limits_config.retention_period"
value = var.table_manager_retention_period
name = "memcached.image.repository"
value = split(":", module.images.images[local.memcached_key].dest_full_path)[0]
}
set {
name = "loki.limits_config.ingestion_rate_strategy"
value = "local"
name = "memcached.image.tag"
value = module.images.images[local.memcached_key].tag
}

##### memcachedExporter #####
set {
name = "loki.limits_config.max_global_streams_per_user"
value = "5000"
name = "memcachedExporter.image.repository"
value = split(":", module.images.images[local.exporter_key].dest_full_path)[0]
}
set {
name = "loki.limits_config.max_query_length"
value = var.table_manager_retention_period
name = "memcachedExporter.image.tag"
value = module.images.images[local.exporter_key].tag
}

##### Auth #####
set {
name = "loki.limits_config.max_query_parallelism"
value = "32"
name = "loki.auth_enabled"
value = "true"
}
set {
name = "loki.limits_config.max_streams_per_user"
value = "10000"
name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
value = module.loki-irsa-role.iam_role_arn
}
set {
name = "memberlist.service.publishNotReadyAddresses"
value = "true"
}

##### Storage #####
set {
name = "loki.storage.bucketNames.chunks"
value = module.loki-s3.s3_requested_bucket_name
Expand All @@ -158,11 +183,39 @@ resource "helm_release" "loki" {
name = "loki.storage.s3.region"
value = var.region
}
set {
name = "loki.storage_config.aws.s3"
value = format("s3://%v/%v",
var.region,
module.loki-s3.s3_requested_bucket_name
)
}
set {
name = "write.extraVolumesMounts[0].name"
value = "data"
}
set {
name = "write.extraVolumesMounts[0].mountPath"
value = "/loki"
}
set {
name = "write.extraVolumes[0].name"
value = "loki"
}

set {
name = "write.persistence.storageClass"
value = var.rwo_storage_class
}
set {
name = "backend.persistence.storageClass"
value = var.rwo_storage_class
}

##### Schema Config #####
set {
name = "loki.schemaConfig.configs[0].from"
value = "2023-09-09"
value = "2024-04-01"
}
set {
name = "loki.schemaConfig.configs[0].index.period"
Expand All @@ -178,41 +231,25 @@ resource "helm_release" "loki" {
}
set {
name = "loki.schemaConfig.configs[0].schema"
value = "v12"
value = "v13"
}
set {
name = "loki.schemaConfig.configs[0].store"
value = "boltdb-shipper"
}


set {
name = "loki.storage_config.aws.s3"
value = format("s3://%v/%v",
var.region,
module.loki-s3.s3_requested_bucket_name
)
}
set {
name = "loki.storage_config.boltdb_shipper.active_index_directory"
value = "/loki/index"
}
set {
name = "loki.storage_config.boltdb_shipper.shared_store"
value = "s3"
value = "tsdb"
}
set {
name = "loki.storage_config.boltdb_shipper.cache_location"
value = "/loki/boltdb-cache"
name = "test.enabled"
value = "false"
}

##### Compactor Config #####
set {
name = "loki.compactor.working_directory"
value = "/loki/compactor"
}
set {
name = "loki.compactor.shared_store"
value = "aws"
value = "s3"
}
set {
name = "loki.compactor.compaction_interval"
Expand All @@ -231,33 +268,30 @@ resource "helm_release" "loki" {
value = "150"
}

##### Limits #####
set {
name = "loki.analytics.reporting_enabled"
value = "false"
}

set {
name = "loki.provisioner.image.registry"
value = module.images.images[local.provisioner_key].dest_registry
name = "loki.limits_config.ingestion_rate_strategy"
value = "local"
}
set {
name = "loki.provisioner.image.repository"
value = module.images.images[local.provisioner_key].dest_repository
name = "loki.limits_config.max_global_streams_per_user"
value = "5000"
}
set {
name = "loki.provisioner.image.tag"
value = module.images.images[local.provisioner_key].tag
name = "loki.limits_config.max_query_parallelism"
value = "32"
}

set {
name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
value = module.loki-irsa-role.iam_role_arn
name = "loki.limits_config.max_streams_per_user"
value = "10000"
}

##### Monitoring #####
set {
name = "test.enabled"
name = "loki.analytics.reporting_enabled"
value = "false"
}

set {
name = "monitoring.dashboards.enabled"
value = "false"
Expand All @@ -279,47 +313,4 @@ resource "helm_release" "loki" {
value = "false"
}

set {
name = "write.extraVolumesMounts[0].name"
value = "data"
}
set {
name = "write.extraVolumesMounts[0].mountPath"
value = "/loki"
}
set {
name = "write.extraVolumes[0].name"
value = "loki"
}
set {
name = "write.extraVolumes[0].emptyDir"
value = "{}"
}

set {
name = "write.persistence.storageClass"
value = var.rwo_storage_class
}
set {
name = "backend.persistence.storageClass"
value = var.rwo_storage_class
}

set {
name = "memberlist.service.publishNotReadyAddresses"
value = "true"
}

set {
name = "gateway.image.registry"
value = module.images.images[local.gateway_key].dest_registry
}
set {
name = "gateway.image.repository"
value = module.images.images[local.provisioner_key].dest_repository
}
set {
name = "gateway.image.tag"
value = module.images.images[local.provisioner_key].tag
}
}

0 comments on commit 4f16eff

Please sign in to comment.