From 9923678c31075949b392a335b7d484fb986b06bf Mon Sep 17 00:00:00 2001 From: mcgin314 Date: Thu, 6 Mar 2025 14:05:14 -0500 Subject: [PATCH] Initial module development --- .cz.yaml | 8 + .github/dependabot.yml | 11 ++ .gitignore | 37 ++++ .pre-commit-config.yaml | 99 +++++++++++ .releaserc.json | 36 ++++ .terraform-docs.yml | 44 +++++ .tflint.hcl | 22 +++ CHANGELOG.md | 5 + README.md | 3 + copy_images.tf | 82 +++++++++ examples/simple/open-telemetry.tf | 9 + examples/simple/providers.tf | 46 +++++ examples/simple/variables.tf | 21 +++ java_instrumentation.yaml | 16 ++ main.tf | 95 +++++++++++ otelcol_daemonset.yaml | 270 ++++++++++++++++++++++++++++++ otelcol_deployment.yaml | 129 ++++++++++++++ outputs.tf | 13 ++ requirements.tf | 18 ++ variables.tf | 41 +++++ version.tf | 4 + 21 files changed, 1009 insertions(+) create mode 100644 .cz.yaml create mode 100644 .github/dependabot.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .releaserc.json create mode 100644 .terraform-docs.yml create mode 100644 .tflint.hcl create mode 100644 CHANGELOG.md create mode 100644 README.md create mode 100644 copy_images.tf create mode 100644 examples/simple/open-telemetry.tf create mode 100644 examples/simple/providers.tf create mode 100644 examples/simple/variables.tf create mode 100644 java_instrumentation.yaml create mode 100644 main.tf create mode 100644 otelcol_daemonset.yaml create mode 100644 otelcol_deployment.yaml create mode 100644 outputs.tf create mode 100644 requirements.tf create mode 100644 variables.tf create mode 100644 version.tf diff --git a/.cz.yaml b/.cz.yaml new file mode 100644 index 0000000..b1981ec --- /dev/null +++ b/.cz.yaml @@ -0,0 +1,8 @@ +--- +commitizen: + major_version_zero: true + name: cz_gitmoji + tag_format: $version + update_changelog_on_bump: true + version_provider: scm + version_scheme: semver2 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..867570d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "terraform" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "daily" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..403ffa9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +# Local .terraform directories +**/.terraform/* + +# terraform lock file. +**/.terraform.lock.hcl + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, +# such as password, private keys, and other secrets. These should not be +# part of version control as they are data points which are potentially +# sensitive and subject to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources +# locally and so are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6bfca7a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,99 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + # Git style + - id: check-added-large-files + - id: check-merge-conflict + - id: check-vcs-permalinks + - id: forbid-new-submodules + - id: no-commit-to-branch + + # Common errors + - id: end-of-file-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + exclude: CHANGELOG.md + - id: check-yaml + - id: check-merge-conflict + - id: check-executables-have-shebangs + + # Cross platform + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + + # Security + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + - id: detect-private-key + +# JSON5 Linter +- repo: https://github.com/pre-commit/mirrors-prettier + rev: v4.0.0-alpha.8 + hooks: + - id: prettier + # https://prettier.io/docs/en/options.html#parser + files: '.json5$' + +# Terraform Hooks +- repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.92.1 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases + hooks: + - id: terraform_fmt + args: + - --hook-config=--parallelism-ci-cpu-cores=2 + - id: terraform_docs + args: + - --hook-config=--parallelism-ci-cpu-cores=2 + - id: terraform_tflint + name: Terraform validate with tflint + description: Validates all Terraform configuration files with TFLint. + require_serial: true + entry: hooks/terraform_tflint.sh + language: script + files: (\.tf|\.tfvars)$ + exclude: \.(terraform/.*|terragrunt-cache)$ + args: + - --hook-config=--parallelism-ci-cpu-cores=2 + - id: terragrunt_fmt + name: Terragrunt fmt + description: Rewrites all Terragrunt configuration files to a canonical format. + entry: hooks/terragrunt_fmt.sh + language: script + files: (\.hcl)$ + exclude: \.(terraform/.*|terragrunt-cache)$ + args: + - --hook-config=--parallelism-ci-cpu-cores=2 + # - id: terragrunt_validate + # name: Terragrunt validate + # description: Validates all Terragrunt configuration files. + # entry: hooks/terragrunt_validate.sh + # language: script + # files: (\.hcl)$ + # exclude: \.(terraform/.*|terragrunt-cache)$ + # args: + # - --hook-config=--parallelism-ci-cpu-cores=2 + # - id: terragrunt_validate_inputs + # name: Terragrunt validate inputs + # description: Validates Terragrunt unused and undefined inputs. + # entry: hooks/terragrunt_validate_inputs.sh + # language: script + # files: (\.hcl)$ + # exclude: \.(terraform/.*|terragrunt-cache)$ + # args: + # - --hook-config=--parallelism-ci-cpu-cores=2 + # - id: terragrunt_providers_lock + # name: Terragrunt providers lock + # description: Updates provider signatures in dependency lock files using terragrunt. + # entry: hooks/terragrunt_providers_lock.sh + # language: script + # files: (terragrunt|\.terraform\.lock)\.hcl$ + # exclude: \.(terraform/.*|terragrunt-cache)$ + # args: + # - --hook-config=--parallelism-ci-cpu-cores=2 + +- repo: https://github.com/ljnsn/cz-conventional-gitmoji + rev: v0.3.2 + hooks: + - id: conventional-gitmoji diff --git a/.releaserc.json b/.releaserc.json new file mode 100644 index 0000000..6e39031 --- /dev/null +++ b/.releaserc.json @@ -0,0 +1,36 @@ +{ + "branches": [ + "main", + "master" + ], + "ci": false, + "plugins": [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + [ + "@semantic-release/github", + { + "successComment": + "This ${issue.pull_request ? 'PR is included' : 'issue has been resolved'} in version ${nextRelease.version} :tada:", + "labels": false, + "releasedLabels": false + } + ], + [ + "@semantic-release/changelog", + { + "changelogFile": "CHANGELOG.md", + "changelogTitle": "# Changelog\n\nAll notable changes to this project will be documented in this file." + } + ], + [ + "@semantic-release/git", + { + "assets": [ + "CHANGELOG.md" + ], + "message": "chore(release): version ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}" + } + ] + ] +} diff --git a/.terraform-docs.yml b/.terraform-docs.yml new file mode 100644 index 0000000..fabfb8d --- /dev/null +++ b/.terraform-docs.yml @@ -0,0 +1,44 @@ +formatter: markdown table + +header-from: main.tf +footer-from: "" + +sections: +## hide: [] + show: + - data-sources + - header + - footer + - inputs + - modules + - outputs + - providers + - requirements + - resources + +output: + file: README.md + mode: inject + template: |- + + {{ .Content }} + + +output-values: + enabled: false + from: "" + +sort: + enabled: true + by: name + +settings: + anchor: true + color: true + default: true + description: true + escape: true + indent: 2 + required: true + sensitive: true + type: true diff --git a/.tflint.hcl b/.tflint.hcl new file mode 100644 index 0000000..f63db7d --- /dev/null +++ b/.tflint.hcl @@ -0,0 +1,22 @@ +config { + module = true + force = false + disabled_by_default = false + +} + +rule "aws_instance_invalid_type" { + enabled = true +} + +plugin "aws" { + enabled = true + version = "0.32.0" + source = "github.com/terraform-linters/tflint-ruleset-aws" +} + +plugin "terraform" { + enabled = true + version = "0.9.0" + source = "github.com/terraform-linters/tflint-ruleset-terraform" +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..257696f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +## Unreleased + +# CHANGELOG +* 0.0.1 -- 2024-10-17 + - Code genesis diff --git a/README.md b/README.md new file mode 100644 index 0000000..3180ce3 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# tfmod-open-telemetry + +Installs the \ No newline at end of file diff --git a/copy_images.tf b/copy_images.tf new file mode 100644 index 0000000..f7c8f82 --- /dev/null +++ b/copy_images.tf @@ -0,0 +1,82 @@ +locals { + otel_name = "opentelemetry-operator" + otel_version = "0.110.0" + otel_key = format("%v#%v", local.otel_name, local.otel_version) + + collector_name = "opentelemetry-collector-k8s" + collector_version = "0.111.0-amd64" + collector_key = format("%v#%v", local.collector_name, local.collector_version) + + collector_contrib_name = "opentelemetry-collector-contrib" + collector_contrib_version = "0.113.0-amd64" + collector_contrib_key = format("%v#%v", local.collector_contrib_name, local.collector_contrib_version) + + rbac_proxy_name = "kube-rbac-proxy" + rbac_proxy_version = "v0.18.1" + rbac_proxy_key = format("%v#%v", local.rbac_proxy_name, local.rbac_proxy_version) + + auto_instrumentation_java_name = "autoinstrumentation-java" + auto_instrumentation_java_version = "2.9.0" + auto_instrumentation_java_key = format("%v#%v", local.auto_instrumentation_java_name, local.auto_instrumentation_java_version) + + image_config = [ + { + enabled = true + dest_path = null + name = local.otel_name + source_image = "otel/${local.otel_name}" + source_registry = "docker.io" + source_tag = local.otel_version + tag = local.otel_version + }, + { + enabled = true + dest_path = null + name = local.collector_name + source_image = "otel/${local.collector_name}" + source_registry = "docker.io" + source_tag = local.collector_version + tag = local.collector_version + }, + { + enabled = true + dest_path = null + name = local.collector_contrib_name + source_image = "otel/${local.collector_contrib_name}" + source_registry = "docker.io" + source_tag = local.collector_contrib_version + tag = local.collector_contrib_version + }, + { + enabled = true + dest_path = null + name = local.rbac_proxy_name + source_image = "brancz/${local.rbac_proxy_name}" + source_registry = "quay.io" + source_tag = local.rbac_proxy_version + tag = local.rbac_proxy_version + }, + { + enabled = true + dest_path = null + name = local.auto_instrumentation_java_name + source_image = "open-telemetry/opentelemetry-operator/${local.auto_instrumentation_java_name}" + source_registry = "ghcr.io" + source_tag = local.auto_instrumentation_java_version + tag = local.auto_instrumentation_java_version + } + ] +} + +module "images" { + source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" + + profile = var.profile + application_name = var.cluster_name + image_config = local.image_config + tags = {} + + enable_lifecycle_policy = true + lifecycle_policy_all = true + force_delete = true +} diff --git a/examples/simple/open-telemetry.tf b/examples/simple/open-telemetry.tf new file mode 100644 index 0000000..701be0e --- /dev/null +++ b/examples/simple/open-telemetry.tf @@ -0,0 +1,9 @@ +module "metrics-server" { + source = "../.." + + region = var.region + profile = var.profile + cluster_name = var.cluster_name + + tags = var.tags +} diff --git a/examples/simple/providers.tf b/examples/simple/providers.tf new file mode 100644 index 0000000..fd0156f --- /dev/null +++ b/examples/simple/providers.tf @@ -0,0 +1,46 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.14.0" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.11.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.23.0" + } + } +} + +provider "aws" { + profile = var.profile + region = var.region +} + +data "aws_eks_cluster" "cluster" { + name = var.cluster_name +} + +data "aws_eks_cluster_auth" "cluster" { + name = var.cluster_name +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.cluster.endpoint + + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token + } +} diff --git a/examples/simple/variables.tf b/examples/simple/variables.tf new file mode 100644 index 0000000..235d0e1 --- /dev/null +++ b/examples/simple/variables.tf @@ -0,0 +1,21 @@ +variable "cluster_name" { + description = "EKS cluster name name component used through out the EKS cluster describing its purpose (ex: dice-dev)" + type = string +} + +variable "region" { + description = "AWS region" + type = string +} + +variable "profile" { + description = "AWS config profile" + type = string + default = "" +} + +variable "tags" { + description = "AWS Tags to apply to appropriate resources" + type = map(string) + default = {} +} diff --git a/java_instrumentation.yaml b/java_instrumentation.yaml new file mode 100644 index 0000000..1ac1403 --- /dev/null +++ b/java_instrumentation.yaml @@ -0,0 +1,16 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: java-instrumentation + namespace: telemetry +spec: + exporter: + endpoint: http://otel-daemonset-collector.telemetry.svc.cluster.local:4318 + propagators: + - tracecontext + - baggage + sampler: + type: parentbased_traceidratio + argument: "1" + java: + image: 224384469011.dkr.ecr.us-gov-east-1.amazonaws.com/platform-test-z/autoinstrumentation-java:2.9.0 diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..d1b1426 --- /dev/null +++ b/main.tf @@ -0,0 +1,95 @@ +locals { + telemetry_ns = var.namespace +} + +# Install opentelemetry-collector daemonset collector, handles collection of node and workload telemetry data +resource "helm_release" "opentelemetry-operator" { + depends_on = [ + module.images, + ] + + chart = "opentelemetry-operator" + name = "opentelemetry-operator" + namespace = local.telemetry_ns + repository = var.otel_helm_repo + version = var.otel_helm_version + + set { + name = "manager.image.repository" + value = format("%v/%v", + module.images.images[local.otel_key].dest_registry, + module.images.images[local.otel_key].dest_repository + ) + } + set { + name = "manager.image.tag" + value = module.images.images[local.otel_key].tag + } + + set { + name = "manager.collectorImage.repository" + value = format("%v/%v", + module.images.images[local.collector_key].dest_registry, + module.images.images[local.collector_key].dest_repository + ) + } + set { + name = "manager.collectorImage.tag" + value = module.images.images[local.collector_key].tag + } + + set { + name = "kubeRBACProxy.image.repository" + value = format("%v/%v", + module.images.images[local.rbac_proxy_key].dest_registry, + module.images.images[local.rbac_proxy_key].dest_repository + ) + } + set { + name = "kubeRBACProxy.image.tag" + value = module.images.images[local.rbac_proxy_key].tag + } + + set { + name = "admissionWebhooks.certManager.enabled" + value = true + } + set { + name = "admissionWebhooks.certManager.issuerRef.kind" + value = "ClusterIssuer" + } + set { + name = "admissionWebhooks.certManager.issuerRef.name" + value = "clusterissuer" + } + + timeout = 180 +} + +# SHOULD PROBABLY PUT THESE OBJECTS INTO A HELM CHART +# Create karpenter default +# resource "kubectl_manifest" "otelcol_daemonset" { +# depends_on = [ +# helm_release.opentelemetry-operator, +# ] +# yaml_body = templatefile("${path.module}/otelcol_daemonset.yaml.tpl", { + +# # cluster_name = var.cluster_name +# # tags = jsonencode(data.aws_default_tags.default_tags.tags) +# # amd_ami_family = "Bottlerocket" +# # amd_ami_alias = "bottlerocket" +# }) +# } + +# resource "kubectl_manifest" "otelcol_simple" { +# depends_on = [ +# helm_release.opentelemetry-operator, +# ] +# yaml_body = templatefile("${path.module}/collector_simple.yaml", { + +# # cluster_name = var.cluster_name +# # tags = jsonencode(data.aws_default_tags.default_tags.tags) +# # amd_ami_family = "Bottlerocket" +# # amd_ami_alias = "bottlerocket" +# }) +# } \ No newline at end of file diff --git a/otelcol_daemonset.yaml b/otelcol_daemonset.yaml new file mode 100644 index 0000000..a7d434d --- /dev/null +++ b/otelcol_daemonset.yaml @@ -0,0 +1,270 @@ +--- +apiVersion: opentelemetry.io/v1beta1 +kind: OpenTelemetryCollector +metadata: + name: otel-daemonset + namespace: telemetry +spec: + mode: daemonset + config: + connectors: + spanmetrics: {} # Aggregates Request, Error and Duration (R.E.D) OpenTelemetry metrics from span data. + receivers: + otlp: # Used to collect application traces, metrics and logs. + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + kubeletstats: # Used to pull node, pod, and container metrics from the API server on a kubelet. + collection_interval: 10s + auth_type: serviceAccount + endpoint: ${env:K8S_NODE_NAME}:10250 + insecure_skip_verify: true + metric_groups: + - node + - pod + - container + filelog: # Used to collect Kubernetes logs and application logs written to stdout/stderr. + include: + - /var/log/pods/*/*/*.log + exclude: + # Exclude logs from all containers named otel-collector + - /var/log/pods/*/otel-collector/*.log + start_at: beginning + include_file_path: true + include_file_name: false + operators: + # Find out which format is used by kubernetes + - type: router + id: get-format + routes: + - output: parser-docker + expr: 'body matches "^\\{"' + - output: parser-crio + expr: 'body matches "^[^ Z]+ "' + - output: parser-containerd + expr: 'body matches "^[^ Z]+Z"' + # Parse CRI-O format + - type: regex_parser + id: parser-crio + regex: + '^(?P