diff --git a/.github/workflows/terraform-release.yaml b/.github/workflows/terraform-release.yaml new file mode 100644 index 0000000..90910bc --- /dev/null +++ b/.github/workflows/terraform-release.yaml @@ -0,0 +1,73 @@ +name: Terraform CI/CD +on: + workflow_dispatch: + pull_request: + types: [closed] + branches: + - main +jobs: + terraform-ci-cd: + runs-on: 229685449397 + permissions: + contents: write + + steps: + - name: Checkout code + uses: CSVD/gh-actions-checkout@v4 + + - name: Setup Terraform + uses: CSVD/gh-actions-setup-terraform@v3 + with: + terraform_version: "1.9.1" + + - name: Setup GITHUB Credentials + id: github_credentials + uses: CSVD/gh-auth@main + with: + github_app_pem_file: ${{ secrets.GH_APP_PEM_FILE }} + github_app_installation_id: ${{ vars.GH_APP_INSTALLATION_ID }} + github_app_id: ${{ vars.GH_APP_ID }} + + + - name: Debug Authentication + run: | + # Print the GitHub server URL + echo "GitHub Server URL: ${{ github.server_url }}" + + # Extract the host from the URL + HOST="${{ github.server_url }}" + HOST="${HOST#*//}" + HOST="${HOST%%/*}" + echo "GitHub Host: $HOST" + + # Check if token exists + if [[ -n "${{ steps.github_credentials.outputs.github_token }}" ]]; then + echo "Token generated successfully" + # Test the token with a simple GitHub API call (without exposing the token) + STATUS=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer ${{ steps.github_credentials.outputs.github_token }}" "${{ github.server_url }}/api/v3/user") + echo "API Test Status Code: $STATUS" + else + echo "No token was generated!" + fi + + - name: Setup GitHub CLI + run: | + # Force manual authentication since setup-git might not work with GitHub Enterprise + echo "${{ steps.github_credentials.outputs.github_token }}" > /tmp/token.txt + gh auth login --with-token --hostname "github.e.it.census.gov" < /tmp/token.txt + rm /tmp/token.txt + + # Test GitHub CLI auth status + gh auth status || echo "GitHub CLI authentication failed" + + - name: AWS Auth + id: aws_auth + uses: CSVD/aws-auth@main + with: + ecs: true + + - name: Run Terraform Module Release Action + uses: CSVD/terraform-module-release@main + with: + github-token: ${{ steps.github_credentials.outputs.github_token }} + working-directory: '.' diff --git a/.github/workflows/terraform-validate.yaml b/.github/workflows/terraform-validate.yaml new file mode 100644 index 0000000..72829d8 --- /dev/null +++ b/.github/workflows/terraform-validate.yaml @@ -0,0 +1,42 @@ +name: Terraform Validate +on: + pull_request: + workflow_dispatch: + +jobs: + + terraform-validate: + runs-on: "229685449397" + permissions: + contents: write + steps: + - name: Checkout code + uses: CSVD/gh-actions-checkout@v4 + + - name: Setup Terraform + uses: CSVD/gh-actions-setup-terraform@v2 + with: + terraform_version: '1.7.3' + + - name: Validate Terraform Configuration + id: validate + uses: CSVD/terraform-validate@main + + - name: Check Validation/Test Results + if: always() + run: | + # Set default values if outputs are empty + IS_VALID="${{ steps.validate.outputs.is_valid }}" + TESTS_PASSED="${{ steps.validate.outputs.tests_passed }}" + + # If outputs are empty, set them to false + [ -z "$IS_VALID" ] && IS_VALID="false" + [ -z "$TESTS_PASSED" ] && TESTS_PASSED="false" + + if [[ "$IS_VALID" != "true" || "$TESTS_PASSED" != "true" ]]; then + echo "Validation or test errors found:" + echo "${{ steps.validate.outputs.stderr }}" + exit 1 + else + echo "All validations and tests passed successfully!" + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6bfca7a..add5da5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: # Git style - id: check-added-large-files @@ -38,7 +38,7 @@ repos: # Terraform Hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.92.1 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases + rev: v1.98.0 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases hooks: - id: terraform_fmt args: @@ -94,6 +94,6 @@ repos: # - --hook-config=--parallelism-ci-cpu-cores=2 - repo: https://github.com/ljnsn/cz-conventional-gitmoji - rev: v0.3.2 + rev: v0.7.0 hooks: - id: conventional-gitmoji diff --git a/.tflint.hcl b/.tflint.hcl index f63db7d..6a77997 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -5,18 +5,18 @@ config { } -rule "aws_instance_invalid_type" { - enabled = true -} +# rule "aws_instance_invalid_type" { +# enabled = true +# } -plugin "aws" { - enabled = true - version = "0.32.0" - source = "github.com/terraform-linters/tflint-ruleset-aws" -} +# plugin "aws" { +# enabled = true +# version = "0.32.0" +# source = "github.com/terraform-linters/tflint-ruleset-aws" +# } -plugin "terraform" { - enabled = true - version = "0.9.0" - source = "github.com/terraform-linters/tflint-ruleset-terraform" -} +# plugin "terraform" { +# enabled = true +# version = "0.9.0" +# source = "github.com/terraform-linters/tflint-ruleset-terraform" +# } diff --git a/README.md b/README.md index a2aaf2f..0491327 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # tfmod-open-telemetry -Installs the OpenTelemetry (OTel) Collector operator, collectors and autoinstrumentation. [OpenTelemetry](https://opentelemetry.io/docs/) is a vendor neutral, open source observability framework and toolkit providing collection, processing and export of telemetry data, such as traces, metrics, and logs from applications and infrastructure. In the platform context, the OpenTelemetry Collector acts as an intermediary between instrumented applications and observability backends, aggregating telemetry data before forwarding it to various monitoring and logging systems. The Collector acts as a pipeline, utilizing processors, receivers and exporters, allowing fine-grained control over how and where data is sent. +Installs the OpenTelemetry (OTel) Collector operator, collectors and autoinstrumentation. [OpenTelemetry](https://opentelemetry.io/docs/) is a vendor neutral, open source observability framework and toolkit providing collection, processing and export of telemetry data, such as traces, metrics, and logs from applications and infrastructure. In the platform context, the OpenTelemetry Collector acts as an intermediary between instrumented applications and observability backends, aggregating telemetry data before forwarding it to various monitoring and logging systems. The Collector acts as a pipeline, utilizing processors, receivers and exporters, allowing fine-grained control over how and where data is sent. ![OpenTelemetry implementation:](images/opentelemetry_diagrams.drawio.png) -This modules installs the OpenTelemetry Collector operator and two Collectors. One Collector runs as a deployment and its purpose is to collect cluster-wide telemetry content, ie. collect content related to the cluster itself, such as cluster health, utilization, etc. The second Collector runs as a daemonset and is associated with collecting application specific telemetry content for the pods of each node. Lastly, OpenTelemetry Instrumentation is deployed for application specific workloads corresponding to GPLs of the project, ie. Java, .NET, Deno, etc. This Instrumentation allows for auto-instrumentation of application workloads through injection of application libraries integrating with the Collectors, allowing for zero-code configuration of applications with OpenTelemetry and the Platform. The auto-instrumentation is bound by Kubernetes namespaces and configured via annotation, for example the following is annotation for configuration of python libraries: `instrumentation.opentelemetry.io/inject-python: true`. +This modules installs the OpenTelemetry Collector operator and two Collectors. One Collector runs as a deployment and its purpose is to collect cluster-wide telemetry content, ie. collect content related to the cluster itself, such as cluster health, utilization, etc. The second Collector runs as a daemonset and is associated with collecting application specific telemetry content for the pods of each node. Lastly, OpenTelemetry Instrumentation is deployed for application specific workloads corresponding to GPLs of the project, ie. Java, .NET, Deno, etc. This Instrumentation allows for auto-instrumentation of application workloads through injection of application libraries integrating with the Collectors, allowing for zero-code configuration of applications with OpenTelemetry and the Platform. The auto-instrumentation is bound by Kubernetes namespaces and configured via annotation, for example the following is annotation for configuration of python libraries: `instrumentation.opentelemetry.io/inject-python: true`. ## Requirements @@ -14,19 +14,18 @@ This modules installs the OpenTelemetry Collector operator and two Collectors. O | [terraform](#requirement\_terraform) | >= 0.13 | | [aws](#requirement\_aws) | >= 5.14.0 | | [helm](#requirement\_helm) | >= 2.11.0 | -| [kubectl](#requirement\_kubectl) | >= 1.14.0 | ## Providers | Name | Version | |------|---------| -| [helm](#provider\_helm) | >= 2.11.0 | +| [helm](#provider\_helm) | 2.17.0 | ## Modules | Name | Source | Version | |------|--------|---------| -| [images](#module\_images) | git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/ | tf-upgrade | +| [images](#module\_images) | git::https://github.e.it.census.gov/terraform-modules/aws-ecr-copy-images.git/ | tf-upgrade | ## Resources @@ -55,4 +54,4 @@ This modules installs the OpenTelemetry Collector operator and two Collectors. O |------|-------------| | [module\_name](#output\_module\_name) | The name of this module. | | [module\_version](#output\_module\_version) | The version of this module. | - \ No newline at end of file + diff --git a/copy_images.tf b/copy_images.tf index 95b8623..9019947 100644 --- a/copy_images.tf +++ b/copy_images.tf @@ -69,7 +69,7 @@ locals { } module "images" { - source = "git@github.e.it.census.gov:terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" + source = "git::https://github.e.it.census.gov/terraform-modules/aws-ecr-copy-images.git/?ref=tf-upgrade" profile = var.profile application_name = var.cluster_name diff --git a/examples/simple/variables.tf b/examples/simple/variables.tf index 660df39..64e253e 100644 --- a/examples/simple/variables.tf +++ b/examples/simple/variables.tf @@ -36,4 +36,4 @@ variable "tempo_endpoint" { description = "The endpoint for the tempo service" type = string default = "" -} \ No newline at end of file +} diff --git a/main.tf b/main.tf index 7514e7d..d4c7d7d 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,5 @@ locals { - telemetry_ns = var.namespace - - otel_internal_hostname = format("otel-daemonset-collector.%v.svc.cluster.local", local.telemetry_ns) + otel_internal_hostname = format("otel-daemonset-collector.%v.svc.cluster.local", var.namespace) otel_internal_port_number = 4318 otel_internal_url = format("http://%v:%v/", local.otel_internal_hostname, local.otel_internal_port_number) } @@ -14,7 +12,7 @@ resource "helm_release" "opentelemetry-operator" { chart = "opentelemetry-operator" name = "opentelemetry-operator" - namespace = local.telemetry_ns + namespace = var.namespace repository = var.otel_helm_repo version = var.otel_helm_version @@ -78,7 +76,7 @@ resource "helm_release" "otel-resources" { chart = "./otel" name = "otel-resources" - namespace = local.telemetry_ns + namespace = var.namespace version = local.module_version set { @@ -126,4 +124,4 @@ resource "helm_release" "otel-resources" { name = "autoinstrumentation.java.tag" value = module.images.images[local.auto_instrumentation_java_key].tag } -} \ No newline at end of file +} diff --git a/otel/templates/java_instrumentation.yaml b/otel/templates/java_instrumentation.yaml index ec67844..3b98a6b 100644 --- a/otel/templates/java_instrumentation.yaml +++ b/otel/templates/java_instrumentation.yaml @@ -12,4 +12,4 @@ spec: type: parentbased_traceidratio argument: "1" java: - image: "{{ .Values.autoinstrumentation.java.image }}:{{ .Values.autoinstrumentation.java.tag | default .Chart.AppVersion }}" \ No newline at end of file + image: "{{ .Values.autoinstrumentation.java.image }}:{{ .Values.autoinstrumentation.java.tag | default .Chart.AppVersion }}" diff --git a/otel/templates/otelcol_daemonset.yaml b/otel/templates/otelcol_daemonset.yaml index 3eeae79..8a2e13a 100644 --- a/otel/templates/otelcol_daemonset.yaml +++ b/otel/templates/otelcol_daemonset.yaml @@ -11,9 +11,9 @@ spec: receivers: otlp: # Used to collect application traces, metrics and logs. protocols: - grpc: + grpc: endpoint: 0.0.0.0:4317 - http: + http: endpoint: 0.0.0.0:4318 kubeletstats: # Used to pull node, pod, and container metrics from the API server on a kubelet. collection_interval: 10s @@ -141,7 +141,7 @@ spec: name: k8s.pod.uid - sources: # If neither of those work, use the requests connection to get the pod IP. - from: connection - memory_limiter: # Best practice to force telemetry backoff if the collector pushes against memory thresholds, ie. avoid oom + memory_limiter: # Best practice to force telemetry backoff if the collector pushes against memory thresholds, ie. avoid oom check_interval: 1s limit_percentage: 80 spike_limit_percentage: 25 @@ -164,7 +164,7 @@ spec: # prometheusremotewrite: # endpoint: http://mimir-monolithic-headless.mimir.svc.cluster.local:8080/api/v1/push # tls: - # insecure_skip_verify: true + # insecure_skip_verify: true # remote_write_queue: # enabled: true # queue_size: 100000 @@ -264,4 +264,4 @@ roleRef: subjects: - kind: ServiceAccount name: otel-daemon - namespace: {{ .Release.Namespace }} \ No newline at end of file + namespace: {{ .Release.Namespace }} diff --git a/otel/templates/otelcol_deployment.yaml b/otel/templates/otelcol_deployment.yaml index 1511efd..d457187 100644 --- a/otel/templates/otelcol_deployment.yaml +++ b/otel/templates/otelcol_deployment.yaml @@ -103,7 +103,7 @@ rules: - get - list - watch -- apiGroups: +- apiGroups: - "events.k8s.io" resources: - events @@ -124,4 +124,4 @@ roleRef: subjects: - kind: ServiceAccount name: otel-deployment - namespace: {{ .Release.Namespace }} \ No newline at end of file + namespace: {{ .Release.Namespace }} diff --git a/otel/values.yaml b/otel/values.yaml index 5254581..e89063b 100644 --- a/otel/values.yaml +++ b/otel/values.yaml @@ -22,4 +22,4 @@ otel: autoinstrumentation: java: image: "" - tag: "" \ No newline at end of file + tag: "" diff --git a/requirements.tf b/requirements.tf index 4eab89e..2e678e4 100644 --- a/requirements.tf +++ b/requirements.tf @@ -10,9 +10,5 @@ terraform { source = "hashicorp/helm" version = ">= 2.11.0" } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.14.0" - } } }