From 18df1b633db182a0a17bf4f55858a950fb11802f Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Mon, 7 Oct 2019 22:04:23 -0400 Subject: [PATCH] Feature/don't upload p rs to releases (#126) * update ci --- .circleci/config.yml | 400 +++++++++++++----- .env.template | 6 +- .gitignore | 3 +- clair-scan.sh | 68 +++ docker-compose-ci.yml | 2 +- ...SC4S deployment.png => SC4Sdeployment.png} | Bin docs/gettingstarted.md | 3 +- docs/gettingstarted/docker-swarm-general.md | 40 +- docs/gettingstarted/docker-swarm-rhel7.md | 39 +- docs/gettingstarted/docker-systemd-general.md | 38 +- docs/gettingstarted/podman-systemd-general.md | 45 +- docs/performance.md | 20 +- package/Dockerfile | 9 +- .../conflib/_common/compliance_meta.conf | 4 +- .../vendor_product_by_source_context.conf | 4 +- .../conflib/_splunk/splunk_context.conf | 4 +- .../context}/microfocus_arcsight_source.csv | 0 .../local/{ => config}/destinations/README.md | 0 .../local/{ => config}/filters/README.md | 0 .../local/{ => config}/filters/example.conf | 0 .../local/{ => config}/log_paths/README.md | 0 .../{ => config}/log_paths/example.conf.tmpl | 0 .../local/{ => config}/sources/README.md | 0 .../context}/compliance_meta_by_source.conf | 0 .../context}/compliance_meta_by_source.csv | 0 .../local/context}/splunk_index.csv | 0 .../context}/vendor_product_by_source.conf | 0 .../context}/vendor_product_by_source.csv | 0 .../p_rfc3164_microfocus_arcsight.conf.tmpl | 2 +- .../compliance_meta_by_source.conf | 5 + .../compliance_meta_by_source.csv | 2 + .../etc/context_templates/splunk_index.csv | 40 ++ .../vendor_product_by_source.conf | 34 ++ .../vendor_product_by_source.csv | 8 + .../source_network.t | 0 .../etc/local_config/destinations/README.md | 1 + package/etc/local_config/filters/README.md | 1 + package/etc/local_config/filters/example.conf | 4 + package/etc/local_config/log_paths/README.md | 1 + .../local_config/log_paths/example.conf.tmpl | 76 ++++ package/etc/local_config/sources/README.md | 1 + package/etc/syslog-ng.conf | 9 +- package/sbin/entrypoint.sh | 9 +- 43 files changed, 662 insertions(+), 216 deletions(-) create mode 100755 clair-scan.sh rename docs/{SC4S deployment.png => SC4Sdeployment.png} (100%) rename package/etc/{context-local => conf.d/context}/microfocus_arcsight_source.csv (100%) rename package/etc/conf.d/local/{ => config}/destinations/README.md (100%) rename package/etc/conf.d/local/{ => config}/filters/README.md (100%) rename package/etc/conf.d/local/{ => config}/filters/example.conf (100%) rename package/etc/conf.d/local/{ => config}/log_paths/README.md (100%) rename package/etc/conf.d/local/{ => config}/log_paths/example.conf.tmpl (100%) rename package/etc/conf.d/local/{ => config}/sources/README.md (100%) rename package/etc/{context-local => conf.d/local/context}/compliance_meta_by_source.conf (100%) rename package/etc/{context-local => conf.d/local/context}/compliance_meta_by_source.csv (100%) rename package/etc/{context-local => conf.d/local/context}/splunk_index.csv (100%) rename package/etc/{context-local => conf.d/local/context}/vendor_product_by_source.conf (100%) rename package/etc/{context-local => conf.d/local/context}/vendor_product_by_source.csv (100%) create mode 100644 package/etc/context_templates/compliance_meta_by_source.conf create mode 100644 package/etc/context_templates/compliance_meta_by_source.csv create mode 100644 package/etc/context_templates/splunk_index.csv create mode 100644 package/etc/context_templates/vendor_product_by_source.conf create mode 100644 package/etc/context_templates/vendor_product_by_source.csv rename package/etc/{templates => go_templates}/source_network.t (100%) create mode 100644 package/etc/local_config/destinations/README.md create mode 100644 package/etc/local_config/filters/README.md create mode 100644 package/etc/local_config/filters/example.conf create mode 100644 package/etc/local_config/log_paths/README.md create mode 100644 package/etc/local_config/log_paths/example.conf.tmpl create mode 100644 package/etc/local_config/sources/README.md diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d7ddba..65ac936 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,58 +9,60 @@ version: 2.1 orbs: - clair_scanner: ovotech/clair-scanner@1 + snyk: snyk/snyk@0.0.8 + clair-scanner: ovotech/clair-scanner@1.5.0 jobs: build: + machine: + image: ubuntu-1604:201903-01 + docker_layer_caching: true # default - false environment: - IMAGE_NAME: rfaircloth/scs - docker: - - image: circleci/buildpack-deps:stretch + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci +# docker: +# - image: circleci/buildpack-deps:stretch steps: +# - setup_remote_docker: +# docker_layer_caching: true - checkout - run: git submodule sync - run: git submodule update --init --recursive - - setup_remote_docker: - docker_layer_caching: true + - run: mkdir /tmp/artifacts + - run: docker pull gittools/gitversion:latest-linux-netcoreapp2.1 - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Build Docker image command: docker build -f package/Dockerfile --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION -t $IMAGE_NAME:$CIRCLE_SHA1 package - run: - name: Push Docker image + name: Push Docker image by hash command: docker push $IMAGE_NAME:$CIRCLE_SHA1 - build-egb: - environment: - IMAGE_NAME: rfaircloth/scs - docker: - - image: circleci/buildpack-deps:stretch - steps: - - checkout - - run: git submodule sync - - run: git submodule update --init --recursive - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: - name: Build Docker image - command: docker build -f perftests/bundlesrv/Dockerfile -t $IMAGE_NAME:egb-$CIRCLE_SHA1 perftests/bundlesrv - run: - name: Tag Docker image - command: docker tag $IMAGE_NAME:egb-$CIRCLE_SHA1 $IMAGE_NAME:egb-edge + name: Docker tag and push with version label + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + echo image $IMAGE_NAME:$SEMVER + docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$SEMVER + docker push $IMAGE_NAME:$SEMVER + - run: - name: Push Docker image - command: docker push $IMAGE_NAME:egb-$CIRCLE_SHA1 + name: Docker Save + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker save $IMAGE_NAME:$SEMVER | gzip -c > /tmp/artifacts/oci_container.tar.gz - run: - name: Push Docker image - command: docker push $IMAGE_NAME:egb-edge + name: BYOE Config + command: | + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + + - store_artifacts: + path: /tmp/artifacts + dgoss: environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci GOSS_VER: v0.3.7 GOSS_OPTS: "--max-concurrent=1 --format junit" GOSS_FILES_STRATEGY: cp @@ -73,7 +75,7 @@ jobs: docker_layer_caching: true - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 @@ -96,6 +98,8 @@ jobs: - store_test_results: path: test-results test-unit: + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci docker: - image: circleci/buildpack-deps:stretch steps: @@ -106,7 +110,7 @@ jobs: - run: git submodule update --init --recursive - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker-compose pull @@ -148,140 +152,275 @@ jobs: when: always - store_test_results: path: test-results - test-scan_images: + test-scan-synk: + docker: + - image: 'circleci/buildpack-deps:stable' environment: - IMAGE_NAME: rfaircloth/scs - executor: clair_scanner/default + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci steps: - - clair_scanner/scan: - image: $IMAGE_NAME:$CIRCLE_SHA1 - whitelist: clair-whitelist.yml + - checkout + - setup_remote_docker: + docker_layer_caching: true - run: - command: | - mkdir -p /root/project/test-results - pip install -r requirements.txt - python clair_to_junit_parser.py "/clair-reports/$IMAGE_NAME:$CIRCLE_SHA1.json" --output test-results/results.xml - when: on_fail - - store_test_results: - path: test-results/results.xml - - store_artifacts: - path: /clair-reports - - publish-common: - machine: - image: ubuntu-1604:201903-01 - docker_layer_caching: true # default - false + name: Docker Login + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + - run: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - snyk/scan: + docker-image-name: $IMAGE_NAME:$CIRCLE_SHA1 + test-scan-clair: + docker: + - image: 'docker:stable' environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci steps: - checkout + - setup_remote_docker: + docker_layer_caching: true - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - run: - name: Docker tag image - command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$SEMVER - - run: - name: Docker push tag + name: "Vulnerability scan" command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker push $IMAGE_NAME:$SEMVER + + - store_artifacts: + path: clair-reports + publish-edge: environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases docker: - - image: circleci/buildpack-deps:stretch + #- image: circleci/buildpack-deps:stretch + - image: circleci/golang:latest + steps: - setup_remote_docker: docker_layer_caching: true + - checkout + - run: + name: Create Directory + command: mkdir /tmp/artifacts - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - run: name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:edge + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:edge - run: name: Docker push tag - command: docker push $IMAGE_NAME:edge + command: docker push $PUBLIC_IMAGE_NAME:edge + + - run: + name: Docker Save + command: | + docker save $IMAGE_NAME:$CIRCLE_SHA1 | gzip -c > /tmp/artifacts/oci_container.tar.gz + - run: + name: BYOE Config + command: | + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + + - run: + name: "Publish Release on GitHub" + command: | + go get -u github.com/tcnksm/ghr + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete edge /tmp/artifacts/ publish-version: machine: image: ubuntu-1604:201903-01 docker_layer_caching: true # default - false environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases steps: + - checkout - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + name: Create Directory + command: mkdir /tmp/artifacts - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + name: Docker Login + command: | + docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: name: Docker tag image command: | SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker tag $IMAGE_NAME:$CIRCLE_SHA1 splunk/scs:$SEMVER + docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:$SEMVER + docker push $PUBLIC_IMAGE_NAME:$SEMVER + - run: - name: Docker push tag + name: Docker Save + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker save $PUBLIC_IMAGE_NAME:$SEMVER | gzip -c > /tmp/artifacts/oci_container.tar.gz + - run: + name: BYOE Config + command: | + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + - run: + name: "Publish Release on GitHub" command: | + go get -u github.com/tcnksm/ghr SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker push splunk/scs:$SEMVER + ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${SEMVER} /tmp/artifacts/ + + publish-latest: environment: - IMAGE_NAME: rfaircloth/scs + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases docker: - - image: circleci/buildpack-deps:stretch + #- image: circleci/buildpack-deps:stretch + - image: circleci/golang:latest steps: - setup_remote_docker: docker_layer_caching: true + - checkout + - run: + name: Create Directory + command: mkdir /tmp/artifacts - run: name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - run: name: Docker pull command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - run: name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 splunk/scs:latest + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:latest - run: name: Docker push tag - command: docker push splunk/scs:latest + command: docker push $PUBLIC_IMAGE_NAME:latest + + - run: + name: Docker Save + command: | + docker save $IMAGE_NAME:$CIRCLE_SHA1 | gzip -c > /tmp/artifacts/oci_container.tar.gz + - run: + name: BYOE Config + command: | + tar rvf /tmp/artifacts/baremetal.tar -C package etc + tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + + - run: + name: "Publish Release on GitHub" + command: | + go get -u github.com/tcnksm/ghr + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete latest /tmp/artifacts/ + + + dockerhub-version: + machine: + image: ubuntu-1604:201903-01 + docker_layer_caching: true # default - false + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: splunk/scs + steps: + + - checkout + + - run: + name: Create Directory + command: mkdir artifacts + - run: + name: Docker Login + command: | + docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker tag image + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:$SEMVER + docker push $PUBLIC_IMAGE_NAME:$SEMVER + + dockerhub-edge: + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: splunk/scs + docker: + - image: circleci/buildpack-deps:stretch + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Create Directory + command: mkdir /tmp/artifacts + - run: + name: Docker Login + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + - run: + name: Docker pull + command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Docker tag image + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:edge + - run: + name: Docker push tag + command: docker push $PUBLIC_IMAGE_NAME:edge + + dockerhub-latest: + environment: + IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + PUBLIC_IMAGE_NAME: splunk/scs + docker: + - image: circleci/buildpack-deps:stretch + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Create Directory + command: mkdir /tmp/artifacts + - run: + name: Docker Login + command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN + - run: + name: Docker pull + command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Docker tag image + command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:latest + - run: + name: Docker push tag + command: docker push $PUBLIC_IMAGE_NAME:latest + workflows: - build-publish: + version: 2 + build-branches: jobs: - build - - build-egb - dgoss: requires: - build - test-unit: requires: - build - - test-scan_images: + - test-scan-synk: requires: - build - - publish-common: - requires: - - dgoss - - test-unit -# filters: -# branches: -# only: -# - master -# - develop +#Clair scanner image is broken using synk for now +# - test-scan-clair: +# requires: +# - build - publish-edge: requires: - dgoss @@ -290,7 +429,7 @@ workflows: branches: only: - develop - - publish-version: + - publish-latest: requires: - dgoss - test-unit @@ -298,8 +437,15 @@ workflows: branches: only: - master - - /^release\/.*/ - - publish-latest: + - dockerhub-edge: + requires: + - dgoss + - test-unit + filters: + branches: + only: + - develop + - dockerhub-latest: requires: - dgoss - test-unit @@ -307,3 +453,53 @@ workflows: branches: only: - master + build-tags: + jobs: + - build: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + - dgoss: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - build + - test-unit: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - build + - test-scan-synk: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - build + - publish-version: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - dgoss + - test-unit + - dockerhub-version: + filters: + tags: + only: /^\d*\.\d*\.\d*.*$/ + branches: + ignore: /.*/ + requires: + - dgoss + - test-unit \ No newline at end of file diff --git a/.env.template b/.env.template index 8d0cafa..c1f7af2 100644 --- a/.env.template +++ b/.env.template @@ -16,6 +16,6 @@ SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics -SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download -SPLUNKBASE_USERNAME=username -SPLUNKBASE_PASSWORD=password +#SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download +#SPLUNKBASE_USERNAME=username +#SPLUNKBASE_PASSWORD=password diff --git a/.gitignore b/.gitignore index 3ac4e78..4d40b52 100644 --- a/.gitignore +++ b/.gitignore @@ -382,4 +382,5 @@ fabric.properties /.idea/ tests/test_plugin_*.py -package/etc/conf.d/local/ \ No newline at end of file +# package/etc/conf.d/local/ +!package/etc/conf.d/local diff --git a/clair-scan.sh b/clair-scan.sh new file mode 100755 index 0000000..e890ce3 --- /dev/null +++ b/clair-scan.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + + set -e + + REPORT_DIR=clair-reports + mkdir $REPORT_DIR || true + + #DB=$(docker run -p 5432:5432 -d arminc/clair-db:latest) + docker run -p 5432:5432 -d --rm --name db arminc/clair-db:latest + #CLAIR=$(docker run -p 6060:6060 --link "$DB":postgres -d arminc/clair-local-scan:latest)' + sleep 30 + docker run -p 6060:6060 --link db:postgres -d --rm --name clair arminc/clair-local-scan:latest + #CLAIR_SCANNER=$(docker run -v /var/run/docker.sock:/var/run/docker.sock --link clair:clair --name clairscanner --rm -d ovotech/clair-scanner@sha256:53fe8e8ac63af330d2dfc63498d23d8825d07f916f7d230271176de06d12acd6 tail -f /dev/null) + + CLAIR_SCANNER=$(docker run --link clair:clair --name clairscanner --rm -d ovotech/clair-scanner@sha256:53fe8e8ac63af330d2dfc63498d23d8825d07f916f7d230271176de06d12acd6 tail -f /dev/null) + + #clair_ip=$(docker exec -it "$CLAIR" hostname -i | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+') + #scanner_ip=$(docker exec -it "$CLAIR_SCANNER" hostname -i | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+') + + docker cp "clair-whitelist.yml" "$CLAIR_SCANNER:/whitelist.yml" + WHITELIST="-w /whitelist.yml" + + function scan() { + echo Scanning $1 + local image=$1 + # replace forward-slashes and colons with underscores + munged_image=$(echo "$image" | sed 's/\//_/g' | sed 's/:/_/g') + sanitised_image_filename="${munged_image}.json" + local ret=0 + #--ip "$scanner_ip" \ + # + local docker_cmd=(docker exec -it "$CLAIR_SCANNER" clair-scanner \ + --clair=http://clair:6060 \ + -t "high" \ + --report "$REPORT_DIR/$sanitised_image_filename" \ + --log "$REPORT_DIR/log.json" --whitelist=${WHITELIST:+"-x"} + --reportAll=true \ + --exit-when-no-features=false \ + "$image") + + docker pull "$image" + + "${docker_cmd[@]}" 2>&1 || ret=$? + if [ $ret -eq 0 ]; then + echo "No unapproved vulnerabilities" + elif [ $ret -eq 1 ]; then + echo "Unapproved vulnerabilities found" + EXIT_STATUS=1 + elif [ $ret -eq 5 ]; then + echo "Image was not scanned, not supported." + EXIT_STATUS=1 + else + echo "Unknown clair-scanner return code $ret." + EXIT_STATUS=1 + fi + + docker cp "$CLAIR_SCANNER:/$sanitised_image_filename" "$REPORT_DIR/$sanitised_image_filename" || true + } + + EXIT_STATUS=0 + + scan "$IMAGE_NAME:$CIRCLE_SHA1" + + docker kill clairscanner + docker kill clair + docker kill db + + exit $EXIT_STATUS \ No newline at end of file diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index c33122c..b4bb66d 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -19,7 +19,7 @@ services: - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} sc4s: - image: rfaircloth/scs:${CIRCLE_SHA1} + image: ${IMAGE_NAME}:${CIRCLE_SHA1} hostname: sc4s ports: - "514" diff --git a/docs/SC4S deployment.png b/docs/SC4Sdeployment.png similarity index 100% rename from docs/SC4S deployment.png rename to docs/SC4Sdeployment.png diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index 7a29b37..a43e906 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -21,6 +21,7 @@ instance in the same VLAN as the source device. environment. * Avoid TCP except where the source is unable to contain the event to a single UDP packet. * Avoid TLS except where the event may cross a untrusted network. +* Plan for appropriately sized hardware (see)[performance.md] # Implementation @@ -81,4 +82,4 @@ Splunk type. Additional hosts can be deployed for syslog collection from additional network zones and locations: -![SC4S deployment diagram](SC4S%20deployment.png) \ No newline at end of file +![SC4S deployment diagram](SC4Sdeployment.png) \ No newline at end of file diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 9fef61c..f9a6e12 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -35,21 +35,11 @@ services: - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the yml template -above (`volumes` declarations) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. -Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. +* NOTE: If you use the default `volumes` declarations as-is from the `docker-compose.yml` file template example, you must create and/or download all files and directories referenced in the file according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s Environment" section. Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. ## Configure the SC4S environment -Create the following file ``/opt/sc4s/env_file`` and add the environment variables below: - -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment - -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. - -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event @@ -62,32 +52,38 @@ SPLUNK_METRICS_INDEX=em_metrics #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment. + +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example above. + ## Configure index destinations for Splunk -Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow -customization of index destinations. +Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. -* Create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above. From this directory, +* Create a subdirectory called ``default`` in the directory that you created in the previous step (e.g. ``/opt/sc4s/``). Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the command below to download the index context file: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv ``` -* Edit splunk_index.csv review the index configuration and revise as required for sourcetypes utilized in your environment. +* Edit splunk_index.csv to review the index configuration and revise as required for the sourcetypes utilized in your environment. ## Configure sources by source IP or host name -Legacy sources and non-standard-compliant source require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* If not already done in the step immediately above, create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) -created in the first step above. From this directory, execute the commands below to download the vendor context files: +* If not already done, create a subdirectory called ``default`` in the ``/opt/sc4s/`` directory. Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the following commands to download the vendor context files: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv ``` -* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. +* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. ## Start/Restart SC4S @@ -224,4 +220,4 @@ Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', cl ``` If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more -information. \ No newline at end of file +information. diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index eba329c..008dd65 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -63,21 +63,12 @@ services: - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the yml template -above (`volumes` declarations) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. -Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. +* NOTE: If you use the default `volumes` declarations as-is from the `docker-compose.yml` file template example, do create and/or download all files and directories referenced in the file according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s environment" section. Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. -## Configure the SC4S environment - -Create the following file ``/opt/sc4s/env_file`` and add the environment variables below: -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment - -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +## Configure the SC4S environment -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event @@ -90,32 +81,40 @@ SPLUNK_METRICS_INDEX=em_metrics #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment. + +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example below. + + + ## Configure index destinations for Splunk -Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow -customization of index destinations. +Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. -* Create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above. From this directory, +* Create a subdirectory called ``default`` in the directory that you created in the previous step (e.g. ``/opt/sc4s/``). Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the command below to download the index context file: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv ``` -* Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. +* Edit splunk_index.csv to review the index configuration and revise as required for the sourcertypes utilized in your environment. ## Configure sources by source IP or host name Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* If not already done in the step immediately above, create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) -created in the first step above. From this directory, execute the commands below to download the vendor context files: +* If not already done, create a subdirectory called ``default`` in the ``/opt/sc4s/`` directory. Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the following commands to download the vendor context files: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv ``` -* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. +* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. ## Start/Restart SC4S diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index d4cd5c4..0e1fc58 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -42,20 +42,11 @@ ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp \ $SC4S_IMAGE ``` -* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the unit -file above (the `Environment` assignments) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. +* NOTE: If you use the default `Environment` assignments as-is from the `sc4s.service` unit file template example, do create and/or download all files and directories referenced in the file's Service stanza according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s environment" section. ## Configure the SC4S environment -Create the following file ``/opt/sc4s/default/env_file`` and add the environment variables below: - -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment - -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. - -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event @@ -68,33 +59,38 @@ SPLUNK_METRICS_INDEX=em_metrics #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example. + + ## Configure index destinations for Splunk -Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow -customization of index destinations. +Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. -* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the unit file above (the ``-v`` variables) -match the directory you created above. From this directory, execute the following to download the latest index context file: +* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest index context file: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv ``` -* Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. +* Edit splunk_index.csv to review the index configuration and revise as required for the sourcertypes utilized in your environment. ## Configure sources by source IP or host name Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +apply to support such sources. To identify sources that require this step refer to the "sources" section of this documentation. -* If not already done in the step immediately above, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory -references in the unit file above (the ``-v`` variables) match the directory you created above. From this directory, execute the following to -download the latest vendor context files: +* If not already done, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest vendor context files: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv ``` -* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. +* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. ## Configure SC4S for systemd and start SC4S diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 74b6855..361ac12 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -20,8 +20,8 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" #Note Uncomment the following two lines for host and ip based source type mapping AND download the two file templates per getting started -#Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" -#Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" +Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" +Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" #Uncomment the following line if custom TLS certs are provided #Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -42,20 +42,11 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp \ $SC4S_IMAGE ``` -* NOTE: While strictly optional, it is recommended that you create and/or download all files and directories referenced in the unit -file above (the `Environment` assignments) according to the configuration steps that follow. The TLS options are described in the "Configuration" section. +* NOTE: If you use the default `Environment` assignments as-is from the `sc4s.service` unit file template example, do create and/or download all files and directories referenced in the file's Service stanza according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s environment" section. ## Configure the sc4s environment -Create the following file ``/opt/sc4s/default/env_file`` and add the environment variables below: - -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment - -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. - -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event @@ -68,33 +59,37 @@ SPLUNK_METRICS_INDEX=em_metrics #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, +match this value to the total number of indexers behind the load balancer. + +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example. + + ## Configure index destinations for Splunk -Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow -customization of index destinations. +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the unit file above (the ``-v`` variables) -match the directory you created above. From this directory, execute the following to download the latest index context file: +* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest index context file: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv ``` -* Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment. +* Edit splunk_index.csv to review the index configuration and revise as required for the sourcertypes utilized in your environment. ## Configure sources by source IP or host name -Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* If not already done in the step immediately above, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory -references in the unit file above (the ``-v`` variables) match the directory you created above. From this directory, execute the following to -download the latest vendor context files: +* If not already done, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest vendor context files: ```bash sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv ``` -* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax. +* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. ## Configure SC4S for systemd and start SC4S @@ -235,4 +230,4 @@ Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', cl ``` If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more -information. \ No newline at end of file +information. diff --git a/docs/performance.md b/docs/performance.md index 966a318..ea02c72 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -22,5 +22,21 @@ average rate = 9717.58 msg/sec, count=1749420, time=180.026, (average) msg size= ## Limitations -Splunk Enterprise's implementation of the http event collection server responds to the client with a status code 200 and fails to commit the events to disk during a rolling restart. In our testing, 20-30 events per indexer are lost. - +In our tests, if Splunk Enterprise’s implementation of the http event collection server responded to the client with a status code 200 and failed to commit the events to disk during a rolling restart, then 20-30 events per indexer were lost. + +## Guidance on sizing hardware + +The following reference deployment hardware specifications are based on Splunk performance testing results in Amazon Web Services. +The overall load on your deployment hardware will vary based on the percentage of events not handled by a filter or use of +exceptionally complex regex in filters. While we consider the following conservative, actual hardware performance will vary +due to network interface card, driver, kernel version, exact CPU, type of memory and configuration. SYSLOG is a fire +and forget protocol making it sensitive to performance. Given this it is highly recommended that you validate +performance with your hardware and production data samples. The syslog-ng loggen tool available in the SC4S container +and the commands above can be utilized in this effort. + +Deployment Size | Hardware Spec | Average EPS with average msg size 800 k +-- | -- | -- +Small | 2 X 3.1 ghz cores1 GB of memory | 2K msg/sec +Medium | 4 X 3.1 ghz cores2 GB of memory | 4.5K msg/sec +Large | 8 X 3.1 ghz cores4 GB of memory | 9K msg/sec +XL | 16 X 3.1 ghz cores8 GB of memory | 18K msg/sec diff --git a/package/Dockerfile b/package/Dockerfile index a5f67cc..7008a31 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -86,12 +86,13 @@ RUN curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gompl COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d -COPY etc/templates /opt/syslog-ng/etc/templates -COPY etc/context-local /opt/syslog-ng/etc/context-local +COPY etc/go_templates /opt/syslog-ng/etc/go_templates +COPY etc/context_templates /opt/syslog-ng/etc/context_templates +COPY etc/local_config /opt/syslog-ng/etc/local_config + COPY sbin/entrypoint.sh / RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V -#RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t EXPOSE 514 EXPOSE 601/tcp @@ -101,4 +102,4 @@ ENV SPLUNK_CONNECT_METHOD=UF ENTRYPOINT ["/entrypoint.sh", "-F"] -HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate +HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/compliance_meta.conf b/package/etc/conf.d/conflib/_common/compliance_meta.conf index 3529ae5..75479c6 100644 --- a/package/etc/conf.d/conflib/_common/compliance_meta.conf +++ b/package/etc/conf.d/conflib/_common/compliance_meta.conf @@ -1,7 +1,7 @@ parser compliance_meta_by_source { add-contextual-data( - selector(filters("`syslog-ng-sysconfdir`/context-local/compliance_meta_by_source.conf")), - database("context-local/compliance_meta_by_source.csv") + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/compliance_meta_by_source.conf")), + database("conf.d/local/context/compliance_meta_by_source.csv") ignore-case(yes) ); }; diff --git a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf index f3789a7..44d8ff1 100644 --- a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf +++ b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf @@ -1,7 +1,7 @@ block parser vendor_product_by_source() { add-contextual-data( - selector(filters("`syslog-ng-sysconfdir`/context-local/vendor_product_by_source.conf")), - database("context-local/vendor_product_by_source.csv") + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.conf")), + database("conf.d/local/context/vendor_product_by_source.csv") ignore-case(yes) prefix("fields.") ); diff --git a/package/etc/conf.d/conflib/_splunk/splunk_context.conf b/package/etc/conf.d/conflib/_splunk/splunk_context.conf index e2d1fbb..6fb181d 100644 --- a/package/etc/conf.d/conflib/_splunk/splunk_context.conf +++ b/package/etc/conf.d/conflib/_splunk/splunk_context.conf @@ -1,7 +1,7 @@ block parser p_add_context_splunk(key("syslogng-fallback")) { add-contextual-data( selector("`key`"), - database("context-local/splunk_index.csv"), + database("conf.d/local/context/splunk_index.csv"), prefix(".splunk.") ); -}; \ No newline at end of file +}; diff --git a/package/etc/context-local/microfocus_arcsight_source.csv b/package/etc/conf.d/context/microfocus_arcsight_source.csv similarity index 100% rename from package/etc/context-local/microfocus_arcsight_source.csv rename to package/etc/conf.d/context/microfocus_arcsight_source.csv diff --git a/package/etc/conf.d/local/destinations/README.md b/package/etc/conf.d/local/config/destinations/README.md similarity index 100% rename from package/etc/conf.d/local/destinations/README.md rename to package/etc/conf.d/local/config/destinations/README.md diff --git a/package/etc/conf.d/local/filters/README.md b/package/etc/conf.d/local/config/filters/README.md similarity index 100% rename from package/etc/conf.d/local/filters/README.md rename to package/etc/conf.d/local/config/filters/README.md diff --git a/package/etc/conf.d/local/filters/example.conf b/package/etc/conf.d/local/config/filters/example.conf similarity index 100% rename from package/etc/conf.d/local/filters/example.conf rename to package/etc/conf.d/local/config/filters/example.conf diff --git a/package/etc/conf.d/local/log_paths/README.md b/package/etc/conf.d/local/config/log_paths/README.md similarity index 100% rename from package/etc/conf.d/local/log_paths/README.md rename to package/etc/conf.d/local/config/log_paths/README.md diff --git a/package/etc/conf.d/local/log_paths/example.conf.tmpl b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl similarity index 100% rename from package/etc/conf.d/local/log_paths/example.conf.tmpl rename to package/etc/conf.d/local/config/log_paths/example.conf.tmpl diff --git a/package/etc/conf.d/local/sources/README.md b/package/etc/conf.d/local/config/sources/README.md similarity index 100% rename from package/etc/conf.d/local/sources/README.md rename to package/etc/conf.d/local/config/sources/README.md diff --git a/package/etc/context-local/compliance_meta_by_source.conf b/package/etc/conf.d/local/context/compliance_meta_by_source.conf similarity index 100% rename from package/etc/context-local/compliance_meta_by_source.conf rename to package/etc/conf.d/local/context/compliance_meta_by_source.conf diff --git a/package/etc/context-local/compliance_meta_by_source.csv b/package/etc/conf.d/local/context/compliance_meta_by_source.csv similarity index 100% rename from package/etc/context-local/compliance_meta_by_source.csv rename to package/etc/conf.d/local/context/compliance_meta_by_source.csv diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/conf.d/local/context/splunk_index.csv similarity index 100% rename from package/etc/context-local/splunk_index.csv rename to package/etc/conf.d/local/context/splunk_index.csv diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/conf.d/local/context/vendor_product_by_source.conf similarity index 100% rename from package/etc/context-local/vendor_product_by_source.conf rename to package/etc/conf.d/local/context/vendor_product_by_source.conf diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/conf.d/local/context/vendor_product_by_source.csv similarity index 100% rename from package/etc/context-local/vendor_product_by_source.csv rename to package/etc/conf.d/local/context/vendor_product_by_source.csv diff --git a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl index 6170b56..4c89731 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl @@ -22,7 +22,7 @@ parser p_microfocus_arcsight_ts_end { parser p_microfocus_arcsight_source { add-contextual-data( selector("${fields.cef_device_vendor}_${fields.cef_device_product}"), - database("context-local/microfocus_arcsight_source.csv") + database("conf.d/context/microfocus_arcsight_source.csv") ignore-case(yes) prefix(".splunk.") default-selector("unknown") diff --git a/package/etc/context_templates/compliance_meta_by_source.conf b/package/etc/context_templates/compliance_meta_by_source.conf new file mode 100644 index 0000000..1d5acae --- /dev/null +++ b/package/etc/context_templates/compliance_meta_by_source.conf @@ -0,0 +1,5 @@ +@version: 3.23 +filter f_test_test { + host("something-*" type(glob)) or + netmask(192.168.100.1/24) +}; diff --git a/package/etc/context_templates/compliance_meta_by_source.csv b/package/etc/context_templates/compliance_meta_by_source.csv new file mode 100644 index 0000000..6608db0 --- /dev/null +++ b/package/etc/context_templates/compliance_meta_by_source.csv @@ -0,0 +1,2 @@ +#f_test_test,.splunk.index,"badindex" +#f_test_test,fields.compliance,"pci" diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv new file mode 100644 index 0000000..e93911a --- /dev/null +++ b/package/etc/context_templates/splunk_index.csv @@ -0,0 +1,40 @@ +#bluecoat_proxy,index,netproxy +#cef_ArcSight_ArcSight,index,netwaf +#cef_Incapsula_SIEMintegration,index,netwaf +#cef_Microsoft_Microsoft Windows,index,oswinsec +#cef_Microsoft_System or Application Event,index,oswin +#cisco_asa,index,netfw +#cisco_ios,index,netops +#cisco_nx_os,index,netops +#local_example,index,main +#fortinet_fortios_event,index,netops +#fortinet_fortios_log,index,netops +#fortinet_fortios_traffic,index,netfw +#fortinet_fortios_utm,index,netids +#juniper_idp,index,netids +#juniper_structured,index,netops +#juniper_idp_structured,index,netids +#juniper_junos_fw_structured,index,netfw +#juniper_junos_ids_structured,index,netids +#juniper_junos_utm_structured,index,netfw +#juniper_junos_fw,index,netfw +#juniper_junos_ids,index,netids +#juniper_junos_utm,index,netfw +#juniper_sslvpn,index,netfw +#juniper_netscreen,index,netfw +#juniper_nsm,index,netfw +#juniper_nsm_idp,index,netids +#juniper_legacy,index,netops +#pan_traffic,index,netfw +#pan_threat,index,netproxy +#pan_system,index,netops +#pan_config,index,netops +#pan_hipwatch,index,main +#pan_correlation,index,main +#pan_userid,index,netauth +#pan_unknown,index,netops +#proofpoint_pps_filter,index,email +#proofpoint_pps_sendmail,index,email +#sc4s_events,index,main +#sc4s_fallback,index,main +#sc4s_metrics,index,em_metrics diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf new file mode 100644 index 0000000..342c9f4 --- /dev/null +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -0,0 +1,34 @@ +@version: 3.23 + +filter f_test_test { + host("testvp-*" type(glob)) or + netmask(192.168.100.1/24) +}; +filter f_juniper_nsm { + host("jnpnsm-*" type(glob)) or + netmask(192.168.1.0/24) +}; +filter f_juniper_nsm_idp { + host("jnpnsmidp-*" type(glob)) or + netmask(192.168.2.0/24) +}; +filter f_juniper_idp { + host("jnpidp-*" type(glob)) or + netmask(192.168.3.0/24) +}; +filter f_juniper_netscreen { + host("jnpns-*" type(glob)) or + netmask(192.168.4.0/24) +}; +filter f_cisco_nx_os { + host("csconx-*" type(glob)) or + netmask(192.168.5.0/24) +}; +filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) or + netmask(192.168.6.0/24) +}; +filter f_proofpoint_pps_filter { + host("pps-*" type(glob)) or + netmask(192.168.7.0/24) +}; \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv new file mode 100644 index 0000000..3f90603 --- /dev/null +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -0,0 +1,8 @@ +f_test_test,sc4s_vendor_product,"test_test" +f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" +f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" +f_juniper_idp,sc4s_vendor_product,"juniper_idp" +f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" +f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" +f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" +f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/package/etc/templates/source_network.t b/package/etc/go_templates/source_network.t similarity index 100% rename from package/etc/templates/source_network.t rename to package/etc/go_templates/source_network.t diff --git a/package/etc/local_config/destinations/README.md b/package/etc/local_config/destinations/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/destinations/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/filters/README.md b/package/etc/local_config/filters/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/filters/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/filters/example.conf b/package/etc/local_config/filters/example.conf new file mode 100644 index 0000000..047fdc8 --- /dev/null +++ b/package/etc/local_config/filters/example.conf @@ -0,0 +1,4 @@ + +filter f_local_example { + program(sc4splugin); +}; \ No newline at end of file diff --git a/package/etc/local_config/log_paths/README.md b/package/etc/local_config/log_paths/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/log_paths/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl new file mode 100644 index 0000000..a8ac264 --- /dev/null +++ b/package/etc/local_config/log_paths/example.conf.tmpl @@ -0,0 +1,76 @@ +# LOCAL_EXAMPLE + +# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique +# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the +# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ + +# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), +# the template generator will build a custom source based on the value of one or more of the set variables. + +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} + +# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name +# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". +# "parser" can be customized on dedicated ports only +# "common" uses the same parser sequence as the default ports and is the most commonly used + +{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} + +# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path +# The ports used are based on the values of one or more of the environment variables set above. + +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { + +# The first time this template is used the log_path will be linked to the default port + +{{- if eq (.) "yes"}} + source(s_default-ports); + +# Filters should be updated to use the simplest and most effecient logic possible to discard +# the message from this path + + filter(f_is_rfc3164); + filter(f_local_example); +{{- end}} +{{- if eq (.) "no"}} + +# In the second pass through the template a link to the dedicated port is used. This +# normally does not require additional filters + +source (s_dedicated_port_LOCAL_EXAMPLE); +{{- end}} + +#Set a default sourcetype and index + + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + +#using the key "local_example" find any cutomized index,source or sourcetype meta values + + parser {p_add_context_splunk(key("local_example")); }; + +# Any additional logic needed to process the event before sending to Splunk goes here + +# Send it to Splunk + + destination(d_hec); #--HEC-- + +# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will +# forward events to another system + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic + + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic + +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/local_config/sources/README.md b/package/etc/local_config/sources/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/sources/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index e4a0a78..d0422fb 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -62,8 +62,7 @@ options { @include "conf.d/destinations/*.conf" @include "conf.d/log_paths/*.conf" -@include "conf.d/local/filters/*.conf" -@include "conf.d/local/filters/*/*.conf" -@include "conf.d/local/sources/*.conf" -@include "conf.d/local/destinations/*.conf" -@include "conf.d/local/log_paths/*.conf" +@include "conf.d/local/config/filters/*.conf" +@include "conf.d/local/config/sources/*.conf" +@include "conf.d/local/config/destinations/*.conf" +@include "conf.d/local/config/log_paths/*.conf" diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index d100fc0..30537c7 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -7,10 +7,15 @@ do echo Templating conf for $d gomplate \ --input-dir=$d \ - --template t=etc/templates/ \ + --template t=etc/go_templates/ \ --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" done -echo syslog-ng started +mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ +mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ +cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ +cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ + +echo syslog-ng starting exec /opt/syslog-ng/sbin/syslog-ng $@ \ No newline at end of file