diff --git a/.circleci/config.yml b/.circleci/config.yml index 65ac936..a981500 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,117 +7,104 @@ #You should have received a copy of the CC0 legalcode along with this #work. If not, see . version: 2.1 - orbs: + docker: circleci/docker@0.5.13 + go: circleci/go@0.2.0 snyk: snyk/snyk@0.0.8 - clair-scanner: ovotech/clair-scanner@1.5.0 - + versioning: kollex/versioning@1.0.0 jobs: build: - machine: - image: ubuntu-1604:201903-01 - docker_layer_caching: true # default - false + docker: + - image: circleci/buildpack-deps:18.04 environment: - IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci -# docker: -# - image: circleci/buildpack-deps:stretch + REGISTRY: docker.pkg.github.com + CI_IMAGE: splunk/splunk-connect-for-syslog/ci steps: -# - setup_remote_docker: -# docker_layer_caching: true + - setup_remote_docker: + docker_layer_caching: true - checkout - - run: git submodule sync - run: git submodule update --init --recursive - - run: mkdir /tmp/artifacts - - run: docker pull gittools/gitversion:latest-linux-netcoreapp2.1 - - run: - name: Docker Login - command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - - run: - name: Build Docker image - command: docker build -f package/Dockerfile --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION -t $IMAGE_NAME:$CIRCLE_SHA1 package - - run: - name: Push Docker image by hash - command: docker push $IMAGE_NAME:$CIRCLE_SHA1 + - run: mkdir /tmp/workspace + + - versioning/define_version + - versioning/create_version_file: + version_file_path: /tmp/workspace/VERSION + + - docker/install-docker + - docker/check: + registry: $REGISTRY + docker-username: GITHUB_USER + docker-password: GITHUB_TOKEN + - docker/build: + image: $CI_IMAGE + registry: $REGISTRY + path: package + extra_build_args: --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION + + - docker/install-goss - run: - name: Docker tag and push with version label + name: Test Docker command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - echo image $IMAGE_NAME:$SEMVER - docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$SEMVER - docker push $IMAGE_NAME:$SEMVER + mkdir -p /tmp/test-results + # Don't forget path! + export PATH=$PATH:~/bin + pushd package + GOSS_FMT_OPTIONS="perfdata verbose" GOSS_FILES_STRATEGY=cp GOSS_OPTS="--max-concurrent=1 --format junit" dgoss run -t $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 3>&1 1>&2 2>&3 > /tmp/goss.tmp || true ; cat /tmp/goss.tmp | grep -v "<" + popd + cat /tmp/goss.tmp | grep "[<\>]" > /tmp/test-results/goss.xml + - docker/push: + registry: $REGISTRY + image: $CI_IMAGE + + - run: | + CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') + if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi + if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi + docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 $REGISTRY/$CI_IMAGE:$DOCKER_TAG + docker push $REGISTRY/$CI_IMAGE:$DOCKER_TAG - run: name: Docker Save command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker save $IMAGE_NAME:$SEMVER | gzip -c > /tmp/artifacts/oci_container.tar.gz + docker save $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 | gzip -c > /tmp/workspace/oci_container.tar.gz - run: name: BYOE Config command: | - tar rvf /tmp/artifacts/baremetal.tar -C package etc - tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh + tar rvf /tmp/workspace/baremetal.tar -C package etc + tar rvf /tmp/workspace/baremetal.tar -C package/sbin entrypoint.sh - - store_artifacts: - path: /tmp/artifacts + - store_test_results: + path: /tmp/test-results/goss.xml - dgoss: - environment: - IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci - GOSS_VER: v0.3.7 - GOSS_OPTS: "--max-concurrent=1 --format junit" - GOSS_FILES_STRATEGY: cp + - persist_to_workspace: + root: /tmp + paths: + - workspace - docker: - - image: circleci/buildpack-deps:stretch - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Docker Login - command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - run: - name: Install goss - command: | - # rather than give internet scripts SU rights, we install to local user bin and add to path - curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sudo sh - goss -version - - run: - name: Test Docker - command: | - mkdir -p ~/project/test-results - # Don't forget path! - export PATH=$PATH:~/bin - pushd package - GOSS_FMT_OPTIONS="perfdata verbose" GOSS_FILES_STRATEGY=cp GOSS_OPTS="--max-concurrent=1 --format junit" dgoss run -t $IMAGE_NAME:$CIRCLE_SHA1 3>&1 1>&2 2>&3 > /tmp/goss.tmp || true ; cat /tmp/goss.tmp | grep -v "<" - popd - cat /tmp/goss.tmp | grep "[<\>]" > ~/project/test-results/goss.xml - - store_test_results: - path: test-results test-unit: environment: IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci docker: - - image: circleci/buildpack-deps:stretch + - image: circleci/buildpack-deps:18.04 + environment: + REGISTRY: docker.pkg.github.com + CI_IMAGE: splunk/splunk-connect-for-syslog/ci steps: - setup_remote_docker: docker_layer_caching: true - checkout - run: git submodule sync - run: git submodule update --init --recursive - - run: - name: Docker Login - command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - - run: - name: Docker pull - command: docker-compose pull + + - docker/install-docker-tools + - docker/check: + registry: $REGISTRY + docker-username: GITHUB_USER + docker-password: GITHUB_TOKEN - run: name: Setup test volume command: | - mkdir test-results + mkdir /tmp/test-results docker volume create sc4s-tests docker volume create sc4s-results @@ -136,6 +123,7 @@ jobs: - run: name: Docker Compose up command: | + export IMAGE_NAME=$CIRCLE_PROJECT_REPONAME docker-compose -f ./docker-compose-ci.yml up -d splunk sleep 70 docker-compose -f ./docker-compose-ci.yml up --abort-on-container-exit @@ -147,307 +135,253 @@ jobs: -v sc4s-results:/work/test-results \ registry.access.redhat.com/ubi7/ubi - docker cp dummy:/work/test-results/functional test-results + docker cp dummy:/work/test-results/functional /tmp/test-results docker rm dummy when: always - store_test_results: - path: test-results + path: /tmp/test-results + test-scan-synk: docker: - - image: 'circleci/buildpack-deps:stable' + - image: circleci/buildpack-deps:18.04 environment: - IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + REGISTRY: docker.pkg.github.com + CI_IMAGE: splunk/splunk-connect-for-syslog/ci steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Docker Login - command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - - run: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - snyk/scan: - docker-image-name: $IMAGE_NAME:$CIRCLE_SHA1 - test-scan-clair: + - setup_remote_docker: + docker_layer_caching: true + - checkout + - run: git submodule sync + - run: git submodule update --init --recursive + - docker/check: + registry: $REGISTRY + docker-username: GITHUB_USER + docker-password: GITHUB_TOKEN + - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 + - snyk/scan: + docker-image-name: $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 + target-file: package/Dockerfile + project: '${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BRANCH}-container' + + test-scan-synk-nomonitor: docker: - - image: 'docker:stable' + - image: circleci/buildpack-deps:18.04 environment: - IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci + REGISTRY: docker.pkg.github.com + CI_IMAGE: splunk/splunk-connect-for-syslog/ci steps: - - checkout - setup_remote_docker: docker_layer_caching: true - - run: - name: "Vulnerability scan" - command: | - - - store_artifacts: - path: clair-reports - + - checkout + - docker/check: + registry: $REGISTRY + docker-username: GITHUB_USER + docker-password: GITHUB_TOKEN + - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 + - run: git submodule sync + - run: git submodule update --init --recursive + - snyk/scan: + docker-image-name: $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 + target-file: package/Dockerfile + project: '${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BRANCH}-container' + monitor-on-build: false - publish-edge: - environment: - IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci - PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases + publish-branch: docker: - #- image: circleci/buildpack-deps:stretch - - image: circleci/golang:latest + - image: circleci/buildpack-deps:18.04 + environment: + REGISTRY: docker.pkg.github.com + CI_IMAGE: splunk/splunk-connect-for-syslog/ci + REGISTRY_PUBLIC: docker.pkg.github.com + PUBLIC_IMAGE: splunk/splunk-connect-for-syslog/releases steps: - setup_remote_docker: docker_layer_caching: true - checkout - - run: - name: Create Directory - command: mkdir /tmp/artifacts - - run: - name: Docker Login - command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - run: - name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:edge - - run: - name: Docker push tag - command: docker push $PUBLIC_IMAGE_NAME:edge - - - run: - name: Docker Save - command: | - docker save $IMAGE_NAME:$CIRCLE_SHA1 | gzip -c > /tmp/artifacts/oci_container.tar.gz - - run: - name: BYOE Config - command: | - tar rvf /tmp/artifacts/baremetal.tar -C package etc - tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh - - - run: - name: "Publish Release on GitHub" - command: | - go get -u github.com/tcnksm/ghr - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete edge /tmp/artifacts/ + - docker/check: + registry: $REGISTRY + docker-username: GITHUB_USER + docker-password: GITHUB_TOKEN + - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 + - run: | + CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') + if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi + if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi + echo docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 -t $REGISTRY_PUBLIC/$PUBLIC_IMAGE:$DOCKER_TAG + docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 $REGISTRY_PUBLIC/$PUBLIC_IMAGE:$DOCKER_TAG + docker push $REGISTRY_PUBLIC/$PUBLIC_IMAGE:$DOCKER_TAG - publish-version: - machine: - image: ubuntu-1604:201903-01 - docker_layer_caching: true # default - false + publish-edge: + docker: + - image: circleci/buildpack-deps:18.04 environment: - IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci - PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases + REGISTRY: docker.pkg.github.com + CI_IMAGE: splunk/splunk-connect-for-syslog/ci + REGISTRY_PUBLIC: docker.pkg.github.com + PUBLIC_IMAGE: splunk/splunk-connect-for-syslog/releases steps: - + - setup_remote_docker: + docker_layer_caching: true - checkout - - - run: - name: Create Directory - command: mkdir /tmp/artifacts - - run: - name: Docker Login - command: | - docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - - run: - name: Docker tag image + - docker/check: + registry: $REGISTRY + docker-username: GITHUB_USER + docker-password: GITHUB_TOKEN + - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 + - run: docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 $REGISTRY/$PUBLIC_IMAGE:edge + - docker/push: + registry: $REGISTRY + image: $PUBLIC_IMAGE + tag: edge + - docker/check: + docker-username: DOCKER_USER + docker-password: DOCKER_PASS + - run: docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 splunk/scs:edge + - docker/push: + image: splunk/scs + tag: edge + - go/install + - attach_workspace: + at: /tmp + - run: + name: "Publish edge on GitHub" command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:$SEMVER - docker push $PUBLIC_IMAGE_NAME:$SEMVER - - - run: - name: Docker Save - command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker save $PUBLIC_IMAGE_NAME:$SEMVER | gzip -c > /tmp/artifacts/oci_container.tar.gz - - run: - name: BYOE Config - command: | - tar rvf /tmp/artifacts/baremetal.tar -C package etc - tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh - - run: - name: "Publish Release on GitHub" - command: | - go get -u github.com/tcnksm/ghr - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${SEMVER} /tmp/artifacts/ - + PATH=$PATH:/usr/local/go/bin + go get -v -u github.com/tcnksm/ghr + $HOME/go/bin/ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -prerelease -delete edge /tmp/workspace/ + - store_artifacts: + path: /tmp/workspace/ publish-latest: - environment: - IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci - PUBLIC_IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/releases docker: - #- image: circleci/buildpack-deps:stretch - - image: circleci/golang:latest + - image: circleci/buildpack-deps:18.04 + environment: + REGISTRY: docker.pkg.github.com + CI_IMAGE: splunk/splunk-connect-for-syslog/ci + REGISTRY_PUBLIC: docker.pkg.github.com + PUBLIC_IMAGE: splunk/splunk-connect-for-syslog/releases + steps: - setup_remote_docker: docker_layer_caching: true - checkout - - run: - name: Create Directory - command: mkdir /tmp/artifacts - - run: - name: Docker Login - command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - run: - name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:latest - - run: - name: Docker push tag - command: docker push $PUBLIC_IMAGE_NAME:latest + - docker/check: + registry: $REGISTRY + docker-username: GITHUB_USER + docker-password: GITHUB_TOKEN + - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 + - run: docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 $REGISTRY/$PUBLIC_IMAGE:latest + - docker/push: + registry: $REGISTRY + image: $PUBLIC_IMAGE + tag: latest + - docker/check: + docker-username: DOCKER_USER + docker-password: DOCKER_PASS + - run: docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 splunk/scs:latest + - docker/push: + image: splunk/scs + tag: latest + - go/install + - attach_workspace: + at: /tmp - run: - name: Docker Save - command: | - docker save $IMAGE_NAME:$CIRCLE_SHA1 | gzip -c > /tmp/artifacts/oci_container.tar.gz - - run: - name: BYOE Config + name: "Publish edge on GitHub" command: | - tar rvf /tmp/artifacts/baremetal.tar -C package etc - tar rvf /tmp/artifacts/baremetal.tar -C package/sbin entrypoint.sh - - - run: - name: "Publish Release on GitHub" - command: | - go get -u github.com/tcnksm/ghr - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete latest /tmp/artifacts/ - - - dockerhub-version: - machine: - image: ubuntu-1604:201903-01 - docker_layer_caching: true # default - false - environment: - IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci - PUBLIC_IMAGE_NAME: splunk/scs - steps: - - - checkout - - - run: - name: Create Directory - command: mkdir artifacts - - run: - name: Docker Login - command: | - docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - run: - name: Docker tag image - command: | - SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) - docker login -u $DOCKER_USER -p $DOCKER_PASS - docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:$SEMVER - docker push $PUBLIC_IMAGE_NAME:$SEMVER + PATH=$PATH:/usr/local/go/bin + go get -v -u github.com/tcnksm/ghr + $HOME/go/bin/ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete latest /tmp/workspace/ + - store_artifacts: + path: /tmp/workspace/ - dockerhub-edge: - environment: - IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci - PUBLIC_IMAGE_NAME: splunk/scs + publish-tag: docker: - - image: circleci/buildpack-deps:stretch - steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Create Directory - command: mkdir /tmp/artifacts - - run: - name: Docker Login - command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: - name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:edge - - run: - name: Docker push tag - command: docker push $PUBLIC_IMAGE_NAME:edge - - dockerhub-latest: + - image: circleci/buildpack-deps:18.04 environment: - IMAGE_NAME: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci - PUBLIC_IMAGE_NAME: splunk/scs - docker: - - image: circleci/buildpack-deps:stretch + REGISTRY: docker.pkg.github.com + CI_IMAGE: splunk/splunk-connect-for-syslog/ci + REGISTRY_PUBLIC: docker.pkg.github.com + PUBLIC_IMAGE: splunk/splunk-connect-for-syslog/releases + steps: - setup_remote_docker: docker_layer_caching: true - - run: - name: Create Directory - command: mkdir /tmp/artifacts - - run: - name: Docker Login - command: docker login docker.pkg.github.com --username $GITHUB_USER --password $GITHUB_TOKEN - - run: - name: Docker pull - command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - - run: - name: Docker Login - command: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: - name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $PUBLIC_IMAGE_NAME:latest - - run: - name: Docker push tag - command: docker push $PUBLIC_IMAGE_NAME:latest + - checkout + - docker/check: + registry: $REGISTRY + docker-username: GITHUB_USER + docker-password: GITHUB_TOKEN + - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 + + - run: | + CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') + if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi + if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi + docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 $REGISTRY/$PUBLIC_IMAGE:$DOCKER_TAG + docker push $REGISTRY/$PUBLIC_IMAGE:$DOCKER_TAG + - docker/check: + docker-username: DOCKER_USER + docker-password: DOCKER_PASS + - run: | + CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') + if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi + if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi + docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 splunk/scs:$DOCKER_TAG + docker push splunk/scs:$DOCKER_TAG + - go/install + - attach_workspace: + at: /tmp + - run: + name: "Publish edge on GitHub" + command: | + PATH=$PATH:/usr/local/go/bin + go get -v -u github.com/tcnksm/ghr + CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') + if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi + if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi + $HOME/go/bin/ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete $VERSION /tmp/workspace/ + - store_artifacts: + path: /tmp/workspace/ workflows: version: 2 build-branches: jobs: - build - - dgoss: - requires: - - build - test-unit: requires: - build - test-scan-synk: requires: - build -#Clair scanner image is broken using synk for now -# - test-scan-clair: -# requires: -# - build - - publish-edge: - requires: - - dgoss - - test-unit filters: branches: only: - - develop - - publish-latest: + - master + - test-scan-synk-nomonitor: requires: - - dgoss - - test-unit + - build filters: branches: - only: + ignore: - master - - dockerhub-edge: + - publish-branch: + requires: + - build + - publish-edge: requires: - - dgoss + - build - test-unit filters: branches: only: - develop - - dockerhub-latest: + - publish-latest: requires: - - dgoss + - build - test-unit filters: branches: @@ -461,14 +395,6 @@ workflows: only: /^\d*\.\d*\.\d*.*$/ branches: ignore: /.*/ - - dgoss: - filters: - tags: - only: /^\d*\.\d*\.\d*.*$/ - branches: - ignore: /.*/ - requires: - - build - test-unit: filters: tags: @@ -485,21 +411,13 @@ workflows: ignore: /.*/ requires: - build - - publish-version: - filters: - tags: - only: /^\d*\.\d*\.\d*.*$/ - branches: - ignore: /.*/ + - publish-tag: requires: - - dgoss + - build - test-unit - - dockerhub-version: + - test-scan-synk filters: tags: only: /^\d*\.\d*\.\d*.*$/ branches: ignore: /.*/ - requires: - - dgoss - - test-unit \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..ebc927b --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Code of conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at [support@splunk.com](mailto:support@splunk.com). All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 21e497a..e210a11 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,120 +1,82 @@ -# Contributing +#CONTRIBUTING -### Past / Active(marked as *) Contributors +##Prerequisites -Splunk Connect for Syslog is developed by Splunkers and the open-source community. +When contributing to this repository, please first discuss the change you wish to make via a GitHub issue or Slack message with the owners of this repository. -We want to give extra special thanks to Hurricane Labs our seed community contributor +##Setup Development Environment -We thank all of our contributors! +For a basic development environment docker and a bash shell is all you need. For a more complete IDE experience see our wiki (Setup PyCharm)[https://github.com/splunk/splunk-connect-for-syslog/wiki/SC4S-Development-Setup-Using-PyCharm] -[https://github.com/splunk/splunk-connect-for-syslog/graphs/contributors] +##Contribution Workflow -For the detailed history of contributions of a given file, try +SC4S is a community project so please consider contributing your efforts! For example, documentation can always use improvement. There's always code that can be clarified, functionality that can be extended, new data filters to develop. If you see something you think should be fixed or added, go for it. -git blame file -to see line-by-line credits and +#Feature Requests and Bug Reports -git log --follow file -to see the change log even across renames and rewrites. +Have ideas on improvements or found a problem? While the community encourages everyone to contribute code, it is also appreciated when someone reports an issue. Please report any issues or bugs you find through GitHub's issue tracker. +If you are reporting a bug, please include the following details: -## Code of conduct +* Your operating system name and version +* Any details about your local setup that might be helpful in troubleshooting (ex. container runtime you use, etc.) +* Detailed steps to reproduce the bug +* We want to hear about you enhancements as well. Feel free to submit them as issues: +* Explain in detail how they should work +* Keep the scope as narrow as possible. This will make it easier to implement +##Fixing Issues -### Our Pledge +Look through our issue tracker to find problems to fix! Feel free to comment and tag community members of this project with any questions or concerns. -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of experience, -nationality, personal appearance, race, religion, or sexual identity and -orientation. +##Pull Requests -### Our Standards +What is a "pull request"? It informs the project's core developers about the changes you want to review and merge. Once you submit a pull request, it enters a stage of code review where you and others can discuss its potential modifications and even add more commits to it later on. -Examples of behavior that contributes to creating a positive environment -include: +If you want to learn more, please consult this tutorial on how pull requests work in the GitHub Help Center. -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +Here's an overview of how you can make a pull request against this project: -Examples of unacceptable behavior by participants include: +* Fork the Splunk-connect-for-syslog GitHub repository +* Clone your fork using git and create a branch off develop +$ git clone git@github.com:YOUR_GITHUB_USERNAME/splunk-connect-for-syslog.git +$ cd splunk-connect-for-syslog +* This project uses 'develop' for all development activity, so create your branch off that +$ git checkout -b your-bugfix-branch-name develop +* Run all the tests to verify your environment +$ cd splunk-connect-for-syslog +$ ./test-with-compose.sh +* Make your changes, commit and push once your tests have passed +$ git commit -m "" +$ git push +* Submit a pull request through the GitHub website using the changes from your forked codebase -* The use of sexualized language or imagery and unwelcome sexual attention or -advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting +##Code Review +There are two aspects of code review: giving and receiving. +To make it easier for your PR to receive reviews, consider the reviewers will need you to: -### Our Responsibilities +* Follow the project coding conventions +* Write good commit messages +* Break large changes into a logical series of smaller patches which individually make easily understandable changes, and in aggregate solve a broader issue +* Reviewers, the people giving the review, are highly encouraged to revisit the Code of Conduct and must go above and beyond to promote a collaborative, respectful community. +* When reviewing PRs from others The Gentle Art of Patch Review suggests an iterative series of focuses which is designed to lead new contributors to positive collaboration without inundating them initially with nuances: +* Is the idea behind the contribution sound? +* Is the contribution architected correctly? +* Is the contribution polished? +* For this project, we require that at least 2 approvals are given and a build from our continuous integration system is successful off of your branch. Please note that any new changes made with your existing pull request during review will automatically unapprove and retrigger another build/round of tests. -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. +##Testing -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. +Testing is the responsibility of all contributors. In general, we try to adhere to TDD, writing the test first. +There are multiple types of tests. The location of the test code varies with type, as do the specifics of the environment needed to successfully run the test. -### Scope +* Review existing tests in the tests folder of the repo -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. +We could always use improvements to our documentation! Anyone can contribute to these docs - whether you’re new to the project, you’ve been around a long time, and whether you self-identify as a developer, an end user, or someone who just can’t stand seeing typos. What exactly is needed? -### Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at tonyl@splunk.com. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -### Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at [http://contributor-covenant.org/version/1/4][version] - -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ - -## Filing issues - -Please file issues in this project with clear description of the problem. - -## Project Discussion Forums - -splunk-usergroups #splunk_connect_for_syslog - -## Contributing code or data - -Future Docs covering the contribution process and requirements - -By submitting a Contribution to this Work, You agree that Your Contribution is made subject to the license files applicable to this Work. Contributions comprising code shall be made pursuant the BSD2 license; Contributions consisting of configuration information or data shall be made pursuant to the CCO license. In addition, You represent that: You are the copyright owner of the Contribution or (ii) You have the requisite rights to make the Contribution. - -Definitions: - -“You” shall mean: yourself if you are making a Contribution on your own behalf; or (ii) your company, if you are making a Contribution on behalf of your company. If you are making a Contribution on behalf of your company, you represent that you have the requisite authority to do so. - -"Contribution" shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You for inclusion in, or documentation of, this project/repository. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication submitted for inclusion in this project/repository, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the maintainers of the project/repository. - -“Work” shall mean the collective software, data, content, and documentation in this project/repository. +* More complementary documentation. Have you perhaps found something unclear? +* More examples or generic templates that others can use. +* Blog posts, articles and such – they’re all very appreciated. +* You can also edit documentation files directly in the GitHub web interface, without creating a local copy. This can be convenient for small typos or grammar fixes. \ No newline at end of file diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index b4bb66d..6028e6e 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -19,7 +19,7 @@ services: - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} sc4s: - image: ${IMAGE_NAME}:${CIRCLE_SHA1} + image: ${REGISTRY}/${CI_IMAGE}:${CIRCLE_SHA1} hostname: sc4s ports: - "514" diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index f9a6e12..c407091 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -28,14 +28,23 @@ services: env_file: - /opt/sc4s/env_file volumes: - - /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv - - /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv - - /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf + - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local #Uncomment the following line if custom TLS certs are provided - - /opt/sc4s/tls:/opt/syslog-ng/tls +# - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* NOTE: If you use the default `volumes` declarations as-is from the `docker-compose.yml` file template example, you must create and/or download all files and directories referenced in the file according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s Environment" section. Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). + +* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation +of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent +restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of +the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout +to read the local configurations properly. + +* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state +when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, +the direcory will populate as it did when you first installed SC4S. Your previous changes can then +be merged back in and will take effect after another restart. ## Configure the SC4S environment @@ -60,30 +69,31 @@ match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to uncomment the last line in the example above. -## Configure index destinations for Splunk +## Modify index destinations for Splunk -Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Create a subdirectory called ``default`` in the directory that you created in the previous step (e.g. ``/opt/sc4s/``). Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, -execute the command below to download the index context file: +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. -```bash -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv -``` -* Edit splunk_index.csv to review the index configuration and revise as required for the sourcetypes utilized in your environment. - -## Configure sources by source IP or host name +## Configure source filtering by source IP or host name Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* If not already done, create a subdirectory called ``default`` in the ``/opt/sc4s/`` directory. Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the following commands to download the vendor context files: +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. -```bash -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv -``` -* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. +## Configure compliance index/metadata overrides + +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is in +the "Configuration" section. ## Start/Restart SC4S @@ -144,7 +154,7 @@ services: - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment @@ -197,7 +207,7 @@ index=* sourcetype=sc4s:events "starting up" This should yield the following event: ```ini syslog-ng starting up; version='3.22.1' -``` +``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index 008dd65..59b8801 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -63,8 +63,18 @@ services: - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* NOTE: If you use the default `volumes` declarations as-is from the `docker-compose.yml` file template example, do create and/or download all files and directories referenced in the file according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s environment" section. Failure to match the volume specification in the `yml` file with what exists locally will result in startup errors. +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation +of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent +restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of +the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout +to read the local configurations properly. + +* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state +when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, +the direcory will populate as it did when you first installed SC4S. Your previous changes can then +be merged back in and will take effect after another restart. ## Configure the SC4S environment @@ -90,31 +100,32 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. +## Modify index destinations for Splunk -## Configure index destinations for Splunk - -Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Create a subdirectory called ``default`` in the directory that you created in the previous step (e.g. ``/opt/sc4s/``). Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, -execute the command below to download the index context file: +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. -```bash -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv -``` -* Edit splunk_index.csv to review the index configuration and revise as required for the sourcertypes utilized in your environment. -## Configure sources by source IP or host name +## Configure source filtering by source IP or host name Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* If not already done, create a subdirectory called ``default`` in the ``/opt/sc4s/`` directory. Make sure the local directory volume references in the `yml` file match the directory you create here. From this directory, execute the following commands to download the vendor context files: +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. -```bash -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv -``` -* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. +## Configure compliance index/metadata overrides + +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is in +the "Configuration" section. ## Start/Restart SC4S @@ -175,7 +186,7 @@ services: - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment @@ -228,7 +239,7 @@ index=* sourcetype=sc4s:events "starting up" This should yield the following event: ```ini syslog-ng starting up; version='3.22.1' -``` +``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -251,4 +262,4 @@ Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', cl ``` If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more -information. \ No newline at end of file +information. diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 0e1fc58..4e6c216 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -42,7 +42,18 @@ ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp \ $SC4S_IMAGE ``` -* NOTE: If you use the default `Environment` assignments as-is from the `sc4s.service` unit file template example, do create and/or download all files and directories referenced in the file's Service stanza according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s environment" section. +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). + +* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation +of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent +restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of +the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout +to read the local configurations properly. + +* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state +when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, +the direcory will populate as it did when you first installed SC4S. Your previous changes can then +be merged back in and will take effect after another restart. ## Configure the SC4S environment @@ -67,30 +78,31 @@ match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to uncomment the last line in the example. +## Modify index destinations for Splunk -## Configure index destinations for Splunk - -Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest index context file: - -```bash -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv -``` -* Edit splunk_index.csv to review the index configuration and revise as required for the sourcertypes utilized in your environment. +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. -## Configure sources by source IP or host name +## Configure source filtering by source IP or host name Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources that require this step refer to the "sources" section of this documentation. +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* If not already done, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest vendor context files: +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. -```bash -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv -``` -* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. +## Configure compliance index/metadata overrides + +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is in +the "Configuration" section. ## Configure SC4S for systemd and start SC4S @@ -145,7 +157,7 @@ ExecStart=/usr/bin/docker run -p 514:514 -p 5000-5020:5000-5020 \ $SC4S_IMAGE ``` -* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment @@ -182,11 +194,11 @@ sudo systemctl start sc4s sudo systemctl restart sc4s ``` -If changes were made to the configuration Unit file above (e.g. to configure with dedicated ports), you must first stop SC4S and re-run +If changes were made to the configuration Unit file above (e.g. to configure with dedicated ports), you must first stop SC4S and re-run the systemd configuration commands: ```bash sudo systemctl stop sc4s -sudo systemctl daemon-reload +sudo systemctl daemon-reload sudo systemctl enable sc4s sudo systemctl start sc4s ``` @@ -208,7 +220,7 @@ index=* sourcetype=sc4s:events "starting up" This should yield the following event: ```ini syslog-ng starting up; version='3.22.1' -``` +``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -231,4 +243,4 @@ Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', cl ``` If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more -information. \ No newline at end of file +information. diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 361ac12..a26514e 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -42,7 +42,18 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp \ $SC4S_IMAGE ``` -* NOTE: If you use the default `Environment` assignments as-is from the `sc4s.service` unit file template example, do create and/or download all files and directories referenced in the file's Service stanza according to the configuration steps that follow. The TLS-specific options are described in the "Configure the sc4s environment" section. +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). + +* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation +of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent +restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of +the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout +to read the local configurations properly. + +* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state +when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, +the direcory will populate as it did when you first installed SC4S. Your previous changes can then +be merged back in and will take effect after another restart. ## Configure the sc4s environment @@ -67,34 +78,36 @@ match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to uncomment the last line in the example. +## Modify index destinations for Splunk -## Configure index destinations for Splunk +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. -* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest index context file: - -```bash -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv -``` -* Edit splunk_index.csv to review the index configuration and revise as required for the sourcertypes utilized in your environment. +## Configure source filtering by source IP or host name -## Configure sources by source IP or host name +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. -* If not already done, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the `sc4s.service` unit file match the directory you create here (the ``-v`` variables). From this directory, execute the following to download the latest vendor context files: +## Configure compliance index/metadata overrides -```bash -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.csv -``` -* If you have legacy sources and non-standard-compliant sources, edit the file to properly identify these products by host glob or network mask using syslog-ng filter syntax. +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is in +the "Configuration" section. ## Configure SC4S for systemd and start SC4S ```bash -sudo systemctl daemon-reload +sudo systemctl daemon-reload sudo systemctl enable sc4s sudo systemctl start sc4s ``` @@ -144,7 +157,7 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 5000-5020:5000-5020 \ $SC4S_IMAGE ``` -* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" +* Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" section for more information on your specific device(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment @@ -181,11 +194,11 @@ sudo systemctl start sc4s sudo systemctl restart sc4s ``` -If changes were made to the configuration Unit file above (e.g. to configure with dedicated ports), you must first stop SC4S and re-run +If changes were made to the configuration Unit file above (e.g. to configure with dedicated ports), you must first stop SC4S and re-run the systemd configuration commands: ```bash sudo systemctl stop sc4s -sudo systemctl daemon-reload +sudo systemctl daemon-reload sudo systemctl enable sc4s sudo systemctl start sc4s ``` @@ -207,7 +220,7 @@ index=* sourcetype=sc4s:events "starting up" This should yield the following event: ```ini syslog-ng starting up; version='3.22.1' -``` +``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: