diff --git a/.circleci/config.yml b/.circleci/config.yml index 353c0ff..2b6ea4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,421 +7,181 @@ #You should have received a copy of the CC0 legalcode along with this #work. If not, see . version: 2.1 + +test: &test + steps: + - checkout + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Setup for testing + command: | + pip install -r tests/requirements.txt + mkdir test-results + - semver-orb/export-tag + - run: + name: test + command: | + docker-compose -f tests/docker-compose-ci.yml build + docker-compose -f tests/docker-compose-ci.yml up --abort-on-container-exit + no_output_timeout: 1h + - run: + name: collect results + when: always + command: | + docker container create --name dummy \ + -v tests_results:/work/test-results \ + registry.access.redhat.com/ubi7/ubi + docker cp dummy:/work/test-results/test.xml test-results/ + - store_artifacts: + path: test-results + destination: test-results + - store_test_results: + path: test-results + +publish: &publish + steps: + - setup_remote_docker: + docker_layer_caching: true + - checkout + - run: + name: Docker Login + command: | + docker login -u $DOCKER_USER -p $DOCKER_PASS + - semver-orb/export-tag + - run: + name: Build SC4S + command: | + echo $SEMVER_VERSION >package/VERSION + if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${DOCKER_TAG}"; fi + docker build --build-arg BRANCH=${SYSLOG} package \ + -t splunk/scs:${VERSION} + - run: + name: Docker Save + command: | + mkdir -p /tmp/workspace/ + if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${DOCKER_TAG}"; fi + docker save splunk/scs:${VERSION} | gzip -c > /tmp/workspace/oci_container.tar.gz + - run: + name: BYOE Config + command: | + tar rvf /tmp/workspace/baremetal.tar -C package/etc . + tar rvf /tmp/workspace/baremetal.tar -C package/sbin entrypoint.sh + - docker/push: + image: splunk/scs + tag: $DOCKER_TAG + - go/install + - run: + name: "Publish edge on GitHub" + command: | + PATH=$PATH:/usr/local/go/bin + go get -v -u github.com/tcnksm/ghr + if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${DOCKER_TAG}"; fi + $HOME/go/bin/ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} ${GHR_FLAGS} -delete ${VERSION} /tmp/workspace/ + + - store_artifacts: + path: /tmp/workspace/ + orbs: docker: circleci/docker@0.5.20 go: circleci/go@0.2.0 snyk: snyk/snyk@0.0.8 versioning: kollex/versioning@1.0.0 + semver-orb: tv2norge/semver-orb@0.0.1 + jobs: - build: + test-sc4s-3-25-1-splunk-8-0: docker: - - image: circleci/buildpack-deps:18.04 + - image: circleci/python:3.7 environment: - REGISTRY: docker.pkg.github.com - CI_IMAGE: splunk/splunk-connect-for-syslog/ci - steps: - - setup_remote_docker: - docker_layer_caching: true - - checkout - - run: git submodule update --init --recursive - - run: mkdir /tmp/workspace - - - versioning/define_version - - versioning/create_version_file: - version_file_path: /tmp/workspace/VERSION - - run: - name: Record version - command: | - CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') - if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi - if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi - echo $DOCKER_TAG-${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:7}>package/VERSION - - docker/install-docker - - docker/check: - registry: $REGISTRY - docker-username: GITHUB_USER - docker-password: GITHUB_TOKEN - - docker/build: - image: $CI_IMAGE - registry: $REGISTRY - path: package - - - docker/install-goss: - version: v0.3.7 - - run: - name: Test Docker - command: | - mkdir -p /tmp/test-results - # Don't forget path! - export PATH=$PATH:~/bin - pushd package - GOSS_VER=v0.3.7 GOSS_FMT_OPTIONS="perfdata verbose" GOSS_FILES_STRATEGY=cp GOSS_OPTS="--max-concurrent=1 --format junit" dgoss run -t $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 3>&1 1>&2 2>&3 > /tmp/goss.tmp || true ; cat /tmp/goss.tmp | grep -v "<" - popd - cat /tmp/goss.tmp | grep "[<\>]" > /tmp/test-results/goss.xml - - - docker/push: - registry: $REGISTRY - image: $CI_IMAGE - - - run: | - CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') - if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi - if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi - docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 $REGISTRY/$CI_IMAGE:$DOCKER_TAG - docker push $REGISTRY/$CI_IMAGE:$DOCKER_TAG - - run: - name: Docker Save - command: | - docker save $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 | gzip -c > /tmp/workspace/oci_container.tar.gz - - run: - name: BYOE Config - command: | - tar rvf /tmp/workspace/baremetal.tar -C package/etc . - tar rvf /tmp/workspace/baremetal.tar -C package/sbin entrypoint.sh - - - store_test_results: - path: /tmp/test-results/goss.xml - - - persist_to_workspace: - root: /tmp - paths: - - workspace - - test-unit: + SYSLOG: "syslog-ng-3.25.1" + SPLUNK: "8.0" + <<: *test + test-sc4s-master-splunk-8-0: docker: - - image: circleci/buildpack-deps:18.04 + - image: circleci/python:3.7 environment: - REGISTRY: docker.pkg.github.com - CI_IMAGE: splunk/splunk-connect-for-syslog/ci - steps: - - setup_remote_docker: - docker_layer_caching: true - - checkout - - run: git submodule sync - - run: git submodule update --init --recursive - - #- docker/install-docker-tools - - docker/check: - registry: $REGISTRY - docker-username: GITHUB_USER - docker-password: GITHUB_TOKEN - - run: - name: Setup test volume - command: | - mkdir /tmp/test-results - docker volume create sc4s-tests - docker volume create sc4s-results - - docker container create --name dummy \ - -v sc4s-tests:/work/tests \ - -v sc4s-results:/work/test-results \ - -v splunk-etc:/work/splunk-etc \ - registry.access.redhat.com/ubi7/ubi - docker cp tests/ dummy:/work/tests/ - docker cp ./splunk/etc/* dummy:/work/splunk-etc/ - docker rm dummy - - run: - name: Docker Compose build - command: | - docker-compose build test - - run: - name: Docker Compose up - command: | - export IMAGE_NAME=$CIRCLE_PROJECT_REPONAME - docker-compose -f ./docker-compose-ci.yml up -d splunk - sleep 70 - docker-compose -f ./docker-compose-ci.yml up --abort-on-container-exit - - run: - name: Collect test results from Docker Compose - command: | - docker container create --name dummy \ - -v sc4s-tests:/work/tests \ - -v sc4s-results:/work/test-results \ - registry.access.redhat.com/ubi7/ubi - - docker cp dummy:/work/test-results/functional /tmp/test-results - docker rm dummy - when: always - - store_test_results: - path: /tmp/test-results - - test-scan-synk: + SYSLOG: "master" + SPLUNK: "8.0" + <<: *test + test-sc4s-3-25-1-splunk-7-3: docker: - - image: circleci/buildpack-deps:18.04 + - image: circleci/python:3.7 environment: - REGISTRY: docker.pkg.github.com - CI_IMAGE: splunk/splunk-connect-for-syslog/ci - steps: - - setup_remote_docker: - docker_layer_caching: true - - checkout - - run: git submodule sync - - run: git submodule update --init --recursive - - docker/check: - registry: $REGISTRY - docker-username: GITHUB_USER - docker-password: GITHUB_TOKEN - - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 - - snyk/scan: - docker-image-name: $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 - target-file: package/Dockerfile - project: '${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BRANCH}-container' - - test-scan-synk-nomonitor: + SYSLOG: "syslog-ng-3.25.1" + SPLUNK: "7.3" + <<: *test + test-sc4s-3-25-1-splunk-7-2: docker: - - image: circleci/buildpack-deps:18.04 + - image: circleci/python:3.7 environment: - REGISTRY: docker.pkg.github.com - CI_IMAGE: splunk/splunk-connect-for-syslog/ci - steps: - - setup_remote_docker: - docker_layer_caching: true - - checkout - - docker/check: - registry: $REGISTRY - docker-username: GITHUB_USER - docker-password: GITHUB_TOKEN - - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 - - run: git submodule sync - - run: git submodule update --init --recursive - - snyk/scan: - docker-image-name: $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 - target-file: package/Dockerfile - project: '${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BRANCH}-container' - monitor-on-build: false - - publish-branch: - docker: - - image: circleci/buildpack-deps:18.04 - environment: - REGISTRY: docker.pkg.github.com - CI_IMAGE: splunk/splunk-connect-for-syslog/ci - REGISTRY_PUBLIC: docker.pkg.github.com - PUBLIC_IMAGE: splunk/splunk-connect-for-syslog/releases - - steps: - - setup_remote_docker: - docker_layer_caching: true - - checkout - - docker/check: - registry: $REGISTRY - docker-username: GITHUB_USER - docker-password: GITHUB_TOKEN - - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 - - run: | - CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') - if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi - if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi - echo docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 -t $REGISTRY_PUBLIC/$PUBLIC_IMAGE:$DOCKER_TAG - docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 $REGISTRY_PUBLIC/$PUBLIC_IMAGE:$DOCKER_TAG - docker push $REGISTRY_PUBLIC/$PUBLIC_IMAGE:$DOCKER_TAG + SYSLOG: "syslog-ng-3.25.1" + SPLUNK: "7-2" + <<: *test publish-edge: docker: - - image: circleci/buildpack-deps:18.04 + - image: circleci/python:3.7 environment: - REGISTRY: docker.pkg.github.com - CI_IMAGE: splunk/splunk-connect-for-syslog/ci - REGISTRY_PUBLIC: docker.pkg.github.com - PUBLIC_IMAGE: splunk/splunk-connect-for-syslog/releases - steps: - - setup_remote_docker: - docker_layer_caching: true - - checkout - - docker/check: - registry: $REGISTRY - docker-username: GITHUB_USER - docker-password: GITHUB_TOKEN - - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 - - run: docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 $REGISTRY/$PUBLIC_IMAGE:edge - - docker/push: - registry: $REGISTRY - image: $PUBLIC_IMAGE - tag: edge - - docker/check: - docker-username: DOCKER_USER - docker-password: DOCKER_PASS - - run: docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 splunk/scs:edge - - docker/push: - image: splunk/scs - tag: edge - - go/install - - attach_workspace: - at: /tmp - - run: - name: "Publish edge on GitHub" - command: | - PATH=$PATH:/usr/local/go/bin - go get -v -u github.com/tcnksm/ghr - $HOME/go/bin/ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -prerelease -delete edge /tmp/workspace/ - - store_artifacts: - path: /tmp/workspace/ - + SYSLOG: "syslog-ng-3.25.1" + SPLUNK: "8.0" + DOCKER_TAG: edge + GHR_FLAGS: -prerelease + <<: *publish publish-latest: docker: - - image: circleci/buildpack-deps:18.04 + - image: circleci/python:3.7 environment: - REGISTRY: docker.pkg.github.com - CI_IMAGE: splunk/splunk-connect-for-syslog/ci - REGISTRY_PUBLIC: docker.pkg.github.com - PUBLIC_IMAGE: splunk/splunk-connect-for-syslog/releases - - steps: - - setup_remote_docker: - docker_layer_caching: true - - checkout - - docker/check: - registry: $REGISTRY - docker-username: GITHUB_USER - docker-password: GITHUB_TOKEN - - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 - - run: docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 $REGISTRY/$PUBLIC_IMAGE:latest - - docker/push: - registry: $REGISTRY - image: $PUBLIC_IMAGE - tag: latest - - docker/check: - docker-username: DOCKER_USER - docker-password: DOCKER_PASS - - run: docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 splunk/scs:latest - - docker/push: - image: splunk/scs - tag: latest - - - go/install - - attach_workspace: - at: /tmp - - run: - name: "Publish edge on GitHub" - command: | - PATH=$PATH:/usr/local/go/bin - go get -v -u github.com/tcnksm/ghr - $HOME/go/bin/ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete latest /tmp/workspace/ - - store_artifacts: - path: /tmp/workspace/ - + SYSLOG: "syslog-ng-3.25.1" + SPLUNK: "8.0" + DOCKER_TAG: latest + <<: *publish publish-tag: - docker: - - image: circleci/buildpack-deps:18.04 - environment: - REGISTRY: docker.pkg.github.com - CI_IMAGE: splunk/splunk-connect-for-syslog/ci - REGISTRY_PUBLIC: docker.pkg.github.com - PUBLIC_IMAGE: splunk/splunk-connect-for-syslog/releases - - steps: - - setup_remote_docker: - docker_layer_caching: true - - checkout - - docker/check: - registry: $REGISTRY - docker-username: GITHUB_USER - docker-password: GITHUB_TOKEN - - run: docker pull $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 - - - run: | - CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') - if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi - if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi - docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 $REGISTRY/$PUBLIC_IMAGE:$DOCKER_TAG - docker push $REGISTRY/$PUBLIC_IMAGE:$DOCKER_TAG - - docker/check: - docker-username: DOCKER_USER - docker-password: DOCKER_PASS - - run: | - CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') - if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi - if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi - docker tag $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 splunk/scs:$DOCKER_TAG - docker push splunk/scs:$DOCKER_TAG - - - go/install - - attach_workspace: - at: /tmp - - run: - name: "Publish edge on GitHub" - command: | - PATH=$PATH:/usr/local/go/bin - go get -v -u github.com/tcnksm/ghr - CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') - if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi - if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi - $HOME/go/bin/ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete $VERSION /tmp/workspace/ - - store_artifacts: - path: /tmp/workspace/ - + docker: + - image: circleci/python:3.7 + environment: + SYSLOG: "3.25.1" + SPLUNK: "8.0" + <<: *publish workflows: version: 2 - build-branches: + build_and_deploy: jobs: - - build - - test-unit: - requires: - - build - - test-scan-synk: + - test-sc4s-3-25-1-splunk-8-0 + - test-sc4s-master-splunk-8-0 + - test-sc4s-3-25-1-splunk-7-3 + - test-sc4s-3-25-1-splunk-7-2 + - publish-pre-gh-edge: + type: approval requires: - - build + - test-sc4s-3-25-1-splunk-8-0 filters: branches: only: - - master - - test-scan-synk-nomonitor: - requires: - - build - filters: - branches: - ignore: - - master - - publish-branch: - requires: - - build + - develop - publish-edge: requires: - - build - - test-unit - filters: - branches: - only: - - develop - - publish-latest: + - publish-pre-gh-edge + - publish-pre-gh-latest: + type: approval requires: - - build - - test-unit + - test-sc4s-3-25-1-splunk-8-0 filters: branches: only: - master - build-tags: - jobs: - - build: - filters: - tags: - only: /^\d*\.\d*\.\d*.*$/ - branches: - ignore: /.*/ - - test-unit: - filters: - tags: - only: /^\d*\.\d*\.\d*.*$/ - branches: - ignore: /.*/ + - publish-latest: requires: - - build - - test-scan-synk: + - publish-pre-gh-latest + - publish-pre-gh-tag: + type: approval + requires: + - test-sc4s-3-25-1-splunk-8-0 filters: tags: only: /^\d*\.\d*\.\d*.*$/ branches: ignore: /.*/ - requires: - - build - publish-tag: requires: - - build - - test-unit - #- test-scan-synk - filters: - tags: - only: /^\d*\.\d*\.\d*.*$/ - branches: - ignore: /.*/ + - publish-pre-gh-tag diff --git a/.env.template b/.env.template deleted file mode 100644 index b795fde..0000000 --- a/.env.template +++ /dev/null @@ -1,15 +0,0 @@ -#Splunk Connect for Syslog (SC4S) by Splunk, Inc. -# -#To the extent possible under law, the person who associated CC0 with -#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights -#to Splunk Connect for Syslog (SC4S). -# -#You should have received a copy of the CC0 legalcode along with this -#work. If not, see . -SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SPLUNK_PASSWORD=Changed@11 -SPLUNK_START_ARGS=--accept-license -SPLUNK_HEC_URL=https://splunk:8088/services/collector/event -#SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download -#SPLUNKBASE_USERNAME=username -#SPLUNKBASE_PASSWORD=password diff --git a/.github/workflows/review-secrets.yml b/.github/workflows/review-secrets.yml new file mode 100644 index 0000000..bc0c84c --- /dev/null +++ b/.github/workflows/review-secrets.yml @@ -0,0 +1,12 @@ +name: Review Secrets +on: [push] + +jobs: + review_secrets: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Trufflehog Actions Scan + uses: edplato/trufflehog-actions-scan@v0.9f-beta + diff --git a/.gitmodules b/.gitmodules index fd4c8a0..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +0,0 @@ -[submodule "package/syslog-ng"] - path = package/syslog-ng - url = https://github.com/balabit/syslog-ng.git - branch = syslog-ng-3.25.1 -# - diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..09e2786 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python.pythonPath": "venv/bin/python3", + "python.testing.pytestEnabled": true, + "python.testing.autoTestDiscoverOnSaveEnabled": true +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..e5c70bf --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,33 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "Setup step 1: python venv", + "type": "shell", + "command": "python3 -m venv ./venv", + "windows": { + "command": "python venv" + }, + "group": "Setup", + "presentation": { + "reveal": "always", + "panel": "new" + } + }, + { + "label": "Setup step 2: python requirements", + "type": "shell", + "command": "pip3 install -r requirements.txt", + "windows": { + "command": "python requirements" + }, + "group": "Setup", + "presentation": { + "reveal": "always", + "panel": "new" + } + } + ] +} \ No newline at end of file diff --git a/clair-scan.sh b/clair-scan.sh deleted file mode 100755 index e890ce3..0000000 --- a/clair-scan.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - - set -e - - REPORT_DIR=clair-reports - mkdir $REPORT_DIR || true - - #DB=$(docker run -p 5432:5432 -d arminc/clair-db:latest) - docker run -p 5432:5432 -d --rm --name db arminc/clair-db:latest - #CLAIR=$(docker run -p 6060:6060 --link "$DB":postgres -d arminc/clair-local-scan:latest)' - sleep 30 - docker run -p 6060:6060 --link db:postgres -d --rm --name clair arminc/clair-local-scan:latest - #CLAIR_SCANNER=$(docker run -v /var/run/docker.sock:/var/run/docker.sock --link clair:clair --name clairscanner --rm -d ovotech/clair-scanner@sha256:53fe8e8ac63af330d2dfc63498d23d8825d07f916f7d230271176de06d12acd6 tail -f /dev/null) - - CLAIR_SCANNER=$(docker run --link clair:clair --name clairscanner --rm -d ovotech/clair-scanner@sha256:53fe8e8ac63af330d2dfc63498d23d8825d07f916f7d230271176de06d12acd6 tail -f /dev/null) - - #clair_ip=$(docker exec -it "$CLAIR" hostname -i | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+') - #scanner_ip=$(docker exec -it "$CLAIR_SCANNER" hostname -i | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+') - - docker cp "clair-whitelist.yml" "$CLAIR_SCANNER:/whitelist.yml" - WHITELIST="-w /whitelist.yml" - - function scan() { - echo Scanning $1 - local image=$1 - # replace forward-slashes and colons with underscores - munged_image=$(echo "$image" | sed 's/\//_/g' | sed 's/:/_/g') - sanitised_image_filename="${munged_image}.json" - local ret=0 - #--ip "$scanner_ip" \ - # - local docker_cmd=(docker exec -it "$CLAIR_SCANNER" clair-scanner \ - --clair=http://clair:6060 \ - -t "high" \ - --report "$REPORT_DIR/$sanitised_image_filename" \ - --log "$REPORT_DIR/log.json" --whitelist=${WHITELIST:+"-x"} - --reportAll=true \ - --exit-when-no-features=false \ - "$image") - - docker pull "$image" - - "${docker_cmd[@]}" 2>&1 || ret=$? - if [ $ret -eq 0 ]; then - echo "No unapproved vulnerabilities" - elif [ $ret -eq 1 ]; then - echo "Unapproved vulnerabilities found" - EXIT_STATUS=1 - elif [ $ret -eq 5 ]; then - echo "Image was not scanned, not supported." - EXIT_STATUS=1 - else - echo "Unknown clair-scanner return code $ret." - EXIT_STATUS=1 - fi - - docker cp "$CLAIR_SCANNER:/$sanitised_image_filename" "$REPORT_DIR/$sanitised_image_filename" || true - } - - EXIT_STATUS=0 - - scan "$IMAGE_NAME:$CIRCLE_SHA1" - - docker kill clairscanner - docker kill clair - docker kill db - - exit $EXIT_STATUS \ No newline at end of file diff --git a/clair-whitelist.yml b/clair-whitelist.yml deleted file mode 100644 index 586bcb7..0000000 --- a/clair-whitelist.yml +++ /dev/null @@ -1,24 +0,0 @@ -#Splunk Connect for Syslog (SC4S) by Splunk, Inc. -# -#To the extent possible under law, the person who associated CC0 with -#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights -#to Splunk Connect for Syslog (SC4S). -# -#You should have received a copy of the CC0 legalcode along with this -#work. If not, see . -generalwhitelist: - RHSA-2019:1619: False Positive - RHSA-2018:0654: False Positive - RHSA-2018:1967: False Positive - RHSA-2017:0372: False Positive - RHSA-2018:0502: False Positive - RHSA-2018:2772: False Positive - RHSA-2018:1374: False Positive - RHSA-2018:0180: False Positive - RHSA-2019:2029: False Positive - RHSA-2019:1873: False Positive - - - -images: - scs: diff --git a/clair_to_junit_parser.py b/clair_to_junit_parser.py deleted file mode 100644 index 7dbc220..0000000 --- a/clair_to_junit_parser.py +++ /dev/null @@ -1,80 +0,0 @@ -#Copyright 2019 Splunk, Inc. -# -#Use of this source code is governed by a BSD-2-clause-style -#license that can be found in the LICENSE-BSD2 file or at -#https://opensource.org/licenses/BSD-2-Clause -import json -from junit_xml import TestSuite, TestCase -import os -import argparse -import logging - -logger = logging.getLogger('clair_scanner_converter') -logger.setLevel(logging.WARN) -console_logger = logging.StreamHandler() -console_logger.setLevel(logging.WARN) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -console_logger.setFormatter(formatter) -logger.addHandler(console_logger) - -def parse_args(): - parser = argparse.ArgumentParser(description="Process Json File") - parser.add_argument("clairfile", type=str, default=None, help="Location of clair scanner ouptut file to convert to cucumber.json") - parser.add_argument("--output", type=str, default=None, help="name of output file to store in new format. Defaults to clair inputfile") - args = parser.parse_args() - if not args.output: - logger.warning("No output file specified, replacing input file.") - args.output = args.clairfile - return args - -def main(): - cwd = os.getcwd() - args = parse_args() - try: - if os.path.exists(args.clairfile): - with open(args.clairfile) as clairfile: - clair_parsed_file = json.load(clairfile) - if os.path.exists(os.path.join("clair-scanner-logs", "/clair_setup_errors.log")): - with open(os.path.join("clair-scanner-logs", "/clair_setup_errors.log"), 'r') as clairfile_errors: - clair_parsed_error_file = clairfile_errors.readlines() - else: - clair_parsed_error_file = None - except: - logger.exception("Failed to parse clair / clair_error file. Exiting.") - - current_sorted_level = None - current_suite = None - test_suites = [] - if clair_parsed_error_file: - current_suite = TestSuite("SetupError") - new_step = TestCase(name="SetupError", classname="SetupError", status="unapproved", stderr=clair_parsed_error_file) - new_step.log = clair_parsed_error_file - new_step.category = "SetupError" - new_step.failure_type = "unapproved" - new_step.failure_message = "Please have the following security issue reviewed by Splunk: {}".format(vuln["link"]) - new_step.failure_output = clair_parsed_error_file - current_suite.test_cases.append(new_step) - test_suites.append(current_suite) - for vuln in clair_parsed_file["vulnerabilities"]: - if current_sorted_level != vuln["severity"]: - if current_suite: - test_suites.append(current_suite) - current_suite = TestSuite(name=vuln["severity"]) - current_sorted_level = vuln["severity"] - new_step = TestCase(name=vuln["vulnerability"], classname=vuln["severity"], status="unapproved", url=vuln["link"], stderr=vuln["description"]) - new_step.log = vuln - new_step.category = vuln["severity"] - new_step.failure_type = "unapproved" - new_step.failure_message = "Please have the following security issue reviewed by Splunk: {}".format(vuln["link"]) - new_step.failure_output = vuln["description"] - current_suite.test_cases.append(new_step) - # try to write new file - try: - with open(args.output, 'w') as outfile: - outfile.write(TestSuite.to_xml_string(test_suites)) - except: - logger.exception("Filed saving file.") - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/demo-with-compose.sh b/demo-with-compose.sh index 3e0f693..9e5dbac 100755 --- a/demo-with-compose.sh +++ b/demo-with-compose.sh @@ -4,35 +4,11 @@ #Use of this source code is governed by a BSD-2-clause-style #license that can be found in the LICENSE-BSD2 file or at #https://opensource.org/licenses/BSD-2-Clause -WAITON=${1:-test} -compose=${2:-docker-compose-demo.yml} -echo $WAITON $compose -mkdir test-results -docker-compose down -docker volume rm sc4s-results -docker volume rm splunk-etc - -docker volume create sc4s-results -docker volume create splunk-etc - -docker container create --name dummy \ - -v sc4s-results:/work/test-results \ - -v splunk-etc:/work/splunk-etc \ - registry.access.redhat.com/ubi7/ubi -docker cp ./splunk/etc/* dummy:/work/splunk-etc/ -docker rm dummy - -docker-compose -f $compose pull -docker-compose -f $compose up -d splunk -docker-compose -f $compose up -d sc4s +docker-compose -f tests/docker-compose-script.yml build +docker-compose -f tests/docker-compose-script.yml up -d splunk sleep 60 -docker-compose -f $compose build -docker-compose -f $compose up - -docker container create --name dummy \ - -v sc4s-results:/work/test-results \ - registry.access.redhat.com/ubi7/ubi +docker-compose -f tests/docker-compose-script.yml up -d sc4s +sleep 5 +docker-compose -f tests/docker-compose-script.yml up -docker cp dummy:/work/test-results/functional test-results -docker rm dummy EXIT=$0 diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml deleted file mode 100644 index ee8d0e0..0000000 --- a/docker-compose-debug.yml +++ /dev/null @@ -1,53 +0,0 @@ -#Splunk Connect for Syslog (SC4S) by Splunk, Inc. -# -#To the extent possible under law, the person who associated CC0 with -#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights -#to Splunk Connect for Syslog (SC4S). -# -#You should have received a copy of the CC0 legalcode along with this -#work. If not, see . -version: "3.2" -services: - - sc4s: - image: splunk/scs:latest - build: - context: ./package - entrypoint: - - "tail" - - "-f" - - "/dev/null" - hostname: sc4s - ports: - - "514" - - "601" - - "514/udp" - - "5514" - - "5514/udp" - stdin_open: true - tty: true - links: - - splunk - environment: - - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no - splunk: - image: splunk/splunk:latest - hostname: splunk - ports: - - "8000:8000" - - "8088:8088" - - "8089:8089" - environment: - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} - - SPLUNK_START_ARGS=${SPLUNK_START_ARGS} - - SPLUNK_APPS_URL=${SPLUNK_APPS_URL} - - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} - - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} - volumes: - - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng diff --git a/docker-compose-demo.yml b/docker-compose-demo.yml deleted file mode 100644 index 3934fe1..0000000 --- a/docker-compose-demo.yml +++ /dev/null @@ -1,66 +0,0 @@ -#Splunk Connect for Syslog (SC4S) by Splunk, Inc. -# -#To the extent possible under law, the person who associated CC0 with -#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights -#to Splunk Connect for Syslog (SC4S). -# -#You should have received a copy of the CC0 legalcode along with this -#work. If not, see . -version: "3.2" -services: - test: - build: ./tests - links: - - splunk - - sc4s - volumes: - - sc4s-tests:/work/tests - - sc4s-results:/work/test-results - environment: - - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} - - sc4s: - image: splunk/scs:latest - hostname: sc4s - ports: - - "514" - - "601" - - "514/udp" - - "5514" - - "5514/udp" - stdin_open: true - tty: true - links: - - splunk - environment: - - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no - - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 - splunk: - image: splunk/splunk:latest - hostname: splunk - ports: - - "8000:8000" - - "8088:8088" - - "8089:8089" - environment: - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} - - SPLUNK_START_ARGS=${SPLUNK_START_ARGS} - - SPLUNK_APPS_URL=${SPLUNK_APPS_URL} - - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} - - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} - volumes: - - splunk-etc:/opt/splunk/etc - -volumes: - sc4s-tests: - external: true - sc4s-results: - external: true - splunk-etc: - external: true diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 81191cb..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,87 +0,0 @@ -#Splunk Connect for Syslog (SC4S) by Splunk, Inc. -# -#To the extent possible under law, the person who associated CC0 with -#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights -#to Splunk Connect for Syslog (SC4S). -# -#You should have received a copy of the CC0 legalcode along with this -#work. If not, see . -version: "3.7" -services: - test: - build: ./tests - links: - - splunk - - sc4s - environment: - - SPLUNK_USER=admin - - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} - - SPLUNK_HOST=splunk - - SYSLOG_HOST=sc4s - - sc4s: - image: splunk/scs:latest - build: - context: ./package - hostname: sc4s -#When this is enabled test_common will fail -# command: -det - ports: - - "514:514" - - "601:601" - - "514:514/udp" - - "5000" - - "5000:5000/udp" - - "6514:6514" - stdin_open: true - tty: true - links: - - splunk - environment: - - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SC4S_SOURCE_TLS_ENABLE=no - - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no -# - SC4S_LISTEN_DEFAULT_TCP_PORT=514 -# - SC4S_LISTEN_DEFAULT_UDP_PORT=514 -# - SC4S_LISTEN_DEFAULT_TLS_PORT=6514 - - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 - - SC4S_LISTEN_CISCO_ASA_TCP_PORT=5001 - - SC4S_LISTEN_CISCO_IOS_TCP_PORT=5002 - - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003 - - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 - - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 - - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 - - SC4S_ARCHIVE_GLOBAL=yes -# - SC4S_SOURCE_STORE_RAWMSG=yes - volumes: - - ./tls:/opt/syslog-ng/tls - splunk: - image: splunk/splunk:latest - hostname: splunk - ports: - - "8000:8000" - - "8088:8088" - - "8089:8089" - environment: - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} - - SPLUNK_START_ARGS=${SPLUNK_START_ARGS} - - SPLUNK_APPS_URL=${SPLUNK_APPS_URL} - - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} - - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} - volumes: - - splunk-etc:/opt/splunk/etc - pcapreplay: - build: - context: ./utility/pcapreplay - entrypoint: tail -f /dev/null - links: - - splunk - - sc4s - volumes: - - ./replay:/work - -volumes: - splunk-etc: - external: true diff --git a/docs/developing/gh_clone.png b/docs/developing/gh_clone.png new file mode 100644 index 0000000..a654aad Binary files /dev/null and b/docs/developing/gh_clone.png differ diff --git a/docs/developing/gh_fork.png b/docs/developing/gh_fork.png new file mode 100644 index 0000000..a765410 Binary files /dev/null and b/docs/developing/gh_fork.png differ diff --git a/docs/developing/index.md b/docs/developing/index.md new file mode 100644 index 0000000..5cf9063 --- /dev/null +++ b/docs/developing/index.md @@ -0,0 +1,44 @@ +# Development setup (BETA) + +## Get Docker + +Development requires Docker desktop available for windows + and mac or Docker CE available for Linux visit (Docker)[https://www.docker.com/get-started] +for download instructions + +## Setup VS Code IDE + +VS Code provides a free IDE experience that is effective for daily development with SC4S visit (Microsoft)[https://code.visualstudio.com/docs/introvideos/basics] +to download and install for your plaform (windows/mac/linux) + +## Fork and Clone the github repository + +Visit our repository at (Github)[https://github.com/splunk/splunk-connect-for-syslog] and "fork" our repository this will allow you to make changes and submit pull requests. + +![How to Fork](gh_fork.png) + +Click the clone icon and select the location + +![How to Clone](gh_clone.png) + +## Setup the project and install requirements + +The follow steps are only required on the first time run. + +* Install VS Code Extensions S + * Python + * Test Explorer + * "Python Test Explorer" +* From the terminal menu select Run Task +* Select "Setup step 1: python venv" then "go without scanning output" +* From the terminal menu select Run Task +* Select "Setup step 2: python requirements" then "go without scanning output" + +![VS Code setup](vsc_run.png) + +## Click the test lab icon + +* Run all tests this will appear to do nothing for a period system fan may spin loud whiletests are run icons on each test will turn green +or red to indicate pass fail however VS Code does not show the status of status until the tests complete + +![VS Code Debug](vsc_debug.png) + diff --git a/docs/developing/vsc_debug.png b/docs/developing/vsc_debug.png new file mode 100644 index 0000000..e3950d4 Binary files /dev/null and b/docs/developing/vsc_debug.png differ diff --git a/docs/developing/vsc_run.png b/docs/developing/vsc_run.png new file mode 100644 index 0000000..a00fef1 Binary files /dev/null and b/docs/developing/vsc_run.png differ diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md index b0234dd..ae72697 100644 --- a/docs/sources/Cisco/index.md +++ b/docs/sources/Cisco/index.md @@ -48,6 +48,54 @@ index= sourcetype=cisco:acs Verify timestamp, and host values match as expected +## Product - APIC (ACI) + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | No current add-on for syslog events | +| Product Manual | https://community.cisco.com/t5/security-documents/acs-5-x-configuring-the-external-syslog-server/ta-p/3143143 | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cisco:apic:acl: | APIC events from leaf switches | +| cisco:apic:events | APIC events from any component used | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_apic_acl | cisco:apic:acl | netfw | None | +| cisco_apic_events | cisco:apic:events | netops | None | + +### Filter type + +PATTERN MATCH + +### Setup and Configuration + +* No special steps required + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CISCO_APIC_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CISCO_APIC_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CISCO_APIC | no | Enable archive to disk for this specific source | +| SC4S_DEST_CISCO_APIC_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +Use the following search to validate events are present + +``` +index= sourcetype=cisco:apic:* +``` + +Verify timestamp, and host values match as expected ## Product - ASA (Pre Firepower) @@ -283,3 +331,50 @@ index= sourcetype=merkai Verify timestamp, and host values match as expected +## Product - UCM + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | na | +| Product Manual | multiple | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cisco:ucm | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_ucm | cisco:ucm | ucm | None | + + +### Filter type + +PATTERN MATCH + +### Setup and Configuration + +* Refer to Cisco support web site + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CISCO_UCM_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CISCO_UCM_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CISCO_UCM | no | Enable archive to disk for this specific source | +| SC4S_DEST_CISCO_UCM_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +Use the following search to validate events are present + +``` +index= sourcetype=cisco:ucm +``` + +Verify timestamp, and host values match as expected \ No newline at end of file diff --git a/docs/sources/Citrix/index.md b/docs/sources/Citrix/index.md new file mode 100644 index 0000000..3c2fc13 --- /dev/null +++ b/docs/sources/Citrix/index.md @@ -0,0 +1,53 @@ +# Vendor - Citrix + +## Product - Netscaler ADC + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2770/ | +| Product Manual | https://docs.citrix.com/en-us/citrix-adc/12-1/system/audit-logging/configuring-audit-logging.html | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| citrix:netscaler:syslog | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| citrix_netscaler | citrix:netscaler:syslog | netfw | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Follow vendor configuration steps per Product Manual above. Ensure the data format selected is "MMDDYYYY" + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CITRIX_NETSCALER_SPLUNK_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the port number defined | +| SC4S_LISTEN_CITRIX_NETSCALERSPLUNK_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the port number defined | +| SC4S_ARCHIVE_CITRIX_NETSCALER_SPLUNK | no | Enable archive to disk for this specific source | +| SC4S_DEST_CITRIX_NETSCALER_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CITRIX_NETSCALER_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_SOURCE_CITRIX_NETSCALER_USEALT_DATE_FORMAT | no | Use "DDMMYYYY" format rather than "MMDDYYYY" | + +### Verification + +Use the following search to validate events are present + +``` +index= sourcetype=cp_log +``` + +Verify timestamp, and host values match as expected diff --git a/mkdocs.yml b/mkdocs.yml index 8f18791..6a73268 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,20 +1,22 @@ site_name: Splunk Connect for Syslog nav: - - Home: 'index.md' + - Home: "index.md" - Getting Started: - - 'Read First': 'gettingstarted/index.md' - - 'Podman + systemd': 'gettingstarted/podman-systemd-general.md' - - 'Docker CE + systemd': 'gettingstarted/docker-systemd-general.md' - - 'Docker CE + Swarm': 'gettingstarted/docker-swarm-general.md' - - 'Docker CE + Swarm RHEL 7.7': 'gettingstarted/docker-swarm-rhel7.md' - - 'Bring your own Envionment': 'gettingstarted/byoe-rhel7.md' - - Configuration: 'configuration.md' + - "Read First": "gettingstarted/index.md" + - "Podman + systemd": "gettingstarted/podman-systemd-general.md" + - "Docker CE + systemd": "gettingstarted/docker-systemd-general.md" + - "Docker CE + Swarm": "gettingstarted/docker-swarm-general.md" + - "Docker CE + Swarm RHEL 7.7": "gettingstarted/docker-swarm-rhel7.md" + - "Bring your own Envionment": "gettingstarted/byoe-rhel7.md" + - Configuration: "configuration.md" + - Developing: "docs/developing/index.md" - Sources: - About: sources/index.md - Checkpoint: sources/Checkpoint/index.md - Cisco: sources/Cisco/index.md - - 'Common Event Format': sources/CommonEventFormat/index.md + - Citrix: sources/Citrix/index.md + - "Common Event Format": sources/CommonEventFormat/index.md - CyberArk: sources/CyberArk/index.md - Forcepoint: sources/Forcepoint/index.md - Fortinet: sources/Fortinet/index.md @@ -22,30 +24,30 @@ nav: - Juniper: sources/Juniper/index.md - Microfocus: sources/Microfocus/index.md - Nix: sources/nix/index.md - - 'Palo Alto Networks': sources/PaloaltoNetworks/index.md - - 'pfSense': sources/pfSense/index.md + - "Palo Alto Networks": sources/PaloaltoNetworks/index.md + - "pfSense": sources/pfSense/index.md - Proofpoint: sources/Proofpoint/index.md - Symantec: sources/Symantec/index.md - Ubiquiti: sources/Ubiquiti/index.md - - VMware: sources/VMware/index.md + - VMware: sources/VMWare/index.md - Zscaler: sources/Zscaler/index.md - - 'Demo Lab': 'demo.md' - - Performance: 'performance.md' - - Troubleshooting: 'troubleshooting.md' - - 'Upgrading SC4S': 'upgrade.md' + - "Demo Lab": "demo.md" + - Performance: "performance.md" + - Troubleshooting: "troubleshooting.md" + - "Upgrading SC4S": "upgrade.md" markdown_extensions: - - toc: - permalink: True - - smarty - - fenced_code - - sane_lists - - codehilite + - toc: + permalink: True + - smarty + - fenced_code + - sane_lists + - codehilite theme: - name: 'material' + name: "material" palette: - primary: 'black' - accent: 'orange' - favicon: 'logo.png' - logo: 'logo.png' + primary: "black" + accent: "orange" + favicon: "logo.png" + logo: "logo.png" diff --git a/package/Dockerfile b/package/Dockerfile index 75a9c62..0ad5796 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -6,66 +6,12 @@ # #You should have received a copy of the CC0 legalcode along with this #work. If not, see . -FROM centos:centos8 +ARG BRANCH=master +FROM splunk/scs:base-${BRANCH} +COPY --from=hairyhenderson/gomplate:v3.5.0 /gomplate /usr/local/bin/gomplate -ENV CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --disable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --disable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking" - -ENV DISTCHECK_CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --disable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --disable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking" - -RUN dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y ;\ - dnf install 'dnf-command(config-manager)' -y ;\ - dnf config-manager --set-enabled PowerTools -y; \ - dnf update -y ;\ - dnf upgrade - -RUN dnf group install "Development Tools" -y ;\ - dnf install findutils autoconf \ - autoconf automake ca-certificates git libtool pkgconfig bison byacc file \ - flex pcre-devel glib2-devel openssl-devel libcurl-devel \ - python3 python3-devel \ - net-snmp-devel \ - libuuid-devel cmake make libxslt gcc-c++ tzdata libxml2 sqlite \ - gnupg wget curl which bzip2 libsecret ivykis-devel autoconf-archive json-c-devel -y - - -RUN CRITERION_VERSION=2.3.3 ;\ - cd /tmp/;\ - wget https://github.com/Snaipe/Criterion/releases/download/v${CRITERION_VERSION}/criterion-v${CRITERION_VERSION}.tar.bz2 ;\ - tar xvf /tmp/criterion-v${CRITERION_VERSION}.tar.bz2;cd /tmp/criterion-v${CRITERION_VERSION} ;\ - cmake -DCMAKE_INSTALL_PREFIX=/usr . ;\ - make install ;\ - ldconfig ;\ - rm -rf /tmp/criterion.tar.bz2 /tmp/criterion-v${CRITERION_VERSION} - - -COPY syslog-ng /work - -RUN cd /work;\ - pip3 install -r requirements.txt ;\ - ./autogen.sh ;\ - ./configure $CONFIGURE_FLAGS ;\ - make -j -l 2.5 install - - -FROM registry.access.redhat.com/ubi8/ubi - -RUN cd /tmp ;\ - dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y; \ - dnf update -y ;\ - dnf install wget gcc tzdata libdbi libsecret libxml2 sqlite \ - python3 libcurl ivykis scl-utils curl wget openssl -y - -ENV DEBCONF_NONINTERACTIVE_SEEN=true - -RUN curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh -COPY goss.yaml /etc/goss.yaml - -COPY --from=0 /opt/syslog-ng /opt/syslog-ng - -RUN curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.5.0/gomplate_linux-amd64-slim && \ - chmod 755 /usr/local/bin/gomplate - +COPY goss.yaml goss.yaml COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d @@ -74,15 +20,9 @@ COPY etc/context_templates /opt/syslog-ng/etc/context_templates COPY etc/local_config /opt/syslog-ng/etc/local_config COPY sbin/entrypoint.sh / -RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer -RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V COPY VERSION / -EXPOSE 514 -EXPOSE 601/tcp -EXPOSE 6514/tcp - -ENTRYPOINT ["/entrypoint.sh", "-F"] +RUN /opt/syslog-ng/sbin/syslog-ng -V -HEALTHCHECK --start-period=15s --interval=30s --timeout=6s CMD goss -g /etc/goss.yaml validate \ No newline at end of file +#USER [syslog] \ No newline at end of file diff --git a/package/etc/conf.d/filters/cisco/apic.conf b/package/etc/conf.d/filters/cisco/apic.conf new file mode 100644 index 0000000..ea6660d --- /dev/null +++ b/package/etc/conf.d/filters/cisco/apic.conf @@ -0,0 +1,6 @@ + +filter f_cisco_apic { + program('^%LOG_LOCAL\d-\d-'); + or + program('^%ACLLOG-\d-ACLLOG_PKTLOG'); +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/cisco/ucm.conf b/package/etc/conf.d/filters/cisco/ucm.conf new file mode 100644 index 0000000..a193625 --- /dev/null +++ b/package/etc/conf.d/filters/cisco/ucm.conf @@ -0,0 +1,32 @@ + +filter f_cisco_ucm { + message("^%UC\_") + or + message("^%CCM\_") +}; + +filter f_cisco_ucm_message { + message( + '^(<\d{1,3}>)\d*: (?:([^:]+): )?(.*) : (%.*)' + flags(store-matches) + ); +}; + +parser p_cisco_ucm_date { + #Oct 14 2015 05:50:19 AM.484 UTC + #Apr 21 19:01:35.638 UTC + date-parser(format( + '%b %d %Y %I:%M:%S %p.%f %Z', + '%b %d %H:%M:%S.%f %Z' + ) + template("$3") + ); +}; + +rewrite r_cisco_ucm_message { + set("cisco_ucm" value("fields.sc4s_syslog_format")); + set("cisco_ucm" value("fields.sc4s_vendor_product")); + set("$HOST_FROM" value("HOST") ); + set("$2" value("HOST") condition(match("^..." template("${2}"))) ); + set("$4" value("MESSAGE")); +}; diff --git a/package/etc/conf.d/filters/citrix/netscaler.conf.tmpl b/package/etc/conf.d/filters/citrix/netscaler.conf.tmpl new file mode 100644 index 0000000..76c0334 --- /dev/null +++ b/package/etc/conf.d/filters/citrix/netscaler.conf.tmpl @@ -0,0 +1,28 @@ +filter f_citrix_netscaler { + match("^citrix_netscaler", value("fields.sc4s_vendor_product")); +}; +filter f_citrix_netscaler_message { + message( + '^(<\d{1,3}>) (\d\d\/\d\d\/\d\d\d\d\:\d\d:\d\d:\d\d [^ ]{3}+) ([^ ]+) (.*)' + flags(store-matches) + ); +}; + +parser p_citrix_netscaler_date { +{{- if (conv.ToBool (getenv "SC4S_SOURCE_CITRIX_NETSCALER_USEALT_DATE_FORMAT" "yes")) }} + #01/10/2001:01:01:01 GMT + date-parser(format('%d/%m/%Y:%H:%M:%S %Z') + template("$2")); +{{- else }} + #10/01/2001:01:01:01 GMT + date-parser(format('%m/%d/%Y:%H:%M:%S %Z') + template("$2")); +{{- end }} +}; + +rewrite r_citrix_netscaler_message { + set("citrix_netscaler" value("fields.sc4s_syslog_format")); + set("citrix_netscaler" value("fields.sc4s_vendor_product")); + set("$3" value("HOST")); + set("$4" value("MESSAGE")); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-cisco_apic.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_apic.conf.tmpl new file mode 100644 index 0000000..cd3d9f9 --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-cisco_apic.conf.tmpl @@ -0,0 +1,56 @@ +# Cisco APIC +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_APIC" "parser" "rfc3164" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +log { + junction { +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_APIC_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_APIC_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_APIC_TLS_PORT")) }} + channel { + # Listen on the specified dedicated port(s) for CISCO_APIC traffic + source (s_CISCO_APIC); + flags (final); + }; +{{- end}} + channel { + # Listen on the default port (typically 514) for CISCO_APIC traffic + source (s_DEFAULT); + filter(f_cisco_apic); + flags(final); + }; + }; + + rewrite { + guess-time-zone(); + }; + if { + filter { + program('^%ACLLOG-\d-ACLLOG_PKTLOG') + }; + rewrite { + set("cisco_APIC_acl", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:apic:acl"), index("netfw"), template("t_hdr_msg")) + }; + parser { p_add_context_splunk(key("cisco_apic_acl")); }; + + } elif { + rewrite { + set("cisco_APIC_events", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:apic:events"), index("netops"), template("t_hdr_msg")) + }; + parser { p_add_context_splunk(key("cisco_apic_events")); }; + }; + + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CISCO_APIC_HEC" "no")) }} + destination(d_hec); +{{- end}} + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CISCO_APIC" "no")) }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; diff --git a/package/etc/conf.d/log_paths/lp-cisco_ucm.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_ucm.conf.tmpl new file mode 100644 index 0000000..e0823cf --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-cisco_ucm.conf.tmpl @@ -0,0 +1,62 @@ +# Cisco UCM +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_UCM" "parser" "cisco_ucm" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +log { + junction { +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_UCM_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_UCM_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_UCM_TLS_PORT")) }} + channel { + # Listen on the specified dedicated port(s) for CISCO_UCM traffic + source (s_CISCO_UCM); + flags (final); + }; +{{- end}} + channel { + # Listen on the default port (typically 514) for CISCO_UCM traffic + source (s_DEFAULT); + filter(f_cisco_ucm); + flags(final); + }; + }; + + if { + filter { + message( + 'Node ?ID(?:\:|\=)([^ \]]+)' + flags(store-matches) + ); + }; + rewrite { + set("$1" value("HOST") ); + }; + } elif { + filter { + message( + ' on node ([^ ]+\. |[^ ]+ )' + flags(store-matches) + ); + }; + rewrite { + set("$1" value("HOST") ); + }; + }; + + rewrite { + set("cisco_ucm", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:ucm"), index("main")) + }; + parser {p_add_context_splunk(key("cisco_ucm")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CISCO_UCM_HEC" "no")) }} + destination(d_hec); +{{- end}} + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CISCO_UCM" "no")) }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; diff --git a/package/etc/conf.d/log_paths/lp-citrix-netscaler.conf.tmpl b/package/etc/conf.d/log_paths/lp-citrix-netscaler.conf.tmpl new file mode 100644 index 0000000..ed11613 --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-citrix-netscaler.conf.tmpl @@ -0,0 +1,41 @@ +# Citrix Netscaler ADC +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CITRIX_NETSCALER" "parser" "citrix_netscaler" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +log { + junction { +{{- if or (or (getenv (print "SC4S_LISTEN_CITRIX_NETSCALER_TCP_PORT")) (getenv (print "SC4S_LISTEN_CITRIX_NETSCALER_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CITRIX_NETSCALER_TLS_PORT")) }} + channel { + # Listen on the specified dedicated port(s) for CITRIX_NETSCALER traffic + source (s_CITRIX_NETSCALER); + flags (final); + }; +{{- end}} + channel { + # Listen on the default port (typically 514) for CITRIX_NETSCALER traffic + source (s_DEFAULT); + filter(f_citrix_netscaler); + flags(final); + }; + }; + + rewrite { + set("citrix_netscaler", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("citrix:netscaler:syslog"), index("netfw")) + }; + + parser {p_add_context_splunk(key("citrix_netscaler")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CITRIX_NETSCALER_HEC" "no")) }} + destination(d_hec); +{{- end}} + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CITRIX_NETSCALER" "no")) }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; diff --git a/package/etc/conf.d/sources/startup.conf b/package/etc/conf.d/sources/startup.conf index 00fbce7..e4de003 100644 --- a/package/etc/conf.d/sources/startup.conf +++ b/package/etc/conf.d/sources/startup.conf @@ -1,12 +1,12 @@ source s_startup_out { - file("/var/log/syslog-ng.out" + file("/opt/syslog-ng/var/log/syslog-ng.out" program-override("syslog-ng-config") flags(no-hostname,no-parse,assume-utf8) ); }; source s_startup_err { - file("/var/log/syslog-ng.err" + file("/opt/syslog-ng/var/log/syslog-ng.err" program-override("syslog-ng-config") flags(no-hostname,no-parse,assume-utf8) ); diff --git a/package/etc/context_templates/splunk_index.csv.example b/package/etc/context_templates/splunk_index.csv.example index 6d36fa8..bf8e3bb 100644 --- a/package/etc/context_templates/splunk_index.csv.example +++ b/package/etc/context_templates/splunk_index.csv.example @@ -13,11 +13,15 @@ #checkpoint_splunk_web,index,netproxy #checkpoint_splunk,index,netops #checkpoint_splunk,index,netops +#cisco_apic_acl,index,netfw +#cisco_apic_events,index,netops #cisco_acs,index,netauth #cisco_asa,index,netfw #cisco_ios,index,netops #cisco_ise,index,netauth #cisco_nx_os,index,netops +#cisco_ucm,index,main +#citrix_netscaler,index,netfw #local_example,index,main #forcepoint_webprotect,index,netproxy #fortinet_fortios_event,index,netops diff --git a/package/etc/context_templates/vendor_product_by_source.conf.example b/package/etc/context_templates/vendor_product_by_source.conf.example index 591fa77..8be917a 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf.example +++ b/package/etc/context_templates/vendor_product_by_source.conf.example @@ -2,6 +2,10 @@ filter f_test_test { host("testvp-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) }; +filter f_citrix_netscaler { + host("test_ctitrixns-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) +}; filter f_juniper_idp { host("jnpidp-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) diff --git a/package/etc/context_templates/vendor_product_by_source.csv.example b/package/etc/context_templates/vendor_product_by_source.csv.example index 193732e..1901afe 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv.example +++ b/package/etc/context_templates/vendor_product_by_source.csv.example @@ -1,5 +1,6 @@ f_test_test,sc4s_vendor_product,"test_test" f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" +f_citrix_netscaler,sc4s_vendor_product,"citrix_netscaler" f_infoblox,sc4s_vendor_product,"infoblox" f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index a49e89b..ea83f05 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -88,10 +88,20 @@ source s_{{ .port_id }} { {{ else if eq .parser "cisco_meraki_parser" }} parser (p_cisco_meraki); rewrite(set_rfc5424_epochtime); +{{ else if eq .parser "citrix_netscaler" }} + parser(p_citrix_netscaler_date); + rewrite(r_citrix_netscaler_message); +{{ else if eq .parser "cisco_ucm" }} + parser (p_cisco_ucm_date); + rewrite (r_cisco_ucm_message); {{ else if eq .parser "no_parse" }} rewrite(set_no_parse); {{ else }} if { + filter(f_citrix_netscaler_message); + parser(p_citrix_netscaler_date); + rewrite(r_citrix_netscaler_message); + } elif { filter(f_rfc5424_strict); parser { syslog-parser(flags(syslog-protocol)); @@ -100,6 +110,10 @@ source s_{{ .port_id }} { } elif { parser (p_cisco_meraki); rewrite(set_rfc5424_epochtime); + } elif { + filter(f_cisco_ucm_message); + parser (p_cisco_ucm_date); + rewrite (r_cisco_ucm_message); } elif { filter(f_rfc3164_version); rewrite(set_rfc3164_no_version_string); diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 66bdb51..e68bf47 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -20,11 +20,11 @@ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ cp /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ for file in /opt/syslog-ng/etc/conf.d/local/context/*.example ; do cp --verbose -n $file ${file%.example}; done cp --verbose -R /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ - +mkdir -p /opt/syslog-ng/var/log echo syslog-ng checking config echo sc4s version=$(cat /VERSION) -echo sc4s version=$(cat /VERSION) >/var/log/syslog-ng.out -/opt/syslog-ng/sbin/syslog-ng -s >>/var/log/syslog-ng.out 2>/var/log/syslog-ng.err +echo sc4s version=$(cat /VERSION) >/opt/syslog-ng/var/log/syslog-ng.out +/opt/syslog-ng/sbin/syslog-ng -s >>/opt/syslog-ng/var/log/syslog-ng.out 2>/opt/syslog-ng/var/log/syslog-ng.err echo syslog-ng starting exec /opt/syslog-ng/sbin/syslog-ng $@ diff --git a/package/syslog-ng b/package/syslog-ng deleted file mode 160000 index b880087..0000000 --- a/package/syslog-ng +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b880087c95ce44e3d33722a825e895b9351aca58 diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..4fce685 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,11 @@ +[pytest] +testpaths = tests +addopts = + -v --tb=long + --splunk_type=docker + --keepalive + --splunk_hec_token=1ec3c8ac-74b3-46f2-ba44-a7c96b6ab236 + -n=6 +filterwarnings = + ignore::DeprecationWarning +junit_family=xunit1 \ No newline at end of file diff --git a/semtag b/semtag new file mode 100755 index 0000000..d000e3c --- /dev/null +++ b/semtag @@ -0,0 +1,625 @@ +#!/usr/bin/env bash + +PROG=semtag +PROG_VERSION="v0.1.0" + +SEMVER_REGEX="^v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(\-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$" +IDENTIFIER_REGEX="^\-([0-9A-Za-z-]+)\.([0-9A-Za-z-]+)*$" + +# Global variables +FIRST_VERSION="v0.0.0" +finalversion=$FIRST_VERSION +lastversion=$FIRST_VERSION +hasversiontag="false" +scope="patch" +displayonly="false" +forcetag="false" +forcedversion= +versionname= +identifier= + +HELP="\ +Usage: + $PROG + $PROG getlast + $PROG getfinal + $PROG (final|alpha|beta|candidate) [-s (major|minor|patch|auto) | -o] + $PROG --help + $PROG --version +Options: + -s The scope that must be increased, can be major, minor or patch. + The resulting version will match X.Y.Z(-PRERELEASE)(+BUILD) + where X, Y and Z are positive integers, PRERELEASE is an optionnal + string composed of alphanumeric characters describing if the build is + a release candidate, alpha or beta version, with a number. + BUILD is also an optional string composed of alphanumeric + characters and hyphens. + Setting the scope as 'auto', the script will chose the scope between + 'minor' and 'patch', depending on the amount of lines added (<10% will + choose patch). + -v Specifies manually the version to be tagged, must be a valid semantic version + in the format X.Y.Z where X, Y and Z are positive integers. + -o Output the version only, shows the bumped version, but doesn't tag. + -f Forces to tag, even if there are unstaged or uncommited changes. +Commands: + --help Print this help message. + --version Prints the program's version. + get Returns both current final version and last tagged version. + getlast Returns the latest tagged version. + getfinal Returns the latest tagged final version. + getcurrent Returns the current version, based on the latest one, if there are uncommited or + unstaged changes, they will be reflected in the version, adding the number of + pending commits, current branch and commit hash. + final Tags the current build as a final version, this only can be done on the master branch. + candidate Tags the current build as a release candidate, the tag will contain all + the commits from the last final version. + alpha Tags the current build as an alpha version, the tag will contain all + the commits from the last final version. + beta Tags the current build as a beta version, the tag will contain all + the commits from the last final version." + +# Commands and options +ACTION="getlast" +ACTION="$1" +shift + +# We get the parameters +while getopts "v:s:of" opt; do + case $opt in + v) + forcedversion="$OPTARG" + ;; + s) + scope="$OPTARG" + ;; + o) + displayonly="true" + ;; + f) + forcetag="true" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +# Gets a string with the version and returns an array of maximum size of 5 with all the parts of the sematinc version +# $1 The string containing the version in semantic format +# $2 The variable to store the result array: +# position 0: major number +# position 1: minor number +# position 2: patch number +# position 3: identifier (or prerelease identifier) +# position 4: build info +function explode_version { + local __version=$1 + local __result=$2 + if [[ $__version =~ $SEMVER_REGEX ]] ; then + local __major=${BASH_REMATCH[1]} + local __minor=${BASH_REMATCH[2]} + local __patch=${BASH_REMATCH[3]} + local __prere=${BASH_REMATCH[4]} + local __build=${BASH_REMATCH[5]} + eval "$__result=(\"$__major\" \"$__minor\" \"$__patch\" \"$__prere\" \"$__build\")" + else + eval "$__result=" + fi +} + +# Compare two versions and returns -1, 0 or 1 +# $1 The first version to compare +# $2 The second version to compare +# $3 The variable where to store the result +function compare_versions { + local __first + local __second + explode_version $1 __first + explode_version $2 __second + local lv=$3 + + # Compares MAJOR, MINOR and PATCH + for i in 0 1 2; do + local __numberfirst=${__first[$i]} + local __numbersecond=${__second[$i]} + case $(($__numberfirst - $__numbersecond)) in + 0) + ;; + -[0-9]*) + eval "$lv=-1" + return 0 + ;; + [0-9]*) + eval "$lv=1" + return 0 + ;; + esac + done + + # Identifiers should compare with the ASCII order. + local __identifierfirst=${__first[3]} + local __identifiersecond=${__second[3]} + if [[ -n "$__identifierfirst" ]] && [[ -n "$__identifiersecond" ]]; then + if [[ "$__identifierfirst" > "$__identifiersecond" ]]; then + eval "$lv=1" + return 0 + elif [[ "$__identifierfirst" < "$__identifiersecond" ]]; then + eval "$lv=-1" + return 0 + fi + elif [[ -z "$__identifierfirst" ]] && [[ -n "$__identifiersecond" ]]; then + eval "$lv=1" + return 0 + elif [[ -n "$__identifierfirst" ]] && [[ -z "$__identifiersecond" ]]; then + eval "$lv=-1" + return 0 + fi + + eval "$lv=0" +} + +# Returns the last version of two +# $1 The first version to compare +# $2 The second version to compare +# $3 The variable where to store the last one +function get_latest_of_two { + local __first=$1 + local __second=$2 + local __result + local __latest=$3 + compare_versions $__first $__second __result + case $__result in + 0) + eval "$__latest=$__second" + ;; + -1) + eval "$__latest=$__second" + ;; + 1) + eval "$__latest=$__first" + ;; + esac +} + +# Assigns a 2 size array with the identifier, having the identifier at pos 0, and the number in pos 1 +# $1 The identifier in the format -id.# +# $2 The vferiable where to store the 2 size array +function explode_identifier { + local __identifier=$1 + local __result=$2 + if [[ $__identifier =~ $IDENTIFIER_REGEX ]] ; then + local __id=${BASH_REMATCH[1]} + local __number=${BASH_REMATCH[2]} + if [[ -z "$__number" ]]; then + __number=1 + fi + eval "$__result=(\"$__id\" \"$__number\")" + else + eval "$__result=" + fi +} + +# Gets a list of tags and assigns the base and latest versions +# Receives an array with the tags containing the versions +# Assigns to the global variables finalversion and lastversion the final version and the latest version +function get_latest { + local __taglist=("$@") + local __tagsnumber=${#__taglist[@]} + local __current + case $__tagsnumber in + 0) + finalversion=$FIRST_VERSION + lastversion=$FIRST_VERSION + ;; + 1) + __current=${__taglist[0]} + explode_version $__current ver + if [ -n "$ver" ]; then + if [ -n "${ver[3]}" ]; then + finalversion=$FIRST_VERSION + else + finalversion=$__current + fi + lastversion=$__current + else + finalversion=$FIRST_VERSION + lastversion=$FIRST_VERSION + fi + ;; + *) + local __lastpos=$(($__tagsnumber-1)) + for i in $(seq 0 $__lastpos) + do + __current=${__taglist[i]} + explode_version ${__taglist[i]} ver + if [ -n "$ver" ]; then + if [ -z "${ver[3]}" ]; then + get_latest_of_two $finalversion $__current finalversion + get_latest_of_two $lastversion $finalversion lastversion + else + get_latest_of_two $lastversion $__current lastversion + fi + fi + done + ;; + esac + + if git rev-parse -q --verify "refs/tags/$lastversion" >/dev/null; then + hasversiontag="true" + else + hasversiontag="false" + fi +} + +# Gets the next version given the provided scope +# $1 The version that is going to be bumped +# $2 The scope to bump +# $3 The variable where to stoer the result +function get_next_version { + local __exploded + local __fromversion=$1 + local __scope=$2 + local __result=$3 + explode_version $__fromversion __exploded + case $__scope in + major) + __exploded[0]=$((${__exploded[0]}+1)) + __exploded[1]=0 + __exploded[2]=0 + ;; + minor) + __exploded[1]=$((${__exploded[1]}+1)) + __exploded[2]=0 + ;; + patch) + __exploded[2]=$((${__exploded[2]}+1)) + ;; + esac + + eval "$__result=v${__exploded[0]}.${__exploded[1]}.${__exploded[2]}" +} + +function bump_version { + ## First we try to get the next version based on the existing last one + if [ "$scope" == "auto" ]; then + get_scope_auto scope + fi + + local __candidatefromlast=$FIRST_VERSION + local __explodedlast + explode_version $lastversion __explodedlast + if [[ -n "${__explodedlast[3]}" ]]; then + # Last version is not final + local __idlast + explode_identifier ${__explodedlast[3]} __idlast + + # We get the last, given the desired id based on the scope + __candidatefromlast="v${__explodedlast[0]}.${__explodedlast[1]}.${__explodedlast[2]}" + if [[ -n "$identifier" ]]; then + local __nextid="$identifier.1" + if [ "$identifier" == "${__idlast[0]}" ]; then + # We target the same identifier as the last so we increase one + __nextid="$identifier.$(( ${__idlast[1]}+1 ))" + __candidatefromlast="$__candidatefromlast-$__nextid" + else + # Different identifiers, we make sure we are assigning a higher identifier, if not, we increase the version + __candidatefromlast="$__candidatefromlast-$__nextid" + local __comparedwithlast + compare_versions $__candidatefromlast $lastversion __comparedwithlast + if [ "$__comparedwithlast" == -1 ]; then + get_next_version $__candidatefromlast $scope __candidatefromlast + __candidatefromlast="$__candidatefromlast-$__nextid" + fi + fi + fi + fi + + # Then we try to get the version based on the latest final one + local __candidatefromfinal=$FIRST_VERSION + get_next_version $finalversion $scope __candidatefromfinal + if [[ -n "$identifier" ]]; then + __candidatefromfinal="$__candidatefromfinal-$identifier.1" + fi + + # Finally we compare both candidates + local __resultversion + local __result + compare_versions $__candidatefromlast $__candidatefromfinal __result + case $__result in + 0) + __resultversion=$__candidatefromlast + ;; + -1) + __resultversion="$__candidatefromfinal" + ;; + 1) + __resultversion=$__candidatefromlast + ;; + esac + + eval "$1=$__resultversion" +} + +function increase_version { + local __version= + + if [ -z $forcedversion ]; then + bump_version __version + else + if [[ $forcedversion =~ $SEMVER_REGEX ]] ; then + compare_versions $forcedversion $lastversion __result + if [ $__result -le 0 ]; then + echo "Version can't be lower than last version: $lastversion" + exit 1 + fi + else + echo "Non valid version to bump" + exit 1 + fi + __version=$forcedversion + fi + + if [ "$displayonly" == "true" ]; then + echo "$__version" + else + if [ "$forcetag" == "false" ]; then + check_git_dirty_status + fi + local __commitlist + if [ "$finalversion" == "$FIRST_VERSION" ] || [ "$hasversiontag" != "true" ]; then + __commitlist="$(git log --pretty=oneline | cat)" + else + __commitlist="$(git log --pretty=oneline $finalversion... | cat)" + fi + + # If we are forcing a bump, we add bump to the commit list + if [[ -z $__commitlist && "$forcetag" == "true" ]]; then + __commitlist="bump" + fi + + if [[ -z $__commitlist ]]; then + echo "No commits since the last final version, not bumping version" + else + if [[ -z $versionname ]]; then + versionname=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + fi + local __message="$versionname +$__commitlist" + + # We check we have info on the user + local __username=$(git config user.name) + if [ -z "$__username" ]; then + __username=$(id -u -n) + git config user.name $__username + fi + local __useremail=$(git config user.email) + if [ -z "$__useremail" ]; then + __useremail=$(hostname) + git config user.email "$__username@$__useremail" + fi + + git tag -a $__version -m "$__message" + + # If we have a remote, we push there + local __remotes=$(git remote) + if [[ -n $__remotes ]]; then + for __remote in $__remotes; do + git push $__remote $__version > /dev/null + if [ $? -eq 0 ]; then + echo "$__version pushed to $__remote" + else + echo "Error pushing the tag $__version to $__remote" + exit 1 + fi + done + else + echo "$__version" + fi + fi + fi +} + +function check_git_dirty_status { + local __repostatus= + get_work_tree_status __repostatus + + if [ "$__repostatus" == "uncommitted" ]; then + echo "ERROR: You have uncommitted changes" + git status --porcelain + exit 1 + fi + + if [ "$__repostatus" == "unstaged" ]; then + echo "ERROR: You have unstaged changes" + git status --porcelain + exit 1 + fi +} + +# Get the total amount of lines of code in the repo +function get_total_lines { + local __empty_id="$(git hash-object -t tree /dev/null)" + local __changes="$(git diff --numstat $__empty_id | cat)" + local __added_deleted=$1 + get_changed_lines "$__changes" $__added_deleted +} + +# Get the total amount of lines of code since the provided tag +function get_sincetag_lines { + local __sincetag=$1 + local __changes="$(git diff --numstat $__sincetag | cat)" + local __added_deleted=$2 + get_changed_lines "$__changes" $__added_deleted +} + +function get_changed_lines { + local __changes_numstat=$1 + local __result=$2 + IFS=$'\n' read -rd '' -a __changes_array <<<"$__changes_numstat" + local __diff_regex="^([0-9]+)[[:space:]]+([0-9]+)[[:space:]]+.+$" + + local __total_added=0 + local __total_deleted=0 + for i in "${__changes_array[@]}" + do + if [[ $i =~ $__diff_regex ]] ; then + local __added=${BASH_REMATCH[1]} + local __deleted=${BASH_REMATCH[2]} + __total_added=$(( $__total_added+$__added )) + __total_deleted=$(( $__total_deleted+$__deleted )) + fi + done + eval "$2=( $__total_added $__total_deleted )" +} + +function get_scope_auto { + local __verbose=$2 + local __total=0 + local __since=0 + local __scope= + + get_total_lines __total + get_sincetag_lines $finalversion __since + + local __percentage=0 + if [ "$__total" != "0" ]; then + local __percentage=$(( 100*$__since/$__total )) + if [ $__percentage -gt "10" ]; then + __scope="minor" + else + __scope="patch" + fi + fi + + eval "$1=$__scope" + if [[ -n "$__verbose" ]]; then + echo "[Auto Scope] Percentage of lines changed: $__percentage" + echo "[Auto Scope] : $__scope" + fi +} + +function get_work_tree_status { + # Update the index + git update-index -q --ignore-submodules --refresh > /dev/null + eval "$1=" + + if ! git diff-files --quiet --ignore-submodules -- > /dev/null + then + eval "$1=unstaged" + fi + + if ! git diff-index --cached --quiet HEAD --ignore-submodules -- > /dev/null + then + eval "$1=uncommitted" + fi +} + +function get_current { + if [ "$hasversiontag" == "true" ]; then + local __commitcount="$(git rev-list $lastversion.. --count)" + else + local __commitcount="$(git rev-list --count HEAD)" + fi + local __status= + get_work_tree_status __status + + if [ "$__commitcount" == "0" ] && [ -z "$__status" ]; then + eval "$1=$lastversion" + else + local __buildinfo="$(git rev-parse --short HEAD)" + local __currentbranch="$(git rev-parse --abbrev-ref HEAD)" + if [ "$__currentbranch" != "master" ]; then + __buildinfo="$__currentbranch.$__buildinfo" + fi + + local __suffix= + if [ "$__commitcount" != "0" ]; then + if [ -n "$__suffix" ]; then + __suffix="$__suffix." + fi + __suffix="$__suffix$__commitcount" + fi + if [ -n "$__status" ]; then + if [ -n "$__suffix" ]; then + __suffix="$__suffix." + fi + __suffix="$__suffix$__status" + fi + + __suffix="$__suffix+$__buildinfo" + if [ "$lastversion" == "$finalversion" ]; then + scope="patch" + identifier= + local __bumped= + bump_version __bumped + eval "$1=$__bumped-dev.$__suffix" + else + eval "$1=$lastversion.$__suffix" + fi + fi +} + +function init { + git fetch > /dev/null + TAGS="$(git tag)" + IFS=$'\n' read -rd '' -a TAG_ARRAY <<<"$TAGS" + + get_latest ${TAG_ARRAY[@]} + currentbranch="$(git rev-parse --abbrev-ref HEAD)" +} + +case $ACTION in + --help) + echo -e "$HELP" + ;; + --version) + echo -e "${PROG}: $PROG_VERSION" + ;; + final) + init + diff=$(git diff master | cat) + if [ "$forcetag" == "false" ]; then + if [ -n "$diff" ]; then + echo "ERROR: Branch must be updated with master for final versions" + exit 1 + fi + fi + increase_version + ;; + alpha|beta) + init + identifier="$ACTION" + increase_version + ;; + candidate) + init + identifier="rc" + increase_version + ;; + getlast) + init + echo "$lastversion" + ;; + getfinal) + init + echo "$finalversion" + ;; + getcurrent) + init + get_current current + echo "$current" + ;; + get) + init + echo "Current final version: $finalversion" + echo "Last tagged version: $lastversion" + ;; + *) + echo "'$ACTION' is not a valid command, see --help for available commands." + ;; +esac \ No newline at end of file diff --git a/splunk/Dockerfile b/splunk/Dockerfile new file mode 100644 index 0000000..7f1f8c0 --- /dev/null +++ b/splunk/Dockerfile @@ -0,0 +1,5 @@ +ARG SPLUNK_VERSION=latest +FROM splunk/splunk:$SPLUNK_VERSION + + +COPY etc/apps/ /opt/splunk/etc/apps/ \ No newline at end of file diff --git a/test-with-compose.sh b/test-with-compose.sh index 25eeb4f..47911fb 100755 --- a/test-with-compose.sh +++ b/test-with-compose.sh @@ -4,35 +4,8 @@ #Use of this source code is governed by a BSD-2-clause-style #license that can be found in the LICENSE-BSD2 file or at #https://opensource.org/licenses/BSD-2-Clause -WAITON=${1:-test} -compose=${2:-docker-compose.yml} -echo $WAITON $compose -mkdir test-results -docker-compose down -docker volume rm sc4s-results -docker volume rm splunk-etc +docker-compose -f tests/docker-compose-script.yml build +docker-compose -f tests/docker-compose-script.yml up --abort-on-container-exit -docker volume create sc4s-results -docker volume create splunk-etc - -docker container create --name dummy \ - -v sc4s-results:/work/test-results \ - -v splunk-etc:/work/splunk-etc \ - registry.access.redhat.com/ubi7/ubi -docker cp ./splunk/etc/* dummy:/work/splunk-etc/ -docker rm dummy - -docker-compose -f $compose pull -docker-compose -f $compose up -d splunk -docker-compose -f $compose up -d sc4s -sleep 60 -docker-compose -f $compose build -docker-compose -f $compose up --abort-on-container-exit --exit-code-from $WAITON - -docker container create --name dummy \ - -v sc4s-results:/work/test-results \ - registry.access.redhat.com/ubi7/ubi - -docker cp dummy:/work/test-results/functional test-results -docker rm dummy EXIT=$0 + diff --git a/tests/Dockerfile b/tests/Dockerfile index 285d52c..56537b2 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -10,12 +10,13 @@ FROM python:3.7 COPY requirements.txt / RUN pip3 install -r /requirements.txt +RUN pip3 install pytest-xdist RUN mkdir -p /work/tests RUN mkdir -p /work/test-results/functional COPY entrypoint.sh / COPY wait-for /bin/ COPY ./* /work/tests/ -COPY pytest.ini /work +COPY pytest-ci.ini /work/pytest.ini COPY ./data /work/tests/data #WORKDIR /work CMD /entrypoint.sh diff --git a/tests/conftest.py b/tests/conftest.py index 7777889..a02c8a2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,9 +5,12 @@ # https://opensource.org/licenses/BSD-2-Clause import os import random +import socket +import uuid from time import sleep import pytest +import requests import splunklib.client as client @@ -30,16 +33,196 @@ def get_host_key(setup_wordlist): return host -@pytest.fixture -def setup_splunk(): +def pytest_addoption(parser): + group = parser.getgroup('splunk-addon') + + group.addoption( + '--sc4s_host', + action='store', + dest='sc4s_host', + default='127.0.0.1', + help='Address of the sc4s Server' + ) + + group.addoption( + '--splunk_app', + action='store', + dest='splunk_app', + default='package', + help='Path to Splunk app' + ) + group.addoption( + '--splunk_type', + action='store', + dest='splunk_type', + default='docker', + help='Type of Splunk' + ) + group.addoption( + '--splunk_host', + action='store', + dest='splunk_host', + default='127.0.0.1', + help='Address of the Splunk Server' + ) + group.addoption( + '--splunk_port', + action='store', + dest='splunk_port', + default='8089', + help='Splunk rest port' + ) + group.addoption( + '--splunk_user', + action='store', + dest='splunk_user', + default='admin', + help='Splunk login user' + ) + group.addoption( + '--splunk_password', + action='store', + dest='splunk_password', + default='Changed@11', + help='Splunk password' + ) + group.addoption( + '--splunk_hec_token', + action='store', + dest='splunk_hec_token', + default=str(uuid.uuid1()), + help='Splunk HEC token' + ) + group.addoption( + '--splunk_version', + action='store', + dest='splunk_version', + default='latest', + help='Splunk version' + ) + + +def is_responsive(url): + try: + response = requests.get(url) + if response.status_code != 500: + return True + except ConnectionError: + return False + + +def is_responsive_splunk(splunk): + try: + client.connect(username=splunk['username'], password=splunk['password'], host=splunk['host'], + port=splunk['port']) + return True + except Exception: + return False + + +@pytest.fixture(scope="session") +def docker_compose_file(pytestconfig): + """Get an absolute path to the `docker-compose.yml` file. Override this + fixture in your tests if you need a custom location.""" + + return os.path.join(str(pytestconfig.invocation_dir), "tests", "docker-compose.yml") + + +@pytest.fixture(scope="session") +def splunk(request): + if request.config.getoption('splunk_type') == 'external': + request.fixturenames.append('splunk_external') + splunk = request.getfixturevalue("splunk_external") + elif request.config.getoption('splunk_type') == 'docker': + os.environ['SPLUNK_PASSWORD'] = request.config.getoption( + 'splunk_password') + os.environ['SPLUNK_HEC_TOKEN'] = request.config.getoption( + 'splunk_hec_token') + request.fixturenames.append('splunk_docker') + splunk = request.getfixturevalue("splunk_docker") + else: + raise Exception + + yield splunk + + +@pytest.fixture(scope="session") +def sc4s(request): + if request.config.getoption('splunk_type') == 'external': + request.fixturenames.append('sc4s_external') + sc4s = request.getfixturevalue("sc4s_external") + elif request.config.getoption('splunk_type') == 'docker': + request.fixturenames.append('sc4s_docker') + sc4s = request.getfixturevalue("sc4s_docker") + else: + raise Exception + + yield sc4s + + +@pytest.fixture(scope="session") +def splunk_docker(request, docker_services): + docker_services.start('splunk') + port = docker_services.port_for("splunk", 8089) + + splunk = { + 'host': docker_services.docker_ip, + 'port': port, + 'username': request.config.getoption('splunk_user'), + 'password': request.config.getoption('splunk_password'), + } + + docker_services.wait_until_responsive( + timeout=180.0, pause=1.0, check=lambda: is_responsive_splunk(splunk) + ) + + return splunk + + +@pytest.fixture(scope="session") +def splunk_external(request): + splunk = { + 'host': request.config.getoption('splunk_host'), + 'port': request.config.getoption('splunk_port'), + 'username': request.config.getoption('splunk_user'), + 'password': request.config.getoption('splunk_password'), + } + return splunk + + +@pytest.fixture(scope="session") +def sc4s_docker(docker_services): + docker_services.start('sc4s') + + ports = {514: docker_services.port_for("sc4s", 514)} + for x in range(5000, 5006): + ports.update({x: docker_services.port_for("sc4s", x)}) + + return docker_services.docker_ip, ports + + +@pytest.fixture(scope="session") +def sc4s_external(request): + ports = {514: 514} + for x in range(5000, 5050): + ports.update({x: x}) + + return request.config.getoption('sc4s_host'), ports + + +@pytest.fixture() +def setup_sc4s(sc4s): + return sc4s + + +@pytest.fixture(scope="session") +def setup_splunk(splunk): tried = 0 - username = os.getenv('SPLUNK_USER', "admin") - password = os.getenv('SPLUNK_PASSWORD', "Changed@11") - host = os.getenv('SPLUNK_HOST', "splunk") - port = os.getenv('SPLUNK_PORT', "8089") + while True: try: - c = client.connect(username=username, password=password, host=host, port=port) + c = client.connect(username=splunk['username'], password=splunk['password'], host=splunk['host'], + port=splunk['port']) break except ConnectionRefusedError: tried += 1 diff --git a/tests/docker-compose-ci.yml b/tests/docker-compose-ci.yml new file mode 100644 index 0000000..7ce6645 --- /dev/null +++ b/tests/docker-compose-ci.yml @@ -0,0 +1,66 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . +version: "3.7" + +services: + test: + build: . + links: + - splunk + - sc4s + volumes: + - results:/work/test-results + + sc4s: + build: + context: ../package + args: + BRANCH: ${SYSLOG} + hostname: sc4s + #When this is enabled test_common will fail + # command: -det + ports: + - "514" + - "601" + - "514/udp" + - "5000-5050" + - "5000-5050/udp" + - "6514" + stdin_open: true + tty: true + links: + - splunk + environment: + - SPLUNK_HEC_URL=https://splunk:8088 + - SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + - SC4S_SOURCE_TLS_ENABLE=no + - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 + - SC4S_LISTEN_CISCO_ASA_TCP_PORT=5001 + - SC4S_LISTEN_CISCO_IOS_TCP_PORT=5002 + - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003 + - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 + - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 + - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 + - SC4S_ARCHIVE_GLOBAL=no + splunk: + build: ../splunk + hostname: splunk + ports: + - "8000" + - "8088" + - "8089" + environment: + - SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + - SPLUNK_PASSWORD=Changed@11 + - SPLUNK_START_ARGS=--accept-license + +volumes: + results: + external: false diff --git a/tests/docker-compose-script.yml b/tests/docker-compose-script.yml new file mode 100644 index 0000000..7bd4de2 --- /dev/null +++ b/tests/docker-compose-script.yml @@ -0,0 +1,66 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . +version: "3.7" + + +services: + test: + build: . + links: + - splunk + - sc4s + volumes: + - results:/work/test-results + + sc4s: + build: + context: ../package + hostname: sc4s + #When this is enabled test_common will fail + # command: -det + ports: + - "514" + - "601" + - "514/udp" + - "5000-5050" + - "5000-5050/udp" + - "6514" + stdin_open: true + tty: true + links: + - splunk + environment: + - SPLUNK_HEC_URL=https://splunk:8088 + - SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + - SC4S_SOURCE_TLS_ENABLE=no + - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 + - SC4S_LISTEN_CISCO_ASA_TCP_PORT=5001 + - SC4S_LISTEN_CISCO_IOS_TCP_PORT=5002 + - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003 + - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 + - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 + - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 + - SC4S_ARCHIVE_GLOBAL=no + splunk: + build: + context: ../splunk + hostname: splunk + ports: + - "8000" + - "8088" + - "8089" + environment: + - SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + - SPLUNK_PASSWORD=Changed@11 + - SPLUNK_START_ARGS=--accept-license + +volumes: + results: + external: false \ No newline at end of file diff --git a/docker-compose-ci.yml b/tests/docker-compose.yml similarity index 50% rename from docker-compose-ci.yml rename to tests/docker-compose.yml index 1bd512b..9966505 100644 --- a/docker-compose-ci.yml +++ b/tests/docker-compose.yml @@ -7,64 +7,57 @@ #You should have received a copy of the CC0 legalcode along with this #work. If not, see . version: "3.7" -services: - test: - build: ./tests - entrypoint: - - /entrypoint.sh - - --workers - - auto - - --tests-per-worker - - auto - links: - - splunk - - sc4s - volumes: - - sc4s-results:/work/test-results - environment: - - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} + +services: sc4s: - image: ${REGISTRY}/${CI_IMAGE}:${CIRCLE_SHA1} + build: + context: ../package hostname: sc4s + #When this is enabled test_common will fail + # command: -det ports: - "514" - "601" - "514/udp" - - "5000" - - "5000/udp" + - "5000-5050" + - "5000-5050/udp" + - "6514" stdin_open: true tty: true links: - splunk environment: - - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} + - SPLUNK_HEC_URL=https://splunk:8088 - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + - SC4S_SOURCE_TLS_ENABLE=no - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 + - SC4S_LISTEN_CISCO_ASA_TCP_PORT=5001 + - SC4S_LISTEN_CISCO_IOS_TCP_PORT=5002 + - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003 + - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 + - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 - + - SC4S_ARCHIVE_GLOBAL=no + - SC4S_SOURCE_STORE_RAWMSG=yes splunk: - image: splunk/splunk:latest + build: + context: ../splunk hostname: splunk ports: - "8000:8000" - - "8088:8088" - - "8089:8089" + - "8088" + - "8089" environment: - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} - - SPLUNK_START_ARGS=${SPLUNK_START_ARGS} - - SPLUNK_APPS_URL=${SPLUNK_APPS_URL} - - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} - - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} + - SPLUNK_START_ARGS=--accept-license volumes: - - splunk-etc:/opt/splunk/etc + - splunk-var:/opt/splunk/vars + volumes: - sc4s-results: - external: true - splunk-etc: - external: true + results: + external: false + splunk-var: + external: false \ No newline at end of file diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh index 6128096..157ddcc 100755 --- a/tests/entrypoint.sh +++ b/tests/entrypoint.sh @@ -1,23 +1,4 @@ #!/bin/sh -echo Check for sc4s -wait-for sc4s:514 -t 0 -- echo scs is up -echo check for splunk web -wait-for splunk:8000 -t 0 -- echo splunkweb is up -echo check for splunk mgmt -wait-for splunk:8089 -t 0 -- echo splunkmgmt is up -echo check for splunk hec -wait-for splunk:8088 -t 0 -- echo splunkhec is up -sleep 30 - -echo Check for sc4s -wait-for sc4s:514 -t 0 -- echo scs is up -echo check for splunk web -wait-for splunk:8000 -t 0 -- echo splunkweb is up -echo check for splunk mgmt -wait-for splunk:8089 -t 0 -- echo splunkmgmt is up -echo check for splunk hec -wait-for splunk:8088 -t 0 -- echo splunkhec is up - - -cd /work;python -m pytest --junitxml=/work/test-results/functional/functional.xml $@ +cd /work +pytest $@ diff --git a/tests/pytest-ci.ini b/tests/pytest-ci.ini new file mode 100644 index 0000000..988c802 --- /dev/null +++ b/tests/pytest-ci.ini @@ -0,0 +1,11 @@ +[pytest] +addopts = -v --tb=long + --splunk_type=external + --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + --splunk_host=splunk + --sc4s_host=sc4s + --junitxml=/work/test-results/test.xml + -n=10 +junit_family=xunit1 +filterwarnings = + ignore::DeprecationWarning diff --git a/tests/pytest.ini b/tests/pytest.ini deleted file mode 100644 index ce9083e..0000000 --- a/tests/pytest.ini +++ /dev/null @@ -1,5 +0,0 @@ -[pytest] -addopts = -# --force-flaky --max-runs=3 --min-passes=1 -filterwarnings = - ignore::DeprecationWarning diff --git a/tests/requirements.txt b/tests/requirements.txt index df09269..5bb5756 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -13,5 +13,8 @@ splunk-sdk flake8 pytz flaky -#pytest-randomly -pytest-parallel +lovely-pytest-docker +pathlib2 +psutil +requests +pytest-xdist \ No newline at end of file diff --git a/tests/sendmessage.py b/tests/sendmessage.py index 122381f..7618b97 100644 --- a/tests/sendmessage.py +++ b/tests/sendmessage.py @@ -9,8 +9,8 @@ import os def sendsingle(message, - host=os.getenv('SYSLOG_HOST', "sc4s"), - port=514): + host, + port): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_address = (host, port) diff --git a/tests/splunkutils.py b/tests/splunkutils.py index bb77a58..205fcfa 100644 --- a/tests/splunkutils.py +++ b/tests/splunkutils.py @@ -24,12 +24,13 @@ def splunk_single(service, search): if stats["isDone"] == "1": break - sleep(2) + else: + sleep(2) # Get the results and display them resultCount = stats["resultCount"] eventCount = stats["eventCount"] - if resultCount > 0 or tried > 15: + if resultCount > 0 or tried > 5: break else: tried += 1 diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index c9adef0..9e1a9a0 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -14,14 +14,14 @@ # #Oct 8 15:00:25 DEVICENAME time=1570561225|hostname=devicename|severity=Informational|confidence_level=Unknown|product=IPS|action=Drop|ifdir=inbound|ifname=bond2|loguid={0x5d9cdcc9,0x8d159f,0x5f19f392,0x1897a828}|origin=1.1.1.1|time=1570561225|version=1|attack=Streaming Engine: TCP Segment Limit Enforcement|attack_info=TCP segment out of maximum allowed sequence. Packet dropped.|chassis_bladed_system=[ 1_3 ]|dst=10.10.10.10|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|performance_impact=0|protection_id=tcp_segment_limit|protection_name=TCP Segment Limit Enforcement|protection_type=settings_tcp|proto=6|rule=393|rule_name=10.384_..|rule_uid={9F77F944-8DD5-4ADF-803A-785D03B3A2E8}|s_port=46455|service=443|smartdefense_profile=Recommended_Protection_ded9e8d8ee89d|src=1.1.1.2| -def test_checkpoint_splunk_ips(record_property, setup_wordlist, setup_splunk): +def test_checkpoint_splunk_ips(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} time={% now 'utc', '%s' %}|hostname={{ host }}|severity=Informational|confidence_level=Unknown|product=IPS|action=Drop|ifdir=inbound|ifname=bond2|loguid={0x5d9cdcc9,0x8d159f,0x5f19f392,0x1897a828}|origin=1.1.1.1|time={% now 'utc', '%s' %}|version=1|attack=Streaming Engine: TCP Segment Limit Enforcement|attack_info=TCP segment out of maximum allowed sequence. Packet dropped.|chassis_bladed_system=[ 1_3 ]|dst=10.10.10.10|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|performance_impact=0|protection_id=tcp_segment_limit|protection_name=TCP Segment Limit Enforcement|protection_type=settings_tcp|proto=6|rule=393|rule_name=10.384_..|rule_uid={9F77F944-8DD5-4ADF-803A-785D03B3A2E8}|s_port=46455|service=443|smartdefense_profile=Recommended_Protection_ded9e8d8ee89d|src=1.1.1.2|\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"cp_log\" | head 2") search = st.render(host=host) @@ -35,14 +35,14 @@ def test_checkpoint_splunk_ips(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #$Oct 8 15:48:31 DEVICENAME time=1570564111|hostname=devicename|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time=1570564111|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11_..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2| -def test_checkpoint_splunk_firewall(record_property, setup_wordlist, setup_splunk): +def test_checkpoint_splunk_firewall(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} time={% now 'utc', '%s' %}|hostname={{ host }}|product=Firewall|action=Drop|ifdir=inbound|ifname=bond1|loguid={0x5d9ce80f,0x8d0555,0x5f19f392,0x18982828}|origin=1.1.1.1|time={% now 'utc', '%s' %}|version=1|chassis_bladed_system=[ 1_1 ]|dst=10.10.10.10|inzone=External|origin_sic_name=CN=something_03_local,O=devicename.domain.com.p7fdbt|outzone=Internal|proto=6|rule=402|rule_name=11_..|rule_uid={C8CD796E-7BD5-47B6-90CA-B250D062D5E5}|s_port=33687|service=23|src=1.1.1.2|\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cp_log\" | head 2") search = st.render(host=host) @@ -56,14 +56,14 @@ def test_checkpoint_splunk_firewall(record_property, setup_wordlist, setup_splun assert resultCount == 1 #Oct 9 12:01:16 DEVICENAME |hostname=DEVICENAME|product=mds-query-tool|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time=1570641309|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login| -def test_checkpoint_splunk_mds(record_property, setup_wordlist, setup_splunk): +def test_checkpoint_splunk_mds(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} |hostname={{ host }}|time={% now 'utc', '%s' %}|product=mds-query-tool|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time={% now 'utc', '%s' %}|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login|\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"cp_log\" | head 2") search = st.render(host=host) @@ -77,14 +77,14 @@ def test_checkpoint_splunk_mds(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #Oct 9 12:01:16 DEVICENAME|hostname=DEVICENAME|product=CPMI Client|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time=1570641173|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login -def test_checkpoint_splunk_cpmi(record_property, setup_wordlist, setup_splunk): +def test_checkpoint_splunk_cpmi(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} |hostname={{ host }}|product=CPMI Client|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time={% now 'utc', '%s' %}|version=5|administrator=localhost|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"cp_log\" | head 2") search = st.render(host=host) @@ -98,14 +98,14 @@ def test_checkpoint_splunk_cpmi(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #Oct 9 12:01:16 DEVICENAME |hostname=DEVICENAME|product=WEB_API|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time=1570640578|version=5|administrator=tufinapi|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login -def test_checkpoint_splunk_web_api(record_property, setup_wordlist, setup_splunk): +def test_checkpoint_splunk_web_api(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} |hostname={{ host }}|product=WEB_API|action=Accept|ifdir=outbound|origin=1.1.1.1|2.2.2.2|originsicname=cn\=cp_mgmt,o\=DEVICENAME.domain.com.p7fdbt|sequencenum=1|time={% now 'utc', '%s' %}|version=5|administrator=tufinapi|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log Out|operation_number=12|subject=Administrator Login\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"cp_log\" | head 2") search = st.render(host=host) @@ -119,14 +119,14 @@ def test_checkpoint_splunk_web_api(record_property, setup_wordlist, setup_splunk assert resultCount == 1 #Oct 9 11:05:15 DEVICENAME time=1570633513|hostname=DEVICENAME|product=SmartConsole|action=Accept|ifdir=outbound|origin=1.1.1.1|4.4.4.4|sequencenum=1|time=1570633513|version=5|additional_info=Authentication method: Password based application token|administrator=psanadhya|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log In|operation_number=10|subject=Administrator Login| -def test_checkpoint_splunk_smartconsole(record_property, setup_wordlist, setup_splunk): +def test_checkpoint_splunk_smartconsole(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} time={% now 'utc', '%s' %}|hostname={{ host }}|product=SmartConsole|action=Accept|ifdir=outbound|origin=1.1.1.1|4.4.4.4|sequencenum=1|time={% now 'utc', '%s' %}|version=5|additional_info=Authentication method: Password based application token|administrator=psanadhya|client_ip=3.3.3.3|machine=DEVICENAME|operation=Log In|operation_number=10|subject=Administrator Login|\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"cp_log\" | head 2") search = st.render(host=host) @@ -141,7 +141,7 @@ def test_checkpoint_splunk_smartconsole(record_property, setup_wordlist, setup_s #<6>kernel: sd 2:0:0:0: SCSI error: return code = 0x00040000 -def test_checkpoint_splunk_os(record_property, setup_wordlist, setup_splunk): +def test_checkpoint_splunk_os(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) @@ -150,7 +150,7 @@ def test_checkpoint_splunk_os(record_property, setup_wordlist, setup_splunk): "{{ mark }}kernel: sd 2:0:0:0: SCSI error: return code = 0x{{pid}}\n") message = mt.render(mark="<6>", pid=pid) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=osnix \"0x{{ pid }}\" sourcetype=\"nix:syslog\" | head 2") search = st.render(pid=pid) diff --git a/tests/test_cisco_acs.py b/tests/test_cisco_acs.py index 29bf81f..190e985 100644 --- a/tests/test_cisco_acs.py +++ b/tests/test_cisco_acs.py @@ -13,13 +13,13 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) -def test_cisco_acs_single(record_property, setup_wordlist, setup_splunk): +def test_cisco_acs_single(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0765855540 1 0 2019-10-24 21:01:05.028 +00:00 0178632943 5202 NOTICE Device-Administration: Command Authorization succeeded, ACSVersion=acs-5.8.1.4-B.462.x86_64, ConfigVersionId=16489, Device IP Address=10.0.0.93, DestinationIPAddress=10.0.0.10, DestinationPort=49, UserName=nsdevman, CmdSet=[ CmdAV=show CmdArgAV=vpn-sessiondb CmdArgAV=full CmdArgAV=ra-ikev2-ipsec ], Protocol=Tacacs, MatchedCommandSet=fw3, RequestLatency=11, Type=Authorization, Privilege-Level=15, Authen-Type=ASCII, Service=None, User=nsdevman, Port=443, Remote-Address=10.0.0.15, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=mnsvdcfpiuac03/359448835/9871764, AuthenticationIdentityStore=AD1, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedCommandSet=fw3, IdentityGroup=IdentityGroup:All Groups:SystemID, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15004 , Step=15013 , Step=24210 , Step=24212 , Step=24432 , Step=24325 , Step=24313 , Step=24319 , Step=24323 , Step=24420 , Step=24355 , Step=24416 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15018 , Step=13024 , Step=13034 , SelectedAuthenticationIdentityStores=Internal Users, NetworkDeviceName=devicenamehere, NetworkDeviceGroups=Device Type:All Device Types:Firewall:Cisco Systems:Firewall:ASA5545, NetworkDeviceGroups=Location:All Locations:MN, ServiceSelectionMatchedRule=TACACS, IdentityPolicyMatchedRule=Firewall, AuthorizationPolicyMatchedRule=nsdevman, AD-User-Candidate-Identities=nsdevman@ent.example.corp, AD-User-DNS-Domain=ent.example.corp, AD-User-NetBios-Name=AD-ENT, AD-User-Resolved-Identities=nsdevman@ent.example.corp, AD-User-Join-Point=ENT.example.CORP, AD-User-Resolved-DNs=CN=nsdevman\,OU=Service Accounts\,OU=CAO\,OU=ENT\,DC=ent\,DC=wfb\,DC=example\,DC=corp, StepData=10=nsdevman, StepData=11=ent.example.corp, StepData=12=example.corp, StepData=15=ent.example.corp, AD-Domain=ent.example.corp, IdentityAccessRestricted=false, UserIdentityGroup=IdentityGroup:All Groups:SystemID, Cisco-Firewall=Superuser, Firewall=Superuser, NetSec-CSM=User, NetSec-Logging=Engineer, Response={Type=Authorization; Author-Reply-Status=PassAdd; ExternalIdentityStoreName=AD1; }\n") message = mt.render(mark="<165>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\" | head 11") search = st.render(host=host) @@ -32,18 +32,18 @@ def test_cisco_acs_single(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_cisco_acs_multi(record_property, setup_wordlist, setup_splunk): +def test_cisco_acs_multi(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0000000002 2 0 2011-08-01 22:32:53.032 +00:00 0000008450 5203 NOTICE Device-Administration: Session Authorization succeeded, ACSVersion=acs-5.2.0.26-B.3075, ConfigVersionId=117, Device IP Address=192.168.26.137, UserName=edward, CmdSet=[ CmdAV= ], Protocol=Tacacs, RequestLatency=10, NetworkDeviceName=switch, Type=Authorization, Privilege-Level=1, Authen-Type=ASCII, Service=Login, User=edward, Port=tty2, Remote-Address=10.78.167.190, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=ACS41/101085887/112, AuthenticationIdentityStore=Internal Users, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedShellProfile=Permit Access, IdentityGroup=IdentityGroup:All Groups, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15006 , Step=15013 , Step=24210 , Step=24212 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15017 , Step=13034 ,\n") message = mt.render(mark="<165>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0000000002 2 1 Step=13015 , SelectedAuthenticationIdentityStores=Internal Users, NetworkDeviceGroups=s1Migrated_NDGs:All s1Migrated_NDGs, NetworkDeviceGroups=Device Type:All Device Types, NetworkDeviceGroups=Location:All Locations, ServiceSelectionMatchedRule=Rule-2, IdentityPolicyMatchedRule=Default, AuthorizationPolicyMatchedRule=Rule-0, Action=Login, Privilege-Level=1, Authen-Type=ASCII, Service=Login, Remote-Address=10.78.167.190, UserIdentityGroup=IdentityGroup:All\n") message = mt.render(mark="<165>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\" \"Step=13015\" | head 11") search = st.render(host=host) diff --git a/tests/test_cisco_apic.py b/tests/test_cisco_apic.py new file mode 100644 index 0000000..9526f55 --- /dev/null +++ b/tests/test_cisco_apic.py @@ -0,0 +1,53 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<11>July 22 22:45:28 apic1 %LOG_LOCAL0-2-SYSTEM_MSG [F0110][soaking][node-failed][critical][topology/pod-1/node-102/fault-F0110] Node 102 not reachable. unknown +def test_cisco_aci(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} %LOG_LOCAL0-2-SYSTEM_MSG [F0110][soaking][node-failed][critical][topology/pod-1/node-102/fault-F0110]\n") + message = mt.render(mark="<165>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"cisco:apic:events\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#%ACLLOG-5-ACLLOG_PKTLOG +def test_cisco_aci_acl(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} %ACLLOG-5-ACLLOG_PKTLOG unable to locate real message\n") + message = mt.render(mark="<165>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:apic:acl\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_cisco_asa.py b/tests/test_cisco_asa.py index 6bcd920..32d0c00 100644 --- a/tests/test_cisco_asa.py +++ b/tests/test_cisco_asa.py @@ -16,14 +16,14 @@ # Apr 15 2017 00:21:14 192.168.12.1 : %ASA-5-111010: User 'john', running 'CLI' from IP 0.0.0.0, executed 'dir disk0:/dap.xml' # Apr 15 2017 00:22:27 192.168.12.1 : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:81.24.28.226 dst inside:72.142.17.10 (type 3, code 0) on outside interface. Original IP payload: udp src 72.142.17.10/40998 dst 194.153.237.66/53. # Apr 15 2017 00:22:42 192.168.12.1 : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 -def test_cisco_asa_traditional(record_property, setup_wordlist, setup_splunk): +def test_cisco_asa_traditional(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-003164\" | head 2") search = st.render(host=host) @@ -37,16 +37,16 @@ def test_cisco_asa_traditional(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 # <164>Jan 31 2020 17:24:03: %ASA-4-402119: IPSEC: Received an ESP packet (SPI= 0x0C190BF9, sequence number= 0x598243) from 192.0.0.1 (user= 192.0.0.1) to 192.0.0.2 that failed anti-replay checking. -def test_cisco_asa_traditional_nohost(record_property, setup_wordlist, setup_splunk): +def test_cisco_asa_traditional_nohost(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( - "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %}: %ASA-4-402119: IPSEC: Received an ESP packet (SPI= 0x0C190BF9, sequence number= 0x598243) from {host} (user= 192.0.0.1) to 192.0.0.2 that failed anti-replay checking.\n") + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %}: %ASA-4-402119: IPSEC: Received an ESP packet (SPI= 0x0C190BF9, sequence number= 0x598243) from {{host}} (user= 192.0.0.1) to 192.0.0.2 that failed anti-replay checking.\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search index=netfw sourcetype=\"cisco:asa\" \"%ASA-4-402119\" \"{host}\" | head 2") + st = env.from_string("search index=netfw sourcetype=\"cisco:asa\" \"%ASA-4-402119\" \"{{ host }}\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -59,14 +59,14 @@ def test_cisco_asa_traditional_nohost(record_property, setup_wordlist, setup_spl # <166>2018-06-27T12:17:46Z asa : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 -def test_cisco_asa_rfc5424(record_property, setup_wordlist, setup_splunk): +def test_cisco_asa_rfc5424(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} : %ASA-3-005424: TCP access denied by ACL from 179.236.133.160/5424 to outside:72.142.18.38/23\n") message = mt.render(mark="<166>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-005424\"| head 2") search = st.render(host=host) diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py index 13ec3f5..f3c3046 100644 --- a/tests/test_cisco_ios.py +++ b/tests/test_cisco_ios.py @@ -13,14 +13,14 @@ # <190>30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated -def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }}{{ seq }}: {{ host }}: *{% now 'utc', '%b %d %H:%M:%S' %}.100: CET: %SEC-6-IPACCESSLOGP: list 110 denied tcp 54.122.123.124(8932) -> 10.1.0.1(22), 1 packet\n") message = mt.render(mark="<166>", seq=20, host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2") search = st.render(host=host) diff --git a/tests/test_cisco_ise.py b/tests/test_cisco_ise.py index 0093686..000e5fb 100644 --- a/tests/test_cisco_ise.py +++ b/tests/test_cisco_ise.py @@ -17,28 +17,28 @@ #<165>Apr 24 15:00:48 ICDC-ISE03 CISE_Passed_Authentications 0001939187 4 2 PostureAssessmentStatus=NotApplicable, EndPointMatchedProfile=Unknown, DeviceRegistrationStatus=notRegistered, ISEPolicySetName=Wired MAB, IdentitySelectionMatchedRule=Default, StepData=5= Aruba.Aruba-Essid-Name, StepData=6= DEVICE.Device Type, StepData=8=Internal Endpoints, StepData=14= EndPoints.LogicalProfile, StepData=15= Network Access.Protocol, allowEasyWiredSession=false, DTLSSupport=Unknown, HostIdentityGroup=Endpoint Identity Groups:Unknown, Network Device Profile=Cisco, Location=Location#All Locations#Provo, Device Type=Device Type#All Device Types#Switches, Migrated NDGs=Migrated NDGs#All Migrated NDGs, Name=Endpoint Identity Groups:Unknown, #<165>Apr 24 15:00:48 ICDC-ISE03 CISE_Passed_Authentications 0001939187 4 3 Response={UserName=90:1B:0E:34:EA:92; User-Name=90-1B-0E-34-EA-92; State=ReauthSession:0A06400F000006AA6F83C371; Class=CACS:0A06400F000006AA6F83C371:ICDC-ISE03/341048949/1407358; Session-Timeout=300; Idle-Timeout=240; Termination-Action=RADIUS-Request; cisco-av-pair=url-redirect-acl=ACL-WEBAUTH-REDIRECT; cisco-av-pair=url-redirect=https://ICDC-ISE03.example.com:8443/portal/gateway?sessionId=0A06400F000006AA6F83C371&portal=c5a76cb0-6150-11e5-b062-0050568d954e&action=cwa&type=drw&token=6ded1943789b345f7afdd09e91549047; cisco-av-pair=profile-name=Unknown; LicenseTypes=1; }, -def test_cisco_ise(record_property, setup_wordlist, setup_splunk): +def test_cisco_ise(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 0 {% now 'utc', '%Y-%m-%d %H:%M:%S' %}.610 +00:00 0042009748 5200 NOTICE Passed-Authentication: Authentication succeeded, ConfigVersionId=128, Device IP Address=10.6.64.15, DestinationIPAddress=10.16.20.23, DestinationPort=1812, UserName=90-1B-0E-34-EA-92, Protocol=Radius, RequestLatency=8, NetworkDeviceName=ICPAV2-SW15, User-Name=901b0e34ea92, NAS-IP-Address=10.6.64.15, NAS-Port=50104, Service-Type=Call Check, Framed-IP-Address=10.6.226.138, Framed-MTU=1500, Called-Station-ID=B0-FA-EB-11-70-04, Calling-Station-ID=90-1B-0E-34-EA-92, NAS-Port-Type=Ethernet, NAS-Port-Id=GigabitEthernet0/4, EAP-Key-Name=, cisco-av-pair=service-type=Call Check, cisco-av-pair=audit-session-id=0A06400F000006AA6F83C371, cisco-av-pair=method=mab, OriginalUserName=901b0e34ea92, NetworkDeviceProfileName=Cisco, NetworkDeviceProfileId=b2652f13-5b3f-41ba-ada2-8385c8870809, IsThirdPartyDeviceFlow=false, RadiusFlowType=WiredMAB, SSID=B0-FA-EB-11-70-04,\n") message = mt.render(mark="<165>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 1 AcsSessionID=ICDC-ISE03/341048949/1407358, AuthenticationIdentityStore=Internal Endpoints, AuthenticationMethod=Lookup, SelectedAccessService=Default Network Access, SelectedAuthorizationProfiles=WIRED_GUEST_REDIRECT, UseCase=Host Lookup, IdentityGroup=Endpoint Identity Groups:Unknown, Step=11001, Step=11017, Step=11027, Step=15049, Step=15008, Step=15048, Step=15048, Step=15041, Step=15013, Step=24209, Step=24211, Step=22037, Step=24715, Step=15036, Step=15048, Step=15048, Step=15016, Step=11002, SelectedAuthenticationIdentityStores=Internal Endpoints, AuthenticationStatus=AuthenticationPassed, NetworkDeviceGroups=Location#All Locations#Provo, NetworkDeviceGroups=Device Type#All Device Types#Switches, NetworkDeviceGroups=Migrated NDGs#All Migrated NDGs, IdentityPolicyMatchedRule=Default, AuthorizationPolicyMatchedRule=Guest Web Auth, UserType=Host, CPMSessionID=0A06400F000006AA6F83C371, EndPointMACAddress=90-1B-0E-34-EA-92,\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 2 PostureAssessmentStatus=NotApplicable, EndPointMatchedProfile=Unknown, DeviceRegistrationStatus=notRegistered, ISEPolicySetName=Wired MAB, IdentitySelectionMatchedRule=Default, StepData=5= Aruba.Aruba-Essid-Name, StepData=6= DEVICE.Device Type, StepData=8=Internal Endpoints, StepData=14= EndPoints.LogicalProfile, StepData=15= Network Access.Protocol, allowEasyWiredSession=false, DTLSSupport=Unknown, HostIdentityGroup=Endpoint Identity Groups:Unknown, Network Device Profile=Cisco, Location=Location#All Locations#Provo, Device Type=Device Type#All Device Types#Switches, Migrated NDGs=Migrated NDGs#All Migrated NDGs, Name=Endpoint Identity Groups:Unknown,\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 3 Response={UserName=90:1B:0E:34:EA:92; User-Name=90-1B-0E-34-EA-92; State=ReauthSession:0A06400F000006AA6F83C371; Class=CACS:0A06400F000006AA6F83C371:ICDC-ISE03/341048949/1407358; Session-Timeout=300; Idle-Timeout=240; Termination-Action=RADIUS-Request; cisco-av-pair=url-redirect-acl=ACL-WEBAUTH-REDIRECT; cisco-av-pair=url-redirect=https://ICDC-ISE03.example.com:8443/portal/gateway?sessionId=0A06400F000006AA6F83C371&portal=c5a76cb0-6150-11e5-b062-0050568d954e&action=cwa&type=drw&token=6ded1943789b345f7afdd09e91549047; cisco-av-pair=profile-name=Unknown; LicenseTypes=1; },\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:ise:syslog\" | head 11") search = st.render(host=host) @@ -52,13 +52,13 @@ def test_cisco_ise(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #<181>Oct 24 21:00:02 ciscohost CISE_RADIUS_Accounting 0006028545 1 0 2019-10-24 21:00:02.879 +00:00 0088472694 3002 NOTICE Radius-Accounting: RADIUS Accounting watchdog update, ConfigVersionId=336, Device IP Address=10.0.0.3, RequestLatency=3, NetworkDeviceName=nc-aaa-aaa1, User-Name=U100000.ent.corp, NAS-IP-Address=10.0.0.3, NAS-Port=50047, Service-Type=Framed, Framed-IP-Address=10.0.0.80, Class=CACS:0AEF12345677832097B3F362:ncsilsepsuie212/356139633/9969901, Called-Station-ID=00-08-00-00-1B-AF, Calling-Station-ID=00-00-00-00-A0-7E, Acct-Status-Type=Interim-Update, Acct-Delay-Time=0, Acct-Input-Octets=653293631, Acct-Output-Octets=1497972244, Acct-Session-Id=00000B68, Acct-Authentic=RADIUS, Acct-Session-Time=241598, Acct-Input-Packets=2656224, Acct-Output-Packets=7614179, Acct-Input-Gigawords=0, Acct-Output-Gigawords=1, NAS-Port-Type=Ethernet, NAS-Port-Id=FastEthernet0/47, undefined-151=31D7AADD, cisco-av-pair=audit-session-id=0AEF10030000032097B3F362, cisco-av-pair=connect-progress=Auth Open, AcsSessionID=ncsilsepsuie205/359238109/4017186, SelectedAccessService=Default Network Access, Step=11004, Step=11017, Step=15049, Step=15008, Step=15048, Step=15048, Step=15048, Step=15004, Step=22094, Step=11005, NetworkDeviceGroups=Location#All Locations#NC, NetworkDeviceGroups=Device Type#All Device Types#Switch#2960-Switches, NetworkDeviceGroups=All Network Device Groups#All Network Device Groups, CPMSessionID=0AEF10030000032097B3F362, AllowedProtocolMatchedRule=EAP-TLS, All Network Device Groups=All Network Device Groups#All Network Device Groups, Location=Location#All Locations#NC, Device Type=Device Type#All Device Types#Switch#2960-Switches, Network Device Profile=Cisco, -def test_cisco_ise_single(record_property, setup_wordlist, setup_splunk): +def test_cisco_ise_single(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_RADIUS_Accounting 0006028545 1 0 2019-10-24 21:00:02.879 +00:00 0088472694 3002 NOTICE Radius-Accounting: RADIUS Accounting watchdog update, ConfigVersionId=336, Device IP Address=10.0.0.3, RequestLatency=3, NetworkDeviceName=nc-aaa-aaa1, User-Name=U100000.ent.corp, NAS-IP-Address=10.0.0.3, NAS-Port=50047, Service-Type=Framed, Framed-IP-Address=10.0.0.80, Class=CACS:0AEF12345677832097B3F362:ncsilsepsuie212/356139633/9969901, Called-Station-ID=00-08-00-00-1B-AF, Calling-Station-ID=00-00-00-00-A0-7E, Acct-Status-Type=Interim-Update, Acct-Delay-Time=0, Acct-Input-Octets=653293631, Acct-Output-Octets=1497972244, Acct-Session-Id=00000B68, Acct-Authentic=RADIUS, Acct-Session-Time=241598, Acct-Input-Packets=2656224, Acct-Output-Packets=7614179, Acct-Input-Gigawords=0, Acct-Output-Gigawords=1, NAS-Port-Type=Ethernet, NAS-Port-Id=FastEthernet0/47, undefined-151=31D7AADD, cisco-av-pair=audit-session-id=0AEF10030000032097B3F362, cisco-av-pair=connect-progress=Auth Open, AcsSessionID=ncsilsepsuie205/359238109/4017186, SelectedAccessService=Default Network Access, Step=11004, Step=11017, Step=15049, Step=15008, Step=15048, Step=15048, Step=15048, Step=15004, Step=22094, Step=11005, NetworkDeviceGroups=Location#All Locations#NC, NetworkDeviceGroups=Device Type#All Device Types#Switch#2960-Switches, NetworkDeviceGroups=All Network Device Groups#All Network Device Groups, CPMSessionID=0AEF10030000032097B3F362, AllowedProtocolMatchedRule=EAP-TLS, All Network Device Groups=All Network Device Groups#All Network Device Groups, Location=Location#All Locations#NC, Device Type=Device Type#All Device Types#Switch#2960-Switches, Network Device Profile=Cisco,\n") message = mt.render(mark="<165>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:ise:syslog\" | head 11") search = st.render(host=host) diff --git a/tests/test_cisco_meraki.py b/tests/test_cisco_meraki.py index 920714a..34adcbc 100644 --- a/tests/test_cisco_meraki.py +++ b/tests/test_cisco_meraki.py @@ -13,14 +13,14 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #<134>1 1563249630.774247467 devicename security_event ids_alerted signature=1:28423:1 priority=1 timestamp=1468531589.810079 dhost=98:5A:EB:E1:81:2F direction=ingress protocol=tcp/ip src=151.101.52.238:80 dst=192.168.128.2:53023 message: EXPLOIT-KIT Multiple exploit kit single digit exe detection -def test_cisco_meraki_security_event(record_property, setup_wordlist, setup_splunk): +def test_cisco_meraki_security_event(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }}1 {% now 'utc', '%s' %}.123456789 testcm-{{ host }} security_event ids_alerted signature=1:28423:1 priority=1 timestamp={% now 'utc', '%s' %}.123456 dhost=98:5A:EB:E1:81:2F direction=ingress protocol=tcp/ip src=151.101.52.238:80 dst=192.168.128.2:53023 message: EXPLOIT-KIT Multiple exploit kit single digit exe detection\n") message = mt.render(mark="<134>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"testcm-{{ host }}\" sourcetype=\"meraki\" | head 2") search = st.render(host=host) diff --git a/tests/test_cisco_nx_os.py b/tests/test_cisco_nx_os.py index 27b1155..f9c267d 100644 --- a/tests/test_cisco_nx_os.py +++ b/tests/test_cisco_nx_os.py @@ -12,14 +12,14 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) # Nov 1 14:07:58 excal-113 %MODULE-5-MOD_OK: Module 1 is online -def test_cisco_nx_os(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_cisco_nx_os(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} csconx-{{ host }} %MODULE-5-MOD_OK: Module 1 is online") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netops host=\"csconx-{{ host }}\" sourcetype=\"cisco:ios\" | head 2") search = st.render(host=host) @@ -34,7 +34,7 @@ def test_cisco_nx_os(record_property, setup_wordlist, get_host_key, setup_splunk # Nov 1 14:07:58 excal-113 %MODULE-5-MOD_OK: Module 1 is online # @pytest.mark.xfail -#def test_cisco_nx_os_singleport(record_property, setup_wordlist, get_host_key, setup_splunk): +#def test_cisco_nx_os_singleport(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): # host = get_host_key # # mt = env.from_string( diff --git a/tests/test_cisco_ucm.py b/tests/test_cisco_ucm.py new file mode 100644 index 0000000..1a155cd --- /dev/null +++ b/tests/test_cisco_ucm.py @@ -0,0 +1,110 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +# https://www.ciscolive.com/c/dam/r/ciscolive/us/docs/2017/pdf/TECUCC-3000.pdf + +# <189>8103: Oct 14 2015 05:50:19 AM.484 UTC : %UC_AUDITLOG-5-AdministrativeEvent: %[ UserID =administrator][ ClientAddress =10.110.1.2][ Severity =5][ EventType =GeneralConfigurationUpdate][ ResourceAccessed=CUCMAdmin][ EventStatus =Success][ CompulsoryEvent =No][ AuditCategory =AdministrativeEvent][ ComponentID =Cisco CUCM Administration][ AuditDetails =record in table device, with key field name = SEP0000311107A5 deleted][App ID=Cisco Tomcat][Cluster ID=][Node ID=CUCM11PUB]: Audit Event is generated by this application + + +def test_cisco_ucm_nohost_auditlog(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}8103: {% now 'utc', '%b %d %Y %I:%M:%S %p' %}.100 UTC : %UC_AUDITLOG-5-AdministrativeEvent: %[ UserID =administrator][ ClientAddress =192.168.1.1][ Severity =5][ EventType =GeneralConfigurationUpdate][ ResourceAccessed=CUCMAdmin][ EventStatus =Success][ CompulsoryEvent =No][ AuditCategory =AdministrativeEvent][ ComponentID =Cisco CUCM Administration][ AuditDetails =record in table device, with key field name = SEP0000311107A5 deleted][App ID=Cisco Tomcat][Cluster ID=][Node ID={{ host }}]: Audit Event is generated by this application\n") + message = mt.render(mark="<189>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=main host=\"{{ host }}\" sourcetype=\"cisco:ucm\" | head 11") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +# <189>17: Apr 21 19:01:35.638 UTC : %CCM_RTMT-RTMT-2-RTMT-ERROR-ALERT: RTMT Alert Name:SyslogSeverityMatchFound Detail: At Tue Apr 21 14:01:35 CDT 2009 on node ORD-PUB1, the following SyslogSeverityMatchFound events generated: SeverityMatch - Critical ntpRunningStatus.sh: NTP server 10.12.254.33 is inactive. Verify the network to this server, that it is a NTPv4 server and is operational. SeverityMatch - Alert sshd(pam_unix)[20038]: check pass; user unknown App ID:Cisco AMC Service Cluster ID: Node ID:ord-pub1 +def test_cisco_ucm_nohost_rtmt(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}17: {% now 'utc', '%b %d %H:%M:%S' %}.100 UTC : %UC_AUDITLOG-5-AdministrativeEvent: %[ UserID =administrator][ ClientAddress =10.1.1.1][ Severity =5][ EventType =GeneralConfigurationUpdate][ ResourceAccessed=CUCMAdmin][ EventStatus =Success][ CompulsoryEvent =No][ AuditCategory =AdministrativeEvent][ ComponentID =Cisco CUCM Administration][ AuditDetails =record in table device, with key field name = SEP0000311107A5 deleted][App ID=Cisco Tomcat][Cluster ID=][Node ID={{ host }}]: Audit Event is generated by this application\n") + message = mt.render(mark="<189>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=main host=\"{{ host }}\" sourcetype=\"cisco:ucm\" | head 11") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# <189>23813: cucm-pub: Jul 05 2016 04:03:01 PM.688 UTC : %UC_RTMT-2-RTMT_ALERT: %[AlertName=SyslogSeverityMatchFound][AlertDetail= At Tue Jul 05 12:03:01 EDT 2016 on node 1.2.3.4, the following SyslogSeverityMatchFound events generated: #012SeverityMatch : Critical#012MatchedEvent : Jul 5 12:02:29 cucm-sub1 local7 2 ccm: 6838: cucm-sub1: Jul 05 2016 16:02:29.795 UTC : %UC_CALLMANAGER-2-SignalCongestionEntry: %[Thread=SIP Handler Thread] [AverageDelay=22] [EntryLatency=20] [ExitLatency=8] [SampleSize=10] [TotalSignalCongestionEntry=6752][HighPriorityQueueDepth=0][NormalPriorityQueueDepth=1][LowPriorityQueueDepth=0][AppID=Cisco CallManager][ClusterID=UCMCluster1][NodeID=cucm-sub1]: Unified CM has detected signal congestion in an internal thread and has throttled activities for that thread#012AppID : Cisco Syslog Agent#012Cluster + + +def test_cisco_ucm_host_auditlog(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}23813: {% now 'utc', '%b %d %Y %I:%M:%S %p' %}.100 UTC : %UC_AUDITLOG-5-AdministrativeEvent: %[ UserID =administrator][ ClientAddress =192.1.1.1][ Severity =5][ EventType =GeneralConfigurationUpdate][ ResourceAccessed=CUCMAdmin][ EventStatus =Success][ CompulsoryEvent =No][ AuditCategory =AdministrativeEvent][ ComponentID =Cisco CUCM Administration][ AuditDetails =record in table device, with key field name = SEP0000311107A5 deleted][App ID=Cisco Tomcat][Cluster ID=][Node ID={{ host }}]: Audit Event is generated by this application\n") + message = mt.render(mark="<189>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=main host=\"{{ host }}\" sourcetype=\"cisco:ucm\" | head 11") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<121>17: Apr 21 19:01:35.638 UTC : %CCM_RTMT-RTMT-2-RTMT-ERROR-ALERT: RTMT Alert Name:SyslogSeverityMatchFound Detail: At Tue Apr 21 14:01:35 CDT 2009 on node ORD-PUB1, the following SyslogSeverityMatchFound events generated: SeverityMatch - Critical ntpRunningStatus.sh: NTP server 10.12.254.33 is inactive. Verify the network to this server, that it is a NTPv4 server and is operational. SeverityMatch - Alert sshd(pam_unix)[20038]: check pass; user unknown App ID:Cisco AMC Service Cluster ID: Node ID:ord-pub1 + + +def test_cisco_ucm_nohost_alert(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}17: {% now 'utc', '%b %d %H:%M:%S' %}.100 UTC : %CCM_RTMT-RTMT-2-RTMT-ERROR-ALERT: RTMT Alert Name:SyslogSeverityMatchFound Detail: At Tue Apr 21 14:01:35 CDT 2009 on node {{ host }}, the following SyslogSeverityMatchFound events generated: SeverityMatch - Critical ntpRunningStatus.sh: NTP server 10.12.254.33 is inactive. Verify the network to this server, that it is a NTPv4 server and is operational. SeverityMatch - Alert sshd(pam_unix)[20038]: check pass; user unknown App ID:Cisco AMC Service Cluster ID: Node ID:{{host}}\n") + message = mt.render(mark="<189>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=main host=\"{{ host }}\" sourcetype=\"cisco:ucm\" | head 11") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_citrix_netscaler.py b/tests/test_citrix_netscaler.py new file mode 100644 index 0000000..038eede --- /dev/null +++ b/tests/test_citrix_netscaler.py @@ -0,0 +1,37 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import datetime +import random +import pytz + +from jinja2 import Environment, environment + +from .sendmessage import * +from .splunkutils import * +import random + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<12> 01/10/2001:01:01:01 GMT netscaler ABC-D : SSLVPN HTTPREQUEST 1234567 : Context username@192.0.2.1 - SessionId: 12345- example.com User username : Group(s) groupname : Vserver a1b2:c3d4:e5f6:a7b8:c9d0:e1f2:a3b4:c5d6:123 - 01/01/2001:01:01:01 GMT GET file/path.gif - - +def test_citrix_netscaler(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "test-ctitrixns-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%m/%d/%Y:%H:%M:%S' %} GMT {{ host }} ABC-D : SSLVPN HTTPREQUEST 1234567 : Context username@192.0.2.1 - SessionId: 12345- example.com User username : Group(s) groupname : Vserver a1b2:c3d4:e5f6:a7b8:c9d0:e1f2:a3b4:c5d6:123 - 01/01/2001:01:01:01 GMT GET file/path.gif - -\n") + message = mt.render(mark="<12>", host=host, pid=pid) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search index=netfw host={{ host }} sourcetype=\"citrix:netscaler:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_common.py b/tests/test_common.py index 11912b5..ea82d77 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -14,13 +14,13 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) -def test_defaultroute(record_property, setup_wordlist, setup_splunk): +def test_defaultroute(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} test something else\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"sc4s:fallback\" PROGRAM=\"test\" | head 2") search = st.render(host=host) @@ -33,13 +33,13 @@ def test_defaultroute(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_internal(record_property, setup_wordlist, setup_splunk): +def test_internal(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} sc4sdefault[0]: test\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=main NOT host=\"{{ host }}\" sourcetype=\"sc4s:events\" | head 1") search = st.render(host=host) @@ -52,13 +52,13 @@ def test_internal(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_fallback(record_property, setup_wordlist, setup_splunk): +def test_fallback(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} test\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=main host=\"testvp-{{ host }}\" sourcetype=\"sc4s:fallback\" | head 2") search = st.render(host=host) @@ -72,7 +72,7 @@ def test_fallback(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 # -def test_metrics(record_property, setup_wordlist, setup_splunk): +def test_metrics(record_property, setup_wordlist, setup_splunk, setup_sc4s): st = env.from_string('mcatalog values(metric_name) WHERE metric_name="syslogng.d_*#0" AND ("index"="*" OR "index"="_*") BY index | fields index') search = st.render() @@ -83,7 +83,7 @@ def test_metrics(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_tz_guess(record_property, setup_wordlist, setup_splunk): +def test_tz_guess(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) @@ -91,7 +91,7 @@ def test_tz_guess(record_property, setup_wordlist, setup_splunk): "{{ mark }} {% now 'America/Los_Angeles', '%b %d %H:%M:%S' %} {{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-003164\" | head 2") search = st.render(host=host) @@ -105,7 +105,7 @@ def test_tz_guess(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_tz_fix_hst(record_property, setup_wordlist, setup_splunk): +def test_tz_fix_hst(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) @@ -114,7 +114,7 @@ def test_tz_fix_hst(record_property, setup_wordlist, setup_splunk): "{{ mark }} {{ dt }} tzfhst-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") message = mt.render(mark="<111>", host=host, dt=dt.strftime('%b %d %H:%M:%S')) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"tzfhst-{{ host }}\" sourcetype=\"cisco:asa\"") search = st.render(host=host) @@ -127,7 +127,7 @@ def test_tz_fix_hst(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_tz_fix_ny(record_property, setup_wordlist, setup_splunk): +def test_tz_fix_ny(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) @@ -137,7 +137,7 @@ def test_tz_fix_ny(record_property, setup_wordlist, setup_splunk): "{{ mark }} {{ dt }} tzfny-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") message = mt.render(mark="<111>", host=host, dt=dt.strftime('%b %d %H:%M:%S')) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"tzfny-{{ host }}\" sourcetype=\"cisco:asa\"") search = st.render(host=host) @@ -151,7 +151,7 @@ def test_tz_fix_ny(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_check_config_version(record_property, setup_wordlist, setup_splunk): +def test_check_config_version(record_property, setup_wordlist, setup_splunk, setup_sc4s): st = env.from_string("search index=main sourcetype=\"sc4s:events:startup:err\" \"Configuration file format is too old\" ") search = st.render() @@ -162,7 +162,7 @@ def test_check_config_version(record_property, setup_wordlist, setup_splunk): assert resultCount == 0 -def test_check_config_version_multiple(record_property, setup_wordlist, setup_splunk): +def test_check_config_version_multiple(record_property, setup_wordlist, setup_splunk, setup_sc4s): st = env.from_string("search index=main sourcetype=\"sc4s:events:startup:err\" \"you have multiple @version directives\" ") search = st.render() @@ -173,7 +173,7 @@ def test_check_config_version_multiple(record_property, setup_wordlist, setup_sp assert resultCount == 0 -def test_check_sc4s_version(record_property, setup_wordlist, setup_splunk): +def test_check_sc4s_version(record_property, setup_wordlist, setup_splunk, setup_sc4s): st = env.from_string("search index=main sourcetype=\"sc4s:events:startup:out\" \"sc4s version=\" NOT \"UNKNOWN\"") search = st.render() diff --git a/tests/test_cyberark.py b/tests/test_cyberark.py index c8ac7ba..5380d92 100644 --- a/tests/test_cyberark.py +++ b/tests/test_cyberark.py @@ -13,14 +13,14 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #<5>1 2020-01-24T22:53:03Z REDACTEDHOSTNAME CEF:0|Cyber-Ark|Vault|10.9.0000|22|CPM Verify Password|5|act="CPM Verify Password" suser=PasswordManager fname=Root\Operating System-OBO-ISSO-Windows-Domain-Account-redacted dvc= shost=10.0.0.10 dhost= duser=redacted externalId= app= reason= cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2="re-dact-ted" cs3Label="Device Type" cs3="Operating System" cs4Label="Database" cs4= cs5Label="Other info" cs5= cn1Label="Request Id" cn1= cn2Label="Ticket Id" cn2="VerificationPeriod" msg="VerificationPeriod" -def test_cyberark_epv_5424(record_property, setup_wordlist, setup_splunk): +def test_cyberark_epv_5424(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}Z {{ host }} CEF:0|Cyber-Ark|Vault|9.20.0000|7|Logon|5|act=\"Logon\" suser=##USER_NAME## fname= dvc= shost=##SOURCE_IP## dhost= duser= externalId= app= reason= cs1Label=\"Affected User Name\" cs1= cs2Label=\"Safe Name\" cs2= cs3Label=\"Device Type\" cs3=11111 cs4Label=\"Database\" cs4=222222 cs5Label=\"Other info\" cs5= cn1Label=\"Request Id\" cn1= cn2Label=\"Ticket Id\" cn2= msg=\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cyberark:epv:cef\"| head 2") search = st.render(host=host) @@ -34,14 +34,14 @@ def test_cyberark_epv_5424(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #<190>Jul 27 23:31:58 VAULT CEF:0|Cyber-Ark|Vault|9.20.0000|7|Logon|5|act="Logon" suser=##USER_NAME## fname= dvc= shost=##SOURCE_IP## dhost= duser= externalId= app= reason= cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2= cs3Label="Device Type" cs3=11111 cs4Label="Database" cs4=222222 cs5Label="Other info" cs5= cn1Label="Request Id" cn1= cn2Label="Ticket Id" cn2= msg= -def test_cyberark_epv(record_property, setup_wordlist, setup_splunk): +def test_cyberark_epv(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CEF:0|Cyber-Ark|Vault|9.20.0000|7|Logon|5|act=\"Logon\" suser=##USER_NAME## fname= dvc= shost=##SOURCE_IP## dhost= duser= externalId= app= reason= cs1Label=\"Affected User Name\" cs1= cs2Label=\"Safe Name\" cs2= cs3Label=\"Device Type\" cs3=11111 cs4Label=\"Database\" cs4=222222 cs5Label=\"Other info\" cs5= cn1Label=\"Request Id\" cn1= cn2Label=\"Ticket Id\" cn2= msg=\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cyberark:epv:cef\"| head 2") search = st.render(host=host) @@ -55,14 +55,14 @@ def test_cyberark_epv(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #<190>Jul 12 23:44:25 10.0.0.1 CEF:0|CyberArk|PTA|2.6.1|20|Privileged account anomaly|8|cs1Label=incidentId cs1=55a32ed8e4b0e4a90114e12c start=1436755482000 deviceCustomDate1Label=detectionDate deviceCustomDate1=1436759065017 msg=Incident updated. Now contains 7 anomalies cs2Label=link cs2=https://10.0.0.1/incidents/55a32ed8e4b0e4a90114e12c -def test_cyberark_pta(record_property, setup_wordlist, setup_splunk): +def test_cyberark_pta(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CEF:0|CyberArk|PTA|2.6.1|20|Privileged account anomaly|8|cs1Label=incidentId cs1=55a32ed8e4b0e4a90114e12c start=1436755482000 deviceCustomDate1Label=detectionDate deviceCustomDate1=1436759065017 msg=Incident updated. Now contains 7 anomalies cs2Label=link cs2=https://10.0.0.1/incidents/55a32ed8e4b0e4a90114e12c\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cyberark:pta:cef\"| head 2") search = st.render(host=host) diff --git a/tests/test_forcepoint_web.py b/tests/test_forcepoint_web.py index 7ecf4cb..4f4a410 100644 --- a/tests/test_forcepoint_web.py +++ b/tests/test_forcepoint_web.py @@ -13,14 +13,14 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #<134>Oct 16 12:13:06 sourcehost2 vendor=Websense 9f product=Security product_version=7.7.0 action=permitted severity=7 category=755 user=LDAP://user7 OU=Users,OU=Beijing,DC=com/TEST\, TEST_NAME src_host=10.0.0.4 src_port=61435 dst_host=HOST-013 dst_ip=10.0.0.19 dst_port=25404 bytes_out=4074 bytes_in=12328 http_response=200 http_method=POST http_content_type=image/gif;charset=UTF-8 http_user_agent=Mozilla/3.0 (Windows; U; Windows NT 6.1; es-def; rv:1.7.0.11) Gecko/2009060215 Firefox/8.0.11 (.NET CLR 8.5.30729) http_proxy_status_code=200 reason=- disposition=2573 policy=role-8**Default role=4 duration=63 url=http://test_web.com/contents/content1.jpg -def test_forcepoint_webprotect_kv(record_property, setup_wordlist, setup_splunk): +def test_forcepoint_webprotect_kv(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} vendor=Websense 9f product=Security product_version=7.7.0 action=permitted severity=7 category=755 user=LDAP://user7 OU=Users,OU=Beijing,DC=com/TEST\, TEST_NAME src_host=10.0.0.4 src_port=61435 dst_host=HOST-013 dst_ip=10.0.0.19 dst_port=25404 bytes_out=4074 bytes_in=12328 http_response=200 http_method=POST http_content_type=image/gif;charset=UTF-8 http_user_agent=Mozilla/3.0 (Windows; U; Windows NT 6.1; es-def; rv:1.7.0.11) Gecko/2009060215 Firefox/8.0.11 (.NET CLR 8.5.30729) http_proxy_status_code=200 reason=- disposition=2573 policy=role-8**Default role=4 duration=63 url=http://test_web.com/contents/content1.jpg unknownfield=-\n") message = mt.render(mark="<134>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"websense:cg:kv\" | head 2") search = st.render(host=host) diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py index f68f13a..892f87d 100644 --- a/tests/test_fortinet_ngfw.py +++ b/tests/test_fortinet_ngfw.py @@ -13,14 +13,14 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 -def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk): +def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT60D4614044725 logid=0100040704 type=event subtype=system level=notice vd=root logdesc=\"System performance statistics\" action=\"perf-stats\" cpu=2 mem=35 totalsession=61 disk=2 bandwidth=158/138 setuprate=2 disklograte=0 fazlograte=0 msg=\"Performance statistics: average CPU: 2, memory: 35, concurrent sessions: 61, setup-rate: 2\"\n") message = mt.render(mark="<13>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"fgt_event\" | head 2") search = st.render(host=host) @@ -34,13 +34,13 @@ def test_fortinet_fgt_event(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:19:43 devname=Nosey devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf="port3" dstip=ff02::1:ff77:20d4 dstintf="port3" sessionid=408903 proto=58 action=accept policyid=2 dstcountry="Reserved" srccountry="Reserved" trandisp=snat transip=:: transport=0 service="icmp6/131/0" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app="IPv6.ICMP" appcat="Network.Service" apprisk=elevated applist="sniffer-profile" appact=detected utmaction=allow countapp=1 -def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk): +def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FG800C3912801080 logid=0004000017 type=traffic subtype=sniffer level=notice vd=root srcip=fe80::20c:29ff:fe77:20d4 srcintf=\"port3\" dstip=ff02::1:ff77:20d4 dstintf=\"port3\" sessionid=408903 proto=58 action=accept policyid=2 dstcountry=\"Reserved\" srccountry=\"Reserved\" trandisp=snat transip=:: transport=0 service=\"icmp6/131/0\" duration=36 sentbyte=0 rcvdbyte=40 sentpkt=0 rcvdpkt=0 appid=16321 app=\"IPv6.ICMP\" appcat=\"Network.Service\" apprisk=elevated applist=\"sniffer-profile\" appact=detected utmaction=allow countapp=1\n") message = mt.render(mark="<13>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"fgt_traffic\" | head 2") search = st.render(host=host) @@ -54,13 +54,13 @@ def test_fortinet_fgt_traffic(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #<111> Aug 17 00:00:00 fortigate date=2015-08-11 time=19:21:40 logver=52 devname=US-Corp_Main1 devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user="" srcip=172.30.16.119 srcport=53235 srcintf="Internal" dstip=114.112.67.75 dstport=80 dstintf="External-SDC" proto=6 service=HTTP hostname="popo.wan.ijinshan.com" profile="scan" action=passthrough reqtype=direct url="/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl" sentbyte=525 rcvdbyte=325 direction=outgoing msg="URL belongs to an allowed category in policy" method=domain cat=52 catdesc="Information Technology" -def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk): +def test_fortinet_fgt_utm(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }}date={% now 'utc', '%Y-%m-%d' %} time={% now 'utc', '%H:%M:%S' %} devname={{ host }} devid=FGT37D4613800138 logid=0317013312 type=utm subtype=webfilter eventtype=ftgd_allow level=notice vd=root sessionid=1490845588 user=\"\" srcip=172.30.16.119 srcport=53235 srcintf=\"Internal\" dstip=114.112.67.75 dstport=80 dstintf=\"External-SDC\" proto=6 service=HTTP hostname=\"popo.wan.ijinshan.com\" profile=\"scan\" action=passthrough reqtype=direct url=\"/popo/launch?c=cHA9d29vZHMxOTgyQGhvdG1haWwuY29tJnV1aWQ9NDBiNDkyZDRmNzdhNjFmOTNlMjQwMjhiYjE3ZGRlYTYmY29tcGl\" sentbyte=525 rcvdbyte=325 direction=outgoing msg=\"URL belongs to an allowed category in policy\" method=domain cat=52 catdesc=\"Information Technology\"\n") message = mt.render(mark="<13>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"fgt_utm\" | head 2") search = st.render(host=host) diff --git a/tests/test_imperva.py b/tests/test_imperva.py index 9a0005c..a3ec535 100644 --- a/tests/test_imperva.py +++ b/tests/test_imperva.py @@ -12,14 +12,14 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) -def test_imperva_incapsula(record_property, setup_wordlist, setup_splunk): +def test_imperva_incapsula(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Incapsula|SIEMintegration|1|1|Illegal Resource Access|3| fileid=3412341160002518171 sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support src=12.12.12.12 caIP=13.13.13.13 ccode=IL tag=www.elvis.com cn1=200 in=54 xff=44.44.44.44 cs1=NOT_SUPPORTED cs1Label=Cap Support cs4=c2e72124-0e8a-4dd8-b13b-3da246af3ab2 cs4Label=VID cs5=de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4 cs5Label=clappsigdproc=Browser cs6=Firefox cs6Label=clapp ccode=IL cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 start=1453290121336 request=site123.abcd.info/ requestmethod=GET qstr=p\=%2fetc%2fpasswd app=HTTP act=REQ_CHALLENGE_CAPTCHA deviceExternalID=33411452762204224 cpt=443 filetype=30037,1001, filepermission=2,1, cs9=Block Malicious User,High Risk Resources, cs9Label=Rule name' + "\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netwaf host=\"{{ host }}\" sourcetype=\"cef\" source=\"Imperva:Incapsula\" | head 2") search = st.render(host=host) diff --git a/tests/test_infoblox.py b/tests/test_infoblox.py index f7da9c7..f9f29ef 100644 --- a/tests/test_infoblox.py +++ b/tests/test_infoblox.py @@ -16,14 +16,14 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #<142>Oct 25 13:08:00 161.231.218.156 named[6597]: FORMERR resolving 'www.google.com/AAAA/IN': 209.2.230.6#53 -def test_infoblox_dns(record_property, setup_wordlist, setup_splunk): +def test_infoblox_dns(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "vib-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} named[{{ pid }}]: FORMERR resolving 'www.google.com/AAAA/IN': 209.2.230.6#53\n") message = mt.render(mark="<111>", host=host, pid=pid) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netdns host={{ host }} sourcetype=\"infoblox:dns\" | head 2") search = st.render(host=host, pid=pid) @@ -37,14 +37,14 @@ def test_infoblox_dns(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #<150>Oct 24 21:09:00 162.101.157.246 dhcpd[28922]: DHCPREQUEST for 10.130.151.62 from 80:ce:62:9c:0e:70 (DTCCE0826E00C97) via eth2 TransID 802c562c uid 01:80:ce:62:9c:0e:70 (RENEW) -def test_infoblox_dhcp(record_property, setup_wordlist, setup_splunk): +def test_infoblox_dhcp(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "vib-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dhcpd[{{ pri }}]: DHCPREQUEST for 10.00.00.62 from 80:00:00:00:0e:70 (EXAMPLE) via eth2 TransID 802c562c uid 01:80:00:00:00:00:70 (RENEW)\n") message = mt.render(mark="<150>", host=host, pid=pid) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netipam host={{ host }} sourcetype=\"infoblox:dhcp\" | head 2") search = st.render(host=host, pid=pid) diff --git a/tests/test_juniper_junos_rfc3164.py b/tests/test_juniper_junos_rfc3164.py index b127814..9bac164 100644 --- a/tests/test_juniper_junos_rfc3164.py +++ b/tests/test_juniper_junos_rfc3164.py @@ -11,14 +11,14 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) # <23> Mar 18 17:56:52 RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION="URL Permitted" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY="Enhanced_Information_Technology" REASON="BY_PRE_DEFINED" PROFILE="UTM-Wireless-Profile" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A -def test_juniper_utm_standard(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_utm_standard(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION=\"URL Permitted\" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY=\"Enhanced_Information_Technology\" REASON=\"BY_PRE_DEFINED\" PROFILE=\"UTM-Wireless-Profile\" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A") message = mt.render(mark="<23>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") search = st.render(host=host) @@ -32,14 +32,14 @@ def test_juniper_utm_standard(record_property, setup_wordlist, get_host_key, set assert resultCount == 1 # <23> Nov 18 09:56:58 INTERNET-ROUTER RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192 -def test_juniper_firewall_standard(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_firewall_standard(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192") message = mt.render(mark="<23>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall\" | head 2") search = st.render(host=host) @@ -52,14 +52,14 @@ def test_juniper_firewall_standard(record_property, setup_wordlist, get_host_key assert resultCount == 1 -def test_juniper_idp_standard(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_idp_standard(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message -") message = mt.render(mark="<23>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:junos:idp\" | head 2") search = st.render(host=host) diff --git a/tests/test_juniper_junos_rfc5124.py b/tests/test_juniper_junos_rfc5124.py index 2e0b4e1..188e99f 100644 --- a/tests/test_juniper_junos_rfc5124.py +++ b/tests/test_juniper_junos_rfc5124.py @@ -13,14 +13,14 @@ # <165>1 2007-02-15T09:17:15.719Z router1 mgd 3046 UI_DBASE_LOGOUT_EVENT [junos@2636.1.1.1.2.18 username="user"] User 'user' exiting configuration mode # @pytest.mark.xfail -def test_juniper_junos_structured(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_junos_structured(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} mgd 3046 UI_DBASE_LOGOUT_EVENT [junos@2636.1.1.1.2.18 username=\"user\"] User 'user' exiting configuration mode\n") message = mt.render(mark="<165>1", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netops host=\"{{ host }}\" sourcetype=\"juniper:structured\" | head 2") search = st.render(host=host) @@ -35,14 +35,14 @@ def test_juniper_junos_structured(record_property, setup_wordlist, get_host_key, # <165>1 2007-02-15T09:17:15.719Z idp1 RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time="1507845354" message-type="SIG" source-address="183.78.180.27" source-port="45610" destination-address="118.127.xx.xx" destination-port="80" protocol-name="TCP" service-name="SERVICE_IDP" application-name="HTTP" rule-name="9" rulebase-name="IPS" policy-name="Recommended" export-id="15229" repeat-count="0" action="DROP" threat-severity="HIGH" attack-name="TROJAN:ZMEU-BOT-SCAN" nat-source-address="0.0.0.0" nat-source-port="0" nat-destination-address="172.xx.xx.xx" nat-destination-port="0" elapsed-time="0" inbound-bytes="0" outbound-bytes="0" inbound-packets="0" outbound-packets="0" source-zone-name="sec-zone-name-internet" source-interface-name="reth0.XXX" destination-zone-name="dst-sec-zone1-outside" destination-interface-name="reth1.xxx" packet-log-id="0" alert="no" username="N/A" roles="N/A" message="-"] # @pytest.mark.xfail -def test_juniper_junos_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_junos_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time=\"1507845354\" message-type=\"SIG\" source-address=\"183.78.180.27\" source-port=\"45610\" destination-address=\"118.127.xx.xx\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"9\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"15229\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"HIGH\" attack-name=\"TROJAN:ZMEU-BOT-SCAN\" nat-source-address=\"0.0.0.0\" nat-source-port=\"0\" nat-destination-address=\"172.xx.xx.xx\" nat-destination-port=\"0\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"sec-zone-name-internet\" source-interface-name=\"reth0.XXX\" destination-zone-name=\"dst-sec-zone1-outside\" destination-interface-name=\"reth1.xxx\" packet-log-id=\"0\" alert=\"no\" username=\"N/A\" roles=\"N/A\" message=\"-\"]") message = mt.render(mark="<165>1", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:junos:idp:structured\" | head 2") search = st.render(host=host) @@ -57,14 +57,14 @@ def test_juniper_junos_idp_structured(record_property, setup_wordlist, get_host_ # <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos # @pytest.mark.xfail -def test_juniper_junos_fw_structured(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_junos_fw_structured(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} RT_FLOW - RT_FLOW_SESSION_CREATE_LS [junos@2636.1.1.1.2.26 logical-system-name=\"test-lsys\" source-address=\"10.10.10.100\" source-port=\"4206\" destination-address=\"10.20.20.15\" destination-port=\"445\" service-name=\"junos-smb\" nat-source-address=\"10.10.10.100\" nat-source-port=\"4206\" nat-destination-address=\"10.20.20.15\" nat-destination-port=\"445\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"123\" source-zone-name=\"TEST1\" destination-zone-name=\"TEST2\" session-id-32=\"14285714\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth1.100\"]") message = mt.render(mark="<23>1", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:junos:firewall:structured\" | head 2") search = st.render(host=host) diff --git a/tests/test_juniper_legacy.py b/tests/test_juniper_legacy.py index cd9cbcc..f9696d9 100644 --- a/tests/test_juniper_legacy.py +++ b/tests/test_juniper_legacy.py @@ -13,14 +13,14 @@ # <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos -def test_juniper_nsm_standard(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_nsm_standard(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} jnpnsm-{{ host }} 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [FW NAME], [FW IP], traffic, traffic log, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [FW NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos") message = mt.render(mark="<134>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"jnpnsm-{{ host }}\" sourcetype=\"juniper:nsm\" | head 2") search = st.render(host=host) @@ -35,14 +35,14 @@ def test_juniper_nsm_standard(record_property, setup_wordlist, get_host_key, set # THE LOG SAMPLE BELOW IS IMPLIED FROM THE JUNIPER DOCS; need to obtain a real sample. # <134> Aug 02 14:45:04 10.0.0.1 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [IDP NAME], [IDP IP], predefined, rule, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [IDP NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos -def test_juniper_nsm_idp_standard(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_nsm_idp_standard(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} jnpnsmidp-{{ host }} 65.197.254.193 20090320, 17331, 2009/03/20 14:47:45, 2009/03/20 14:47:50, global, 53, [IDP NAME], [IDP IP], predefined, rule, trust, (NULL), 10.1.1.20, 1725, 82.2.19.2, 2383, untrust, (NULL), 84.5.78.4, 80, 84.53.178.64, 80, tcp, global, 53, [IDP NAME], fw/vpn, 4, accepted, info, no, Creation, (NULL), (NULL), (NULL), 0, 0, 0, 0, 0, 0, 0, 1, no, 0, Not Set, sos") message = mt.render(mark="<134>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netids host=\"jnpnsmidp-{{ host }}\" sourcetype=\"juniper:nsm:idp\" | head 2") search = st.render(host=host) @@ -57,14 +57,14 @@ def test_juniper_nsm_idp_standard(record_property, setup_wordlist, get_host_key, # <23> Apr 24 12:30:05 cs-loki3 RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message - # <23> Mar 18 17:56:52 [FW IP] [FW Model]: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time="2009-03-18 16:07:06" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1 -def test_juniper_netscreen_fw(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_netscreen_fw(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} jnpns-{{ host }} ns204: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time=\"2009-03-18 16:07:06\" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1\n") message = mt.render(mark="<23>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"jnpns-{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") search = st.render(host=host) @@ -82,14 +82,14 @@ def test_juniper_netscreen_fw(record_property, setup_wordlist, get_host_key, set # # # @pytest.mark.xfail -def test_juniper_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_idp_structured(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}.700Z {{ host }} Jnpr Syslog 23414 [syslog@juniper.net dayId=\"20100623\" recordId=\"0\" timeRecv=\"2010/06/23 18:05:55\" timeGen=\"2010/06/23 18:05:51\" domain=\"\" devDomVer2=\"0\" device_ip=\"10.209.83.9\" cat=\"Config\" attack=\"\" srcZn=\"NULL\" srcIntf=\"\" srcAddr=\"0.0.0.0\" srcPort=\"0\" natSrcAddr=\"NULL\" natSrcPort=\"0\" dstZn=\"NULL\" dstIntf=\"NULL\" dstAddr=\"0.0.0.0\" dstPort=\"0\" natDstAddr=\"NULL\" natDstPort=\"0\" protocol=\"IP\" ruleDomain=\"\" ruleVer=\"0\" policy=\"\" rulebase=\"NONE\" ruleNo=\"0\" action=\"NONE\" severity=\"INFO\" alert=\"no\" elaspedTime=\"0\" inbytes=\"0\" outbytes=\"0\" totBytes=\"0\" inPak=\"0\" outPak=\"0\" totPak=\"0\" repCount=\"0\" packetData=\"no\" varEnum=\"0\" misc=\"Interaface eth2,eth3 is in Normal State\" user=\"NULL\" app=\"NULL\" uri=\"NULL\"]") message = mt.render(mark="<165>1", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netids host=\"{{ host }}\" sourcetype=\"juniper:idp\" | head 2") search = st.render(host=host) @@ -104,14 +104,14 @@ def test_juniper_idp_structured(record_property, setup_wordlist, get_host_key, s # <23> Apr 24 12:30:05 cs-loki3 RT_IDP: IDP_ATTACK_LOG_EVENT: IDP: at 1303673404, ANOMALY Attack log <64.1.2.1/48397->198.87.233.110/80> for TCP protocol and service HTTP application NONE by rule 3 of rulebase IPS in policy Recommended. attack: repeat=0, action=DROP, threat-severity=HIGH, name=HTTP:INVALID:MSNG-HTTP-VER, NAT <46.0.3.254:55870->0.0.0.0:0>, time-elapsed=0, inbytes=0, outbytes=0, inpackets=0, outpackets=0, intf:trust:fe-0/0/2.0->untrust:fe-0/0/3.0, packet-log-id: 0 and misc-message - # <23> Mar 18 17:56:52 [FW IP] [FW Model]: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time="2009-03-18 16:07:06" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1 -def test_juniper_netscreen_fw_singleport(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_netscreen_fw_singleport(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} ns204: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time=\"2009-03-18 16:07:06\" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1 singleport=5000\n") message = mt.render(mark="<23>", host=host) - sendsingle(message, port=5000) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][5000]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") search = st.render(host=host) diff --git a/tests/test_juniper_sslvpn.py b/tests/test_juniper_sslvpn.py index e301be1..867a082 100644 --- a/tests/test_juniper_sslvpn.py +++ b/tests/test_juniper_sslvpn.py @@ -13,14 +13,14 @@ # <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[] - Session timed out for xxx@xxx.xxx.xxx/Users (session:00000000) due to inactivity (last access at 13:59:31 2013/02/27). Idle session identified during routine system scan. # <23> Feb 27 15:00:00 vpn-001 Juniper: 2013-02-27 15:00:00 - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied. -def test_juniper_sslvpn_standard(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_juniper_sslvpn_standard(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} Juniper: {% now 'utc', '%Y-%m-%d %H:%M:%S' %} - ive - [000.000.000.000] SAMPLE::xxx@xxx.xxx(Users)[User_Role] - Remote address for user xxx@xxx.xxx/Users changed from 000.000.000.000 to 000.000.000.000. Access denied.") message = mt.render(mark="<23>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"juniper:sslvpn\" | head 2") search = st.render(host=host) diff --git a/tests/test_linux_syslog.py b/tests/test_linux_syslog.py index 9297929..e467eea 100644 --- a/tests/test_linux_syslog.py +++ b/tests/test_linux_syslog.py @@ -16,14 +16,14 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #<78>Oct 25 09:10:00 /usr/sbin/cron[54928]: (root) CMD (/usr/libexec/atrun) -def test_linux__nohost_program_as_path(record_property, setup_wordlist, setup_splunk): +def test_linux__nohost_program_as_path(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} /usr/sbin/cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") message = mt.render(mark="<111>", host=host, pid=pid) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) @@ -36,14 +36,14 @@ def test_linux__nohost_program_as_path(record_property, setup_wordlist, setup_sp assert resultCount == 1 -def test_linux__host_program_as_path(record_property, setup_wordlist, setup_splunk): +def test_linux__host_program_as_path(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} /usr/sbin/cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") message = mt.render(mark="<111>", host=host, pid=pid) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) @@ -56,14 +56,14 @@ def test_linux__host_program_as_path(record_property, setup_wordlist, setup_splu assert resultCount == 1 -def test_linux__nohost_program_conforms(record_property, setup_wordlist, setup_splunk): +def test_linux__nohost_program_conforms(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") message = mt.render(mark="<111>", host=host, pid=pid) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) @@ -76,14 +76,14 @@ def test_linux__nohost_program_conforms(record_property, setup_wordlist, setup_s assert resultCount == 1 -def test_linux__host_program_conforms(record_property, setup_wordlist, setup_splunk): +def test_linux__host_program_conforms(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") message = mt.render(mark="<111>", host=host, pid=pid) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) diff --git a/tests/test_microfocus_arcsight.py b/tests/test_microfocus_arcsight.py index 507db99..0825d05 100644 --- a/tests/test_microfocus_arcsight.py +++ b/tests/test_microfocus_arcsight.py @@ -16,14 +16,14 @@ # Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=30 msg=Connected to Host mrt=1539321123071 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321124967 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321123071 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= # Mar 19 15:19:15 root CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:030|Agent [PAN1_WUC_UDP8000] type [windowsfg] started|Low| eventId=26 mrt=1539321122832 categorySignificance=/Normal categoryBehavior=/Execute/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application/Service art=1539321124967 cat=/Agent/Started deviceSeverity=Warning rt=1539321122832 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= # Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321047341 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= -def test_microfocus_arcsight_ts_rt(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_ts_rt(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=' + "{% now 'utc', '%s' %}" + ' dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\\=\\=' + "\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cef\" source=ArcSight:ArcSight | head 2") search = st.render(host=host) @@ -36,14 +36,14 @@ def test_microfocus_arcsight_ts_rt(record_property, setup_wordlist, setup_splunk assert resultCount == 1 -def test_microfocus_arcsight_ts_end(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_ts_end(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning end=' + "{% now 'utc', '%s' %}" + ' dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\\=\\=' + "\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cef\" source=ArcSight:ArcSight| head 2") search = st.render(host=host) @@ -56,14 +56,14 @@ def test_microfocus_arcsight_ts_end(record_property, setup_wordlist, setup_splun assert resultCount == 1 -def test_microfocus_arcsight_ts_syslog(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_ts_syslog(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\\=\\=' + "\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cef\" source=ArcSight:ArcSight | head 2") search = st.render(host=host) @@ -76,14 +76,14 @@ def test_microfocus_arcsight_ts_syslog(record_property, setup_wordlist, setup_sp assert resultCount == 1 -def test_microfocus_arcsight_windows(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_windows(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Microsoft|Microsoft Windows||Microsoft-Windows-Security-Auditing:4634|An account was logged off.|Low| eventId=1 externalId=4634 msg=Network: A user or computer logged on to this computer from the network. rawEvent=EventlogType\=Security&&EventIndex\=1031&&WindowsVersion\=Windows Server 2012 R2&&WindowsKeyMapFamily\=Windows 2012 R2&&WindowsParserFamily\=Windows 2012 R2|2012|8&&DetectTime\=2018-10-12 7:25:11&&EventSource\=Microsoft-Windows-Security-Auditing&&EventID\=4634&&EventType\=Audit_success&&EventCategory\=12545&&User\=null&&ComputerName\=WIN-PAN1&&Description\=An account was logged off.&&Message\=This event is generated when a logon session is destroyed. It may be positively correlated with a logon event using the Logon ID value. Logon IDs are only unique between reboots on the same computer.&&Subject:Security ID\=S-1-5-21-750061412-3179291162-3140434184-500&&Subject:Account Name\=Administrator&&Subject:Account Domain\=WIN-PAN1&&Subject:Logon ID\=0x373c2&&Logon Type\=3 categorySignificance=/Informational categoryBehavior=/Access/Stop categoryDeviceGroup=/Operating System catdt=Operating System categoryOutcome=/Success categoryObject=/Host/Operating System art=1539321047369 cat=Security deviceSeverity=Audit_success rt=' + "{% now 'utc', '%s' %}" + ' dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dntdom=WIN-PAN1 duser=Administrator duid=0x373c2 cs2=Logon/Logoff:Logoff cn1=3 cs1Label=Accesses cs2Label=EventlogCategory cs4Label=Reason or Error Code cs5Label=Authentication Package Name cn1Label=LogonType cn2Label=CrashOnAuditFail cn3Label=Count ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=WIN-PAN1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 deviceNtDomain=WIN-PAN1 dtz=Asia/Riyadh _cefVer=0.1 ad.WindowsVersion=Windows Server 2012 R2 ad.WindowsParserFamily=Windows 2012 R2|2012|8 ad.WindowsKeyMapFamily=Windows 2012 R2 ad.EventIndex=1031 aid=3o0OiZmYBABCACGN9CiyuGQ\=\=' + "\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=oswinsec host=\"{{ host }}\" sourcetype=\"cef\" source=\"CEFEventLog:Microsoft Windows\" | head 2") search = st.render(host=host) @@ -96,14 +96,14 @@ def test_microfocus_arcsight_windows(record_property, setup_wordlist, setup_splu assert resultCount == 1 -def test_microfocus_arcsight_windows_system(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_windows_system(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Microsoft|System or Application Event||Software Protection Platform Service:902|Software Protection Platform Service|Unknown| eventId=39 externalId=902 rawEvent=EventlogType\=Application&&EventIndex\=1604&&WindowsVersion\=Windows Server 2012 R2&&WindowsKeyMapFamily\=Windows 2012 R2&&WindowsParserFamily\=Windows 2012 R2|2012|8&&DetectTime\=2018-10-12 8:12:31&&EventSource\=Software Protection Platform Service&&EventID\=902&&EventType\=null&&EventCategory\=0&&User\=null&&ComputerName\=WIN-PAN1&&Key[0]\=6.3.9600.16384 art=1539321151610 cat=Application rt=' + "{% now 'utc', '%s' %}" + ' cs2=0 cs3=Software Protection Platform Service cs2Label=EventlogCategory cs3Label=EventSource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=WIN-PAN1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dtz=Asia/Riyadh _cefVer=0.1 ad.WindowsVersion=Windows Server 2012 R2 ad.WindowsParserFamily=Windows 2012 R2|2012|8 ad.WindowsKeyMapFamily=Windows 2012 R2 ad.Key[0]=6.3.9600.16384 ad.EventIndex=1604 aid=3o0OiZmYBABCACGN9CiyuGQ\=\=' + "\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=oswin host=\"{{ host }}\" sourcetype=\"cef\" source=\"CEFEventLog:System or Application Event\" | head 2") search = st.render(host=host) diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py index 6211590..400442a 100644 --- a/tests/test_palo_alto.py +++ b/tests/test_palo_alto.py @@ -14,14 +14,14 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #<190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0 -def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): +def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") search = st.render(host=host) @@ -36,14 +36,14 @@ def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): # <190>Oct 30 09:46:17 1,2012/10/30 09:46:17,01606001116,THREAT,url,1,2012/04/10 04:39:55,192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:57,22860,1,59303,80,0,0,0x208000,tcp,alert,"litetopdetect.cn/index.php",(9999),not-resolved,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,text/html -def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk): +def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},01606001116,THREAT,url,1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:57,22860,1,59303,80,0,0,0x208000,tcp,alert,\"litetopdetect.cn/index.php\",(9999),not-resolved,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,text/html\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"pan:threat\" | head 2") search = st.render(host=host) @@ -56,14 +56,14 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk): +def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }}1 {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") search = st.render(host=host) @@ -78,14 +78,14 @@ def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk @mark.skip() -def test_palo_alto_traffic_mstime(record_property, setup_wordlist, setup_splunk): +def test_palo_alto_traffic_mstime(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") search = st.render(host=host) diff --git a/tests/test_pfsense.py b/tests/test_pfsense.py index a5cc41b..2f8ee6a 100644 --- a/tests/test_pfsense.py +++ b/tests/test_pfsense.py @@ -12,13 +12,13 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #<27>Jan 25 01:58:06 filterlog: 82,,,1000002666,mvneta2,match,pass,out,6,0x00,0x00000,64,ICMPv6,58,8,fe80::208:a2ff:fe0f:cb66,fe80::56a6:5cff:fe7d:1d43, -def test_pfsense_filterlog(record_property, setup_wordlist, setup_splunk): +def test_pfsense_filterlog(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} filterlog: 82,,,1000002666,mvneta2,match,pass,out,6,0x00,0x00000,64,ICMPv6,58,8,{{key}},\n") message = mt.render(mark="<27>", key=host) - sendsingle(message, port=5006) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][5006]) st = env.from_string("search index=netfw sourcetype=pfsense:filterlog \"{{key}}\" earliest=-2m | head 2") search = st.render(key=host) @@ -32,13 +32,13 @@ def test_pfsense_filterlog(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #<27>Jan 25 01:58:06 kqueue error: unknown -def test_pfsense_other(record_property, setup_wordlist, setup_splunk): +def test_pfsense_other(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} kqueue error: {{key}}\n") message = mt.render(mark="<27>", key=host) - sendsingle(message, port=5006) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][5006]) st = env.from_string("search index=netops sourcetype=pfsense:* \"{{key}}\" earliest=-2m | head 2") search = st.render(key=host) @@ -52,13 +52,13 @@ def test_pfsense_other(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #<27>Jan 25 01:58:06 syslogd: restart -def test_pfsense_syslogd(record_property, setup_wordlist, setup_splunk): +def test_pfsense_syslogd(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} syslogd: restart {{key}}\n") message = mt.render(mark="<27>", key=host) - sendsingle(message, port=5006) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][5006]) st = env.from_string("search index=netops sourcetype=pfsense:syslogd \"{{key}}\" earliest=-2m | head 2") search = st.render(key=host) diff --git a/tests/test_plugin_example.py b/tests/test_plugin_example.py index 852dec5..b510ca8 100644 --- a/tests/test_plugin_example.py +++ b/tests/test_plugin_example.py @@ -12,13 +12,13 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) -def test_plugin_local_example(record_property, setup_wordlist, setup_splunk): +def test_plugin_local_example(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} sc4splugin[0]: test\n") message = mt.render(mark="<111>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"sc4s:local_example\" | head 2") search = st.render(host=host) diff --git a/tests/test_proofpoint.py b/tests/test_proofpoint.py index 4b04935..512a406 100644 --- a/tests/test_proofpoint.py +++ b/tests/test_proofpoint.py @@ -13,14 +13,14 @@ # Apr 17 18:33:26 aplegw01 filter_instance1[195529]: rprt s=2hdryp02r6 m=1 x=2hdryp02r6-1 cmd=send profile=mail qid=w3HMWjG3039079 rcpts=rfaircloth@splunk.com -def test_proofpoint_pps_filter(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_proofpoint_pps_filter(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} filter_instance1[195529]: rprt s=2hdryp02r6 m=1 x=2hdryp02r6-1 cmd=send profile=mail qid=w3HMWjG3039079 rcpts=rfaircloth@splunk.com\n") message = mt.render(mark="<166>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"pps_filter_log\" | head 2") search = st.render(host=host) @@ -34,14 +34,14 @@ def test_proofpoint_pps_filter(record_property, setup_wordlist, get_host_key, se assert resultCount == 1 # Apr 17 18:35:26 aplegw02 sendmail[56106]: w3HMZPVT056101: to=, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=FAIL, pri=133527, relay=mx1.splunk.iphmx.com. [216.71.153.223], dsn=2.0.0, stat=Sent (ok: Message 22675962 accepted) -def test_proofpoint_pps_mail(record_property, setup_wordlist, get_host_key, setup_splunk): +def test_proofpoint_pps_mail(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s): host = get_host_key mt = env.from_string( "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} pps-{{ host }} sendmail[195529]: w3HMZPVT056101: to=, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=FAIL, pri=133527, relay=mx1.splunk.iphmx.com. [216.71.153.223], dsn=2.0.0, stat=Sent (ok: Message 22675962 accepted)\n") message = mt.render(mark="<166>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=email host=\"pps-{{ host }}\" sourcetype=\"pps_mail_log\" | head 2") search = st.render(host=host) diff --git a/tests/test_symantec_brightmail.py b/tests/test_symantec_brightmail.py index 6e877f9..d73bae8 100644 --- a/tests/test_symantec_brightmail.py +++ b/tests/test_symantec_brightmail.py @@ -13,13 +13,13 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) # <141>Oct 24 21:05:43 smg-1 conduit: [Brightmail] (NOTICE:7500.3119331456): [12066] 'BrightSig3 Newsletter Rules' were updated successfully. -def test_symantec_brightmail(record_property, setup_wordlist, setup_splunk): +def test_symantec_brightmail(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} conduit: [Brightmail] (NOTICE:7500.3119331456): [12066] 'BrightSig3 Newsletter Rules' were updated successfully.") message = mt.render(mark="<134>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"symantec:smg\" | head 2") search = st.render(host=host) @@ -32,7 +32,7 @@ def test_symantec_brightmail(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_symantec_brightmail_msg(record_property, setup_wordlist, setup_splunk): +def test_symantec_brightmail_msg(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) msgid = uuid.uuid4() @@ -55,7 +55,7 @@ def test_symantec_brightmail_msg(record_property, setup_wordlist, setup_splunk): {{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|SOURCE|external\n {{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195987|{{ MSGID }}|VERDICT||connection_class_1|default|static connection class 1\n""") message = mt.render(mark="<1>", host=host, MSGID=msgid) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"symantec:smg:mail\" | head 2") search = st.render(host=host) diff --git a/tests/test_symantec_proxy.py b/tests/test_symantec_proxy.py index 0dddfac..8f29a59 100644 --- a/tests/test_symantec_proxy.py +++ b/tests/test_symantec_proxy.py @@ -12,13 +12,13 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) # <134>1 2019-08-21T17:42:08.000z "sample_logs bluecoat[0]:SPLV5.1 c-ip=192.0.0.6 cs-bytes=6269 cs-categories="unavailable" cs-host=gg.hhh.iii.com cs-ip=192.0.0.6 cs-method=GET cs-uri-path=/Sample/abc-xyz-01.pqr_sample_Internal.crt/MFAwTqADAgEAMEcwRTBDMAkGBSsOAwIaBQAEFOoaVMtyzC9gObESY9g1eXf1VM8VBBTl1mBq2WFf4cYqBI6c08kr4S302gIKUCIZdgAAAAAnQA%3D%3D cs-uri-port=8000 cs-uri-scheme=http cs-User-Agent="ocspd/1.0.3" cs-username=user4 clientduration=0 rs-status=0 s-action=TCP_HIT s-ip=10.0.0.6 serveripservice.name="Explicit HTTP" service.group="Standard" s-supplier-ip=10.0.0.6 s-supplier-name=gg.hhh.iii.com sc-bytes=9469 sc-filter-result=OBSERVED sc-status=200 time-taken=20 x-bluecoat-appliance-name="10.0.0.6-sample_logs" x-bluecoat-appliance-primary-address=10.0.0.6 x-bluecoat-proxy-primary-address=10.0.0.6 x-bluecoat-transaction-uuid=35d24c931c0erecta-0003000012161a77e70-00042100041002145cc859ed c-url="http://randomserver:8000/en-US/app/examples/" -def test_bluecoatproxySG_kv(record_property, setup_wordlist, setup_splunk): +def test_bluecoatproxySG_kv(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{host}} bluecoat[0]: SPLV5.1 c-ip=192.0.0.6 cs-bytes=6269 cs-categories=\"unavailable\" cs-host=gg.hhh.iii.com cs-ip=192.0.0.6 cs-method=GET cs-uri-path=/Sample/abc-xyz-01.pqr_sample_Internal.crt/MFAwTqADAgEAMEcwRTBDMAkGBSsOAwIaBQAEFOoaVMtyzC9gObESY9g1eXf1VM8VBBTl1mBq2WFf4cYqBI6c08kr4S302gIKUCIZdgAAAAAnQA%3D%3D cs-uri-port=8000 cs-uri-scheme=http cs-User-Agent=\"ocspd/1.0.3\" cs-username=user4 clientduration=0 rs-status=0 s-action=TCP_HIT s-ip=10.0.0.6 serveripservice.name=\"Explicit HTTP\" service.group=\"Standard\" s-supplier-ip=10.0.0.6 s-supplier-name=gg.hhh.iii.com sc-bytes=9469 sc-filter-result=OBSERVED sc-status=200 time-taken=20 x-bluecoat-appliance-name=\"10.0.0.6-sample_logs\" x-bluecoat-appliance-primary-address=10.0.0.6 x-bluecoat-proxy-primary-address=10.0.0.6 x-bluecoat-transaction-uuid=35d24c931c0erecta-0003000012161a77e70-00042100041002145cc859ed c-url=\"http://randomserver:8000/en-US/app/examples/\"") message = mt.render(mark="<134>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"bluecoat:proxysg:access:kv\" | head 2") search = st.render(host=host) diff --git a/tests/test_ubiquiti_unifi.py b/tests/test_ubiquiti_unifi.py index 0a69280..3298a58 100644 --- a/tests/test_ubiquiti_unifi.py +++ b/tests/test_ubiquiti_unifi.py @@ -13,13 +13,13 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #<27>Nov 8 17:28:43 US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type -def test_ubiquiti_unifi_us8p60(record_property, setup_wordlist, setup_splunk): +def test_ubiquiti_unifi_us8p60(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type {{key}}") message = mt.render(mark="<27>", key=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netops sourcetype=ubnt:switch \"{{key}}\" earliest=-2m | head 2") search = st.render(key=host) @@ -33,13 +33,13 @@ def test_ubiquiti_unifi_us8p60(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #<29>Nov 10 20:46:02 US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: 0 -def test_ubiquiti_unifi_switch_us24p250(record_property, setup_wordlist, setup_splunk): +def test_ubiquiti_unifi_switch_us24p250(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: {{key}}") message = mt.render(mark="<27>", key=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netops sourcetype=ubnt:switch \"{{key}}\" earliest=-2m | head 2") search = st.render(key=host) @@ -53,13 +53,13 @@ def test_ubiquiti_unifi_switch_us24p250(record_property, setup_wordlist, setup_s assert resultCount == 1 #<30>Nov 10 11:49:46 U7PG2,788a2056b181,v4.0.66.10832: logread[5495]: Logread connected to 10.2.0.9:514 -def test_ubiquiti_unifi_ap_u7pg2(record_property, setup_wordlist, setup_splunk): +def test_ubiquiti_unifi_ap_u7pg2(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} U7PG2,788a2056b181,v4.0.66.10832: logread[5495]: Logread connected to 10.2.0.9:514") message = mt.render(mark="<27>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netops sourcetype=ubnt:wireless earliest=-2m | head 2") search = st.render(host=host) @@ -73,13 +73,13 @@ def test_ubiquiti_unifi_ap_u7pg2(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #<4>Nov 10 23:04:06 USG kernel: [LAN_LOCAL-default-A]IN=eth0.2004 OUT= MAC= SRC=10.254.3.1 DST=224.0.0.251 LEN=348 TOS=0x00 PREC=0x00 TTL=255 ID=32463 DF PROTO=UDP SPT=5353 DPT=5353 LEN=328 -def test_ubiquiti_unifi_usg(record_property, setup_wordlist, setup_splunk): +def test_ubiquiti_unifi_usg(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} usg-{{host}} kernel: [LAN_LOCAL-default-A]IN=eth0.2004 OUT= MAC= SRC=10.254.3.1 DST=224.0.0.251 LEN=348 TOS=0x00 PREC=0x00 TTL=255 ID=32463 DF PROTO=UDP SPT=5353 DPT=5353 LEN=328") message = mt.render(mark="<27>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netfw sourcetype=ubnt:fw host=usg-{{host}} | head 2") search = st.render(host=host) diff --git a/tests/test_vmware.py b/tests/test_vmware.py index 235798b..8649ade 100644 --- a/tests/test_vmware.py +++ b/tests/test_vmware.py @@ -16,14 +16,14 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #vpxd 123 - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)] -def test_linux_vmware(record_property, setup_wordlist, setup_splunk): +def test_linux_vmware(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} vpxd {{ pid }} - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)]\n") message = mt.render(mark="<144>", host=host, pid=pid) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=main {{ pid }} sourcetype=\"vmware:vsphere:esx\" | head 2") search = st.render(host=host, pid=pid) @@ -37,14 +37,14 @@ def test_linux_vmware(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 #<46>1 2019-10-24T21:00:02.403Z {{ host }} NSXV 5996 - [nsxv@6876 comp="nsx-manager" subcomp="manager"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704 -def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk): +def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSX - SYSTEM [nsx@6876 comp=\"nsx-manager\" errorCode=\"MP4039\" subcomp=\"manager\"] Connection verification failed for broker '10.160.108.196'. Marking broker unhealthy.\n") message = mt.render(mark="<144>", host=host, pid=pid) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=main host={{ host }} sourcetype=\"vmware:vsphere:nsx\" | head 2") search = st.render(host=host, pid=pid) @@ -58,14 +58,14 @@ def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 # -def test_linux_vmware_nsx_fw(record_property, setup_wordlist, setup_splunk): +def test_linux_vmware_nsx_fw(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dfwpktlogs: {{ pid }} INET match PASS domain-c7/1001 IN 60 TCP 10.33.24.50/45926->10.33.24.9/8140 S\n") message = mt.render(mark="<144>", host=host, pid=pid) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=main host={{ host }} {{ pid }} sourcetype=\"vmware:vsphere:nsx\" | head 2") search = st.render(host=host, pid=pid) diff --git a/tests/test_zscaler_proxy.py b/tests/test_zscaler_proxy.py index e0f7fb1..e5581bb 100644 --- a/tests/test_zscaler_proxy.py +++ b/tests/test_zscaler_proxy.py @@ -13,13 +13,13 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #Note the long white space is a \t #2019-10-16 15:44:36 reason=Allowed event_id=6748427317914894361 protocol=HTTPS action=Allowed transactionsize=663 responsesize=65 requestsize=598 urlcategory=UK_ALLOW_Pharmacies serverip=216.58.204.70 clienttranstime=0 requestmethod=CONNECT refererURL=None useragent=Windows Windows 10 Enterprise ZTunnel/1.0 product=NSS location=UK_Wynyard_VPN->other ClientIP=192.168.0.38 status=200 user=first.last@example.com url=4171764.fls.doubleclick.net:443 vendor=Zscaler hostname=4171764.fls.doubleclick.net clientpublicIP=213.86.221.94 threatcategory=None threatname=None filetype=None appname=DoubleClick pagerisk=0 department=Procurement, Generics urlsupercategory=User-defined appclass=Sales and Marketing dlpengine=None urlclass=Bandwidth Loss threatclass=None dlpdictionaries=None fileclass=None bwthrottle=NO servertranstime=0 md5=None -def test_zscaler_proxy(record_property, setup_wordlist, setup_splunk): +def test_zscaler_proxy(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894361\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None") message = mt.render(mark="<134>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2") search = st.render(host=host) @@ -33,13 +33,13 @@ def test_zscaler_proxy(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 # -def test_zscaler_proxy_pri(record_property, setup_wordlist, setup_splunk): +def test_zscaler_proxy_pri(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( "{{mark}}{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894362\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None") message = mt.render(mark="<134>", host=host) - sendsingle(message) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2") search = st.render(host=host)