From 8d73e1d6e3f0916a627f6aec86034b56f89300ac Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Tue, 30 Jul 2019 15:32:36 -0400 Subject: [PATCH] Feature/perftest (#35) This merge will restore broken metrics functionality Define metrics using the a hierarchy syslogng.${SourceName} emulating Splunk Connect for Kafka Configure persistant queuing Define a new dest for metrics Remove early metrics collection scripts --- .circleci/config.yml | 38 ++++++++- .gitignore | 1 + .idea/vcs.xml | 10 +++ docker-compose-perf.yml | 77 +++++++++++++++++++ package/Dockerfile | 5 +- .../conf.d/destinations/d_destinations.conf | 42 ++++++++-- .../filters/cisco_asa_tradditional.conf | 2 +- package/etc/conf.d/filters/cisco_ios.conf | 2 +- package/etc/conf.d/filters/fortinet_fgt.conf | 2 +- .../etc/conf.d/filters/paloalto_networks.conf | 2 +- package/etc/context/splunk_index.csv | 8 +- package/etc/syslog-ng.conf | 58 ++++++++++++-- package/sbin/entrypoint.sh | 3 +- package/scripts/splunkmetrics.sh | 37 --------- perftests/bundlesrv/Dockerfile | 14 ++++ perftests/bundlesrv/ngnix.conf | 10 +++ perftests/bundlesrv/scenarios/build.sh | 8 ++ .../scenarios/test1/default/eventgen.conf | 14 ++++ .../scenarios/test1/samples/ciscoasa.sample | 1 + .../scenarios/test2/default/eventgen.conf | 13 ++++ .../scenarios/test2/samples/ciscoasa.sample | 1 + tests/test_palo_alto.py | 2 +- 22 files changed, 286 insertions(+), 64 deletions(-) create mode 100644 .idea/vcs.xml create mode 100644 docker-compose-perf.yml delete mode 100755 package/scripts/splunkmetrics.sh create mode 100644 perftests/bundlesrv/Dockerfile create mode 100644 perftests/bundlesrv/ngnix.conf create mode 100755 perftests/bundlesrv/scenarios/build.sh create mode 100644 perftests/bundlesrv/scenarios/test1/default/eventgen.conf create mode 100644 perftests/bundlesrv/scenarios/test1/samples/ciscoasa.sample create mode 100644 perftests/bundlesrv/scenarios/test2/default/eventgen.conf create mode 100644 perftests/bundlesrv/scenarios/test2/samples/ciscoasa.sample diff --git a/.circleci/config.yml b/.circleci/config.yml index a811e2a..1dedbc9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,13 +28,48 @@ jobs: command: docker login -u $DOCKER_USER -p $DOCKER_PASS - run: name: Build Docker image - command: pushd package; docker build -f Dockerfile --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION -t $IMAGE_NAME:build . + command: docker build -f package/Dockerfile --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION -t $IMAGE_NAME:build package - run: name: Tag Docker image command: docker tag $IMAGE_NAME:build $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Tag Docker image + command: docker tag $IMAGE_NAME:build $IMAGE_NAME:edge - run: name: Push Docker image command: docker push $IMAGE_NAME:$CIRCLE_SHA1 + - run: + name: Push Docker image + command: docker push $IMAGE_NAME:edge + build-egb: + environment: + IMAGE_NAME: rfaircloth/scs + docker: + - image: circleci/buildpack-deps:stretch + steps: + - checkout + - run: git submodule sync + - run: git submodule update --init --recursive + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Docker Login + command: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Build Docker image + command: docker build -f perftests/bundlesrv/Dockerfile -t $IMAGE_NAME:egb-build perftests/bundlesrv + - run: + name: Tag Docker image + command: docker tag $IMAGE_NAME:egb-build $IMAGE_NAME:egb-$CIRCLE_SHA1 + - run: + name: Tag Docker image + command: docker tag $IMAGE_NAME:egb-build $IMAGE_NAME:egb-edge + - run: + name: Push Docker image + command: docker push $IMAGE_NAME:egb-$CIRCLE_SHA1 + - run: + name: Push Docker image + command: docker push $IMAGE_NAME:egb-edge dgoss: environment: IMAGE_NAME: rfaircloth/scs @@ -164,6 +199,7 @@ workflows: build-publish: jobs: - build + - build-egb - dgoss: requires: - build diff --git a/.gitignore b/.gitignore index a63f961..602ef23 100644 --- a/.gitignore +++ b/.gitignore @@ -376,3 +376,4 @@ fabric.properties !/package/scripts/switch_transport.sh !/package/scripts/splunkmetrics.sh !/package/scripts/ +.ecs diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..6310177 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml new file mode 100644 index 0000000..67f3844 --- /dev/null +++ b/docker-compose-perf.yml @@ -0,0 +1,77 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . +version: "3" +services: + sc4s: + image: rfaircloth/scs:edge + build: + context: ./package + args: + RH_ORG: ${RH_ORG} + RH_ACTIVATION: ${RH_ACTIVATION} + hostname: sc4s + ports: + - "514" + - "601" + - "514/udp" + - "5514" + - "5514/udp" + links: + - splunk + environment: + - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} + - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} + - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} + - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} +# logging: +# driver: splunk +# options: +# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94 +# splunk-url: https://splunk:8088/services/collector/event +# splunk-index: main +# splunk-insecureskipverify: true +# splunk-verify-connection: false + splunk: + image: splunk/splunk:latest + hostname: splunk + ports: + - "8000:8000" + - "8088:8088" + - "8089:8089" + environment: + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} + - SPLUNK_START_ARGS=${SPLUNK_START_ARGS} + - SPLUNK_APPS_URL=${SPLUNK_APPS_URL} + - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} + - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} +# logging: +# driver: splunk +# options: +# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94 +# splunk-url: https://splunk:8088/services/collector/event +# splunk-index: main +# splunk-insecureskipverify: true +# splunk-verify-connection: false + egbundles: + image: rfaircloth/scs:egb-edge + hostname: egbundles + build: + context: perftests/bundlesrv + eventgen: + image: rfaircloth/eventgen:edge + command: standalone + ports: + - "9500:9500" + links: + - egbundles + - sc4s + diff --git a/package/Dockerfile b/package/Dockerfile index 5ecc921..a72311f 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -76,21 +76,20 @@ RUN cd /tmp ;\ ENV DEBCONF_NONINTERACTIVE_SEEN=true ENV SPLUNK_CONNECT_METHOD=hec -ENV SYSLOGNG_HEC_WORKERS=3 +ENV SYSLOGNG_HEC_WORKERS=10 RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh COPY goss.yaml /etc/goss.yaml COPY --from=0 /opt/syslog-ng /opt/syslog-ng -COPY scripts/splunkmetrics.sh /opt/syslog-ng/sbin/splunkmetrics.sh COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d COPY etc/context /opt/syslog-ng/etc/context COPY sbin/entrypoint.sh /sbin/entrypoint.sh +RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t -RUN mkdir -p /var/log/syslog-ng/data/disk-buffer EXPOSE 514 EXPOSE 601/tcp diff --git a/package/etc/conf.d/destinations/d_destinations.conf b/package/etc/conf.d/destinations/d_destinations.conf index 1d373d4..4f03b5f 100644 --- a/package/etc/conf.d/destinations/d_destinations.conf +++ b/package/etc/conf.d/destinations/d_destinations.conf @@ -17,18 +17,18 @@ destination d_hec { method("POST") log-fifo-size(`splunk-log-fifo-size`) workers(`SYSLOGNG_HEC_WORKERS`) - batch-lines(1000) + batch-lines(500) batch-bytes(512Kb) - batch-timeout(1000) - timeout(5) + batch-timeout(3) + timeout(15) user_agent("syslog-ng User Agent") user("syslog-ng") password("`SPLUNK_HEC_TOKEN`") -# persist-name("t_standard") -# disk-buffer(mem-buf-length(5000) -# disk-buf-size(107374182400) -# reliable(no) -# dir("/var/log/syslog-ng/data/disk-buffer")) + persist-name("splunk") + disk-buffer(mem-buf-length(500) + disk-buf-size(20000) + reliable(no) + dir("/opt/syslog-ng/var/data/disk-buffer/")) tls(peer-verify(no) # ca-dir("dir") # ca-file("ca") @@ -48,3 +48,29 @@ destination d_hec { fields.*)') ); }; + + +destination d_hecmetrics { + http( + url("`SPLUNK_HEC_URL`") + method("POST") + batch-lines(5) + batch-bytes(512Kb) + batch-timeout(1) + timeout(15) + user_agent("syslog-ng User Agent") + user("syslog-ng") + password("`SPLUNK_HEC_TOKEN`") + persist-name("splunk_metrics") + tls(peer-verify(no) +# ca-dir("dir") +# ca-file("ca") +# cert-file("cert") +# cipher-suite("cipher") +# key-file("key") +# peer-verify(yes|no) +# ssl-version() + ) + body('$MESSAGE') + ); + }; diff --git a/package/etc/conf.d/filters/cisco_asa_tradditional.conf b/package/etc/conf.d/filters/cisco_asa_tradditional.conf index a912baf..57d25ce 100644 --- a/package/etc/conf.d/filters/cisco_asa_tradditional.conf +++ b/package/etc/conf.d/filters/cisco_asa_tradditional.conf @@ -20,5 +20,5 @@ log { destination(d_hec); #--HEC-- - flags(final); + flags(flow-control,final); }; diff --git a/package/etc/conf.d/filters/cisco_ios.conf b/package/etc/conf.d/filters/cisco_ios.conf index 98427f1..cac0bb7 100644 --- a/package/etc/conf.d/filters/cisco_ios.conf +++ b/package/etc/conf.d/filters/cisco_ios.conf @@ -12,5 +12,5 @@ log { rewrite { r_set_splunk_basic(template("t_msg_only"))}; #--HEC-- destination(d_hec); #--HEC-- - flags(final); + flags(flow-control,final); }; diff --git a/package/etc/conf.d/filters/fortinet_fgt.conf b/package/etc/conf.d/filters/fortinet_fgt.conf index 5a4afcf..f862fa9 100644 --- a/package/etc/conf.d/filters/fortinet_fgt.conf +++ b/package/etc/conf.d/filters/fortinet_fgt.conf @@ -36,5 +36,5 @@ log { rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- destination(d_hec); #--HEC-- - flags(final); + flags(flow-control,final); }; diff --git a/package/etc/conf.d/filters/paloalto_networks.conf b/package/etc/conf.d/filters/paloalto_networks.conf index 1ef160c..a3e1016 100644 --- a/package/etc/conf.d/filters/paloalto_networks.conf +++ b/package/etc/conf.d/filters/paloalto_networks.conf @@ -57,5 +57,5 @@ log { destination(d_hec); #--HEC-- - flags(final); + flags(flow-control,final); }; diff --git a/package/etc/context/splunk_index.csv b/package/etc/context/splunk_index.csv index eae8cf3..efa7216 100644 --- a/package/etc/context/splunk_index.csv +++ b/package/etc/context/splunk_index.csv @@ -26,5 +26,9 @@ pan:hipwatch,sourcetype,pan:hipwatch pan:correlation,sourcetype,pan:correlation pan:userid,sourcetype,pan:userid pan:unknown,sourcetype,pan:unknown -syslogng-fallback,index,syslogng-fallback -syslogng-fallback,sourcetype,syslogng-fallback +syslog-ng:events,index,main +syslog-ng:events,sourcetype,syslog-ng:events +syslog-ng:fallback,index,main +syslog-ng:fallback,sourcetype,syslog-ng:fallback +syslog-ng:metrics,index,em_metrics +syslog-ng:metrics,sourcetype,httpevent diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index ab28af9..ba0768c 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -15,15 +15,21 @@ options { log_msg_size (65536); - flush_lines (0); + flush_lines (100); time_reopen (10); log_fifo_size (10000); chain_hostnames (off); use_dns (no); use_fqdn (no); - dns-cache(no); create_dirs (no); keep-hostname (yes); + create_dirs(yes); + dir_perm(0750); + stats-freq(30); + stats-level(1); + stats-max-dynamics(2000); + normalize-hostnames(yes); + on-error(fallback-to-string); }; # =============================================================================================== @@ -54,10 +60,10 @@ options { # =============================================================================================== @define ip-version 4 @define splunk-max-connections 1000 -@define splunk-log-fifo-size 10000 -@define splunk-fetch-limit 100 +@define splunk-log-fifo-size 180000000 +@define splunk-fetch-limit 20000 # make sure splunk-window-size >= splunk-max-connections * splunk-fetch-limit -@define splunk-window-size 100000 +@define splunk-window-size 20000000 @define splunk-rcvbuf 425984 @define default-timezone "GMT" @@ -70,6 +76,45 @@ options { @include "conf.d/sources/*.conf" @include "conf.d/blocks/*.conf" @include "conf.d/destinations/*.conf" + +log { + source(s_internal); + + if (match("^Log statistics; " value("MESSAGE"))) { + + parser {p_add_context_splunk(key("syslog-ng:metrics")); }; + rewrite { + subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?', + ' + {"time": "$S_UNIXTIME.$S_MSEC", + "event": "metric", + "host": "$HOST", + "index": "${.splunk.index}", + "source": "internal", + "sourcetype": "${.splunk.sourcetype}", + "fields": { + "source_name": "${SourceName}", + "source_instance": "${SourceInstance}", + "state": "${State}", + "type": "${Type}", + "_value": ${Number}, + "metric_name": "syslogng.${SourceId}" + } + } + ', + value("MESSAGE") flags("utf8" "global") + ); + }; + destination(d_hecmetrics); #--HEC-- + } else { + parser {p_add_context_splunk(key("syslog-ng:events")); }; + rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- + destination(d_hec); #--HEC-- + }; + flags(final); +}; + + @include "conf.d/prefilters/*.conf" @include "conf.d/filters/*.conf" @@ -79,9 +124,10 @@ options { log { source(s_default-ports); + parser {p_add_context_splunk(key("syslog-ng:fallback")); }; rewrite { r_set_splunk(template("t_JSON"))}; #--HEC-- destination(d_hec); #--HEC-- - flags(fallback); + flags(flow-control,fallback); }; diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 3578897..5d37dd1 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -7,5 +7,4 @@ #Run syslog mkdir /opt/syslog-ng/var rm /opt/syslog-ng/var/syslog-ng.ctl || true -/opt/syslog-ng/sbin/syslog-ng --process-mode=background -/opt/syslog-ng/sbin/splunkmetrics.sh +/opt/syslog-ng/sbin/syslog-ng -F diff --git a/package/scripts/splunkmetrics.sh b/package/scripts/splunkmetrics.sh deleted file mode 100755 index 7a497eb..0000000 --- a/package/scripts/splunkmetrics.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -#Copyright 2019 Splunk, Inc. -# -#Use of this source code is governed by a BSD-2-clause-style -#license that can be found in the LICENSE-BSD2 file or at -#https://opensource.org/licenses/BSD-2-Clause -sleep 30 -while sleep 10 -do - TS=$(date +"%s") - tmpfilecsv=$(mktemp /tmp/syslog-ng-stats-XXXXXXX-csv) - tmpfilejson=$(mktemp /tmp/syslog-ng-stats-XXXXXXX-json) - /opt/syslog-ng/sbin/syslog-ng-ctl query get "*" --reset | tail -n +2 | grep \^. | grep -v "^The selected counters" >$tmpfilecsv - IFS=';' - while read -r SourceName SourceId SourceInstance State Type Number - do - echo "{ - \"time\":\"$TS\", - \"event\":\"metric\", - \"source\":\"syslog-ng\", - \"host\":\"$HOSTNAME\", - \"index\":\"$SPLUNK_METRICS_INDEX\", - \"fields\":{ - \"SourceId\":\"$SourceId\", - \"SourceInstance\":\"$SourceInstance\", - \"State\":\"$State\", - \"Type\":\"$Type\", - \"_value\":$Number, - \"metric_name\":\"$SourceName\" - } - } ">>$tmpfilejson - done < $tmpfilecsv - - curl -s -S -k $SPLUNK_HEC_STATSURL -H "Authorization: Splunk $SPLUNK_HEC_TOKEN" -d "@$tmpfilejson" - rm $tmpfilejson - rm $tmpfilecsv -done diff --git a/perftests/bundlesrv/Dockerfile b/perftests/bundlesrv/Dockerfile new file mode 100644 index 0000000..e5ed403 --- /dev/null +++ b/perftests/bundlesrv/Dockerfile @@ -0,0 +1,14 @@ +FROM registry.access.redhat.com/ubi7/ubi as build + +COPY scenarios /scenarios + +RUN yum install zip -y + +RUN mkdir /bundles + +RUN cd /scenarios;./build.sh;cp *.zip /bundles + +FROM nginx:stable + +COPY ./ngnix.conf /etc/nginx/conf.d/default.conf +COPY --from=build /bundles/ /var/www diff --git a/perftests/bundlesrv/ngnix.conf b/perftests/bundlesrv/ngnix.conf new file mode 100644 index 0000000..f0c3ae0 --- /dev/null +++ b/perftests/bundlesrv/ngnix.conf @@ -0,0 +1,10 @@ +server { + listen 80; + + root /var/www/; + autoindex on; + # Force all paths to load either itself (js files) or go through index.html. + location / { + try_files $uri /index.html; + } +} \ No newline at end of file diff --git a/perftests/bundlesrv/scenarios/build.sh b/perftests/bundlesrv/scenarios/build.sh new file mode 100755 index 0000000..8d151e8 --- /dev/null +++ b/perftests/bundlesrv/scenarios/build.sh @@ -0,0 +1,8 @@ + #!/bin/bash + + IFS=$'\n' + + for f in $(find . -mindepth 1 -maxdepth 1 -type d ); do + zip -x .* -x dist -r "$f" "$f/" + done + diff --git a/perftests/bundlesrv/scenarios/test1/default/eventgen.conf b/perftests/bundlesrv/scenarios/test1/default/eventgen.conf new file mode 100644 index 0000000..6fb3cb7 --- /dev/null +++ b/perftests/bundlesrv/scenarios/test1/default/eventgen.conf @@ -0,0 +1,14 @@ +[ciscoasa.sample] +sampleDir = samples +sampletype = raw +outputMode = tcpout +tcpDestinationHost = sc4s +tcpDestinationPort = 514 +end = 10 +mode = sample +interval = 1 +count = 100 +#2018-06-27T12:17:46 +token.0.token = \d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ +token.0.replacementType = timestamp +token.0.replacement = %Y-%m-%dT%H:%M:%SZ diff --git a/perftests/bundlesrv/scenarios/test1/samples/ciscoasa.sample b/perftests/bundlesrv/scenarios/test1/samples/ciscoasa.sample new file mode 100644 index 0000000..cd37166 --- /dev/null +++ b/perftests/bundlesrv/scenarios/test1/samples/ciscoasa.sample @@ -0,0 +1 @@ +<166>2018-06-27T12:17:46Z asa : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 \ No newline at end of file diff --git a/perftests/bundlesrv/scenarios/test2/default/eventgen.conf b/perftests/bundlesrv/scenarios/test2/default/eventgen.conf new file mode 100644 index 0000000..98872e2 --- /dev/null +++ b/perftests/bundlesrv/scenarios/test2/default/eventgen.conf @@ -0,0 +1,13 @@ +[ciscoasa.sample] +sampleDir = samples +sampletype = raw +outputMode = tcpout +tcpDestinationHost = sc4s +tcpDestinationPort = 514 +end = 300 +mode = sample +interval = 1 +#2018-06-27T12:17:46 +token.0.token = \d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ +token.0.replacementType = timestamp +token.0.replacement = %Y-%m-%dT%H:%M:%SZ diff --git a/perftests/bundlesrv/scenarios/test2/samples/ciscoasa.sample b/perftests/bundlesrv/scenarios/test2/samples/ciscoasa.sample new file mode 100644 index 0000000..cd37166 --- /dev/null +++ b/perftests/bundlesrv/scenarios/test2/samples/ciscoasa.sample @@ -0,0 +1 @@ +<166>2018-06-27T12:17:46Z asa : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 \ No newline at end of file diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py index ec8744e..5a601f0 100644 --- a/tests/test_palo_alto.py +++ b/tests/test_palo_alto.py @@ -14,7 +14,7 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) -# <190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0 +#<190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0 @flaky(max_runs=3, min_passes=2) def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))