diff --git a/.circleci/config.yml b/.circleci/config.yml
index a811e2a..1dedbc9 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -28,13 +28,48 @@ jobs:
command: docker login -u $DOCKER_USER -p $DOCKER_PASS
- run:
name: Build Docker image
- command: pushd package; docker build -f Dockerfile --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION -t $IMAGE_NAME:build .
+ command: docker build -f package/Dockerfile --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION -t $IMAGE_NAME:build package
- run:
name: Tag Docker image
command: docker tag $IMAGE_NAME:build $IMAGE_NAME:$CIRCLE_SHA1
+ - run:
+ name: Tag Docker image
+ command: docker tag $IMAGE_NAME:build $IMAGE_NAME:edge
- run:
name: Push Docker image
command: docker push $IMAGE_NAME:$CIRCLE_SHA1
+ - run:
+ name: Push Docker image
+ command: docker push $IMAGE_NAME:edge
+ build-egb:
+ environment:
+ IMAGE_NAME: rfaircloth/scs
+ docker:
+ - image: circleci/buildpack-deps:stretch
+ steps:
+ - checkout
+ - run: git submodule sync
+ - run: git submodule update --init --recursive
+ - setup_remote_docker:
+ docker_layer_caching: true
+ - run:
+ name: Docker Login
+ command: docker login -u $DOCKER_USER -p $DOCKER_PASS
+ - run:
+ name: Build Docker image
+ command: docker build -f perftests/bundlesrv/Dockerfile -t $IMAGE_NAME:egb-build perftests/bundlesrv
+ - run:
+ name: Tag Docker image
+ command: docker tag $IMAGE_NAME:egb-build $IMAGE_NAME:egb-$CIRCLE_SHA1
+ - run:
+ name: Tag Docker image
+ command: docker tag $IMAGE_NAME:egb-build $IMAGE_NAME:egb-edge
+ - run:
+ name: Push Docker image
+ command: docker push $IMAGE_NAME:egb-$CIRCLE_SHA1
+ - run:
+ name: Push Docker image
+ command: docker push $IMAGE_NAME:egb-edge
dgoss:
environment:
IMAGE_NAME: rfaircloth/scs
@@ -164,6 +199,7 @@ workflows:
build-publish:
jobs:
- build
+ - build-egb
- dgoss:
requires:
- build
diff --git a/.gitignore b/.gitignore
index a63f961..602ef23 100644
--- a/.gitignore
+++ b/.gitignore
@@ -376,3 +376,4 @@ fabric.properties
!/package/scripts/switch_transport.sh
!/package/scripts/splunkmetrics.sh
!/package/scripts/
+.ecs
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..6310177
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,10 @@
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml
new file mode 100644
index 0000000..67f3844
--- /dev/null
+++ b/docker-compose-perf.yml
@@ -0,0 +1,77 @@
+#Splunk Connect for Syslog (SC4S) by Splunk, Inc.
+#
+#To the extent possible under law, the person who associated CC0 with
+#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights
+#to Splunk Connect for Syslog (SC4S).
+#
+#You should have received a copy of the CC0 legalcode along with this
+#work. If not, see .
+version: "3"
+services:
+ sc4s:
+ image: rfaircloth/scs:edge
+ build:
+ context: ./package
+ args:
+ RH_ORG: ${RH_ORG}
+ RH_ACTIVATION: ${RH_ACTIVATION}
+ hostname: sc4s
+ ports:
+ - "514"
+ - "601"
+ - "514/udp"
+ - "5514"
+ - "5514/udp"
+ links:
+ - splunk
+ environment:
+ - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
+ - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
+ - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+ - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
+ - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
+ - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
+# logging:
+# driver: splunk
+# options:
+# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
+# splunk-url: https://splunk:8088/services/collector/event
+# splunk-index: main
+# splunk-insecureskipverify: true
+# splunk-verify-connection: false
+ splunk:
+ image: splunk/splunk:latest
+ hostname: splunk
+ ports:
+ - "8000:8000"
+ - "8088:8088"
+ - "8089:8089"
+ environment:
+ - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+ - SPLUNK_PASSWORD=${SPLUNK_PASSWORD}
+ - SPLUNK_START_ARGS=${SPLUNK_START_ARGS}
+ - SPLUNK_APPS_URL=${SPLUNK_APPS_URL}
+ - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME}
+ - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
+# logging:
+# driver: splunk
+# options:
+# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
+# splunk-url: https://splunk:8088/services/collector/event
+# splunk-index: main
+# splunk-insecureskipverify: true
+# splunk-verify-connection: false
+ egbundles:
+ image: rfaircloth/scs:egb-edge
+ hostname: egbundles
+ build:
+ context: perftests/bundlesrv
+ eventgen:
+ image: rfaircloth/eventgen:edge
+ command: standalone
+ ports:
+ - "9500:9500"
+ links:
+ - egbundles
+ - sc4s
+
diff --git a/package/Dockerfile b/package/Dockerfile
index 5ecc921..a72311f 100644
--- a/package/Dockerfile
+++ b/package/Dockerfile
@@ -76,21 +76,20 @@ RUN cd /tmp ;\
ENV DEBCONF_NONINTERACTIVE_SEEN=true
ENV SPLUNK_CONNECT_METHOD=hec
-ENV SYSLOGNG_HEC_WORKERS=3
+ENV SYSLOGNG_HEC_WORKERS=10
RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh
COPY goss.yaml /etc/goss.yaml
COPY --from=0 /opt/syslog-ng /opt/syslog-ng
-COPY scripts/splunkmetrics.sh /opt/syslog-ng/sbin/splunkmetrics.sh
COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf
COPY etc/conf.d /opt/syslog-ng/etc/conf.d
COPY etc/context /opt/syslog-ng/etc/context
COPY sbin/entrypoint.sh /sbin/entrypoint.sh
+RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer
RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V
RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t
-RUN mkdir -p /var/log/syslog-ng/data/disk-buffer
EXPOSE 514
EXPOSE 601/tcp
diff --git a/package/etc/conf.d/destinations/d_destinations.conf b/package/etc/conf.d/destinations/d_destinations.conf
index 1d373d4..4f03b5f 100644
--- a/package/etc/conf.d/destinations/d_destinations.conf
+++ b/package/etc/conf.d/destinations/d_destinations.conf
@@ -17,18 +17,18 @@ destination d_hec {
method("POST")
log-fifo-size(`splunk-log-fifo-size`)
workers(`SYSLOGNG_HEC_WORKERS`)
- batch-lines(1000)
+ batch-lines(500)
batch-bytes(512Kb)
- batch-timeout(1000)
- timeout(5)
+ batch-timeout(3)
+ timeout(15)
user_agent("syslog-ng User Agent")
user("syslog-ng")
password("`SPLUNK_HEC_TOKEN`")
-# persist-name("t_standard")
-# disk-buffer(mem-buf-length(5000)
-# disk-buf-size(107374182400)
-# reliable(no)
-# dir("/var/log/syslog-ng/data/disk-buffer"))
+ persist-name("splunk")
+ disk-buffer(mem-buf-length(500)
+ disk-buf-size(20000)
+ reliable(no)
+ dir("/opt/syslog-ng/var/data/disk-buffer/"))
tls(peer-verify(no)
# ca-dir("dir")
# ca-file("ca")
@@ -48,3 +48,29 @@ destination d_hec {
fields.*)')
);
};
+
+
+destination d_hecmetrics {
+ http(
+ url("`SPLUNK_HEC_URL`")
+ method("POST")
+ batch-lines(5)
+ batch-bytes(512Kb)
+ batch-timeout(1)
+ timeout(15)
+ user_agent("syslog-ng User Agent")
+ user("syslog-ng")
+ password("`SPLUNK_HEC_TOKEN`")
+ persist-name("splunk_metrics")
+ tls(peer-verify(no)
+# ca-dir("dir")
+# ca-file("ca")
+# cert-file("cert")
+# cipher-suite("cipher")
+# key-file("key")
+# peer-verify(yes|no)
+# ssl-version()
+ )
+ body('$MESSAGE')
+ );
+ };
diff --git a/package/etc/conf.d/filters/cisco_asa_tradditional.conf b/package/etc/conf.d/filters/cisco_asa_tradditional.conf
index a912baf..57d25ce 100644
--- a/package/etc/conf.d/filters/cisco_asa_tradditional.conf
+++ b/package/etc/conf.d/filters/cisco_asa_tradditional.conf
@@ -20,5 +20,5 @@ log {
destination(d_hec); #--HEC--
- flags(final);
+ flags(flow-control,final);
};
diff --git a/package/etc/conf.d/filters/cisco_ios.conf b/package/etc/conf.d/filters/cisco_ios.conf
index 98427f1..cac0bb7 100644
--- a/package/etc/conf.d/filters/cisco_ios.conf
+++ b/package/etc/conf.d/filters/cisco_ios.conf
@@ -12,5 +12,5 @@ log {
rewrite { r_set_splunk_basic(template("t_msg_only"))}; #--HEC--
destination(d_hec); #--HEC--
- flags(final);
+ flags(flow-control,final);
};
diff --git a/package/etc/conf.d/filters/fortinet_fgt.conf b/package/etc/conf.d/filters/fortinet_fgt.conf
index 5a4afcf..f862fa9 100644
--- a/package/etc/conf.d/filters/fortinet_fgt.conf
+++ b/package/etc/conf.d/filters/fortinet_fgt.conf
@@ -36,5 +36,5 @@ log {
rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC--
destination(d_hec); #--HEC--
- flags(final);
+ flags(flow-control,final);
};
diff --git a/package/etc/conf.d/filters/paloalto_networks.conf b/package/etc/conf.d/filters/paloalto_networks.conf
index 1ef160c..a3e1016 100644
--- a/package/etc/conf.d/filters/paloalto_networks.conf
+++ b/package/etc/conf.d/filters/paloalto_networks.conf
@@ -57,5 +57,5 @@ log {
destination(d_hec); #--HEC--
- flags(final);
+ flags(flow-control,final);
};
diff --git a/package/etc/context/splunk_index.csv b/package/etc/context/splunk_index.csv
index eae8cf3..efa7216 100644
--- a/package/etc/context/splunk_index.csv
+++ b/package/etc/context/splunk_index.csv
@@ -26,5 +26,9 @@ pan:hipwatch,sourcetype,pan:hipwatch
pan:correlation,sourcetype,pan:correlation
pan:userid,sourcetype,pan:userid
pan:unknown,sourcetype,pan:unknown
-syslogng-fallback,index,syslogng-fallback
-syslogng-fallback,sourcetype,syslogng-fallback
+syslog-ng:events,index,main
+syslog-ng:events,sourcetype,syslog-ng:events
+syslog-ng:fallback,index,main
+syslog-ng:fallback,sourcetype,syslog-ng:fallback
+syslog-ng:metrics,index,em_metrics
+syslog-ng:metrics,sourcetype,httpevent
diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf
index ab28af9..ba0768c 100644
--- a/package/etc/syslog-ng.conf
+++ b/package/etc/syslog-ng.conf
@@ -15,15 +15,21 @@
options {
log_msg_size (65536);
- flush_lines (0);
+ flush_lines (100);
time_reopen (10);
log_fifo_size (10000);
chain_hostnames (off);
use_dns (no);
use_fqdn (no);
- dns-cache(no);
create_dirs (no);
keep-hostname (yes);
+ create_dirs(yes);
+ dir_perm(0750);
+ stats-freq(30);
+ stats-level(1);
+ stats-max-dynamics(2000);
+ normalize-hostnames(yes);
+ on-error(fallback-to-string);
};
# ===============================================================================================
@@ -54,10 +60,10 @@ options {
# ===============================================================================================
@define ip-version 4
@define splunk-max-connections 1000
-@define splunk-log-fifo-size 10000
-@define splunk-fetch-limit 100
+@define splunk-log-fifo-size 180000000
+@define splunk-fetch-limit 20000
# make sure splunk-window-size >= splunk-max-connections * splunk-fetch-limit
-@define splunk-window-size 100000
+@define splunk-window-size 20000000
@define splunk-rcvbuf 425984
@define default-timezone "GMT"
@@ -70,6 +76,45 @@ options {
@include "conf.d/sources/*.conf"
@include "conf.d/blocks/*.conf"
@include "conf.d/destinations/*.conf"
+
+log {
+ source(s_internal);
+
+ if (match("^Log statistics; " value("MESSAGE"))) {
+
+ parser {p_add_context_splunk(key("syslog-ng:metrics")); };
+ rewrite {
+ subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?',
+ '
+ {"time": "$S_UNIXTIME.$S_MSEC",
+ "event": "metric",
+ "host": "$HOST",
+ "index": "${.splunk.index}",
+ "source": "internal",
+ "sourcetype": "${.splunk.sourcetype}",
+ "fields": {
+ "source_name": "${SourceName}",
+ "source_instance": "${SourceInstance}",
+ "state": "${State}",
+ "type": "${Type}",
+ "_value": ${Number},
+ "metric_name": "syslogng.${SourceId}"
+ }
+ }
+ ',
+ value("MESSAGE") flags("utf8" "global")
+ );
+ };
+ destination(d_hecmetrics); #--HEC--
+ } else {
+ parser {p_add_context_splunk(key("syslog-ng:events")); };
+ rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC--
+ destination(d_hec); #--HEC--
+ };
+ flags(final);
+};
+
+
@include "conf.d/prefilters/*.conf"
@include "conf.d/filters/*.conf"
@@ -79,9 +124,10 @@ options {
log {
source(s_default-ports);
+ parser {p_add_context_splunk(key("syslog-ng:fallback")); };
rewrite { r_set_splunk(template("t_JSON"))}; #--HEC--
destination(d_hec); #--HEC--
- flags(fallback);
+ flags(flow-control,fallback);
};
diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh
index 3578897..5d37dd1 100755
--- a/package/sbin/entrypoint.sh
+++ b/package/sbin/entrypoint.sh
@@ -7,5 +7,4 @@
#Run syslog
mkdir /opt/syslog-ng/var
rm /opt/syslog-ng/var/syslog-ng.ctl || true
-/opt/syslog-ng/sbin/syslog-ng --process-mode=background
-/opt/syslog-ng/sbin/splunkmetrics.sh
+/opt/syslog-ng/sbin/syslog-ng -F
diff --git a/package/scripts/splunkmetrics.sh b/package/scripts/splunkmetrics.sh
deleted file mode 100755
index 7a497eb..0000000
--- a/package/scripts/splunkmetrics.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-#Copyright 2019 Splunk, Inc.
-#
-#Use of this source code is governed by a BSD-2-clause-style
-#license that can be found in the LICENSE-BSD2 file or at
-#https://opensource.org/licenses/BSD-2-Clause
-sleep 30
-while sleep 10
-do
- TS=$(date +"%s")
- tmpfilecsv=$(mktemp /tmp/syslog-ng-stats-XXXXXXX-csv)
- tmpfilejson=$(mktemp /tmp/syslog-ng-stats-XXXXXXX-json)
- /opt/syslog-ng/sbin/syslog-ng-ctl query get "*" --reset | tail -n +2 | grep \^. | grep -v "^The selected counters" >$tmpfilecsv
- IFS=';'
- while read -r SourceName SourceId SourceInstance State Type Number
- do
- echo "{
- \"time\":\"$TS\",
- \"event\":\"metric\",
- \"source\":\"syslog-ng\",
- \"host\":\"$HOSTNAME\",
- \"index\":\"$SPLUNK_METRICS_INDEX\",
- \"fields\":{
- \"SourceId\":\"$SourceId\",
- \"SourceInstance\":\"$SourceInstance\",
- \"State\":\"$State\",
- \"Type\":\"$Type\",
- \"_value\":$Number,
- \"metric_name\":\"$SourceName\"
- }
- } ">>$tmpfilejson
- done < $tmpfilecsv
-
- curl -s -S -k $SPLUNK_HEC_STATSURL -H "Authorization: Splunk $SPLUNK_HEC_TOKEN" -d "@$tmpfilejson"
- rm $tmpfilejson
- rm $tmpfilecsv
-done
diff --git a/perftests/bundlesrv/Dockerfile b/perftests/bundlesrv/Dockerfile
new file mode 100644
index 0000000..e5ed403
--- /dev/null
+++ b/perftests/bundlesrv/Dockerfile
@@ -0,0 +1,14 @@
+FROM registry.access.redhat.com/ubi7/ubi as build
+
+COPY scenarios /scenarios
+
+RUN yum install zip -y
+
+RUN mkdir /bundles
+
+RUN cd /scenarios;./build.sh;cp *.zip /bundles
+
+FROM nginx:stable
+
+COPY ./ngnix.conf /etc/nginx/conf.d/default.conf
+COPY --from=build /bundles/ /var/www
diff --git a/perftests/bundlesrv/ngnix.conf b/perftests/bundlesrv/ngnix.conf
new file mode 100644
index 0000000..f0c3ae0
--- /dev/null
+++ b/perftests/bundlesrv/ngnix.conf
@@ -0,0 +1,10 @@
+server {
+ listen 80;
+
+ root /var/www/;
+ autoindex on;
+ # Force all paths to load either itself (js files) or go through index.html.
+ location / {
+ try_files $uri /index.html;
+ }
+}
\ No newline at end of file
diff --git a/perftests/bundlesrv/scenarios/build.sh b/perftests/bundlesrv/scenarios/build.sh
new file mode 100755
index 0000000..8d151e8
--- /dev/null
+++ b/perftests/bundlesrv/scenarios/build.sh
@@ -0,0 +1,8 @@
+ #!/bin/bash
+
+ IFS=$'\n'
+
+ for f in $(find . -mindepth 1 -maxdepth 1 -type d ); do
+ zip -x .* -x dist -r "$f" "$f/"
+ done
+
diff --git a/perftests/bundlesrv/scenarios/test1/default/eventgen.conf b/perftests/bundlesrv/scenarios/test1/default/eventgen.conf
new file mode 100644
index 0000000..6fb3cb7
--- /dev/null
+++ b/perftests/bundlesrv/scenarios/test1/default/eventgen.conf
@@ -0,0 +1,14 @@
+[ciscoasa.sample]
+sampleDir = samples
+sampletype = raw
+outputMode = tcpout
+tcpDestinationHost = sc4s
+tcpDestinationPort = 514
+end = 10
+mode = sample
+interval = 1
+count = 100
+#2018-06-27T12:17:46
+token.0.token = \d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%dT%H:%M:%SZ
diff --git a/perftests/bundlesrv/scenarios/test1/samples/ciscoasa.sample b/perftests/bundlesrv/scenarios/test1/samples/ciscoasa.sample
new file mode 100644
index 0000000..cd37166
--- /dev/null
+++ b/perftests/bundlesrv/scenarios/test1/samples/ciscoasa.sample
@@ -0,0 +1 @@
+<166>2018-06-27T12:17:46Z asa : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23
\ No newline at end of file
diff --git a/perftests/bundlesrv/scenarios/test2/default/eventgen.conf b/perftests/bundlesrv/scenarios/test2/default/eventgen.conf
new file mode 100644
index 0000000..98872e2
--- /dev/null
+++ b/perftests/bundlesrv/scenarios/test2/default/eventgen.conf
@@ -0,0 +1,13 @@
+[ciscoasa.sample]
+sampleDir = samples
+sampletype = raw
+outputMode = tcpout
+tcpDestinationHost = sc4s
+tcpDestinationPort = 514
+end = 300
+mode = sample
+interval = 1
+#2018-06-27T12:17:46
+token.0.token = \d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%dT%H:%M:%SZ
diff --git a/perftests/bundlesrv/scenarios/test2/samples/ciscoasa.sample b/perftests/bundlesrv/scenarios/test2/samples/ciscoasa.sample
new file mode 100644
index 0000000..cd37166
--- /dev/null
+++ b/perftests/bundlesrv/scenarios/test2/samples/ciscoasa.sample
@@ -0,0 +1 @@
+<166>2018-06-27T12:17:46Z asa : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23
\ No newline at end of file
diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py
index ec8744e..5a601f0 100644
--- a/tests/test_palo_alto.py
+++ b/tests/test_palo_alto.py
@@ -14,7 +14,7 @@
env = Environment(extensions=['jinja2_time.TimeExtension'])
-# <190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0
+#<190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0
@flaky(max_runs=3, min_passes=2)
def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))