Feature/perftest (#35)

This merge will restore broken metrics functionality Define metrics using the a hierarchy syslogng.${SourceName} emulating Splunk Connect for Kafka Configure persistant queuing Define a new dest for metrics Remove early metrics collection scripts
CSVD · Jul 30, 2019 · 8d73e1d · 8d73e1d
1 parent 422071a
commit 8d73e1d
Show file tree

Hide file tree

Showing 22 changed files with 286 additions and 64 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -28,13 +28,48 @@ jobs:
           command: docker login -u $DOCKER_USER -p $DOCKER_PASS
       - run:
           name: Build Docker image
-          command: pushd package; docker build -f Dockerfile --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION -t $IMAGE_NAME:build .
+          command: docker build -f package/Dockerfile --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION -t $IMAGE_NAME:build package
       - run:
           name: Tag Docker image
           command: docker tag $IMAGE_NAME:build $IMAGE_NAME:$CIRCLE_SHA1
+      - run:
+          name: Tag Docker image
+          command: docker tag $IMAGE_NAME:build $IMAGE_NAME:edge
       - run:
           name: Push Docker image
           command: docker push $IMAGE_NAME:$CIRCLE_SHA1
+      - run:
+          name: Push Docker image
+          command: docker push $IMAGE_NAME:edge
+  build-egb:
+    environment:
+      IMAGE_NAME: rfaircloth/scs
+    docker:
+      - image: circleci/buildpack-deps:stretch
+    steps:
+      - checkout
+      - run: git submodule sync
+      - run: git submodule update --init --recursive
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: Docker Login
+          command: docker login -u $DOCKER_USER -p $DOCKER_PASS
+      - run:
+          name: Build Docker image
+          command: docker build -f perftests/bundlesrv/Dockerfile  -t $IMAGE_NAME:egb-build perftests/bundlesrv
+      - run:
+          name: Tag Docker image
+          command: docker tag $IMAGE_NAME:egb-build $IMAGE_NAME:egb-$CIRCLE_SHA1
+      - run:
+          name: Tag Docker image
+          command: docker tag $IMAGE_NAME:egb-build $IMAGE_NAME:egb-edge
+      - run:
+          name: Push Docker image
+          command: docker push $IMAGE_NAME:egb-$CIRCLE_SHA1
+      - run:
+          name: Push Docker image
+          command: docker push $IMAGE_NAME:egb-edge
   dgoss:
     environment:
       IMAGE_NAME: rfaircloth/scs
@@ -164,6 +199,7 @@ workflows:
   build-publish:
     jobs:
       - build
+      - build-egb
       - dgoss:
           requires:
             - build

diff --git a/.gitignore b/.gitignore
@@ -376,3 +376,4 @@ fabric.properties
 !/package/scripts/switch_transport.sh
 !/package/scripts/splunkmetrics.sh
 !/package/scripts/
+.ecs
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml
@@ -0,0 +1,77 @@
+#Splunk Connect for Syslog (SC4S) by Splunk, Inc.
+#
+#To the extent possible under law, the person who associated CC0 with
+#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights
+#to Splunk Connect for Syslog (SC4S).
+#
+#You should have received a copy of the CC0 legalcode along with this
+#work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
+version: "3"
+services:
+  sc4s:
+    image: rfaircloth/scs:edge
+    build:
+      context: ./package
+      args:
+        RH_ORG: ${RH_ORG}
+        RH_ACTIVATION: ${RH_ACTIVATION}
+    hostname: sc4s
+    ports:
+      - "514"
+      - "601"
+      - "514/udp"
+      - "5514"
+      - "5514/udp"
+    links:
+      - splunk
+    environment:
+      - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
+      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
+      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+      - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
+      - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
+      - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
+#    logging:
+#      driver: splunk
+#      options:
+#        splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
+#        splunk-url: https://splunk:8088/services/collector/event
+#        splunk-index: main
+#        splunk-insecureskipverify: true
+#        splunk-verify-connection: false
+  splunk:
+    image: splunk/splunk:latest
+    hostname: splunk
+    ports:
+      - "8000:8000"
+      - "8088:8088"
+      - "8089:8089"
+    environment:
+      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+      - SPLUNK_PASSWORD=${SPLUNK_PASSWORD}
+      - SPLUNK_START_ARGS=${SPLUNK_START_ARGS}
+      - SPLUNK_APPS_URL=${SPLUNK_APPS_URL}
+      - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME}
+      - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
+#    logging:
+#      driver: splunk
+#      options:
+#        splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
+#        splunk-url: https://splunk:8088/services/collector/event
+#        splunk-index: main
+#        splunk-insecureskipverify: true
+#        splunk-verify-connection: false
+  egbundles:
+    image: rfaircloth/scs:egb-edge
+    hostname: egbundles
+    build:
+      context: perftests/bundlesrv
+  eventgen:
+    image: rfaircloth/eventgen:edge
+    command: standalone
+    ports:
+      - "9500:9500"
+    links:
+      - egbundles
+      - sc4s
+
diff --git a/package/Dockerfile b/package/Dockerfile
@@ -76,21 +76,20 @@ RUN cd /tmp ;\
 
 ENV DEBCONF_NONINTERACTIVE_SEEN=true
 ENV SPLUNK_CONNECT_METHOD=hec
-ENV SYSLOGNG_HEC_WORKERS=3
+ENV SYSLOGNG_HEC_WORKERS=10
 
 RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh
 COPY goss.yaml /etc/goss.yaml
 
 COPY --from=0 /opt/syslog-ng /opt/syslog-ng
-COPY scripts/splunkmetrics.sh /opt/syslog-ng/sbin/splunkmetrics.sh
 COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf
 COPY etc/conf.d /opt/syslog-ng/etc/conf.d
 COPY etc/context /opt/syslog-ng/etc/context
 COPY sbin/entrypoint.sh /sbin/entrypoint.sh
 
+RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer
 RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng  -V
 RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t
-RUN mkdir -p /var/log/syslog-ng/data/disk-buffer
 
 EXPOSE 514
 EXPOSE 601/tcp

diff --git a/package/etc/conf.d/destinations/d_destinations.conf b/package/etc/conf.d/destinations/d_destinations.conf
@@ -17,18 +17,18 @@ destination d_hec {
          method("POST")
          log-fifo-size(`splunk-log-fifo-size`)
          workers(`SYSLOGNG_HEC_WORKERS`)
-         batch-lines(1000)
+         batch-lines(500)
          batch-bytes(512Kb)
-         batch-timeout(1000)
-         timeout(5)
+         batch-timeout(3)
+         timeout(15)
          user_agent("syslog-ng User Agent")
          user("syslog-ng")
          password("`SPLUNK_HEC_TOKEN`")
-#        persist-name("t_standard")
-#         disk-buffer(mem-buf-length(5000)
-#                     disk-buf-size(107374182400)
-#                     reliable(no)
-#                     dir("/var/log/syslog-ng/data/disk-buffer"))
+         persist-name("splunk")
+         disk-buffer(mem-buf-length(500)
+                     disk-buf-size(20000)
+                     reliable(no)
+                     dir("/opt/syslog-ng/var/data/disk-buffer/"))
          tls(peer-verify(no)
 #            ca-dir("dir")
 #            ca-file("ca")
@@ -48,3 +48,29 @@ destination d_hec {
                  fields.*)')
         );
     };
+
+
+destination d_hecmetrics {
+    http(
+         url("`SPLUNK_HEC_URL`")
+         method("POST")
+         batch-lines(5)
+         batch-bytes(512Kb)
+         batch-timeout(1)
+         timeout(15)
+         user_agent("syslog-ng User Agent")
+         user("syslog-ng")
+         password("`SPLUNK_HEC_TOKEN`")
+         persist-name("splunk_metrics")
+         tls(peer-verify(no)
+#            ca-dir("dir")
+#            ca-file("ca")
+#            cert-file("cert")
+#            cipher-suite("cipher")
+#            key-file("key")
+#            peer-verify(yes|no)
+#            ssl-version(<the permitted SSL/TLS version>)
+            )
+         body('$MESSAGE')
+        );
+    };
diff --git a/package/etc/conf.d/filters/cisco_asa_tradditional.conf b/package/etc/conf.d/filters/cisco_asa_tradditional.conf
@@ -20,5 +20,5 @@ log {
 
     destination(d_hec);  #--HEC--
 
-    flags(final);
+    flags(flow-control,final);
 };
diff --git a/package/etc/conf.d/filters/cisco_ios.conf b/package/etc/conf.d/filters/cisco_ios.conf
@@ -12,5 +12,5 @@ log {
     rewrite { r_set_splunk_basic(template("t_msg_only"))}; #--HEC--
     destination(d_hec);  #--HEC--
 
-    flags(final);
+    flags(flow-control,final);
 };
diff --git a/package/etc/conf.d/filters/fortinet_fgt.conf b/package/etc/conf.d/filters/fortinet_fgt.conf
@@ -36,5 +36,5 @@ log {
     rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC--
     destination(d_hec);  #--HEC--
 
-    flags(final);
+    flags(flow-control,final);
 };
diff --git a/package/etc/conf.d/filters/paloalto_networks.conf b/package/etc/conf.d/filters/paloalto_networks.conf
@@ -57,5 +57,5 @@ log {
 
     destination(d_hec);  #--HEC--
 
-    flags(final);
+    flags(flow-control,final);
 };
diff --git a/package/etc/context/splunk_index.csv b/package/etc/context/splunk_index.csv
@@ -26,5 +26,9 @@ pan:hipwatch,sourcetype,pan:hipwatch
 pan:correlation,sourcetype,pan:correlation
 pan:userid,sourcetype,pan:userid
 pan:unknown,sourcetype,pan:unknown
-syslogng-fallback,index,syslogng-fallback
-syslogng-fallback,sourcetype,syslogng-fallback
+syslog-ng:events,index,main
+syslog-ng:events,sourcetype,syslog-ng:events
+syslog-ng:fallback,index,main
+syslog-ng:fallback,sourcetype,syslog-ng:fallback
+syslog-ng:metrics,index,em_metrics
+syslog-ng:metrics,sourcetype,httpevent
diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf
@@ -15,15 +15,21 @@
 
 options {
         log_msg_size (65536);
-        flush_lines (0);
+        flush_lines (100);
         time_reopen (10);
         log_fifo_size (10000);
         chain_hostnames (off);
         use_dns (no);
         use_fqdn (no);
-        dns-cache(no);
         create_dirs (no);
         keep-hostname (yes);
+        create_dirs(yes);
+        dir_perm(0750);
+        stats-freq(30);
+        stats-level(1);
+        stats-max-dynamics(2000);
+        normalize-hostnames(yes);
+        on-error(fallback-to-string);
 };
 
 # ===============================================================================================
@@ -54,10 +60,10 @@ options {
 # ===============================================================================================
 @define ip-version 4
 @define splunk-max-connections 1000
-@define splunk-log-fifo-size 10000
-@define splunk-fetch-limit 100
+@define splunk-log-fifo-size 180000000
+@define splunk-fetch-limit 20000
 # make sure splunk-window-size >= splunk-max-connections * splunk-fetch-limit
-@define splunk-window-size 100000
+@define splunk-window-size 20000000
 @define splunk-rcvbuf 425984
 @define default-timezone "GMT"
 
@@ -70,6 +76,45 @@ options {
 @include "conf.d/sources/*.conf"
 @include "conf.d/blocks/*.conf"
 @include "conf.d/destinations/*.conf"
+
+log {
+    source(s_internal);
+
+    if (match("^Log statistics; " value("MESSAGE"))) {
+
+        parser {p_add_context_splunk(key("syslog-ng:metrics")); };
+        rewrite {
+            subst('(?:Log statistics; )?(?<Type>[^= ]+)=\x27(?<SourceName>[^\(]+)\((?<SourceId>[^,\)]+)(?:,(?<SourceInstance>[^,]+),(?<State>[^\)]+))?\)\=(?<Number>\d+)\x27(?:, )?',
+                    '
+                    {"time": "$S_UNIXTIME.$S_MSEC",
+                      "event": "metric",
+                      "host": "$HOST",
+                      "index": "${.splunk.index}",
+                      "source": "internal",
+                      "sourcetype": "${.splunk.sourcetype}",
+                      "fields": {
+                         "source_name": "${SourceName}",
+                         "source_instance": "${SourceInstance}",
+                         "state": "${State}",
+                         "type": "${Type}",
+                         "_value": ${Number},
+                         "metric_name": "syslogng.${SourceId}"
+                       }
+                     }
+                     ',
+                     value("MESSAGE") flags("utf8" "global")
+                  );
+        };
+        destination(d_hecmetrics);  #--HEC--
+    } else {
+        parser {p_add_context_splunk(key("syslog-ng:events")); };
+        rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC--
+        destination(d_hec);  #--HEC--
+    };
+    flags(final);
+};
+
+
 @include "conf.d/prefilters/*.conf"
 @include "conf.d/filters/*.conf"
 
@@ -79,9 +124,10 @@ options {
 
 log {
     source(s_default-ports);
+    parser {p_add_context_splunk(key("syslog-ng:fallback")); };
     rewrite { r_set_splunk(template("t_JSON"))}; #--HEC--
     destination(d_hec);  #--HEC--
-    flags(fallback);
+    flags(flow-control,fallback);
 };
 
 
diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh
@@ -7,5 +7,4 @@
 #Run syslog
 mkdir /opt/syslog-ng/var
 rm /opt/syslog-ng/var/syslog-ng.ctl || true
-/opt/syslog-ng/sbin/syslog-ng --process-mode=background
-/opt/syslog-ng/sbin/splunkmetrics.sh 
+/opt/syslog-ng/sbin/syslog-ng -F