Merge pull request #282 from splunk/release/1.6.2

Release/1.6.2
CSVD · Jan 22, 2020 · 5351a45 · 5351a45
2 parents c381d2d + afd177f
commit 5351a45
Show file tree

Hide file tree

Showing 37 changed files with 498 additions and 613 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -8,7 +8,7 @@
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
 version: 2.1
 orbs:
-  docker: circleci/docker@0.5.13
+  docker: circleci/docker@0.5.20
   go: circleci/go@0.2.0
   snyk: snyk/snyk@0.0.8
   versioning: kollex/versioning@1.0.0
@@ -94,7 +94,7 @@ jobs:
       - run: git submodule sync
       - run: git submodule update --init --recursive
 
-      - docker/install-docker-tools
+      #- docker/install-docker-tools
       - docker/check:
           registry: $REGISTRY
           docker-username: GITHUB_USER
@@ -418,4 +418,4 @@ workflows:
             tags:
               only: /^\d*\.\d*\.\d*.*$/
             branches:
-              ignore: /.*/
+              ignore: /.*/
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -48,8 +48,11 @@ services:
 #     - SC4S_LISTEN_DEFAULT_UDP_PORT=514
 #     - SC4S_LISTEN_DEFAULT_TLS_PORT=6514
       - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
-      - SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT=6000
-#     - SC4S_ARCHIVE_CHECKPOINT=yes
+      - SC4S_LISTEN_CISCO_ASA_TCP_PORT=5001
+      - SC4S_LISTEN_CISCO_IOS_TCP_PORT=5002
+      - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003
+      - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004
+      - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005
       - SC4S_ARCHIVE_GLOBAL=yes
     volumes:
       - ./tls:/opt/syslog-ng/tls

diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf
@@ -4,15 +4,9 @@ filter f_rfc5424_strict{
 filter f_rfc5424_noversion{
     message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>) ?(?<TIMESTAMP>(?<FULLDATE>(?<FULLDATEYEAR>\d{4})-(?<FULLDATEMONTH>\d\d)-(?<FULLDATEDAY>\d\d))T(?<FULLTIME>(?<PARTIALTIME>(?<TIMEHOUR>[0-2]\d):(?<TIMEMINUTE>[0-5]\d):(?<TIMESECOND>[0-5]\d)(?:.(?<TIMESECFRAC>\d{1,6}))?)(?<TIMEOFFSET>Z|(?<TIMENUMOFFSET>[+\-][0-2]\d:[0-5]\d))))))');
 };
-filter f_rfc5424_epochtime{
-    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>(?<EPOCH>\d{10})(?:.(?<TIMESECFRAC>\d{1,9})?)) (?<FROMHOST>[^ ]+) ))');
-};
 filter f_rfc3164_version{
     message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?<FROMHOST>[^ ]+) ))');
 };
-rewrite set_rfcnonconformant{
-    set("rfc5424_nonconform" value("fields.sc4s_syslog_format"));
-};
 rewrite set_rfc5424_strict{
     set("rfc5424_strict" value("fields.sc4s_syslog_format"));
 };
@@ -28,9 +22,6 @@ filter f_is_rfc5424_noversion{
 rewrite set_rfc5424_epochtime{
     set("rfc5424_epochtime" value("fields.sc4s_syslog_format"));
 };
-filter f_is_rfc5424_epochtime{
-    match("rfc5424_epochtime" value("fields.sc4s_syslog_format"))
-};
 rewrite set_rfc3164_version{
     set("rfc3164_version" value("fields.sc4s_syslog_format"));
 };

diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl
@@ -6,7 +6,7 @@ destination d_hec {
          workers({{- getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10"}})
          batch-lines({{- getenv "SC4S_DEST_SPLUNK_HEC_BATCH_LINES" "1000"}})
          batch-bytes({{- getenv "SC4S_DEST_SPLUNK_HEC_BATCH_BYTES" "4096kb"}})
-         batch-timeout({{- getenv "SC4S_DEST_SPLUNK_HEC_BATCH_TIMEOUT" "1"}})
+         batch-timeout({{- getenv "SC4S_DEST_SPLUNK_HEC_BATCH_TIMEOUT" "3000"}})
          timeout({{- getenv "SC4S_DEST_SPLUNK_HEC_TIMEOUT" "30"}})
          user_agent("sc4s/1.0 (events)")
          user("sc4s")
@@ -45,4 +45,4 @@ destination d_hec {
                  event="$MSG"
                  fields.*)')
         );
-};
+};
diff --git a/package/etc/conf.d/filters/VMware/vsphere.conf b/package/etc/conf.d/filters/VMware/vsphere.conf
@@ -23,7 +23,8 @@ filter f_vmware_all {
     or program("NSX", flags(ignore-case))
     or program("NSXV", flags(ignore-case))
     or program("dfwpktlogs", flags(ignore-case))
-    or program("nsx-.*", flags(ignore-case))};
+    or program("nsx-.*", flags(ignore-case))
+};
 
 filter f_vmware_vsphere {
     program("cimslp", flags(ignore-case))

diff --git a/.../local/config/log_paths/example.conf.tmpl → ...cal/config/log_paths/lp-example.conf.tmpl b/.../local/config/log_paths/example.conf.tmpl → ...cal/config/log_paths/lp-example.conf.tmpl
@@ -29,24 +29,23 @@
 {{- $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
-{{- /* The following is an inline template to generate the actual log path */}}
-{{- define "log_path"}}
 log {
-
-{{- /* The first time this template is used the log_path will be linked to the default port */}}
-
-{{- if eq (.) "yes"}}
-    source(s_DEFAULT);
-    filter(f_is_rfc3164);
-    filter(f_local_example);
-{{- end}}
-
-{{- /* In the second pass through the template a link to the dedicated port is used. This */}}
-{{- /* normally does not require additional filters */}}
-
-{{- if eq (.) "no"}}
-    source (s_LOCAL_EXAMPLE);
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT")) (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic
+            source (s_LOCAL_EXAMPLE);
+            flags (final);
+	    };
 {{- end}}
+        channel {
+        # Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic
+            source (s_DEFAULT);
+            filter(f_is_rfc3164);
+            filter(f_local_example);
+            flags(final);
+        };
+    };
 
 # Set a default sourcetype and index, as well as an appropriate value for the field
 # "sc4s_vendor_product".  This field is sent as an indexed field to Splunk,
@@ -86,16 +85,4 @@ log {
 
 # All passes through any matching log path will be final
     flags(flow-control,final);
-};
-{{- end}}
-
-{{- /* Prepare to run two passes through this template, one for default traffic and another for */}}
-{{- /* "unique ports" if they are configured. */}}
-
-{{- if or (or (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT")) (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT")) }}
-# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic
-{{ tmpl.Exec "log_path" "no" }}
-{{- end }}
-
-# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic
-{{ tmpl.Exec "log_path" "yes" }}
+};
diff --git a/...ths/p_rfc3164-checkpoint_splunk.conf.tmpl → .../log_paths/lp-checkpoint_splunk.conf.tmpl b/...ths/p_rfc3164-checkpoint_splunk.conf.tmpl → .../log_paths/lp-checkpoint_splunk.conf.tmpl
@@ -1,19 +1,25 @@
 # Checkpoint
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }}
+{{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
-{{- /* The following is an inline template to generate the actual log path */}}
-{{- define "log_path"}}
 log {
-{{- if eq (.) "yes"}}
-    source(s_DEFAULT);
-    filter(f_is_rfc3164);
-    filter(f_checkpoint_splunk);
-{{- end}}
-{{- if eq (.) "no"}}
-    source (s_CHECKPOINT_SPLUNK);
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for CHECKPOINT_SPLUNK traffic
+            source (s_CHECKPOINT_SPLUNK);
+            flags (final);
+	    };
 {{- end}}
+        channel {
+        # Listen on the default port (typically 514) for CHECKPOINT_SPLUNK traffic
+            source(s_DEFAULT);
+            filter(f_is_rfc3164);
+            filter(f_checkpoint_splunk);
+            flags(final);
+        };
+    };
 
     if {
         filter(f_nix_syslog);
@@ -83,12 +89,4 @@ log {
 {{- end}}
 
     flags(flow-control,final);
-};
-{{- end}}
-{{- if or (or (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }}
-# Listen on the specified dedicated port(s) for CHECKPOINT_SPLUNK traffic
-{{ tmpl.Exec "log_path" "no" }}
-{{- end}}
-
-# Listen on the default port (typically 514) for CHECKPOINT_SPLUNK traffic
-{{ tmpl.Exec "log_path" "yes" }}
+};
diff --git a/...d/log_paths/p_rfc3164-cisco_acs.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_acs.conf.tmpl b/...d/log_paths/p_rfc3164-cisco_acs.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_acs.conf.tmpl
@@ -1,6 +1,6 @@
 # Cisco ACS
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }}
+{{- $context := dict "port_id" "CISCO_ACS" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 # This filter uses a field we set to prevent the original messages before aggregation from being
@@ -46,17 +46,24 @@ parser acs_event_time {
             template("${ACS.DATE} ${ACS.TIME} ${ACS.TZ}")
     );
 };
-{{- /* The following is an inline template to generate the actual log path */}}
-{{ define "log_path" }}
+
 log {
-{{- if eq (.) "yes"}}
-    source(s_DEFAULT);
-    filter(f_is_rfc3164);
-    filter(f_cisco_acs);
-{{- end}}
-{{- if eq (.) "no"}}
-    source (s_CISCO_ACS);
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ACS_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ACS_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ACS_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for CISCO_ACS traffic
+            source (s_CISCO_ACS);
+            flags (final);
+	    };
 {{- end}}
+        channel {
+        # Listen on the default port (typically 514) for CISCO_ACS traffic
+            source (s_DEFAULT);
+            filter(f_is_rfc3164);
+            filter(f_cisco_acs);
+            flags(final);
+        };
+    };
 
     parser(acs_grouping);
 
@@ -83,15 +90,4 @@ log {
         flags(flow-control,final);
     };
 
-
-};
-{{- end}}
-
-{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ACS_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ACS_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ACS_TLS_PORT")) }}
-
-# Listen on the specified dedicated port(s) for CISCO_ACS traffic
-    {{tmpl.Exec "log_path" "no" }}
-{{- end}}
-
-# Listen on the default port (typically 514) for CISCO_ACS traffic
-{{tmpl.Exec "log_path" "yes" }}
+};
diff --git a/...s/p_rfc5424-noversion_cisco_asa.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_asa.conf.tmpl b/...s/p_rfc5424-noversion_cisco_asa.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_asa.conf.tmpl
@@ -1,19 +1,25 @@
 # Cisco ASA RFC5424
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CISCO_ASA" "parser" "common" }}
+{{- $context := dict "port_id" "CISCO_ASA" "parser" "rfc5424_noversion" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
-{{- /* The following is an inline template to generate the actual log path */}}
-{{- define "log_path"}}
 log {
-{{- if eq (.) "yes"}}
-    source(s_DEFAULT);
-    filter(f_is_rfc5424_noversion);
-    filter(f_cisco_asa);
-{{- end}}
-{{- if eq (.) "no"}}
-    source (s_CISCO_ASA);
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for CISCO_ASA traffic
+            source (s_CISCO_ASA);
+            flags (final);
+	    };
 {{- end}}
+        channel {
+        # Listen on the default port (typically 514) for CISCO_ASA traffic
+            source (s_DEFAULT);
+            filter(f_is_rfc5424_noversion);
+            filter(f_cisco_asa);
+            flags(final);
+        };
+    };
 
     rewrite {
         set("cisco_asa", value("fields.sc4s_vendor_product"));
@@ -33,12 +39,3 @@ log {
 
     flags(flow-control,final);
 };
-{{- end}}
-
-{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT")) }}
-# Listen on the specified dedicated port(s) for CISCO_ASA traffic
-{{ tmpl.Exec "log_path" "no" }}
-{{- end}}
-
-# Listen on the default port (typically 514) for CISCO_ASA traffic
-{{ tmpl.Exec "log_path" "yes" }}
diff --git a/...d/log_paths/p_rfc3164-cisco_asa.conf.tmpl → ...d/log_paths/lp-cisco_asa_legacy.conf.tmpl b/...d/log_paths/p_rfc3164-cisco_asa.conf.tmpl → ...d/log_paths/lp-cisco_asa_legacy.conf.tmpl
@@ -1,19 +1,25 @@
 # Cisco ASA
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common" }}
+{{- $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
-{{- /* The following is an inline template to generate the actual log path */}}
-{{- define "log_path"}}
 log {
-{{- if eq (.) "yes"}}
-    source(s_DEFAULT);
-    filter(f_is_rfc3164);
-    filter(f_cisco_asa);
-{{- end}}
-{{- if eq (.) "no"}}
-    source (s_CISCO_ASA_LEGACY);
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for CISCO_ASA_LEGACY traffic
+            source (s_CISCO_ASA_LEGACY);
+            flags (final);
+	    };
 {{- end}}
+        channel {
+        # Listen on the default port (typically 514) for CISCO_ASA_LEGACY traffic
+            source (s_DEFAULT);
+            filter(f_is_rfc3164);
+            filter(f_cisco_asa);
+            flags(final);
+        };
+    };
 
     rewrite {
         set("cisco_asa", value("fields.sc4s_vendor_product"));
@@ -31,16 +37,5 @@ log {
     destination(d_archive);
 {{- end}}
 
-
     flags(flow-control,final);
-
 };
-{{- end}}
-{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT")) }}
-
-# Listen on the specified dedicated port(s) for CISCO_ASA_LEGACY traffic
-    {{tmpl.Exec "log_path" "no" }}
-{{- end}}
-
-# Listen on the default port (typically 514) for CISCO_ASA_LEGACY traffic
-{{tmpl.Exec "log_path" "yes" }}