Merge pull request #279 from splunk/update/network_source

Update network source template/log path unique port parsers

docker-compose.yml

@@ -48,8 +48,11 @@ services:
 #     - SC4S_LISTEN_DEFAULT_UDP_PORT=514
 #     - SC4S_LISTEN_DEFAULT_TLS_PORT=6514
       - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
-      - SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT=6000
-#     - SC4S_ARCHIVE_CHECKPOINT=yes
+      - SC4S_LISTEN_CISCO_ASA_TCP_PORT=5001
+      - SC4S_LISTEN_CISCO_IOS_TCP_PORT=5002
+      - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003
+      - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004
+      - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005
       - SC4S_ARCHIVE_GLOBAL=yes
     volumes:
       - ./tls:/opt/syslog-ng/tls

package/etc/conf.d/conflib/_common/syslog_format.conf

@@ -4,15 +4,9 @@ filter f_rfc5424_strict{
 filter f_rfc5424_noversion{
     message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>) ?(?<TIMESTAMP>(?<FULLDATE>(?<FULLDATEYEAR>\d{4})-(?<FULLDATEMONTH>\d\d)-(?<FULLDATEDAY>\d\d))T(?<FULLTIME>(?<PARTIALTIME>(?<TIMEHOUR>[0-2]\d):(?<TIMEMINUTE>[0-5]\d):(?<TIMESECOND>[0-5]\d)(?:.(?<TIMESECFRAC>\d{1,6}))?)(?<TIMEOFFSET>Z|(?<TIMENUMOFFSET>[+\-][0-2]\d:[0-5]\d))))))');
 };
-filter f_rfc5424_epochtime{
-    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>(?<EPOCH>\d{10})(?:.(?<TIMESECFRAC>\d{1,9})?)) (?<FROMHOST>[^ ]+) ))');
-};
 filter f_rfc3164_version{
     message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?<FROMHOST>[^ ]+) ))');
 };
-rewrite set_rfcnonconformant{
-    set("rfc5424_nonconform" value("fields.sc4s_syslog_format"));
-};
 rewrite set_rfc5424_strict{
     set("rfc5424_strict" value("fields.sc4s_syslog_format"));
 };
@@ -28,9 +22,6 @@ filter f_is_rfc5424_noversion{
 rewrite set_rfc5424_epochtime{
     set("rfc5424_epochtime" value("fields.sc4s_syslog_format"));
 };
-filter f_is_rfc5424_epochtime{
-    match("rfc5424_epochtime" value("fields.sc4s_syslog_format"))
-};
 rewrite set_rfc3164_version{
     set("rfc3164_version" value("fields.sc4s_syslog_format"));
 };

package/etc/conf.d/filters/VMware/vsphere.conf

@@ -23,7 +23,8 @@ filter f_vmware_all {
     or program("NSX", flags(ignore-case))
     or program("NSXV", flags(ignore-case))
     or program("dfwpktlogs", flags(ignore-case))
-    or program("nsx-.*", flags(ignore-case))};
+    or program("nsx-.*", flags(ignore-case))
+};
 
 filter f_vmware_vsphere {
     program("cimslp", flags(ignore-case))

package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl

@@ -1,6 +1,6 @@
 # Checkpoint
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }}
+{{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl

@@ -1,6 +1,6 @@
 # Cisco ACS
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }}
+{{- $context := dict "port_id" "CISCO_ACS" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 # This filter uses a field we set to prevent the original messages before aggregation from being
@@ -49,15 +49,15 @@ parser acs_event_time {
 
 log {
     junction {
-{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT")) }}
+{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ACS_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ACS_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ACS_TLS_PORT")) }}
         channel {
-        # Listen on the specified dedicated port(s) for CISCO_NX_OS traffic
-            source (s_CISCO_NX_OS);
+        # Listen on the specified dedicated port(s) for CISCO_ACS traffic
+            source (s_CISCO_ACS);
             flags (final);
 	    };
 {{- end}}
         channel {
-        # Listen on the default port (typically 514) for CISCO_NX_OS traffic
+        # Listen on the default port (typically 514) for CISCO_ACS traffic
             source (s_DEFAULT);
             filter(f_is_rfc3164);
             filter(f_cisco_acs);

package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl

@@ -1,6 +1,6 @@
 # Cisco ASA RFC5424
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CISCO_ASA" "parser" "common" }}
+{{- $context := dict "port_id" "CISCO_ASA" "parser" "rfc5424_noversion" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl

@@ -1,6 +1,6 @@
 # Cisco ASA
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common" }}
+{{- $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl

@@ -1,6 +1,6 @@
 # Cisco IOS
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CISCO_IOS" "parser" "common" }}
+{{- $context := dict "port_id" "CISCO_IOS" "parser" "cisco_parser" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl

@@ -1,6 +1,6 @@
 # Cisco ISE
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CISCO_ISE" "parser" "common" }}
+{{- $context := dict "port_id" "CISCO_ISE" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 # This filter uses a field we set to prevent the original messages before aggregation from being

package/etc/conf.d/log_paths/lp-epoch-cisco_meraki.conf.tmpl → package/etc/conf.d/log_paths/lp-cisco_meraki.conf.tmpl

@@ -1,6 +1,6 @@
 # Cisco Meraki
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }}
+{{- $context := dict "port_id" "CISCO_MERAKI" "parser" "cisco_meraki_parser" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl

@@ -1,6 +1,6 @@
 # Cisco NX_OS
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }}
+{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-forcepoint_webprotect.conf.tmpl

@@ -1,6 +1,6 @@
 # Forcepoint Webprotect
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }}
+{{- $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-fortinet_fortios.conf.tmpl

@@ -1,6 +1,6 @@
 # Fortinet Fortios
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "FORTINET_FORTIOS" "parser" "common" }}
+{{- $context := dict "port_id" "FORTINET_FORTIOS" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-infoblox.conf.tmpl

@@ -1,6 +1,6 @@
 # Infoblox
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "INFOBLOX" "parser" "common" }}
+{{- $context := dict "port_id" "INFOBLOX" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-juniper_idp.conf.tmpl

@@ -1,6 +1,6 @@
 # Juniper IDP
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "JUNIPER_IDP" "parser" "common" }}
+{{- $context := dict "port_id" "JUNIPER_IDP" "parser" "rfc5424_strict" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-juniper_junos.conf.tmpl

@@ -1,6 +1,6 @@
 # Juniper JunOS
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "JUNIPER_JUNOS" "parser" "common" }}
+{{- $context := dict "port_id" "JUNIPER_JUNOS" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl

@@ -1,6 +1,6 @@
 # Juniper JunOS (Structured, RFC5424-compliant)
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "JUNIPER_JUNOS_STRUCTURED" "parser" "common" }}
+{{- $context := dict "port_id" "JUNIPER_JUNOS_STRUCTURED" "parser" "rfc5424_strict" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-juniper_netscreen.conf.tmpl

@@ -1,6 +1,6 @@
 # Juniper Netscreen
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "JUNIPER_NETSCREEN" "parser" "common" }}
+{{- $context := dict "port_id" "JUNIPER_NETSCREEN" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-juniper_nsm.conf.tmpl

@@ -1,6 +1,6 @@
 # Juniper NSM
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "JUNIPER_NSM" "parser" "common" }}
+{{- $context := dict "port_id" "JUNIPER_NSM" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-juniper_nsm_idp.conf.tmpl

@@ -1,6 +1,6 @@
 # Juniper NSM IDP
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "JUNIPER_NSM_IDP" "parser" "common" }}
+{{- $context := dict "port_id" "JUNIPER_NSM_IDP" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl

@@ -1,6 +1,6 @@
 # Microfocus ArcSight
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common" }}
+{{- $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 parser p_microfocus_arcsight_header {

package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl

@@ -1,6 +1,6 @@
 # PaloAlto PanOS
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "PALOALTO_PANOS" "parser" "common" }}
+{{- $context := dict "port_id" "PALOALTO_PANOS" "parser" "rfc3164_version" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-proofpoint_pps.conf.tmpl

@@ -1,6 +1,6 @@
 # Proofpoint Protection Server
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "PROOFPOINT_PPS" "parser" "common" }}
+{{- $context := dict "port_id" "PROOFPOINT_PPS" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-symantec_brightmail.conf.tmpl

@@ -24,7 +24,7 @@ parser symantec_brightmail_grouping {
 {{- end }}
 
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "common" }}
+{{- $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl

@@ -1,6 +1,6 @@
 # Symantec Proxy (Bluecoat)
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "SYMANTEC_PROXY" "parser" "common" }}
+{{- $context := dict "port_id" "SYMANTEC_PROXY" "parser" "rfc5424_noversion" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-ubiquiti_unifi.conf.tmpl

@@ -1,6 +1,6 @@
 # Ubiquiti unifi
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "common" }}
+{{- $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl

@@ -1,6 +1,6 @@
 # Zscaler
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }}
+{{- $context := dict "port_id" "ZSCALER_NSS" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/conf.d/log_paths/lp-zzy-nix_syslog.conf.tmpl

@@ -1,6 +1,6 @@
 # Linux/Unix OS system logs
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }}
+{{- $context := dict "port_id" "NIX_SYSLOG" "parser" "rfc3164" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 log {

package/etc/go_templates/source_network.t

@@ -58,35 +58,48 @@ source s_{{ .port_id }} {
             );
 {{- end}}
         };
-        #TODO: #60 Remove this function with enhancement
-        rewrite(set_rfcnonconformant);
-{{- if eq .parser "rfc5424_strict" }}
-        filter(f_rfc5424_strict);
+{{ if eq .parser "rfc3164" }}
+        parser {
+            syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
+        };
+        rewrite(set_rfc3164);
+{{ else if eq .parser "rfc3164_version" }}
+#       filter(f_rfc3164_version);
+        rewrite(set_rfc3164_no_version_string);
+        parser {
+            syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
+        };
+        rewrite(set_rfc3164_version);
+{{ else if eq .parser "rfc5424_strict" }}
+#       filter(f_rfc5424_strict);
         parser {
                 syslog-parser(flags(syslog-protocol));
             };
         rewrite(set_rfc5424_strict);
-{{- else if eq .parser "rfc5424_noversion" }}
-        filter(f_rfc5424_noversion);
+{{ else if eq .parser "rfc5424_noversion" }}
+#       filter(f_rfc5424_noversion);
         parser {
                 syslog-parser(flags(syslog-protocol));
             };
         rewrite(set_rfc5424_noversion);
-{{- else if eq .parser "cisco_parser" }}
+{{ else if eq .parser "cisco_parser" }}
         parser {cisco-parser()};
         rewrite(set_cisco_ios);
-{{- else if eq .parser "cisco_meraki_parser" }}
+{{ else if eq .parser "cisco_meraki_parser" }}
         parser (p_cisco_meraki);
         rewrite(set_rfc5424_epochtime);
-{{- else if eq .parser "rfc3164" }}
-        parser {
-            syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) );
-        };
-        rewrite(set_rfc3164);
-{{- else if eq .parser "no_parse" }}
+{{ else if eq .parser "no_parse" }}
         rewrite(set_no_parse);
-{{- else }}
-        if {filter(f_rfc5424_strict);
+{{ else }}
+        if {
+            filter(f_rfc3164_version);
+            rewrite(set_rfc3164_no_version_string);
+            parser {
+                    syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
+                };
+            rewrite(set_rfc3164_version);
+        } elif {
+            filter(f_rfc5424_strict);
             parser {
                     syslog-parser(flags(syslog-protocol));
                 };
@@ -100,13 +113,6 @@ source s_{{ .port_id }} {
         } elif {
             parser {cisco-parser()};
             rewrite(set_cisco_ios);
-        } elif {
-            filter(f_rfc3164_version);
-            rewrite(set_rfc3164_no_version_string);
-            parser {
-                    syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
-                };
-            rewrite(set_rfc3164_version);
         } elif {
             parser (p_cisco_meraki);
             rewrite(set_rfc5424_epochtime);
@@ -116,9 +122,8 @@ source s_{{ .port_id }} {
             };
             rewrite(set_rfc3164);
         };
-{{- end }}
+{{ end }}
         rewrite(r_set_splunk_default);
-
         parser {
             vendor_product_by_source();
         };