Improve time stamp validation in test routines

CSVD · Mar 9, 2020 · 6fdbeb0 · 6fdbeb0
2 parents de96be8 + b30ffb3
commit 6fdbeb0
Show file tree

Hide file tree

Showing 45 changed files with 1,226 additions and 635 deletions.
diff --git a/docs/sources/PaloaltoNetworks/index.md b/docs/sources/PaloaltoNetworks/index.md
@@ -53,7 +53,6 @@ MSG Parse: This filter parses message content
 | SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_PALOALTO_PANOS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_PALOALTO_PANOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
-| SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS | no | Use custom time stamp parsing with ms added |
 
 ### Verification
 

diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf
@@ -31,7 +31,7 @@ template t_msg_trim {
 # ===============================================================================================
 
 template t_everything {
-        template("${ISODATE} ${HOST} ${MSGHDR}${MESSAGE}");
+        template("${ISODATE} ${HOST} ${LEGACY_MSGHDR}${MESSAGE}");
         };
 
 # ===============================================================================================

diff --git a/package/etc/conf.d/context/common_event_format_source.csv b/package/etc/conf.d/context/common_event_format_source.csv
@@ -1,4 +1,6 @@
 ArcSight_ArcSight,source,ArcSight:ArcSight
+Carbon Black_Protection,sourcetype,carbonblack:protection:cef
+Carbon Black_Protection,index,cb:cef
 Cyber-Ark_Vault,sourcetype,cyberark:epv:cef
 Cyber-Ark_Vault,index,netauth
 CyberArk_PTA,sourcetype,cyberark:pta:cef
@@ -9,4 +11,4 @@ Microsoft_Microsoft Windows,source,CEFEventLog:Microsoft Windows
 Microsoft_Microsoft Windows,index,oswinsec
 Incapsula_SIEMintegration,source,Imperva:Incapsula
 Incapsula_SIEMintegration,index,netwaf
-unknown,source,ArcSight:unknown
+unknown,source,CEF:unknown
diff --git a/package/etc/conf.d/filters/cisco/ios.conf b/package/etc/conf.d/filters/cisco/ios.conf
@@ -9,26 +9,30 @@ parser cisco-parser-ex{
     channel {
         filter {
             #message('^<\d*>(?:(?<ciscoseq>\d+)\: )?(?:(?<HOST>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(?<ciscorule>\d+): )?(?:(?<ciscotimereliable>\*)?(?<ciscotime>(?<time>\w\w\w {1,2}\d{1,2} \d\d:\d\d:\d\d)(?<ciscofrac>\.\d{3,6})? ?(?<ciscotz>\w+)?): )?(?:(?<ciscouptime>\d\d:\d\d:\d\d|\d{1,6} \d{1,2}): )?(?<cisomsg>(?<ciscoprogram>%.{2,15}\-\d{1,3}\-[^:]{3,}): (?<ciscodescription>.*))' flags(store-matches));            
-            message('^<\d*>(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?(?: [AP]M)? ?(?:\w+)?): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}): )?(%.{2,15}\-\d{1,3}\-[^:]+): (.*)' flags(store-matches));            
+            message('^<\d*>(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\*)?((?:\w\w\w {1,2}\d{1,2} (?:\d{2,4} )?\d\d:\d\d:\d\d)(?:\.\d{3,6})?( [AP]M)?)( \w+)?: )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}): )?(%.{2,15}\-\d{1,3}\-[^:]+): (.*)' flags(store-matches));
         };        
         if {
             #Mar  4 11:45:20
             #Apr 29 13:58:46.000001
             #Apr 29 13:58:46.411
-            #Mar  1 18:48:50.483 UTC
+            #Mar  1 18:48:50.483 UTC NOTE: Reverse TZ "%Z" parsing will not work for non-local timezones.
+            # guess-timezone() will be used to reconcile timezones
             parser {
-                    date-parser(format('%b %d %I:%M:%S %p.%f',
-                                    '%b %d %I:%M:%S %p',                                            
+                    date-parser(format(
                                     '%b %d %H:%M:%S.%f',
                                     '%b %d %H:%M:%S',
-                                    '%b %d %H:%M:%S %Z',
-                                    '%b %d %Y %H:%M:%S.%f',
-                                    '%b %d %Y %H:%M:%S.%f %Z',
-                                    '%b %d %Y %H:%M:%S')
-                template("$7")                        
+                                    '%b %d %I:%M:%S %p.%f',
+                                    '%b %d %I:%M:%S %p',
+                                    '%b %d %Y %H:%M:%S.%f'
+                                    '%b %d %Y %H:%M:%S',
+                                    )
+                template("$7")
                 flags(guess-timezone)
                 );
             };
+        } else {
+#           rewrite { set("date/time parser failed", value("fields.sc4s_error")); };
+            rewrite { set("date/time parser failed on string $7" value("fields.sc4s_error")); };
         };
         rewrite {
             set(
@@ -37,11 +41,11 @@ parser cisco-parser-ex{
                 condition(match('..' value('4')))
             );            
             set(        
-                "$9",
+                "$11",
                 value("PROGRAM")                
             );            
             set(
-                "$10",
+                "$12",
                 value("MSG")                
             );
         };

diff --git a/package/etc/conf.d/filters/cisco/ucm.conf b/package/etc/conf.d/filters/cisco/ucm.conf
@@ -7,26 +7,37 @@ filter f_cisco_ucm {
 
 filter f_cisco_ucm_message {
     message(
-        '^(<\d{1,3}>)\d*: (?:([^:]+): )?(.*) : (%.*)' 
+#        '^(<\d{1,3}>)\d*: (?:([^:]+): )?(.*[A|P]M\.\d{3}) (\w{3}) : (%.*)'
+#       '^(<\d{1,3}>)\d*: (?:([^:]+): )?(.*) : (%.*)'
+        '^(<\d{1,3}>)\d*: (?:([^:]+): )?(\w{3} \d{1,2} (\d{4} )?\d\d:\d\d:\d\d(?: [AP]M)?(\.\d{3,6})?)([^:]*): (%[A-Z].*)'
         flags(store-matches)
     );
 };
 
 parser p_cisco_ucm_date {
-    #Oct 14 2015 05:50:19 AM.484 UTC
-    #Apr 21 19:01:35.638 UTC
-    date-parser(format(
-        '%b %d %Y %I:%M:%S %p.%f %Z',
-        '%b %d %H:%M:%S.%f %Z'
-        )
-        template("$3")
-    );
+    channel {
+        #Oct 14 2015 05:50:19 AM.484 (timezone skipped; '%Z' date-parser only works for local TZ)
+        #Apr 21 19:01:35.638 (timezone skipped; '%Z' date-parser only works for local TZ)
+        if {
+            parser {
+                date-parser(format(
+                    '%b %d %H:%M:%S.%f',
+                    '%b %d %Y %I:%M:%S %p.%f'
+                    )
+                    template("$3")
+                    flags(guess-timezone)
+                );
+            };
+        } else {
+            rewrite { set("date/time parser failed on string $3" value("fields.sc4s_error")); };
+        };
+    };
 };
 
 rewrite r_cisco_ucm_message {
     set("cisco_ucm" value("fields.sc4s_syslog_format"));
     set("cisco_ucm" value("fields.sc4s_vendor_product"));
     set("$HOST_FROM" value("HOST") );
     set("$2" value("HOST") condition(match("^..." template("${2}"))) );
-    set("$4" value("MESSAGE"));
+    set("$7" value("MESSAGE"));
 };
diff --git a/package/etc/conf.d/filters/citrix/netscaler.conf.tmpl b/package/etc/conf.d/filters/citrix/netscaler.conf.tmpl
@@ -3,26 +3,28 @@ filter f_citrix_netscaler {
 };
 filter f_citrix_netscaler_message {
     message(
-        '^(<\d{1,3}>) (\d\d\/\d\d\/\d\d\d\d\:\d\d:\d\d:\d\d [^ ]{3}+) ([^ ]+) (.*)' 
+        '^(<\d{1,3}>) (\d\d\/\d\d\/\d\d\d\d\:\d\d:\d\d:\d\d) ([^ ]{3}+) ([^ ]+) (.*)'
         flags(store-matches)
     );
 };
 
 parser p_citrix_netscaler_date {
 {{- if (conv.ToBool (getenv "SC4S_SOURCE_CITRIX_NETSCALER_USEALT_DATE_FORMAT" "no")) }}
     #10/01/2001:01:01:01 GMT
-    date-parser(format('%m/%d/%Y:%H:%M:%S %Z')
-                template("$2"));
+    date-parser(format("%m/%d/%Y:%H:%M:%S")
+                template("$2")
+                flags(guess-timezone));
 {{- else }}
     #01/10/2001:01:01:01 GMT
-    date-parser(format('%d/%m/%Y:%H:%M:%S %Z')
-                template("$2"));
+    date-parser(format("%d/%m/%Y:%H:%M:%S")
+                template("$2")
+                flags(guess-timezone));
 {{- end }}
 };
 
 rewrite r_citrix_netscaler_message {
     set("citrix_netscaler" value("fields.sc4s_syslog_format"));
     set("citrix_netscaler" value("fields.sc4s_vendor_product"));
-    set("$3" value("HOST"));
-    set("$4" value("MESSAGE"));
+    set("$4" value("HOST"));
+    set("$5" value("MESSAGE"));
 };
diff --git a/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl
@@ -10,29 +10,36 @@ filter f_cisco_acs_complete{
     match("yes", value("ACS.COMPLETE") type(glob));
 };
 
-#This parser adds messages from ACS to a context without sending them
-#forward to Splunk
-parser acs_grouping {
+#This grabs the sequence numbers from the event; message gets group-parsed
+#if the total number of events (ACS.seq) is greater than 1
+
+parser acs_get_sequence {
     csv-parser(
         columns(PID, ACS.num, ACS.seq, MESSAGE)
         delimiters(chars(" "))
         flags(greedy)
     );
+};
+
+#This parser adds messages from ACS to a context without sending them on to Splunk
+
+parser acs_grouping {
     grouping-by(
         scope(program)
         key("$PID")
         trigger("$(+ ${ACS.seq} 1)" == "${ACS.num}")
         sort-key("${ACS.seq}")
         aggregate(
-             value("MESSAGE" "$(implode '' $(context-values ${MESSAGE}))")
-             value("ACS.COMPLETE" "yes")
+            value("MESSAGE" "$(implode '' $(context-values ${MESSAGE}))")
+            value("ACS.COMPLETE" "yes")
         )
-        timeout(10)
+        timeout(30)
     );
 };
 
-# The syslog message includes a date with nano seconds and TZ which is not in the header
-# So must reparse the date
+#The syslog message includes a date with milliseconds and TZ which is not in the header
+#So must reparse the date
+
 parser acs_event_time {
     csv-parser(
         columns(ACS.DATE, ACS.TIME, ACS.TZ, MESSAGE)
@@ -65,7 +72,14 @@ log {
         };
     };
 
-    parser(acs_grouping);
+# Do not run the events through the group parser if there is only one event
+    parser(acs_get_sequence);
+    if (match("1" value("ACS.num"))) {
+        rewrite { set("yes" value("ACS.COMPLETE"));
+        };
+    } else {
+        parser(acs_grouping);
+    };
 
     if {
         filter(f_cisco_acs_complete);
@@ -87,14 +101,6 @@ log {
         destination(d_archive);
 {{- end}}
 
-{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
-    {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
-{{- end }}
-
-{{- if (print (getenv "SC4S_DEST_CISCO_ACS_ALTERNATES")) }}
-    {{ getenv "SC4S_DEST_CISCO_ACS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
-{{- end }}
-
         flags(flow-control,final);
     };
 

diff --git a/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl
@@ -10,29 +10,34 @@ filter f_cisco_ise_complete{
     match("yes", value("ISE.COMPLETE") type(glob));
 };
 
-#This parser adds messages from ISE to a context without sending them
-#forward to Splunk
+#This grabs the sequence numbers from the event; message gets group-parsed
+#if the total number of events (ISE.seq) is greater than 1
 
-parser ise_grouping {
+parser ise_get_sequence {
     csv-parser(
         columns(PID, ISE.num, ISE.seq, MESSAGE)
         delimiters(chars(" "))
         flags(greedy)
     );
+};
+
+#This parser adds messages from ISE to a context without sending them on to Splunk
+
+parser ise_grouping {
     grouping-by(
         scope(program)
         key("$PID")
         trigger("$(+ ${ISE.seq} 1)" == "${ISE.num}")
         sort-key("${ISE.seq}")
         aggregate(
-             value("MESSAGE" "$(implode '' $(context-values ${MESSAGE}))")
-             value("ISE.COMPLETE" "yes")
+            value("MESSAGE" "$(implode '' $(context-values ${MESSAGE}))")
+            value("ISE.COMPLETE" "yes")
         )
-        timeout(10)
+        timeout(30)
     );
 };
 
-#The syslog message includes a date with nano seconds and TZ which is not in the header
+#The syslog message includes a date with milliseconds and TZ which is not in the header
 #So must reparse the date
 
 parser ise_event_time {
@@ -67,7 +72,14 @@ log {
         };
     };
 
-    parser(ise_grouping);
+# Do not run the events through the group parser if there is only one event
+    parser(ise_get_sequence);
+    if (match("1" value("ISE.num"))) {
+        rewrite { set("yes" value("ISE.COMPLETE"));
+        };
+    } else {
+        parser(ise_grouping);
+    };
 
     if {
         filter(f_cisco_ise_complete);

diff --git a/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl
@@ -39,15 +39,13 @@ log {
             prefix(".pan.")
             delimiters(',')
             );
+
         #2012/04/10 04:39:55
         #parse the date
-        date-parser(
-            {{- if (conv.ToBool (getenv "SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS")) }}
-                format("%Y/%m/%d %H:%M:%S.%f")
-            {{- else}}
-                format("%Y/%m/%d %H:%M:%S")
-            {{- end}}
-
+        date-parser(format(
+                '%Y/%m/%d %H:%M:%S.%f',
+                '%Y/%m/%d %H:%M:%S'
+                )
                 template("${.pan.GeneratedTime}")
                 time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})
                 flags(guess-timezone)

diff --git a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl
@@ -23,10 +23,8 @@ log {
     parser {
         #.jsonLog.Timestamp Mar 04 20:37:53 2020
                 date-parser(
-                format("%b %d %H:%M:%S %Y",
-                       "%h %d %H:%M:%S %Y",
-                       "%b %d %k:%M:%S %Y",
-                       "%h %d %k:%M:%S %Y")
+                format("%a %b %d %H:%M:%S %Y",
+                       "%a %b %d %k:%M:%S %Y")
                 template("${.json.LogTimestamp}")
                 time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})
                 flags(guess-timezone)

diff --git a/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl
@@ -35,7 +35,6 @@ log {
             #basic parsing
             kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}"));
         };
-
 
         if (match("dns" value(".kv.product"))) {
         rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"))};

diff --git a/pytest.ini b/pytest.ini
@@ -5,7 +5,7 @@ addopts =
     --splunk_type=docker
     --keepalive
     --splunk_hec_token=1ec3c8ac-74b3-46f2-ba44-a7c96b6ab236
-    -n=1
+    -n=10
 filterwarnings =
     ignore::DeprecationWarning
 junit_family=xunit1