Merge pull request #233 from splunk/feature/palo-mstime-and-nonietf

Improve palo alto support

docs/sources/PaloaltoNetworks/index.md

@@ -53,6 +53,7 @@ MSG Parse: This filter parses message content
 | SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_PALOALTO_PANOS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_PALOALTO_PANOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS | no | Use custom time stamp parsing with ms added |
 
 ### Verification
 

package/etc/conf.d/conflib/_common/syslog_format.conf

@@ -7,6 +7,9 @@ filter f_rfc5424_noversion{
 filter f_rfc5424_epochtime{
     message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>(?<EPOCH>\d{10})(?:.(?<TIMESECFRAC>\d{1,9})?)) (?<FROMHOST>[^ ]+) ))');
 };
+filter f_rfc5424_bsdtime{
+    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?<FROMHOST>[^ ]+) ))');
+};
 rewrite set_rfcnonconformant{
     set("rfc5424_nonconform" value("fields.sc4s_syslog_format"));
 };
@@ -28,6 +31,12 @@ rewrite set_rfc5424_epochtime{
 filter f_is_rfc5424_epochtime{
     match("rfc5424_epochtime" value("fields.sc4s_syslog_format"))
 };
+rewrite set_rfc5424_bsdtime{
+    set("rfc5424_bsdtime" value("fields.sc4s_syslog_format"));
+};
+filter f_is_rfc5424_bsdtime{
+    match("rfc5424_bsdtime" value("fields.sc4s_syslog_format"))
+};
 rewrite set_rfc3164{
     set("rfc3164" value("fields.sc4s_syslog_format"));
 };
@@ -45,4 +54,8 @@ rewrite set_no_parse{
 };
 filter f_is_no_parse{
     match("no_parse" value("fields.sc4s_syslog_format"))
+};
+
+rewrite set_rfc3894_bsdtime{
+    subst('^<\d+>(\d)', "", value("MESSAGE"));
 };

package/etc/conf.d/filters/paloalto/panos.conf

@@ -1,3 +1,9 @@
+filter f_is_palalto_format{
+    match("rfc5424_bsdtime" value("fields.sc4s_syslog_format"))
+    or
+    match("rfc3164" value("fields.sc4s_syslog_format"))
+};
+
 filter f_paloalto_panos {
     message(',\d+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPWATCH|CORRELATION|USERID),');
 };

package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl

@@ -6,7 +6,7 @@
 log {
 {{- if eq (.) "yes"}}
     source(s_DEFAULT);
-    filter(f_is_rfc3164);
+    filter(f_is_palalto_format);
     filter(f_paloalto_panos);
 {{- end}}
 {{- if eq (.) "no"}}
@@ -36,7 +36,12 @@ log {
         #2012/04/10 04:39:55
         #parse the date
         date-parser(
+            {{- if ((getenv "SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS") | conv.ToBool) }}
+                format("%Y/%m/%d %H:%M:%S.%f")
+            {{- else}}
                 format("%Y/%m/%d %H:%M:%S")
+            {{- end}}
+
                 template("${.pan.GeneratedTime}")
                 time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})
                 flags(guess-timezone)

package/etc/go_templates/source_network.t

@@ -99,6 +99,13 @@ source s_{{ .port_id}} {
         } elif {
             parser {cisco-parser()};
             rewrite(set_cisco_ios);
+        } elif {
+            filter(f_rfc5424_bsdtime);
+            rewrite(set_rfc3894_bsdtime);
+            parser {
+                    syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message, guess-timezone));
+                };
+            rewrite(set_rfc5424_bsdtime);
         } elif {
             parser (p_cisco_meraki);
             rewrite(set_rfc5424_epochtime);

tests/test_palo_alto.py

@@ -6,6 +6,7 @@
 import random
 
 from jinja2 import Environment
+from pytest import mark
 
 from .sendmessage import *
 from .splunkutils import *
@@ -54,3 +55,45 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk):
     record_property("message", message)
 
     assert resultCount == 1
+
+def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{ mark }}1 {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n")
+    message = mt.render(mark="<111>", host=host)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+
+@mark.skip()
+def test_palo_alto_traffic_mstime(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n")
+    message = mt.render(mark="<111>", host=host)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1