From a2d5a277d8e049853fef16508c04549e70df982b Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 12 Dec 2019 17:14:39 -0500 Subject: [PATCH] Update Paloalto for IETF format In most cases palo alto customers use BSD format over UDP for performance if a customer uses IETF. The firewall sends a malformed message which fails to parse the host. This change also add a feature flag --- docs/sources/PaloaltoNetworks/index.md | 1 + .../conf.d/conflib/_common/syslog_format.conf | 13 ++++++ .../etc/conf.d/filters/paloalto/panos.conf | 6 +++ .../p_rfc3164-paloalto_panos.conf.tmpl | 7 ++- package/etc/go_templates/source_network.t | 7 +++ tests/test_palo_alto.py | 43 +++++++++++++++++++ 6 files changed, 76 insertions(+), 1 deletion(-) diff --git a/docs/sources/PaloaltoNetworks/index.md b/docs/sources/PaloaltoNetworks/index.md index bc27602..814344b 100644 --- a/docs/sources/PaloaltoNetworks/index.md +++ b/docs/sources/PaloaltoNetworks/index.md @@ -53,6 +53,7 @@ MSG Parse: This filter parses message content | SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | | SC4S_ARCHIVE_PALOALTO_PANOS | no | Enable archive to disk for this specific source | | SC4S_DEST_PALOALTO_PANOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS | no | Use custom time stamp parsing with ms added | ### Verification diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index 5b69e71..49e6529 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -7,6 +7,9 @@ filter f_rfc5424_noversion{ filter f_rfc5424_epochtime{ message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) ))'); }; +filter f_rfc5424_bsdtime{ + message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?[^ ]+) ))'); +}; rewrite set_rfcnonconformant{ set("rfc5424_nonconform" value("fields.sc4s_syslog_format")); }; @@ -28,6 +31,12 @@ rewrite set_rfc5424_epochtime{ filter f_is_rfc5424_epochtime{ match("rfc5424_epochtime" value("fields.sc4s_syslog_format")) }; +rewrite set_rfc5424_bsdtime{ + set("rfc5424_bsdtime" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc5424_bsdtime{ + match("rfc5424_bsdtime" value("fields.sc4s_syslog_format")) +}; rewrite set_rfc3164{ set("rfc3164" value("fields.sc4s_syslog_format")); }; @@ -45,4 +54,8 @@ rewrite set_no_parse{ }; filter f_is_no_parse{ match("no_parse" value("fields.sc4s_syslog_format")) +}; + +rewrite set_rfc3894_bsdtime{ + subst('^<\d+>(\d)', "", value("MESSAGE")); }; \ No newline at end of file diff --git a/package/etc/conf.d/filters/paloalto/panos.conf b/package/etc/conf.d/filters/paloalto/panos.conf index da0cb04..6b645b1 100644 --- a/package/etc/conf.d/filters/paloalto/panos.conf +++ b/package/etc/conf.d/filters/paloalto/panos.conf @@ -1,3 +1,9 @@ +filter f_is_palalto_format{ + match("rfc5424_bsdtime" value("fields.sc4s_syslog_format")) + or + match("rfc3164" value("fields.sc4s_syslog_format")) +}; + filter f_paloalto_panos { message(',\d+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPWATCH|CORRELATION|USERID),'); }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 6ee0ad0..31b5013 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -6,7 +6,7 @@ log { {{- if eq (.) "yes"}} source(s_DEFAULT); - filter(f_is_rfc3164); + filter(f_is_palalto_format); filter(f_paloalto_panos); {{- end}} {{- if eq (.) "no"}} @@ -36,7 +36,12 @@ log { #2012/04/10 04:39:55 #parse the date date-parser( + {{- if ((getenv "SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS") | conv.ToBool) }} + format("%Y/%m/%d %H:%M:%S.%f") + {{- else}} format("%Y/%m/%d %H:%M:%S") + {{- end}} + template("${.pan.GeneratedTime}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index ff5be59..8a71d29 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -99,6 +99,13 @@ source s_{{ .port_id}} { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); + } elif { + filter(f_rfc5424_bsdtime); + rewrite(set_rfc3894_bsdtime); + parser { + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message, guess-timezone)); + }; + rewrite(set_rfc5424_bsdtime); } elif { parser (p_cisco_meraki); rewrite(set_rfc5424_epochtime); diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py index 2d40c8f..6211590 100644 --- a/tests/test_palo_alto.py +++ b/tests/test_palo_alto.py @@ -6,6 +6,7 @@ import random from jinja2 import Environment +from pytest import mark from .sendmessage import * from .splunkutils import * @@ -54,3 +55,45 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk): record_property("message", message) assert resultCount == 1 + +def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}1 {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +@mark.skip() +def test_palo_alto_traffic_mstime(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1