diff --git a/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl index 8f9907d..77c15a1 100644 --- a/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl @@ -35,18 +35,17 @@ log { #we need to actual even time from the field GeneratedTime. Use csv-parser to extract it. csv-parser( - columns('FUTURE_USE', 'ReceiveTime', 'SerialNumber', 'Type', 'Subtype', 'FUTURE_USE2', 'GeneratedTime') + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time") prefix(".pan.") delimiters(',') ); - #2012/04/10 04:39:55 #parse the date date-parser(format( '%Y/%m/%d %H:%M:%S.%f', '%Y/%m/%d %H:%M:%S' ) - template("${.pan.GeneratedTime}") + template("${.pan.generated_time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone) ); @@ -54,31 +53,91 @@ log { #set the source type based on program field and lookup index from the splunk_context csv - if (message(',[0-9A-F]+,THREAT')) { + if (match('THREAT', value('.pan.type'))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","dest_ip","src_translated_ip","dest_translated_ip","rule","src_user","dest_user","app","vsys","src_zone","dest_zone","src_interface","dest_interface","log_forwarding_profile","future_use3","session_id","repeat_count","src_port","dest_port","src_translated_port","dest_translated_port","session_flags","transport","action","misc","threat","raw_category","severity","direction","sequence_number","action_flags","src_location","dest_location","future_use4","content_type","pcap_id","file_hash","cloud_address","url_index","user_agent","file_type","xff","referrer","sender","subject","recipient","report_id","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".pan.") + delimiters(',') + ); + }; rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"))}; parser {p_add_context_splunk(key("pan_threat")); }; - } elif (message(',[0-9A-F]+,TRAFFIC')) { + } elif (match('TRAFFIC', value('.pan.type'))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","dest_ip","src_translated_ip","dest_translated_ip","rule","src_user","dest_user","app","vsys","src_zone","dest_zone","src_interface","dest_interface","log_forwarding_profile","future_use3","session_id","repeat_count","src_port","dest_port","src_translated_port","dest_translated_port","session_flags","transport","action","bytes","bytes_out","bytes_in","packets","start_time","duration","http_category","future_use4","sequence_number","action_flags","src_location","dest_location","future_use5","packets_out","packets_in","session_end_reason","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".pan.") + delimiters(',') + ); + }; rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"))}; parser {p_add_context_splunk(key("pan_traffic")); }; - } elif (message(',[0-9A-F]+,SYSTEM')) { + } elif (match('SYSTEM', value('.pan.type'))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","vsys","event_id","object","future_use3","future_use4","module","severity","description","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".pan.") + delimiters(',') + ); + }; rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"))}; parser {p_add_context_splunk(key("pan_system")); }; - } elif (message(',[0-9A-F]+,CONFIG')) { + } elif (match('CONFIG', value('.pan.type'))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","host_name","vsys","command","admin","client","result","configuration_path","sequence_number","action_flags","before_change_detail","after_change_detail","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".pan.") + delimiters(',') + ); + }; rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"))}; parser {p_add_context_splunk(key("pan_config")); }; - } elif (message(',[0-9A-F]+,HIPWATCH')) { + } elif (match('HIPWATCH', value('.pan.type'))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_user","vsys","host_name","os","src_ip","hip_name","hip_count","hip_type","future_use3","future_use4","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".pan.") + delimiters(',') + ); + }; rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"))}; parser {p_add_context_splunk(key("pan_hipwatch")); }; - } elif (message(',[0-9A-F]+,CORRELATION')) { + } elif (match('CORRELATION', value('.pan.type'))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","src_user","vsys","category","severity","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name","vsys_id","object","object_id","evidence") + prefix(".pan.") + delimiters(',') + ); + }; rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"))}; parser {p_add_context_splunk(key("pan_correlation")); }; - } elif (message(',[0-9A-F]+,USERID')) { + } elif (match('USERID', value('.pan.type'))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","vsys","src_ip","source_name","event_id","repeat_count","timeout_threshold","src_port","dest_port","source","source_type","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".pan.") + delimiters(',') + ); + }; rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"))}; parser {p_add_context_splunk(key("pan_userid")); }; } else { + parser { + csv-parser( + columns() + prefix(".pan.") + delimiters(',') + ); + }; rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"))}; parser {p_add_context_splunk(key("pan_log")); }; }; + rewrite { + set("${.pan.dvc_name}" value("HOST") + condition( match('^.' value('.pan.dvc_name') )) ); + }; parser (compliance_meta_by_source); rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 9966505..7a09dd8 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -8,14 +8,13 @@ #work. If not, see . version: "3.7" - services: sc4s: build: context: ../package hostname: sc4s #When this is enabled test_common will fail - # command: -det + command: -det ports: - "514" - "601" @@ -60,4 +59,4 @@ volumes: results: external: false splunk-var: - external: false \ No newline at end of file + external: false diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py index 419e1b9..25dbecc 100644 --- a/tests/test_palo_alto.py +++ b/tests/test_palo_alto.py @@ -14,9 +14,12 @@ env = Environment() -#<190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0 +# <190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0 + + def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) dt = datetime.datetime.now() iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) @@ -31,7 +34,38 @@ def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk, setup_ sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"") + st = env.from_string( + "search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# Oct 30 09:46:12 1,2012/10/30 09:46:12,01606001116,TRAFFIC,start,1,2012/04/10 04:39:58,192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:59,11449,1,59324,80,0,0,0x200000,tcp,allow,78,78,0,1,2012/04/10 04:39:59,0,any,0,0,0x0,192.168.0.0-192.168.255.255,United States,0,1,0 +def test_palo_alto_traffic_dvc_name(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + time = dt.strftime("%Y/%m/%d %H:%M:%S") + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }}-no 1,{{ time }},007200C01056,TRAFFIC,start,1,{{ time }},192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,{{ time }},11449,1,59324,80,0,0,0x200000,tcp,allow,78,78,0,1,2012/04/10 04:39:59,0,any,0,0,0x0,192.168.0.0-192.168.255.255,United States,0,1,0,unknown,dg1,dg2,dg3,dg4,vsys_n13,{{ host }},action_source,src_vm,dest_vm,tunnel_id,tunnel_monitor_tag,tunnel_session_id,tunnel_start_time,tunnel_type\n") + message = mt.render(mark="<111>", bsd=bsd, host=host, time=time) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"") search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -45,7 +79,8 @@ def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk, setup_ # <190>Oct 30 09:46:17 1,2012/10/30 09:46:17,01606001116,THREAT,url,1,2012/04/10 04:39:55,192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:57,22860,1,59303,80,0,0,0x208000,tcp,alert,"litetopdetect.cn/index.php",(9999),not-resolved,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,text/html def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) dt = datetime.datetime.now() iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) @@ -60,7 +95,8 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk, setup_s sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netproxy host=\"{{ host }}\" sourcetype=\"pan:threat\"") + st = env.from_string( + "search _time={{ epoch }} index=netproxy host=\"{{ host }}\" sourcetype=\"pan:threat\"") search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -71,8 +107,10 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk, setup_s assert resultCount == 1 + def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) dt = datetime.datetime.now() iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) @@ -87,7 +125,8 @@ def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"") + st = env.from_string( + "search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"") search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -99,9 +138,9 @@ def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk assert resultCount == 1 -@mark.skip() def test_palo_alto_traffic_mstime(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + host = "{}-{}".format(random.choice(setup_wordlist), + random.choice(setup_wordlist)) dt = datetime.datetime.now() iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) @@ -112,12 +151,13 @@ def test_palo_alto_traffic_mstime(record_property, setup_wordlist, setup_splunk, epoch = epoch[:-3] mt = env.from_string( - "{{ mark }} {{ bsd }} {{ host }} 1,{{ time }},007200001056,TRAFFIC,end,1,{{ time }},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,{% now 'local', '%Y/%m/%d %H:%M:%S.%f' %},2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n") + "{{ mark }} {{ bsd }} {{ host }} 1,{{ time }},007200001056,TRAFFIC,end,1,{{ time }},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,{{ time }},2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n") message = mt.render(mark="<111>", bsd=bsd, host=host, time=time) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"") + st = env.from_string( + "search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"") search = st.render(epoch=epoch, host=host) resultCount, eventCount = splunk_single(setup_splunk, search)