Skip to content

Commit

Permalink
Fixes #554 handle commas inside double quoted fields (#560)
Browse files Browse the repository at this point in the history
  • Loading branch information
Ryan Faircloth authored and GitHub committed Jul 10, 2020
1 parent 68a2383 commit 95cdafe
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 1 deletion.
16 changes: 16 additions & 0 deletions package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ log {
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","dest_ip","src_translated_ip","dest_translated_ip","rule","src_user","dest_user","app","vsys","src_zone","dest_zone","src_interface","dest_interface","log_forwarding_profile","future_use3","session_id","repeat_count","src_port","dest_port","src_translated_port","dest_translated_port","session_flags","transport","action","misc","threat","raw_category","severity","direction","sequence_number","action_flags","src_location","dest_location","future_use4","content_type","pcap_id","file_hash","cloud_address","url_index","user_agent","file_type","xff","referrer","sender","subject","recipient","report_id","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
quote-pairs('""')
flags(escape-double-char)
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"))};
Expand All @@ -66,6 +68,8 @@ log {
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","dest_ip","src_translated_ip","dest_translated_ip","rule","src_user","dest_user","app","vsys","src_zone","dest_zone","src_interface","dest_interface","log_forwarding_profile","future_use3","session_id","repeat_count","src_port","dest_port","src_translated_port","dest_translated_port","session_flags","transport","action","bytes","bytes_out","bytes_in","packets","start_time","duration","http_category","future_use4","sequence_number","action_flags","src_location","dest_location","future_use5","packets_out","packets_in","session_end_reason","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
quote-pairs('""')
flags(escape-double-char)
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"))};
Expand All @@ -76,6 +80,8 @@ log {
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","vsys","event_id","object","future_use3","future_use4","module","severity","description","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
quote-pairs('""')
flags(escape-double-char)
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:system"))};
Expand All @@ -86,6 +92,8 @@ log {
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","host_name","vsys","command","admin","client","result","configuration_path","sequence_number","action_flags","before_change_detail","after_change_detail","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
quote-pairs('""')
flags(escape-double-char)
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:config"))};
Expand All @@ -96,6 +104,8 @@ log {
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_user","vsys","host_name","os","src_ip","hip_name","hip_count","hip_type","future_use3","future_use4","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
quote-pairs('""')
flags(escape-double-char)
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:hipmatch"))};
Expand All @@ -106,6 +116,8 @@ log {
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","src_user","vsys","category","severity","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name","vsys_id","object","object_id","evidence")
prefix(".pan.")
delimiters(',')
quote-pairs('""')
flags(escape-double-char)
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"))};
Expand All @@ -116,6 +128,8 @@ log {
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","vsys","src_ip","source_name","event_id","repeat_count","timeout_threshold","src_port","dest_port","source","source_type","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
quote-pairs('""')
flags(escape-double-char)
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"))};
Expand All @@ -126,6 +140,8 @@ log {
columns()
prefix(".pan.")
delimiters(',')
quote-pairs('""')
flags(escape-double-char)
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:log"))};
Expand Down
30 changes: 29 additions & 1 deletion tests/test_palo_alto.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

# <190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0


def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))
Expand Down Expand Up @@ -107,6 +106,35 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk, setup_s

assert resultCount == 1

# <190>Jan 28 01:28:35 fooooo 1,2020/07/08 16:48:50,013201020735,THREAT,url,2049,2020/07/08 16:48:48,10.1.1.1,1.1.1.2,1.1.1.1,1.1.1.3,URLFilter_CatchAll_Internet,testuser,,arcgis,vsys1,DMZ,Outside,ae3,ae1,Panorama-Only,2020/07/08 16:48:48,357728,1,61066,80,33396,80,0x8403000,tcp,alert,"geocode.arcgis.com/arcgis/rest/services/World/GeocodeServer/reverseGeocode?distance=100&f=json&location={""x"":-33,""y"":22.3,""spatialReference"":{""wkid"":111}}",(9999),ALL-WhitelistedURLs,informational,client-to-server,6816029286804555581,0xa000000000000000,Internal,United States,0,application/json,0,,,1,,,,,,,,0,11,16,0,0,,TESTFW01,,,,get,0,,0,,N/A,unknown,AppThreat-0-0,0x0,0,4294967295,
def test_palo_alto_threat2(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))

dt = datetime.datetime.now()
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)

# Tune time functions
time = dt.strftime("%Y/%m/%d %H:%M:%S")
epoch = epoch[:-7]

mt = env.from_string(
'{{ mark }} {{ bsd }} {{ host }} 1,{{ time }},01606001116,THREAT,url,1,{{ time }},10.1.1.1,1.1.1.2,1.1.1.1,1.1.1.3,URLFilter_CatchAll_Internet,testuser,,arcgis,vsys1,DMZ,Outside,ae3,ae1,Panorama-Only,2020/07/08 16:48:48,357728,1,61066,80,33396,80,0x8403000,tcp,alert,"geocode.arcgis.com/arcgis/rest/services/World/GeocodeServer/reverseGeocode?distance=100&f=json&location={""x"":-33,""y"":22.3,""spatialReference"":{""wkid"":111}}",(9999),ALL-WhitelistedURLs,informational,client-to-server,6816029286804555581,0xa000000000000000,Internal,United States,0,application/json,0,,,1,,,,,,,,0,11,16,0,0,,{{ host }},,,,get,0,,0,,N/A,unknown,AppThreat-0-0,0x0,0,4294967295,\n')
message = mt.render(mark="<111>", bsd=bsd, host=host, time=time)

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string(
"search _time={{ epoch }} index=netproxy host=\"{{ host }}\" sourcetype=\"pan:threat\"")
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)

record_property("host", host)
record_property("resultCount", resultCount)
record_property("message", message)

assert resultCount == 1

def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist),
Expand Down

0 comments on commit 95cdafe

Please sign in to comment.