Skip to content

Commit

Permalink
Merge pull request #340 from splunk/feature/palo-devicename
Browse files Browse the repository at this point in the history 
For Palo Alto networks devices use the device_name field as hostname
  • Loading branch information
Ryan Faircloth authored and GitHub committed Mar 12, 2020
2 parents 0d1aa11 + d2800a8 commit a4eee4c
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 24 deletions.
79 changes: 69 additions & 10 deletions package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,50 +35,109 @@ log {

#we need to actual even time from the field GeneratedTime. Use csv-parser to extract it.
csv-parser(
columns('FUTURE_USE', 'ReceiveTime', 'SerialNumber', 'Type', 'Subtype', 'FUTURE_USE2', 'GeneratedTime')
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time")
prefix(".pan.")
delimiters(',')
);

#2012/04/10 04:39:55
#parse the date
date-parser(format(
'%Y/%m/%d %H:%M:%S.%f',
'%Y/%m/%d %H:%M:%S'
)
template("${.pan.GeneratedTime}")
template("${.pan.generated_time}")
time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})
flags(guess-timezone)
);
};

#set the source type based on program field and lookup index from the splunk_context csv

if (message(',[0-9A-F]+,THREAT')) {
if (match('THREAT', value('.pan.type'))) {
parser {
csv-parser(
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","dest_ip","src_translated_ip","dest_translated_ip","rule","src_user","dest_user","app","vsys","src_zone","dest_zone","src_interface","dest_interface","log_forwarding_profile","future_use3","session_id","repeat_count","src_port","dest_port","src_translated_port","dest_translated_port","session_flags","transport","action","misc","threat","raw_category","severity","direction","sequence_number","action_flags","src_location","dest_location","future_use4","content_type","pcap_id","file_hash","cloud_address","url_index","user_agent","file_type","xff","referrer","sender","subject","recipient","report_id","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"))};
parser {p_add_context_splunk(key("pan_threat")); };
} elif (message(',[0-9A-F]+,TRAFFIC')) {
} elif (match('TRAFFIC', value('.pan.type'))) {
parser {
csv-parser(
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","dest_ip","src_translated_ip","dest_translated_ip","rule","src_user","dest_user","app","vsys","src_zone","dest_zone","src_interface","dest_interface","log_forwarding_profile","future_use3","session_id","repeat_count","src_port","dest_port","src_translated_port","dest_translated_port","session_flags","transport","action","bytes","bytes_out","bytes_in","packets","start_time","duration","http_category","future_use4","sequence_number","action_flags","src_location","dest_location","future_use5","packets_out","packets_in","session_end_reason","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"))};
parser {p_add_context_splunk(key("pan_traffic")); };
} elif (message(',[0-9A-F]+,SYSTEM')) {
} elif (match('SYSTEM', value('.pan.type'))) {
parser {
csv-parser(
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","vsys","event_id","object","future_use3","future_use4","module","severity","description","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"))};
parser {p_add_context_splunk(key("pan_system")); };
} elif (message(',[0-9A-F]+,CONFIG')) {
} elif (match('CONFIG', value('.pan.type'))) {
parser {
csv-parser(
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","host_name","vsys","command","admin","client","result","configuration_path","sequence_number","action_flags","before_change_detail","after_change_detail","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"))};
parser {p_add_context_splunk(key("pan_config")); };
} elif (message(',[0-9A-F]+,HIPWATCH')) {
} elif (match('HIPWATCH', value('.pan.type'))) {
parser {
csv-parser(
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_user","vsys","host_name","os","src_ip","hip_name","hip_count","hip_type","future_use3","future_use4","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"))};
parser {p_add_context_splunk(key("pan_hipwatch")); };
} elif (message(',[0-9A-F]+,CORRELATION')) {
} elif (match('CORRELATION', value('.pan.type'))) {
parser {
csv-parser(
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","src_user","vsys","category","severity","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name","vsys_id","object","object_id","evidence")
prefix(".pan.")
delimiters(',')
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"))};
parser {p_add_context_splunk(key("pan_correlation")); };
} elif (message(',[0-9A-F]+,USERID')) {
} elif (match('USERID', value('.pan.type'))) {
parser {
csv-parser(
columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","vsys","src_ip","source_name","event_id","repeat_count","timeout_threshold","src_port","dest_port","source","source_type","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name")
prefix(".pan.")
delimiters(',')
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"))};
parser {p_add_context_splunk(key("pan_userid")); };
} else {
parser {
csv-parser(
columns()
prefix(".pan.")
delimiters(',')
);
};
rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"))};
parser {p_add_context_splunk(key("pan_log")); };
};
rewrite {
set("${.pan.dvc_name}" value("HOST")
condition( match('^.' value('.pan.dvc_name') )) );
};

parser (compliance_meta_by_source);
rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
Expand Down
5 changes: 2 additions & 3 deletions tests/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,13 @@
#work. If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
version: "3.7"


services:
sc4s:
build:
context: ../package
hostname: sc4s
#When this is enabled test_common will fail
# command: -det
command: -det
ports:
- "514"
- "601"
Expand Down Expand Up @@ -60,4 +59,4 @@ volumes:
results:
external: false
splunk-var:
external: false
external: false
62 changes: 51 additions & 11 deletions tests/test_palo_alto.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@

env = Environment()

#<190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0
# <190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0


def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))

dt = datetime.datetime.now()
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
Expand All @@ -31,7 +34,38 @@ def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk, setup_

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"")
st = env.from_string(
"search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"")
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)

record_property("host", host)
record_property("resultCount", resultCount)
record_property("message", message)

assert resultCount == 1

# Oct 30 09:46:12 1,2012/10/30 09:46:12,01606001116,TRAFFIC,start,1,2012/04/10 04:39:58,192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:59,11449,1,59324,80,0,0,0x200000,tcp,allow,78,78,0,1,2012/04/10 04:39:59,0,any,0,0,0x0,192.168.0.0-192.168.255.255,United States,0,1,0
def test_palo_alto_traffic_dvc_name(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))

dt = datetime.datetime.now()
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)

# Tune time functions
time = dt.strftime("%Y/%m/%d %H:%M:%S")
epoch = epoch[:-7]

mt = env.from_string(
"{{ mark }} {{ bsd }} {{ host }}-no 1,{{ time }},007200C01056,TRAFFIC,start,1,{{ time }},192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,{{ time }},11449,1,59324,80,0,0,0x200000,tcp,allow,78,78,0,1,2012/04/10 04:39:59,0,any,0,0,0x0,192.168.0.0-192.168.255.255,United States,0,1,0,unknown,dg1,dg2,dg3,dg4,vsys_n13,{{ host }},action_source,src_vm,dest_vm,tunnel_id,tunnel_monitor_tag,tunnel_session_id,tunnel_start_time,tunnel_type\n")
message = mt.render(mark="<111>", bsd=bsd, host=host, time=time)

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string(
"search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"")
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -45,7 +79,8 @@ def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk, setup_

# <190>Oct 30 09:46:17 1,2012/10/30 09:46:17,01606001116,THREAT,url,1,2012/04/10 04:39:55,192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:57,22860,1,59303,80,0,0,0x208000,tcp,alert,"litetopdetect.cn/index.php",(9999),not-resolved,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,text/html
def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))

dt = datetime.datetime.now()
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
Expand All @@ -60,7 +95,8 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk, setup_s

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string("search _time={{ epoch }} index=netproxy host=\"{{ host }}\" sourcetype=\"pan:threat\"")
st = env.from_string(
"search _time={{ epoch }} index=netproxy host=\"{{ host }}\" sourcetype=\"pan:threat\"")
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -71,8 +107,10 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk, setup_s

assert resultCount == 1


def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))

dt = datetime.datetime.now()
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
Expand All @@ -87,7 +125,8 @@ def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"")
st = env.from_string(
"search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"")
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -99,9 +138,9 @@ def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk
assert resultCount == 1


@mark.skip()
def test_palo_alto_traffic_mstime(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))

dt = datetime.datetime.now()
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
Expand All @@ -112,12 +151,13 @@ def test_palo_alto_traffic_mstime(record_property, setup_wordlist, setup_splunk,
epoch = epoch[:-3]

mt = env.from_string(
"{{ mark }} {{ bsd }} {{ host }} 1,{{ time }},007200001056,TRAFFIC,end,1,{{ time }},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,{% now 'local', '%Y/%m/%d %H:%M:%S.%f' %},2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n")
"{{ mark }} {{ bsd }} {{ host }} 1,{{ time }},007200001056,TRAFFIC,end,1,{{ time }},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,{{ time }},2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n")
message = mt.render(mark="<111>", bsd=bsd, host=host, time=time)

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string("search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"")
st = env.from_string(
"search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\"")
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand Down

0 comments on commit a4eee4c

Please sign in to comment.