From 9b0945397ce7408b711263550cf792272d55e8dc Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Sun, 29 Sep 2019 12:47:09 -0400 Subject: [PATCH] Add support for CEF syslog (#102) Support CEF format --- .gitmodules | 1 - docs/sources.md | 146 ++++++++++++++++++ .../etc/conf.d/conflib/_common/templates.conf | 7 + .../conf.d/filters/microfocus/arcsight.conf | 4 + .../p_rfc3164_microfocus_arcsight.conf.tmpl | 84 ++++++++++ .../microfocus_arcsight_source.csv | 8 + package/etc/context-local/splunk_index.csv | 4 + .../apps/SA-syslog-ng/default/indexes.conf | 14 ++ tests/test_microfocus_arcsight_cef.py | 137 ++++++++++++++++ 9 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 package/etc/conf.d/filters/microfocus/arcsight.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl create mode 100644 package/etc/context-local/microfocus_arcsight_source.csv create mode 100644 tests/test_microfocus_arcsight_cef.py diff --git a/.gitmodules b/.gitmodules index 61760d8..a25cb07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,5 +1,4 @@ [submodule "package/syslog-ng"] path = package/syslog-ng url = https://github.com/balabit/syslog-ng.git -[submodule "Submod"] branch = syslog-ng-3.23.1 diff --git a/docs/sources.md b/docs/sources.md index 19f26d1..7919645 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -233,6 +233,55 @@ index= (sourcetype=fgt_log OR sourcetype=fgt_traffic OR sourcetype Verify timestamp, and host values match as expected +# Vendor - Imperva + +## Product - Incapsula + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/ | +| Splunk Add-on Source Specific | https://bitbucket.org/SPLServices/ta-cef-imperva-incapsula/downloads/ | +| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cef | Common sourcetype | + +### Source + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| Imperva:Incapsula | Common sourcetype | + +### Index Configuration + +| key | source | index | notes | +|----------------|----------------|----------------|----------------| +| cef_Incapsula_SIEMintegration | Imperva:Incapsula | netwaf | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active site will generate frequent events use the following search to check for new events + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=cef source="Imperva:Incapsula") +``` + # Vendor - Juniper ## Product - Juniper JunOS @@ -432,6 +481,103 @@ index= sourcetype=juniper:sslvpn | stats count by host Verify timestamp, and host values match as expected +# Vendor - Microfocus ArcSight + +## Product - Internal Agent Events + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/ | +| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cef | Common sourcetype | + +### Source + +| source | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| ArcSight:ArcSight | Internal logs | + +### Index Configuration + +| key | source | index | notes | +|----------------|----------------|----------------|----------------| +| cef_ArcSight_ArcSight | ArcSight:ArcSight | main | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active site will generate frequent events use the following search to check for new events + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=cef source="ArcSight:ArcSight") +``` + +## Product - Microsoft Windows + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/ | +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-microsoft-windows-for-splunk/downloads/ | +| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cef | Common sourcetype | + +### Source + +| source | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| CEFEventLog:System or Application Event | Windows Application and System Event Logs | +| CEFEventLog:Microsoft Windows | Windows Security Event Logs | + +### Index Configuration + +| key | source | index | notes | +|----------------|----------------|----------------|----------------| +| cef_Microsoft_System or Application Event | CEFEventLog:System or Application Event | oswin | none | +| cef_Microsoft_Microsoft Windows | CEFEventLog:Microsoft Windows | oswinsec | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active site will generate frequent events use the following search to check for new events + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=cef (source="CEFEventLog:Microsoft Windows" OR source="CEFEventLog:System or Application Event")) +``` + # Vendor - PaloAlto ## Product - NGFW diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index 5efd586..a08f36d 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -20,6 +20,13 @@ template t_msg_only { }; # =============================================================================================== +# Message Only trim leading or trailing white space +# =============================================================================================== + +template t_msg_trim { + template("$(strip $MSGONLY )"); + }; +# =============================================================================================== # Everything; useful for bluecoat proxy and others # =============================================================================================== diff --git a/package/etc/conf.d/filters/microfocus/arcsight.conf b/package/etc/conf.d/filters/microfocus/arcsight.conf new file mode 100644 index 0000000..287d7a4 --- /dev/null +++ b/package/etc/conf.d/filters/microfocus/arcsight.conf @@ -0,0 +1,4 @@ + +filter f_microfocus_arcsight { + program(CEF); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl new file mode 100644 index 0000000..76d3e1e --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl @@ -0,0 +1,84 @@ +parser p_microfocus_arcsight_header { + csv-parser( + columns("fields.sc4s_cef_version", "fields.cef_device_vendor", "fields.cef_device_product", "fields.cef_device_version", "fields.cef_device_event_class", "fields.cef_name", "fields.cef_severity", MESSAGE) + delimiters(chars("|")) + flags(strip-whitespace, escape-none, greedy) + ); + + kv-parser(prefix(".cef.") template("t_msg_trim") + ); + +}; + +parser p_microfocus_arcsight_ts_rt { + date-parser(format("%s") template("${.cef.rt}") + ); +}; +parser p_microfocus_arcsight_ts_end { + date-parser(format("%s") template("${.cef.end}") + ); +}; + +parser p_microfocus_arcsight_source { + add-contextual-data( + selector("${fields.cef_device_vendor}_${fields.cef_device_product}"), + database("context-local/microfocus_arcsight_source.csv") + ignore-case(yes) + prefix(".splunk.") + default-selector("unknown") + ); +}; + +# Microfocus ArcSight +{{- if (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_microfocus_arcsight); +{{- end}} +{{- if eq (.) "no"}} + source (s_dedicated_port_MICROFOCUS_ARCSIGHT); +{{- end}} + + rewrite { + r_set_splunk_dest_default(sourcetype("cef"), index("main"), template("t_msg_trim")) + }; + + parser { + p_add_context_splunk(key("cef_{fields.cef_device_vendor}_${fields.cef_device_product}")); + + }; + + parser (p_microfocus_arcsight_header); + + # We already have the syslog msg time stamp however that may not be the best one + # If we have an rt or end field that is best we use the If trick here so if this parser fails + # We don't get sent to fallback. + if { + parser (p_microfocus_arcsight_ts_rt); + } elif { + parser (p_microfocus_arcsight_ts_end); + } else {}; #Do nothing this is allows for both rt and end to be missing and still pass with the message ts + + #CEF TAs use the source as their bounds in props.conf + parser(p_microfocus_arcsight_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for MICROFOCUS_ARCSIGHT traffic + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for MICROFOCUS_ARCSIGHT traffic +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/context-local/microfocus_arcsight_source.csv b/package/etc/context-local/microfocus_arcsight_source.csv new file mode 100644 index 0000000..6c5bd73 --- /dev/null +++ b/package/etc/context-local/microfocus_arcsight_source.csv @@ -0,0 +1,8 @@ +ArcSight_ArcSight,source,ArcSight:ArcSight +Microsoft_System or Application Event,source,CEFEventLog:System or Application Event +Microsoft_System or Application Event,index,oswin +Microsoft_Microsoft Windows,source,CEFEventLog:Microsoft Windows +Microsoft_Microsoft Windows,index,oswinsec +Incapsula_SIEMintegration,source,Imperva:Incapsula +Incapsula_SIEMintegration,index,netwaf +unknown,source,ArcSight:unknown diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv index 566c436..766067b 100644 --- a/package/etc/context-local/splunk_index.csv +++ b/package/etc/context-local/splunk_index.csv @@ -1,4 +1,8 @@ #bluecoat_proxy,index,netproxy +#cef_ArcSight_ArcSight,index,netwaf +#cef_Incapsula_SIEMintegration,index,netwaf +#cef_Microsoft_Microsoft Windows,index,oswinsec +#cef_Microsoft_System or Application Event,index,oswin #cisco_asa,index,netfw #cisco_ios,index,netops #cisco_nx_os,index,netops diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf index 60d7cae..09e86e5 100644 --- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -20,6 +20,15 @@ homePath = $SPLUNK_DB/test2/db coldPath = $SPLUNK_DB/test2/colddb thawedPath = $SPLUNK_DB/test2/thaweddb +[oswin] +homePath = $SPLUNK_DB/oswin/db +coldPath = $SPLUNK_DB/oswin/colddb +thawedPath = $SPLUNK_DB/oswin/thaweddb + +[oswinsec] +homePath = $SPLUNK_DB/oswinsec/db +coldPath = $SPLUNK_DB/oswinsec/colddb +thawedPath = $SPLUNK_DB/oswinsec/thaweddb [netfw] homePath = $SPLUNK_DB/netfw/db @@ -41,6 +50,11 @@ homePath = $SPLUNK_DB/netproxy/db coldPath = $SPLUNK_DB/netproxy/colddb thawedPath = $SPLUNK_DB/netproxy/thaweddb +[netwaf] +homePath = $SPLUNK_DB/netwaf/db +coldPath = $SPLUNK_DB/netwaf/colddb +thawedPath = $SPLUNK_DB/netwaf/thaweddb + [email] homePath = $SPLUNK_DB/email/db coldPath = $SPLUNK_DB/email/colddb diff --git a/tests/test_microfocus_arcsight_cef.py b/tests/test_microfocus_arcsight_cef.py new file mode 100644 index 0000000..eb3dd6d --- /dev/null +++ b/tests/test_microfocus_arcsight_cef.py @@ -0,0 +1,137 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +# Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:050|Connector Raw Event Statistics|Low| eventId=77 mrt=1539321422787 categorySignificance=/Informational categoryBehavior=/Execute/Response categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321424798 cat=/Agent/RawEvent/Statistics deviceSeverity=Warning rt=1539321422787 fileType=Agent cs1=0.23 cs2=157.72333333333333 cs3=0.23 cs4=47317 cs5=157.72333333333333 cs6=3o0OiZmYBABCACGN9CiyuGQ\=\= cn1=69 cn2=47317 cn3=69 deviceCustomDate1=1539321122775 cs1Label=Event throughput cs2Label=Raw event character throughput cs3Label=Event throughput (SLC) cs4Label=Raw event length (SLC) cs5Label=Raw event character throughput (SLC) cs6Label=Destination ID cn1Label=Total event count cn2Label=Total raw event length cn3Label=Event count (SLC) deviceCustomDate1Label=Last time ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= +# Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=30 msg=Connected to Host mrt=1539321123071 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321124967 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321123071 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= +# Mar 19 15:19:15 root CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:030|Agent [PAN1_WUC_UDP8000] type [windowsfg] started|Low| eventId=26 mrt=1539321122832 categorySignificance=/Normal categoryBehavior=/Execute/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application/Service art=1539321124967 cat=/Agent/Started deviceSeverity=Warning rt=1539321122832 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= +# Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321047341 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= +def test_microfocus_arcsight_cef_ts_rt(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=' + "{% now 'utc', '%s' %}" + ' dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\\=\\=' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cef\" source=ArcSight:ArcSight | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_microfocus_arcsight_cef_ts_end(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning end=' + "{% now 'utc', '%s' %}" + ' dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\\=\\=' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cef\" source=ArcSight:ArcSight| head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_microfocus_arcsight_cef_ts_syslog(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\\=\\=' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cef\" source=ArcSight:ArcSight | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_microfocus_arcsight_windows(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Microsoft|Microsoft Windows||Microsoft-Windows-Security-Auditing:4634|An account was logged off.|Low| eventId=1 externalId=4634 msg=Network: A user or computer logged on to this computer from the network. rawEvent=EventlogType\=Security&&EventIndex\=1031&&WindowsVersion\=Windows Server 2012 R2&&WindowsKeyMapFamily\=Windows 2012 R2&&WindowsParserFamily\=Windows 2012 R2|2012|8&&DetectTime\=2018-10-12 7:25:11&&EventSource\=Microsoft-Windows-Security-Auditing&&EventID\=4634&&EventType\=Audit_success&&EventCategory\=12545&&User\=null&&ComputerName\=WIN-PAN1&&Description\=An account was logged off.&&Message\=This event is generated when a logon session is destroyed. It may be positively correlated with a logon event using the Logon ID value. Logon IDs are only unique between reboots on the same computer.&&Subject:Security ID\=S-1-5-21-750061412-3179291162-3140434184-500&&Subject:Account Name\=Administrator&&Subject:Account Domain\=WIN-PAN1&&Subject:Logon ID\=0x373c2&&Logon Type\=3 categorySignificance=/Informational categoryBehavior=/Access/Stop categoryDeviceGroup=/Operating System catdt=Operating System categoryOutcome=/Success categoryObject=/Host/Operating System art=1539321047369 cat=Security deviceSeverity=Audit_success rt=' + "{% now 'utc', '%s' %}" + ' dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dntdom=WIN-PAN1 duser=Administrator duid=0x373c2 cs2=Logon/Logoff:Logoff cn1=3 cs1Label=Accesses cs2Label=EventlogCategory cs4Label=Reason or Error Code cs5Label=Authentication Package Name cn1Label=LogonType cn2Label=CrashOnAuditFail cn3Label=Count ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=WIN-PAN1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 deviceNtDomain=WIN-PAN1 dtz=Asia/Riyadh _cefVer=0.1 ad.WindowsVersion=Windows Server 2012 R2 ad.WindowsParserFamily=Windows 2012 R2|2012|8 ad.WindowsKeyMapFamily=Windows 2012 R2 ad.EventIndex=1031 aid=3o0OiZmYBABCACGN9CiyuGQ\=\=' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=oswinsec host=\"{{ host }}\" sourcetype=\"cef\" source=\"CEFEventLog:Microsoft Windows\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_microfocus_arcsight_windows_system(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Microsoft|System or Application Event||Software Protection Platform Service:902|Software Protection Platform Service|Unknown| eventId=39 externalId=902 rawEvent=EventlogType\=Application&&EventIndex\=1604&&WindowsVersion\=Windows Server 2012 R2&&WindowsKeyMapFamily\=Windows 2012 R2&&WindowsParserFamily\=Windows 2012 R2|2012|8&&DetectTime\=2018-10-12 8:12:31&&EventSource\=Software Protection Platform Service&&EventID\=902&&EventType\=null&&EventCategory\=0&&User\=null&&ComputerName\=WIN-PAN1&&Key[0]\=6.3.9600.16384 art=1539321151610 cat=Application rt=' + "{% now 'utc', '%s' %}" + ' cs2=0 cs3=Software Protection Platform Service cs2Label=EventlogCategory cs3Label=EventSource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=WIN-PAN1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dtz=Asia/Riyadh _cefVer=0.1 ad.WindowsVersion=Windows Server 2012 R2 ad.WindowsParserFamily=Windows 2012 R2|2012|8 ad.WindowsKeyMapFamily=Windows 2012 R2 ad.Key[0]=6.3.9600.16384 ad.EventIndex=1604 aid=3o0OiZmYBABCACGN9CiyuGQ\=\=' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=oswin host=\"{{ host }}\" sourcetype=\"cef\" source=\"CEFEventLog:System or Application Event\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_microfocus_arcsight_imperva_incapsula(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Incapsula|SIEMintegration|1|1|Illegal Resource Access|3| fileid=3412341160002518171 sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support src=12.12.12.12 caIP=13.13.13.13 ccode=IL tag=www.elvis.com cn1=200 in=54 xff=44.44.44.44 cs1=NOT_SUPPORTED cs1Label=Cap Support cs4=c2e72124-0e8a-4dd8-b13b-3da246af3ab2 cs4Label=VID cs5=de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4 cs5Label=clappsigdproc=Browser cs6=Firefox cs6Label=clapp ccode=IL cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 start=1453290121336 request=site123.abcd.info/ requestmethod=GET qstr=p\=%2fetc%2fpasswd app=HTTP act=REQ_CHALLENGE_CAPTCHA deviceExternalID=33411452762204224 cpt=443 filetype=30037,1001, filepermission=2,1, cs9=Block Malicious User,High Risk Resources, cs9Label=Rule name' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netwaf host=\"{{ host }}\" sourcetype=\"cef\" source=\"Imperva:Incapsula\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file