From 645732b2d3d204149f3cc8911fb0d25d684cc7c9 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 23 Jan 2020 13:04:40 -0800 Subject: [PATCH] Refactor old MICROFOCUS_ARCSIGHT log path to CEF * Update MICROFOCUS_ARCSIGHT to be generic CEF * CyberArk and Imperva docs updated with new CEF env vars * splunk_indexes.conf sample entries updated with new key format * TODO: Windows CEF needs its own source doc entry * TODO: Arcsight Internal Agent needs its own source doc entry * TODO: CEF source doc entry should have _no_ products listed; consider removing --- .../index.md | 24 ++++++------- docs/sources/CyberArk/index.md | 4 +-- docs/sources/Imperva/index.md | 12 +++---- mkdocs.yml | 4 +-- ...rce.csv => common_event_format_source.csv} | 0 .../filters/common_event_format/cef.conf | 4 +++ .../conf.d/filters/microfocus/arcsight.conf | 4 --- ....tmpl => lp-common_event_format.conf.tmpl} | 36 +++++++++---------- .../etc/context_templates/splunk_index.csv | 10 +++--- ...ght_cef.py => test_common_event_format.py} | 14 ++++---- 10 files changed, 57 insertions(+), 55 deletions(-) rename docs/sources/{Microfocus => CommonEventFormat}/index.md (79%) rename package/etc/conf.d/context/{microfocus_arcsight_source.csv => common_event_format_source.csv} (100%) create mode 100644 package/etc/conf.d/filters/common_event_format/cef.conf delete mode 100644 package/etc/conf.d/filters/microfocus/arcsight.conf rename package/etc/conf.d/log_paths/{lp-microfocus_arcsight.conf.tmpl => lp-common_event_format.conf.tmpl} (68%) rename tests/{test_microfocus_arcsight_cef.py => test_common_event_format.py} (96%) diff --git a/docs/sources/Microfocus/index.md b/docs/sources/CommonEventFormat/index.md similarity index 79% rename from docs/sources/Microfocus/index.md rename to docs/sources/CommonEventFormat/index.md index 5909324..c6c26bd 100644 --- a/docs/sources/Microfocus/index.md +++ b/docs/sources/CommonEventFormat/index.md @@ -1,6 +1,6 @@ -# Vendor - Microfocus ArcSight +# Vendor - Common Event Format Data Sources -## Product - Internal Agent Events +## Product - Arcsight Internal Agent | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| @@ -24,7 +24,7 @@ | key | source | index | notes | |----------------|----------------|----------------|----------------| -| cef_ArcSight_ArcSight | ArcSight:ArcSight | main | none | +| ArcSight_ArcSight | ArcSight:ArcSight | main | none | ### Filter type @@ -34,7 +34,7 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | ### Verification @@ -46,7 +46,7 @@ Verify timestamp, and host values match as expected index= (sourcetype=cef source="ArcSight:ArcSight") ``` -## Product - Microsoft Windows +## Product - Microsoft Windows (CEF) | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| @@ -72,8 +72,8 @@ index= (sourcetype=cef source="ArcSight:ArcSight") | key | source | index | notes | |----------------|----------------|----------------|----------------| -| cef_Microsoft_System or Application Event | CEFEventLog:System or Application Event | oswin | none | -| cef_Microsoft_Microsoft Windows | CEFEventLog:Microsoft Windows | oswinsec | none | +| Microsoft_System or Application Event | CEFEventLog:System or Application Event | oswin | none | +| Microsoft_Microsoft Windows | CEFEventLog:Microsoft Windows | oswinsec | none | ### Filter type @@ -83,10 +83,10 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT | no | Enable archive to disk for this specific source | -| SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source | +| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -96,4 +96,4 @@ Verify timestamp, and host values match as expected ``` index= (sourcetype=cef (source="CEFEventLog:Microsoft Windows" OR source="CEFEventLog:System or Application Event")) -``` \ No newline at end of file +``` diff --git a/docs/sources/CyberArk/index.md b/docs/sources/CyberArk/index.md index dd497d0..1a113ea 100644 --- a/docs/sources/CyberArk/index.md +++ b/docs/sources/CyberArk/index.md @@ -28,7 +28,7 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | ### Verification @@ -68,7 +68,7 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | ### Verification diff --git a/docs/sources/Imperva/index.md b/docs/sources/Imperva/index.md index 2ae9eea..ad0e0e9 100644 --- a/docs/sources/Imperva/index.md +++ b/docs/sources/Imperva/index.md @@ -25,7 +25,7 @@ | key | source | index | notes | |----------------|----------------|----------------|----------------| -| cef_Incapsula_SIEMintegration | Imperva:Incapsula | netwaf | none | +| Incapsula_SIEMintegration | Imperva:Incapsula | netwaf | none | ### Filter type @@ -37,10 +37,10 @@ Note listed for reference processing utilizes the Microsoft ArcSight log path as | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT | no | Enable archive to disk for this specific source | -| SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source | +| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -50,4 +50,4 @@ Verify timestamp, and host values match as expected ``` index= (sourcetype=cef source="Imperva:Incapsula") -``` \ No newline at end of file +``` diff --git a/mkdocs.yml b/mkdocs.yml index 3407538..018c557 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,14 +14,14 @@ nav: - About: sources/index.md - Checkpoint: sources/Checkpoint/index.md - Cisco: sources/Cisco/index.md + - 'Common Event Format': sources/CommonEventFormat/index.md - CyberArk: sources/CyberArk/index.md - Forcepoint: sources/Forcepoint/index.md - Fortinet: sources/Fortinet/index.md - Imperva: sources/Imperva/index.md - Juniper: sources/Juniper/index.md - Nix: sources/nix/index.md - - Microfocus: sources/Microfocus/index.md - - 'Paloalto Networks': sources/PaloaltoNetworks/index.md + - 'Palo Alto Networks': sources/PaloaltoNetworks/index.md - Proofpoint: sources/Proofpoint/index.md - Symantec: sources/Symantec/index.md - Ubiquiti: sources/Ubiquiti/index.md diff --git a/package/etc/conf.d/context/microfocus_arcsight_source.csv b/package/etc/conf.d/context/common_event_format_source.csv similarity index 100% rename from package/etc/conf.d/context/microfocus_arcsight_source.csv rename to package/etc/conf.d/context/common_event_format_source.csv diff --git a/package/etc/conf.d/filters/common_event_format/cef.conf b/package/etc/conf.d/filters/common_event_format/cef.conf new file mode 100644 index 0000000..e180b31 --- /dev/null +++ b/package/etc/conf.d/filters/common_event_format/cef.conf @@ -0,0 +1,4 @@ + +filter f_cef { + program(CEF); +}; diff --git a/package/etc/conf.d/filters/microfocus/arcsight.conf b/package/etc/conf.d/filters/microfocus/arcsight.conf deleted file mode 100644 index 287d7a4..0000000 --- a/package/etc/conf.d/filters/microfocus/arcsight.conf +++ /dev/null @@ -1,4 +0,0 @@ - -filter f_microfocus_arcsight { - program(CEF); -}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl similarity index 68% rename from package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl rename to package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl index fd5a97a..64e9577 100644 --- a/package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl @@ -1,9 +1,9 @@ -# Microfocus ArcSight +# Common Event Format {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "rfc3164" }} +{{- $context := dict "port_id" "CEF" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} -parser p_microfocus_arcsight_header { +parser p_cef_header { csv-parser( columns("fields.sc4s_cef_version", "fields.cef_device_vendor", "fields.cef_device_product", "fields.cef_device_version", "fields.cef_device_event_class", "fields.cef_name", "fields.cef_severity", MESSAGE) delimiters(chars("|")) @@ -15,19 +15,19 @@ parser p_microfocus_arcsight_header { }; -parser p_microfocus_arcsight_ts_rt { +parser p_cef_ts_rt { date-parser(format("%s") template("${.cef.rt}") ); }; -parser p_microfocus_arcsight_ts_end { +parser p_cef_ts_end { date-parser(format("%s") template("${.cef.end}") ); }; -parser p_microfocus_arcsight_source { +parser p_cef_source { add-contextual-data( selector("${fields.cef_device_vendor}_${fields.cef_device_product}"), - database("conf.d/context/microfocus_arcsight_source.csv") + database("conf.d/context/common_event_format_source.csv") ignore-case(yes) prefix(".splunk.") default-selector("unknown") @@ -36,18 +36,18 @@ parser p_microfocus_arcsight_source { log { junction { -{{- if or (or (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT")) (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT"))) (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT")) }} +{{- if or (or (getenv (print "SC4S_LISTEN_CEF_TCP_PORT")) (getenv (print "SC4S_LISTEN_CEF_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CEF_TLS_PORT")) }} channel { - # Listen on the specified dedicated port(s) for MICROFOCUS_ARCSIGHT traffic - source (s_MICROFOCUS_ARCSIGHT); + # Listen on the specified dedicated port(s) for CEF traffic + source (s_CEF); flags (final); }; {{- end}} channel { - # Listen on the default port (typically 514) for MICROFOCUS_ARCSIGHT traffic + # Listen on the default port (typically 514) for CEF traffic source (s_DEFAULT); filter(f_is_rfc3164); - filter(f_microfocus_arcsight); + filter(f_cef); flags(final); }; }; @@ -56,7 +56,7 @@ log { r_set_splunk_dest_default(sourcetype("cef"), index("main")) }; - parser (p_microfocus_arcsight_header); + parser (p_cef_header); rewrite { set("${fields.cef_device_vendor}_${fields.cef_device_product}", value("fields.sc4s_vendor_product")); @@ -70,13 +70,13 @@ log { # If we have an rt or end field that is best we use the If trick here so if this parser fails # We don't get sent to fallback. if { - parser (p_microfocus_arcsight_ts_rt); + parser (p_cef_ts_rt); } elif { - parser (p_microfocus_arcsight_ts_end); + parser (p_cef_ts_end); } else {}; #Do nothing this is allows for both rt and end to be missing and still pass with the message ts #CEF TAs use the source as their bounds in props.conf - parser(p_microfocus_arcsight_source); + parser(p_cef_source); parser (compliance_meta_by_source); @@ -85,11 +85,11 @@ log { #if we don't rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; -{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC" "no")) }} +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CEF_HEC" "no")) }} destination(d_hec); {{- end}} -{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT" "no")) }} +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CEF" "no")) }} destination(d_archive); {{- end}} diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index 51b71c0..a1cbaa5 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -1,8 +1,10 @@ #bluecoat_proxy,index,netproxy -#cef_ArcSight_ArcSight,index,netwaf -#cef_Incapsula_SIEMintegration,index,netwaf -#cef_Microsoft_Microsoft Windows,index,oswinsec -#cef_Microsoft_System or Application Event,index,oswin +#ArcSight_ArcSight,index,netwaf +#Cyber-Ark_Vault,index,netauth +#CyberArk_PTA,index,main +#Incapsula_SIEMintegration,index,netwaf +#Microsoft_Microsoft Windows,index,oswinsec +#Microsoft_System or Application Event,index,oswin #checkpoint_splunk,index,netops #checkpoint_splunk_dlp,index,netdlp #checkpoint_splunk_email,index,email diff --git a/tests/test_microfocus_arcsight_cef.py b/tests/test_common_event_format.py similarity index 96% rename from tests/test_microfocus_arcsight_cef.py rename to tests/test_common_event_format.py index eb3dd6d..510f46c 100644 --- a/tests/test_microfocus_arcsight_cef.py +++ b/tests/test_common_event_format.py @@ -16,7 +16,7 @@ # Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=30 msg=Connected to Host mrt=1539321123071 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321124967 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321123071 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= # Mar 19 15:19:15 root CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:030|Agent [PAN1_WUC_UDP8000] type [windowsfg] started|Low| eventId=26 mrt=1539321122832 categorySignificance=/Normal categoryBehavior=/Execute/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application/Service art=1539321124967 cat=/Agent/Started deviceSeverity=Warning rt=1539321122832 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= # Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321047341 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= -def test_microfocus_arcsight_cef_ts_rt(record_property, setup_wordlist, setup_splunk): +def test_cef_ts_rt(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -36,7 +36,7 @@ def test_microfocus_arcsight_cef_ts_rt(record_property, setup_wordlist, setup_sp assert resultCount == 1 -def test_microfocus_arcsight_cef_ts_end(record_property, setup_wordlist, setup_splunk): +def test_cef_ts_end(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -56,7 +56,7 @@ def test_microfocus_arcsight_cef_ts_end(record_property, setup_wordlist, setup_s assert resultCount == 1 -def test_microfocus_arcsight_cef_ts_syslog(record_property, setup_wordlist, setup_splunk): +def test_cef_ts_syslog(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -76,7 +76,7 @@ def test_microfocus_arcsight_cef_ts_syslog(record_property, setup_wordlist, setu assert resultCount == 1 -def test_microfocus_arcsight_windows(record_property, setup_wordlist, setup_splunk): +def test_cef_windows(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -96,7 +96,7 @@ def test_microfocus_arcsight_windows(record_property, setup_wordlist, setup_splu assert resultCount == 1 -def test_microfocus_arcsight_windows_system(record_property, setup_wordlist, setup_splunk): +def test_cef_windows_system(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -116,7 +116,7 @@ def test_microfocus_arcsight_windows_system(record_property, setup_wordlist, set assert resultCount == 1 -def test_microfocus_arcsight_imperva_incapsula(record_property, setup_wordlist, setup_splunk): +def test_cef_imperva_incapsula(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -134,4 +134,4 @@ def test_microfocus_arcsight_imperva_incapsula(record_property, setup_wordlist, record_property("resultCount", resultCount) record_property("message", message) - assert resultCount == 1 \ No newline at end of file + assert resultCount == 1