From 645732b2d3d204149f3cc8911fb0d25d684cc7c9 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 23 Jan 2020 13:04:40 -0800 Subject: [PATCH 01/10] Refactor old MICROFOCUS_ARCSIGHT log path to CEF * Update MICROFOCUS_ARCSIGHT to be generic CEF * CyberArk and Imperva docs updated with new CEF env vars * splunk_indexes.conf sample entries updated with new key format * TODO: Windows CEF needs its own source doc entry * TODO: Arcsight Internal Agent needs its own source doc entry * TODO: CEF source doc entry should have _no_ products listed; consider removing --- .../index.md | 24 ++++++------- docs/sources/CyberArk/index.md | 4 +-- docs/sources/Imperva/index.md | 12 +++---- mkdocs.yml | 4 +-- ...rce.csv => common_event_format_source.csv} | 0 .../filters/common_event_format/cef.conf | 4 +++ .../conf.d/filters/microfocus/arcsight.conf | 4 --- ....tmpl => lp-common_event_format.conf.tmpl} | 36 +++++++++---------- .../etc/context_templates/splunk_index.csv | 10 +++--- ...ght_cef.py => test_common_event_format.py} | 14 ++++---- 10 files changed, 57 insertions(+), 55 deletions(-) rename docs/sources/{Microfocus => CommonEventFormat}/index.md (79%) rename package/etc/conf.d/context/{microfocus_arcsight_source.csv => common_event_format_source.csv} (100%) create mode 100644 package/etc/conf.d/filters/common_event_format/cef.conf delete mode 100644 package/etc/conf.d/filters/microfocus/arcsight.conf rename package/etc/conf.d/log_paths/{lp-microfocus_arcsight.conf.tmpl => lp-common_event_format.conf.tmpl} (68%) rename tests/{test_microfocus_arcsight_cef.py => test_common_event_format.py} (96%) diff --git a/docs/sources/Microfocus/index.md b/docs/sources/CommonEventFormat/index.md similarity index 79% rename from docs/sources/Microfocus/index.md rename to docs/sources/CommonEventFormat/index.md index 5909324..c6c26bd 100644 --- a/docs/sources/Microfocus/index.md +++ b/docs/sources/CommonEventFormat/index.md @@ -1,6 +1,6 @@ -# Vendor - Microfocus ArcSight +# Vendor - Common Event Format Data Sources -## Product - Internal Agent Events +## Product - Arcsight Internal Agent | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| @@ -24,7 +24,7 @@ | key | source | index | notes | |----------------|----------------|----------------|----------------| -| cef_ArcSight_ArcSight | ArcSight:ArcSight | main | none | +| ArcSight_ArcSight | ArcSight:ArcSight | main | none | ### Filter type @@ -34,7 +34,7 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | ### Verification @@ -46,7 +46,7 @@ Verify timestamp, and host values match as expected index= (sourcetype=cef source="ArcSight:ArcSight") ``` -## Product - Microsoft Windows +## Product - Microsoft Windows (CEF) | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| @@ -72,8 +72,8 @@ index= (sourcetype=cef source="ArcSight:ArcSight") | key | source | index | notes | |----------------|----------------|----------------|----------------| -| cef_Microsoft_System or Application Event | CEFEventLog:System or Application Event | oswin | none | -| cef_Microsoft_Microsoft Windows | CEFEventLog:Microsoft Windows | oswinsec | none | +| Microsoft_System or Application Event | CEFEventLog:System or Application Event | oswin | none | +| Microsoft_Microsoft Windows | CEFEventLog:Microsoft Windows | oswinsec | none | ### Filter type @@ -83,10 +83,10 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT | no | Enable archive to disk for this specific source | -| SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source | +| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -96,4 +96,4 @@ Verify timestamp, and host values match as expected ``` index= (sourcetype=cef (source="CEFEventLog:Microsoft Windows" OR source="CEFEventLog:System or Application Event")) -``` \ No newline at end of file +``` diff --git a/docs/sources/CyberArk/index.md b/docs/sources/CyberArk/index.md index dd497d0..1a113ea 100644 --- a/docs/sources/CyberArk/index.md +++ b/docs/sources/CyberArk/index.md @@ -28,7 +28,7 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | ### Verification @@ -68,7 +68,7 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | ### Verification diff --git a/docs/sources/Imperva/index.md b/docs/sources/Imperva/index.md index 2ae9eea..ad0e0e9 100644 --- a/docs/sources/Imperva/index.md +++ b/docs/sources/Imperva/index.md @@ -25,7 +25,7 @@ | key | source | index | notes | |----------------|----------------|----------------|----------------| -| cef_Incapsula_SIEMintegration | Imperva:Incapsula | netwaf | none | +| Incapsula_SIEMintegration | Imperva:Incapsula | netwaf | none | ### Filter type @@ -37,10 +37,10 @@ Note listed for reference processing utilizes the Microsoft ArcSight log path as | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT | no | Enable archive to disk for this specific source | -| SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source | +| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -50,4 +50,4 @@ Verify timestamp, and host values match as expected ``` index= (sourcetype=cef source="Imperva:Incapsula") -``` \ No newline at end of file +``` diff --git a/mkdocs.yml b/mkdocs.yml index 3407538..018c557 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,14 +14,14 @@ nav: - About: sources/index.md - Checkpoint: sources/Checkpoint/index.md - Cisco: sources/Cisco/index.md + - 'Common Event Format': sources/CommonEventFormat/index.md - CyberArk: sources/CyberArk/index.md - Forcepoint: sources/Forcepoint/index.md - Fortinet: sources/Fortinet/index.md - Imperva: sources/Imperva/index.md - Juniper: sources/Juniper/index.md - Nix: sources/nix/index.md - - Microfocus: sources/Microfocus/index.md - - 'Paloalto Networks': sources/PaloaltoNetworks/index.md + - 'Palo Alto Networks': sources/PaloaltoNetworks/index.md - Proofpoint: sources/Proofpoint/index.md - Symantec: sources/Symantec/index.md - Ubiquiti: sources/Ubiquiti/index.md diff --git a/package/etc/conf.d/context/microfocus_arcsight_source.csv b/package/etc/conf.d/context/common_event_format_source.csv similarity index 100% rename from package/etc/conf.d/context/microfocus_arcsight_source.csv rename to package/etc/conf.d/context/common_event_format_source.csv diff --git a/package/etc/conf.d/filters/common_event_format/cef.conf b/package/etc/conf.d/filters/common_event_format/cef.conf new file mode 100644 index 0000000..e180b31 --- /dev/null +++ b/package/etc/conf.d/filters/common_event_format/cef.conf @@ -0,0 +1,4 @@ + +filter f_cef { + program(CEF); +}; diff --git a/package/etc/conf.d/filters/microfocus/arcsight.conf b/package/etc/conf.d/filters/microfocus/arcsight.conf deleted file mode 100644 index 287d7a4..0000000 --- a/package/etc/conf.d/filters/microfocus/arcsight.conf +++ /dev/null @@ -1,4 +0,0 @@ - -filter f_microfocus_arcsight { - program(CEF); -}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl similarity index 68% rename from package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl rename to package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl index fd5a97a..64e9577 100644 --- a/package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl @@ -1,9 +1,9 @@ -# Microfocus ArcSight +# Common Event Format {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "rfc3164" }} +{{- $context := dict "port_id" "CEF" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} -parser p_microfocus_arcsight_header { +parser p_cef_header { csv-parser( columns("fields.sc4s_cef_version", "fields.cef_device_vendor", "fields.cef_device_product", "fields.cef_device_version", "fields.cef_device_event_class", "fields.cef_name", "fields.cef_severity", MESSAGE) delimiters(chars("|")) @@ -15,19 +15,19 @@ parser p_microfocus_arcsight_header { }; -parser p_microfocus_arcsight_ts_rt { +parser p_cef_ts_rt { date-parser(format("%s") template("${.cef.rt}") ); }; -parser p_microfocus_arcsight_ts_end { +parser p_cef_ts_end { date-parser(format("%s") template("${.cef.end}") ); }; -parser p_microfocus_arcsight_source { +parser p_cef_source { add-contextual-data( selector("${fields.cef_device_vendor}_${fields.cef_device_product}"), - database("conf.d/context/microfocus_arcsight_source.csv") + database("conf.d/context/common_event_format_source.csv") ignore-case(yes) prefix(".splunk.") default-selector("unknown") @@ -36,18 +36,18 @@ parser p_microfocus_arcsight_source { log { junction { -{{- if or (or (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT")) (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT"))) (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT")) }} +{{- if or (or (getenv (print "SC4S_LISTEN_CEF_TCP_PORT")) (getenv (print "SC4S_LISTEN_CEF_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CEF_TLS_PORT")) }} channel { - # Listen on the specified dedicated port(s) for MICROFOCUS_ARCSIGHT traffic - source (s_MICROFOCUS_ARCSIGHT); + # Listen on the specified dedicated port(s) for CEF traffic + source (s_CEF); flags (final); }; {{- end}} channel { - # Listen on the default port (typically 514) for MICROFOCUS_ARCSIGHT traffic + # Listen on the default port (typically 514) for CEF traffic source (s_DEFAULT); filter(f_is_rfc3164); - filter(f_microfocus_arcsight); + filter(f_cef); flags(final); }; }; @@ -56,7 +56,7 @@ log { r_set_splunk_dest_default(sourcetype("cef"), index("main")) }; - parser (p_microfocus_arcsight_header); + parser (p_cef_header); rewrite { set("${fields.cef_device_vendor}_${fields.cef_device_product}", value("fields.sc4s_vendor_product")); @@ -70,13 +70,13 @@ log { # If we have an rt or end field that is best we use the If trick here so if this parser fails # We don't get sent to fallback. if { - parser (p_microfocus_arcsight_ts_rt); + parser (p_cef_ts_rt); } elif { - parser (p_microfocus_arcsight_ts_end); + parser (p_cef_ts_end); } else {}; #Do nothing this is allows for both rt and end to be missing and still pass with the message ts #CEF TAs use the source as their bounds in props.conf - parser(p_microfocus_arcsight_source); + parser(p_cef_source); parser (compliance_meta_by_source); @@ -85,11 +85,11 @@ log { #if we don't rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; -{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC" "no")) }} +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CEF_HEC" "no")) }} destination(d_hec); {{- end}} -{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT" "no")) }} +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CEF" "no")) }} destination(d_archive); {{- end}} diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index 51b71c0..a1cbaa5 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -1,8 +1,10 @@ #bluecoat_proxy,index,netproxy -#cef_ArcSight_ArcSight,index,netwaf -#cef_Incapsula_SIEMintegration,index,netwaf -#cef_Microsoft_Microsoft Windows,index,oswinsec -#cef_Microsoft_System or Application Event,index,oswin +#ArcSight_ArcSight,index,netwaf +#Cyber-Ark_Vault,index,netauth +#CyberArk_PTA,index,main +#Incapsula_SIEMintegration,index,netwaf +#Microsoft_Microsoft Windows,index,oswinsec +#Microsoft_System or Application Event,index,oswin #checkpoint_splunk,index,netops #checkpoint_splunk_dlp,index,netdlp #checkpoint_splunk_email,index,email diff --git a/tests/test_microfocus_arcsight_cef.py b/tests/test_common_event_format.py similarity index 96% rename from tests/test_microfocus_arcsight_cef.py rename to tests/test_common_event_format.py index eb3dd6d..510f46c 100644 --- a/tests/test_microfocus_arcsight_cef.py +++ b/tests/test_common_event_format.py @@ -16,7 +16,7 @@ # Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=30 msg=Connected to Host mrt=1539321123071 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321124967 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321123071 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= # Mar 19 15:19:15 root CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:030|Agent [PAN1_WUC_UDP8000] type [windowsfg] started|Low| eventId=26 mrt=1539321122832 categorySignificance=/Normal categoryBehavior=/Execute/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application/Service art=1539321124967 cat=/Agent/Started deviceSeverity=Warning rt=1539321122832 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= # Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321047341 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= -def test_microfocus_arcsight_cef_ts_rt(record_property, setup_wordlist, setup_splunk): +def test_cef_ts_rt(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -36,7 +36,7 @@ def test_microfocus_arcsight_cef_ts_rt(record_property, setup_wordlist, setup_sp assert resultCount == 1 -def test_microfocus_arcsight_cef_ts_end(record_property, setup_wordlist, setup_splunk): +def test_cef_ts_end(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -56,7 +56,7 @@ def test_microfocus_arcsight_cef_ts_end(record_property, setup_wordlist, setup_s assert resultCount == 1 -def test_microfocus_arcsight_cef_ts_syslog(record_property, setup_wordlist, setup_splunk): +def test_cef_ts_syslog(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -76,7 +76,7 @@ def test_microfocus_arcsight_cef_ts_syslog(record_property, setup_wordlist, setu assert resultCount == 1 -def test_microfocus_arcsight_windows(record_property, setup_wordlist, setup_splunk): +def test_cef_windows(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -96,7 +96,7 @@ def test_microfocus_arcsight_windows(record_property, setup_wordlist, setup_splu assert resultCount == 1 -def test_microfocus_arcsight_windows_system(record_property, setup_wordlist, setup_splunk): +def test_cef_windows_system(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -116,7 +116,7 @@ def test_microfocus_arcsight_windows_system(record_property, setup_wordlist, set assert resultCount == 1 -def test_microfocus_arcsight_imperva_incapsula(record_property, setup_wordlist, setup_splunk): +def test_cef_imperva_incapsula(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -134,4 +134,4 @@ def test_microfocus_arcsight_imperva_incapsula(record_property, setup_wordlist, record_property("resultCount", resultCount) record_property("message", message) - assert resultCount == 1 \ No newline at end of file + assert resultCount == 1 From 71a8862db28cd04c50505a2983da4d51fd85d4b4 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Thu, 23 Jan 2020 22:27:15 -0800 Subject: [PATCH 02/10] Add backward compatibity for MF ARCSIGHT env vars * entrypoint.sh: Add backward compatibilty for deprecated MICROFOCUS_ARCSIGHT environment variables * Revise documentation to highlight deprecated variables * Add separate Arcsight source document --- docs/sources/Arcsight/index.md | 101 ++++++++++++++++++++++++ docs/sources/CommonEventFormat/index.md | 68 ++++------------ mkdocs.yml | 1 + package/sbin/entrypoint.sh | 8 ++ 4 files changed, 125 insertions(+), 53 deletions(-) create mode 100644 docs/sources/Arcsight/index.md diff --git a/docs/sources/Arcsight/index.md b/docs/sources/Arcsight/index.md new file mode 100644 index 0000000..15b6dda --- /dev/null +++ b/docs/sources/Arcsight/index.md @@ -0,0 +1,101 @@ +# Vendor - MicroFocus Arcsight + +## Product - Arcsight Internal Agent + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/ | +| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cef | Common sourcetype | + +### Source + +| source | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| ArcSight:ArcSight | Internal logs | + +### Index Configuration + +| key | source | index | notes | +|----------------|----------------|----------------|----------------| +| ArcSight_ArcSight | ArcSight:ArcSight | main | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Deprecated equivalent of above variable. This is included for backward compatibility and will be removed in a future version. _Do not use_ in new installations. | + +### Verification + +An active site will generate frequent events use the following search to check for new events + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=cef source="ArcSight:ArcSight") +``` + +## Product - Microsoft Windows (CEF) + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/ | +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-microsoft-windows-for-splunk/downloads/ | +| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cef | Common sourcetype | + +### Source + +| source | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| CEFEventLog:System or Application Event | Windows Application and System Event Logs | +| CEFEventLog:Microsoft Windows | Windows Security Event Logs | + +### Index Configuration + +| key | source | index | notes | +|----------------|----------------|----------------|----------------| +| Microsoft_System or Application Event | CEFEventLog:System or Application Event | oswin | none | +| Microsoft_Microsoft Windows | CEFEventLog:Microsoft Windows | oswinsec | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source | +| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_WWW_XXX_MICROFOCUS_ARCSIGHT_YYY_ZZZ | no | Deprecated equivalents of the above variables. These are included for backward compatibility, and will be removed in a future version. _Do not use_ in new installations. | + +### Verification + +An active site will generate frequent events use the following search to check for new events + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=cef (source="CEFEventLog:Microsoft Windows" OR source="CEFEventLog:System or Application Event")) +``` diff --git a/docs/sources/CommonEventFormat/index.md b/docs/sources/CommonEventFormat/index.md index c6c26bd..d98e78b 100644 --- a/docs/sources/CommonEventFormat/index.md +++ b/docs/sources/CommonEventFormat/index.md @@ -1,57 +1,20 @@ # Vendor - Common Event Format Data Sources -## Product - Arcsight Internal Agent +## Product - Various products that send CEF-format messages via syslog -| Ref | Link | -|----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/ | -| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm | - - -### Sourcetypes - -| sourcetype | notes | -|----------------|---------------------------------------------------------------------------------------------------------| -| cef | Common sourcetype | - -### Source - -| source | notes | -|----------------|---------------------------------------------------------------------------------------------------------| -| ArcSight:ArcSight | Internal logs | +Each CEF product should have their own source entry in this documentation set. In a departure +from normal configuration, all CEF products should use the "CEF" version of the unique port and +archive envrionmetn variable settings (rather than a unique one per product), as the CEF log path +handles all products sending events to SC4S in the CEF format. Examples of this include Arcsight, +Imperva, and Cyberark. -### Index Configuration +The source documentation included below is a reference baseline for any product that sends data +using the CEF log path. -| key | source | index | notes | -|----------------|----------------|----------------|----------------| -| ArcSight_ArcSight | ArcSight:ArcSight | main | none | - -### Filter type - -MSG Parse: This filter parses message content - -### Options - -| Variable | default | description | -|----------------|----------------|----------------| -| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | - -### Verification - -An active site will generate frequent events use the following search to check for new events - -Verify timestamp, and host values match as expected - -``` -index= (sourcetype=cef source="ArcSight:ArcSight") -``` - -## Product - Microsoft Windows (CEF) | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| | Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/ | -| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-microsoft-windows-for-splunk/downloads/ | | Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm | @@ -61,19 +24,17 @@ index= (sourcetype=cef source="ArcSight:ArcSight") |----------------|---------------------------------------------------------------------------------------------------------| | cef | Common sourcetype | -### Source +### Typical Source | source | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| CEFEventLog:System or Application Event | Windows Application and System Event Logs | -| CEFEventLog:Microsoft Windows | Windows Security Event Logs | +| Varies | Varies | -### Index Configuration +### Typical Index Configuration | key | source | index | notes | |----------------|----------------|----------------|----------------| -| Microsoft_System or Application Event | CEFEventLog:System or Application Event | oswin | none | -| Microsoft_Microsoft Windows | CEFEventLog:Microsoft Windows | oswinsec | none | +| Vendor_Product | Varies | main | none | ### Filter type @@ -83,8 +44,9 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | | SC4S_LISTEN_CEF_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | | SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source | | SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | @@ -95,5 +57,5 @@ An active site will generate frequent events use the following search to check f Verify timestamp, and host values match as expected ``` -index= (sourcetype=cef (source="CEFEventLog:Microsoft Windows" OR source="CEFEventLog:System or Application Event")) +index= (sourcetype=cef source=) ``` diff --git a/mkdocs.yml b/mkdocs.yml index 018c557..5a467aa 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,6 +12,7 @@ nav: - Configuration: 'configuration.md' - Sources: - About: sources/index.md + - ArcSight: sources/Arcsight/index.md - Checkpoint: sources/Checkpoint/index.md - Cisco: sources/Cisco/index.md - 'Common Event Format': sources/CommonEventFormat/index.md diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index b7d5b2d..229a384 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -1,6 +1,14 @@ #!/usr/bin/env bash source scl_source enable rh-python36 +# The MICROFOCUS_ARCSIGHT unique port environment variables are currently deprecated +# This will be removed when the MICROFOCUS_ARCSIGHT unique port environment variables are removed in version 2.0 +if [ ${SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT} ]; then export SC4S_LISTEN_CEF_UDP_PORT=$SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT; fi +if [ ${SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT} ]; then export SC4S_LISTEN_CEF_TCP_PORT=$SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT; fi +if [ ${SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT} ]; then export SC4S_LISTEN_CEF_TLS_PORT=$SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT; fi +if [ ${SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT} ]; then export SC4S_ARCHIVE_CEF=$SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT; fi +if [ ${SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC} ]; then export SC4S_DEST_CEF_HEC=$SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC; fi + cd /opt/syslog-ng gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ From 362c3bee3ed06e2755a3a37e0b25b8f713ce72c3 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 24 Jan 2020 07:59:20 -0800 Subject: [PATCH 03/10] docs/CEF variable detail * Update CEF source doc (and CEF device docs) with note that CEF variables should be set only once for the _entire_ deployment. --- docs/sources/Arcsight/index.md | 8 ++++++++ docs/sources/CommonEventFormat/index.md | 11 ++++++++++- docs/sources/CyberArk/index.md | 8 ++++++++ docs/sources/Imperva/index.md | 4 ++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/sources/Arcsight/index.md b/docs/sources/Arcsight/index.md index 15b6dda..6724e31 100644 --- a/docs/sources/Arcsight/index.md +++ b/docs/sources/Arcsight/index.md @@ -37,6 +37,10 @@ MSG Parse: This filter parses message content | SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | | SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Deprecated equivalent of above variable. This is included for backward compatibility and will be removed in a future version. _Do not use_ in new installations. | +* NOTE: Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how +many ports are in use by this CEF source (or any others). See the "Common Event Format" source +documentation for more information. + ### Verification An active site will generate frequent events use the following search to check for new events @@ -90,6 +94,10 @@ MSG Parse: This filter parses message content | SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | | SC4S_WWW_XXX_MICROFOCUS_ARCSIGHT_YYY_ZZZ | no | Deprecated equivalents of the above variables. These are included for backward compatibility, and will be removed in a future version. _Do not use_ in new installations. | +* NOTE: Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how +many ports are in use by this CEF source (or any others). See the "Common Event Format" source +documentation for more information. + ### Verification An active site will generate frequent events use the following search to check for new events diff --git a/docs/sources/CommonEventFormat/index.md b/docs/sources/CommonEventFormat/index.md index d98e78b..b88b329 100644 --- a/docs/sources/CommonEventFormat/index.md +++ b/docs/sources/CommonEventFormat/index.md @@ -6,7 +6,16 @@ Each CEF product should have their own source entry in this documentation set. from normal configuration, all CEF products should use the "CEF" version of the unique port and archive envrionmetn variable settings (rather than a unique one per product), as the CEF log path handles all products sending events to SC4S in the CEF format. Examples of this include Arcsight, -Imperva, and Cyberark. +Imperva, and Cyberark. Therefore, the CEF environment varialbes for unique port, archive, etc. +should be set only _once_. + +If your deployment has multiple CEF devices that send to more than one port, +set the CEF unique port variable(s) to just one of the ports in use. Then, map the others with +container networking to the port chosen. Example: If you have three CEF devices, sending on TCP +ports 2000,2001, and 2002, set `SC4S_LISTEN_CEF_TCP_PORT=2000`. Then, map the other two with +container networking, e.g. `-p 2000:2000 -p 2001:2000 -p 2002:2000`. This will route all +three ports to TCP port 2000 inside the container, and the single CEF log path will properly +process data from all three devices. The source documentation included below is a reference baseline for any product that sends data using the CEF log path. diff --git a/docs/sources/CyberArk/index.md b/docs/sources/CyberArk/index.md index 1a113ea..40aee14 100644 --- a/docs/sources/CyberArk/index.md +++ b/docs/sources/CyberArk/index.md @@ -30,6 +30,10 @@ MSG Parse: This filter parses message content |----------------|----------------|----------------| | SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +* NOTE: Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how +many ports are in use by this CEF source (or any others). See the "Common Event Format" source +documentation for more information. + ### Verification An active site will generate frequent events use the following search to check for new events @@ -70,6 +74,10 @@ MSG Parse: This filter parses message content |----------------|----------------|----------------| | SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +* NOTE: Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how +many ports are in use by this CEF source (or any others). See the "Common Event Format" source +documentation for more information. + ### Verification An active site will generate frequent events use the following search to check for new events diff --git a/docs/sources/Imperva/index.md b/docs/sources/Imperva/index.md index ad0e0e9..1ba0667 100644 --- a/docs/sources/Imperva/index.md +++ b/docs/sources/Imperva/index.md @@ -42,6 +42,10 @@ Note listed for reference processing utilizes the Microsoft ArcSight log path as | SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source | | SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +* NOTE: Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how +many ports are in use by this CEF source (or any others). See the "Common Event Format" source +documentation for more information. + ### Verification An active site will generate frequent events use the following search to check for new events From 07d756a07c9a4472a4b9f5cbc00e5303a34fb13b Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 24 Jan 2020 21:05:17 -0500 Subject: [PATCH 04/10] Fixes #287 --- .../conf.d/conflib/_common/syslog_format.conf | 6 +++--- package/etc/go_templates/source_network.t | 18 ++++++++--------- tests/test_cyberark.py | 20 +++++++++++++++++++ tests/test_vmware.py | 4 ++-- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index b461e0e..304b03c 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -1,11 +1,11 @@ filter f_rfc5424_strict{ - message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); -}; + message('^\< *(?\d+) *\> *(?\d+)? *(?\d+) *- *(?\d+) *- *(?\d+)T(?\d+): *(?\d+):(?\d+)(?:\.(?\d+))?(?Z|(?: *)[\+-] *\d+:\d+) *(?(-)|[^ ]+) *(?(?:-)|\b\w+\b) *(?(?:-)|\b\w+\b) *(?(?:-)|\b\w+\b) *(?(?:-)|\[.*?\]) *(?(?:-)|\b.*)?$'); + }; filter f_rfc5424_noversion{ message('^(?(?
(?<\d{1,3}>) ?(?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); }; filter f_rfc3164_version{ - message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?[^ ]+) ))'); + message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) ))'); }; rewrite set_rfc5424_strict{ set("rfc5424_strict" value("fields.sc4s_syslog_format")); diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 480130a..7b201eb 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -92,18 +92,21 @@ source s_{{ .port_id }} { rewrite(set_no_parse); {{ else }} if { + filter(f_rfc5424_strict); + parser { + syslog-parser(flags(syslog-protocol)); + }; + rewrite(set_rfc5424_strict); + } elif { + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); + } elif { filter(f_rfc3164_version); rewrite(set_rfc3164_no_version_string); parser { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); }; rewrite(set_rfc3164_version); - } elif { - filter(f_rfc5424_strict); - parser { - syslog-parser(flags(syslog-protocol)); - }; - rewrite(set_rfc5424_strict); } elif { filter(f_rfc5424_noversion); parser { @@ -113,9 +116,6 @@ source s_{{ .port_id }} { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); - } elif { - parser (p_cisco_meraki); - rewrite(set_rfc5424_epochtime); } else { parser { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); diff --git a/tests/test_cyberark.py b/tests/test_cyberark.py index edb6985..c8ac7ba 100644 --- a/tests/test_cyberark.py +++ b/tests/test_cyberark.py @@ -12,6 +12,26 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) +#<5>1 2020-01-24T22:53:03Z REDACTEDHOSTNAME CEF:0|Cyber-Ark|Vault|10.9.0000|22|CPM Verify Password|5|act="CPM Verify Password" suser=PasswordManager fname=Root\Operating System-OBO-ISSO-Windows-Domain-Account-redacted dvc= shost=10.0.0.10 dhost= duser=redacted externalId= app= reason= cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2="re-dact-ted" cs3Label="Device Type" cs3="Operating System" cs4Label="Database" cs4= cs5Label="Other info" cs5= cn1Label="Request Id" cn1= cn2Label="Ticket Id" cn2="VerificationPeriod" msg="VerificationPeriod" +def test_cyberark_epv_5424(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}Z {{ host }} CEF:0|Cyber-Ark|Vault|9.20.0000|7|Logon|5|act=\"Logon\" suser=##USER_NAME## fname= dvc= shost=##SOURCE_IP## dhost= duser= externalId= app= reason= cs1Label=\"Affected User Name\" cs1= cs2Label=\"Safe Name\" cs2= cs3Label=\"Device Type\" cs3=11111 cs4Label=\"Database\" cs4=222222 cs5Label=\"Other info\" cs5= cn1Label=\"Request Id\" cn1= cn2Label=\"Ticket Id\" cn2= msg=\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cyberark:epv:cef\"| head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 #<190>Jul 27 23:31:58 VAULT CEF:0|Cyber-Ark|Vault|9.20.0000|7|Logon|5|act="Logon" suser=##USER_NAME## fname= dvc= shost=##SOURCE_IP## dhost= duser= externalId= app= reason= cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2= cs3Label="Device Type" cs3=11111 cs4Label="Database" cs4=222222 cs5Label="Other info" cs5= cn1Label="Request Id" cn1= cn2Label="Ticket Id" cn2= msg= def test_cyberark_epv(record_property, setup_wordlist, setup_splunk): diff --git a/tests/test_vmware.py b/tests/test_vmware.py index 4ed57ca..f128205 100644 --- a/tests/test_vmware.py +++ b/tests/test_vmware.py @@ -41,12 +41,12 @@ def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk): host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) - mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSXV {{ pid }} - [nsxv@6876 comp=\"nsx-manager\" subcomp=\"manager\"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704\n") + mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSXV NSX - SYSTEM [nsx@6876 comp=\"nsx-manager\" errorCode=\"MP4039\" subcomp=\"manager\"] Connection verification failed for broker '10.160.108.196'. Marking broker unhealthy.\n") message = mt.render(mark="<144>", host=host, pid=pid) sendsingle(message) - st = env.from_string("search index=main host={{ host }} PID={{ pid }} sourcetype=\"vmware:vsphere:nsx\" | head 2") + st = env.from_string("search index=main host={{ host }} sourcetype=\"vmware:vsphere:nsx\" | head 2") search = st.render(host=host, pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) From 9bb5a0e01371fd84f496971b2eb5f470c47a3ba3 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 24 Jan 2020 21:24:22 -0500 Subject: [PATCH 05/10] second fix vmware --- package/etc/conf.d/conflib/_common/syslog_format.conf | 2 +- tests/test_vmware.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index 304b03c..96ab5dc 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -1,5 +1,5 @@ filter f_rfc5424_strict{ - message('^\< *(?\d+) *\> *(?\d+)? *(?\d+) *- *(?\d+) *- *(?\d+)T(?\d+): *(?\d+):(?\d+)(?:\.(?\d+))?(?Z|(?: *)[\+-] *\d+:\d+) *(?(-)|[^ ]+) *(?(?:-)|\b\w+\b) *(?(?:-)|\b\w+\b) *(?(?:-)|\b\w+\b) *(?(?:-)|\[.*?\]) *(?(?:-)|\b.*)?$'); + message('^\<(?\d+)\>(?\d{1,2})? (?\d+)-(?\d+)-(?\d+)T(?\d+):(?\d+):(?\d+)(?:\.(?\d+))?(?Z|[\+-] *\d+:\d+) (?(-)|[^ ]+) (?(?:-)|\b\w+\b) (?(?:-)|\b\w+\b) (?(?:-)|\b\w+\b) *(?(?:-)|\[.*?\]) *(?(?:-)|\b.*)?$'); }; filter f_rfc5424_noversion{ message('^(?(?
(?<\d{1,3}>) ?(?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); diff --git a/tests/test_vmware.py b/tests/test_vmware.py index f128205..235798b 100644 --- a/tests/test_vmware.py +++ b/tests/test_vmware.py @@ -41,7 +41,7 @@ def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk): host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) - mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSXV NSX - SYSTEM [nsx@6876 comp=\"nsx-manager\" errorCode=\"MP4039\" subcomp=\"manager\"] Connection verification failed for broker '10.160.108.196'. Marking broker unhealthy.\n") + mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSX - SYSTEM [nsx@6876 comp=\"nsx-manager\" errorCode=\"MP4039\" subcomp=\"manager\"] Connection verification failed for broker '10.160.108.196'. Marking broker unhealthy.\n") message = mt.render(mark="<144>", host=host, pid=pid) sendsingle(message) From e5478f639f72e8b4ace8fc399eca04c3844f912b Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sat, 25 Jan 2020 09:34:44 -0800 Subject: [PATCH 06/10] Update test names/files * Remove prior (incorrect) change to Arcsight test file name/test names * Create a separate test file name/test name for Imperva --- docs/sources/Arcsight/index.md | 2 +- tests/test_imperva.py | 33 +++++++++++++++++++ ..._format.py => test_microfocus_arcsight.py} | 30 +++-------------- 3 files changed, 39 insertions(+), 26 deletions(-) create mode 100644 tests/test_imperva.py rename tests/{test_common_event_format.py => test_microfocus_arcsight.py} (87%) diff --git a/docs/sources/Arcsight/index.md b/docs/sources/Arcsight/index.md index 6724e31..953f3e6 100644 --- a/docs/sources/Arcsight/index.md +++ b/docs/sources/Arcsight/index.md @@ -51,7 +51,7 @@ Verify timestamp, and host values match as expected index= (sourcetype=cef source="ArcSight:ArcSight") ``` -## Product - Microsoft Windows (CEF) +## Product - Arcsight Microsoft Windows (CEF) | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| diff --git a/tests/test_imperva.py b/tests/test_imperva.py new file mode 100644 index 0000000..9a0005c --- /dev/null +++ b/tests/test_imperva.py @@ -0,0 +1,33 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +def test_imperva_incapsula(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Incapsula|SIEMintegration|1|1|Illegal Resource Access|3| fileid=3412341160002518171 sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support src=12.12.12.12 caIP=13.13.13.13 ccode=IL tag=www.elvis.com cn1=200 in=54 xff=44.44.44.44 cs1=NOT_SUPPORTED cs1Label=Cap Support cs4=c2e72124-0e8a-4dd8-b13b-3da246af3ab2 cs4Label=VID cs5=de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4 cs5Label=clappsigdproc=Browser cs6=Firefox cs6Label=clapp ccode=IL cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 start=1453290121336 request=site123.abcd.info/ requestmethod=GET qstr=p\=%2fetc%2fpasswd app=HTTP act=REQ_CHALLENGE_CAPTCHA deviceExternalID=33411452762204224 cpt=443 filetype=30037,1001, filepermission=2,1, cs9=Block Malicious User,High Risk Resources, cs9Label=Rule name' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netwaf host=\"{{ host }}\" sourcetype=\"cef\" source=\"Imperva:Incapsula\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_common_event_format.py b/tests/test_microfocus_arcsight.py similarity index 87% rename from tests/test_common_event_format.py rename to tests/test_microfocus_arcsight.py index 510f46c..507db99 100644 --- a/tests/test_common_event_format.py +++ b/tests/test_microfocus_arcsight.py @@ -16,7 +16,7 @@ # Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=30 msg=Connected to Host mrt=1539321123071 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321124967 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321123071 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= # Mar 19 15:19:15 root CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:030|Agent [PAN1_WUC_UDP8000] type [windowsfg] started|Low| eventId=26 mrt=1539321122832 categorySignificance=/Normal categoryBehavior=/Execute/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application/Service art=1539321124967 cat=/Agent/Started deviceSeverity=Warning rt=1539321122832 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= # Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321047341 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= -def test_cef_ts_rt(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_ts_rt(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -36,7 +36,7 @@ def test_cef_ts_rt(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_cef_ts_end(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_ts_end(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -56,7 +56,7 @@ def test_cef_ts_end(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_cef_ts_syslog(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_ts_syslog(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -76,7 +76,7 @@ def test_cef_ts_syslog(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_cef_windows(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_windows(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -96,7 +96,7 @@ def test_cef_windows(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_cef_windows_system(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_windows_system(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -115,23 +115,3 @@ def test_cef_windows_system(record_property, setup_wordlist, setup_splunk): record_property("message", message) assert resultCount == 1 - -def test_cef_imperva_incapsula(record_property, setup_wordlist, setup_splunk): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - - mt = env.from_string( - "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Incapsula|SIEMintegration|1|1|Illegal Resource Access|3| fileid=3412341160002518171 sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support src=12.12.12.12 caIP=13.13.13.13 ccode=IL tag=www.elvis.com cn1=200 in=54 xff=44.44.44.44 cs1=NOT_SUPPORTED cs1Label=Cap Support cs4=c2e72124-0e8a-4dd8-b13b-3da246af3ab2 cs4Label=VID cs5=de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4 cs5Label=clappsigdproc=Browser cs6=Firefox cs6Label=clapp ccode=IL cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 start=1453290121336 request=site123.abcd.info/ requestmethod=GET qstr=p\=%2fetc%2fpasswd app=HTTP act=REQ_CHALLENGE_CAPTCHA deviceExternalID=33411452762204224 cpt=443 filetype=30037,1001, filepermission=2,1, cs9=Block Malicious User,High Risk Resources, cs9Label=Rule name' + "\n") - message = mt.render(mark="<111>", host=host) - - sendsingle(message) - - st = env.from_string("search index=netwaf host=\"{{ host }}\" sourcetype=\"cef\" source=\"Imperva:Incapsula\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 From 80846d77ae182352c2f8c27335a4c8a291acbcc2 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sat, 25 Jan 2020 10:55:37 -0800 Subject: [PATCH 07/10] Update docs for Microfocus * Replace "Arcsight" with "Microfocus" in docs sources and TOC --- docs/sources/{Arcsight => Microfocus}/index.md | 0 mkdocs.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/sources/{Arcsight => Microfocus}/index.md (100%) diff --git a/docs/sources/Arcsight/index.md b/docs/sources/Microfocus/index.md similarity index 100% rename from docs/sources/Arcsight/index.md rename to docs/sources/Microfocus/index.md diff --git a/mkdocs.yml b/mkdocs.yml index 5a467aa..eb1ad96 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,7 +12,6 @@ nav: - Configuration: 'configuration.md' - Sources: - About: sources/index.md - - ArcSight: sources/Arcsight/index.md - Checkpoint: sources/Checkpoint/index.md - Cisco: sources/Cisco/index.md - 'Common Event Format': sources/CommonEventFormat/index.md @@ -21,6 +20,7 @@ nav: - Fortinet: sources/Fortinet/index.md - Imperva: sources/Imperva/index.md - Juniper: sources/Juniper/index.md + - Microfocus: sources/Microfocus/index.md - Nix: sources/nix/index.md - 'Palo Alto Networks': sources/PaloaltoNetworks/index.md - Proofpoint: sources/Proofpoint/index.md From fecc5497af9ff93be000e614ad5c0bb039f5801d Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Sat, 25 Jan 2020 17:40:52 -0500 Subject: [PATCH 08/10] Support pfsense --- docker-compose.yml | 1 + docs/sources/Pfsense/index.md | 57 +++++++++++++++ mkdocs.yml | 1 + .../etc/conf.d/filters/pfsense/syslog.conf | 4 ++ .../etc/conf.d/log_paths/lp-pfsense.conf.tmpl | 58 +++++++++++++++ .../etc/context_templates/splunk_index.csv | 2 + .../vendor_product_by_source.conf | 4 ++ .../vendor_product_by_source.csv | 1 + tests/test_pfsense.py | 72 +++++++++++++++++++ 9 files changed, 200 insertions(+) create mode 100644 docs/sources/Pfsense/index.md create mode 100644 package/etc/conf.d/filters/pfsense/syslog.conf create mode 100644 package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl create mode 100644 tests/test_pfsense.py diff --git a/docker-compose.yml b/docker-compose.yml index 3d936bc..323da58 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,6 +53,7 @@ services: - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003 - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 + - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 - SC4S_ARCHIVE_GLOBAL=yes volumes: - ./tls:/opt/syslog-ng/tls diff --git a/docs/sources/Pfsense/index.md b/docs/sources/Pfsense/index.md new file mode 100644 index 0000000..46e1af4 --- /dev/null +++ b/docs/sources/Pfsense/index.md @@ -0,0 +1,57 @@ +# Vendor - pfSense + +All pfSense based firewalls + + +## Product + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/1527/ | +| Product Manual | https://docs.netgate.com/pfsense/en/latest/monitoring/copying-logs-to-a-remote-host-with-syslog.html?highlight=syslog | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| pfsense:filterlog | None | +| pfsense:* | All programs other than filterlog | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| pfsense | pfsense | netops | none | +| pfsense_filterlog | pfsense:filterlog | netfw | none | + +### Filter type + +Source does not provide a hostname, port or IP based filter is required + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Configure a dedicated SC4S port OR configure IP filter +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_PFSENSE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_PFSENSE_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_PFSENSE | no | Enable archive to disk for this specific source | +| SC4S_DEST_PFSENSE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=pfsense:filterlog | stats count by host +``` diff --git a/mkdocs.yml b/mkdocs.yml index 3407538..1d356c6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,6 +22,7 @@ nav: - Nix: sources/nix/index.md - Microfocus: sources/Microfocus/index.md - 'Paloalto Networks': sources/PaloaltoNetworks/index.md + - 'pfSense': sources/pfSense/index.md - Proofpoint: sources/Proofpoint/index.md - Symantec: sources/Symantec/index.md - Ubiquiti: sources/Ubiquiti/index.md diff --git a/package/etc/conf.d/filters/pfsense/syslog.conf b/package/etc/conf.d/filters/pfsense/syslog.conf new file mode 100644 index 0000000..bd6f6fa --- /dev/null +++ b/package/etc/conf.d/filters/pfsense/syslog.conf @@ -0,0 +1,4 @@ +filter f_pfsense { + match("^pfsense", value("fields.sc4s_vendor_product")); + +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl b/package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl new file mode 100644 index 0000000..b9ea159 --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl @@ -0,0 +1,58 @@ +# pfSense +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "PFSENSE" "parser" "rfc3164" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +log { + junction { +{{- if or (or (getenv (print "SC4S_LISTEN_PFSENSE_TCP_PORT")) (getenv (print "SC4S_LISTEN_PFSENSE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PFSENSE_TLS_PORT")) }} + channel { + # Listen on the specified dedicated port(s) for PFSENSE traffic + source (s_PFSENSE); + flags (final); + }; +{{- end}} + channel { + # Listen on the default port (typically 514) for PFSENSE traffic + source (s_DEFAULT); + filter(f_is_rfc3164); + filter(f_pfsense); + flags(final); + }; + }; + + if { + filter{program("filterlog")}; + rewrite { + set("pfsense_filterlog", value("fields.sc4s_vendor_product")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("pfsense:filterlog"), index("netfw"), source("program:${.PROGRAM}")) + }; + parser { p_add_context_splunk(key("pfsense_filterlog")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + } else { + rewrite { + set("pfsense", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("pfsense:${.PROGRAM}"), index("netops"), source("program:${.PROGRAM}")) + }; + parser { p_add_context_splunk(key("pfsense")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; + }; + + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_PFSENSE_HEC" "no")) }} + destination(d_hec); +{{- end}} + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_PFSENSE" "no")) }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index 51b71c0..41c7121 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -48,6 +48,8 @@ #pan_correlation,index,main #pan_userid,index,netauth #pan_unknown,index,netops +#pfsense,index,netops +#pfsense_filterlog,index,netfw #proofpoint_pps_filter,index,email #proofpoint_pps_sendmail,index,email #sc4s_events,index,main diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 1f5f13f..591fa77 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -30,6 +30,10 @@ filter f_infoblox { host("vib-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) }; +filter f_pfsense { + host("pfsense-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) +}; filter f_proofpoint_pps_filter { host("pps-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 510cc19..193732e 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -6,6 +6,7 @@ f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" f_juniper_idp,sc4s_vendor_product,"juniper_idp" f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" +f_pfsense,sc4s_vendor_product,"pfsense" f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" diff --git a/tests/test_pfsense.py b/tests/test_pfsense.py new file mode 100644 index 0000000..a5cc41b --- /dev/null +++ b/tests/test_pfsense.py @@ -0,0 +1,72 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +#<27>Jan 25 01:58:06 filterlog: 82,,,1000002666,mvneta2,match,pass,out,6,0x00,0x00000,64,ICMPv6,58,8,fe80::208:a2ff:fe0f:cb66,fe80::56a6:5cff:fe7d:1d43, +def test_pfsense_filterlog(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} filterlog: 82,,,1000002666,mvneta2,match,pass,out,6,0x00,0x00000,64,ICMPv6,58,8,{{key}},\n") + message = mt.render(mark="<27>", key=host) + sendsingle(message, port=5006) + + st = env.from_string("search index=netfw sourcetype=pfsense:filterlog \"{{key}}\" earliest=-2m | head 2") + search = st.render(key=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<27>Jan 25 01:58:06 kqueue error: unknown +def test_pfsense_other(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} kqueue error: {{key}}\n") + message = mt.render(mark="<27>", key=host) + sendsingle(message, port=5006) + + st = env.from_string("search index=netops sourcetype=pfsense:* \"{{key}}\" earliest=-2m | head 2") + search = st.render(key=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<27>Jan 25 01:58:06 syslogd: restart +def test_pfsense_syslogd(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} syslogd: restart {{key}}\n") + message = mt.render(mark="<27>", key=host) + sendsingle(message, port=5006) + + st = env.from_string("search index=netops sourcetype=pfsense:syslogd \"{{key}}\" earliest=-2m | head 2") + search = st.render(key=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file From 0f3c4befd7920438156a8b8252691e47c44988af Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Sat, 25 Jan 2020 18:54:39 -0500 Subject: [PATCH 09/10] update filename and ci vars --- docker-compose-ci.yml | 2 ++ .../etc/conf.d/filters/infoblox/{syslog.conf => pfsense.conf} | 0 2 files changed, 2 insertions(+) rename package/etc/conf.d/filters/infoblox/{syslog.conf => pfsense.conf} (100%) diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index 7566f3d..1bd512b 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -45,6 +45,8 @@ services: - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 + - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 + splunk: image: splunk/splunk:latest hostname: splunk diff --git a/package/etc/conf.d/filters/infoblox/syslog.conf b/package/etc/conf.d/filters/infoblox/pfsense.conf similarity index 100% rename from package/etc/conf.d/filters/infoblox/syslog.conf rename to package/etc/conf.d/filters/infoblox/pfsense.conf From dce928e408f8c993b786940502c387daf7c088ba Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Sat, 25 Jan 2020 19:01:26 -0500 Subject: [PATCH 10/10] Update mkdocs.yml --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index c546229..869a54c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,7 +22,7 @@ nav: - Juniper: sources/Juniper/index.md - Microfocus: sources/Microfocus/index.md - Nix: sources/nix/index.md - - 'Paloalto Networks': sources/PaloaltoNetworks/index.md + - 'Palo Alto Networks': sources/PaloaltoNetworks/index.md - 'pfSense': sources/pfSense/index.md - Proofpoint: sources/Proofpoint/index.md - Symantec: sources/Symantec/index.md