diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index 7566f3d..1bd512b 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -45,6 +45,8 @@ services: - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 + - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 + splunk: image: splunk/splunk:latest hostname: splunk diff --git a/docker-compose.yml b/docker-compose.yml index 3d936bc..323da58 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,6 +53,7 @@ services: - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003 - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 + - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 - SC4S_ARCHIVE_GLOBAL=yes volumes: - ./tls:/opt/syslog-ng/tls diff --git a/docs/sources/CommonEventFormat/index.md b/docs/sources/CommonEventFormat/index.md new file mode 100644 index 0000000..b88b329 --- /dev/null +++ b/docs/sources/CommonEventFormat/index.md @@ -0,0 +1,70 @@ +# Vendor - Common Event Format Data Sources + +## Product - Various products that send CEF-format messages via syslog + +Each CEF product should have their own source entry in this documentation set. In a departure +from normal configuration, all CEF products should use the "CEF" version of the unique port and +archive envrionmetn variable settings (rather than a unique one per product), as the CEF log path +handles all products sending events to SC4S in the CEF format. Examples of this include Arcsight, +Imperva, and Cyberark. Therefore, the CEF environment varialbes for unique port, archive, etc. +should be set only _once_. + +If your deployment has multiple CEF devices that send to more than one port, +set the CEF unique port variable(s) to just one of the ports in use. Then, map the others with +container networking to the port chosen. Example: If you have three CEF devices, sending on TCP +ports 2000,2001, and 2002, set `SC4S_LISTEN_CEF_TCP_PORT=2000`. Then, map the other two with +container networking, e.g. `-p 2000:2000 -p 2001:2000 -p 2002:2000`. This will route all +three ports to TCP port 2000 inside the container, and the single CEF log path will properly +process data from all three devices. + +The source documentation included below is a reference baseline for any product that sends data +using the CEF log path. + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/ | +| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cef | Common sourcetype | + +### Typical Source + +| source | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| Varies | Varies | + +### Typical Index Configuration + +| key | source | index | notes | +|----------------|----------------|----------------|----------------| +| Vendor_Product | Varies | main | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CEF_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source | +| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active site will generate frequent events use the following search to check for new events + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=cef source=) +``` diff --git a/docs/sources/CyberArk/index.md b/docs/sources/CyberArk/index.md index dd497d0..40aee14 100644 --- a/docs/sources/CyberArk/index.md +++ b/docs/sources/CyberArk/index.md @@ -28,7 +28,11 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +* NOTE: Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how +many ports are in use by this CEF source (or any others). See the "Common Event Format" source +documentation for more information. ### Verification @@ -68,7 +72,11 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +* NOTE: Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how +many ports are in use by this CEF source (or any others). See the "Common Event Format" source +documentation for more information. ### Verification diff --git a/docs/sources/Imperva/index.md b/docs/sources/Imperva/index.md index 2ae9eea..1ba0667 100644 --- a/docs/sources/Imperva/index.md +++ b/docs/sources/Imperva/index.md @@ -25,7 +25,7 @@ | key | source | index | notes | |----------------|----------------|----------------|----------------| -| cef_Incapsula_SIEMintegration | Imperva:Incapsula | netwaf | none | +| Incapsula_SIEMintegration | Imperva:Incapsula | netwaf | none | ### Filter type @@ -37,10 +37,14 @@ Note listed for reference processing utilizes the Microsoft ArcSight log path as | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT | no | Enable archive to disk for this specific source | -| SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source | +| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +* NOTE: Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how +many ports are in use by this CEF source (or any others). See the "Common Event Format" source +documentation for more information. ### Verification @@ -50,4 +54,4 @@ Verify timestamp, and host values match as expected ``` index= (sourcetype=cef source="Imperva:Incapsula") -``` \ No newline at end of file +``` diff --git a/docs/sources/Microfocus/index.md b/docs/sources/Microfocus/index.md index 5909324..953f3e6 100644 --- a/docs/sources/Microfocus/index.md +++ b/docs/sources/Microfocus/index.md @@ -1,6 +1,6 @@ -# Vendor - Microfocus ArcSight +# Vendor - MicroFocus Arcsight -## Product - Internal Agent Events +## Product - Arcsight Internal Agent | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| @@ -24,7 +24,7 @@ | key | source | index | notes | |----------------|----------------|----------------|----------------| -| cef_ArcSight_ArcSight | ArcSight:ArcSight | main | none | +| ArcSight_ArcSight | ArcSight:ArcSight | main | none | ### Filter type @@ -34,7 +34,12 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Deprecated equivalent of above variable. This is included for backward compatibility and will be removed in a future version. _Do not use_ in new installations. | + +* NOTE: Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how +many ports are in use by this CEF source (or any others). See the "Common Event Format" source +documentation for more information. ### Verification @@ -46,7 +51,7 @@ Verify timestamp, and host values match as expected index= (sourcetype=cef source="ArcSight:ArcSight") ``` -## Product - Microsoft Windows +## Product - Arcsight Microsoft Windows (CEF) | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| @@ -72,8 +77,8 @@ index= (sourcetype=cef source="ArcSight:ArcSight") | key | source | index | notes | |----------------|----------------|----------------|----------------| -| cef_Microsoft_System or Application Event | CEFEventLog:System or Application Event | oswin | none | -| cef_Microsoft_Microsoft Windows | CEFEventLog:Microsoft Windows | oswinsec | none | +| Microsoft_System or Application Event | CEFEventLog:System or Application Event | oswin | none | +| Microsoft_Microsoft Windows | CEFEventLog:Microsoft Windows | oswinsec | none | ### Filter type @@ -83,10 +88,15 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | -| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT | no | Enable archive to disk for this specific source | -| SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_CEF_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CEF_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source | +| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_WWW_XXX_MICROFOCUS_ARCSIGHT_YYY_ZZZ | no | Deprecated equivalents of the above variables. These are included for backward compatibility, and will be removed in a future version. _Do not use_ in new installations. | + +* NOTE: Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how +many ports are in use by this CEF source (or any others). See the "Common Event Format" source +documentation for more information. ### Verification @@ -96,4 +106,4 @@ Verify timestamp, and host values match as expected ``` index= (sourcetype=cef (source="CEFEventLog:Microsoft Windows" OR source="CEFEventLog:System or Application Event")) -``` \ No newline at end of file +``` diff --git a/docs/sources/Pfsense/index.md b/docs/sources/Pfsense/index.md new file mode 100644 index 0000000..46e1af4 --- /dev/null +++ b/docs/sources/Pfsense/index.md @@ -0,0 +1,57 @@ +# Vendor - pfSense + +All pfSense based firewalls + + +## Product + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/1527/ | +| Product Manual | https://docs.netgate.com/pfsense/en/latest/monitoring/copying-logs-to-a-remote-host-with-syslog.html?highlight=syslog | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| pfsense:filterlog | None | +| pfsense:* | All programs other than filterlog | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| pfsense | pfsense | netops | none | +| pfsense_filterlog | pfsense:filterlog | netfw | none | + +### Filter type + +Source does not provide a hostname, port or IP based filter is required + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Configure a dedicated SC4S port OR configure IP filter +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_PFSENSE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_PFSENSE_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_PFSENSE | no | Enable archive to disk for this specific source | +| SC4S_DEST_PFSENSE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=pfsense:filterlog | stats count by host +``` diff --git a/mkdocs.yml b/mkdocs.yml index 3407538..869a54c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,14 +14,16 @@ nav: - About: sources/index.md - Checkpoint: sources/Checkpoint/index.md - Cisco: sources/Cisco/index.md + - 'Common Event Format': sources/CommonEventFormat/index.md - CyberArk: sources/CyberArk/index.md - Forcepoint: sources/Forcepoint/index.md - Fortinet: sources/Fortinet/index.md - Imperva: sources/Imperva/index.md - Juniper: sources/Juniper/index.md - - Nix: sources/nix/index.md - Microfocus: sources/Microfocus/index.md - - 'Paloalto Networks': sources/PaloaltoNetworks/index.md + - Nix: sources/nix/index.md + - 'Palo Alto Networks': sources/PaloaltoNetworks/index.md + - 'pfSense': sources/pfSense/index.md - Proofpoint: sources/Proofpoint/index.md - Symantec: sources/Symantec/index.md - Ubiquiti: sources/Ubiquiti/index.md diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index b461e0e..96ab5dc 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -1,11 +1,11 @@ filter f_rfc5424_strict{ - message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); -}; + message('^\<(?\d+)\>(?\d{1,2})? (?\d+)-(?\d+)-(?\d+)T(?\d+):(?\d+):(?\d+)(?:\.(?\d+))?(?Z|[\+-] *\d+:\d+) (?(-)|[^ ]+) (?(?:-)|\b\w+\b) (?(?:-)|\b\w+\b) (?(?:-)|\b\w+\b) *(?(?:-)|\[.*?\]) *(?(?:-)|\b.*)?$'); + }; filter f_rfc5424_noversion{ message('^(?(?
(?<\d{1,3}>) ?(?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); }; filter f_rfc3164_version{ - message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?[^ ]+) ))'); + message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) ))'); }; rewrite set_rfc5424_strict{ set("rfc5424_strict" value("fields.sc4s_syslog_format")); diff --git a/package/etc/conf.d/context/microfocus_arcsight_source.csv b/package/etc/conf.d/context/common_event_format_source.csv similarity index 100% rename from package/etc/conf.d/context/microfocus_arcsight_source.csv rename to package/etc/conf.d/context/common_event_format_source.csv diff --git a/package/etc/conf.d/filters/common_event_format/cef.conf b/package/etc/conf.d/filters/common_event_format/cef.conf new file mode 100644 index 0000000..e180b31 --- /dev/null +++ b/package/etc/conf.d/filters/common_event_format/cef.conf @@ -0,0 +1,4 @@ + +filter f_cef { + program(CEF); +}; diff --git a/package/etc/conf.d/filters/infoblox/syslog.conf b/package/etc/conf.d/filters/infoblox/pfsense.conf similarity index 100% rename from package/etc/conf.d/filters/infoblox/syslog.conf rename to package/etc/conf.d/filters/infoblox/pfsense.conf diff --git a/package/etc/conf.d/filters/microfocus/arcsight.conf b/package/etc/conf.d/filters/microfocus/arcsight.conf deleted file mode 100644 index 287d7a4..0000000 --- a/package/etc/conf.d/filters/microfocus/arcsight.conf +++ /dev/null @@ -1,4 +0,0 @@ - -filter f_microfocus_arcsight { - program(CEF); -}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/pfsense/syslog.conf b/package/etc/conf.d/filters/pfsense/syslog.conf new file mode 100644 index 0000000..bd6f6fa --- /dev/null +++ b/package/etc/conf.d/filters/pfsense/syslog.conf @@ -0,0 +1,4 @@ +filter f_pfsense { + match("^pfsense", value("fields.sc4s_vendor_product")); + +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl similarity index 68% rename from package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl rename to package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl index fd5a97a..64e9577 100644 --- a/package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl @@ -1,9 +1,9 @@ -# Microfocus ArcSight +# Common Event Format {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "rfc3164" }} +{{- $context := dict "port_id" "CEF" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} -parser p_microfocus_arcsight_header { +parser p_cef_header { csv-parser( columns("fields.sc4s_cef_version", "fields.cef_device_vendor", "fields.cef_device_product", "fields.cef_device_version", "fields.cef_device_event_class", "fields.cef_name", "fields.cef_severity", MESSAGE) delimiters(chars("|")) @@ -15,19 +15,19 @@ parser p_microfocus_arcsight_header { }; -parser p_microfocus_arcsight_ts_rt { +parser p_cef_ts_rt { date-parser(format("%s") template("${.cef.rt}") ); }; -parser p_microfocus_arcsight_ts_end { +parser p_cef_ts_end { date-parser(format("%s") template("${.cef.end}") ); }; -parser p_microfocus_arcsight_source { +parser p_cef_source { add-contextual-data( selector("${fields.cef_device_vendor}_${fields.cef_device_product}"), - database("conf.d/context/microfocus_arcsight_source.csv") + database("conf.d/context/common_event_format_source.csv") ignore-case(yes) prefix(".splunk.") default-selector("unknown") @@ -36,18 +36,18 @@ parser p_microfocus_arcsight_source { log { junction { -{{- if or (or (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT")) (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT"))) (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT")) }} +{{- if or (or (getenv (print "SC4S_LISTEN_CEF_TCP_PORT")) (getenv (print "SC4S_LISTEN_CEF_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CEF_TLS_PORT")) }} channel { - # Listen on the specified dedicated port(s) for MICROFOCUS_ARCSIGHT traffic - source (s_MICROFOCUS_ARCSIGHT); + # Listen on the specified dedicated port(s) for CEF traffic + source (s_CEF); flags (final); }; {{- end}} channel { - # Listen on the default port (typically 514) for MICROFOCUS_ARCSIGHT traffic + # Listen on the default port (typically 514) for CEF traffic source (s_DEFAULT); filter(f_is_rfc3164); - filter(f_microfocus_arcsight); + filter(f_cef); flags(final); }; }; @@ -56,7 +56,7 @@ log { r_set_splunk_dest_default(sourcetype("cef"), index("main")) }; - parser (p_microfocus_arcsight_header); + parser (p_cef_header); rewrite { set("${fields.cef_device_vendor}_${fields.cef_device_product}", value("fields.sc4s_vendor_product")); @@ -70,13 +70,13 @@ log { # If we have an rt or end field that is best we use the If trick here so if this parser fails # We don't get sent to fallback. if { - parser (p_microfocus_arcsight_ts_rt); + parser (p_cef_ts_rt); } elif { - parser (p_microfocus_arcsight_ts_end); + parser (p_cef_ts_end); } else {}; #Do nothing this is allows for both rt and end to be missing and still pass with the message ts #CEF TAs use the source as their bounds in props.conf - parser(p_microfocus_arcsight_source); + parser(p_cef_source); parser (compliance_meta_by_source); @@ -85,11 +85,11 @@ log { #if we don't rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; -{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC" "no")) }} +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CEF_HEC" "no")) }} destination(d_hec); {{- end}} -{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT" "no")) }} +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CEF" "no")) }} destination(d_archive); {{- end}} diff --git a/package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl b/package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl new file mode 100644 index 0000000..b9ea159 --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl @@ -0,0 +1,58 @@ +# pfSense +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "PFSENSE" "parser" "rfc3164" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +log { + junction { +{{- if or (or (getenv (print "SC4S_LISTEN_PFSENSE_TCP_PORT")) (getenv (print "SC4S_LISTEN_PFSENSE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PFSENSE_TLS_PORT")) }} + channel { + # Listen on the specified dedicated port(s) for PFSENSE traffic + source (s_PFSENSE); + flags (final); + }; +{{- end}} + channel { + # Listen on the default port (typically 514) for PFSENSE traffic + source (s_DEFAULT); + filter(f_is_rfc3164); + filter(f_pfsense); + flags(final); + }; + }; + + if { + filter{program("filterlog")}; + rewrite { + set("pfsense_filterlog", value("fields.sc4s_vendor_product")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("pfsense:filterlog"), index("netfw"), source("program:${.PROGRAM}")) + }; + parser { p_add_context_splunk(key("pfsense_filterlog")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + } else { + rewrite { + set("pfsense", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("pfsense:${.PROGRAM}"), index("netops"), source("program:${.PROGRAM}")) + }; + parser { p_add_context_splunk(key("pfsense")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; + }; + + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_PFSENSE_HEC" "no")) }} + destination(d_hec); +{{- end}} + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_PFSENSE" "no")) }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index 51b71c0..6d36fa8 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -1,8 +1,10 @@ #bluecoat_proxy,index,netproxy -#cef_ArcSight_ArcSight,index,netwaf -#cef_Incapsula_SIEMintegration,index,netwaf -#cef_Microsoft_Microsoft Windows,index,oswinsec -#cef_Microsoft_System or Application Event,index,oswin +#ArcSight_ArcSight,index,netwaf +#Cyber-Ark_Vault,index,netauth +#CyberArk_PTA,index,main +#Incapsula_SIEMintegration,index,netwaf +#Microsoft_Microsoft Windows,index,oswinsec +#Microsoft_System or Application Event,index,oswin #checkpoint_splunk,index,netops #checkpoint_splunk_dlp,index,netdlp #checkpoint_splunk_email,index,email @@ -48,6 +50,8 @@ #pan_correlation,index,main #pan_userid,index,netauth #pan_unknown,index,netops +#pfsense,index,netops +#pfsense_filterlog,index,netfw #proofpoint_pps_filter,index,email #proofpoint_pps_sendmail,index,email #sc4s_events,index,main diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 1f5f13f..591fa77 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -30,6 +30,10 @@ filter f_infoblox { host("vib-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) }; +filter f_pfsense { + host("pfsense-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) +}; filter f_proofpoint_pps_filter { host("pps-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 510cc19..193732e 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -6,6 +6,7 @@ f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" f_juniper_idp,sc4s_vendor_product,"juniper_idp" f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" +f_pfsense,sc4s_vendor_product,"pfsense" f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 480130a..7b201eb 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -92,18 +92,21 @@ source s_{{ .port_id }} { rewrite(set_no_parse); {{ else }} if { + filter(f_rfc5424_strict); + parser { + syslog-parser(flags(syslog-protocol)); + }; + rewrite(set_rfc5424_strict); + } elif { + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); + } elif { filter(f_rfc3164_version); rewrite(set_rfc3164_no_version_string); parser { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); }; rewrite(set_rfc3164_version); - } elif { - filter(f_rfc5424_strict); - parser { - syslog-parser(flags(syslog-protocol)); - }; - rewrite(set_rfc5424_strict); } elif { filter(f_rfc5424_noversion); parser { @@ -113,9 +116,6 @@ source s_{{ .port_id }} { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); - } elif { - parser (p_cisco_meraki); - rewrite(set_rfc5424_epochtime); } else { parser { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index b7d5b2d..229a384 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -1,6 +1,14 @@ #!/usr/bin/env bash source scl_source enable rh-python36 +# The MICROFOCUS_ARCSIGHT unique port environment variables are currently deprecated +# This will be removed when the MICROFOCUS_ARCSIGHT unique port environment variables are removed in version 2.0 +if [ ${SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT} ]; then export SC4S_LISTEN_CEF_UDP_PORT=$SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT; fi +if [ ${SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT} ]; then export SC4S_LISTEN_CEF_TCP_PORT=$SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT; fi +if [ ${SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT} ]; then export SC4S_LISTEN_CEF_TLS_PORT=$SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT; fi +if [ ${SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT} ]; then export SC4S_ARCHIVE_CEF=$SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT; fi +if [ ${SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC} ]; then export SC4S_DEST_CEF_HEC=$SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC; fi + cd /opt/syslog-ng gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ diff --git a/tests/test_cyberark.py b/tests/test_cyberark.py index edb6985..c8ac7ba 100644 --- a/tests/test_cyberark.py +++ b/tests/test_cyberark.py @@ -12,6 +12,26 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) +#<5>1 2020-01-24T22:53:03Z REDACTEDHOSTNAME CEF:0|Cyber-Ark|Vault|10.9.0000|22|CPM Verify Password|5|act="CPM Verify Password" suser=PasswordManager fname=Root\Operating System-OBO-ISSO-Windows-Domain-Account-redacted dvc= shost=10.0.0.10 dhost= duser=redacted externalId= app= reason= cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2="re-dact-ted" cs3Label="Device Type" cs3="Operating System" cs4Label="Database" cs4= cs5Label="Other info" cs5= cn1Label="Request Id" cn1= cn2Label="Ticket Id" cn2="VerificationPeriod" msg="VerificationPeriod" +def test_cyberark_epv_5424(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}Z {{ host }} CEF:0|Cyber-Ark|Vault|9.20.0000|7|Logon|5|act=\"Logon\" suser=##USER_NAME## fname= dvc= shost=##SOURCE_IP## dhost= duser= externalId= app= reason= cs1Label=\"Affected User Name\" cs1= cs2Label=\"Safe Name\" cs2= cs3Label=\"Device Type\" cs3=11111 cs4Label=\"Database\" cs4=222222 cs5Label=\"Other info\" cs5= cn1Label=\"Request Id\" cn1= cn2Label=\"Ticket Id\" cn2= msg=\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cyberark:epv:cef\"| head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 #<190>Jul 27 23:31:58 VAULT CEF:0|Cyber-Ark|Vault|9.20.0000|7|Logon|5|act="Logon" suser=##USER_NAME## fname= dvc= shost=##SOURCE_IP## dhost= duser= externalId= app= reason= cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2= cs3Label="Device Type" cs3=11111 cs4Label="Database" cs4=222222 cs5Label="Other info" cs5= cn1Label="Request Id" cn1= cn2Label="Ticket Id" cn2= msg= def test_cyberark_epv(record_property, setup_wordlist, setup_splunk): diff --git a/tests/test_imperva.py b/tests/test_imperva.py new file mode 100644 index 0000000..9a0005c --- /dev/null +++ b/tests/test_imperva.py @@ -0,0 +1,33 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +def test_imperva_incapsula(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Incapsula|SIEMintegration|1|1|Illegal Resource Access|3| fileid=3412341160002518171 sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support src=12.12.12.12 caIP=13.13.13.13 ccode=IL tag=www.elvis.com cn1=200 in=54 xff=44.44.44.44 cs1=NOT_SUPPORTED cs1Label=Cap Support cs4=c2e72124-0e8a-4dd8-b13b-3da246af3ab2 cs4Label=VID cs5=de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4 cs5Label=clappsigdproc=Browser cs6=Firefox cs6Label=clapp ccode=IL cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 start=1453290121336 request=site123.abcd.info/ requestmethod=GET qstr=p\=%2fetc%2fpasswd app=HTTP act=REQ_CHALLENGE_CAPTCHA deviceExternalID=33411452762204224 cpt=443 filetype=30037,1001, filepermission=2,1, cs9=Block Malicious User,High Risk Resources, cs9Label=Rule name' + "\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netwaf host=\"{{ host }}\" sourcetype=\"cef\" source=\"Imperva:Incapsula\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_microfocus_arcsight_cef.py b/tests/test_microfocus_arcsight.py similarity index 88% rename from tests/test_microfocus_arcsight_cef.py rename to tests/test_microfocus_arcsight.py index eb3dd6d..507db99 100644 --- a/tests/test_microfocus_arcsight_cef.py +++ b/tests/test_microfocus_arcsight.py @@ -16,7 +16,7 @@ # Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=30 msg=Connected to Host mrt=1539321123071 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321124967 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321123071 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= # Mar 19 15:19:15 root CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:030|Agent [PAN1_WUC_UDP8000] type [windowsfg] started|Low| eventId=26 mrt=1539321122832 categorySignificance=/Normal categoryBehavior=/Execute/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application/Service art=1539321124967 cat=/Agent/Started deviceSeverity=Warning rt=1539321122832 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= # Mar 19 15:19:15 syslog1 CEF:0|ArcSight|ArcSight|7.9.0.8084.0|agent:016|Device connection up|Low| eventId=77 msg=Connected to Host mrt=1539321047341 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Management categoryOutcome=/Success categoryObject=/Host/Application art=1539321049259 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1539321047341 dhost=WIN-PAN1 dst=192.168.13.152 destinationZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 fileType=Agent cs2= cs2Label=Configuration Resource ahost=win-pan1 agt=192.168.13.152 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 amac=00-0C-29-98-8D-D7 av=7.9.0.8084.0 atz=Asia/Riyadh at=windowsfg dvchost=win-pan1 dvc=192.168.13.152 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dvcmac=00-0C-29-98-8D-D7 dtz=Asia/Riyadh _cefVer=0.1 aid=3o0OiZmYBABCACGN9CiyuGQ\=\= -def test_microfocus_arcsight_cef_ts_rt(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_ts_rt(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -36,7 +36,7 @@ def test_microfocus_arcsight_cef_ts_rt(record_property, setup_wordlist, setup_sp assert resultCount == 1 -def test_microfocus_arcsight_cef_ts_end(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_ts_end(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -56,7 +56,7 @@ def test_microfocus_arcsight_cef_ts_end(record_property, setup_wordlist, setup_s assert resultCount == 1 -def test_microfocus_arcsight_cef_ts_syslog(record_property, setup_wordlist, setup_splunk): +def test_microfocus_arcsight_ts_syslog(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) mt = env.from_string( @@ -115,23 +115,3 @@ def test_microfocus_arcsight_windows_system(record_property, setup_wordlist, set record_property("message", message) assert resultCount == 1 - -def test_microfocus_arcsight_imperva_incapsula(record_property, setup_wordlist, setup_splunk): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - - mt = env.from_string( - "{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} " + 'CEF:0|Incapsula|SIEMintegration|1|1|Illegal Resource Access|3| fileid=3412341160002518171 sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support src=12.12.12.12 caIP=13.13.13.13 ccode=IL tag=www.elvis.com cn1=200 in=54 xff=44.44.44.44 cs1=NOT_SUPPORTED cs1Label=Cap Support cs4=c2e72124-0e8a-4dd8-b13b-3da246af3ab2 cs4Label=VID cs5=de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4 cs5Label=clappsigdproc=Browser cs6=Firefox cs6Label=clapp ccode=IL cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 start=1453290121336 request=site123.abcd.info/ requestmethod=GET qstr=p\=%2fetc%2fpasswd app=HTTP act=REQ_CHALLENGE_CAPTCHA deviceExternalID=33411452762204224 cpt=443 filetype=30037,1001, filepermission=2,1, cs9=Block Malicious User,High Risk Resources, cs9Label=Rule name' + "\n") - message = mt.render(mark="<111>", host=host) - - sendsingle(message) - - st = env.from_string("search index=netwaf host=\"{{ host }}\" sourcetype=\"cef\" source=\"Imperva:Incapsula\" | head 2") - search = st.render(host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_pfsense.py b/tests/test_pfsense.py new file mode 100644 index 0000000..a5cc41b --- /dev/null +++ b/tests/test_pfsense.py @@ -0,0 +1,72 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +#<27>Jan 25 01:58:06 filterlog: 82,,,1000002666,mvneta2,match,pass,out,6,0x00,0x00000,64,ICMPv6,58,8,fe80::208:a2ff:fe0f:cb66,fe80::56a6:5cff:fe7d:1d43, +def test_pfsense_filterlog(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} filterlog: 82,,,1000002666,mvneta2,match,pass,out,6,0x00,0x00000,64,ICMPv6,58,8,{{key}},\n") + message = mt.render(mark="<27>", key=host) + sendsingle(message, port=5006) + + st = env.from_string("search index=netfw sourcetype=pfsense:filterlog \"{{key}}\" earliest=-2m | head 2") + search = st.render(key=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<27>Jan 25 01:58:06 kqueue error: unknown +def test_pfsense_other(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} kqueue error: {{key}}\n") + message = mt.render(mark="<27>", key=host) + sendsingle(message, port=5006) + + st = env.from_string("search index=netops sourcetype=pfsense:* \"{{key}}\" earliest=-2m | head 2") + search = st.render(key=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<27>Jan 25 01:58:06 syslogd: restart +def test_pfsense_syslogd(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} syslogd: restart {{key}}\n") + message = mt.render(mark="<27>", key=host) + sendsingle(message, port=5006) + + st = env.from_string("search index=netops sourcetype=pfsense:syslogd \"{{key}}\" earliest=-2m | head 2") + search = st.render(key=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_vmware.py b/tests/test_vmware.py index 4ed57ca..235798b 100644 --- a/tests/test_vmware.py +++ b/tests/test_vmware.py @@ -41,12 +41,12 @@ def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk): host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) - mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSXV {{ pid }} - [nsxv@6876 comp=\"nsx-manager\" subcomp=\"manager\"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704\n") + mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSX - SYSTEM [nsx@6876 comp=\"nsx-manager\" errorCode=\"MP4039\" subcomp=\"manager\"] Connection verification failed for broker '10.160.108.196'. Marking broker unhealthy.\n") message = mt.render(mark="<144>", host=host, pid=pid) sendsingle(message) - st = env.from_string("search index=main host={{ host }} PID={{ pid }} sourcetype=\"vmware:vsphere:nsx\" | head 2") + st = env.from_string("search index=main host={{ host }} sourcetype=\"vmware:vsphere:nsx\" | head 2") search = st.render(host=host, pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search)