From 2fb4009d215ee36f648549dd4a17dbbce0b871ad Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Tue, 15 Oct 2019 15:48:55 -0400 Subject: [PATCH 1/9] Support Cisco Meraki (#150) --- docker-compose.yml | 1 + docs/sources.md | 57 ++++++++++++++++--- .../conf.d/conflib/_common/syslog_format.conf | 9 +++ package/etc/conf.d/filters/cisco/meraki.conf | 22 +++++++ package/etc/conf.d/local/context/README.md | 1 + .../context/compliance_meta_by_source.conf | 5 -- .../context/compliance_meta_by_source.csv | 2 - .../etc/conf.d/local/context/splunk_index.csv | 40 ------------- .../context/vendor_product_by_source.conf | 37 ------------ .../context/vendor_product_by_source.csv | 8 --- .../p_rfc5424_epoch-cisco_merkai.conf.tmpl | 42 ++++++++++++++ package/etc/conf.d/sources/network.conf.tmpl | 3 + .../vendor_product_by_source.conf | 4 ++ .../vendor_product_by_source.csv | 1 + package/etc/go_templates/source_network.t | 6 ++ tests/test_cisco_meraki.py | 35 ++++++++++++ 16 files changed, 174 insertions(+), 99 deletions(-) create mode 100644 package/etc/conf.d/filters/cisco/meraki.conf create mode 100644 package/etc/conf.d/local/context/README.md delete mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.conf delete mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.csv delete mode 100644 package/etc/conf.d/local/context/splunk_index.csv delete mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.conf delete mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.csv create mode 100644 package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl create mode 100644 tests/test_cisco_meraki.py diff --git a/docker-compose.yml b/docker-compose.yml index 1765168..a9e5e74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,7 @@ services: RH_ORG: ${RH_ORG} RH_ACTIVATION: ${RH_ACTIVATION} hostname: sc4s + command: -det ports: - "514:514" - "601:601" diff --git a/docs/sources.md b/docs/sources.md index 12f4f3c..7ec8f00 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -41,12 +41,7 @@ MSG Parse: This filter parses message content * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. -* Follow vendor configuration steps per Product Manual above ensure: - * Log Level is 6 "Informational" - * Protocol is TCP/IP - * permit-hostdown is on - * device-id is hostname and included - * timestamp is included +* Follow vendor configuration steps per Product Manual above ### Options @@ -107,7 +102,7 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_JUNIPER_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | | SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC3164 format | ### Verification @@ -189,6 +184,54 @@ Use the following search to validate events are present, for NX-OS, WLC and ACI index= sourcetype=cisco:ios | stats count by host ``` +## Product - Meraki Product Line MR, MS, MX, MV + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3018/ | +| Product Manual | https://documentation.meraki.com/zGeneral_Administration/Monitoring_and_Reporting/Syslog_Server_Overview_and_Configuration | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| merkai | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_meraki | meraki | netfw | The current TA does not sub sourcetype or utilize source preventing segmenation into more appropriate indexes | + + +### Filter type + +IP, Netmask, Host or Port + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Follow vendor configuration steps per Product Manual above + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CISCO_MERAKI_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_MERAKI_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | + +### Verification + +Use the following search to validate events are present + +``` +index= sourcetype=merkai +``` + +Verify timestamp, and host values match as expected + Verify timestamp, and host values match as expected diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index 8c5794c..5b69e71 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -4,6 +4,9 @@ filter f_rfc5424_strict{ filter f_rfc5424_noversion{ message('^(?(?
(?<\d{1,3}>) ?(?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); }; +filter f_rfc5424_epochtime{ + message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) ))'); +}; rewrite set_rfcnonconformant{ set("rfc5424_nonconform" value("fields.sc4s_syslog_format")); }; @@ -19,6 +22,12 @@ rewrite set_rfc5424_noversion{ filter f_is_rfc5424_noversion{ match("rfc5424_noversion" value("fields.sc4s_syslog_format")) }; +rewrite set_rfc5424_epochtime{ + set("rfc5424_epochtime" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc5424_epochtime{ + match("rfc5424_epochtime" value("fields.sc4s_syslog_format")) +}; rewrite set_rfc3164{ set("rfc3164" value("fields.sc4s_syslog_format")); }; diff --git a/package/etc/conf.d/filters/cisco/meraki.conf b/package/etc/conf.d/filters/cisco/meraki.conf new file mode 100644 index 0000000..3d93cb6 --- /dev/null +++ b/package/etc/conf.d/filters/cisco/meraki.conf @@ -0,0 +1,22 @@ +# Meraki + +filter f_cisco_meraki { + match("cisco_meraki", value("fields.sc4s_vendor_product") type(glob)) +}; + +parser p_cisco_meraki { + channel { + filter { + match( + #'(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+ )(?.*))' + '(?:(?:<(?\d{1,3})>(?[1-9][0-9]?) (?:(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) )(?.*))' + flags(store-matches) + ); + }; + parser { + date-parser(format('%s') + template("${EPOCH}")); + }; + }; + +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/context/README.md b/package/etc/conf.d/local/context/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/context/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.conf b/package/etc/conf.d/local/context/compliance_meta_by_source.conf deleted file mode 100644 index f325b4f..0000000 --- a/package/etc/conf.d/local/context/compliance_meta_by_source.conf +++ /dev/null @@ -1,5 +0,0 @@ -@version: 3.24 -filter f_test_test { - host("something-*" type(glob)) or - netmask(192.168.100.1/24) -}; diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.csv b/package/etc/conf.d/local/context/compliance_meta_by_source.csv deleted file mode 100644 index 6608db0..0000000 --- a/package/etc/conf.d/local/context/compliance_meta_by_source.csv +++ /dev/null @@ -1,2 +0,0 @@ -#f_test_test,.splunk.index,"badindex" -#f_test_test,fields.compliance,"pci" diff --git a/package/etc/conf.d/local/context/splunk_index.csv b/package/etc/conf.d/local/context/splunk_index.csv deleted file mode 100644 index e93911a..0000000 --- a/package/etc/conf.d/local/context/splunk_index.csv +++ /dev/null @@ -1,40 +0,0 @@ -#bluecoat_proxy,index,netproxy -#cef_ArcSight_ArcSight,index,netwaf -#cef_Incapsula_SIEMintegration,index,netwaf -#cef_Microsoft_Microsoft Windows,index,oswinsec -#cef_Microsoft_System or Application Event,index,oswin -#cisco_asa,index,netfw -#cisco_ios,index,netops -#cisco_nx_os,index,netops -#local_example,index,main -#fortinet_fortios_event,index,netops -#fortinet_fortios_log,index,netops -#fortinet_fortios_traffic,index,netfw -#fortinet_fortios_utm,index,netids -#juniper_idp,index,netids -#juniper_structured,index,netops -#juniper_idp_structured,index,netids -#juniper_junos_fw_structured,index,netfw -#juniper_junos_ids_structured,index,netids -#juniper_junos_utm_structured,index,netfw -#juniper_junos_fw,index,netfw -#juniper_junos_ids,index,netids -#juniper_junos_utm,index,netfw -#juniper_sslvpn,index,netfw -#juniper_netscreen,index,netfw -#juniper_nsm,index,netfw -#juniper_nsm_idp,index,netids -#juniper_legacy,index,netops -#pan_traffic,index,netfw -#pan_threat,index,netproxy -#pan_system,index,netops -#pan_config,index,netops -#pan_hipwatch,index,main -#pan_correlation,index,main -#pan_userid,index,netauth -#pan_unknown,index,netops -#proofpoint_pps_filter,index,email -#proofpoint_pps_sendmail,index,email -#sc4s_events,index,main -#sc4s_fallback,index,main -#sc4s_metrics,index,em_metrics diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.conf b/package/etc/conf.d/local/context/vendor_product_by_source.conf deleted file mode 100644 index 37e3412..0000000 --- a/package/etc/conf.d/local/context/vendor_product_by_source.conf +++ /dev/null @@ -1,37 +0,0 @@ -@version: 3.22 -#TODO: #60 The syntax below uses regex and an indirect reference to a variable due to a -#bug/limitation of selector files. The better syntax should be as follows -#filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; - -filter f_test_test { - host("testvp-*" type(glob)) or - netmask(192.168.100.1/24) -}; -filter f_juniper_nsm { - host("jnpnsm-*" type(glob)) or - netmask(192.168.1.0/24) -}; -filter f_juniper_nsm_idp { - host("jnpnsmidp-*" type(glob)) or - netmask(192.168.2.0/24) -}; -filter f_juniper_idp { - host("jnpidp-*" type(glob)) or - netmask(192.168.3.0/24) -}; -filter f_juniper_netscreen { - host("jnpns-*" type(glob)) or - netmask(192.168.4.0/24) -}; -filter f_cisco_nx_os { - host("csconx-*" type(glob)) or - netmask(192.168.5.0/24) -}; -filter f_proofpoint_pps_sendmail { - host("pps-*" type(glob)) or - netmask(192.168.6.0/24) -}; -filter f_proofpoint_pps_filter { - host("pps-*" type(glob)) or - netmask(192.168.7.0/24) -}; \ No newline at end of file diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.csv b/package/etc/conf.d/local/context/vendor_product_by_source.csv deleted file mode 100644 index 3f90603..0000000 --- a/package/etc/conf.d/local/context/vendor_product_by_source.csv +++ /dev/null @@ -1,8 +0,0 @@ -f_test_test,sc4s_vendor_product,"test_test" -f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" -f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" -f_juniper_idp,sc4s_vendor_product,"juniper_idp" -f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" -f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" -f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" -f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl new file mode 100644 index 0000000..dbbb1f6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl @@ -0,0 +1,42 @@ +# Checkpoint Splunk format +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_default-ports); + filter(f_cisco_meraki); +{{- end}} +{{- if eq (.) "no"}} + source (s_dedicated_port_CISCO_MERAKI); +{{- end}} + + #parser { + # kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); + # date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); + # + # }; + + #rewrite { set("${.kv.hostname}", value("HOST")); }; + + rewrite { r_set_splunk_dest_default(sourcetype("meraki"), index("netfw"), template("t_hdr_msg"))}; + parser {p_add_context_splunk(key("cisco_meraki")); }; + + + + parser (compliance_meta_by_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for CISCO_MERAKI traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for CISCO_MERAKI traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index a9e8b78..234e87c 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -82,6 +82,9 @@ source s_default-ports { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); + } elif { + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); } else { parser { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 0903ca3..b736ac3 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -1,5 +1,9 @@ @version: 3.24 +filter f_cisco_meraki { + host("testcm-*" type(glob)) or + netmask(192.168.100.1/24) +}; filter f_test_test { host("testvp-*" type(glob)) or netmask(192.168.100.1/24) diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 3f90603..2f85bc4 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -1,4 +1,5 @@ f_test_test,sc4s_vendor_product,"test_test" +f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" f_juniper_idp,sc4s_vendor_product,"juniper_idp" diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index f804816..09ea120 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -74,6 +74,9 @@ source s_dedicated_port_{{ .port_id}} { {{- else if eq .parser "cisco_parser" }} parser {cisco-parser()}; rewrite(set_cisco_ios); +{{- else if eq .parser "cisco_meraki_parser" }} + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); {{- else if eq .parser "rfc3164" }} parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); @@ -96,6 +99,9 @@ source s_dedicated_port_{{ .port_id}} { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); + } elif { + parser (p_cisco_meraki); + rewrite(set_rfc5424_epochtime); } else { parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); diff --git a/tests/test_cisco_meraki.py b/tests/test_cisco_meraki.py new file mode 100644 index 0000000..920714a --- /dev/null +++ b/tests/test_cisco_meraki.py @@ -0,0 +1,35 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<134>1 1563249630.774247467 devicename security_event ids_alerted signature=1:28423:1 priority=1 timestamp=1468531589.810079 dhost=98:5A:EB:E1:81:2F direction=ingress protocol=tcp/ip src=151.101.52.238:80 dst=192.168.128.2:53023 message: EXPLOIT-KIT Multiple exploit kit single digit exe detection +def test_cisco_meraki_security_event(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}1 {% now 'utc', '%s' %}.123456789 testcm-{{ host }} security_event ids_alerted signature=1:28423:1 priority=1 timestamp={% now 'utc', '%s' %}.123456 dhost=98:5A:EB:E1:81:2F direction=ingress protocol=tcp/ip src=151.101.52.238:80 dst=192.168.128.2:53023 message: EXPLOIT-KIT Multiple exploit kit single digit exe detection\n") + message = mt.render(mark="<134>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"testcm-{{ host }}\" sourcetype=\"meraki\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +#<134>1 Dec 6 08:41:44 192.168.1.1 1 1386337316.207232138 MX84 events Cellular connection up From 9e6bd1ee42f48c26ee5bccd8db3bdc65af46089b Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Tue, 15 Oct 2019 16:03:57 -0400 Subject: [PATCH 2/9] Feature/improve startup time (#151) * Improve startup time by using a single invocation of gomplate --- package/sbin/entrypoint.sh | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 30537c7..ff9a112 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -2,15 +2,18 @@ source scl_source enable rh-python36 cd /opt/syslog-ng -for d in $(find /opt/syslog-ng/etc -type d) -do - echo Templating conf for $d - gomplate \ - --input-dir=$d \ - --template t=etc/go_templates/ \ - --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ - --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" -done +#The following is no longer needed but retained as a comment just in case we run into command line length issues +#for d in $(find /opt/syslog-ng/etc -type d) +#do +# echo Templating conf for $d +# gomplate \ +# --input-dir=$d \ +# --template t=etc/go_templates/ \ +# --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ +# --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" +#done +gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ + mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ From 8e7a3ed7f42cfafd27b652283d63f44abedffc8f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2019 08:23:46 -0400 Subject: [PATCH 3/9] Bump package/syslog-ng from `26c0fe2` to `f219fbb` (#155) Bumps [package/syslog-ng](https://github.com/balabit/syslog-ng) from `26c0fe2` to `f219fbb`. - [Release notes](https://github.com/balabit/syslog-ng/releases) - [Commits](https://github.com/balabit/syslog-ng/compare/26c0fe20a7169a2007de561ff3c4bc7df93ca86b...f219fbbb12dad0b0b6cb458a0bb4415b2cd94d8a) Signed-off-by: dependabot-preview[bot] --- package/syslog-ng | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/syslog-ng b/package/syslog-ng index 26c0fe2..f219fbb 160000 --- a/package/syslog-ng +++ b/package/syslog-ng @@ -1 +1 @@ -Subproject commit 26c0fe20a7169a2007de561ff3c4bc7df93ca86b +Subproject commit f219fbbb12dad0b0b6cb458a0bb4415b2cd94d8a From d9394a8c925236846f7a9372e62a4299b087c00b Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 16 Oct 2019 12:14:53 -0400 Subject: [PATCH 4/9] Fixes #156 (#157) Support forcepoint webprotect aka websense --- docs/sources.md | 49 +++++++++++++++++++ .../conf.d/filters/fortinet/webprotect.conf | 3 ++ .../p_rfc3164-forcepoint_webprotect.conf.tmpl | 36 ++++++++++++++ .../etc/context_templates/splunk_index.csv | 1 + tests/test_forcepoint_web.py | 35 +++++++++++++ 5 files changed, 124 insertions(+) create mode 100644 package/etc/conf.d/filters/fortinet/webprotect.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl create mode 100644 tests/test_forcepoint_web.py diff --git a/docs/sources.md b/docs/sources.md index 7ec8f00..16c7f0c 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -235,6 +235,55 @@ Verify timestamp, and host values match as expected Verify timestamp, and host values match as expected +# Vendor - Forcepoint + +## Product - Webprotect (Websense) + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2966/ | +| Product Manual | http://www.websense.com/content/support/library/web/v85/siem/siem.pdf | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| websense:cg:kv | None | + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| forcepoint_webprotect | websense:cg:kv | netproxy | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the admin manual for specific details of configuration to send Reliable syslog using RFC 3195 format, a typical logging configuration will include the following features. + + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active proxy will generate frequent events, in addition WebProtect has the ability to test logging functionality using a built in command + + +``` +index= sourcetype=websense:cg:kv +``` + # Vendor - Fortinet ## Product - Fortigate diff --git a/package/etc/conf.d/filters/fortinet/webprotect.conf b/package/etc/conf.d/filters/fortinet/webprotect.conf new file mode 100644 index 0000000..2d669e3 --- /dev/null +++ b/package/etc/conf.d/filters/fortinet/webprotect.conf @@ -0,0 +1,3 @@ +filter f_forcepoint_webprotect_kv { + program('vendor=[Ww]ebsense'); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl new file mode 100644 index 0000000..ac0cb48 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -0,0 +1,36 @@ +# Forcepoint Webprotect +{{- if (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_forcepoint_webprotect_kv); +{{- end}} +{{- if eq (.) "no"}} + source (s_dedicated_port_FORCEPOINT_WEBPROTECT); +{{- end}} + + rewrite { + subst(" [^ =]+\=\-", "", value("MESSAGE"), flags("global")); + }; + rewrite { r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg"))}; + parser {p_add_context_splunk(key("forcepoint_webprotect")); }; + + parser (compliance_meta_by_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for FORCEPOINT_WEBPROTECT traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for FORCEPOINT_WEBPROTECT traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index ec4f02c..7a418e3 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -15,6 +15,7 @@ #cisco_ios,index,netops #cisco_nx_os,index,netops #local_example,index,main +#forcepoint_webprotect,index,netproxy #fortinet_fortios_event,index,netops #fortinet_fortios_log,index,netops #fortinet_fortios_traffic,index,netfw diff --git a/tests/test_forcepoint_web.py b/tests/test_forcepoint_web.py new file mode 100644 index 0000000..7ecf4cb --- /dev/null +++ b/tests/test_forcepoint_web.py @@ -0,0 +1,35 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<134>Oct 16 12:13:06 sourcehost2 vendor=Websense 9f product=Security product_version=7.7.0 action=permitted severity=7 category=755 user=LDAP://user7 OU=Users,OU=Beijing,DC=com/TEST\, TEST_NAME src_host=10.0.0.4 src_port=61435 dst_host=HOST-013 dst_ip=10.0.0.19 dst_port=25404 bytes_out=4074 bytes_in=12328 http_response=200 http_method=POST http_content_type=image/gif;charset=UTF-8 http_user_agent=Mozilla/3.0 (Windows; U; Windows NT 6.1; es-def; rv:1.7.0.11) Gecko/2009060215 Firefox/8.0.11 (.NET CLR 8.5.30729) http_proxy_status_code=200 reason=- disposition=2573 policy=role-8**Default role=4 duration=63 url=http://test_web.com/contents/content1.jpg +def test_forcepoint_webprotect_kv(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{ host }} vendor=Websense 9f product=Security product_version=7.7.0 action=permitted severity=7 category=755 user=LDAP://user7 OU=Users,OU=Beijing,DC=com/TEST\, TEST_NAME src_host=10.0.0.4 src_port=61435 dst_host=HOST-013 dst_ip=10.0.0.19 dst_port=25404 bytes_out=4074 bytes_in=12328 http_response=200 http_method=POST http_content_type=image/gif;charset=UTF-8 http_user_agent=Mozilla/3.0 (Windows; U; Windows NT 6.1; es-def; rv:1.7.0.11) Gecko/2009060215 Firefox/8.0.11 (.NET CLR 8.5.30729) http_proxy_status_code=200 reason=- disposition=2573 policy=role-8**Default role=4 duration=63 url=http://test_web.com/contents/content1.jpg unknownfield=-\n") + message = mt.render(mark="<134>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netproxy host=\"{{ host }}\" sourcetype=\"websense:cg:kv\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +#<134>1 Dec 6 08:41:44 192.168.1.1 1 1386337316.207232138 MX84 events Cellular connection up From 3a5e588c54f4e3410c737c2d8b56d7d7ec4cd0f7 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 16 Oct 2019 14:08:49 -0400 Subject: [PATCH 5/9] Fixes #144 Add ZScaler support (#159) --- docs/sources.md | 68 +++++++++++++++++ package/etc/conf.d/filters/zscaler/nss.conf | 3 + .../log_paths/p_rfc3164-zscaler_nss.conf.tmpl | 75 +++++++++++++++++++ package/syslog-ng | 2 +- tests/test_zscaler_proxy.py | 55 ++++++++++++++ 5 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 package/etc/conf.d/filters/zscaler/nss.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl create mode 100644 tests/test_zscaler_proxy.py diff --git a/docs/sources.md b/docs/sources.md index 16c7f0c..f7e85cb 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -896,3 +896,71 @@ An active proxy will generate frequent events. Use the following search to valid ``` index= sourcetype=bluecoat:proxysg:access:kv | stats count by host ``` + + +# Vendor - Zscaler + +## Product - All Products + +The ZScaler product manual includes and extensive section of configuration for multiple Splunk TCP input ports around page +26. When using SC4S these ports are not required and should not be used. Simply configure all outputs from the NSS to utilize +the IP or host name of the SC4S instance and port 514 + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3865/ | +| Product Manual | https://community.zscaler.com/t/zscaler-splunk-app-design-and-installation-documentation/4728 | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| zscalernss-alerts | Requires format customization add ``\tvendor=Zscaler\tproduct=alerts`` immediately prior to the ``\n`` in the NSS Alert Web format. See Zscaler manual for more info. | +| zscalernss-dns | Requires format customization add ``\tvendor=Zscaler\tproduct=dns`` immediately prior to the ``\n`` in the NSS DNS format. See Zscaler manual for more info. | +| zscalernss-web | None | +| zscalernss-zpa-app | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | +| zscalernss-zpa-auth | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa_auth`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | +| zscalernss-zpa-connector | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa_auth_connector`` immediately prior to the ``\n`` in the LSS Connector format. See Zscaler manual for more info. | +| zscalernss-fw | Requires format customization add ``\tvendor=Zscaler\tproduct=fw`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| zscalernss_alerts | zscalernss-alerts | main | none | +| zscalernss_dns | zscalernss-dns | netdns | none | +| zscalernss_fw | zscalernss-fw | netfw | none | +| zscalernss_web | zscalernss-web | netproxy | none | +| zscalernss-zpa-app | zscalernss_zpa-app | netids | none | +| zscalernss-zpa-auth | zscalernss_zpa_auth | netauth | none | +| zscalernss-zpa-connector | zscalernss_zpa_connector | netops | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_ZSCALER_NSS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=zscalernss-* | stats count by host +``` diff --git a/package/etc/conf.d/filters/zscaler/nss.conf b/package/etc/conf.d/filters/zscaler/nss.conf new file mode 100644 index 0000000..9ee4e1a --- /dev/null +++ b/package/etc/conf.d/filters/zscaler/nss.conf @@ -0,0 +1,3 @@ +filter f_zscaler_nss { + message('\tvendor=Zscaler\t'); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl new file mode 100644 index 0000000..7bd86fc --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -0,0 +1,75 @@ +# Proofpoint +{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} +{{ $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_default-ports); + filter(f_zscaler_nss); +{{- end }} +{{- if eq (.) "no" }} + source (s_dedicated_port_ZSCALER_NSS); +{{- end }} + + rewrite { + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + }; + parser { + #basic parsing + kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}")); + }; + + if (match("alerts" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_alerts")); }; + } elif (match("dns" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_dns")); }; + } elif (match("fw" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_fw")); }; + } elif (match("NSS" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_web")); }; + } elif (match("audit" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zia_audit")); }; + } elif (match("sandbox" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zia_sandbox")); }; + } elif (match("zpa" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa")); }; + } elif (match("zpa_auth" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_auth")); }; + } elif (match("zpa_auth_connector" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_connector")); }; + } elif (match("zpa_bba" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"), template("t_msg_only"))}; + parser { p_add_context_splunk(key("zscaler_zpa_bba")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("zscaler_nss")); + }; + }; + + + parser (compliance_meta_by_source); + + destination(d_hec); #--HEC-- + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for ZSCALER_NSS traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for ZSCALER_NSS traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/syslog-ng b/package/syslog-ng index f219fbb..26c0fe2 160000 --- a/package/syslog-ng +++ b/package/syslog-ng @@ -1 +1 @@ -Subproject commit f219fbbb12dad0b0b6cb458a0bb4415b2cd94d8a +Subproject commit 26c0fe20a7169a2007de561ff3c4bc7df93ca86b diff --git a/tests/test_zscaler_proxy.py b/tests/test_zscaler_proxy.py new file mode 100644 index 0000000..e0f7fb1 --- /dev/null +++ b/tests/test_zscaler_proxy.py @@ -0,0 +1,55 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +#Note the long white space is a \t +#2019-10-16 15:44:36 reason=Allowed event_id=6748427317914894361 protocol=HTTPS action=Allowed transactionsize=663 responsesize=65 requestsize=598 urlcategory=UK_ALLOW_Pharmacies serverip=216.58.204.70 clienttranstime=0 requestmethod=CONNECT refererURL=None useragent=Windows Windows 10 Enterprise ZTunnel/1.0 product=NSS location=UK_Wynyard_VPN->other ClientIP=192.168.0.38 status=200 user=first.last@example.com url=4171764.fls.doubleclick.net:443 vendor=Zscaler hostname=4171764.fls.doubleclick.net clientpublicIP=213.86.221.94 threatcategory=None threatname=None filetype=None appname=DoubleClick pagerisk=0 department=Procurement, Generics urlsupercategory=User-defined appclass=Sales and Marketing dlpengine=None urlclass=Bandwidth Loss threatclass=None dlpdictionaries=None fileclass=None bwthrottle=NO servertranstime=0 md5=None +def test_zscaler_proxy(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894361\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None") + message = mt.render(mark="<134>", host=host) + sendsingle(message) + + st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# +def test_zscaler_proxy_pri(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%Y-%m-%d %H:%M:%S' %}\treason=Allowed\tevent_id=6748427317914894362\tprotocol=HTTPS\taction=Allowed\ttransactionsize=663\tresponsesize=65\trequestsize=598\turlcategory=UK_ALLOW_Pharmacies\tserverip=216.58.204.70\tclienttranstime=0\trequestmethod=CONNECT\trefererURL=None\tuseragent=Windows Windows 10 Enterprise ZTunnel/1.0\tproduct=NSS\tlocation=UK_Wynyard_VPN->other\tClientIP=192.168.0.38\tstatus=200\tuser=first.last@example.com\turl=4171764.fls.doubleclick.net:443\tvendor=Zscaler\thostname={{host}}.fls.doubleclick.net\tclientpublicIP=213.86.221.94\tthreatcategory=None\tthreatname=None\tfiletype=None\tappname=DoubleClick\tpagerisk=0\tdepartment=Procurement, Generics\turlsupercategory=User-defined\tappclass=Sales and Marketing\tdlpengine=None\turlclass=Bandwidth Loss\tthreatclass=None\tdlpdictionaries=None\tfileclass=None\tbwthrottle=NO\tservertranstime=0\tmd5=None") + message = mt.render(mark="<134>", host=host) + sendsingle(message) + + st = env.from_string("search index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# From 0c761c2d942baba283ffaa88f201fe519bc72491 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 16 Oct 2019 15:27:39 -0400 Subject: [PATCH 6/9] Feature/cleanup logpaths (#158) --- ...x-os.conf.tmpl => p_rfc3164-cisco_nxos.conf.tmpl} | 0 ....tmpl => p_rfc3164-microfocus_arcsight.conf.tmpl} | 0 ....tmpl => p_rfc5424-noversion_cisco_asa.conf.tmpl} | 0 ... => p_rfc5424-noversion_symantec_proxy.conf.tmpl} | 0 ...tmpl => p_rfc5424-strict_juniper_junos.conf.tmpl} | 0 .../context_templates/compliance_meta_by_source.conf | 8 ++++---- .../context_templates/vendor_product_by_source.conf | 12 ++++++------ 7 files changed, 10 insertions(+), 10 deletions(-) rename package/etc/conf.d/log_paths/{p_rfc3164-cisco_nx-os.conf.tmpl => p_rfc3164-cisco_nxos.conf.tmpl} (100%) rename package/etc/conf.d/log_paths/{p_rfc3164_microfocus_arcsight.conf.tmpl => p_rfc3164-microfocus_arcsight.conf.tmpl} (100%) rename package/etc/conf.d/log_paths/{p_rfc5424_noversion-cisco_asa.conf.tmpl => p_rfc5424-noversion_cisco_asa.conf.tmpl} (100%) rename package/etc/conf.d/log_paths/{p_rfc_5424_noversion-symantec_proxy.conf.tmpl => p_rfc5424-noversion_symantec_proxy.conf.tmpl} (100%) rename package/etc/conf.d/log_paths/{p_rfc_5424_strict-juniper_junos.conf.tmpl => p_rfc5424-strict_juniper_junos.conf.tmpl} (100%) diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl similarity index 100% rename from package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl rename to package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl diff --git a/package/etc/context_templates/compliance_meta_by_source.conf b/package/etc/context_templates/compliance_meta_by_source.conf index f325b4f..322b938 100644 --- a/package/etc/context_templates/compliance_meta_by_source.conf +++ b/package/etc/context_templates/compliance_meta_by_source.conf @@ -1,5 +1,5 @@ @version: 3.24 -filter f_test_test { - host("something-*" type(glob)) or - netmask(192.168.100.1/24) -}; +#filter f_test_test { +# host("something-*" type(glob)) or +# netmask(192.168.100.1/24) +#}; diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index b736ac3..0f9ca65 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -1,9 +1,5 @@ @version: 3.24 -filter f_cisco_meraki { - host("testcm-*" type(glob)) or - netmask(192.168.100.1/24) -}; filter f_test_test { host("testvp-*" type(glob)) or netmask(192.168.100.1/24) @@ -20,6 +16,10 @@ filter f_juniper_idp { host("jnpidp-*" type(glob)) or netmask(192.168.3.0/24) }; +filter f_cisco_meraki { + host("testcm-*" type(glob)) or + netmask(192.168.4.0/24) +}; filter f_juniper_netscreen { host("jnpns-*" type(glob)) or netmask(192.168.4.0/24) @@ -27,7 +27,7 @@ filter f_juniper_netscreen { filter f_cisco_nx_os { host("csconx-*" type(glob)) or netmask(192.168.5.0/24) -}; +}; filter f_proofpoint_pps_sendmail { host("pps-*" type(glob)) or netmask(192.168.6.0/24) @@ -35,4 +35,4 @@ filter f_proofpoint_pps_sendmail { filter f_proofpoint_pps_filter { host("pps-*" type(glob)) or netmask(192.168.7.0/24) -}; \ No newline at end of file +}; From 65d3c9ab11345648a16c751a3e75d033bb7d18c5 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 16 Oct 2019 15:48:20 -0400 Subject: [PATCH 7/9] Update README.md (#161) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c7c8559..64043d5 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Splunk Connect for Syslog (SC4S) is a community project focused on reducing the ## Usage -For full usage instructions, please visit the Splunk Connect for Syslog documentation page. +For full usage instructions, please visit the Splunk Connect for Syslog [documentation pages over at readthedocs](https://splunk-connect-for-syslog.readthedocs.io/en/master/). ## Support From 07fe2744edcebbf8cf9eee9bac585d793f8650f1 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 16 Oct 2019 16:10:41 -0400 Subject: [PATCH 8/9] Feature/cleanup logpaths 2 (#160) Sort config --- .../vendor_product_by_source.conf | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 0f9ca65..57c73ac 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -4,6 +4,14 @@ filter f_test_test { host("testvp-*" type(glob)) or netmask(192.168.100.1/24) }; +filter f_juniper_idp { + host("jnpidp-*" type(glob)) or + netmask(192.168.3.0/24) +}; +filter f_juniper_netscreen { + host("jnpns-*" type(glob)) or + netmask(192.168.4.0/24) +}; filter f_juniper_nsm { host("jnpnsm-*" type(glob)) or netmask(192.168.1.0/24) @@ -12,27 +20,19 @@ filter f_juniper_nsm_idp { host("jnpnsmidp-*" type(glob)) or netmask(192.168.2.0/24) }; -filter f_juniper_idp { - host("jnpidp-*" type(glob)) or - netmask(192.168.3.0/24) -}; filter f_cisco_meraki { host("testcm-*" type(glob)) or netmask(192.168.4.0/24) }; -filter f_juniper_netscreen { - host("jnpns-*" type(glob)) or - netmask(192.168.4.0/24) -}; filter f_cisco_nx_os { host("csconx-*" type(glob)) or netmask(192.168.5.0/24) }; -filter f_proofpoint_pps_sendmail { - host("pps-*" type(glob)) or - netmask(192.168.6.0/24) -}; filter f_proofpoint_pps_filter { host("pps-*" type(glob)) or netmask(192.168.7.0/24) }; +filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) or + netmask(192.168.6.0/24) +}; From 9c283e56adde46a3d4544b54d1ae2acaad89d945 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 16 Oct 2019 18:29:03 -0400 Subject: [PATCH 9/9] Add persist path to docs (#162) * Add persist path to docs * Update meraki.conf resolve error on match syntax --- docs/gettingstarted/byoe-rhel7.md | 20 +++++++------ docs/gettingstarted/docker-swarm-general.md | 27 ++++++++++++++---- docs/gettingstarted/docker-swarm-rhel7.md | 28 ++++++++++++++----- docs/gettingstarted/docker-systemd-general.md | 25 ++++++++++++++--- docs/gettingstarted/podman-systemd-general.md | 27 ++++++++++++++---- package/etc/conf.d/filters/cisco/meraki.conf | 2 +- 6 files changed, 97 insertions(+), 32 deletions(-) diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 2567d89..47d2f7c 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -100,15 +100,17 @@ WantedBy=multi-user.target source scl_source enable rh-python36 cd /opt/syslog-ng -for d in $(find /opt/syslog-ng/etc -type d) -do - echo Templating conf for $d - gomplate \ - --input-dir=$d \ - --template t=etc/go_templates/ \ - --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ - --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" -done +#The following is no longer needed but retained as a comment just in case we run into command line length issues +#for d in $(find /opt/syslog-ng/etc -type d) +#do +# echo Templating conf for $d +# gomplate \ +# --input-dir=$d \ +# --template t=etc/go_templates/ \ +# --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ +# --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" +#done +gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index a1c0f72..420065f 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -5,8 +5,10 @@ Refer to [Getting Started](https://docs.docker.com/get-started/) # SC4S Configuration -* Create a directory on the server for local configurations. This should be available to all administrators, for example: +* Create a directory on the server for local configurations and disk buffering. This should be available to all +administrators, for example: ``/opt/sc4s/`` + * Create a docker-compose.yml file in the directory created above, based on the following template: ```yaml @@ -29,23 +31,35 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer # Uncomment the following line if custom TLS certs are provided # - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new + set of files will be created in addition to the original ones. _The original ones will not be removed_. + If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created + again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +`docker-compose.yml` file. Failure to do this will cause SC4S to abort at startup. + ## Configure the SC4S environment Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: @@ -148,8 +162,9 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer #Uncomment the following line if custom TLS certs are provided - - /opt/sc4s/tls:/opt/syslog-ng/tls +# - /opt/sc4s/tls:/opt/syslog-ng/tls ``` * Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" @@ -222,7 +237,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index c06849b..e8ee2ac 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -33,8 +33,9 @@ sudo docker swarm init # SC4S Configuration -* Create a directory on the server for local configurations. This should be available to all administrators, for example: +* Create a directory on the server for local configurations and disk buffering. This should be available to all administrators, for example: ``/opt/sc4s/`` + * Create a docker-compose.yml file in the directory created above, based on the following template: ```yaml @@ -57,23 +58,35 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer # Uncomment the following line if custom TLS certs are provided - - /opt/sc4s/tls:/opt/syslog-ng/tls +# - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new +set of files will be created in addition to the original ones. _The original ones will not be removed_. +If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created +again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +`docker-compose.yml` file. Failure to do this will cause SC4S to abort at startup. + ## Configure the SC4S environment Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: @@ -178,8 +191,9 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local + - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer #Uncomment the following line if custom TLS certs are provided - - /opt/sc4s/tls:/opt/syslog-ng/tls +# - /opt/sc4s/tls:/opt/syslog-ng/tls ``` * Modify the following file ``/opt/sc4s/default/env_file`` to include the port-specific environment variable(s). See the "Sources" @@ -252,7 +266,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 32eb897..20b5a3c 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -20,6 +20,8 @@ Environment="SC4S_IMAGE=splunk/sc4s:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -35,23 +37,35 @@ ExecStartPre=/usr/bin/docker run \ ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S --rm \ $SC4S_IMAGE ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new +set of files will be created in addition to the original ones. _The original ones will not be removed_. +If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created +again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +unit file above. Failure to do this will cause SC4S to abort at startup. + ## Configure the SC4S environment Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: @@ -134,6 +148,8 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -148,6 +164,7 @@ ExecStartPre=/usr/bin/docker run \ ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S \ --rm \ $SC4S_IMAGE @@ -233,7 +250,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 394762d..a70cd2d 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -14,12 +14,14 @@ After=network.service Requires=network.service [Service] -Environment="SC4S_IMAGE=splunk/scs:latest" +Environment="SC4S_IMAGE=splunk/sc4s:latest" # Optional mount point for local overrides and configurations; see notes in docs Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -35,23 +37,35 @@ ExecStartPre=/usr/bin/podman run \ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S --rm \ $SC4S_IMAGE ``` -* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations (below). +* Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. -* NOTE: The empty ``local`` directory created above will populate with templates at the first invocation + * The empty ``local`` directory created above will populate with templates at the first invocation of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout to read the local configurations properly. -* NOTE: You can back up the contents of this directory elsewhere and return the directory to an empty state + * You can back up the contents of this directory elsewhere and return the directory to an empty state when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, the direcory will populate as it did when you first installed SC4S. Your previous changes can then be merged back in and will take effect after another restart. +* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering +of events in the event of network failure to the Splunk infrastructure. + + * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new +set of files will be created in addition to the original ones. _The original ones will not be removed_. +If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created +again upon restart. + +* IMPORTANT: When creating the two directories above, ensure the directories created match the volume mounts specified in the +unit file above. Failure to do this will cause SC4S to abort at startup. + ## Configure the sc4s environment Create a file named ``/opt/sc4s/default/env_file`` and add the following environment variables: @@ -134,6 +148,8 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local" +# Mount point for local disk buffer (required) +Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer" # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -148,6 +164,7 @@ ExecStartPre=/usr/bin/podman run \ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/default/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ + "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ --name SC4S \ --rm \ $SC4S_IMAGE @@ -233,7 +250,7 @@ podman logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.22.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/package/etc/conf.d/filters/cisco/meraki.conf b/package/etc/conf.d/filters/cisco/meraki.conf index 3d93cb6..c0573b5 100644 --- a/package/etc/conf.d/filters/cisco/meraki.conf +++ b/package/etc/conf.d/filters/cisco/meraki.conf @@ -7,7 +7,7 @@ filter f_cisco_meraki { parser p_cisco_meraki { channel { filter { - match( + message( #'(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+ )(?.*))' '(?:(?:<(?\d{1,3})>(?[1-9][0-9]?) (?:(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) )(?.*))' flags(store-matches)