From 62eda439c8e2f47c9d8294600d88d0d849f05f29 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Wed, 11 Dec 2019 14:12:15 -0500 Subject: [PATCH 01/10] WIP Nix support --- .gitignore | 1 + docker-compose.yml | 15 ++++- .../etc/conf.d/conflib/_common/templates.conf | 3 + package/etc/conf.d/filters/nix/syslog.conf | 5 ++ .../p_rfc3164-checkpoint_splunk.conf.tmpl | 2 +- .../log_paths/p_rfc3164-cisco_asa.conf.tmpl | 2 +- .../log_paths/p_rfc3164-cisco_ios.conf.tmpl | 2 +- .../log_paths/p_rfc3164-cisco_ise.conf.tmpl | 2 +- .../log_paths/p_rfc3164-cisco_nxos.conf.tmpl | 2 +- .../p_rfc3164-forcepoint_webprotect.conf.tmpl | 2 +- .../p_rfc3164-fortinet_fortios.conf.tmpl | 2 +- .../log_paths/p_rfc3164-juniper_idp.conf.tmpl | 2 +- .../p_rfc3164-juniper_junos.conf.tmpl | 2 +- .../p_rfc3164-juniper_netscreen.conf.tmpl | 2 +- .../log_paths/p_rfc3164-juniper_nsm.conf.tmpl | 2 +- .../p_rfc3164-juniper_nsm_idp.conf.tmpl | 2 +- .../p_rfc3164-microfocus_arcsight.conf.tmpl | 2 +- .../p_rfc3164-paloalto_panos.conf.tmpl | 2 +- .../p_rfc3164-proofpoint_pps.conf.tmpl | 2 +- .../p_rfc3164-proofpoint_pps_filter.conf.tmpl | 2 +- .../p_rfc3164-ubiquiti_unifi.conf.tmpl | 2 +- .../log_paths/p_rfc3164-zscaler_nss.conf.tmpl | 2 +- .../p_rfc5424-noversion_cisco_asa.conf.tmpl | 2 +- ...rfc5424-noversion_symantec_proxy.conf.tmpl | 2 +- .../p_rfc5424-strict_juniper_junos.conf.tmpl | 2 + .../p_rfc5424_epoch-cisco_meraki.conf.tmpl | 2 +- .../log_paths/p_za_nix_syslog.conf.tmpl | 58 +++++++++++++++++++ ...back.conf.tmpl => p_zz_fallback.conf.tmpl} | 4 +- package/etc/syslog-ng.conf | 11 ++-- tests/pytest.ini | 2 +- tests/test_common.py | 4 +- tests/test_linux_syslog.py | 57 ++++++++++++++++++ utility/udpreplay/Dockerfile | 40 +++++++++++++ 33 files changed, 213 insertions(+), 31 deletions(-) create mode 100644 package/etc/conf.d/filters/nix/syslog.conf create mode 100644 package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl rename package/etc/conf.d/log_paths/{zfallback.conf.tmpl => p_zz_fallback.conf.tmpl} (91%) create mode 100644 tests/test_linux_syslog.py create mode 100644 utility/udpreplay/Dockerfile diff --git a/.gitignore b/.gitignore index 4d40b52..921f415 100644 --- a/.gitignore +++ b/.gitignore @@ -384,3 +384,4 @@ fabric.properties tests/test_plugin_*.py # package/etc/conf.d/local/ !package/etc/conf.d/local +replay \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d1e1558..2a37a59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,7 @@ services: RH_ACTIVATION: ${RH_ACTIVATION} hostname: sc4s #When this is enabled test_common will fail -# command: -det + command: -det ports: - "514:514" - "601:601" @@ -72,6 +72,19 @@ services: - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} volumes: - splunk-etc:/opt/splunk/etc + udpreplay: + build: + context: ./utility/udpreplay + args: + RH_ORG: ${RH_ORG} + RH_ACTIVATION: ${RH_ACTIVATION} + entrypoint: tail -f /dev/null + links: + - splunk + - sc4s + volumes: + - ./replay:/work + volumes: sc4s-results: external: true diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index a08f36d..729e5e2 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -42,6 +42,9 @@ template t_hdr_msg { template("${MSGHDR}${MESSAGE}"); }; +template t_legacy_hdr_msg { + template("${LEGACY_MSGHDR}${MESSAGE}"); + }; # =============================================================================================== # Message Header, Structured Data (from RFC5424 parse) and Message; for Juniper # =============================================================================================== diff --git a/package/etc/conf.d/filters/nix/syslog.conf b/package/etc/conf.d/filters/nix/syslog.conf new file mode 100644 index 0000000..bfeeb2a --- /dev/null +++ b/package/etc/conf.d/filters/nix/syslog.conf @@ -0,0 +1,5 @@ +filter f_nix_syslog { + program("[a-zA-Z0-9\/]+") + and + match('[a-zA-Z\]]: $' value("LEGACY_MSGHDR")) +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index 1f5c620..316bd6d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -78,7 +78,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} {{- if or (or (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl index a07d6a1..26dfaea 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -41,7 +41,7 @@ log { {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index 554277b..d03081a 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -43,7 +43,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl index cca9afd..bdd92d4 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl @@ -92,7 +92,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index fd7a8b4..0d28a3a 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -44,7 +44,7 @@ log { {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl index b35e857..d86957d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -41,7 +41,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index 1be63f1..02be55a 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -60,7 +60,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl index 761a8c0..b9d1ca1 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -45,7 +45,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index d461e5e..0ecaee2 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -58,7 +58,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl index b735a73..ca717e8 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -42,7 +42,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl index 5571ba0..94bbba5 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -43,7 +43,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl index bb717e0..e571083 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -40,7 +40,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl index db6cd6a..0114932 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl @@ -88,7 +88,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 05c45ad..6ee0ad0 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -92,7 +92,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl index 78f234c..4b383e8 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl @@ -50,7 +50,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl index 09dd410..2325290 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl @@ -39,7 +39,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl index 55bd6a7..7ca3176 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl @@ -126,7 +126,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl index 47a77c1..21ffa8b 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -81,7 +81,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl index a7f069c..0e803a7 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl @@ -39,7 +39,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl index 855d390..2c3c547 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl @@ -42,7 +42,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl index 40bce61..766bbff 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl @@ -61,6 +61,8 @@ log { destination(d_archive); {{- end}} + + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl index 18a37b4..949fcce 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl @@ -39,7 +39,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl new file mode 100644 index 0000000..1d3aef3 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl @@ -0,0 +1,58 @@ +# Proofpoint +{{ $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_DEFAULT); + filter(f_nix_syslog); +{{- end }} +{{- if eq (.) "no" }} + source (s_NIX_SYSLOG); +{{- end }} + + rewrite { + set("zscaler_nss", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + }; + parser { + #basic parsing + kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}")); + }; + + rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"))}; + parser { p_add_context_splunk(key("nix_syslog")); }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + +{{- if ((getenv "SC4S_NIX_SYSLOG_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_NIX_SYSLOG_HEC" "no") | conv.ToBool) }} + destination(d_hec); +{{- end}} + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_NIX_SYSLOG") }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; +{{- end}} + +{{- if or (or (getenv (print "SC4S_LISTEN_NIX_SYSLOG_TCP_PORT")) (getenv (print "SC4S_LISTEN_NIX_SYSLOG_UDP_PORT"))) (getenv (print "SC4S_NIX_SYSLOG_NSS_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for NIX_SYSLOG traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for NIX_SYSLOG traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/zfallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl similarity index 91% rename from package/etc/conf.d/log_paths/zfallback.conf.tmpl rename to package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl index 0f36b1f..15c3931 100644 --- a/package/etc/conf.d/log_paths/zfallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl @@ -16,7 +16,7 @@ log { #in fallback archive only write rawmsg as msg rewrite { - set("value(RAWMSG)" value("MSG")); + set("$RAWMSG" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -27,5 +27,5 @@ log { destination(d_archive); {{- end}} - flags(flow-control,fallback); + flags(flow-control,fallback,final); }; diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 4f19083..84c177b 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -52,11 +52,14 @@ options { @include "conf.d/conflib/blocks/*/*.conf" @include "conf.d/filters/*/*.conf" -@include "conf.d/sources/*.conf" -@include "conf.d/destinations/*.conf" -@include "conf.d/log_paths/*.conf" - @include "conf.d/local/config/filters/*.conf" + +@include "conf.d/sources/*.conf" @include "conf.d/local/config/sources/*.conf" +@include "conf.d/destinations/*.conf" @include "conf.d/local/config/destinations/*.conf" + @include "conf.d/local/config/log_paths/*.conf" +@include "conf.d/log_paths/*.conf" + + diff --git a/tests/pytest.ini b/tests/pytest.ini index 00b64d9..ce9083e 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,5 +1,5 @@ [pytest] addopts = - --force-flaky --max-runs=3 --min-passes=1 +# --force-flaky --max-runs=3 --min-passes=1 filterwarnings = ignore::DeprecationWarning diff --git a/tests/test_common.py b/tests/test_common.py index ce279a6..9855cc9 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -17,7 +17,7 @@ def test_defaultroute(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} sc4sdefault[0]: test\n") + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} test something else\n") message = mt.render(mark="<111>", host=host) sendsingle(message) @@ -55,7 +55,7 @@ def test_internal(record_property, setup_wordlist, setup_splunk): def test_tag(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} sc4sdefault[0]: test\n") + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} test\n") message = mt.render(mark="<111>", host=host) sendsingle(message) diff --git a/tests/test_linux_syslog.py b/tests/test_linux_syslog.py new file mode 100644 index 0000000..3731db0 --- /dev/null +++ b/tests/test_linux_syslog.py @@ -0,0 +1,57 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import datetime +import random +import pytz + +from jinja2 import Environment, environment + +from .sendmessage import * +from .splunkutils import * +import random + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<78>Oct 25 09:10:00 /usr/sbin/cron[54928]: (root) CMD (/usr/libexec/atrun) +def test_linux_program_as_path(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} /usr/sbin/cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_linux_program_conforms(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file diff --git a/utility/udpreplay/Dockerfile b/utility/udpreplay/Dockerfile new file mode 100644 index 0000000..2dbd80f --- /dev/null +++ b/utility/udpreplay/Dockerfile @@ -0,0 +1,40 @@ +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . +FROM registry.access.redhat.com/rhel7/rhel + +ARG RH_ORG +ARG RH_ACTIVATION + +RUN subscription-manager register --org=$RH_ORG --activationkey=$RH_ACTIVATION --force +RUN subscription-manager repos --enable=rhel-7-server-extras-rpms --enable=rhel-7-server-optional-rpms --enable=rhel-server-rhscl-7-rpms + +RUN yum -y install libpcap-devel gcc-c++ git findutils autoconf \ + autoconf-archive automake ca-certificates git libtool pkgconfig bison byacc file \ + flex pcre-devel glib2-devel openssl-devel librdkafka-devel libcurl-devel \ + rh-python36 rh-python36-python-tools rh-python36-scldevel\ + net-snmp-devel \ + libuuid-devel make libxslt docbook-style-xsl gcc-c++ tzdata libxml2 sqlite \ + json-c-devel gnupg wget curl which bzip2 doxygen libsecret ivykis-devel -y + +RUN cd /tmp ;\ + wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm ;\ + yum install epel-release-latest-7.noarch.rpm -y ;\ + rm epel-release-latest-7.noarch.rpm ;\ + yum install -y cmake3 boost-devel + + + +RUN cd ~ ; git clone https://github.com/ska-sa/udpreplay.git +RUN cd ~/udpreplay && \ + ./bootstrap.sh && \ + ./configure && \ + make && \ + make install && \ + cd ~ ; rm -Rf udpreplay + +RUN subscription-manager unregister +ENTRYPOINT ["/bin/tail", "-f", "/dev/null"] From b48c445c8e4b04f7687e51c92f2d31fa26b14996 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Wed, 11 Dec 2019 19:22:19 -0500 Subject: [PATCH 02/10] Update p_za_nix_syslog.conf.tmpl --- package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl index 1d3aef3..b809caf 100644 --- a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl @@ -13,13 +13,9 @@ log { {{- end }} rewrite { - set("zscaler_nss", value("fields.sc4s_vendor_product")); + set("nix_syslog", value("fields.sc4s_vendor_product")); subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); }; - parser { - #basic parsing - kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}")); - }; rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"))}; parser { p_add_context_splunk(key("nix_syslog")); }; @@ -34,7 +30,6 @@ log { unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); }; {{- if ((getenv "SC4S_NIX_SYSLOG_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_NIX_SYSLOG_HEC" "no") | conv.ToBool) }} From 85a59d857a6f85d79b36c6e2e21282d49042b822 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 12 Dec 2019 10:38:34 -0500 Subject: [PATCH 03/10] Update replay tool --- tests/test_symantec_ep.py | 76 +++++++++++++++++++++++++++++++++ utility/udpreplay/Dockerfile | 30 +++---------- utility/udpreplay/entrypoint.sh | 3 ++ 3 files changed, 86 insertions(+), 23 deletions(-) create mode 100644 tests/test_symantec_ep.py create mode 100755 utility/udpreplay/entrypoint.sh diff --git a/tests/test_symantec_ep.py b/tests/test_symantec_ep.py new file mode 100644 index 0000000..b3c906e --- /dev/null +++ b/tests/test_symantec_ep.py @@ -0,0 +1,76 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import datetime +import random +import pytz + +from jinja2 import Environment, environment + +from .sendmessage import * +from .splunkutils import * +import random + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +def test_symantec_ep_msg_1(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: WORK1-PC,Local Host: 0.0.0.0,Local Port: 29555,Local Host MAC: FFFFFFFFFFFF,Remote Host IP: 0.0.0.0,Remote Host Name: ,Remote Port: 0,Remote Host MAC: WORK2-PC,7,Inbound,Begin: 2019-10-25 00:06:22,End: 2019-10-25 00:06:22,Occurrences: 1,Application: ,Rule: B-ALL-B,Location: Untrusted,User: johndoe,Domain: AD-ENT,Action: Blocked,SHA-256: ,MD-5:\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_symantec_ep_msg_two(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: WORK1-PC,Local Host: 192.168.1.85,Local Port: 59929,Local Host MAC: D4D252E652BA,Remote Host IP: 10.217.138.110,Remote Host Name: host.example.com,Remote Port: 9000,Remote Host MAC: D4B17A775938,TCP,Outbound,Begin: 2019-10-25 00:06:09,End: 2019-10-25 00:06:18,Occurrences: 3,Application: C:/Program Files/Preton/PretonSaver/PretonService.exe,Rule: B-ALL-B,Location: Untrusted,User: SYSTEM,Domain: NT AUTHORITY,Action: Blocked,SHA-256: ba532f64bd6a31cf5f1938820f458d31fed8faa01733c9de3a1d313198b0dd9c,MD-5: 1AE7578A3CF3EABE492463C2AB7D7318\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_symantec_ep_msg_two(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: Site: WORK-A,Server: FOOFOO,Domain: Desktop,The client has downloaded the content package successfully,FOOFO,USERNAME,ENT.EXAMPLE.CORP\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file diff --git a/utility/udpreplay/Dockerfile b/utility/udpreplay/Dockerfile index 2dbd80f..62a5397 100644 --- a/utility/udpreplay/Dockerfile +++ b/utility/udpreplay/Dockerfile @@ -4,28 +4,9 @@ # #You should have received a copy of the CC0 legalcode along with this #work. If not, see . -FROM registry.access.redhat.com/rhel7/rhel - -ARG RH_ORG -ARG RH_ACTIVATION - -RUN subscription-manager register --org=$RH_ORG --activationkey=$RH_ACTIVATION --force -RUN subscription-manager repos --enable=rhel-7-server-extras-rpms --enable=rhel-7-server-optional-rpms --enable=rhel-server-rhscl-7-rpms - -RUN yum -y install libpcap-devel gcc-c++ git findutils autoconf \ - autoconf-archive automake ca-certificates git libtool pkgconfig bison byacc file \ - flex pcre-devel glib2-devel openssl-devel librdkafka-devel libcurl-devel \ - rh-python36 rh-python36-python-tools rh-python36-scldevel\ - net-snmp-devel \ - libuuid-devel make libxslt docbook-style-xsl gcc-c++ tzdata libxml2 sqlite \ - json-c-devel gnupg wget curl which bzip2 doxygen libsecret ivykis-devel -y - -RUN cd /tmp ;\ - wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm ;\ - yum install epel-release-latest-7.noarch.rpm -y ;\ - rm epel-release-latest-7.noarch.rpm ;\ - yum install -y cmake3 boost-devel +FROM ubuntu:latest +RUN apt-get update ;apt-get install -y build-essential autoconf wget git libboost-dev libpcap-dev libtool autogen RUN cd ~ ; git clone https://github.com/ska-sa/udpreplay.git @@ -36,5 +17,8 @@ RUN cd ~/udpreplay && \ make install && \ cd ~ ; rm -Rf udpreplay -RUN subscription-manager unregister -ENTRYPOINT ["/bin/tail", "-f", "/dev/null"] +RUN cd ~ ; git clone https://github.com/GabrielGanne/tcpreplay.git +RUN cd ~/tcpreplay && \ + ./autogen.sh; ./configure; make install +COPY entrypoint.sh / +ENTRYPOINT ["/entrypoint.sh"] diff --git a/utility/udpreplay/entrypoint.sh b/utility/udpreplay/entrypoint.sh new file mode 100755 index 0000000..fb099fc --- /dev/null +++ b/utility/udpreplay/entrypoint.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +tail -f /dev/null From 3ebe4ddbbb4190a0d753d0577ca4854885918332 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 12 Dec 2019 11:21:20 -0500 Subject: [PATCH 04/10] update utility --- docker-compose.yml | 7 ++----- utility/{udpreplay => pcapreplay}/Dockerfile | 0 utility/{udpreplay => pcapreplay}/entrypoint.sh | 0 3 files changed, 2 insertions(+), 5 deletions(-) rename utility/{udpreplay => pcapreplay}/Dockerfile (100%) rename utility/{udpreplay => pcapreplay}/entrypoint.sh (100%) diff --git a/docker-compose.yml b/docker-compose.yml index 2a37a59..d24be63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,12 +72,9 @@ services: - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} volumes: - splunk-etc:/opt/splunk/etc - udpreplay: + pcapreplay: build: - context: ./utility/udpreplay - args: - RH_ORG: ${RH_ORG} - RH_ACTIVATION: ${RH_ACTIVATION} + context: ./utility/pcapreplay entrypoint: tail -f /dev/null links: - splunk diff --git a/utility/udpreplay/Dockerfile b/utility/pcapreplay/Dockerfile similarity index 100% rename from utility/udpreplay/Dockerfile rename to utility/pcapreplay/Dockerfile diff --git a/utility/udpreplay/entrypoint.sh b/utility/pcapreplay/entrypoint.sh similarity index 100% rename from utility/udpreplay/entrypoint.sh rename to utility/pcapreplay/entrypoint.sh From df88021b95ab7a811573633951bf67419158616e Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 12 Dec 2019 11:41:22 -0500 Subject: [PATCH 05/10] Delete test_symantec_ep.py --- tests/test_symantec_ep.py | 76 --------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 tests/test_symantec_ep.py diff --git a/tests/test_symantec_ep.py b/tests/test_symantec_ep.py deleted file mode 100644 index b3c906e..0000000 --- a/tests/test_symantec_ep.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2019 Splunk, Inc. -# -# Use of this source code is governed by a BSD-2-clause-style -# license that can be found in the LICENSE-BSD2 file or at -# https://opensource.org/licenses/BSD-2-Clause -import datetime -import random -import pytz - -from jinja2 import Environment, environment - -from .sendmessage import * -from .splunkutils import * -import random - -env = Environment(extensions=['jinja2_time.TimeExtension']) - -def test_symantec_ep_msg_1(record_property, setup_wordlist, setup_splunk): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - pid = random.randint(1000, 32000) - - mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: WORK1-PC,Local Host: 0.0.0.0,Local Port: 29555,Local Host MAC: FFFFFFFFFFFF,Remote Host IP: 0.0.0.0,Remote Host Name: ,Remote Port: 0,Remote Host MAC: WORK2-PC,7,Inbound,Begin: 2019-10-25 00:06:22,End: 2019-10-25 00:06:22,Occurrences: 1,Application: ,Rule: B-ALL-B,Location: Untrusted,User: johndoe,Domain: AD-ENT,Action: Blocked,SHA-256: ,MD-5:\n") - message = mt.render(mark="<111>", host=host, pid=pid) - - sendsingle(message) - - st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") - search = st.render(host=host, pid=pid) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -def test_symantec_ep_msg_two(record_property, setup_wordlist, setup_splunk): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - pid = random.randint(1000, 32000) - - mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: WORK1-PC,Local Host: 192.168.1.85,Local Port: 59929,Local Host MAC: D4D252E652BA,Remote Host IP: 10.217.138.110,Remote Host Name: host.example.com,Remote Port: 9000,Remote Host MAC: D4B17A775938,TCP,Outbound,Begin: 2019-10-25 00:06:09,End: 2019-10-25 00:06:18,Occurrences: 3,Application: C:/Program Files/Preton/PretonSaver/PretonService.exe,Rule: B-ALL-B,Location: Untrusted,User: SYSTEM,Domain: NT AUTHORITY,Action: Blocked,SHA-256: ba532f64bd6a31cf5f1938820f458d31fed8faa01733c9de3a1d313198b0dd9c,MD-5: 1AE7578A3CF3EABE492463C2AB7D7318\n") - message = mt.render(mark="<111>", host=host, pid=pid) - - sendsingle(message) - - st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") - search = st.render(host=host, pid=pid) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 - -def test_symantec_ep_msg_two(record_property, setup_wordlist, setup_splunk): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - pid = random.randint(1000, 32000) - - mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} SymantecServer: Site: WORK-A,Server: FOOFOO,Domain: Desktop,The client has downloaded the content package successfully,FOOFO,USERNAME,ENT.EXAMPLE.CORP\n") - message = mt.render(mark="<111>", host=host, pid=pid) - - sendsingle(message) - - st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") - search = st.render(host=host, pid=pid) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 \ No newline at end of file From 1233911d5a6f7b2bd2dfee8b9fb6c25a599a9256 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 12 Dec 2019 12:29:45 -0500 Subject: [PATCH 06/10] final --- docs/sources/nix/index.md | 50 +++++++++++++++++++ mkdocs.yml | 1 + .../log_paths/p_za_nix_syslog.conf.tmpl | 8 ++- .../etc/context_templates/splunk_index.csv | 1 + .../apps/SA-syslog-ng/default/indexes.conf | 5 ++ tests/test_linux_syslog.py | 44 +++++++++++++++- 6 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 docs/sources/nix/index.md diff --git a/docs/sources/nix/index.md b/docs/sources/nix/index.md new file mode 100644 index 0000000..64e9e4b --- /dev/null +++ b/docs/sources/nix/index.md @@ -0,0 +1,50 @@ +# Vendor - Nix Generic + +## Product - All Products + +Many appliance vendor utilize Linux and BSD distributions as the foundation of the solution when configured to provide +syslog output these devices can be monitored using the common Splunk Nix TA + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/833/ | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| nix:syslog | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| nix_syslog | nix:syslog | osnix | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. + + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_ARCHIVE_NIX_SYSLOG | no | Enable archive to disk for this specific source | +| SC4S_DEST_NIX_SYSLOG_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index=osnix sourcetype=nix:syslog | stats count by host +``` diff --git a/mkdocs.yml b/mkdocs.yml index 6b82a92..3848849 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -17,6 +17,7 @@ nav: - Fortinet: sources/Fortinet/index.md - Imperva: sources/Imperva/index.md - Juniper: sources/Juniper/index.md + - Nix: sources/nix/index.md - Microfocus: sources/Microfocus/index.md - 'Paloalto Networks': sources/PaloaltoNetworks/index.md - Proofpoint: sources/Proofpoint/index.md diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl index b809caf..9e2547e 100644 --- a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl @@ -15,9 +15,15 @@ log { rewrite { set("nix_syslog", value("fields.sc4s_vendor_product")); subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + + rewrite { + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"), source("program:${.PROGRAM}") ) + }; - rewrite { r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"))}; parser { p_add_context_splunk(key("nix_syslog")); }; parser (compliance_meta_by_source); diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index d4c05b9..eaf846d 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -35,6 +35,7 @@ #juniper_nsm,index,netfw #juniper_nsm_idp,index,netids #juniper_legacy,index,netops +#nix_syslog,index,osnix #pan_traffic,index,netfw #pan_threat,index,netproxy #pan_system,index,netops diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf index 04c2df6..9c64a84 100644 --- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -26,6 +26,11 @@ homePath = $SPLUNK_DB/test2/db coldPath = $SPLUNK_DB/test2/colddb thawedPath = $SPLUNK_DB/test2/thaweddb +[osnix] +homePath = $SPLUNK_DB/osnix/db +coldPath = $SPLUNK_DB/osnix/colddb +thawedPath = $SPLUNK_DB/osnix/thaweddb + [oswin] homePath = $SPLUNK_DB/oswin/db coldPath = $SPLUNK_DB/oswin/colddb diff --git a/tests/test_linux_syslog.py b/tests/test_linux_syslog.py index 3731db0..914dea6 100644 --- a/tests/test_linux_syslog.py +++ b/tests/test_linux_syslog.py @@ -16,7 +16,7 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) #<78>Oct 25 09:10:00 /usr/sbin/cron[54928]: (root) CMD (/usr/libexec/atrun) -def test_linux_program_as_path(record_property, setup_wordlist, setup_splunk): +def test_linux__nohost_program_as_path(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) @@ -36,7 +36,27 @@ def test_linux_program_as_path(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 -def test_linux_program_conforms(record_property, setup_wordlist, setup_splunk): +def test_linux__host_program_as_path(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} /usr/sbin/cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_linux__nohost_program_conforms(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) @@ -54,4 +74,24 @@ def test_linux_program_conforms(record_property, setup_wordlist, setup_splunk): record_property("resultCount", resultCount) record_property("message", message) + assert resultCount == 1 + +def test_linux__host_program_conforms(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + assert resultCount == 1 \ No newline at end of file From 5fff7a0eb07c05ebcb9b4f92ddd92486d375904f Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 12 Dec 2019 13:02:24 -0500 Subject: [PATCH 07/10] Update index.md --- docs/sources/nix/index.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/sources/nix/index.md b/docs/sources/nix/index.md index 64e9e4b..2dc0416 100644 --- a/docs/sources/nix/index.md +++ b/docs/sources/nix/index.md @@ -3,7 +3,11 @@ ## Product - All Products Many appliance vendor utilize Linux and BSD distributions as the foundation of the solution when configured to provide -syslog output these devices can be monitored using the common Splunk Nix TA +syslog output these devices can be monitored using the common Splunk Nix TA. + +Note: This is not a replacement or alternative for use of the Splunk Universal forwarder on Linux and Unix. For server applications +the syslog only prevents full collection of events and metrics appropriate for security and operations use cases. + | Ref | Link | From d9c2bc56c92b7437c8e73c58a29808dcd6eb568b Mon Sep 17 00:00:00 2001 From: mbonsack Date: Thu, 12 Dec 2019 10:14:17 -0800 Subject: [PATCH 08/10] Update index.md --- docs/sources/nix/index.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/sources/nix/index.md b/docs/sources/nix/index.md index 2dc0416..7be21b0 100644 --- a/docs/sources/nix/index.md +++ b/docs/sources/nix/index.md @@ -2,11 +2,12 @@ ## Product - All Products -Many appliance vendor utilize Linux and BSD distributions as the foundation of the solution when configured to provide -syslog output these devices can be monitored using the common Splunk Nix TA. +Many appliance vendor utilize Linux and BSD distributions as the foundation of the solution. When configured to log via +syslog, these devices' OS logs (from a security perspective) can be monitored using the common Splunk Nix TA. -Note: This is not a replacement or alternative for use of the Splunk Universal forwarder on Linux and Unix. For server applications -the syslog only prevents full collection of events and metrics appropriate for security and operations use cases. +Note: This is NOT a replacement for or alternative to the Splunk Universal forwarder on Linux and Unix. For general-purpose +server applications, the Universal Forwarder offers more comprehensive collection of events and metrics appropriate for both +security and operations use cases. From f82fa5365fe8dd74a724a6bd1a2f0346f4d19c2f Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 12 Dec 2019 14:54:24 -0500 Subject: [PATCH 09/10] Update Dockerfile --- utility/pcapreplay/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utility/pcapreplay/Dockerfile b/utility/pcapreplay/Dockerfile index 62a5397..322eb6a 100644 --- a/utility/pcapreplay/Dockerfile +++ b/utility/pcapreplay/Dockerfile @@ -6,7 +6,7 @@ #work. If not, see . FROM ubuntu:latest -RUN apt-get update ;apt-get install -y build-essential autoconf wget git libboost-dev libpcap-dev libtool autogen +RUN apt-get update ;apt-get install -y build-essential autoconf wget git libboost-all-dev libpcap-dev libtool autogen RUN cd ~ ; git clone https://github.com/ska-sa/udpreplay.git From 428386d979b058ba6936f85022f8e930d32ff92b Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 12 Dec 2019 15:07:57 -0500 Subject: [PATCH 10/10] Update index.md add missing default indexes --- docs/gettingstarted/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/gettingstarted/index.md b/docs/gettingstarted/index.md index c3822dc..f686dc0 100644 --- a/docs/gettingstarted/index.md +++ b/docs/gettingstarted/index.md @@ -42,6 +42,8 @@ using the SC4S defaults. SC4S can be easily customized to use different indexes * netops * netproxy * netipam +* oswinsec +* osnix * em_metrics (ensure this is created as a metrics index) #### Install Related Splunk Apps