From 891ae1df46137aad98469f4e0f3f5905e8f783b4 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 16 Dec 2019 07:55:43 -0500 Subject: [PATCH 1/8] Add support vmware nsx --- docs/sources/VMWare/index.md | 52 +++++++++++ .../etc/conf.d/conflib/_common/templates.conf | 2 + .../conf.d/destinations/splunk_hec.conf.tmpl | 2 +- package/etc/conf.d/filters/VMware/nsx.conf | 8 ++ .../log_paths/p_multi-vmware_nsx.conf.tmpl | 92 +++++++++++++++++++ .../conf.d/log_paths/p_zz_fallback.conf.tmpl | 70 +++++++++----- .../etc/context_templates/splunk_index.csv | 1 + tests/test_vmware.py | 59 ++++++++++++ 8 files changed, 264 insertions(+), 22 deletions(-) create mode 100644 docs/sources/VMWare/index.md create mode 100644 package/etc/conf.d/filters/VMware/nsx.conf create mode 100644 package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl create mode 100644 tests/test_vmware.py diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md new file mode 100644 index 0000000..665b105 --- /dev/null +++ b/docs/sources/VMWare/index.md @@ -0,0 +1,52 @@ +# Vendor - Dell - VMWare + +## Product - NSX Controller, Manager, Edge + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | None | +| Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| vmware:nsx:vsphere:syslog | None | +| nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| vmware_nsx | vmware:nsx:vsphere:syslog | main | none | + +### Filter type + +MSG Parse: This filter parses message content when using the default configuration + +### Setup and Configuration + +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_VMWARE_NSX_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_NSX_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_NSX_TLS_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_VMWARE_NSX | no | Enable archive to disk for this specific source | +| SC4S_DEST_VMWARE_NSX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=vmware:nsx:vsphere:syslog | stats count by host +``` diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index 729e5e2..97db7ef 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -63,6 +63,7 @@ template t_JSON { --exclude .splunk.* --exclude HOST --exclude HOST_FROM + --exclude SOURCE )"); }; @@ -77,5 +78,6 @@ template t_JSON_5424 { --exclude HOST --exclude HOST_FROM --exclude RAWMSG + --exclude SOURCE )"); }; diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index bc92162..1b2d8fe 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -42,7 +42,7 @@ destination d_hec { source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$MSG + event="$MSG" fields.*)') ); }; \ No newline at end of file diff --git a/package/etc/conf.d/filters/VMware/nsx.conf b/package/etc/conf.d/filters/VMware/nsx.conf new file mode 100644 index 0000000..88744dd --- /dev/null +++ b/package/etc/conf.d/filters/VMware/nsx.conf @@ -0,0 +1,8 @@ + +filter f_vmware_nsx { + program("NSX") + or + program("NSXV") + or + program("dfwpktlogs") +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl new file mode 100644 index 0000000..6e623a3 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl @@ -0,0 +1,92 @@ +# Generate the custom port if defined +{{ $context := dict "port_id" "VMWARE_NSX" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} + +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + + filter(f_vmware_nsx); +{{- end}} +{{- if eq (.) "no"}} + source (s_VMWARE_NSX); +{{- end}} + + + if { + filter(f_is_rfc5424_strict); + filter(f_vmware_nsx); + + rewrite { + r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}")); + set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_nsx")); + }; + } elif { + + filter(f_vmware_nsx); + + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}")); + set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_nsx")); + }; + + } else { + + rewrite { + set("nix_syslog", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + + rewrite { + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") ) + + }; + + parser { p_add_context_splunk(key("nix_syslog")); }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + }; + + parser (compliance_meta_by_source); + +{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_NSX_HEC" "no") | conv.ToBool) }} + destination(d_hec); +{{- end}} + + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE_NSX") }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; +{{- end}} +{{- if or (or (getenv (print "SC4S_LISTEN_VMWARE_NSX_TCP_PORT")) (getenv (print "SC4S_LISTEN_VMWARE_NSX_UDP_PORT"))) (getenv (print "SC4S_LISTEN_VMWARE_NSX_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for VMWARE_NSX traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for VMWARE_NSX traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl index 15c3931..b6e1f40 100644 --- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl @@ -1,31 +1,59 @@ log { source(s_DEFAULT); - rewrite { - r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); - set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG")); - }; - parser { - p_add_context_splunk(key("sc4s_fallback")); - }; + if { + filter(f_is_rfc5424_strict); + rewrite { + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); + set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + }; + parser { + p_add_context_splunk(key("sc4s_fallback")); + }; + {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} + destination(d_hec); + {{- end}} + + + #in fallback archive only write rawmsg as msg + rewrite { + unset(value("RAWMSG")); + groupunset(values(".kv.*")); + }; -{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} - destination(d_hec); -{{- end}} + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} + destination(d_archive); + {{- end}} + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); + set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG")); + }; + parser { + p_add_context_splunk(key("sc4s_fallback")); + }; - #in fallback archive only write rawmsg as msg - rewrite { - set("$RAWMSG" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); + {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} + destination(d_hec); + {{- end}} + + + #in fallback archive only write rawmsg as msg + rewrite { + set("$RAWMSG" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} + destination(d_archive); + {{- end}} }; -{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} - destination(d_archive); -{{- end}} - flags(flow-control,fallback,final); + + flags(flow-control,fallback); }; diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index 6c19bc3..51b71c0 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -53,3 +53,4 @@ #sc4s_events,index,main #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics +#vmware_nsx,index,main \ No newline at end of file diff --git a/tests/test_vmware.py b/tests/test_vmware.py new file mode 100644 index 0000000..e64b3ca --- /dev/null +++ b/tests/test_vmware.py @@ -0,0 +1,59 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import datetime +import random +import pytz + +from jinja2 import Environment, environment + +from .sendmessage import * +from .splunkutils import * +import random + +env = Environment(extensions=['jinja2_time.TimeExtension']) + + +#<46>1 2019-10-24T21:00:02.403Z {{ host }} NSXV 5996 - [nsxv@6876 comp="nsx-manager" subcomp="manager"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704 +def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSXV {{ pid }} - [nsxv@6876 comp=\"nsx-manager\" subcomp=\"manager\"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main host={{ host }} PID={{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# +def test_linux_vmware_nsx_fw(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dfwpktlogs: {{ pid }} INET match PASS domain-c7/1001 IN 60 TCP 10.33.24.50/45926->10.33.24.9/8140 S\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main host={{ host }} {{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From f3d76e0a0bd7d86901b29bce0c1dad837a0c7dec Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 16 Dec 2019 12:11:10 -0500 Subject: [PATCH 2/8] fix osnix index in tests --- tests/test_checkpoint.py | 2 +- tests/test_linux_syslog.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index ab0a630..c9adef0 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -152,7 +152,7 @@ def test_checkpoint_splunk_os(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main \"0x{{ pid }}\" sourcetype=\"nix:syslog\" | head 2") + st = env.from_string("search index=osnix \"0x{{ pid }}\" sourcetype=\"nix:syslog\" | head 2") search = st.render(pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_linux_syslog.py b/tests/test_linux_syslog.py index 914dea6..9297929 100644 --- a/tests/test_linux_syslog.py +++ b/tests/test_linux_syslog.py @@ -25,7 +25,7 @@ def test_linux__nohost_program_as_path(record_property, setup_wordlist, setup_sp sendsingle(message) - st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") + st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -45,7 +45,7 @@ def test_linux__host_program_as_path(record_property, setup_wordlist, setup_splu sendsingle(message) - st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") + st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -65,7 +65,7 @@ def test_linux__nohost_program_conforms(record_property, setup_wordlist, setup_s sendsingle(message) - st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") + st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -85,7 +85,7 @@ def test_linux__host_program_conforms(record_property, setup_wordlist, setup_spl sendsingle(message) - st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") + st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) From 5d4e597abd3ca42bf6ef07abb57872bacb3a3491 Mon Sep 17 00:00:00 2001 From: mbonsack Date: Mon, 16 Dec 2019 09:23:30 -0800 Subject: [PATCH 3/8] Update index.md --- docs/sources/VMWare/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md index 665b105..d8954da 100644 --- a/docs/sources/VMWare/index.md +++ b/docs/sources/VMWare/index.md @@ -39,7 +39,7 @@ MSG Parse: This filter parses message content when using the default configurati |----------------|----------------|----------------| | SC4S_LISTEN_VMWARE_NSX_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | | SC4S_LISTEN_VMWARE_NSX_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_LISTEN_VMWARE_NSX_TLS_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_NSX_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | | SC4S_ARCHIVE_VMWARE_NSX | no | Enable archive to disk for this specific source | | SC4S_DEST_VMWARE_NSX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | From 9e1992c8d08f69a0f2d1aa66c954aba7b0348b1c Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 16 Dec 2019 15:13:17 -0500 Subject: [PATCH 4/8] vmware support --- docs/sources/VMWare/index.md | 17 +++--- package/etc/conf.d/filters/VMware/nsx.conf | 8 --- .../etc/conf.d/filters/VMware/vsphere.conf | 58 +++++++++++++++++++ .../log_paths/p_multi-vmware_nsx.conf.tmpl | 41 ++++++++++--- tests/test_vmware.py | 20 +++++++ 5 files changed, 120 insertions(+), 24 deletions(-) delete mode 100644 package/etc/conf.d/filters/VMware/nsx.conf create mode 100644 package/etc/conf.d/filters/VMware/vsphere.conf diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md index d8954da..3b9205e 100644 --- a/docs/sources/VMWare/index.md +++ b/docs/sources/VMWare/index.md @@ -1,6 +1,6 @@ # Vendor - Dell - VMWare -## Product - NSX Controller, Manager, Edge +## Product - vSphwere - ESX NSX (Controller, Manager, Edge) | Ref | Link | @@ -8,12 +8,12 @@ | Splunk Add-on | None | | Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html | - ### Sourcetypes | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| | vmware:nsx:vsphere:syslog | None | +| vmware:esx:vsphere:syslog | None | | nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx | ### Sourcetype and Index Configuration @@ -21,6 +21,7 @@ | key | sourcetype | index | notes | |----------------|----------------|----------------|----------------| | vmware_nsx | vmware:nsx:vsphere:syslog | main | none | +| vmware_esx | vmware:esx:vsphere:syslog | main | none | ### Filter type @@ -37,16 +38,16 @@ MSG Parse: This filter parses message content when using the default configurati | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_VMWARE_NSX_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | -| SC4S_LISTEN_VMWARE_NSX_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_LISTEN_VMWARE_NSX_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | -| SC4S_ARCHIVE_VMWARE_NSX | no | Enable archive to disk for this specific source | -| SC4S_DEST_VMWARE_NSX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_VMWARE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_VMWARE | no | Enable archive to disk for this specific source | +| SC4S_DEST_VMWARE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification An active proxy will generate frequent events. Use the following search to validate events are present per source device ``` -index= sourcetype=vmware:nsx:vsphere:syslog | stats count by host +index= sourcetype="vmware:*:vsphere:*" | stats count by host ``` diff --git a/package/etc/conf.d/filters/VMware/nsx.conf b/package/etc/conf.d/filters/VMware/nsx.conf deleted file mode 100644 index 88744dd..0000000 --- a/package/etc/conf.d/filters/VMware/nsx.conf +++ /dev/null @@ -1,8 +0,0 @@ - -filter f_vmware_nsx { - program("NSX") - or - program("NSXV") - or - program("dfwpktlogs") -}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/VMware/vsphere.conf b/package/etc/conf.d/filters/VMware/vsphere.conf new file mode 100644 index 0000000..624c6e0 --- /dev/null +++ b/package/etc/conf.d/filters/VMware/vsphere.conf @@ -0,0 +1,58 @@ +filter f_vmware_all { + #begin base vmware + program("cimslp", flags(ignore-case)) + or program("Fdm", flags(ignore-case)) + or program("Hostd", flags(ignore-case)) + or program("hostd-probe", flags(ignore-case)) + or program("indcfg", flags(ignore-case)) + or program("lwsmd", flags(ignore-case)) + or program("netcpa", flags(ignore-case)) + or program("pktcap-agent", flags(ignore-case)) + or program("Rhttpproxy", flags(ignore-case)) + or program("sdrsInjector", flags(ignore-case)) + or program("sfcb-.*", flags(ignore-case)) + or program("storageRM", flags(ignore-case)) + or program("vmkernel", flags(ignore-case)) + or program("vmkwarning", flags(ignore-case)) + or program("vobd", flags(ignore-case)) + or program("Vpxa", flags(ignore-case)) + or program("Vpxd", flags(ignore-case)) + or program("VSANMGMTSVC", flags(ignore-case)) + or program("vsfwd", flags(ignore-case)) + #begin nsx + or program("NSX", flags(ignore-case)) + or program("NSXV", flags(ignore-case)) + or program("dfwpktlogs", flags(ignore-case)) + or program("nsx-.*", flags(ignore-case))}; + +filter f_vmware_vsphere { + program("cimslp", flags(ignore-case)) + or program("Fdm", flags(ignore-case)) + or program("Hostd", flags(ignore-case)) + or program("hostd-probe", flags(ignore-case)) + or program("indcfg", flags(ignore-case)) + or program("lwsmd", flags(ignore-case)) + or program("netcpa", flags(ignore-case)) + or program("pktcap-agent", flags(ignore-case)) + or program("Rhttpproxy", flags(ignore-case)) + or program("sdrsInjector", flags(ignore-case)) + or program("sfcb-.*", flags(ignore-case)) + or program("storageRM", flags(ignore-case)) + or program("vmkernel", flags(ignore-case)) + or program("vmkwarning", flags(ignore-case)) + or program("vobd", flags(ignore-case)) + or program("Vpxa", flags(ignore-case)) + or program("Vpxd", flags(ignore-case)) + or program("VSANMGMTSVC", flags(ignore-case)) + or program("vsfwd", flags(ignore-case)) +}; + +filter f_vmware_nsx { + program("NSX", flags(ignore-case)) + or + program("NSXV", flags(ignore-case)) + or + program("dfwpktlogs", flags(ignore-case)) + or + program("nsx-.*", flags(ignore-case)) +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl index 6e623a3..23a480f 100644 --- a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl @@ -1,5 +1,5 @@ # Generate the custom port if defined -{{ $context := dict "port_id" "VMWARE_NSX" "parser" "common" }} +{{ $context := dict "port_id" "VMWARE" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} # The following is an inline template; we will use this to generate the actual log path @@ -8,13 +8,14 @@ log { {{- if eq (.) "yes"}} source(s_DEFAULT); - filter(f_vmware_nsx); + filter(f_vmware_all); {{- end}} {{- if eq (.) "no"}} - source (s_VMWARE_NSX); + source (s_VMWARE); {{- end}} + #NSX first because its the cheapest check if { filter(f_is_rfc5424_strict); filter(f_vmware_nsx); @@ -39,7 +40,31 @@ log { parser { p_add_context_splunk(key("vmware_nsx")); }; + #esx things + } elif { + filter(f_is_rfc5424_strict); + filter(f_vmware_vsphere); + + rewrite { + r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}")); + set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_esx")); + }; + } elif { + filter(f_vmware_vsphere); + + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}")); + set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_esx")); + }; } else { rewrite { @@ -71,22 +96,22 @@ log { parser (compliance_meta_by_source); -{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_NSX_HEC" "no") | conv.ToBool) }} +{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_HEC" "no") | conv.ToBool) }} destination(d_hec); {{- end}} -{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE_NSX") }} +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE") }} destination(d_archive); {{- end}} flags(flow-control,final); }; {{- end}} -{{- if or (or (getenv (print "SC4S_LISTEN_VMWARE_NSX_TCP_PORT")) (getenv (print "SC4S_LISTEN_VMWARE_NSX_UDP_PORT"))) (getenv (print "SC4S_LISTEN_VMWARE_NSX_TLS_PORT")) }} -# Listen on the specified dedicated port(s) for VMWARE_NSX traffic +{{- if or (or (getenv (print "SC4S_LISTEN_VMWARE_TCP_PORT")) (getenv (print "SC4S_LISTEN_VMWARE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_VMWARE_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for VMWARE traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} -# Listen on the default port (typically 514) for VMWARE_NSX traffic +# Listen on the default port (typically 514) for VMWARE traffic {{ tmpl.Exec "log_path" "yes" }} diff --git a/tests/test_vmware.py b/tests/test_vmware.py index e64b3ca..e406ee8 100644 --- a/tests/test_vmware.py +++ b/tests/test_vmware.py @@ -15,6 +15,26 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) +#vpxd 123 - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)] +def test_linux_vmware(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} vpxd {{ pid }} - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)]\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main {{ pid }} sourcetype=\"vmware:esx:vsphere:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 #<46>1 2019-10-24T21:00:02.403Z {{ host }} NSXV 5996 - [nsxv@6876 comp="nsx-manager" subcomp="manager"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704 def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk): From 8ddb10e2204a0c69bb707fdcf5cb942218bcf371 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 17 Dec 2019 09:42:19 -0500 Subject: [PATCH 5/8] Update p_multi-vmware_nsx.conf.tmpl --- .../etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl index 23a480f..797b513 100644 --- a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl @@ -22,7 +22,7 @@ log { rewrite { r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}")); - set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; parser { p_add_context_splunk(key("vmware_nsx")); @@ -35,7 +35,7 @@ log { set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}")); - set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; parser { p_add_context_splunk(key("vmware_nsx")); @@ -47,7 +47,7 @@ log { rewrite { r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}")); - set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; parser { p_add_context_splunk(key("vmware_esx")); @@ -60,7 +60,7 @@ log { set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}")); - set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; parser { p_add_context_splunk(key("vmware_esx")); @@ -87,7 +87,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); From 4b87e50304ef54445afb41633eefad0bc21f99e5 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 17 Dec 2019 11:04:42 -0500 Subject: [PATCH 6/8] Update p_multi-vmware_nsx.conf.tmpl --- package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl index 797b513..f5c1fa0 100644 --- a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl @@ -21,7 +21,7 @@ log { filter(f_vmware_nsx); rewrite { - r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}")); + r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), source("program:${PROGRAM}")); set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; parser { @@ -34,7 +34,7 @@ log { rewrite { set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); - r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}")); + r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), source("program:${.PROGRAM}")); set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; parser { @@ -46,7 +46,7 @@ log { filter(f_vmware_vsphere); rewrite { - r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}")); + r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), source("program:${PROGRAM}")); set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); }; parser { @@ -59,7 +59,7 @@ log { rewrite { set("${PROGRAM}", value(".PROGRAM")); subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); - r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}")); + r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), source("program:${.PROGRAM}")); set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; parser { From b6397f53ea4a1aa9ed5b6279733c3756b2b59741 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 17 Dec 2019 11:21:22 -0500 Subject: [PATCH 7/8] Update p_zz_fallback.conf.tmpl --- package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl index 11f90fa..011f670 100644 --- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl @@ -4,7 +4,7 @@ log { if { filter(f_is_rfc5424_strict); rewrite { - r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main")); set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG")); }; parser { @@ -27,7 +27,7 @@ log { } else { rewrite { - r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main") ); set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG")); }; parser { From 857b6d0081ac2881ebb2331844e530d5c4e6bfd2 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 17 Dec 2019 11:50:22 -0500 Subject: [PATCH 8/8] Update p_zz_fallback.conf.tmpl --- package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl index 011f670..cb9ed87 100644 --- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl @@ -28,7 +28,7 @@ log { } else { rewrite { r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main") ); - set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG")); }; parser { p_add_context_splunk(key("sc4s_fallback"));