From 891ae1df46137aad98469f4e0f3f5905e8f783b4 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 16 Dec 2019 07:55:43 -0500 Subject: [PATCH] Add support vmware nsx --- docs/sources/VMWare/index.md | 52 +++++++++++ .../etc/conf.d/conflib/_common/templates.conf | 2 + .../conf.d/destinations/splunk_hec.conf.tmpl | 2 +- package/etc/conf.d/filters/VMware/nsx.conf | 8 ++ .../log_paths/p_multi-vmware_nsx.conf.tmpl | 92 +++++++++++++++++++ .../conf.d/log_paths/p_zz_fallback.conf.tmpl | 70 +++++++++----- .../etc/context_templates/splunk_index.csv | 1 + tests/test_vmware.py | 59 ++++++++++++ 8 files changed, 264 insertions(+), 22 deletions(-) create mode 100644 docs/sources/VMWare/index.md create mode 100644 package/etc/conf.d/filters/VMware/nsx.conf create mode 100644 package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl create mode 100644 tests/test_vmware.py diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md new file mode 100644 index 0000000..665b105 --- /dev/null +++ b/docs/sources/VMWare/index.md @@ -0,0 +1,52 @@ +# Vendor - Dell - VMWare + +## Product - NSX Controller, Manager, Edge + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | None | +| Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| vmware:nsx:vsphere:syslog | None | +| nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| vmware_nsx | vmware:nsx:vsphere:syslog | main | none | + +### Filter type + +MSG Parse: This filter parses message content when using the default configuration + +### Setup and Configuration + +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_VMWARE_NSX_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_NSX_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_NSX_TLS_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_VMWARE_NSX | no | Enable archive to disk for this specific source | +| SC4S_DEST_VMWARE_NSX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=vmware:nsx:vsphere:syslog | stats count by host +``` diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index 729e5e2..97db7ef 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -63,6 +63,7 @@ template t_JSON { --exclude .splunk.* --exclude HOST --exclude HOST_FROM + --exclude SOURCE )"); }; @@ -77,5 +78,6 @@ template t_JSON_5424 { --exclude HOST --exclude HOST_FROM --exclude RAWMSG + --exclude SOURCE )"); }; diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index bc92162..1b2d8fe 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -42,7 +42,7 @@ destination d_hec { source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$MSG + event="$MSG" fields.*)') ); }; \ No newline at end of file diff --git a/package/etc/conf.d/filters/VMware/nsx.conf b/package/etc/conf.d/filters/VMware/nsx.conf new file mode 100644 index 0000000..88744dd --- /dev/null +++ b/package/etc/conf.d/filters/VMware/nsx.conf @@ -0,0 +1,8 @@ + +filter f_vmware_nsx { + program("NSX") + or + program("NSXV") + or + program("dfwpktlogs") +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl new file mode 100644 index 0000000..6e623a3 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl @@ -0,0 +1,92 @@ +# Generate the custom port if defined +{{ $context := dict "port_id" "VMWARE_NSX" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} + +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + + filter(f_vmware_nsx); +{{- end}} +{{- if eq (.) "no"}} + source (s_VMWARE_NSX); +{{- end}} + + + if { + filter(f_is_rfc5424_strict); + filter(f_vmware_nsx); + + rewrite { + r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}")); + set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_nsx")); + }; + } elif { + + filter(f_vmware_nsx); + + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}")); + set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_nsx")); + }; + + } else { + + rewrite { + set("nix_syslog", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + + rewrite { + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") ) + + }; + + parser { p_add_context_splunk(key("nix_syslog")); }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + }; + + parser (compliance_meta_by_source); + +{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_NSX_HEC" "no") | conv.ToBool) }} + destination(d_hec); +{{- end}} + + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE_NSX") }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; +{{- end}} +{{- if or (or (getenv (print "SC4S_LISTEN_VMWARE_NSX_TCP_PORT")) (getenv (print "SC4S_LISTEN_VMWARE_NSX_UDP_PORT"))) (getenv (print "SC4S_LISTEN_VMWARE_NSX_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for VMWARE_NSX traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for VMWARE_NSX traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl index 15c3931..b6e1f40 100644 --- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl @@ -1,31 +1,59 @@ log { source(s_DEFAULT); - rewrite { - r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); - set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG")); - }; - parser { - p_add_context_splunk(key("sc4s_fallback")); - }; + if { + filter(f_is_rfc5424_strict); + rewrite { + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); + set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + }; + parser { + p_add_context_splunk(key("sc4s_fallback")); + }; + {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} + destination(d_hec); + {{- end}} + + + #in fallback archive only write rawmsg as msg + rewrite { + unset(value("RAWMSG")); + groupunset(values(".kv.*")); + }; -{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} - destination(d_hec); -{{- end}} + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} + destination(d_archive); + {{- end}} + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); + set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG")); + }; + parser { + p_add_context_splunk(key("sc4s_fallback")); + }; - #in fallback archive only write rawmsg as msg - rewrite { - set("$RAWMSG" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); + {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} + destination(d_hec); + {{- end}} + + + #in fallback archive only write rawmsg as msg + rewrite { + set("$RAWMSG" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} + destination(d_archive); + {{- end}} }; -{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} - destination(d_archive); -{{- end}} - flags(flow-control,fallback,final); + + flags(flow-control,fallback); }; diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index 6c19bc3..51b71c0 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -53,3 +53,4 @@ #sc4s_events,index,main #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics +#vmware_nsx,index,main \ No newline at end of file diff --git a/tests/test_vmware.py b/tests/test_vmware.py new file mode 100644 index 0000000..e64b3ca --- /dev/null +++ b/tests/test_vmware.py @@ -0,0 +1,59 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import datetime +import random +import pytz + +from jinja2 import Environment, environment + +from .sendmessage import * +from .splunkutils import * +import random + +env = Environment(extensions=['jinja2_time.TimeExtension']) + + +#<46>1 2019-10-24T21:00:02.403Z {{ host }} NSXV 5996 - [nsxv@6876 comp="nsx-manager" subcomp="manager"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704 +def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSXV {{ pid }} - [nsxv@6876 comp=\"nsx-manager\" subcomp=\"manager\"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main host={{ host }} PID={{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# +def test_linux_vmware_nsx_fw(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dfwpktlogs: {{ pid }} INET match PASS domain-c7/1001 IN 60 TCP 10.33.24.50/45926->10.33.24.9/8140 S\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main host={{ host }} {{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1