diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md new file mode 100644 index 0000000..3b9205e --- /dev/null +++ b/docs/sources/VMWare/index.md @@ -0,0 +1,53 @@ +# Vendor - Dell - VMWare + +## Product - vSphwere - ESX NSX (Controller, Manager, Edge) + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | None | +| Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html | + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| vmware:nsx:vsphere:syslog | None | +| vmware:esx:vsphere:syslog | None | +| nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| vmware_nsx | vmware:nsx:vsphere:syslog | main | none | +| vmware_esx | vmware:esx:vsphere:syslog | main | none | + +### Filter type + +MSG Parse: This filter parses message content when using the default configuration + +### Setup and Configuration + +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_VMWARE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_VMWARE | no | Enable archive to disk for this specific source | +| SC4S_DEST_VMWARE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype="vmware:*:vsphere:*" | stats count by host +``` diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index 729e5e2..97db7ef 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -63,6 +63,7 @@ template t_JSON { --exclude .splunk.* --exclude HOST --exclude HOST_FROM + --exclude SOURCE )"); }; @@ -77,5 +78,6 @@ template t_JSON_5424 { --exclude HOST --exclude HOST_FROM --exclude RAWMSG + --exclude SOURCE )"); }; diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index bc92162..1b2d8fe 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -42,7 +42,7 @@ destination d_hec { source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$MSG + event="$MSG" fields.*)') ); }; \ No newline at end of file diff --git a/package/etc/conf.d/filters/VMware/vsphere.conf b/package/etc/conf.d/filters/VMware/vsphere.conf new file mode 100644 index 0000000..624c6e0 --- /dev/null +++ b/package/etc/conf.d/filters/VMware/vsphere.conf @@ -0,0 +1,58 @@ +filter f_vmware_all { + #begin base vmware + program("cimslp", flags(ignore-case)) + or program("Fdm", flags(ignore-case)) + or program("Hostd", flags(ignore-case)) + or program("hostd-probe", flags(ignore-case)) + or program("indcfg", flags(ignore-case)) + or program("lwsmd", flags(ignore-case)) + or program("netcpa", flags(ignore-case)) + or program("pktcap-agent", flags(ignore-case)) + or program("Rhttpproxy", flags(ignore-case)) + or program("sdrsInjector", flags(ignore-case)) + or program("sfcb-.*", flags(ignore-case)) + or program("storageRM", flags(ignore-case)) + or program("vmkernel", flags(ignore-case)) + or program("vmkwarning", flags(ignore-case)) + or program("vobd", flags(ignore-case)) + or program("Vpxa", flags(ignore-case)) + or program("Vpxd", flags(ignore-case)) + or program("VSANMGMTSVC", flags(ignore-case)) + or program("vsfwd", flags(ignore-case)) + #begin nsx + or program("NSX", flags(ignore-case)) + or program("NSXV", flags(ignore-case)) + or program("dfwpktlogs", flags(ignore-case)) + or program("nsx-.*", flags(ignore-case))}; + +filter f_vmware_vsphere { + program("cimslp", flags(ignore-case)) + or program("Fdm", flags(ignore-case)) + or program("Hostd", flags(ignore-case)) + or program("hostd-probe", flags(ignore-case)) + or program("indcfg", flags(ignore-case)) + or program("lwsmd", flags(ignore-case)) + or program("netcpa", flags(ignore-case)) + or program("pktcap-agent", flags(ignore-case)) + or program("Rhttpproxy", flags(ignore-case)) + or program("sdrsInjector", flags(ignore-case)) + or program("sfcb-.*", flags(ignore-case)) + or program("storageRM", flags(ignore-case)) + or program("vmkernel", flags(ignore-case)) + or program("vmkwarning", flags(ignore-case)) + or program("vobd", flags(ignore-case)) + or program("Vpxa", flags(ignore-case)) + or program("Vpxd", flags(ignore-case)) + or program("VSANMGMTSVC", flags(ignore-case)) + or program("vsfwd", flags(ignore-case)) +}; + +filter f_vmware_nsx { + program("NSX", flags(ignore-case)) + or + program("NSXV", flags(ignore-case)) + or + program("dfwpktlogs", flags(ignore-case)) + or + program("nsx-.*", flags(ignore-case)) +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl new file mode 100644 index 0000000..f5c1fa0 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl @@ -0,0 +1,117 @@ +# Generate the custom port if defined +{{ $context := dict "port_id" "VMWARE" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} + +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + + filter(f_vmware_all); +{{- end}} +{{- if eq (.) "no"}} + source (s_VMWARE); +{{- end}} + + + #NSX first because its the cheapest check + if { + filter(f_is_rfc5424_strict); + filter(f_vmware_nsx); + + rewrite { + r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), source("program:${PROGRAM}")); + set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_nsx")); + }; + } elif { + + filter(f_vmware_nsx); + + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), source("program:${.PROGRAM}")); + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_nsx")); + }; + #esx things + } elif { + filter(f_is_rfc5424_strict); + filter(f_vmware_vsphere); + + rewrite { + r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), source("program:${PROGRAM}")); + set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_esx")); + }; + } elif { + + filter(f_vmware_vsphere); + + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), source("program:${.PROGRAM}")); + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_esx")); + }; + } else { + + rewrite { + set("nix_syslog", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + + rewrite { + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") ) + + }; + + parser { p_add_context_splunk(key("nix_syslog")); }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + }; + + parser (compliance_meta_by_source); + +{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_HEC" "no") | conv.ToBool) }} + destination(d_hec); +{{- end}} + + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE") }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; +{{- end}} +{{- if or (or (getenv (print "SC4S_LISTEN_VMWARE_TCP_PORT")) (getenv (print "SC4S_LISTEN_VMWARE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_VMWARE_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for VMWARE traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for VMWARE traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl index f0abfaa..cb9ed87 100644 --- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl @@ -1,31 +1,59 @@ log { source(s_DEFAULT); - rewrite { - r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); - set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG")); - }; - parser { - p_add_context_splunk(key("sc4s_fallback")); - }; + if { + filter(f_is_rfc5424_strict); + rewrite { + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main")); + set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG")); + }; + parser { + p_add_context_splunk(key("sc4s_fallback")); + }; + {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} + destination(d_hec); + {{- end}} + + + #in fallback archive only write rawmsg as msg + rewrite { + unset(value("RAWMSG")); + groupunset(values(".kv.*")); + }; -{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} - destination(d_hec); -{{- end}} + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} + destination(d_archive); + {{- end}} + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main") ); + set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG")); + }; + parser { + p_add_context_splunk(key("sc4s_fallback")); + }; - #in fallback archive only write rawmsg as msg - rewrite { - set("$RAWMSG" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); + {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} + destination(d_hec); + {{- end}} + + + #in fallback archive only write rawmsg as msg + rewrite { + set("$RAWMSG" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} + destination(d_archive); + {{- end}} }; -{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} - destination(d_archive); -{{- end}} - flags(flow-control,fallback,final); + + flags(flow-control,fallback); }; diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index 6c19bc3..51b71c0 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -53,3 +53,4 @@ #sc4s_events,index,main #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics +#vmware_nsx,index,main \ No newline at end of file diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index ab0a630..c9adef0 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -152,7 +152,7 @@ def test_checkpoint_splunk_os(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main \"0x{{ pid }}\" sourcetype=\"nix:syslog\" | head 2") + st = env.from_string("search index=osnix \"0x{{ pid }}\" sourcetype=\"nix:syslog\" | head 2") search = st.render(pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_linux_syslog.py b/tests/test_linux_syslog.py index 914dea6..9297929 100644 --- a/tests/test_linux_syslog.py +++ b/tests/test_linux_syslog.py @@ -25,7 +25,7 @@ def test_linux__nohost_program_as_path(record_property, setup_wordlist, setup_sp sendsingle(message) - st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") + st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -45,7 +45,7 @@ def test_linux__host_program_as_path(record_property, setup_wordlist, setup_splu sendsingle(message) - st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") + st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -65,7 +65,7 @@ def test_linux__nohost_program_conforms(record_property, setup_wordlist, setup_s sendsingle(message) - st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") + st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -85,7 +85,7 @@ def test_linux__host_program_conforms(record_property, setup_wordlist, setup_spl sendsingle(message) - st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") + st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") search = st.render(host=host, pid=pid) resultCount, eventCount = splunk_single(setup_splunk, search) diff --git a/tests/test_vmware.py b/tests/test_vmware.py new file mode 100644 index 0000000..e406ee8 --- /dev/null +++ b/tests/test_vmware.py @@ -0,0 +1,79 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import datetime +import random +import pytz + +from jinja2 import Environment, environment + +from .sendmessage import * +from .splunkutils import * +import random + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#vpxd 123 - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)] +def test_linux_vmware(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} vpxd {{ pid }} - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)]\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main {{ pid }} sourcetype=\"vmware:esx:vsphere:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<46>1 2019-10-24T21:00:02.403Z {{ host }} NSXV 5996 - [nsxv@6876 comp="nsx-manager" subcomp="manager"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704 +def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSXV {{ pid }} - [nsxv@6876 comp=\"nsx-manager\" subcomp=\"manager\"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main host={{ host }} PID={{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# +def test_linux_vmware_nsx_fw(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dfwpktlogs: {{ pid }} INET match PASS domain-c7/1001 IN 60 TCP 10.33.24.50/45926->10.33.24.9/8140 S\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main host={{ host }} {{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1