From 9e1992c8d08f69a0f2d1aa66c954aba7b0348b1c Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 16 Dec 2019 15:13:17 -0500 Subject: [PATCH] vmware support --- docs/sources/VMWare/index.md | 17 +++--- package/etc/conf.d/filters/VMware/nsx.conf | 8 --- .../etc/conf.d/filters/VMware/vsphere.conf | 58 +++++++++++++++++++ .../log_paths/p_multi-vmware_nsx.conf.tmpl | 41 ++++++++++--- tests/test_vmware.py | 20 +++++++ 5 files changed, 120 insertions(+), 24 deletions(-) delete mode 100644 package/etc/conf.d/filters/VMware/nsx.conf create mode 100644 package/etc/conf.d/filters/VMware/vsphere.conf diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md index d8954da..3b9205e 100644 --- a/docs/sources/VMWare/index.md +++ b/docs/sources/VMWare/index.md @@ -1,6 +1,6 @@ # Vendor - Dell - VMWare -## Product - NSX Controller, Manager, Edge +## Product - vSphwere - ESX NSX (Controller, Manager, Edge) | Ref | Link | @@ -8,12 +8,12 @@ | Splunk Add-on | None | | Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html | - ### Sourcetypes | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| | vmware:nsx:vsphere:syslog | None | +| vmware:esx:vsphere:syslog | None | | nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx | ### Sourcetype and Index Configuration @@ -21,6 +21,7 @@ | key | sourcetype | index | notes | |----------------|----------------|----------------|----------------| | vmware_nsx | vmware:nsx:vsphere:syslog | main | none | +| vmware_esx | vmware:esx:vsphere:syslog | main | none | ### Filter type @@ -37,16 +38,16 @@ MSG Parse: This filter parses message content when using the default configurati | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_VMWARE_NSX_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | -| SC4S_LISTEN_VMWARE_NSX_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | -| SC4S_LISTEN_VMWARE_NSX_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | -| SC4S_ARCHIVE_VMWARE_NSX | no | Enable archive to disk for this specific source | -| SC4S_DEST_VMWARE_NSX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_LISTEN_VMWARE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_VMWARE | no | Enable archive to disk for this specific source | +| SC4S_DEST_VMWARE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification An active proxy will generate frequent events. Use the following search to validate events are present per source device ``` -index= sourcetype=vmware:nsx:vsphere:syslog | stats count by host +index= sourcetype="vmware:*:vsphere:*" | stats count by host ``` diff --git a/package/etc/conf.d/filters/VMware/nsx.conf b/package/etc/conf.d/filters/VMware/nsx.conf deleted file mode 100644 index 88744dd..0000000 --- a/package/etc/conf.d/filters/VMware/nsx.conf +++ /dev/null @@ -1,8 +0,0 @@ - -filter f_vmware_nsx { - program("NSX") - or - program("NSXV") - or - program("dfwpktlogs") -}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/VMware/vsphere.conf b/package/etc/conf.d/filters/VMware/vsphere.conf new file mode 100644 index 0000000..624c6e0 --- /dev/null +++ b/package/etc/conf.d/filters/VMware/vsphere.conf @@ -0,0 +1,58 @@ +filter f_vmware_all { + #begin base vmware + program("cimslp", flags(ignore-case)) + or program("Fdm", flags(ignore-case)) + or program("Hostd", flags(ignore-case)) + or program("hostd-probe", flags(ignore-case)) + or program("indcfg", flags(ignore-case)) + or program("lwsmd", flags(ignore-case)) + or program("netcpa", flags(ignore-case)) + or program("pktcap-agent", flags(ignore-case)) + or program("Rhttpproxy", flags(ignore-case)) + or program("sdrsInjector", flags(ignore-case)) + or program("sfcb-.*", flags(ignore-case)) + or program("storageRM", flags(ignore-case)) + or program("vmkernel", flags(ignore-case)) + or program("vmkwarning", flags(ignore-case)) + or program("vobd", flags(ignore-case)) + or program("Vpxa", flags(ignore-case)) + or program("Vpxd", flags(ignore-case)) + or program("VSANMGMTSVC", flags(ignore-case)) + or program("vsfwd", flags(ignore-case)) + #begin nsx + or program("NSX", flags(ignore-case)) + or program("NSXV", flags(ignore-case)) + or program("dfwpktlogs", flags(ignore-case)) + or program("nsx-.*", flags(ignore-case))}; + +filter f_vmware_vsphere { + program("cimslp", flags(ignore-case)) + or program("Fdm", flags(ignore-case)) + or program("Hostd", flags(ignore-case)) + or program("hostd-probe", flags(ignore-case)) + or program("indcfg", flags(ignore-case)) + or program("lwsmd", flags(ignore-case)) + or program("netcpa", flags(ignore-case)) + or program("pktcap-agent", flags(ignore-case)) + or program("Rhttpproxy", flags(ignore-case)) + or program("sdrsInjector", flags(ignore-case)) + or program("sfcb-.*", flags(ignore-case)) + or program("storageRM", flags(ignore-case)) + or program("vmkernel", flags(ignore-case)) + or program("vmkwarning", flags(ignore-case)) + or program("vobd", flags(ignore-case)) + or program("Vpxa", flags(ignore-case)) + or program("Vpxd", flags(ignore-case)) + or program("VSANMGMTSVC", flags(ignore-case)) + or program("vsfwd", flags(ignore-case)) +}; + +filter f_vmware_nsx { + program("NSX", flags(ignore-case)) + or + program("NSXV", flags(ignore-case)) + or + program("dfwpktlogs", flags(ignore-case)) + or + program("nsx-.*", flags(ignore-case)) +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl index 6e623a3..23a480f 100644 --- a/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl @@ -1,5 +1,5 @@ # Generate the custom port if defined -{{ $context := dict "port_id" "VMWARE_NSX" "parser" "common" }} +{{ $context := dict "port_id" "VMWARE" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} # The following is an inline template; we will use this to generate the actual log path @@ -8,13 +8,14 @@ log { {{- if eq (.) "yes"}} source(s_DEFAULT); - filter(f_vmware_nsx); + filter(f_vmware_all); {{- end}} {{- if eq (.) "no"}} - source (s_VMWARE_NSX); + source (s_VMWARE); {{- end}} + #NSX first because its the cheapest check if { filter(f_is_rfc5424_strict); filter(f_vmware_nsx); @@ -39,7 +40,31 @@ log { parser { p_add_context_splunk(key("vmware_nsx")); }; + #esx things + } elif { + filter(f_is_rfc5424_strict); + filter(f_vmware_vsphere); + + rewrite { + r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}")); + set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_esx")); + }; + } elif { + filter(f_vmware_vsphere); + + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}")); + set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_esx")); + }; } else { rewrite { @@ -71,22 +96,22 @@ log { parser (compliance_meta_by_source); -{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_NSX_HEC" "no") | conv.ToBool) }} +{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_HEC" "no") | conv.ToBool) }} destination(d_hec); {{- end}} -{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE_NSX") }} +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE") }} destination(d_archive); {{- end}} flags(flow-control,final); }; {{- end}} -{{- if or (or (getenv (print "SC4S_LISTEN_VMWARE_NSX_TCP_PORT")) (getenv (print "SC4S_LISTEN_VMWARE_NSX_UDP_PORT"))) (getenv (print "SC4S_LISTEN_VMWARE_NSX_TLS_PORT")) }} -# Listen on the specified dedicated port(s) for VMWARE_NSX traffic +{{- if or (or (getenv (print "SC4S_LISTEN_VMWARE_TCP_PORT")) (getenv (print "SC4S_LISTEN_VMWARE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_VMWARE_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for VMWARE traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} -# Listen on the default port (typically 514) for VMWARE_NSX traffic +# Listen on the default port (typically 514) for VMWARE traffic {{ tmpl.Exec "log_path" "yes" }} diff --git a/tests/test_vmware.py b/tests/test_vmware.py index e64b3ca..e406ee8 100644 --- a/tests/test_vmware.py +++ b/tests/test_vmware.py @@ -15,6 +15,26 @@ env = Environment(extensions=['jinja2_time.TimeExtension']) +#vpxd 123 - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)] +def test_linux_vmware(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} vpxd {{ pid }} - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)]\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main {{ pid }} sourcetype=\"vmware:esx:vsphere:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 #<46>1 2019-10-24T21:00:02.403Z {{ host }} NSXV 5996 - [nsxv@6876 comp="nsx-manager" subcomp="manager"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704 def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk):