Add support vmware nsx

docs/sources/VMWare/index.md

@@ -0,0 +1,52 @@
+# Vendor - Dell - VMWare
+
+## Product - NSX Controller, Manager, Edge
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | None                                                                |
+| Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| vmware:nsx:vsphere:syslog | None |
+| nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx  |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| vmware_nsx      | vmware:nsx:vsphere:syslog | main          | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content when using the default configuration
+
+### Setup and Configuration
+
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_VMWARE_NSX_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_NSX_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_NSX_TLS_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_VMWARE_NSX | no | Enable archive to disk for this specific source |
+| SC4S_DEST_VMWARE_NSX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active proxy will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=vmware:nsx:vsphere:syslog | stats count by host
+```

package/etc/conf.d/conflib/_common/templates.conf

@@ -63,6 +63,7 @@ template t_JSON {
 	            --exclude .splunk.*
 	            --exclude HOST
 	            --exclude HOST_FROM
+	            --exclude SOURCE
 	            )");
 	};
 
@@ -77,5 +78,6 @@ template t_JSON_5424 {
 	            --exclude HOST
 	            --exclude HOST_FROM
 	            --exclude RAWMSG
+	            --exclude SOURCE
 	            )");
 	};

package/etc/conf.d/destinations/splunk_hec.conf.tmpl

@@ -42,7 +42,7 @@ destination d_hec {
                  source=${.splunk.source}
                  sourcetype=${.splunk.sourcetype}
                  index=${.splunk.index}
-                 event=$MSG
+                 event="$MSG"
                  fields.*)')
         );
 };

package/etc/conf.d/filters/VMware/nsx.conf

@@ -0,0 +1,8 @@
+
+filter f_vmware_nsx {
+    program("NSX")
+    or
+    program("NSXV")
+    or
+    program("dfwpktlogs")
+};

package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl

@@ -0,0 +1,92 @@
+# Generate the custom port if defined
+{{ $context := dict "port_id" "VMWARE_NSX" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes"}}
+    source(s_DEFAULT);
+
+    filter(f_vmware_nsx);
+{{- end}}
+{{- if eq (.) "no"}}
+    source (s_VMWARE_NSX);
+{{- end}}
+
+
+    if {
+        filter(f_is_rfc5424_strict);
+        filter(f_vmware_nsx);
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_JSON_5424"), source("program:${PROGRAM}"));
+            set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("vmware_nsx"));
+        };
+    } elif {
+
+        filter(f_vmware_nsx);
+
+        rewrite {
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), template("t_legacy_hdr_msg"), source("program:${.PROGRAM}"));
+            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("vmware_nsx"));
+        };
+
+    } else {
+
+        rewrite {
+            set("nix_syslog", value("fields.sc4s_vendor_product"));
+            subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+        };
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") )
+
+        };
+
+        parser { p_add_context_splunk(key("nix_syslog")); };
+
+        parser (compliance_meta_by_source);
+
+        #We want to unset the fields we won't need, as this is copied into the
+        #disk queue for network destinations. This can be very disk expensive
+        #if we don't
+        rewrite {
+            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+            unset(value("RAWMSG"));
+            unset(value("PROGRAM"));
+            unset(value("LEGACY_MSGHDR"));
+        };
+    };
+
+    parser (compliance_meta_by_source);
+
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_NSX_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
+
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE_NSX") }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,final);
+};
+{{- end}}
+{{- if or (or (getenv  (print "SC4S_LISTEN_VMWARE_NSX_TCP_PORT")) (getenv  (print "SC4S_LISTEN_VMWARE_NSX_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_VMWARE_NSX_TLS_PORT")) }}
+# Listen on the specified dedicated port(s) for VMWARE_NSX traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for VMWARE_NSX traffic
+{{ tmpl.Exec "log_path" "yes" }}

package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl

@@ -1,31 +1,59 @@
 log {
     source(s_DEFAULT);
 
-    rewrite {
-        r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
-        set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG"));
-    };
-    parser {
-        p_add_context_splunk(key("sc4s_fallback"));
-    };
+    if {
+        filter(f_is_rfc5424_strict);
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
+            set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("sc4s_fallback"));
+        };
+    {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }}
+        destination(d_hec);
+    {{- end}}
+
+
+        #in fallback archive only write rawmsg as msg
+        rewrite {
+            unset(value("RAWMSG"));
+            groupunset(values(".kv.*"));
+        };
 
-{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }}
-    destination(d_hec);
-{{- end}}
+    {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }}
+        destination(d_archive);
+    {{- end}}
 
+    } else {
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
+            set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("sc4s_fallback"));
+        };
 
-    #in fallback archive only write rawmsg as msg
-    rewrite {
-        set("$RAWMSG" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-        groupunset(values(".kv.*"));
+    {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }}
+        destination(d_hec);
+    {{- end}}
+
+
+        #in fallback archive only write rawmsg as msg
+        rewrite {
+            set("$RAWMSG" value("MSG"));
+            unset(value("RAWMSG"));
+            unset(value("PROGRAM"));
+            unset(value("LEGACY_MSGHDR"));
+            groupunset(values(".kv.*"));
+        };
+
+    {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }}
+        destination(d_archive);
+    {{- end}}
     };
 
-{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }}
-    destination(d_archive);
-{{- end}}
 
-    flags(flow-control,fallback,final);
+
+    flags(flow-control,fallback);
 };

package/etc/context_templates/splunk_index.csv

@@ -53,3 +53,4 @@
 #sc4s_events,index,main
 #sc4s_fallback,index,main
 #sc4s_metrics,index,em_metrics
+#vmware_nsx,index,main

tests/test_vmware.py

@@ -0,0 +1,59 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import datetime
+import random
+import pytz
+
+from jinja2 import Environment, environment
+
+from .sendmessage import *
+from .splunkutils import *
+import random
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+
+#<46>1 2019-10-24T21:00:02.403Z {{ host }} NSXV 5996 - [nsxv@6876 comp="nsx-manager" subcomp="manager"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704
+def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk):
+    host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSXV {{ pid }} - [nsxv@6876 comp=\"nsx-manager\" subcomp=\"manager\"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704\n")
+    message = mt.render(mark="<144>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main host={{ host }} PID={{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#
+def test_linux_vmware_nsx_fw(record_property, setup_wordlist, setup_splunk):
+    host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dfwpktlogs: {{ pid }} INET match PASS domain-c7/1001 IN 60 TCP 10.33.24.50/45926->10.33.24.9/8140 S\n")
+    message = mt.render(mark="<144>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=main host={{ host }} {{ pid }} sourcetype=\"vmware:nsx:vsphere:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1