Merge pull request #242 from splunk/feature/support-esxi

Feature/support vmware vsphere, nsx, and esx

docs/sources/VMWare/index.md

@@ -0,0 +1,53 @@
+# Vendor - Dell - VMWare
+
+## Product - vSphwere - ESX NSX (Controller, Manager, Edge)
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | None                                                                |
+| Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html |
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| vmware:nsx:vsphere:syslog | None |
+| vmware:esx:vsphere:syslog | None |
+| nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx  |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| vmware_nsx      | vmware:nsx:vsphere:syslog | main          | none          |
+| vmware_esx      | vmware:esx:vsphere:syslog | main          | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content when using the default configuration
+
+### Setup and Configuration
+
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_VMWARE_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_VMWARE | no | Enable archive to disk for this specific source |
+| SC4S_DEST_VMWARE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active proxy will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype="vmware:*:vsphere:*" | stats count by host
+```

package/etc/conf.d/conflib/_common/templates.conf

@@ -63,6 +63,7 @@ template t_JSON {
 	            --exclude .splunk.*
 	            --exclude HOST
 	            --exclude HOST_FROM
+	            --exclude SOURCE
 	            )");
 	};
 
@@ -77,5 +78,6 @@ template t_JSON_5424 {
 	            --exclude HOST
 	            --exclude HOST_FROM
 	            --exclude RAWMSG
+	            --exclude SOURCE
 	            )");
 	};

package/etc/conf.d/destinations/splunk_hec.conf.tmpl

@@ -42,7 +42,7 @@ destination d_hec {
                  source=${.splunk.source}
                  sourcetype=${.splunk.sourcetype}
                  index=${.splunk.index}
-                 event=$MSG
+                 event="$MSG"
                  fields.*)')
         );
 };

package/etc/conf.d/filters/VMware/vsphere.conf

@@ -0,0 +1,58 @@
+filter f_vmware_all {
+    #begin base vmware
+    program("cimslp", flags(ignore-case))
+    or program("Fdm", flags(ignore-case))
+    or program("Hostd", flags(ignore-case))
+    or program("hostd-probe", flags(ignore-case))
+    or program("indcfg", flags(ignore-case))
+    or program("lwsmd", flags(ignore-case))
+    or program("netcpa", flags(ignore-case))
+    or program("pktcap-agent", flags(ignore-case))
+    or program("Rhttpproxy", flags(ignore-case))
+    or program("sdrsInjector", flags(ignore-case))
+    or program("sfcb-.*", flags(ignore-case))
+    or program("storageRM", flags(ignore-case))
+    or program("vmkernel", flags(ignore-case))
+    or program("vmkwarning", flags(ignore-case))
+    or program("vobd", flags(ignore-case))
+    or program("Vpxa", flags(ignore-case))
+    or program("Vpxd", flags(ignore-case))
+    or program("VSANMGMTSVC", flags(ignore-case))
+    or program("vsfwd", flags(ignore-case))
+    #begin nsx
+    or program("NSX", flags(ignore-case))
+    or program("NSXV", flags(ignore-case))
+    or program("dfwpktlogs", flags(ignore-case))
+    or program("nsx-.*", flags(ignore-case))};
+
+filter f_vmware_vsphere {
+    program("cimslp", flags(ignore-case))
+    or program("Fdm", flags(ignore-case))
+    or program("Hostd", flags(ignore-case))
+    or program("hostd-probe", flags(ignore-case))
+    or program("indcfg", flags(ignore-case))
+    or program("lwsmd", flags(ignore-case))
+    or program("netcpa", flags(ignore-case))
+    or program("pktcap-agent", flags(ignore-case))
+    or program("Rhttpproxy", flags(ignore-case))
+    or program("sdrsInjector", flags(ignore-case))
+    or program("sfcb-.*", flags(ignore-case))
+    or program("storageRM", flags(ignore-case))
+    or program("vmkernel", flags(ignore-case))
+    or program("vmkwarning", flags(ignore-case))
+    or program("vobd", flags(ignore-case))
+    or program("Vpxa", flags(ignore-case))
+    or program("Vpxd", flags(ignore-case))
+    or program("VSANMGMTSVC", flags(ignore-case))
+    or program("vsfwd", flags(ignore-case))
+};
+
+filter f_vmware_nsx {
+    program("NSX", flags(ignore-case))
+    or
+    program("NSXV", flags(ignore-case))
+    or
+    program("dfwpktlogs", flags(ignore-case))
+    or
+    program("nsx-.*", flags(ignore-case))
+};

package/etc/conf.d/log_paths/p_multi-vmware_nsx.conf.tmpl

@@ -0,0 +1,117 @@
+# Generate the custom port if defined
+{{ $context := dict "port_id" "VMWARE" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes"}}
+    source(s_DEFAULT);
+
+    filter(f_vmware_all);
+{{- end}}
+{{- if eq (.) "no"}}
+    source (s_VMWARE);
+{{- end}}
+
+
+    #NSX first because its the cheapest check
+    if {
+        filter(f_is_rfc5424_strict);
+        filter(f_vmware_nsx);
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), source("program:${PROGRAM}"));
+            set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("vmware_nsx"));
+        };
+    } elif {
+
+        filter(f_vmware_nsx);
+
+        rewrite {
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("vmware:nsx:vsphere:syslog"), index("main"), source("program:${.PROGRAM}"));
+            set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("vmware_nsx"));
+        };
+    #esx things
+    } elif {
+        filter(f_is_rfc5424_strict);
+        filter(f_vmware_vsphere);
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), source("program:${PROGRAM}"));
+            set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("vmware_esx"));
+        };
+    } elif {
+
+        filter(f_vmware_vsphere);
+
+        rewrite {
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("vmware:esx:vsphere:syslog"), index("main"), source("program:${.PROGRAM}"));
+            set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("vmware_esx"));
+        };
+    } else {
+
+        rewrite {
+            set("nix_syslog", value("fields.sc4s_vendor_product"));
+            subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+        };
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") )
+
+        };
+
+        parser { p_add_context_splunk(key("nix_syslog")); };
+
+        parser (compliance_meta_by_source);
+
+        #We want to unset the fields we won't need, as this is copied into the
+        #disk queue for network destinations. This can be very disk expensive
+        #if we don't
+        rewrite {
+            set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+            unset(value("RAWMSG"));
+            unset(value("PROGRAM"));
+            unset(value("LEGACY_MSGHDR"));
+        };
+    };
+
+    parser (compliance_meta_by_source);
+
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
+
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE") }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,final);
+};
+{{- end}}
+{{- if or (or (getenv  (print "SC4S_LISTEN_VMWARE_TCP_PORT")) (getenv  (print "SC4S_LISTEN_VMWARE_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_VMWARE_TLS_PORT")) }}
+# Listen on the specified dedicated port(s) for VMWARE traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for VMWARE traffic
+{{ tmpl.Exec "log_path" "yes" }}

package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl

@@ -1,31 +1,59 @@
 log {
     source(s_DEFAULT);
 
-    rewrite {
-        r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON"));
-        set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG"));
-    };
-    parser {
-        p_add_context_splunk(key("sc4s_fallback"));
-    };
+    if {
+        filter(f_is_rfc5424_strict);
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"));
+            set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("sc4s_fallback"));
+        };
+    {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }}
+        destination(d_hec);
+    {{- end}}
+
+
+        #in fallback archive only write rawmsg as msg
+        rewrite {
+            unset(value("RAWMSG"));
+            groupunset(values(".kv.*"));
+        };
 
-{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }}
-    destination(d_hec);
-{{- end}}
+    {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }}
+        destination(d_archive);
+    {{- end}}
 
+    } else {
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main") );
+            set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("sc4s_fallback"));
+        };
 
-    #in fallback archive only write rawmsg as msg
-    rewrite {
-        set("$RAWMSG" value("MSG"));
-        unset(value("RAWMSG"));
-        unset(value("PROGRAM"));
-        unset(value("LEGACY_MSGHDR"));
-        groupunset(values(".kv.*"));
+    {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }}
+        destination(d_hec);
+    {{- end}}
+
+
+        #in fallback archive only write rawmsg as msg
+        rewrite {
+            set("$RAWMSG" value("MSG"));
+            unset(value("RAWMSG"));
+            unset(value("PROGRAM"));
+            unset(value("LEGACY_MSGHDR"));
+            groupunset(values(".kv.*"));
+        };
+
+    {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }}
+        destination(d_archive);
+    {{- end}}
     };
 
-{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }}
-    destination(d_archive);
-{{- end}}
 
-    flags(flow-control,fallback,final);
+
+    flags(flow-control,fallback);
 };

package/etc/context_templates/splunk_index.csv

@@ -53,3 +53,4 @@
 #sc4s_events,index,main
 #sc4s_fallback,index,main
 #sc4s_metrics,index,em_metrics
+#vmware_nsx,index,main

tests/test_checkpoint.py

@@ -152,7 +152,7 @@ def test_checkpoint_splunk_os(record_property, setup_wordlist, setup_splunk):
 
     sendsingle(message)
 
-    st = env.from_string("search index=main \"0x{{ pid }}\" sourcetype=\"nix:syslog\" | head 2")
+    st = env.from_string("search index=osnix \"0x{{ pid }}\" sourcetype=\"nix:syslog\" | head 2")
     search = st.render(pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)

tests/test_linux_syslog.py

@@ -25,7 +25,7 @@ def test_linux__nohost_program_as_path(record_property, setup_wordlist, setup_sp
 
     sendsingle(message)
 
-    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
+    st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
@@ -45,7 +45,7 @@ def test_linux__host_program_as_path(record_property, setup_wordlist, setup_splu
 
     sendsingle(message)
 
-    st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2")
+    st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
@@ -65,7 +65,7 @@ def test_linux__nohost_program_conforms(record_property, setup_wordlist, setup_s
 
     sendsingle(message)
 
-    st = env.from_string("search index=main \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
+    st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
@@ -85,7 +85,7 @@ def test_linux__host_program_conforms(record_property, setup_wordlist, setup_spl
 
     sendsingle(message)
 
-    st = env.from_string("search index=main \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2")
+    st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)