Merge pull request #380 from splunk/feature/f5

Support F5 and fix vmware docs

docs/sources/F5/index.md

@@ -0,0 +1,53 @@
+# Vendor - F5
+
+
+## Product - BigIP
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/2680/                                                                 |
+| Product Manual | unknown   |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| f5:bigip:syslog        | None                                                                                                    |
+| f5:bigip:irule    | None                                                                                         |
+| nix:syslog     | None                                                                                          |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| f5_bigip      | f5:bigip:syslog       | netops          | none          |
+| f5_bigip_irule    | f5:bigip:syslog      | netops          | none          |
+| f5_bigip_nix    | nix:syslog      | netops          | if `f_f5_bigip` is not set the index osnix will be used          |
+
+### Filter type
+
+Must be identified by host or ip assignment. Update the filter `f_f5_bigip` or configure a dedicated port as required
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the admin manual for specific details of configuration
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_F5_BIGIP_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_F5_BIGIP_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_F5_BIGIP | no | Enable archive to disk for this specific source |
+| SC4S_DEST_F5_BIGIP_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active device will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=f5:bigip:*| stats count by host
+```

docs/sources/VMWare/index.md

@@ -38,11 +38,11 @@ MSG Parse: This filter parses message content when using the default configurati
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_VMWARE_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
-| SC4S_LISTEN_VMWARE_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
-| SC4S_LISTEN_VMWARE_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
-| SC4S_ARCHIVE_VMWARE | no | Enable archive to disk for this specific source |
-| SC4S_DEST_VMWARE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_LISTEN_VMWARE_VSPHERE_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_VSPHERE_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_VSPHERE_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_VMWARE_VSPHERE | no | Enable archive to disk for this specific source |
+| SC4S_DEST_VMWARE_VSPHERE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 
 ### Verification
 

mkdocs.yml

@@ -20,6 +20,7 @@ nav:
       - "Common Event Format": sources/CommonEventFormat/index.md
       - CyberArk: sources/CyberArk/index.md
       - "Dell RSA": sources/Dell_RSA/index.md
+      - F5: sources/F5/index.md
       - Forcepoint: sources/Forcepoint/index.md
       - Fortinet: sources/Fortinet/index.md
       - Imperva: sources/Imperva/index.md

package/etc/conf.d/filters/f5/bigip.conf.tmpl

@@ -0,0 +1,15 @@
+filter f_f5_bigip {
+    match("^f5_bigip", value("fields.sc4s_vendor_product"));
+};
+filter f_f5_bigip_message {
+    message(
+        '^(?i)(<\d+> ?[[:alpha:]]+ \d\d \d\d:\d\d:\d\d +[^ ]+) +(?:notice|err|error|warning|info) +?(.*)'
+        flags(store-matches)
+    );
+};
+parser f_f5_bigip_message {
+    syslog-parser(
+        template("$1 $2")
+        flags(guess-timezone,assume-utf8,{{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})
+    );
+};

package/etc/conf.d/log_paths/lp-f5_bigip.conf.tmpl

@@ -0,0 +1,80 @@
+# F5_BIGIP
+{{- /* The following provides a unique port source configuration if env var(s) are set */}}
+{{- $context := dict "port_id" "F5_BIGIP" "parser" "common" }}
+{{- tmpl.Exec "t/source_network.t" $context }}
+
+log {
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_F5_BIGIP_TCP_PORT")) (getenv  (print "SC4S_LISTEN_F5_BIGIP_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_F5_BIGIP_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for F5_BIGIP traffic
+            source (s_F5_BIGIP);
+            flags (final);
+	    };
+{{- end}}
+        channel {
+        # Listen on the default port (typically 514) for F5_BIGIP traffic
+            source (s_DEFAULT);
+            flags(final);
+        };
+    };
+
+    if {
+        filter{
+            program("tmsh")
+            or program("mcpd")
+            or program("tmm\d?")
+        };
+        rewrite {
+            set("f5_bigip", value("fields.sc4s_vendor_product"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("f5:bigip:syslog"), index("netops"), source("program:${.PROGRAM}"))
+        };
+        parser { p_add_context_splunk(key("f5_bigip")); };
+        parser (compliance_meta_by_source);
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); };
+    } elif {
+        filter {
+            program('^,f5_irule')
+        };
+        rewrite {
+            set("f5_bigip_irule", value("fields.sc4s_vendor_product"));
+            r_set_splunk_dest_default(sourcetype("f5:bigip:irule"), index("netops"))
+        };
+        parser { p_add_context_splunk(key("f5_bigip_irule")); };
+        parser (compliance_meta_by_source);
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); };
+    } elif {
+        filter(f_f5_bigip);
+        rewrite {
+            set("f5_bigip_nix", value("fields.sc4s_vendor_product"));
+            subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}"))
+        };
+        parser { p_add_context_splunk(key("f5_bigip")); };
+        parser (compliance_meta_by_source);
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); };
+    };
+
+
+{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_F5_BIGIP_HEC" "no")) }}
+    destination(d_hec);
+{{- end}}
+
+{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_F5_BIGIP" "no")) }}
+    destination(d_archive);
+{{- end}}
+
+{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
+    {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+{{- end }}
+
+{{- if (print (getenv "SC4S_DEST_F5_BIGIP_ALTERNATES")) }}
+    {{ getenv "SC4S_DEST_F5_BIGIP_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+{{- end }}
+
+    flags(flow-control,final);
+};

package/etc/context_templates/splunk_index.csv.example

@@ -25,6 +25,9 @@
 #citrix_netscaler,index,netfw
 #local_example,index,main
 #forcepoint_webprotect,index,netproxy
+#f5_bigip,index,netops
+#f5_bigip_irule,index,netops
+#f5_bigip_nix,index,netops
 #fortinet_fortios_event,index,netops
 #fortinet_fortios_log,index,netops
 #fortinet_fortios_traffic,index,netfw

package/etc/context_templates/vendor_product_by_source.conf.example

@@ -43,6 +43,10 @@ filter f_cisco_nx_os {
     host("csconx-*" type(glob))
     #or netmask(xxx.xxx.xxx.xxx/xx)
 };
+filter f_f5_bigip {
+    host("test_f5-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
+};
 filter f_infoblox {
     host("vib-*" type(glob))
     #or netmask(xxx.xxx.xxx.xxx/xx)

package/etc/context_templates/vendor_product_by_source.csv.example

@@ -4,6 +4,7 @@ f_catch_first,sc4s_vendor_product,"catch_first"
 f_cisco_meraki,sc4s_vendor_product,"cisco_meraki"
 f_citrix_netscaler,sc4s_vendor_product,"citrix_netscaler"
 f_dell_rsa_secureid,sc4s_vendor_product,"dell_rsa_secureid"
+f_f5_bigip,sc4s_vendor_product,"f5_bigip"
 f_infoblox,sc4s_vendor_product,"infoblox"
 f_juniper_nsm,sc4s_vendor_product,"juniper_nsm"
 f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp"

package/etc/go_templates/source_network.t

@@ -16,7 +16,7 @@ source s_{{ .port_id }} {
                 use-dns(no)
                 use-fqdn(no)
                 chain-hostnames(off)
-                flags(no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})
+                flags(validate-utf8, no-parse {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})
             );
 {{- end}}
 {{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }}
@@ -109,6 +109,10 @@ source s_{{ .port_id }} {
             filter(f_citrix_netscaler_message);
             parser(p_citrix_netscaler_date);
             rewrite(r_citrix_netscaler_message);
+        } elif {
+            filter(f_f5_bigip_message);
+            parser(f_f5_bigip_message);
+            rewrite(set_rfc3164);
         } elif {
             #JSON over IP its not syslog but it can work
             filter { message('^{') and message('}$') };