Merge pull request #239 from splunk/feature/infoblox

Support Infoblox DNS

docs/sources/InfoBlox/index.md

@@ -0,0 +1,54 @@
+# Vendor - Infoblox
+
+## Product - NIOS
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/2934/                                                                 |
+| Product Manual | http://dloads.infoblox.com/direct/appliance//NIOS/NIOS_AdminGuide_6.10.pdf    |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| infoblox:dns        | None                                                                                                    |
+| infoblox:dhcp    | None                                                                                         |
+| infoblox:threat     | None                                                                                          |
+| nix:syslog     | None                                                                                          |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| infoblox_dns      | infoblox:dns       | netdns          | none          |
+| infoblox_dhcp    | infoblox:dhcp      | netipam          | none          |
+| infoblox_threat    | infoblox:threat      | netids          | none          |
+| nix_syslog    | nix:syslog      | osnix          | none          |
+
+### Filter type
+
+Must be identified by host or ip assignment. Update the filter `f_infoblox` or configure a dedicated port as required
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the admin manual for specific details of configuration
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_INFOBLOX_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_INFOBLOX_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_INFOBLOX | no | Enable archive to disk for this specific source |
+| SC4S_DEST_INFOBLOX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active device will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=infoblox:*| stats count by host
+```

package/etc/conf.d/filters/infoblox/syslog.conf

@@ -0,0 +1,4 @@
+filter f_infoblox {
+    match("^infoblox", value("fields.sc4s_vendor_product"));
+
+};

package/etc/conf.d/filters/juniper/legacy.conf

@@ -14,7 +14,7 @@ filter f_juniper_netscreen {
 
 };
 
- filter f_juniper_idp {
+filter f_juniper_idp {
     match("juniper_idp", value("fields.sc4s_vendor_product") type(glob))
       or match('^\[syslog@juniper' value("SDATA"))
 };

package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl

@@ -0,0 +1,104 @@
+# Juniper IDP
+{{ $context := dict "port_id" "INFOBLOX" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes"}}
+    source(s_DEFAULT);
+    filter(f_is_rfc3164);
+    filter(f_infoblox);
+{{- end}}
+{{- if eq (.) "no"}}
+    source (s_INFOBLOX);
+{{- end}}
+
+
+    if {
+        filter{program("named")};
+        rewrite {
+            set("infoblox_dns", value("fields.sc4s_vendor_product"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("infoblox:dns"), index("netdns"), source("program:${.PROGRAM}"))
+            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("infoblox_dns"));
+        };
+    } elif {
+        filter{program("dhcpd")};
+        rewrite {
+            set("infoblox_dhcp", value("fields.sc4s_vendor_product"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("infoblox:dhcp"), index("netipam"), source("program:${.PROGRAM}"))
+            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("infoblox_dhcp"));
+        };
+    } elif {
+        filter{program("threat-protect-log")};
+        rewrite {
+            set("infoblox_dns", value("fields.sc4s_vendor_product"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+            r_set_splunk_dest_default(sourcetype("infoblox:threat"), index("netids"), source("program:${.PROGRAM}"))
+            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        };
+        parser {
+            p_add_context_splunk(key("infoblox_threat"));
+        };
+    } else {
+        rewrite {
+            set("nix_syslog", value("fields.sc4s_vendor_product"));
+            subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+            set("${PROGRAM}", value(".PROGRAM"));
+            subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+        };
+
+        rewrite {
+            r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") )
+            set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+
+        };
+
+        parser { p_add_context_splunk(key("nix_syslog")); };
+    };
+
+
+    parser (compliance_meta_by_source);
+
+    #We want to unset the fields we won't need, as this is copied into the
+    #disk queue for network destinations. This can be very disk expensive
+    #if we don't
+    rewrite {
+        unset(value("RAWMSG"));
+        unset(value("PROGRAM"));
+        unset(value("LEGACY_MSGHDR"));
+        unset(value("PID"));
+        groupunset(values(".kv.*"));
+    };
+
+
+{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_INFOBLOX_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_INFOBLOX") }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,final);
+};
+{{- end}}
+
+{{- if or (or (getenv  (print "SC4S_LISTEN_INFOBLOX_TCP_PORT")) (getenv  (print "SC4S_LISTEN_INFOBLOX_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_INFOBLOX_TLS_PORT")) }}
+# Listen on the specified dedicated port(s) for INFOBLOX traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for INFOBLOX traffic
+{{ tmpl.Exec "log_path" "yes" }}

package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl

@@ -20,7 +20,7 @@ log {
     };
 
     rewrite {
-        r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"), source("program:${.PROGRAM}") )
+        r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") )
 
     };
 

package/etc/context_templates/splunk_index.csv

@@ -22,6 +22,9 @@
 #fortinet_fortios_log,index,netops
 #fortinet_fortios_traffic,index,netfw
 #fortinet_fortios_utm,index,netids
+#infoblox_dns,index,netdns
+#infoblox_dhcp,index,netipam
+#infoblox_threat,index,netids
 #juniper_idp,index,netids
 #juniper_structured,index,netops
 #juniper_idp_structured,index,netids

package/etc/context_templates/vendor_product_by_source.conf

@@ -1,50 +1,54 @@
 @version: 3.24
 
 filter f_test_test {
-    host("testvp-*" type(glob)) or
-    netmask(169.254.1.0/24)
+    host("testvp-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_juniper_idp {
-    host("jnpidp-*" type(glob)) or
-    netmask(169.254.2.0/24)
+    host("jnpidp-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_juniper_netscreen {
-    host("jnpns-*" type(glob)) or
-    netmask(169.254.3.0/24)
+    host("jnpns-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_juniper_nsm {
-    host("jnpnsm-*" type(glob)) or
-    netmask(169.254.4.0/24)
+    host("jnpnsm-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_juniper_nsm_idp {
-    host("jnpnsmidp-*" type(glob)) or
-    netmask(169.254.5.0/24)
+    host("jnpnsmidp-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_cisco_meraki {
-    host("testcm-*" type(glob)) or
-    netmask(169.254.6.0/24)
+    host("testcm-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_cisco_nx_os {
-    host("csconx-*" type(glob)) or
-    netmask(169.254.7.0/24)
+    host("csconx-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
+};
+filter f_infoblox {
+    host("vib-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_proofpoint_pps_filter {
-    host("pps-*" type(glob)) or
-    netmask(169.254.8.0/24)
+    host("pps-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_proofpoint_pps_sendmail {
-    host("pps-*" type(glob)) or
-    netmask(169.254.9.0/24)
+    host("pps-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_ubiquiti_unifi_fw {
-    host("usg-*" type(glob)) or
-    netmask(169.254.10.0/24)
+    host("usg-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_tzfixhst {
-    host("tzfhst-*" type(glob)) or
-    netmask(169.254.11.0/24)
+    host("tzfhst-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };
 filter f_tzfixny {
-    host("tzfny-*" type(glob)) or
-    netmask(169.254.12.0/24)
+    host("tzfny-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
 };

package/etc/context_templates/vendor_product_by_source.csv

@@ -1,5 +1,6 @@
 f_test_test,sc4s_vendor_product,"test_test"
 f_cisco_meraki,sc4s_vendor_product,"cisco_meraki"
+f_infoblox,sc4s_vendor_product,"infoblox"
 f_juniper_nsm,sc4s_vendor_product,"juniper_nsm"
 f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp"
 f_juniper_idp,sc4s_vendor_product,"juniper_idp"

splunk/etc/apps/SA-syslog-ng/default/indexes.conf

@@ -56,6 +56,11 @@ homePath   = $SPLUNK_DB/netdlp/db
 coldPath   = $SPLUNK_DB/netdlp/colddb
 thawedPath = $SPLUNK_DB/netdlp/thaweddb
 
+[netdns]
+homePath   = $SPLUNK_DB/netdns/db
+coldPath   = $SPLUNK_DB/netdns/colddb
+thawedPath = $SPLUNK_DB/netdns/thaweddb
+
 [netfw]
 homePath   = $SPLUNK_DB/netfw/db
 coldPath   = $SPLUNK_DB/netfw/colddb
@@ -66,6 +71,11 @@ homePath   = $SPLUNK_DB/netids/db
 coldPath   = $SPLUNK_DB/netids/colddb
 thawedPath = $SPLUNK_DB/netids/thaweddb
 
+[netipam]
+homePath   = $SPLUNK_DB/netipam/db
+coldPath   = $SPLUNK_DB/netipam/colddb
+thawedPath = $SPLUNK_DB/netipam/thaweddb
+
 [netops]
 homePath   = $SPLUNK_DB/netops/db
 coldPath   = $SPLUNK_DB/netops/colddb

tests/test_infoblox.py

@@ -0,0 +1,58 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import datetime
+import random
+import pytz
+
+from jinja2 import Environment, environment
+
+from .sendmessage import *
+from .splunkutils import *
+import random
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+#<142>Oct 25 13:08:00 161.231.218.156 named[6597]: FORMERR resolving 'www.google.com/AAAA/IN': 209.2.230.6#53
+def test_infoblox_dns(record_property, setup_wordlist, setup_splunk):
+    host = "vib-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} named[{{ pid }}]: FORMERR resolving 'www.google.com/AAAA/IN': 209.2.230.6#53\n")
+    message = mt.render(mark="<111>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netdns host={{ host }} sourcetype=\"infoblox:dns\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+#<150>Oct 24 21:09:00 162.101.157.246 dhcpd[28922]: DHCPREQUEST for 10.130.151.62 from 80:ce:62:9c:0e:70 (DTCCE0826E00C97) via eth2 TransID 802c562c uid 01:80:ce:62:9c:0e:70 (RENEW)
+def test_infoblox_dhcp(record_property, setup_wordlist, setup_splunk):
+    host = "vib-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dhcpd[{{ pri }}]: DHCPREQUEST for 10.00.00.62 from 80:00:00:00:0e:70 (EXAMPLE) via eth2 TransID 802c562c uid 01:80:00:00:00:00:70 (RENEW)\n")
+    message = mt.render(mark="<150>", host=host, pid=pid)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netipam host={{ host }} sourcetype=\"infoblox:dhcp\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1