From 17e972bdb344b237f705393e540cf9b3949d64fa Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 13 Dec 2019 20:02:34 -0500 Subject: [PATCH 1/3] Support InfoBlox DNS --- docs/sources/InfoBlox/index.md | 54 +++++++++ .../etc/conf.d/filters/infoblox/syslog.conf | 4 + .../etc/conf.d/filters/juniper/legacy.conf | 2 +- .../log_paths/p_rfc3164-infoblox.conf.tmpl | 104 ++++++++++++++++++ .../log_paths/p_za_nix_syslog.conf.tmpl | 2 +- .../etc/context_templates/splunk_index.csv | 3 + .../vendor_product_by_source.conf | 4 + .../vendor_product_by_source.csv | 1 + .../apps/SA-syslog-ng/default/indexes.conf | 10 ++ tests/test_infoblox.py | 58 ++++++++++ 10 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 docs/sources/InfoBlox/index.md create mode 100644 package/etc/conf.d/filters/infoblox/syslog.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl create mode 100644 tests/test_infoblox.py diff --git a/docs/sources/InfoBlox/index.md b/docs/sources/InfoBlox/index.md new file mode 100644 index 0000000..5e24f28 --- /dev/null +++ b/docs/sources/InfoBlox/index.md @@ -0,0 +1,54 @@ +# Vendor - Infoblox + +## Product - NIOS + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2934/ | +| Product Manual | http://dloads.infoblox.com/direct/appliance//NIOS/NIOS_AdminGuide_6.10.pdf | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| infoblox:dns | None | +| infoblox:dhcp | None | +| infoblox:threat | None | +| nix:syslog | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| infoblox_dns | infoblox:dns | netdns | none | +| infoblox_dhcp | infoblox:dhcp | netipam | none | +| infoblox_threat | infoblox:threat | netids | none | +| nix_syslog | nix:syslog | osnix | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the admin manual for specific details of configuration + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_INFOBLOX_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_INFOBLOX_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_INFOBLOX | no | Enable archive to disk for this specific source | +| SC4S_DEST_INFOBLOX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active device will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=infoblox:*| stats count by host +``` diff --git a/package/etc/conf.d/filters/infoblox/syslog.conf b/package/etc/conf.d/filters/infoblox/syslog.conf new file mode 100644 index 0000000..f131781 --- /dev/null +++ b/package/etc/conf.d/filters/infoblox/syslog.conf @@ -0,0 +1,4 @@ +filter f_infoblox { + match("^infoblox", value("fields.sc4s_vendor_product")); + +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/juniper/legacy.conf b/package/etc/conf.d/filters/juniper/legacy.conf index 27b3f52..50383ab 100644 --- a/package/etc/conf.d/filters/juniper/legacy.conf +++ b/package/etc/conf.d/filters/juniper/legacy.conf @@ -14,7 +14,7 @@ filter f_juniper_netscreen { }; - filter f_juniper_idp { +filter f_juniper_idp { match("juniper_idp", value("fields.sc4s_vendor_product") type(glob)) or match('^\[syslog@juniper' value("SDATA")) }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl new file mode 100644 index 0000000..14bb05a --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl @@ -0,0 +1,104 @@ +# Juniper IDP +{{ $context := dict "port_id" "INFOBLOX" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} + +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_is_rfc3164); + filter(f_infoblox); +{{- end}} +{{- if eq (.) "no"}} + source (s_INFOBLOX); +{{- end}} + + + if { + filter{program("named")}; + rewrite { + set("infoblox_dns", value("fields.sc4s_vendor_product")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("infoblox:dns"), index("netdns"), source("program:${.PROGRAM}")) + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + }; + parser { + p_add_context_splunk(key("infoblox_dns")); + }; + } elif { + filter{program("dhcpd")}; + rewrite { + set("infoblox_dhcp", value("fields.sc4s_vendor_product")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("infoblox:dhcp"), index("netipam"), source("program:${.PROGRAM}")) + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + }; + parser { + p_add_context_splunk(key("infoblox_dhcp")); + }; + } elif { + filter{program("threat-protect-log")}; + rewrite { + set("infoblox_dns", value("fields.sc4s_vendor_product")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("infoblox:threat"), index("netids"), source("program:${.PROGRAM}")) + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + }; + parser { + p_add_context_splunk(key("infoblox_threat")); + }; + } else { + rewrite { + set("nix_syslog", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + + rewrite { + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") ) + set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + + }; + + parser { p_add_context_splunk(key("nix_syslog")); }; + }; + + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + unset(value("PID")); + groupunset(values(".kv.*")); + }; + + +{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_INFOBLOX_HEC" "no") | conv.ToBool) }} + destination(d_hec); +{{- end}} + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_INFOBLOX") }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; +{{- end}} + +{{- if or (or (getenv (print "SC4S_LISTEN_INFOBLOX_TCP_PORT")) (getenv (print "SC4S_LISTEN_INFOBLOX_UDP_PORT"))) (getenv (print "SC4S_LISTEN_INFOBLOX_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for INFOBLOX traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for INFOBLOX traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl index 9e2547e..805ceff 100644 --- a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl @@ -20,7 +20,7 @@ log { }; rewrite { - r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"), source("program:${.PROGRAM}") ) + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") ) }; diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index 12927fb..6c19bc3 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -22,6 +22,9 @@ #fortinet_fortios_log,index,netops #fortinet_fortios_traffic,index,netfw #fortinet_fortios_utm,index,netids +#infoblox_dns,index,netdns +#infoblox_dhcp,index,netipam +#infoblox_threat,index,netids #juniper_idp,index,netids #juniper_structured,index,netops #juniper_idp_structured,index,netids diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 4c8127f..911117d 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -28,6 +28,10 @@ filter f_cisco_nx_os { host("csconx-*" type(glob)) or netmask(169.254.7.0/24) }; +filter f_infoblox { + host("vib-*" type(glob)) or + netmask(169.254.7.0/24) +}; filter f_proofpoint_pps_filter { host("pps-*" type(glob)) or netmask(169.254.8.0/24) diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 975d558..510cc19 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -1,5 +1,6 @@ f_test_test,sc4s_vendor_product,"test_test" f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" +f_infoblox,sc4s_vendor_product,"infoblox" f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" f_juniper_idp,sc4s_vendor_product,"juniper_idp" diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf index b2ecfa9..1a37448 100644 --- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -56,6 +56,11 @@ homePath = $SPLUNK_DB/netdlp/db coldPath = $SPLUNK_DB/netdlp/colddb thawedPath = $SPLUNK_DB/netdlp/thaweddb +[netdns] +homePath = $SPLUNK_DB/netdns/db +coldPath = $SPLUNK_DB/netdns/colddb +thawedPath = $SPLUNK_DB/netdns/thaweddb + [netfw] homePath = $SPLUNK_DB/netfw/db coldPath = $SPLUNK_DB/netfw/colddb @@ -66,6 +71,11 @@ homePath = $SPLUNK_DB/netids/db coldPath = $SPLUNK_DB/netids/colddb thawedPath = $SPLUNK_DB/netids/thaweddb +[netipam] +homePath = $SPLUNK_DB/netipam/db +coldPath = $SPLUNK_DB/netipam/colddb +thawedPath = $SPLUNK_DB/netipam/thaweddb + [netops] homePath = $SPLUNK_DB/netops/db coldPath = $SPLUNK_DB/netops/colddb diff --git a/tests/test_infoblox.py b/tests/test_infoblox.py new file mode 100644 index 0000000..f7da9c7 --- /dev/null +++ b/tests/test_infoblox.py @@ -0,0 +1,58 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import datetime +import random +import pytz + +from jinja2 import Environment, environment + +from .sendmessage import * +from .splunkutils import * +import random + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<142>Oct 25 13:08:00 161.231.218.156 named[6597]: FORMERR resolving 'www.google.com/AAAA/IN': 209.2.230.6#53 +def test_infoblox_dns(record_property, setup_wordlist, setup_splunk): + host = "vib-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} named[{{ pid }}]: FORMERR resolving 'www.google.com/AAAA/IN': 209.2.230.6#53\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=netdns host={{ host }} sourcetype=\"infoblox:dns\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<150>Oct 24 21:09:00 162.101.157.246 dhcpd[28922]: DHCPREQUEST for 10.130.151.62 from 80:ce:62:9c:0e:70 (DTCCE0826E00C97) via eth2 TransID 802c562c uid 01:80:ce:62:9c:0e:70 (RENEW) +def test_infoblox_dhcp(record_property, setup_wordlist, setup_splunk): + host = "vib-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dhcpd[{{ pri }}]: DHCPREQUEST for 10.00.00.62 from 80:00:00:00:0e:70 (EXAMPLE) via eth2 TransID 802c562c uid 01:80:00:00:00:00:70 (RENEW)\n") + message = mt.render(mark="<150>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=netipam host={{ host }} sourcetype=\"infoblox:dhcp\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file From fb9c900950d408da42686af8c58bc8df41605bc6 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Fri, 13 Dec 2019 20:33:01 -0500 Subject: [PATCH 2/3] Update index.md --- docs/sources/InfoBlox/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/InfoBlox/index.md b/docs/sources/InfoBlox/index.md index 5e24f28..92cea8f 100644 --- a/docs/sources/InfoBlox/index.md +++ b/docs/sources/InfoBlox/index.md @@ -28,7 +28,7 @@ ### Filter type -MSG Parse: This filter parses message content +Must be identified by host or ip assignment. Update the filter `f_infoblox` or configure a dedicated port as required ### Setup and Configuration From 007965854464f9449722f9de5fbd655154289daf Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Sat, 14 Dec 2019 07:19:28 -0500 Subject: [PATCH 3/3] change all netmasks to place holders --- .../vendor_product_by_source.conf | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 911117d..ac87c2e 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -1,54 +1,54 @@ @version: 3.24 filter f_test_test { - host("testvp-*" type(glob)) or - netmask(169.254.1.0/24) + host("testvp-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_juniper_idp { - host("jnpidp-*" type(glob)) or - netmask(169.254.2.0/24) + host("jnpidp-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_juniper_netscreen { - host("jnpns-*" type(glob)) or - netmask(169.254.3.0/24) + host("jnpns-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_juniper_nsm { - host("jnpnsm-*" type(glob)) or - netmask(169.254.4.0/24) + host("jnpnsm-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_juniper_nsm_idp { - host("jnpnsmidp-*" type(glob)) or - netmask(169.254.5.0/24) + host("jnpnsmidp-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_cisco_meraki { - host("testcm-*" type(glob)) or - netmask(169.254.6.0/24) + host("testcm-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_cisco_nx_os { - host("csconx-*" type(glob)) or - netmask(169.254.7.0/24) + host("csconx-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_infoblox { - host("vib-*" type(glob)) or - netmask(169.254.7.0/24) + host("vib-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_proofpoint_pps_filter { - host("pps-*" type(glob)) or - netmask(169.254.8.0/24) + host("pps-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_proofpoint_pps_sendmail { - host("pps-*" type(glob)) or - netmask(169.254.9.0/24) + host("pps-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_ubiquiti_unifi_fw { - host("usg-*" type(glob)) or - netmask(169.254.10.0/24) + host("usg-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_tzfixhst { - host("tzfhst-*" type(glob)) or - netmask(169.254.11.0/24) + host("tzfhst-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_tzfixny { - host("tzfny-*" type(glob)) or - netmask(169.254.12.0/24) + host("tzfny-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) };