From f4d696e7fe01b692e90f26642d789205493ffa3f Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Wed, 2 Oct 2019 08:08:59 -0700 Subject: [PATCH] Feature/pluginwork (#107) * Add a new plugin test case * Add a commented example plugin --- .gitignore | 5 +- .../etc/conf.d/local/destinations/README.md | 1 + package/etc/conf.d/local/filters/README.md | 1 + package/etc/conf.d/local/filters/example.conf | 4 + package/etc/conf.d/local/log_paths/README.md | 1 + .../conf.d/local/log_paths/example.conf.tmpl | 76 +++++++++++++++++++ package/etc/conf.d/local/sources/README.md | 1 + package/etc/context-local/splunk_index.csv | 1 + package/etc/syslog-ng.conf | 3 +- tests/test_plugin_example.py | 32 ++++++++ 10 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 package/etc/conf.d/local/destinations/README.md create mode 100644 package/etc/conf.d/local/filters/README.md create mode 100644 package/etc/conf.d/local/filters/example.conf create mode 100644 package/etc/conf.d/local/log_paths/README.md create mode 100644 package/etc/conf.d/local/log_paths/example.conf.tmpl create mode 100644 package/etc/conf.d/local/sources/README.md create mode 100644 tests/test_plugin_example.py diff --git a/.gitignore b/.gitignore index 59343f4..3ac4e78 100644 --- a/.gitignore +++ b/.gitignore @@ -379,4 +379,7 @@ fabric.properties !/package/scripts/ .ecs /test-results/ -/.idea/ \ No newline at end of file +/.idea/ + +tests/test_plugin_*.py +package/etc/conf.d/local/ \ No newline at end of file diff --git a/package/etc/conf.d/local/destinations/README.md b/package/etc/conf.d/local/destinations/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/destinations/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/filters/README.md b/package/etc/conf.d/local/filters/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/filters/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/filters/example.conf b/package/etc/conf.d/local/filters/example.conf new file mode 100644 index 0000000..047fdc8 --- /dev/null +++ b/package/etc/conf.d/local/filters/example.conf @@ -0,0 +1,4 @@ + +filter f_local_example { + program(sc4splugin); +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/log_paths/README.md b/package/etc/conf.d/local/log_paths/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/log_paths/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/log_paths/example.conf.tmpl b/package/etc/conf.d/local/log_paths/example.conf.tmpl new file mode 100644 index 0000000..a8ac264 --- /dev/null +++ b/package/etc/conf.d/local/log_paths/example.conf.tmpl @@ -0,0 +1,76 @@ +# LOCAL_EXAMPLE + +# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique +# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the +# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ + +# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), +# the template generator will build a custom source based on the value of one or more of the set variables. + +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} + +# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name +# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". +# "parser" can be customized on dedicated ports only +# "common" uses the same parser sequence as the default ports and is the most commonly used + +{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} + +# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path +# The ports used are based on the values of one or more of the environment variables set above. + +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { + +# The first time this template is used the log_path will be linked to the default port + +{{- if eq (.) "yes"}} + source(s_default-ports); + +# Filters should be updated to use the simplest and most effecient logic possible to discard +# the message from this path + + filter(f_is_rfc3164); + filter(f_local_example); +{{- end}} +{{- if eq (.) "no"}} + +# In the second pass through the template a link to the dedicated port is used. This +# normally does not require additional filters + +source (s_dedicated_port_LOCAL_EXAMPLE); +{{- end}} + +#Set a default sourcetype and index + + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + +#using the key "local_example" find any cutomized index,source or sourcetype meta values + + parser {p_add_context_splunk(key("local_example")); }; + +# Any additional logic needed to process the event before sending to Splunk goes here + +# Send it to Splunk + + destination(d_hec); #--HEC-- + +# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will +# forward events to another system + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic + + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic + +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/local/sources/README.md b/package/etc/conf.d/local/sources/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/sources/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv index 766067b..711baf0 100644 --- a/package/etc/context-local/splunk_index.csv +++ b/package/etc/context-local/splunk_index.csv @@ -6,6 +6,7 @@ #cisco_asa,index,netfw #cisco_ios,index,netops #cisco_nx_os,index,netops +#local_example,index,main #fortinet_fortios_event,index,netops #fortinet_fortios_log,index,netops #fortinet_fortios_traffic,index,netfw diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 349a9be..e4a0a78 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -63,6 +63,7 @@ options { @include "conf.d/log_paths/*.conf" @include "conf.d/local/filters/*.conf" -@include "conf.d/local/log_paths/*.conf" +@include "conf.d/local/filters/*/*.conf" @include "conf.d/local/sources/*.conf" @include "conf.d/local/destinations/*.conf" +@include "conf.d/local/log_paths/*.conf" diff --git a/tests/test_plugin_example.py b/tests/test_plugin_example.py new file mode 100644 index 0000000..852dec5 --- /dev/null +++ b/tests/test_plugin_example.py @@ -0,0 +1,32 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +def test_plugin_local_example(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} sc4splugin[0]: test\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"sc4s:local_example\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file