[Filteradd] syslog-ng loggen filter (#576)

* test added for loggen_data * loggen filter added * documentation for loggen filter added * Update product name to loggen * Update product name to loggen Co-authored-by: mbonsack <mbonsack@splunk.com>
CSVD · Jul 23, 2020 · b9fc6a7 · b9fc6a7
1 parent 6650b8c
commit b9fc6a7
Show file tree

Hide file tree

Showing 5 changed files with 155 additions and 0 deletions.
diff --git a/docs/sources/Loggen/index.md b/docs/sources/Loggen/index.md
@@ -0,0 +1,42 @@
+# Vendor - Syslog-ng
+
+## Product -  syslog-ng loggen
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Product Manual | https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.16/administration-guide/87   |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| syslogng:loggen | None                                                                                                    |
+
+
+### Index Configuration
+
+| key            | index          | notes          |
+|----------------|----------------|----------------|
+| syslogng_loggen | main          | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_SYSLOGNG_LOGGEN_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_SYSLOGNG_LOGGEN_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_SYSLOGNG_LOGGEN | no | Enable archive to disk for this specific source |
+| SC4S_DEST_SYSLOGNG_LOGGEN_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active device will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=main sourcetype="syslogng:loggen"| stats count by host
+```
diff --git a/package/etc/conf.d/filters/loggen/syslogng_loggen.conf b/package/etc/conf.d/filters/loggen/syslogng_loggen.conf
@@ -0,0 +1,3 @@
+filter f_syslogng_loggen {
+  program("prg00000");
+};
diff --git a/package/etc/conf.d/log_paths/lp-syslogng_loggen.conf.tmpl b/package/etc/conf.d/log_paths/lp-syslogng_loggen.conf.tmpl
@@ -0,0 +1,75 @@
+# SYSLOGNG_LOGGEN SYSLOG
+{{- /* The following provides a unique port source configuration if env var(s) are set */}}
+{{- $context := dict "port_id" "SYSLOGNG_LOGGEN" "parser" "common" }}
+{{- tmpl.Exec "t/source_network.t" $context }}
+
+log {
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_SYSLOGNG_LOGGEN_TCP_PORT")) (getenv  (print "SC4S_LISTEN_SYSLOGNG_LOGGEN_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_SYSLOGNG_LOGGEN_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for SYSLOGNG_LOGGEN traffic
+            source (s_SYSLOGNG_LOGGEN);
+            flags (final);
+	    };
+{{- end}}
+        channel {
+        # Listen on the default port (typically 514) for SYSLOGNG_LOGGEN traffic
+            source (s_DEFAULT);
+            filter(f_is_rfc5424_strict);
+            filter(f_syslogng_loggen);
+            flags(final);
+        };
+    };
+
+# Set a default sourcetype and index, as well as an appropriate value for the field
+# "sc4s_vendor_product".  This field is sent as an indexed field to Splunk,
+# and is useful for downstream analysis.
+
+    rewrite {
+        set("syslogng_loggen", value("fields.sc4s_vendor_product"));
+        r_set_splunk_dest_default(sourcetype("syslogng:loggen"));
+        subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+        set("${PROGRAM}", value(".PROGRAM"));
+        subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+    };
+
+    parser { p_add_context_splunk(key("syslogng_loggen")); };
+    parser (compliance_meta_by_source);
+    if {
+        filter(f_is_rfc5424_strict);
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); };
+    } else {
+        rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); };
+
+    };
+
+{{- /* Check environment variables (and defaults if unset) for sending to the HEC */}}
+{{- /* destination. When more destination options are offered in SC4S, this is where */}}
+{{- /* output to them will be configured */}}
+
+{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_SYSLOGNG_LOGGEN_HEC" "no")) }}
+    destination(d_hec);
+{{- end}}
+
+{{- /* Check environment variables (and defaults if unset) for sending to the local EWMM-format */}}
+{{- /* disk archive */}}
+
+{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_SYSLOGNG_LOGGEN" "no")) }}
+    destination(d_archive);
+{{- end}}
+
+{{- /* Check environment variables for sending to a global list of alternate destinations */}}
+
+{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }}
+    {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+{{- end }}
+
+{{- /* Check environment variables for sending to a list of alternate destinations only for this specific source */}}
+
+{{- if (print (getenv "SC4S_DEST_SYSLOGNG_LOGGEN_ALTERNATES")) }}
+    {{ getenv "SC4S_DEST_SYSLOGNG_LOGGEN_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n    destination(" }});
+{{- end }}
+
+# All passes through any matching log path will be final
+    flags(flow-control,final);
+};
diff --git a/package/etc/context_templates/splunk_metadata.csv.example b/package/etc/context_templates/splunk_metadata.csv.example
@@ -79,6 +79,7 @@ sc4s_fallback,index,main
 sc4s_metrics,index,em_metrics
 symantec_ep,index,epav
 symantec_brightmail,index,email
+syslogng_loggen,index,main
 ubiquiti_unifi,index,netops
 ubiquiti_unifi_fw,index,netfw
 ubiquiti_unifi_link,index,netops

diff --git a/tests/test_loggen.py b/tests/test_loggen.py
@@ -0,0 +1,34 @@
+import random
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+from .timeutils import *
+import logging
+logger = logging.getLogger(__name__)
+env = Environment()
+
+
+# <38>1 2020-07-21T21:05:56+02:00 localhost prg00000 1234 - - seq: 0000000000, thread: 0000, runid: 1595365556, stamp: 2020-07-21T21:05:56 PADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPAD
+def test_loggen(record_property, setup_wordlist, setup_splunk, setup_sc4s):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    dt = datetime.datetime.now()
+    iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
+
+    epoch = epoch[:-3]
+    mt = env.from_string("<38>1 {{ iso }} {{ host }} prg00000 1234 - - seq: 0000000000, thread: 0000, runid: 1595365556, stamp: {{iso}} PADDPADDPADDPADDPADDP\n")
+    message = mt.render(iso=iso, host=host)
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+    st = env.from_string("search _time={{ epoch }} index=main host=\"{{ host }}\"  sourcetype=\"syslogng:loggen\"")
+    search = st.render(epoch=epoch, host=host)
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+