From 1e94d7f72712bf61f81dad7268ad8fcff6170d4b Mon Sep 17 00:00:00 2001
From: rfaircloth-splunk <rfaircloth@splunk.com>
Date: Wed, 4 Mar 2020 16:47:45 -0500
Subject: [PATCH] Enhance cisco IOS

Enhance to address extra field before time stamp, uptime, and no host formats
---
 .../etc/conf.d/conflib/_common/templates.conf |   5 +
 package/etc/conf.d/filters/cisco/ios.conf     | 118 ++++++++++++++++++
 .../conf.d/log_paths/lp-cisco_ios.conf.tmpl   |   2 +-
 package/etc/go_templates/source_network.t     |   8 +-
 pytest.ini                                    |   2 +-
 tests/test_cisco_ios.py                       |  43 ++++++-
 6 files changed, 166 insertions(+), 12 deletions(-)

diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf
index 5746a1c..cc49e3b 100644
--- a/package/etc/conf.d/conflib/_common/templates.conf
+++ b/package/etc/conf.d/conflib/_common/templates.conf
@@ -45,6 +45,11 @@ template t_hdr_msg {
 template t_legacy_hdr_msg {
         template("${LEGACY_MSGHDR}${MESSAGE}");
         };
+
+template t_program_msg {
+        template("${PROGRAM}: ${MESSAGE}");
+        };
+
 # ===============================================================================================
 # Message Header, Structured Data (from RFC5424 parse) and Message; for Juniper
 # ===============================================================================================
diff --git a/package/etc/conf.d/filters/cisco/ios.conf b/package/etc/conf.d/filters/cisco/ios.conf
index 0e4c478..5a0b589 100644
--- a/package/etc/conf.d/filters/cisco/ios.conf
+++ b/package/etc/conf.d/filters/cisco/ios.conf
@@ -4,3 +4,121 @@ filter f_cisco_ios{
     match("cisco_ios", value("fields.sc4s_vendor_product") type(glob));
 };
 
+#6339985: foo: 6340004: *Mar  4 11:45:20: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet
+#30: foo: *Apr 29 13:58:46.000001: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
+#30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
+#29: foo: *Apr 29 13:58:40.411: %SYS-5-CONFIG_I: Configured from console by console
+#30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
+#31: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 started - CLI initiated
+#32: 0.0.0.0: *Apr 29 13:59:12.491: %SYS-5-CONFIG_I: Configured from console by console
+#33: 0.0.0.0: *Apr 29 13:59:26.415: %SYS-5-CONFIG_I: Configured from console by console
+#34: 0.0.0.0: *Apr 29 13:59:56.603: %SYS-5-CONFIG_I: Configured from console by console
+#35: *Apr 29 14:00:16.059: %SYS-5-CONFIG_I: Configured from console by console
+#32: foo: *Apr 29 13:58:46.411: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.
+#foo: *Apr 29 13:58:46.411: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has 
+#started the shutdown procedure.
+#foo: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the 
+#foo: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.
+#foo: 101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.
+#*Mar  1 18:48:50.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36)
+
+# block parser cisco-timestamp-parser-ex(template()) {
+#     channel {
+#         filter {
+#             match('^\*?([A-Za-z]{3} [0-9 ]\d (\d{4} )?\d{2}:\d{2}:\d{2}(\.\d{3})?( (AM|PM))?)' template(`template`) flags(store-matches));
+#         };
+#         parser {
+#             date-parser(format('%b %d %I:%M:%S %p.%f',
+#                                '%b %d %I:%M:%S %p',
+#                                '%b %d %H:%M:%S.%f',
+#                                '%b %d %H:%M:%S',
+#                                '%b %d %Y %H:%M:%S.%f',
+#                                '%b %d %Y %H:%M:%S')
+#                         template("$1"));
+#         };
+#     };
+# };
+
+# block parser cisco-triplet-parser-ex(template() prefix()) {
+#     channel {
+#         if {
+#             parser {
+#                 csv-parser(delimiters(chars('-')) template(`template`)
+#                            columns('`prefix`facility', '`prefix`severity', '`prefix`mnemonic')
+#                            flags(drop-invalid));
+#             };
+#         } else {
+# 	    parser {
+#                 csv-parser(delimiters(chars('-')) template(`template`)
+#                            columns('`prefix`facility', '1', '`prefix`severity', '`prefix`mnemonic')
+#                            flags(drop-invalid));
+#             };
+#             rewrite { set("${`prefix`facility}-$1" value('`prefix`facility')); };
+#         };
+#     };
+# };
+
+
+parser cisco-parser-ex{
+    channel {
+        filter {
+            #message('^<\d*>(?:(?<ciscoseq>\d+)\: )?(?:(?<HOST>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(?<ciscorule>\d+): )?(?:(?<ciscotimereliable>\*)?(?<ciscotime>(?<time>\w\w\w {1,2}\d{1,2} \d\d:\d\d:\d\d)(?<ciscofrac>\.\d{3,6})? ?(?<ciscotz>\w+)?): )?(?:(?<ciscouptime>\d\d:\d\d:\d\d|\d{1,6} \d{1,2}): )?(?<cisomsg>(?<ciscoprogram>%.{2,15}\-\d{1,3}\-[^:]{3,}): (?<ciscodescription>.*))' flags(store-matches));            
+            message('^<\d*>(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\*)?((\w\w\w {1,2}\d{1,2} \d\d:\d\d:\d\d)(\.\d{3,6})? ?(?<tz>\w+)?): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}): )?(%.{2,15}\-\d{1,3}\-[^:]{3,}): (.*)' flags(store-matches));            
+        };        
+        if {
+            parser {
+                date-parser(format('%b %d %I:%M:%S %p.%f',
+                                            '%b %d %I:%M:%S %p',
+                                            '%b %d %H:%M:%S.%f',
+                                            '%b %d %H:%M:%S',
+                                            '%b %d %Y %H:%M:%S.%f',
+                                            '%b %d %Y %H:%M:%S')
+                                    template("$7"));
+            };
+        };
+        rewrite {
+            set(
+                "$4",
+                value("HOST")                
+                condition(match('..' value('4')))
+            );            
+            set(
+                "$12",
+                value("PROGRAM")                
+            );            
+            set(
+                "$13",
+                value("MSG")                
+            );
+        };
+
+    #     parser {
+    #         # split msg and header right before the '%', Cisco messages may
+    #         # have a variable number of ': ' terminated values
+    #         csv-parser(delimiters(chars('') strings(': %'))
+    #                    columns('1', '2', '3') flags(greedy, drop-invalid));
+
+    #         csv-parser(delimiters(chars(':')) template("$2") columns('3'));
+    #         cisco-triplet-parser_ex(template("$3") prefix(`prefix`));
+    #     };
+    #     rewrite {
+    #         set('%$2', value("MSG"));
+
+	#     # drop "<pri>seqno: " if present
+    #         subst("^(<[0-9]+>)?([0-9]+)?(: )?", "", value('1'));
+
+    #     };
+
+    #     if {
+    #         parser { cisco-timestamp-parser_ex(template("$1")); };
+    #     } elif {
+    #         filter { match("^(?'HOST'[^:]+): (.*)" template('$1') flags(store-matches) type(pcre)); };
+    #         parser { cisco-timestamp-parser_ex(template("$2")); };
+    #     } elif {
+    #         filter { match("^(?'HOST'[^:]+)$" template('$1') flags(store-matches) type(pcre)); };
+    #     } else {
+    #         filter { match("^$" template('$1') flags(store-matches) type(pcre)); };
+    #     };
+	# rewrite { set-severity("${`prefix`severity}"); };
+    };
+};
\ No newline at end of file
diff --git a/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl
index 89186e2..218afbc 100644
--- a/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl
+++ b/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl
@@ -27,7 +27,7 @@ log {
     };
     parser { p_add_context_splunk(key("cisco_ios")); };
     parser (compliance_meta_by_source);
-    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); };
 
 {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CISCO_IOS_HEC" "no")) }}
     destination(d_hec);
diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t
index ea83f05..c648dce 100644
--- a/package/etc/go_templates/source_network.t
+++ b/package/etc/go_templates/source_network.t
@@ -83,7 +83,7 @@ source s_{{ .port_id }} {
             };
         rewrite(set_rfc5424_noversion);
 {{ else if eq .parser "cisco_parser" }}
-        parser {cisco-parser()};
+        parser (cisco-parser-ex);
         rewrite(set_cisco_ios);
 {{ else if eq .parser "cisco_meraki_parser" }}
         parser (p_cisco_meraki);
@@ -110,6 +110,9 @@ source s_{{ .port_id }} {
         } elif {
             parser (p_cisco_meraki);
             rewrite(set_rfc5424_epochtime);
+        } elif {
+            parser(cisco-parser-ex);
+            rewrite(set_cisco_ios);
         } elif {
             filter(f_cisco_ucm_message);
             parser (p_cisco_ucm_date);
@@ -127,9 +130,6 @@ source s_{{ .port_id }} {
                     syslog-parser(flags(syslog-protocol));
                 };
             rewrite(set_rfc5424_noversion);
-        } elif {
-            parser {cisco-parser()};
-            rewrite(set_cisco_ios);
         } else {
             parser {
                 syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}));
diff --git a/pytest.ini b/pytest.ini
index 4fce685..d33097b 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -5,7 +5,7 @@ addopts =
     --splunk_type=docker
     --keepalive
     --splunk_hec_token=1ec3c8ac-74b3-46f2-ba44-a7c96b6ab236
-    -n=6
+    -n=1
 filterwarnings =
     ignore::DeprecationWarning
 junit_family=xunit1
\ No newline at end of file
diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py
index 587559c..8d25926 100644
--- a/tests/test_cisco_ios.py
+++ b/tests/test_cisco_ios.py
@@ -8,21 +8,52 @@
 
 from .sendmessage import *
 from .splunkutils import *
-
+import pytest
 env = Environment(extensions=['jinja2_time.TimeExtension'])
 
 
-# <190>30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
-def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s):
+#30: foo: 6340004: *Mar  4 11:45:20: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet
+#30: foo: *Apr 29 13:58:46.000001: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
+#30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
+#foo: *Apr 29 13:58:46.411: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.
+#30: foo: 6340004: Mar  4 11:45:20: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet
+#30: foo: Apr 29 13:58:46.000001: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
+#30: foo: Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
+#foo: Apr 29 13:58:46.411: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.
+#foo: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the 
+#00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the 
+#foo: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.
+#101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.
+#*Mar  1 18:48:50.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36)
+
+testdata = [
+        "{{ mark }}{{ seq }}: {{ host }}: 6340004: *{% now 'local', '%b %d %H:%M:%S' %}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet",
+        "{{ mark }}{{ seq }}: {{ host }}: *{% now 'local', '%b %d %H:%M:%S' %}.000001: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated",
+        "{{ mark }}{{ seq }}: {{ host }}: *{% now 'local', '%b %d %H:%M:%S' %}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated",
+        "{{ mark }}{{ host }}: *{% now 'local', '%b %d %H:%M:%S' %}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.",
+        "{{ mark }}{{ seq }}: {{ host }}: 6340004: {% now 'local', '%b %d %H:%M:%S' %}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet",
+        "{{ mark }}{{ seq }}: {{ host }}: {% now 'local', '%b %d %H:%M:%S' %}.000001: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated",
+        "{{ mark }}{{ seq }}: {{ host }}: {% now 'local', '%b %d %H:%M:%S' %}.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated",
+        "{{ mark }}{{ host }}: {% now 'local', '%b %d %H:%M:%S' %}.411: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.",
+        "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ",
+        "{{ mark }}00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}",
+        "{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ",
+        "{{ mark }}{{ seq }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}",
+        "{{ mark }}{{ seq }}: {{ host }}: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.",
+        "{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}",
+        "{{ mark }}*{% now 'local', '%b %d %H:%M:%S' %}.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}"
+    ]
+
+@pytest.mark.parametrize("event", testdata)
+def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s,event):
     host = get_host_key
 
-    mt = env.from_string(
-        "{{ mark }}{{ seq }}: {{ host }}: *{% now 'local', '%b %d %H:%M:%S' %}.100: CET: %SEC-6-IPACCESSLOGP: list 110 denied tcp 54.122.123.124(8932) -> 10.1.0.1(22), 1 packet\n")
+    mt = env.from_string(event + "\n")
     message = mt.render(mark="<166>", seq=20, host=host)
 
     sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
 
-    st = env.from_string("search earliest=-1m@m latest=+1m@m index=netops host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2")
+    st = env.from_string("search earliest=-1m@m latest=+1m@m index=netops sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\") | head 2")
     search = st.render(host=host)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)