Skip to content

Commit

Permalink
Enhance cisco IOS
Browse files Browse the repository at this point in the history 
Enhance to address extra field before time stamp, uptime, and no host formats
  • Loading branch information
rfaircloth-splunk committed Mar 4, 2020
1 parent 649250f commit 1e94d7f
Show file tree
Hide file tree
Showing 6 changed files with 166 additions and 12 deletions.
5 changes: 5 additions & 0 deletions package/etc/conf.d/conflib/_common/templates.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ template t_hdr_msg {
template t_legacy_hdr_msg {
template("${LEGACY_MSGHDR}${MESSAGE}");
};

template t_program_msg {
template("${PROGRAM}: ${MESSAGE}");
};

# ===============================================================================================
# Message Header, Structured Data (from RFC5424 parse) and Message; for Juniper
# ===============================================================================================
Expand Down
118 changes: 118 additions & 0 deletions package/etc/conf.d/filters/cisco/ios.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,121 @@ filter f_cisco_ios{
match("cisco_ios", value("fields.sc4s_vendor_product") type(glob));
};

#6339985: foo: 6340004: *Mar 4 11:45:20: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet
#30: foo: *Apr 29 13:58:46.000001: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
#30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
#29: foo: *Apr 29 13:58:40.411: %SYS-5-CONFIG_I: Configured from console by console
#30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
#31: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 started - CLI initiated
#32: 0.0.0.0: *Apr 29 13:59:12.491: %SYS-5-CONFIG_I: Configured from console by console
#33: 0.0.0.0: *Apr 29 13:59:26.415: %SYS-5-CONFIG_I: Configured from console by console
#34: 0.0.0.0: *Apr 29 13:59:56.603: %SYS-5-CONFIG_I: Configured from console by console
#35: *Apr 29 14:00:16.059: %SYS-5-CONFIG_I: Configured from console by console
#32: foo: *Apr 29 13:58:46.411: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.
#foo: *Apr 29 13:58:46.411: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has
#started the shutdown procedure.
#foo: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the
#foo: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.
#foo: 101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.
#*Mar 1 18:48:50.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36)

# block parser cisco-timestamp-parser-ex(template()) {
# channel {
# filter {
# match('^\*?([A-Za-z]{3} [0-9 ]\d (\d{4} )?\d{2}:\d{2}:\d{2}(\.\d{3})?( (AM|PM))?)' template(`template`) flags(store-matches));
# };
# parser {
# date-parser(format('%b %d %I:%M:%S %p.%f',
# '%b %d %I:%M:%S %p',
# '%b %d %H:%M:%S.%f',
# '%b %d %H:%M:%S',
# '%b %d %Y %H:%M:%S.%f',
# '%b %d %Y %H:%M:%S')
# template("$1"));
# };
# };
# };

# block parser cisco-triplet-parser-ex(template() prefix()) {
# channel {
# if {
# parser {
# csv-parser(delimiters(chars('-')) template(`template`)
# columns('`prefix`facility', '`prefix`severity', '`prefix`mnemonic')
# flags(drop-invalid));
# };
# } else {
# parser {
# csv-parser(delimiters(chars('-')) template(`template`)
# columns('`prefix`facility', '1', '`prefix`severity', '`prefix`mnemonic')
# flags(drop-invalid));
# };
# rewrite { set("${`prefix`facility}-$1" value('`prefix`facility')); };
# };
# };
# };


parser cisco-parser-ex{
channel {
filter {
#message('^<\d*>(?:(?<ciscoseq>\d+)\: )?(?:(?<HOST>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(?<ciscorule>\d+): )?(?:(?<ciscotimereliable>\*)?(?<ciscotime>(?<time>\w\w\w {1,2}\d{1,2} \d\d:\d\d:\d\d)(?<ciscofrac>\.\d{3,6})? ?(?<ciscotz>\w+)?): )?(?:(?<ciscouptime>\d\d:\d\d:\d\d|\d{1,6} \d{1,2}): )?(?<cisomsg>(?<ciscoprogram>%.{2,15}\-\d{1,3}\-[^:]{3,}): (?<ciscodescription>.*))' flags(store-matches));
message('^<\d*>(?:(\d+)\: )?(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]): )?(?:(\d+): )?(?:(\*)?((\w\w\w {1,2}\d{1,2} \d\d:\d\d:\d\d)(\.\d{3,6})? ?(?<tz>\w+)?): )?(?:(\d\d:\d\d:\d\d|\d{1,6} \d{1,2}): )?(%.{2,15}\-\d{1,3}\-[^:]{3,}): (.*)' flags(store-matches));
};
if {
parser {
date-parser(format('%b %d %I:%M:%S %p.%f',
'%b %d %I:%M:%S %p',
'%b %d %H:%M:%S.%f',
'%b %d %H:%M:%S',
'%b %d %Y %H:%M:%S.%f',
'%b %d %Y %H:%M:%S')
template("$7"));
};
};
rewrite {
set(
"$4",
value("HOST")
condition(match('..' value('4')))
);
set(
"$12",
value("PROGRAM")
);
set(
"$13",
value("MSG")
);
};

# parser {
# # split msg and header right before the '%', Cisco messages may
# # have a variable number of ': ' terminated values
# csv-parser(delimiters(chars('') strings(': %'))
# columns('1', '2', '3') flags(greedy, drop-invalid));

# csv-parser(delimiters(chars(':')) template("$2") columns('3'));
# cisco-triplet-parser_ex(template("$3") prefix(`prefix`));
# };
# rewrite {
# set('%$2', value("MSG"));

# # drop "<pri>seqno: " if present
# subst("^(<[0-9]+>)?([0-9]+)?(: )?", "", value('1'));

# };

# if {
# parser { cisco-timestamp-parser_ex(template("$1")); };
# } elif {
# filter { match("^(?'HOST'[^:]+): (.*)" template('$1') flags(store-matches) type(pcre)); };
# parser { cisco-timestamp-parser_ex(template("$2")); };
# } elif {
# filter { match("^(?'HOST'[^:]+)$" template('$1') flags(store-matches) type(pcre)); };
# } else {
# filter { match("^$" template('$1') flags(store-matches) type(pcre)); };
# };
# rewrite { set-severity("${`prefix`severity}"); };
};
};
2 changes: 1 addition & 1 deletion package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ log {
};
parser { p_add_context_splunk(key("cisco_ios")); };
parser (compliance_meta_by_source);
rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
rewrite { set("$(template ${.splunk.sc4s_template} $(template t_program_msg))" value("MSG")); };

{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CISCO_IOS_HEC" "no")) }}
destination(d_hec);
Expand Down
8 changes: 4 additions & 4 deletions package/etc/go_templates/source_network.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ source s_{{ .port_id }} {
};
rewrite(set_rfc5424_noversion);
{{ else if eq .parser "cisco_parser" }}
parser {cisco-parser()};
parser (cisco-parser-ex);
rewrite(set_cisco_ios);
{{ else if eq .parser "cisco_meraki_parser" }}
parser (p_cisco_meraki);
Expand All @@ -110,6 +110,9 @@ source s_{{ .port_id }} {
} elif {
parser (p_cisco_meraki);
rewrite(set_rfc5424_epochtime);
} elif {
parser(cisco-parser-ex);
rewrite(set_cisco_ios);
} elif {
filter(f_cisco_ucm_message);
parser (p_cisco_ucm_date);
Expand All @@ -127,9 +130,6 @@ source s_{{ .port_id }} {
syslog-parser(flags(syslog-protocol));
};
rewrite(set_rfc5424_noversion);
} elif {
parser {cisco-parser()};
rewrite(set_cisco_ios);
} else {
parser {
syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}}));
Expand Down
2 changes: 1 addition & 1 deletion pytest.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ addopts =
--splunk_type=docker
--keepalive
--splunk_hec_token=1ec3c8ac-74b3-46f2-ba44-a7c96b6ab236
-n=6
-n=1
filterwarnings =
ignore::DeprecationWarning
junit_family=xunit1
43 changes: 37 additions & 6 deletions tests/test_cisco_ios.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,52 @@

from .sendmessage import *
from .splunkutils import *

import pytest
env = Environment(extensions=['jinja2_time.TimeExtension'])


# <190>30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s):
#30: foo: 6340004: *Mar 4 11:45:20: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet
#30: foo: *Apr 29 13:58:46.000001: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
#30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
#foo: *Apr 29 13:58:46.411: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.
#30: foo: 6340004: Mar 4 11:45:20: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet
#30: foo: Apr 29 13:58:46.000001: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
#30: foo: Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated
#foo: Apr 29 13:58:46.411: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.
#foo: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the
#00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the
#foo: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.
#101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.
#*Mar 1 18:48:50.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36)

testdata = [
"{{ mark }}{{ seq }}: {{ host }}: 6340004: *{% now 'local', '%b %d %H:%M:%S' %}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet",
"{{ mark }}{{ seq }}: {{ host }}: *{% now 'local', '%b %d %H:%M:%S' %}.000001: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated",
"{{ mark }}{{ seq }}: {{ host }}: *{% now 'local', '%b %d %H:%M:%S' %}: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated",
"{{ mark }}{{ host }}: *{% now 'local', '%b %d %H:%M:%S' %}: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.",
"{{ mark }}{{ seq }}: {{ host }}: 6340004: {% now 'local', '%b %d %H:%M:%S' %}: %SEC-6-IPACCESSLOGP: list INET-BLOCK permitted tcp 192.168.20.252(55244) -> 10.54.3.178(44818), 1 packet",
"{{ mark }}{{ seq }}: {{ host }}: {% now 'local', '%b %d %H:%M:%S' %}.000001: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated",
"{{ mark }}{{ seq }}: {{ host }}: {% now 'local', '%b %d %H:%M:%S' %}.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated",
"{{ mark }}{{ host }}: {% now 'local', '%b %d %H:%M:%S' %}.411: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.",
"{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ",
"{{ mark }}00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}",
"{{ mark }}{{ host }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the ",
"{{ mark }}{{ seq }}: 00:01:01: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the {{ host }}",
"{{ mark }}{{ seq }}: {{ host }}: 1 2: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure.",
"{{ mark }}101 21: %SYSMGR-STANDBY-3-SHUTDOWN_START: The System Manager has started the shutdown procedure.shutdown procedure. {{ host }}",
"{{ mark }}*{% now 'local', '%b %d %H:%M:%S' %}.483 UTC: %SYS-5-CONFIG_I: Configured from console by vty2 (10.34.195.36) {{ host }}"
]

@pytest.mark.parametrize("event", testdata)
def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s,event):
host = get_host_key

mt = env.from_string(
"{{ mark }}{{ seq }}: {{ host }}: *{% now 'local', '%b %d %H:%M:%S' %}.100: CET: %SEC-6-IPACCESSLOGP: list 110 denied tcp 54.122.123.124(8932) -> 10.1.0.1(22), 1 packet\n")
mt = env.from_string(event + "\n")
message = mt.render(mark="<166>", seq=20, host=host)

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string("search earliest=-1m@m latest=+1m@m index=netops host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2")
st = env.from_string("search earliest=-1m@m latest=+1m@m index=netops sourcetype=\"cisco:ios\" (host=\"{{ host }}\" OR \"{{ host }}\") | head 2")
search = st.render(host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand Down

0 comments on commit 1e94d7f

Please sign in to comment.