From b71a27dd73cbfecc57299cbfad8ff1d8163bc62e Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Fri, 7 Aug 2020 12:45:09 -0400 Subject: [PATCH] [fix] Incorrect host resolution (#610) * [fix] Incorrect host resolution When the log source includes an IP as host resolve using connection IP rather than field IP. When the host resolves to a single name rather than FQDN do not set the host value as this can't be trusted * Update test_common.py * Update fix_dns.conf --- .../conf.d/conflib/_common/syslog_format.conf | 4 ++ .../etc/conf.d/conflib/_splunk/fix_dns.conf | 11 +++-- package/etc/conf.d/sources/rfc5687.conf.tmpl | 1 - package/etc/go_templates/source_network.t | 10 +---- tests/test_common.py | 42 +++++++------------ 5 files changed, 28 insertions(+), 40 deletions(-) diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index 69652b9..e5f19cd 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -66,4 +66,8 @@ filter f_msg_is_tcp_json{ match("rfc3164_json" value("fields.sc4s_syslog_format")) or match("tcp_json" value("fields.sc4s_syslog_format")) +}; + +filter f_host_is_ip{ + host('^(((([1]?\d)?\d|2[0-4]\d|25[0-5])\.){3}(([1]?\d)?\d|2[0-4]\d|25[0-5]))|([\da-fA-F]{1,4}(\:[\da-fA-F]{1,4}){7})|(([\da-fA-F]{1,4}:){0,5}::([\da-fA-F]{1,4}:){0,5}[\da-fA-F]{1,4})$') }; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_splunk/fix_dns.conf b/package/etc/conf.d/conflib/_splunk/fix_dns.conf index c956c33..28005f5 100644 --- a/package/etc/conf.d/conflib/_splunk/fix_dns.conf +++ b/package/etc/conf.d/conflib/_splunk/fix_dns.conf @@ -18,11 +18,16 @@ class FixHostResolver(object): # try to resolve the IP address try: - ipaddr = log_message['HOST'].decode('utf-8') + ipaddr = log_message['SOURCEIP'].decode('utf-8') hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) - name = str(hostname).split('.')[0] - log_message['HOST'] = name + #print(ipaddr) + #print(hostname) + parts=str(hostname).split('.') + name = parts[0] + #print(name) + if len(parts)>1: + log_message['HOST'] = name except: pass diff --git a/package/etc/conf.d/sources/rfc5687.conf.tmpl b/package/etc/conf.d/sources/rfc5687.conf.tmpl index b5044b9..43fddc8 100644 --- a/package/etc/conf.d/sources/rfc5687.conf.tmpl +++ b/package/etc/conf.d/sources/rfc5687.conf.tmpl @@ -5,7 +5,6 @@ source s_ietf { transport("tcp") port(601) ip-protocol(4) - keep-hostname(yes) keep-timestamp(yes) use-dns(no) use-fqdn(no) diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index af318ab..2b8ea54 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -42,7 +42,6 @@ source s_{{ .port_id }} { max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) log-fetch-limit({{getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}}) - keep-hostname(yes) keep-timestamp(yes) use-dns(no) use-fqdn(no) @@ -60,7 +59,6 @@ source s_{{ .port_id }} { max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) log-fetch-limit({{getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}}) - keep-hostname(yes) keep-timestamp(yes) use-dns(no) use-fqdn(no) @@ -267,15 +265,11 @@ source s_{{ .port_id }} { rewrite(r_set_splunk_default); {{ if eq (getenv "SC4S_USE_REVERSE_DNS" "yes") "yes" }} if { - filter { - host('((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$))') - }; + filter(f_host_is_ip); parser(p_add_context_host); }; if { - filter { - host('((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$))') - }; + filter(f_host_is_ip); parser(p_fix_host_resolver); }; {{ end }} diff --git a/tests/test_common.py b/tests/test_common.py index b0b65b1..b0f3314 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -104,6 +104,7 @@ def test_fallback(record_property, setup_wordlist, setup_splunk, setup_sc4s): # + def test_fix_dns_context(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) pid = random.randint(1000, 32000) @@ -114,12 +115,16 @@ def test_fix_dns_context(record_property, setup_wordlist, setup_splunk, setup_sc # Tune time functions epoch = epoch[:-7] - mt = env.from_string("{{ mark }} {{ bsd }} 169.254.0.2 dnstest[{{ pid }}]: {{ host }}\n") + mt = env.from_string( + "{{ mark }} {{ bsd }} 169.254.0.2 dnstest[{{ pid }}]: {{ host }}\n" + ) message = mt.render(mark="<111>", bsd=bsd, host=host, pid=pid) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} host=foo.example index=osnix \"[{{ pid }}]\" {{ host }} sourcetype=\"nix:syslog\"") + st = env.from_string( + 'search _time={{ epoch }} host=foo.example index=osnix "[{{ pid }}]" {{ host }} sourcetype="nix:syslog"' + ) search = st.render(epoch=epoch, pid=pid, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -129,31 +134,7 @@ def test_fix_dns_context(record_property, setup_wordlist, setup_splunk, setup_sc record_property("message", message) assert resultCount == 1 -def test_fix_dns(record_property, setup_wordlist, setup_splunk, setup_sc4s): - host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - pid = random.randint(1000, 32000) - - dt = datetime.datetime.now() - iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) - - # Tune time functions - epoch = epoch[:-7] - - mt = env.from_string("{{ mark }} {{ bsd }} 8.8.4.4 dnstest[{{ pid }}]: {{ host }}\n") - message = mt.render(mark="<111>", bsd=bsd, host=host, pid=pid) - sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - - st = env.from_string("search _time={{ epoch }} host=dns index=osnix \"[{{ pid }}]\" {{ host }} sourcetype=\"nix:syslog\"") - search = st.render(epoch=epoch, pid=pid, host=host) - - resultCount, eventCount = splunk_single(setup_splunk, search) - - record_property("host", host) - record_property("resultCount", resultCount) - record_property("message", message) - - assert resultCount == 1 def test_fix_dns_notfound(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) @@ -165,12 +146,16 @@ def test_fix_dns_notfound(record_property, setup_wordlist, setup_splunk, setup_s # Tune time functions epoch = epoch[:-7] - mt = env.from_string("{{ mark }} {{ bsd }} 169.254.0.1 dnstest[{{ pid }}]: {{ host }}\n") + mt = env.from_string( + "{{ mark }} {{ bsd }} 169.254.0.1 dnstest[{{ pid }}]: {{ host }}\n" + ) message = mt.render(mark="<111>", bsd=bsd, host=host, pid=pid) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) - st = env.from_string("search _time={{ epoch }} host=169.254.0.1 index=osnix \"[{{ pid }}]\" {{ host }} sourcetype=\"nix:syslog\"") + st = env.from_string( + 'search _time={{ epoch }} host=169.254.0.1 index=osnix "[{{ pid }}]" {{ host }} sourcetype="nix:syslog"' + ) search = st.render(epoch=epoch, pid=pid, host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -181,6 +166,7 @@ def test_fix_dns_notfound(record_property, setup_wordlist, setup_splunk, setup_s assert resultCount == 1 + def test_metrics(record_property, setup_wordlist, setup_splunk, setup_sc4s): st = env.from_string(