[fix] Incorrect host resolution (#610)

* [fix] Incorrect host resolution When the log source includes an IP as host resolve using connection IP rather than field IP. When the host resolves to a single name rather than FQDN do not set the host value as this can't be trusted * Update test_common.py * Update fix_dns.conf
CSVD · Aug 7, 2020 · b71a27d · b71a27d
1 parent 5531372
commit b71a27d
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 40 deletions.
diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf
@@ -66,4 +66,8 @@ filter f_msg_is_tcp_json{
     match("rfc3164_json" value("fields.sc4s_syslog_format"))
     or
     match("tcp_json" value("fields.sc4s_syslog_format"))
+};
+
+filter f_host_is_ip{
+    host('^(((([1]?\d)?\d|2[0-4]\d|25[0-5])\.){3}(([1]?\d)?\d|2[0-4]\d|25[0-5]))|([\da-fA-F]{1,4}(\:[\da-fA-F]{1,4}){7})|(([\da-fA-F]{1,4}:){0,5}::([\da-fA-F]{1,4}:){0,5}[\da-fA-F]{1,4})$')
 };
diff --git a/package/etc/conf.d/conflib/_splunk/fix_dns.conf b/package/etc/conf.d/conflib/_splunk/fix_dns.conf
@@ -18,11 +18,16 @@ class FixHostResolver(object):
 
         # try to resolve the IP address
         try:
-            ipaddr = log_message['HOST'].decode('utf-8')    
+            ipaddr = log_message['SOURCEIP'].decode('utf-8')    
 
             hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr)
-            name = str(hostname).split('.')[0]
-            log_message['HOST'] = name
+            #print(ipaddr)
+            #print(hostname)
+            parts=str(hostname).split('.')
+            name = parts[0]
+            #print(name)                        
+            if len(parts)>1:                
+                log_message['HOST'] = name
         except:
             pass
 

diff --git a/package/etc/conf.d/sources/rfc5687.conf.tmpl b/package/etc/conf.d/sources/rfc5687.conf.tmpl
@@ -5,7 +5,6 @@ source s_ietf {
                 transport("tcp")
                 port(601)
                 ip-protocol(4)
-                keep-hostname(yes)
                 keep-timestamp(yes)
                 use-dns(no)
                 use-fqdn(no)

diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t
@@ -42,7 +42,6 @@ source s_{{ .port_id }} {
                 max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}})
                 log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}})
                 log-fetch-limit({{getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}})
-                keep-hostname(yes)
                 keep-timestamp(yes)
                 use-dns(no)
                 use-fqdn(no)
@@ -60,7 +59,6 @@ source s_{{ .port_id }} {
                 max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}})
                 log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}})
                 log-fetch-limit({{getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}})
-                keep-hostname(yes)
                 keep-timestamp(yes)
                 use-dns(no)
                 use-fqdn(no)
@@ -267,15 +265,11 @@ source s_{{ .port_id }} {
         rewrite(r_set_splunk_default);
         {{ if eq (getenv "SC4S_USE_REVERSE_DNS" "yes") "yes" }}
         if {
-            filter {
-                host('((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$))')
-            };
+            filter(f_host_is_ip);
             parser(p_add_context_host);
         };        
         if {
-            filter {
-                host('((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$))')
-            };
+            filter(f_host_is_ip);
             parser(p_fix_host_resolver);
         };
         {{ end }}

diff --git a/tests/test_common.py b/tests/test_common.py
@@ -104,6 +104,7 @@ def test_fallback(record_property, setup_wordlist, setup_splunk, setup_sc4s):
 
 #
 
+
 def test_fix_dns_context(record_property, setup_wordlist, setup_splunk, setup_sc4s):
     host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
     pid = random.randint(1000, 32000)
@@ -114,12 +115,16 @@ def test_fix_dns_context(record_property, setup_wordlist, setup_splunk, setup_sc
     # Tune time functions
     epoch = epoch[:-7]
 
-    mt = env.from_string("{{ mark }} {{ bsd }} 169.254.0.2 dnstest[{{ pid }}]: {{ host }}\n")
+    mt = env.from_string(
+        "{{ mark }} {{ bsd }} 169.254.0.2 dnstest[{{ pid }}]: {{ host }}\n"
+    )
     message = mt.render(mark="<111>", bsd=bsd, host=host, pid=pid)
 
     sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
 
-    st = env.from_string("search _time={{ epoch }} host=foo.example index=osnix \"[{{ pid }}]\" {{ host }} sourcetype=\"nix:syslog\"")
+    st = env.from_string(
+        'search _time={{ epoch }} host=foo.example index=osnix "[{{ pid }}]" {{ host }} sourcetype="nix:syslog"'
+    )
     search = st.render(epoch=epoch, pid=pid, host=host)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
@@ -129,31 +134,7 @@ def test_fix_dns_context(record_property, setup_wordlist, setup_splunk, setup_sc
     record_property("message", message)
 
     assert resultCount == 1
-def test_fix_dns(record_property, setup_wordlist, setup_splunk, setup_sc4s):
-    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
-    pid = random.randint(1000, 32000)
-
-    dt = datetime.datetime.now()
-    iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
-
-    # Tune time functions
-    epoch = epoch[:-7]
-
-    mt = env.from_string("{{ mark }} {{ bsd }} 8.8.4.4 dnstest[{{ pid }}]: {{ host }}\n")
-    message = mt.render(mark="<111>", bsd=bsd, host=host, pid=pid)
 
-    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
-
-    st = env.from_string("search _time={{ epoch }} host=dns index=osnix \"[{{ pid }}]\" {{ host }} sourcetype=\"nix:syslog\"")
-    search = st.render(epoch=epoch, pid=pid, host=host)
-
-    resultCount, eventCount = splunk_single(setup_splunk, search)
-
-    record_property("host", host)
-    record_property("resultCount", resultCount)
-    record_property("message", message)
-
-    assert resultCount == 1
 
 def test_fix_dns_notfound(record_property, setup_wordlist, setup_splunk, setup_sc4s):
     host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
@@ -165,12 +146,16 @@ def test_fix_dns_notfound(record_property, setup_wordlist, setup_splunk, setup_s
     # Tune time functions
     epoch = epoch[:-7]
 
-    mt = env.from_string("{{ mark }} {{ bsd }} 169.254.0.1 dnstest[{{ pid }}]: {{ host }}\n")
+    mt = env.from_string(
+        "{{ mark }} {{ bsd }} 169.254.0.1 dnstest[{{ pid }}]: {{ host }}\n"
+    )
     message = mt.render(mark="<111>", bsd=bsd, host=host, pid=pid)
 
     sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
 
-    st = env.from_string("search _time={{ epoch }} host=169.254.0.1 index=osnix \"[{{ pid }}]\" {{ host }} sourcetype=\"nix:syslog\"")
+    st = env.from_string(
+        'search _time={{ epoch }} host=169.254.0.1 index=osnix "[{{ pid }}]" {{ host }} sourcetype="nix:syslog"'
+    )
     search = st.render(epoch=epoch, pid=pid, host=host)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
@@ -181,6 +166,7 @@ def test_fix_dns_notfound(record_property, setup_wordlist, setup_splunk, setup_s
 
     assert resultCount == 1
 
+
 def test_metrics(record_property, setup_wordlist, setup_splunk, setup_sc4s):
 
     st = env.from_string(