Merge pull request #494 from splunk/feature/fix-host

Dyanamic reverse DNS
CSVD · May 28, 2020 · 1022740 · 1022740
2 parents 789650b + 2c7e65d
commit 1022740
Show file tree

Hide file tree

Showing 6 changed files with 204 additions and 51 deletions.
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -261,6 +261,11 @@ In some cases rogue or port-probing data can be sent to SC4S from misconfigured
 the `vendor_product_by_source.conf` filter `f_null_queue` with one or more ip/subnet masks to drop events without
 logging. Note that drop metrics will be recorded.
 
+## Fixing (overriding) the host field
+
+In some cases the host value is not present or an IP address in the syslog even analysts and users prefer host names. SC4S
+will first check `host.csv` and replace the value of `host` with the value specified. If a value is not found in `dns.csv`
+reverse dns lookup will be attempted. IP will only be used as the host value as a last result.
 
 ## Splunk Connect for Syslog output templates (syslog-ng templates)
 

diff --git a/package/etc/conf.d/conflib/_splunk/fix_dns.conf b/package/etc/conf.d/conflib/_splunk/fix_dns.conf
@@ -0,0 +1,46 @@
+python {
+
+"""
+simple syslog-ng Python parser example
+resolves IP to hostname
+value pair names are hard-coded
+"""
+import re
+import socket
+
+class FixHostResolver(object):
+
+    def parse(self, log_message):
+        """
+        Resolves IP to hostname
+        """
+
+
+        # try to resolve the IP address
+        try:
+            ipaddr = log_message['HOST'].decode('utf-8')    
+
+            resolved = socket.gethostbyaddr(ipaddr)
+            hostname = resolved[0]
+            log_message['HOST'] = hostname
+        except:
+            pass
+
+        # return True, other way message is dropped
+        return True
+
+};
+
+
+parser p_fix_host_resolver {
+    python(
+        class("FixHostResolver")
+    );
+};
+
+parser p_add_context_host {
+    add-contextual-data(
+        selector("${HOST}"),
+        database("conf.d/local/context/host.csv"),
+    );
+};
diff --git a/package/etc/context_templates/host.csv.example b/package/etc/context_templates/host.csv.example
@@ -0,0 +1 @@
+169.254.0.2,HOST,foo.example
diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t
@@ -207,8 +207,20 @@ source s_{{ .port_id }} {
         };
 {{ end }}
         rewrite(r_set_splunk_default);
-        parser {
-            vendor_product_by_source();
+        if {
+            filter {
+                host('((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$))')
+            };
+            parser(p_add_context_host);
+        };        
+        if {
+            filter {
+                host('((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$))')
+            };
+            parser(p_fix_host_resolver);
+        };        
+        parser {            
+            vendor_product_by_source();            
         };
 
         if {

diff --git a/package/etc/syslog-ng.conf.tmpl b/package/etc/syslog-ng.conf.tmpl
@@ -18,11 +18,11 @@ options {
         time_reopen (10);
         log_fifo_size (10000);
         chain_hostnames (yes);
-        use_dns ({{getenv "SC4S_GLOBAL_DNS_USE" "no"}});
+        use_dns (no);
         use_fqdn (no);
-        dns-cache({{getenv "SC4S_GLOBAL_DNS_CACHE" "yes"}});
+        dns-cache(no);
         create_dirs (no);
-        keep-hostname (no);
+        keep-hostname (yes);
         create_dirs(yes);
         dir_perm(0750);
         stats-freq(30);