Merge pull request #229 from splunk/feature/linux-syslog

Support linux OS events from appliances

.gitignore

@@ -384,3 +384,4 @@ fabric.properties
 tests/test_plugin_*.py
 # package/etc/conf.d/local/
 !package/etc/conf.d/local
+replay

docker-compose.yml

@@ -30,7 +30,7 @@ services:
         RH_ACTIVATION: ${RH_ACTIVATION}
     hostname: sc4s
 #When this is enabled test_common will fail
-#    command: -det
+    command: -det
     ports:
       - "514:514"
       - "601:601"
@@ -72,6 +72,16 @@ services:
       - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
     volumes:
       - splunk-etc:/opt/splunk/etc
+  pcapreplay:
+    build:
+      context: ./utility/pcapreplay
+    entrypoint: tail -f /dev/null
+    links:
+      - splunk
+      - sc4s
+    volumes:
+      - ./replay:/work
+
 volumes:
   sc4s-results:
     external: true

docs/gettingstarted/index.md

@@ -42,6 +42,8 @@ using the SC4S defaults. SC4S can be easily customized to use different indexes
 * netops
 * netproxy
 * netipam
+* oswinsec
+* osnix
 * em_metrics (ensure this is created as a metrics index)
 
 #### Install Related Splunk Apps

docs/sources/nix/index.md

@@ -0,0 +1,55 @@
+# Vendor - Nix Generic
+
+## Product - All Products
+
+Many appliance vendor utilize Linux and BSD distributions as the foundation of the solution. When configured to log via
+syslog, these devices' OS logs (from a security perspective) can be monitored using the common Splunk Nix TA.
+
+Note: This is NOT a replacement for or alternative to the Splunk Universal forwarder on Linux and Unix. For general-purpose
+server applications, the Universal Forwarder offers more comprehensive collection of events and metrics appropriate for both
+security and operations use cases.
+
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/833/                                                                 |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| nix:syslog  | None |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| nix_syslog      | nix:syslog       | osnix          | none          |
+
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_ARCHIVE_NIX_SYSLOG | no | Enable archive to disk for this specific source |
+| SC4S_DEST_NIX_SYSLOG_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active proxy will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=osnix sourcetype=nix:syslog | stats count by host
+```

mkdocs.yml

@@ -17,6 +17,7 @@ nav:
       - Fortinet: sources/Fortinet/index.md
       - Imperva: sources/Imperva/index.md
       - Juniper: sources/Juniper/index.md
+      - Nix: sources/nix/index.md
       - Microfocus: sources/Microfocus/index.md
       - 'Paloalto Networks': sources/PaloaltoNetworks/index.md
       - Proofpoint: sources/Proofpoint/index.md

package/etc/conf.d/conflib/_common/templates.conf

@@ -42,6 +42,9 @@ template t_hdr_msg {
         template("${MSGHDR}${MESSAGE}");
         };
 
+template t_legacy_hdr_msg {
+        template("${LEGACY_MSGHDR}${MESSAGE}");
+        };
 # ===============================================================================================
 # Message Header, Structured Data (from RFC5424 parse) and Message; for Juniper
 # ===============================================================================================

package/etc/conf.d/filters/nix/syslog.conf

@@ -0,0 +1,5 @@
+filter f_nix_syslog {
+    program("[a-zA-Z0-9\/]+")
+    and
+    match('[a-zA-Z\]]: $' value("LEGACY_MSGHDR"))
+};

package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl

@@ -78,7 +78,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 {{- if or (or (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }}

package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl

@@ -41,7 +41,7 @@ log {
 {{- end}}
 
 
-    flags(flow-control);
+    flags(flow-control,final);
 
 };
 {{- end}}

package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl

@@ -43,7 +43,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl

@@ -92,7 +92,7 @@ log {
         destination(d_archive);
     {{- end}}
 
-        flags(flow-control);
+        flags(flow-control,final);
     };
 
 

package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl

@@ -44,7 +44,7 @@ log {
 {{- end}}
 
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl

@@ -41,7 +41,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl

@@ -60,7 +60,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl

@@ -45,7 +45,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl

@@ -58,7 +58,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl

@@ -42,7 +42,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl

@@ -43,7 +43,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl

@@ -40,7 +40,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl

@@ -88,7 +88,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 
 };
 {{- end}}

package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl

@@ -92,7 +92,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl

@@ -50,7 +50,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl

@@ -126,7 +126,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 
 };
 {{- end}}

package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl

@@ -81,7 +81,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl

@@ -39,7 +39,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl

@@ -42,7 +42,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl

@@ -61,6 +61,8 @@ log {
     destination(d_archive);
 {{- end}}
 
+
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl

@@ -39,7 +39,7 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control);
+    flags(flow-control,final);
 };
 {{- end}}
 

package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl

@@ -0,0 +1,59 @@
+# Proofpoint
+{{ $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+# The following is an inline template; we will use this to generate the actual log path
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes" }}
+    source(s_DEFAULT);
+    filter(f_nix_syslog);
+{{- end }}
+{{- if eq (.) "no" }}
+    source (s_NIX_SYSLOG);
+{{- end }}
+
+    rewrite {
+        set("nix_syslog", value("fields.sc4s_vendor_product"));
+        subst("^[^\t]+\t", "", value("MESSAGE"), flags("global"));
+        set("${PROGRAM}", value(".PROGRAM"));
+        subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM"));
+    };
+
+    rewrite {
+        r_set_splunk_dest_default(sourcetype("nix:syslog"), index("main"), source("program:${.PROGRAM}") )
+
+    };
+
+    parser { p_add_context_splunk(key("nix_syslog")); };
+
+    parser (compliance_meta_by_source);
+
+    #We want to unset the fields we won't need, as this is copied into the
+    #disk queue for network destinations. This can be very disk expensive
+    #if we don't
+    rewrite {
+        set("$(template ${fields.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG"));
+        unset(value("RAWMSG"));
+        unset(value("PROGRAM"));
+        unset(value("LEGACY_MSGHDR"));
+    };
+
+{{- if ((getenv "SC4S_NIX_SYSLOG_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_NIX_SYSLOG_HEC" "no") | conv.ToBool) }}
+    destination(d_hec);
+{{- end}}
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_NIX_SYSLOG") }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,final);
+};
+{{- end}}
+
+{{- if or (or (getenv  (print "SC4S_LISTEN_NIX_SYSLOG_TCP_PORT")) (getenv  (print "SC4S_LISTEN_NIX_SYSLOG_UDP_PORT"))) (getenv  (print "SC4S_NIX_SYSLOG_NSS_TLS_PORT")) }}
+# Listen on the specified dedicated port(s) for NIX_SYSLOG traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for NIX_SYSLOG traffic
+{{ tmpl.Exec "log_path" "yes" }}

package/etc/conf.d/log_paths/zfallback.conf.tmpl → package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl

@@ -16,7 +16,7 @@ log {
 
     #in fallback archive only write rawmsg as msg
     rewrite {
-        set("value(RAWMSG)" value("MSG"));
+        set("$RAWMSG" value("MSG"));
         unset(value("RAWMSG"));
         unset(value("PROGRAM"));
         unset(value("LEGACY_MSGHDR"));
@@ -27,5 +27,5 @@ log {
     destination(d_archive);
 {{- end}}
 
-    flags(flow-control,fallback);
+    flags(flow-control,fallback,final);
 };

package/etc/context_templates/splunk_index.csv

@@ -35,6 +35,7 @@
 #juniper_nsm,index,netfw
 #juniper_nsm_idp,index,netids
 #juniper_legacy,index,netops
+#nix_syslog,index,osnix
 #pan_traffic,index,netfw
 #pan_threat,index,netproxy
 #pan_system,index,netops