Support for archival file destinations (#179)

This merge add support for file archival in syslog-ng EWMM format

docker-compose.yml

@@ -50,7 +50,13 @@ services:
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SC4S_SOURCE_TLS_ENABLE=no
       - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
+      - SC4S_LISTEN_DEFAULT_TCP_PORT=514
+      - SC4S_LISTEN_DEFAULT_UDP_PORT=514
+#      - SC4S_LISTEN_DEFAULT_TLS_PORT=6514
       - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
+      - SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT=6000
+#      - SC4S_ARCHIVE_CHECKPOINT=yes
+      - SC4S_ARCHIVE_GLOBAL=yes
     volumes:
       - ./tls:/opt/syslog-ng/tls
   splunk:

docs/configuration.md

@@ -21,11 +21,30 @@ and variables needed to properly configure SC4S for your environment.
 | SC4S_DEST_SPLUNK_HEC_SSL_VERSION |  comma separated list | Open SSL version list |
 | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file |
 
+## Archive File Configuration
+
+This feature is designed to support "compliance" archival of all messages. To enable this feature update the Unit file
+or docker compose to mount an appropriate host folder to the container folder ``/opt/syslog-ng/var/archive``.
+The files will be stored in a folder structure using the naming pattern
+``${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log"``.
+This pattern will create one file per "vendor_product" per minute with records formatted using syslog-ng's EWMM template. 
+
+**WARNING POTENTIAL OUTAGE CAUSING CONSEQUENCE**
+
+SC4S does not prune the files that are created. The administrator must provide a means of log rotation to prune files
+and/or move them to an archival system to avoid disk space failures.
+
+| Variable | Values        | Description |
+|----------|---------------|-------------|
+| SC4S_ARCHIVE_GLOBAL | yes or undefined | Enable archive of all vendor_products |
+| SC4S_ARCHIVE_LISTEN_<VENDOR_PRODUCT> | yes(default) or undefined | See sources section of documentation enables selective archival |
+
+
 ## Syslog Source Configuration
 
 | Variable | Values/Default | Description |
 |----------|----------------|-------------|
-| SC4S_SOURCE_TLS_ENABLE | no(default) or yes | Enable a TLS listener on port 6514 |
+| SC4S_LISTEN_DEFAULT_TLS_PORT | undefined or 6514 | Enable a TLS listener on port 6514 |
 | SC4S_SOURCE_TLS_OPTIONS | See openssl | List of SSl/TLS protocol versions to support | 
 | SC4S_SOURCE_TLS_CIPHER_SUITE | See openssl | List of Ciphers to support |
 | SC4S_SOURCE_TCP_MAX_CONNECTIONS | 2000 | Max number of TCP Connections |

package/etc/conf.d/conflib/_splunk/splunkfields.conf

@@ -17,7 +17,6 @@ block rewrite r_set_splunk_dest_default(
       set("`index`", value(".splunk.index"));
       set("`source`", value(".splunk.source"));
       set("`sourcetype`", value(".splunk.sourcetype"));
-      set("`template`", value("fields.sc4s_template"));
 };
 
 

package/etc/conf.d/destinations/archive_file.conf

@@ -0,0 +1,5 @@
+destination d_archive {
+    file("/opt/syslog-ng/var/archive/${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log"
+         template('$(format-ewmm)')
+     );
+};

package/etc/conf.d/destinations/splunk_hec.conf.tmpl

@@ -43,7 +43,7 @@ destination d_hec {
                  source=${.splunk.source}
                  sourcetype=${.splunk.sourcetype}
                  index=${.splunk.index}
-                 event=$(template ${fields.sc4s_template} $(template "t_standard"))
+                 event=$MSG
                  fields.*)')
         );
 };

package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl

@@ -28,7 +28,7 @@ destination d_hec_internal {
                  source=${.splunk.source}
                  sourcetype=${.splunk.sourcetype}
                  index=${.splunk.index}
-                 event=$(template ${fields.sc4s_template} $(template "t_standard"))
+                 event=$MSG
                  fields.*)')
         );
 };

package/etc/conf.d/local/config/log_paths/example.conf.tmpl

@@ -27,7 +27,7 @@ log {
 # The first time this template is used the log_path will be linked to the default port
 
 {{- if eq (.) "yes"}}
-    source(s_default-ports);
+    source(s_DEFAULT);
 
 # Filters should be updated to use the simplest and most effecient logic possible to discard
 # the message from this path
@@ -40,12 +40,12 @@ log {
 # In the second pass through the template a link to the dedicated port is used. This
 # normally does not require additional filters
 
-source (s_dedicated_port_LOCAL_EXAMPLE);
+source (s_LOCAL_EXAMPLE);
 {{- end}}
 
 #Set a default sourcetype and index
 
-    rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))};
+    rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"))};
 
 #using the key "local_example" find any cutomized index,source or sourcetype meta values
 

package/etc/conf.d/log_paths/internal.conf.tmpl

@@ -32,11 +32,10 @@ log {
         destination(d_hecmetrics);  #--HEC--
     } else {
 
-
         {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes"}}
         destination(d_stdout);
         {{- end}}
-        rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"), template("t_msg_only"))};
+        rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))};
         parser {p_add_context_splunk(key("sc4s_events")); };
         destination(d_hec_internal);
     };

package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl

@@ -1,64 +1,80 @@
-# Checkpoint Splunk format
-{{- if (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT") "no") "no") }}
+# Generate the custom port if defined
 {{ $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }}
 {{ tmpl.Exec "t/source_network.t" $context }}
-{{- end -}}
+
+# The following is an inline template; we will use this to generate the actual log path
 {{ define "log_path" }}
 log {
 {{- if eq (.) "yes"}}
-    source(s_default-ports);
+    source(s_DEFAULT);
     filter(f_is_rfc3164);
     filter(f_checkpoint_splunk);
 {{- end}}
 {{- if eq (.) "no"}}
-    source (s_dedicated_port_CHECKPOINT_SPLUNK);
+    source (s_CHECKPOINT_SPLUNK);
 {{- end}}
 
     parser {
         kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}"));
         date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}));
-
     };
 
-    rewrite { set("${.kv.hostname}", value("HOST")); };
+    rewrite {
+        set("${.kv.hostname}", value("HOST"));
+        set("checkpoint_splunk", value("fields.sc4s_vendor_product"));
+        r_set_splunk_dest_default(sourcetype("cp_log"), index("netops"))
+    };
 
-    rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), index("netops"), template("t_hdr_msg"))};
     parser {p_add_context_splunk(key("checkpoint_splunk")); };
 
     if  {
        filter(f_checkpoint_splunk_NetworkTraffic);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"), template("t_standard"))};
+       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))};
        parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); };
     } elif  {
        filter(f_checkpoint_splunk_Web);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"), template("t_standard"))};
+       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))};
        parser {p_add_context_splunk(key("checkpoint_splunk_web")); };
     } elif  {
        filter(f_checkpoint_splunk_NetworkSessions);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"), template("t_standard"))};
+       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))};
        parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); };
     } elif  {
        filter(f_checkpoint_splunk_IDS_Malware);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"), template("t_standard"))};
+       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))};
        parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
     } elif  {
        filter(f_checkpoint_splunk_IDS);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"), template("t_standard"))};
+       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))};
        parser {p_add_context_splunk(key("checkpoint_splunk_ids")); };
     } elif  {
        filter(f_checkpoint_splunk_email);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"), template("t_standard"))};
+       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))};
        parser {p_add_context_splunk(key("checkpoint_splunk_email")); };
     } elif  {
        filter(f_checkpoint_splunk_DLP);
-       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"), template("t_standard"))};
+       rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))};
        parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); };
     };
 
-
     parser (compliance_meta_by_source);
 
-    destination(d_hec);  #--HEC--
+    #We want to unset the fields we won't need, as this is copied into the
+    #disk queue for network destinations. This can be very disk expensive
+    #if we don't
+    rewrite {
+        set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG"));
+        unset(value("RAWMSG"));
+        unset(value("PROGRAM"));
+        unset(value("LEGACY_MSGHDR"));
+        groupunset(values(".kv.*"));
+    };
+
+    destination(d_hec);
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK") }}
+    destination(d_archive);
+{{- end}}
 
     flags(flow-control);
 };

package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl

@@ -1,25 +1,43 @@
 # Cisco ASA
-{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT") "no") "no") }}
 {{ $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common"}}
 {{ tmpl.Exec "t/source_network.t" $context }}
-{{- end -}}
+
+# The following is an inline template; we will use this to generate the actual log path
 {{ define "log_path" }}
 log {
 {{- if eq (.) "yes"}}
-    source(s_default-ports);
+    source(s_DEFAULT);
     filter(f_is_rfc3164);
     filter(f_cisco_asa);
 {{- end}}
 {{- if eq (.) "no"}}
-    source (s_dedicated_port_CISCO_ASA_LEGACY);
+    source (s_CISCO_ASA_LEGACY);
 {{- end}}
 
-    rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))};
+    rewrite {
+        set("cisco_asa", value("fields.sc4s_vendor_product"));
+        r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))
+    };
     parser {p_add_context_splunk(key("cisco_asa")); };
 
     parser (compliance_meta_by_source);
 
-    destination(d_hec);  #--HEC--
+    #We want to unset the fields we won't need, as this is copied into the
+    #disk queue for network destinations. This can be very disk expensive
+    #if we don't
+    rewrite {
+        set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        unset(value("RAWMSG"));
+        unset(value("PROGRAM"));
+        unset(value("LEGACY_MSGHDR"));
+    };
+
+    destination(d_hec);
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ASA_LEGACY") }}
+    destination(d_archive);
+{{- end}}
+
 
     flags(flow-control);
 

package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl

@@ -1,26 +1,44 @@
 # Cisco IOS
-{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT") "no") "no") }}
 {{ $context := dict "port_id" "CISCO_IOS" "parser" "common" }}
 {{ tmpl.Exec "t/source_network.t" $context }}
-{{- end -}}
+
+# The following is an inline template; we will use this to generate the actual log path
 {{ define "log_path" }}
 log {
 {{- if eq (.) "yes" }}
-    source(s_default-ports);
+    source(s_DEFAULT);
     filter(f_is_cisco_ios);
 {{- end }}
 {{- if eq (.) "no" }}
-    source (s_dedicated_port_CISCO_IOS);
+    source (s_CISCO_IOS);
 {{- end }}
 
-    rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_msg_only"))};
+    rewrite {
+        set("cisco_ios", value("fields.sc4s_vendor_product"));
+        r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"))
+    };
     parser {
         p_add_context_splunk(key("cisco_ios"));
     };
 
     parser (compliance_meta_by_source);
 
-    destination(d_hec);  #--HEC--
+    #We want to unset the fields we won't need, as this is copied into the
+    #disk queue for network destinations. This can be very disk expensive
+    #if we don't
+    rewrite {
+        set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+        unset(value("RAWMSG"));
+        unset(value("PROGRAM"));
+        unset(value("LEGACY_MSGHDR"));
+        groupunset(values(".cisco.*"));
+    };
+
+    destination(d_hec);
+
+{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_IOS") }}
+    destination(d_archive);
+{{- end}}
 
     flags(flow-control);
 };

package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl

@@ -1,3 +1,7 @@
+# Cisco ISE
+{{ $context := dict "port_id" "CISCO_ISE" "parser" "common"}}
+{{ tmpl.Exec "t/source_network.t" $context }}
+
 #This filter uses a field we set to prevent the original messages before aggregation from being
 #sent to Splunk
 filter f_cisco_ise_complete{
@@ -40,33 +44,52 @@ parser ise_event_time {
             template("${ISE.DATE} ${ISE.TIME} ${ISE.TZ}")
     );
 };
-# Cisco ISE
-{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TLS_PORT") "no") "no") }}
-{{ $context := dict "port_id" "CISCO_ISE" "parser" "common"}}
-{{ tmpl.Exec "t/source_network.t" $context }}
-{{- end -}}
+# The following is an inline template; we will use this to generate the actual log path
 {{ define "log_path" }}
 log {
 {{- if eq (.) "yes"}}
-    source(s_default-ports);
+    source(s_DEFAULT);
     filter(f_is_rfc3164);
     filter(f_cisco_ise);
 {{- end}}
 {{- if eq (.) "no"}}
-    source (s_dedicated_port_CISCO_ISE);
+    source (s_CISCO_ISE);
 {{- end}}
 
     parser(ise_grouping);
 
     if {
         filter(f_cisco_ise_complete);
         parser(ise_event_time);
-        rewrite { r_set_splunk_dest_default(sourcetype("cisco:ise:syslog"), index("netauth"), template("t_msg_only"))};
-        parser {p_add_context_splunk(key("cisco_ise")); };
+        rewrite {
+            set("cisco_ise", value("fields.sc4s_vendor_product"));
+            r_set_splunk_dest_default(sourcetype("cisco:ise:syslog"), index("netauth"))
+        };
 
+        parser {p_add_context_splunk(key("cisco_ise")); };
         parser (compliance_meta_by_source);
 
-        destination(d_hec);  #--HEC--
+        #We want to unset the fields we won't need, as this is copied into the
+        #disk queue for network destinations. This can be very disk expensive
+        #if we don't
+        rewrite {
+            set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG"));
+            unset(value("RAWMSG"));
+            unset(value("PROGRAM"));
+            unset(value("PID"));
+            unset(value("LEGACY_MSGHDR"));
+            unset(value("EPOCH"));
+            unset(value("VERSION"));
+            unset(value("TIMESECFRAC"));
+            groupunset(values("ISE.*"));
+        };
+
+        destination(d_hec);
+
+    {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ISE") }}
+        destination(d_archive);
+    {{- end}}
+
         flags(flow-control);
     };