Merge branch 'release/0.7.0'

CSVD · Sep 25, 2019 · eb483ba · eb483ba
2 parents 79715ab + 2caca6d
commit eb483ba
Show file tree

Hide file tree

Showing 57 changed files with 484 additions and 604 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,5 @@
 [submodule "package/syslog-ng"]
 	path = package/syslog-ng
 	url = https://github.com/balabit/syslog-ng.git
+[submodule "Submod"]
+	branch = syslog-ng-3.23.1
diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md
@@ -5,9 +5,11 @@ Refer to [Getting Started](https://docs.docker.com/get-started/)
 
 # Setup
 
-* Create a systemd unit file use to start the container with the host os. ``/lib/systemd/system/sc4s.service``
+* Create a systemd unit file use to start the container with the host os. 
 
-*NOTE*: Replace the URL and HEC tokens with the appropriate values for our environment
+``sudo vi /lib/systemd/system/sc4s.service``
+
+Enter insert mode and paste the following
 
 ```ini
 [Unit]

diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md
@@ -5,9 +5,11 @@ Refer to [Installation](https://podman.io/getting-started/installation)
 
 # Setup
 
-* Create a systemd unit file use to start the container with the host os. ``/lib/systemd/system/sc4s.service``
+* Create a systemd unit file use to start the container with the host os. 
 
-*NOTE*: Replace the URL and HEC tokens with the appropriate values for our environment
+``sudo vi /lib/systemd/system/sc4s.service``
+
+Enter insert mode and paste the following
 
 ```ini
 [Unit]

diff --git a/package/Dockerfile b/package/Dockerfile
@@ -86,18 +86,19 @@ RUN curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gompl
 
 COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf
 COPY etc/conf.d /opt/syslog-ng/etc/conf.d
+COPY etc/templates /opt/syslog-ng/etc/templates
 COPY etc/context-local /opt/syslog-ng/etc/context-local
-
+COPY sbin/entrypoint.sh /
 RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer
 RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V
-RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t
+#RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t
 
 EXPOSE 514
 EXPOSE 601/tcp
 EXPOSE 6514/tcp
 
 ENV SPLUNK_CONNECT_METHOD=UF
 
-ENTRYPOINT ["/opt/syslog-ng/sbin/syslog-ng", "-F"]
+ENTRYPOINT ["/entrypoint.sh", "-F"]
 
 HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate
diff --git a/package/etc/conf.d/conflib/_common/network.conf b/package/etc/conf.d/conflib/_common/network.conf
diff --git a/package/etc/conf.d/conflib/_common/network.conf.tmpl b/package/etc/conf.d/conflib/_common/network.conf.tmpl
@@ -1,93 +0,0 @@
-# ===============================================================================================
-# source definition for remote devices
-# ===============================================================================================
-
-# ===============================================================================================
-# Defaults for the default-network-drivers() source:
-# 514, both TCP and UDP, for RFC3164 (BSD-syslog) formatted traffic
-# 601 TCP, for RFC5424 (IETF-syslog) formatted traffic
-# 6514 TCP, for TLS-encrypted traffic
-# ===============================================================================================
-
-    channel {
-        source {
-{{ if ne (getenv "confgen_udp_port") "no" }}
-            syslog (
-                transport("udp")
-                port({{getenv "confgen_udp_port"}})
-                ip-protocol(4)
-                so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "425984"}})
-                keep-hostname(yes)
-                keep-timestamp(yes)
-                use-dns(no)
-                use-fqdn(no)
-                chain-hostnames(off)
-                flags(no-parse)
-            );
-{{ end }}
-{{ if ne (getenv "confgen_tcp_port") "no" }}
-            network (
-                transport("tcp")
-                port({{getenv "confgen_tcp_port"}})
-                ip-protocol(4)
-                max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}})
-                log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}})
-                log-fetch-limit({{getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}})
-                keep-hostname(yes)
-                keep-timestamp(yes)
-                use-dns(no)
-                use-fqdn(no)
-                chain-hostnames(off)
-                flags(no-parse)
-            );
-{{ end }}
-        };
-        #TODO: #60 Remove this function with enhancement
-        rewrite(set_rfcnonconformant);
-
-{{ if eq (getenv "confgen_parser") "rfc5424_strict" }}
-        filter(f_rfc5424_strict);
-        parser {
-                syslog-parser(flags(syslog-protocol  store-raw-message));
-            };
-        rewrite(set_rfc5424_strict);
-{{ else if eq (getenv "confgen_parser") "rfc5424_noversion" }}
-        filter(f_rfc5424_noversion);
-        parser {
-                syslog-parser(flags(syslog-protocol  store-raw-message));
-            };
-        rewrite(set_rfc5424_noversion);
-{{ else if eq (getenv "confgen_parser") "cisco_parser" }}
-        parser {cisco-parser()};
-        rewrite(set_metadata_vendor_product_cisco_ios);
-{{ else if eq (getenv "confgen_parser") "rfc3164" }}
-        parser {
-            syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message));
-        };
-        rewrite(set_rfc3164);
-{{ else }}
-        if {
-            filter(f_rfc5424_strict);
-            parser {
-                    syslog-parser(flags(syslog-protocol  store-raw-message));
-                };
-            rewrite(set_rfc5424_strict);
-        } elif {
-            filter(f_rfc5424_noversion);
-            parser {
-                    syslog-parser(flags(syslog-protocol  store-raw-message));
-                };
-            rewrite(set_rfc5424_noversion);
-        } elif {
-            parser {cisco-parser()};
-            rewrite(set_metadata_vendor_product_cisco_ios);
-        } else {
-            parser {
-                syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message));
-            };
-            rewrite(set_rfc3164);
-        };
-{{ end }}
-        rewrite(r_set_splunk_default);
-
-   };

diff --git a/package/etc/conf.d/destinations/splunk_hec.conf b/package/etc/conf.d/destinations/splunk_hec.conf
diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl
@@ -1,38 +1,42 @@
+destination d_hec {
     http(
-         url("{{getenv "SPLUNK_HEC_URL"}}")
+         url("{{- getenv "SPLUNK_HEC_URL"}}")
          method("POST")
-         log-fifo-size({{getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}})
-         workers({{getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10"}})
-         batch-lines({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_LINES" "1000"}})
-         batch-bytes({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_BYTES" "4096kb"}})
-         batch-timeout({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_TIMEOUT" "1"}})
-         timeout({{getenv "SC4S_DEST_SPLUNK_HEC_TIMEOUT" "30"}})
+         log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}})
+         workers({{- getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10"}})
+         batch-lines({{- getenv "SC4S_DEST_SPLUNK_HEC_BATCH_LINES" "1000"}})
+         batch-bytes({{- getenv "SC4S_DEST_SPLUNK_HEC_BATCH_BYTES" "4096kb"}})
+         batch-timeout({{- getenv "SC4S_DEST_SPLUNK_HEC_BATCH_TIMEOUT" "1"}})
+         timeout({{- getenv "SC4S_DEST_SPLUNK_HEC_TIMEOUT" "30"}})
          user_agent("sc4s/1.0 (events)")
          user("sc4s")
-         headers("{{getenv "SC4S_DEST_SPLUNK_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}")
-         password("{{getenv "SPLUNK_HEC_TOKEN"}}")
+         headers("{{- getenv "SC4S_DEST_SPLUNK_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}")
+         password("{{- getenv "SPLUNK_HEC_TOKEN"}}")
 
-        {{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE" "yes") "yes" }}
+         {{- if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE" "yes") "yes"}}
          persist-name("splunk_hec")
          disk-buffer(
 
-            {{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE" "no") "yes" }}
-            mem-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024"}})
+            {{- if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE" "no") "yes"}}
+            mem-buf-size({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024"}})
             reliable(yes)
-            {{else}}
-            mem-buf-length({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000"}})
+            {{- else}}
+            mem-buf-length({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000"}})
             reliable(no)
-            {{ end }}
+            {{- end}}
 
-             disk-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "1048576"}})
+             disk-buf-size({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "1048576"}})
              dir("/opt/syslog-ng/var/data/disk-buffer/")
              )
-        {{ end }}
-         tls(peer-verify({{getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}})
-{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}}             cipher-suite("{{getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE"}}"){{end}}
-{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION") ""}}             ssl-version("{{getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION"}}"){{end}}
-             ca-file("{{getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}")
-            )
+         {{- end}}
+         tls(peer-verify({{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}})
+         {{- if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}}
+         cipher-suite("{{- getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE"}}")
+         {{- end}}
+         {{- if ne (getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION") ""}}
+         ssl-version("{{- getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION"}}")
+         {{- end}}
+         ca-file("{{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}"))
          body('$(format-json
                  time=$S_UNIXTIME.$S_MSEC
                  host=${HOST}
@@ -42,3 +46,4 @@
                  event=$(template ${fields.sc4s_template} $(template "t_standard"))
                  fields.*)')
         );
+};
diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf b/package/etc/conf.d/destinations/splunk_hec_internal.conf
diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl
@@ -1,23 +1,27 @@
+destination d_hec_internal {
     http(
-         url("{{getenv "SPLUNK_HEC_URL"}}")
+         url("{{- getenv "SPLUNK_HEC_URL"}}")
          method("POST")
-         log-fifo-size({{getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}})
+         log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}})
          workers(1)
-         batch-lines({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_LINES" "1000"}})
-         batch-bytes({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_BYTES" "4096kb"}})
-         batch-timeout({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_TIMEOUT" "1"}})
-         timeout({{getenv "SC4S_DEST_SPLUNK_HEC_TIMEOUT" "30"}})
+         batch-lines({{- getenv "SC4S_DEST_SPLUNK_HEC_BATCH_LINES" "1000"}})
+         batch-bytes({{- getenv "SC4S_DEST_SPLUNK_HEC_BATCH_BYTES" "4096kb"}})
+         batch-timeout({{- getenv "SC4S_DEST_SPLUNK_HEC_BATCH_TIMEOUT" "1"}})
+         timeout({{- getenv "SC4S_DEST_SPLUNK_HEC_TIMEOUT" "30"}})
          user_agent("sc4s/1.0 (events)")
          user("sc4s")
-         headers("{{getenv "SC4S_DEST_SPLUNK_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}")
-         password("{{getenv "SPLUNK_HEC_TOKEN"}}")
+         headers("{{- getenv "SC4S_DEST_SPLUNK_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}")
+         password("{{- getenv "SPLUNK_HEC_TOKEN"}}")
 
 
-         tls(peer-verify({{getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}})
-{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}}             cipher-suite("{{getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE"}}"){{end}}
-{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION") ""}}             ssl-version("{{getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION"}}"){{end}}
-             ca-file("{{getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}")
-            )
+         tls(peer-verify({{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}})
+         {{- if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}}
+         cipher-suite("{{- getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE"}}")
+         {{- end}}
+         {{- if ne (getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION") ""}}
+         ssl-version("{{- getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION"}}")
+         {{- end}}
+         ca-file("{{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}"))
          body('$(format-json
                  time=$S_UNIXTIME.$S_MSEC
                  host=${HOST}
@@ -27,3 +31,4 @@
                  event=$(template ${fields.sc4s_template} $(template "t_standard"))
                  fields.*)')
         );
+};
diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf b/package/etc/conf.d/destinations/splunk_hec_metrics.conf
diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl
@@ -1,20 +1,25 @@
+destination d_hecmetrics {
     http(
-         url("{{getenv "SPLUNK_HEC_URL"}}")
+         url("{{- getenv "SPLUNK_HEC_URL"}}")
          method("POST")
          batch-lines(50)
          batch-bytes(1024Kb)
          batch-timeout(1)
          timeout(15)
          user_agent("sc4s/1.0 (internal metrics)")
          user("sc4s")
-         headers("{{getenv "SC4S_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}")
-         password("{{getenv "SPLUNK_HEC_TOKEN"}}")
+         headers("{{- getenv "SC4S_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}")
+         password("{{- getenv "SPLUNK_HEC_TOKEN"}}")
          persist-name("splunk_metrics")
 
-         tls(peer-verify({{getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}})
-{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}}             cipher-suite("{{getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE"}}"){{end}}
-{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION") ""}}             ssl-version("{{getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION"}}"){{end}}
-             ca-file("{{getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}")
-            )
+         tls(peer-verify({{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}})
+         {{- if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}}
+         cipher-suite("{{- getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE"}}")
+         {{- end}}
+         {{- if ne (getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION") ""}}
+         ssl-version("{{- getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION"}}")
+         {{- end}}
+         ca-file("{{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}"))
          body('$MESSAGE')
         );
+};
diff --git a/package/etc/conf.d/destinations/stdout.conf b/package/etc/conf.d/destinations/stdout.conf
diff --git a/package/etc/conf.d/destinations/stdout.conf.tmpl b/package/etc/conf.d/destinations/stdout.conf.tmpl
@@ -1 +1,3 @@
-    file("/dev/stdout");
+destination d_stdout {
+    file("/dev/stdout");
+};
diff --git a/.../etc/conf.d/conflib/vendor_cisco/asa.conf → package/etc/conf.d/filters/cisco/asa.conf b/.../etc/conf.d/conflib/vendor_cisco/asa.conf → package/etc/conf.d/filters/cisco/asa.conf
diff --git a/.../etc/conf.d/conflib/vendor_cisco/ios.conf → package/etc/conf.d/filters/cisco/ios.conf b/.../etc/conf.d/conflib/vendor_cisco/ios.conf → package/etc/conf.d/filters/cisco/ios.conf
diff --git a/...tc/conf.d/conflib/vendor_cisco/nx-os.conf → package/etc/conf.d/filters/cisco/nx-os.conf b/...tc/conf.d/conflib/vendor_cisco/nx-os.conf → package/etc/conf.d/filters/cisco/nx-os.conf
diff --git a/...nf.d/conflib/vendor_fortinet/fortios.conf → .../etc/conf.d/filters/fortinet/fortios.conf b/...nf.d/conflib/vendor_fortinet/fortios.conf → .../etc/conf.d/filters/fortinet/fortios.conf
diff --git a/.../conf.d/conflib/vendor_juniper/junos.conf → ...age/etc/conf.d/filters/juniper/junos.conf b/.../conf.d/conflib/vendor_juniper/junos.conf → ...age/etc/conf.d/filters/juniper/junos.conf
diff --git a/...conf.d/conflib/vendor_juniper/legacy.conf → ...ge/etc/conf.d/filters/juniper/legacy.conf b/...conf.d/conflib/vendor_juniper/legacy.conf → ...ge/etc/conf.d/filters/juniper/legacy.conf
diff --git a/.../etc/conf.d/conflib/vendor_pan/panos.conf → ...ge/etc/conf.d/filters/paloalto/panos.conf b/.../etc/conf.d/conflib/vendor_pan/panos.conf → ...ge/etc/conf.d/filters/paloalto/panos.conf
diff --git a/...nflib/vendor_symantec/bluecoat_proxy.conf → ...nf.d/filters/symantec/bluecoat_proxy.conf b/...nflib/vendor_symantec/bluecoat_proxy.conf → ...nf.d/filters/symantec/bluecoat_proxy.conf
diff --git a/package/etc/conf.d/log_paths/internal.conf b/package/etc/conf.d/log_paths/internal.conf
diff --git a/package/etc/conf.d/log_paths/internal.conf.tmpl b/package/etc/conf.d/log_paths/internal.conf.tmpl
@@ -1,3 +1,5 @@
+{{- define "log_path"}}
+log {
     source(s_internal);
 
     if (match("^Log statistics; " value("MESSAGE"))) {
@@ -30,11 +32,14 @@
         destination(d_hecmetrics);  #--HEC--
     } else {
 
-        parser {p_add_context_splunk(key("sc4s_events")); };
 
-        {{ if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes" }}
+        {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes"}}
         destination(d_stdout);
-        {{ end }}
+        {{- end}}
         rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"), template("t_msg_only"))};
+        parser {p_add_context_splunk(key("sc4s_events")); };
         destination(d_hec_internal);
-    };
+    };
+ };
+{{- end}}
+{{- tmpl.Exec "log_path" "yes" }}