Merge branch 'release/0.8.0'

CSVD · Sep 28, 2019 · dfe3193 · dfe3193
2 parents e6e5e34 + 59c0050
commit dfe3193
Show file tree

Hide file tree

Showing 15 changed files with 244 additions and 53 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -30,11 +30,12 @@ services:
         RH_ACTIVATION: ${RH_ACTIVATION}
     hostname: sc4s
     ports:
-      - "514"
-      - "601"
-      - "514/udp"
+      - "514:514"
+      - "601:601"
+      - "514:514/udp"
       - "5000"
-      - "5000/udp"
+      - "5000:5000/udp"
+      - "6514:6514"
     stdin_open: true
     tty: true
     links:
@@ -46,8 +47,11 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
+      - SC4S_SOURCE_TLS_ENABLE=yes
       - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
       - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
+    volumes:
+      - ./tls:/opt/syslog-ng/tls
   splunk:
     image: splunk/splunk:latest
     hostname: splunk

diff --git a/docs/Configuration.md b/docs/Configuration.md
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -0,0 +1,41 @@
+# Global Configuration
+
+| Variable | Values        | Description |
+|----------|---------------|-------------|
+| SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list |
+| SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token |
+
+
+# Splunk HEC destination Configuration
+
+| Variable | Values        | Description |
+|----------|---------------|-------------|
+| SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate |
+| SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE | comma separated list | Open SSL cipher suite list |
+| SC4S_DEST_SPLUNK_HEC_SSL_VERSION |  comma separated list | Open SSL version list |
+| SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file |
+
+# Syslog Source Configuration
+
+| Variable | Values/Default | Description |
+|----------|----------------|-------------|
+| SC4S_SOURCE_TLS_ENABLE | no(default) or yes | Enable a TLS listener on port 6514 |
+| SC4S_SOURCE_TLS_OPTIONS | See openssl | List of SSl/TLS protocol versions to support | 
+| SC4S_SOURCE_TLS_CIPHER_SUITE | See openssl | List of Ciphers to support |
+| SC4S_SOURCE_TCP_MAX_CONNECTIONS | 2000 | Max number of TCP Connections |
+| SC4S_SOURCE_TCP_IW_SIZE | 20000000 | Initial Window size |
+| SC4S_SOURCE_TCP_FETCH_LIMIT | 2000 | Number of events to fetch from server buffer at once |
+| SC4S_SOURCE_UDP_SO_RCVBUFF | 425984 | UDP server buffer size in bytes |
+
+
+# Syslog Source TLS Certificate Configuration
+
+* Create a folder ``/opt/sc4s/tls``
+* Save the server private key in PEM format with NO PASSWORD to ``/opt/sc4s/tls/server.key``
+* Save the server certificate in PEM format to ``/opt/sc4s/tls/server.pem``
+* Add the following line to ``/opt/sc4s/default/env_file``
+
+```dotenv
+SC4S_SOURCE_TLS_ENABLE=yes
+```
+
diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md
@@ -28,11 +28,11 @@ services:
     env_file:
       - /opt/sc4s/env_file
     volumes:
-#Uncomment the following line if overriding index destinations    
-#      - ./context-local/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
-#Uncomment the following lines if using a host or network based filter and log_path
-#      - ./context-local/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv
-#      - ./context-local/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf
+      - /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
+      - /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv
+      - /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf
+#Uncomment the following line if custom TLS certs are provided
+      - /opt/sc4s/tls:/opt/syslog-ng/tls
 
 ```
 
@@ -55,7 +55,7 @@ SPLUNK_METRICS_INDEX=em_metrics
 Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow
 customization of index destinations.
 
-* Create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/scs/``) created in the first step above.  From this directory,
+* Create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above.  From this directory,
 execute the command below to download the index context file:
 
 ```bash
@@ -68,7 +68,7 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas
 Legacy sources and non-standard-compliant source require configuration by source IP or hostname as included in the event. The following steps
 apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. 
 
-* If not already done in the step immediately above, create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``)
+* If not already done in the step immediately above, create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``)
 created in the first step above.  From this directory, execute the commands below to download the vendor context files:
 
 ```bash
@@ -130,11 +130,12 @@ services:
     env_file:
       - /opt/sc4s/env_file
     volumes:
-#Uncomment the following line if overriding index destinations    
-#      - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
-#Uncomment the following lines if using a host or network based filter and log_path
-#      - ./sc4s-juniper/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv
-#      - ./sc4s-juniper/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf
+      - /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
+      - /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv
+      - /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf
+#Uncomment the following line if custom TLS certs are provided
+      - /opt/sc4s/tls:/opt/syslog-ng/tls
+
 ```
 
 * Modify the following file ``/opt/sc4s/default/env_file`` after including the port-specific environment variable(s).

diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md
@@ -56,12 +56,11 @@ services:
     env_file:
       - /opt/sc4s/env_file
     volumes:
-#Uncomment the following line if overriding index destinations    
-#      - ./context-local/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
-#Uncomment the following lines if using a host or network based filter and log_path
-#      - ./context-local/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv
-#      - ./context-local/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf
-
+      - /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
+      - /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv
+      - /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf
+#Uncomment the following line if custom TLS certs are provided
+      - /opt/sc4s/tls:/opt/syslog-ng/tls
 ```
 
 ## Configure the SC4S environment
@@ -84,7 +83,7 @@ SPLUNK_METRICS_INDEX=em_metrics
 Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow
 customization of index destinations.
 
-* Create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above.  From this directory,
+* Create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above.  From this directory,
 execute the command below to download the index context file:
 
 ```bash
@@ -97,7 +96,7 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas
 Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps
 apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. 
 
-* If not already done in the step immediately above, create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``)
+* If not already done in the step immediately above, create a subdirectory called ``default`` in the directory (e.g. ``/opt/sc4s/``)
 created in the first step above.  From this directory, execute the commands below to download the vendor context files:
 
 ```bash
@@ -158,11 +157,11 @@ services:
     env_file:
       - /opt/sc4s/env_file
     volumes:
-#Uncomment the following line if overriding index destinations    
-#      - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
-#Uncomment the following lines if using a host or network based filter and log_path
-#      - ./sc4s-juniper/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv
-#      - ./sc4s-juniper/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf
+      - /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
+      - /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv
+      - /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf
+#Uncomment the following line if custom TLS certs are provided
+      - /opt/sc4s/tls:/opt/syslog-ng/tls
 ```
 
 * Modify the following file ``/opt/sc4s/default/env_file`` 

diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md
@@ -22,22 +22,23 @@ Environment="SC4S_IMAGE=splunk/sc4s:latest"
 
 #Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started
 Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv"
-#Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started
-#Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv"
-#Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf"
+Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv"
+Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf"
+#Uncomment the following line if custom TLS certs are provided
+#Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls"
 
 TimeoutStartSec=0
 Restart=always
 
 ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE
 ExecStartPre=/usr/bin/docker run \
         --env-file=/opt/sc4s/default/env_file \
-        "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \
+        "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" "$SC4S_TLS_DIR" \
         --name SC4S_preflight --rm \
         $SC4S_IMAGE -s
 ExecStart=/usr/bin/docker run -p 514:514 \
         --env-file=/opt/sc4s/default/env_file \
-        "$SC4S_UNIT_SPLUNK_INDEX"  "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \
+        "$SC4S_UNIT_SPLUNK_INDEX"  "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" "$SC4S_TLS_DIR" \
         --name SC4S \
         --rm \
 $SC4S_IMAGE
@@ -122,18 +123,20 @@ Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/s
 #Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started
 #Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv"
 #Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf"
+#Uncomment the following line if custom TLS certs are provided
+#Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls"
 
 TimeoutStartSec=0
 Restart=always
 ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE
 ExecStartPre=/usr/bin/docker run \
         --env-file=/opt/sc4s/default/env_file \
-        "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \
+        "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" "$SC4S_TLS_DIR" \
         --name SC4S_preflight --rm \
         $SC4S_IMAGE -s
 ExecStart=/usr/bin/docker run -p 514:514 -p 5000-5020:5000-5020 \
         --env-file=/opt/sc4s/default/env_file \
-        "$SC4S_UNIT_SPLUNK_INDEX"  "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \
+        "$SC4S_UNIT_SPLUNK_INDEX"  "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" "$SC4S_TLS_DIR" \
         --name SC4S \
         --rm \
 $SC4S_IMAGE

diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md
@@ -25,20 +25,21 @@ Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/s
 #Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started
 #Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv"
 #Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf"
+#Uncomment the following line if custom TLS certs are provided
+#Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls"
 
 TimeoutStartSec=0
 Restart=always
 ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE
 ExecStartPre=/usr/bin/podman run \
         --env-file=/opt/sc4s/default/env_file \
-        "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \
+        "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" "$SC4S_TLS_DIR" \
         --name SC4S_preflight --rm \
         $SC4S_IMAGE -s
 ExecStart=/usr/bin/podman run -p 514:514 \
         --env-file=/opt/sc4s/default/env_file \
-        "$SC4S_UNIT_SPLUNK_INDEX"  "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \
-        --name SC4S \
-        --rm \
+        "$SC4S_UNIT_SPLUNK_INDEX"  "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" "$SC4S_TLS_DIR" \
+        --name SC4S --rm \
 $SC4S_IMAGE
 
 ```
@@ -121,18 +122,20 @@ Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/s
 #Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started
 #Environment="SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv"
 #Environment="SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf"
+#Uncomment the following line if custom TLS certs are provided
+#Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls"
 
 TimeoutStartSec=0
 Restart=always
 ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE
 ExecStartPre=/usr/bin/podman run \
         --env-file=/opt/sc4s/default/env_file \
-        "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \
+        "$SC4S_UNIT_SPLUNK_INDEX" "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" "$SC4S_TLS_DIR" \
         --name SC4S_preflight --rm \
         $SC4S_IMAGE -s
 ExecStart=/usr/bin/podman run -p 514:514 -p 5000-5020:5000-5020 \
         --env-file=/opt/sc4s/default/env_file \
-        "$SC4S_UNIT_SPLUNK_INDEX"  "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" \
+        "$SC4S_UNIT_SPLUNK_INDEX"  "$SC4S_UNIT_VP_CSV" "$SC4S_UNIT_VP_CONF" "$SC4S_TLS_DIR" \
         --name SC4S \
         --rm \
 $SC4S_IMAGE

diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
@@ -1,5 +1,22 @@
 #Troubleshooting 
 
+## General
+
+
+### Verification of TLS Server
+
+To verify the correct configuration of the TLS server use the following command. Replace the IP, FQDN, and port as appropriate
+
+* Docker
+```
+docker run -ti drwetter/testssl.sh --severity MEDIUM --ip 127.0.0.1 selfsigned.example.com:6510
+```
+
+* Podman
+```
+podman run -ti drwetter/testssl.sh --severity MEDIUM --ip 127.0.0.1 selfsigned.example.com:6510
+```
+
 ## Syslog-ng Metrics 
 
 ## Syslog-NG Events

diff --git a/mkdocs.yml b/mkdocs.yml
@@ -4,5 +4,6 @@ nav:
   - Home: 'index.md'
   - Performance: 'performance.md'
   - Getting Started: 'gettingstarted.md'
+  - Configuration: 'configuration.md'
   - Sources: 'sources.md'
-  - Troubleshooting: 'troubleshooting.md'
+  - Troubleshooting: 'troubleshooting.md'
diff --git a/package/Dockerfile b/package/Dockerfile
@@ -67,7 +67,7 @@ RUN cd /tmp ;\
     rm epel-release-latest-7.noarch.rpm ;\
     rpm --import https://packages.confluent.io/rpm/5.2/archive.key ;\
     yum install gcc tzdata libdbi libsecret libxml2 sqlite tcp_wrappers librdkafka \
-    rh-python36 rh-python36-python-tools libcurl ivykis scl-utils tcp_wrappers-libs curl wget -y;\
+    rh-python36 rh-python36-python-tools libcurl ivykis scl-utils tcp_wrappers-libs curl wget openssl -y;\
     echo source scl_source enable rh-python36 >>/etc/profile.d/enablepython36.sh ;\
     source scl_source enable rh-python36
 

diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl
@@ -39,6 +39,30 @@ source s_default-ports {
                 chain-hostnames(off)
                 flags(no-parse)
             );
+
+            {{- if eq (getenv "SC4S_SOURCE_TLS_ENABLE") "yes"}}
+            network(
+                port(6514)
+                transport("tls")
+                ip-protocol(4)
+                max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}})
+                log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}})
+                log-fetch-limit({{getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}})
+                keep-hostname(yes)
+                keep-timestamp(yes)
+                use-dns(no)
+                use-fqdn(no)
+                chain-hostnames(off)
+                flags(no-parse)
+                tls(allow-compress(yes)
+                    key-file("/opt/syslog-ng/tls/server.key")
+                    cert-file("/opt/syslog-ng/tls/server.pem")
+                    ssl-options({{- getenv "SC4S_SOURCE_TLS_OPTIONS" "no-sslv2, no-sslv3, no-tlsv1, no_tls1_1" }})
+                    cipher-suite("{{- getenv "SC4S_SOURCE_TLS_CIPHER_SUITE" "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH" }}")
+                    )
+            );
+
+            {{- end }}
         };
         #TODO: #60 Remove this function with enhancement
         rewrite(set_rfcnonconformant);