Merge branch 'develop' into fix/version

.circleci/config.yml

@@ -76,7 +76,7 @@ jobs:
       - run:
           name: BYOE Config
           command: |
-            tar rvf /tmp/workspace/baremetal.tar -C package etc
+            tar rvf /tmp/workspace/baremetal.tar -C package/etc .
             tar rvf /tmp/workspace/baremetal.tar -C package/sbin entrypoint.sh
 
       - store_test_results:

docker-compose.yml

@@ -13,8 +13,6 @@ services:
     links:
       - splunk
       - sc4s
-    volumes:
-      - sc4s-results:/work/test-results
     environment:
       - SPLUNK_USER=admin
       - SPLUNK_PASSWORD=${SPLUNK_PASSWORD}
@@ -55,6 +53,7 @@ services:
       - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005
       - SC4S_LISTEN_PFSENSE_TCP_PORT=5006
       - SC4S_ARCHIVE_GLOBAL=yes
+#     - SC4S_SOURCE_STORE_RAWMSG=yes
     volumes:
       - ./tls:/opt/syslog-ng/tls
   splunk:
@@ -84,7 +83,5 @@ services:
       - ./replay:/work
 
 volumes:
-  sc4s-results:
-    external: true
   splunk-etc:
     external: true

docs/configuration.md

@@ -73,7 +73,7 @@ and/or move them to an archival system to avoid exhaustion of disk space.
 | SC4S_SOURCE_TCP_IW_SIZE | 20000000 | Initial Window size |
 | SC4S_SOURCE_TCP_FETCH_LIMIT | 2000 | Number of events to fetch from server buffer at once |
 | SC4S_SOURCE_UDP_SO_RCVBUFF | 425984 | UDP server buffer size in bytes |
-
+| SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype.  Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. |
 
 ## Syslog Source TLS Certificate Configuration
 

docs/gettingstarted/byoe-rhel7.md

@@ -21,7 +21,16 @@ for the reason why syslog-ng builds are so dated in almost all RHEL/Debian distr
 
 # BYOE Installation Instructions
 
-* Install CentOS or RHEL 7.7
+These installation instructions assume a recent RHEL or CentOS-based release.  Minor adjustments may have to be made for
+Debian/Ubuntu.  In addition, almost _all_ pre-compiled binaries for syslog-ng assume installation in `etc/syslog-ng`; these instructions
+will reflect that.
+
+The following installation instructions are summarized from a 
+[blog](https://www.syslog-ng.com/community/b/blog/posts/introducing-the-syslog-ng-stable-rpm-repositories)
+maintained by a developer at One Identity (formerly Balabit), who is the owner of the syslog-ng Open Source project.
+It is always adivisable to review the blog for the latest changes to the repo(s), as changes here are quite dynamic.
+
+* Install CentOS or RHEL 7.7/8.0
 
 * Enable EPEL (Centos 7)
 
@@ -55,15 +64,14 @@ sudo systemctl stop syslog-ng
 sudo systemctl disable syslog-ng
 ```        
 
-* Download the latest bare_metal.tar from [releases](https://github.com/splunk/splunk-connect-for-syslog/releases) on github and untar the package
+* Download the latest bare_metal.tar from [releases](https://github.com/splunk/splunk-connect-for-syslog/releases) on github and untar the package in `/etc/syslog-ng`
+
+* NOTE:  The `wget` process below will unpack a tarball with the sc4s version of the syslog-ng config files in the standard
+`/etc/syslog-ng` location, and _will_ overwrite existing content.  Ensure that any previous configurations of syslog-ng are saved
+if needed prior to executing the download step.
 
 ```bash
-cd /tmp
-sudo wget https://github.com/splunk/splunk-connect-for-syslog/releases/download/0.12.1/baremetal.tar
-tar -xf baremetal.tar 
-sudo mkdir -p /opt/syslog-ng/etc
-sudo mkdir -p /opt/syslog-ng/var
-sudo cp -R etc/* /opt/syslog-ng/etc/
+sudo wget -c https://github.com/splunk/splunk-connect-for-syslog/releases/download/latest/baremetal.tar -O - | sudo tar -x -C /etc/syslog-ng
 ```
 
 * Install gomplate and confirm that the version is 3.5.0 or newer 
@@ -78,7 +86,7 @@ gomplate --version
 
 ```scl enable rh-python36 bash```
 
-* create the sc4s unit file drop in ``/etc/systemd/system/sc4s.service`` and add the following content
+* create the sc4s unit file ``/lib/systemd/system/sc4s.service`` and add the following content
 
 ```ini
 [Unit]
@@ -103,13 +111,20 @@ Restart=on-failure
 WantedBy=multi-user.target
 ```
 
-* create the file ``/opt/sc4s/bin/preconfig.sh``.  This file should be made executable according to your file permission standards. Add the following content: 
+* NOTE:  The `wget` process above will download a file called `entrypoint.sh` and place it in `/etc/syslog-ng`.  This is the
+preconfiguration file that is used for the container version of sc4s, and forms the foundation of the BYOE version of the file you will
+create below.  Do _not_ use it verbatim as there are differences between them (most notably the install location).  However, it does include
+the "latest and greatest" updates from the container, and should be used (with appropriate modifications) as the basis of the contents of
+`preconfig.sh` below.
+
+* create the file ``/opt/sc4s/bin/preconfig.sh``.  This file should be made executable according to your file permission standards.
+Add the following content (but be sure to check the note above to ensure the latest updates are included):
 
 ```bash
 #!/usr/bin/env bash
 source scl_source enable rh-python36
 
-cd /opt/syslog-ng
+cd /etc/syslog-ng
 #The following is no longer needed but retained as a comment just in case we run into command line length issues
 #for d in $(find /opt/syslog-ng/etc -type d)
 #do
@@ -120,15 +135,14 @@ cd /opt/syslog-ng
 #    --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\
 #    --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}"
 #done
-gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/
-
-mkdir -p /opt/syslog-ng/etc/conf.d/local/context/
-mkdir -p /opt/syslog-ng/etc/conf.d/local/config/
-cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/
-cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/
-mkdir -p /opt/syslog-ng/var/data/disk-buffer/
-mkdir -p /opt/syslog-ng/var/archive/
-mkdir -p /opt/syslog-ng/tls/
+
+gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=go_templates/
+
+mkdir -p /etc/syslog-ng/conf.d/local/context/
+mkdir -p /etc/syslog-ng/conf.d/local/config/
+cp /etc/syslog-ng/context_templates/* /etc/syslog-ng/conf.d/local/context/
+for file in /etc/syslog-ng/conf.d/local/context/*.example ; do cp -v -n $file ${file%.example}; done
+cp -v -R /etc/syslog-ng/local_config/* /etc/syslog-ng/conf.d/local/config/
 ```
 
 * (Optional) Execute the preconfiguration shell script created above.  You may also optionally execute it as part of the unit
@@ -142,7 +156,7 @@ sudo bash /opt/sc4s/bin/preconfig.sh
 * Create the file ``/opt/sc4s/env_file`` and add the following environment variables:
 
 ```dotenv
-SYSLOGNG_OPTS=-f /opt/syslog-ng/etc/syslog-ng.conf 
+SYSLOGNG_OPTS=-f /etc/syslog-ng/syslog-ng.conf 
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SC4S_DEST_SPLUNK_HEC_WORKERS=6

docs/gettingstarted/docker-swarm-general.md

@@ -55,33 +55,34 @@ services:
       - /opt/sc4s/env_file
     volumes:
       - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z
-      - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z
+      - splunk-sc4s-var:/opt/syslog-ng/var
 # Uncomment the following line if local disk archiving is desired
 #     - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z
 # Uncomment the following line if custom TLS certs are provided
 #     - /opt/sc4s/tls:/opt/syslog-ng/tls:z
 ```
 
+* Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication
+failure to the upstream destination(s).  This will also be used to keep track of the state of syslog-ng between restarts, and in
+particular the state of the disk buffer.  This is a required step.
+```
+sudo docker volume create splunk-sc4s-var
+```
+
 * Create the subdirectory ``/opt/sc4s/local``.  This will be used as a mount point for local overrides and configurations.
 
-    * The empty ``local`` directory created above will populate with templates at the first invocation 
-of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent 
-restarts (i.e. a "no-clobber" copy is performed for any missing files).  _Do not_ change the directory structure of 
+    * The empty ``local`` directory created above will populate with defaults and examples at the first invocation 
+of SC4S for local configurations and context overrides. _Do not_ change the directory structure of 
 the files that are laid down; change (or add) only individual files if desired.  SC4S depends on the directory layout
-to read the local configurations properly.
-
-    * You can back up the contents of this directory elsewhere and return the directory to an empty state
-when a new version of SC4S is released to pick up any new changes provided by Splunk.  Upon a restart,
-the direcory will populate as it did when you first installed SC4S.  Your previous changes can then
-be merged back in and will take effect after another restart.
+to read the local configurations properly.  See the notes below for which files will be preserved on restarts.
 
-* Create the subdirectory ``/opt/sc4s/disk-buffer``.  This will be used as a mount point for local disk buffering
-of events in the event of network failure to the Splunk infrastructure.
+    * In the `local/config` directory, there are example log path files (`lp-example.*`) and a filter (`example.conf`) in the
+appropriate subdirectories.  These should _not_ be used directly, but copied as examples for your own log path development.
+They _will_ get overwritten at each SC4S start.    
 
-    * This directory will populate with the disk buffer files upon SC4S startup.  If SC4S restarts for any reason, a new
-    set of files will be created in addition to the original ones.  _The original ones will not be removed_.
-    If you are sure, after stopping SC4S, that all data has been sent, these files can be removed.  They will be created
-    again upon restart.
+    * In the `local/context` directory, if you change the "non-example" version of a file (e.g. `splunk_index.csv`) the changes
+will be preserved on a restart.  However, the "example" files _themselves_ (e.g. `splunk_index.csv.example`) will be updated
+regularly, and should be used as a template to merge new/changed functionality into existing context files.  
 
 * Create the subdirectory ``/opt/sc4s/archive``.  This will be used as a mount point for local storage of syslog events
 (if the optional mount is uncommented above).  The events will be written in the syslog-ng EWMM format. See the "configuration"
@@ -183,7 +184,7 @@ services:
       - /opt/sc4s/env_file
     volumes:
       - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z
-      - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z
+      - splunk-sc4s-var:/opt/syslog-ng/var
 # Uncomment the following line if local disk archiving is desired
 #     - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z
 # Uncomment the following line if custom TLS certs are provided

docs/gettingstarted/docker-swarm-rhel7.md

@@ -63,34 +63,35 @@ services:
       - /opt/sc4s/env_file
     volumes:
       - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z
-      - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z
+      - splunk-sc4s-var:/opt/syslog-ng/var
 # Uncomment the following line if local disk archiving is desired
 #     - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z
 # Uncomment the following line if custom TLS certs are provided
 #     - /opt/sc4s/tls:/opt/syslog-ng/tls:z
 ```
 
+* Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication
+failure to the upstream destination(s).  This will also be used to keep track of the state of syslog-ng between restarts, and in
+particular the state of the disk buffer.  This is a required step.
+```
+sudo docker volume create splunk-sc4s-var
+```
+
 * Create the subdirectory ``/opt/sc4s/local``.  This will be used as a mount point for local overrides and configurations.
 
-    * The empty ``local`` directory created above will populate with templates at the first invocation 
-of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent 
-restarts (i.e. a "no-clobber" copy is performed for any missing files).  _Do not_ change the directory structure of 
+    * The empty ``local`` directory created above will populate with defaults and examples at the first invocation 
+of SC4S for local configurations and context overrides. _Do not_ change the directory structure of 
 the files that are laid down; change (or add) only individual files if desired.  SC4S depends on the directory layout
-to read the local configurations properly.
+to read the local configurations properly.  See the notes below for which files will be preserved on restarts.
 
-    * You can back up the contents of this directory elsewhere and return the directory to an empty state
-when a new version of SC4S is released to pick up any new changes provided by Splunk.  Upon a restart,
-the direcory will populate as it did when you first installed SC4S.  Your previous changes can then
-be merged back in and will take effect after another restart.
-
-* Create the subdirectory ``/opt/sc4s/disk-buffer``.  This will be used as a mount point for local disk buffering
-of events in the event of network failure to the Splunk infrastructure.
-
-    * This directory will populate with the disk buffer files upon SC4S startup.  If SC4S restarts for any reason, a new
-set of files will be created in addition to the original ones.  _The original ones will not be removed_.
-If you are sure, after stopping SC4S, that all data has been sent, these files can be removed.  They will be created
-again upon restart
+    * In the `local/config` directory, there are example log path files (`lp-example.*`) and a filter (`example.conf`) in the
+appropriate subdirectories.  These should _not_ be used directly, but copied as examples for your own log path development.
+They _will_ get overwritten at each SC4S start.    
 
+    * In the `local/context` directory, if you change the "non-example" version of a file (e.g. `splunk_index.csv`) the changes
+will be preserved on a restart.  However, the "example" files _themselves_ (e.g. `splunk_index.csv.example`) will be updated
+regularly, and should be used as a template to merge new/changed functionality into existing context files.  
+
 * Create the subdirectory ``/opt/sc4s/archive``.  This will be used as a mount point for local storage of syslog events
 (if the optional mount is uncommented above).  The events will be written in the syslog-ng EWMM format. See the "configuration"
 document for details on the directory structure the archive uses.
@@ -191,7 +192,7 @@ services:
       - /opt/sc4s/env_file
     volumes:
       - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z
-      - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z
+      - splunk-sc4s-var:/opt/syslog-ng/var
 # Uncomment the following line if local disk archiving is desired
 #     - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z
 # Uncomment the following line if custom TLS certs are provided