diff --git a/.circleci/config.yml b/.circleci/config.yml index d13a074..353c0ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,13 @@ jobs: - versioning/define_version - versioning/create_version_file: version_file_path: /tmp/workspace/VERSION - + - run: + name: Record version + command: | + CIRCLE_BRANCH_CLEAN=$(echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9\._-]//g') + if [ -n "${CIRCLE_TAG}" ]; then DOCKER_TAG=${CIRCLE_TAG}; else DOCKER_TAG="${CIRCLE_BRANCH_CLEAN}"; fi + if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:6}"; fi + echo $DOCKER_TAG-${CIRCLE_BRANCH_CLEAN}@${CIRCLE_SHA1:0:7}>package/VERSION - docker/install-docker - docker/check: registry: $REGISTRY @@ -70,7 +76,7 @@ jobs: - run: name: BYOE Config command: | - tar rvf /tmp/workspace/baremetal.tar -C package etc + tar rvf /tmp/workspace/baremetal.tar -C package/etc . tar rvf /tmp/workspace/baremetal.tar -C package/sbin entrypoint.sh - store_test_results: diff --git a/docker-compose.yml b/docker-compose.yml index 323da58..81191cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,8 +13,6 @@ services: links: - splunk - sc4s - volumes: - - sc4s-results:/work/test-results environment: - SPLUNK_USER=admin - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} @@ -55,6 +53,7 @@ services: - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 - SC4S_LISTEN_PFSENSE_TCP_PORT=5006 - SC4S_ARCHIVE_GLOBAL=yes +# - SC4S_SOURCE_STORE_RAWMSG=yes volumes: - ./tls:/opt/syslog-ng/tls splunk: @@ -84,7 +83,5 @@ services: - ./replay:/work volumes: - sc4s-results: - external: true splunk-etc: external: true diff --git a/docs/configuration.md b/docs/configuration.md index f19488b..75b7e82 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -73,7 +73,7 @@ and/or move them to an archival system to avoid exhaustion of disk space. | SC4S_SOURCE_TCP_IW_SIZE | 20000000 | Initial Window size | | SC4S_SOURCE_TCP_FETCH_LIMIT | 2000 | Number of events to fetch from server buffer at once | | SC4S_SOURCE_UDP_SO_RCVBUFF | 425984 | UDP server buffer size in bytes | - +| SC4S_SOURCE_STORE_RAWMSG | undefined or "no" | Store unprocessed "on the wire" raw message in the RAWMSG macro for use with the "fallback" sourcetype. Do _not_ set this in production; substantial memory and disk overhead will result. Use for log path/filter development only. | ## Syslog Source TLS Certificate Configuration diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index e91f5b1..544178d 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -21,7 +21,16 @@ for the reason why syslog-ng builds are so dated in almost all RHEL/Debian distr # BYOE Installation Instructions -* Install CentOS or RHEL 7.7 +These installation instructions assume a recent RHEL or CentOS-based release. Minor adjustments may have to be made for +Debian/Ubuntu. In addition, almost _all_ pre-compiled binaries for syslog-ng assume installation in `etc/syslog-ng`; these instructions +will reflect that. + +The following installation instructions are summarized from a +[blog](https://www.syslog-ng.com/community/b/blog/posts/introducing-the-syslog-ng-stable-rpm-repositories) +maintained by a developer at One Identity (formerly Balabit), who is the owner of the syslog-ng Open Source project. +It is always adivisable to review the blog for the latest changes to the repo(s), as changes here are quite dynamic. + +* Install CentOS or RHEL 7.7/8.0 * Enable EPEL (Centos 7) @@ -55,15 +64,14 @@ sudo systemctl stop syslog-ng sudo systemctl disable syslog-ng ``` -* Download the latest bare_metal.tar from [releases](https://github.com/splunk/splunk-connect-for-syslog/releases) on github and untar the package +* Download the latest bare_metal.tar from [releases](https://github.com/splunk/splunk-connect-for-syslog/releases) on github and untar the package in `/etc/syslog-ng` + +* NOTE: The `wget` process below will unpack a tarball with the sc4s version of the syslog-ng config files in the standard +`/etc/syslog-ng` location, and _will_ overwrite existing content. Ensure that any previous configurations of syslog-ng are saved +if needed prior to executing the download step. ```bash -cd /tmp -sudo wget https://github.com/splunk/splunk-connect-for-syslog/releases/download/0.12.1/baremetal.tar -tar -xf baremetal.tar -sudo mkdir -p /opt/syslog-ng/etc -sudo mkdir -p /opt/syslog-ng/var -sudo cp -R etc/* /opt/syslog-ng/etc/ +sudo wget -c https://github.com/splunk/splunk-connect-for-syslog/releases/download/latest/baremetal.tar -O - | sudo tar -x -C /etc/syslog-ng ``` * Install gomplate and confirm that the version is 3.5.0 or newer @@ -78,7 +86,7 @@ gomplate --version ```scl enable rh-python36 bash``` -* create the sc4s unit file drop in ``/etc/systemd/system/sc4s.service`` and add the following content +* create the sc4s unit file ``/lib/systemd/system/sc4s.service`` and add the following content ```ini [Unit] @@ -103,13 +111,20 @@ Restart=on-failure WantedBy=multi-user.target ``` -* create the file ``/opt/sc4s/bin/preconfig.sh``. This file should be made executable according to your file permission standards. Add the following content: +* NOTE: The `wget` process above will download a file called `entrypoint.sh` and place it in `/etc/syslog-ng`. This is the +preconfiguration file that is used for the container version of sc4s, and forms the foundation of the BYOE version of the file you will +create below. Do _not_ use it verbatim as there are differences between them (most notably the install location). However, it does include +the "latest and greatest" updates from the container, and should be used (with appropriate modifications) as the basis of the contents of +`preconfig.sh` below. + +* create the file ``/opt/sc4s/bin/preconfig.sh``. This file should be made executable according to your file permission standards. +Add the following content (but be sure to check the note above to ensure the latest updates are included): ```bash #!/usr/bin/env bash source scl_source enable rh-python36 -cd /opt/syslog-ng +cd /etc/syslog-ng #The following is no longer needed but retained as a comment just in case we run into command line length issues #for d in $(find /opt/syslog-ng/etc -type d) #do @@ -120,15 +135,14 @@ cd /opt/syslog-ng # --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ # --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" #done -gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ - -mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ -mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ -cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ -cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ -mkdir -p /opt/syslog-ng/var/data/disk-buffer/ -mkdir -p /opt/syslog-ng/var/archive/ -mkdir -p /opt/syslog-ng/tls/ + +gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=go_templates/ + +mkdir -p /etc/syslog-ng/conf.d/local/context/ +mkdir -p /etc/syslog-ng/conf.d/local/config/ +cp /etc/syslog-ng/context_templates/* /etc/syslog-ng/conf.d/local/context/ +for file in /etc/syslog-ng/conf.d/local/context/*.example ; do cp -v -n $file ${file%.example}; done +cp -v -R /etc/syslog-ng/local_config/* /etc/syslog-ng/conf.d/local/config/ ``` * (Optional) Execute the preconfiguration shell script created above. You may also optionally execute it as part of the unit @@ -142,7 +156,7 @@ sudo bash /opt/sc4s/bin/preconfig.sh * Create the file ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SYSLOGNG_OPTS=-f /opt/syslog-ng/etc/syslog-ng.conf +SYSLOGNG_OPTS=-f /etc/syslog-ng/syslog-ng.conf SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index a26a1c1..9e1a85f 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -55,33 +55,34 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z - - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z + - splunk-sc4s-var:/opt/syslog-ng/var # Uncomment the following line if local disk archiving is desired # - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z # Uncomment the following line if custom TLS certs are provided # - /opt/sc4s/tls:/opt/syslog-ng/tls:z ``` +* Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication +failure to the upstream destination(s). This will also be used to keep track of the state of syslog-ng between restarts, and in +particular the state of the disk buffer. This is a required step. +``` +sudo docker volume create splunk-sc4s-var +``` + * Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. - * The empty ``local`` directory created above will populate with templates at the first invocation -of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent -restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of + * The empty ``local`` directory created above will populate with defaults and examples at the first invocation +of SC4S for local configurations and context overrides. _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout -to read the local configurations properly. - - * You can back up the contents of this directory elsewhere and return the directory to an empty state -when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, -the direcory will populate as it did when you first installed SC4S. Your previous changes can then -be merged back in and will take effect after another restart. +to read the local configurations properly. See the notes below for which files will be preserved on restarts. -* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering -of events in the event of network failure to the Splunk infrastructure. + * In the `local/config` directory, there are example log path files (`lp-example.*`) and a filter (`example.conf`) in the +appropriate subdirectories. These should _not_ be used directly, but copied as examples for your own log path development. +They _will_ get overwritten at each SC4S start. - * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new - set of files will be created in addition to the original ones. _The original ones will not be removed_. - If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created - again upon restart. + * In the `local/context` directory, if you change the "non-example" version of a file (e.g. `splunk_index.csv`) the changes +will be preserved on a restart. However, the "example" files _themselves_ (e.g. `splunk_index.csv.example`) will be updated +regularly, and should be used as a template to merge new/changed functionality into existing context files. * Create the subdirectory ``/opt/sc4s/archive``. This will be used as a mount point for local storage of syslog events (if the optional mount is uncommented above). The events will be written in the syslog-ng EWMM format. See the "configuration" @@ -183,7 +184,7 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z - - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z + - splunk-sc4s-var:/opt/syslog-ng/var # Uncomment the following line if local disk archiving is desired # - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z # Uncomment the following line if custom TLS certs are provided diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index 251c9fa..c276f64 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -63,34 +63,35 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z - - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z + - splunk-sc4s-var:/opt/syslog-ng/var # Uncomment the following line if local disk archiving is desired # - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z # Uncomment the following line if custom TLS certs are provided # - /opt/sc4s/tls:/opt/syslog-ng/tls:z ``` +* Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication +failure to the upstream destination(s). This will also be used to keep track of the state of syslog-ng between restarts, and in +particular the state of the disk buffer. This is a required step. +``` +sudo docker volume create splunk-sc4s-var +``` + * Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. - * The empty ``local`` directory created above will populate with templates at the first invocation -of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent -restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of + * The empty ``local`` directory created above will populate with defaults and examples at the first invocation +of SC4S for local configurations and context overrides. _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout -to read the local configurations properly. +to read the local configurations properly. See the notes below for which files will be preserved on restarts. - * You can back up the contents of this directory elsewhere and return the directory to an empty state -when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, -the direcory will populate as it did when you first installed SC4S. Your previous changes can then -be merged back in and will take effect after another restart. - -* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering -of events in the event of network failure to the Splunk infrastructure. - - * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new -set of files will be created in addition to the original ones. _The original ones will not be removed_. -If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created -again upon restart + * In the `local/config` directory, there are example log path files (`lp-example.*`) and a filter (`example.conf`) in the +appropriate subdirectories. These should _not_ be used directly, but copied as examples for your own log path development. +They _will_ get overwritten at each SC4S start. + * In the `local/context` directory, if you change the "non-example" version of a file (e.g. `splunk_index.csv`) the changes +will be preserved on a restart. However, the "example" files _themselves_ (e.g. `splunk_index.csv.example`) will be updated +regularly, and should be used as a template to merge new/changed functionality into existing context files. + * Create the subdirectory ``/opt/sc4s/archive``. This will be used as a mount point for local storage of syslog events (if the optional mount is uncommented above). The events will be written in the syslog-ng EWMM format. See the "configuration" document for details on the directory structure the archive uses. @@ -191,7 +192,7 @@ services: - /opt/sc4s/env_file volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z - - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z + - splunk-sc4s-var:/opt/syslog-ng/var # Uncomment the following line if local disk archiving is desired # - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z # Uncomment the following line if custom TLS certs are provided diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 55a4788..2f25337 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -28,21 +28,24 @@ net.ipv4.ip_forward=1 ```ini [Unit] Description=SC4S Container -Wants=network.target network-online.target -After=network.target network-online.target +Wants=NetworkManager.service network-online.target +After=NetworkManager.service network-online.target + +[Install] +WantedBy=multi-user.target [Service] Environment="SC4S_IMAGE=splunk/scs:latest" +# Required mount point for syslog-ng persist data (including disk buffer) +Environment="SC4S_PERSIST_VOLUME=-v splunk-sc4s-var:/opt/syslog-ng/var" + # Optional mount point for local overrides and configurations; see notes in docs Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Optional mount point for local disk archive (EWMM output) files # Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" -# Mount point for local disk buffer (required) -Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" - # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls:z" @@ -57,35 +60,36 @@ ExecStartPre=/usr/bin/docker run \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 \ --env-file=/opt/sc4s/env_file \ + "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ "$SC4S_LOCAL_ARCHIVE_MOUNT" \ "$SC4S_TLS_DIR" \ --name SC4S \ --rm $SC4S_IMAGE ``` +* Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication +failure to the upstream destination(s). This will also be used to keep track of the state of syslog-ng between restarts, and in +particular the state of the disk buffer. This is a required step. +``` +sudo docker volume create splunk-sc4s-var +``` + * Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. - * The empty ``local`` directory created above will populate with templates at the first invocation -of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent -restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of + * The empty ``local`` directory created above will populate with defaults and examples at the first invocation +of SC4S for local configurations and context overrides. _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout -to read the local configurations properly. - - * You can back up the contents of this directory elsewhere and return the directory to an empty state -when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, -the direcory will populate as it did when you first installed SC4S. Your previous changes can then -be merged back in and will take effect after another restart. +to read the local configurations properly. See the notes below for which files will be preserved on restarts. -* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering -of events in the event of network failure to the Splunk infrastructure. - - * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new -set of files will be created in addition to the original ones. _The original ones will not be removed_. -If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created -again upon restart + * In the `local/config` directory, there are example log path files (`lp-example.*`) and a filter (`example.conf`) in the +appropriate subdirectories. These should _not_ be used directly, but copied as examples for your own log path development. +They _will_ get overwritten at each SC4S start. + * In the `local/context` directory, if you change the "non-example" version of a file (e.g. `splunk_index.csv`) the changes +will be preserved on a restart. However, the "example" files _themselves_ (e.g. `splunk_index.csv.example`) will be updated +regularly, and should be used as a template to merge new/changed functionality into existing context files. + * Create the subdirectory ``/opt/sc4s/archive``. This will be used as a mount point for local storage of syslog events (if the optional mount is uncommented above). The events will be written in the syslog-ng EWMM format. See the "configuration" document for details on the directory structure the archive uses. @@ -149,26 +153,30 @@ documentation to identify the specific environment variables that are mapped to ```ini [Unit] Description=SC4S Container -After=network.service -Requires=network.service +Wants=NetworkManager.service network-online.target +After=NetworkManager.service network-online.target + +[Install] +WantedBy=multi-user.target [Service] Environment="SC4S_IMAGE=splunk/scs:latest" +# Required mount point for syslog-ng persist data (including disk buffer) +Environment="SC4S_PERSIST_VOLUME=-v splunk-sc4s-var:/opt/syslog-ng/var" + # Optional mount point for local overrides and configurations; see notes in docs Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Optional mount point for local disk archive (EWMM output) files # Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" -# Mount point for local disk buffer (required) -Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" - # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" TimeoutStartSec=0 Restart=always + ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE ExecStartPre=/usr/bin/docker run \ --env-file=/opt/sc4s/env_file \ @@ -177,8 +185,8 @@ ExecStartPre=/usr/bin/docker run \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/env_file \ + "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ "$SC4S_LOCAL_ARCHIVE_MOUNT" \ "$SC4S_TLS_DIR" \ --name SC4S \ @@ -283,4 +291,5 @@ Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', cl ``` If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more -information. \ No newline at end of file +information. + diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 7b49762..f38c5b3 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -10,21 +10,24 @@ Refer to [Installation](https://podman.io/getting-started/installation) ```ini [Unit] Description=SC4S Container -Wants=network.target network-online.target -After=network.target network-online.target +Wants=NetworkManager.service network-online.target +After=NetworkManager.service network-online.target + +[Install] +WantedBy=multi-user.target [Service] Environment="SC4S_IMAGE=splunk/scs:latest" +# Required mount point for syslog-ng persist data (including disk buffer) +Environment="SC4S_PERSIST_VOLUME=-v splunk-sc4s-var:/opt/syslog-ng/var" + # Optional mount point for local overrides and configurations; see notes in docs Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Optional mount point for local disk archive (EWMM output) files # Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" -# Mount point for local disk buffer (required) -Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" - # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls:z" @@ -39,35 +42,36 @@ ExecStartPre=/usr/bin/podman run \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ --env-file=/opt/sc4s/env_file \ + "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ "$SC4S_LOCAL_ARCHIVE_MOUNT" \ "$SC4S_TLS_DIR" \ --name SC4S \ --rm $SC4S_IMAGE ``` +* Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication +failure to the upstream destination(s). This will also be used to keep track of the state of syslog-ng between restarts, and in +particular the state of the disk buffer. This is a required step. +``` +sudo podman volume create splunk-sc4s-var +``` + * Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. - * The empty ``local`` directory created above will populate with templates at the first invocation -of SC4S for local configurations and overrides. Changes made to these files will be preserved on subsequent -restarts (i.e. a "no-clobber" copy is performed for any missing files). _Do not_ change the directory structure of + * The empty ``local`` directory created above will populate with defaults and examples at the first invocation +of SC4S for local configurations and context overrides. _Do not_ change the directory structure of the files that are laid down; change (or add) only individual files if desired. SC4S depends on the directory layout -to read the local configurations properly. +to read the local configurations properly. See the notes below for which files will be preserved on restarts. - * You can back up the contents of this directory elsewhere and return the directory to an empty state -when a new version of SC4S is released to pick up any new changes provided by Splunk. Upon a restart, -the direcory will populate as it did when you first installed SC4S. Your previous changes can then -be merged back in and will take effect after another restart. - -* Create the subdirectory ``/opt/sc4s/disk-buffer``. This will be used as a mount point for local disk buffering -of events in the event of network failure to the Splunk infrastructure. - - * This directory will populate with the disk buffer files upon SC4S startup. If SC4S restarts for any reason, a new -set of files will be created in addition to the original ones. _The original ones will not be removed_. -If you are sure, after stopping SC4S, that all data has been sent, these files can be removed. They will be created -again upon restart + * In the `local/config` directory, there are example log path files (`lp-example.*`) and a filter (`example.conf`) in the +appropriate subdirectories. These should _not_ be used directly, but copied as examples for your own log path development. +They _will_ get overwritten at each SC4S start. + * In the `local/context` directory, if you change the "non-example" version of a file (e.g. `splunk_index.csv`) the changes +will be preserved on a restart. However, the "example" files _themselves_ (e.g. `splunk_index.csv.example`) will be updated +regularly, and should be used as a template to merge new/changed functionality into existing context files. + * Create the subdirectory ``/opt/sc4s/archive``. This will be used as a mount point for local storage of syslog events (if the optional mount is uncommented above). The events will be written in the syslog-ng EWMM format. See the "configuration" document for details on the directory structure the archive uses. @@ -131,26 +135,30 @@ documentation to identify the specific environment variables that are mapped to ```ini [Unit] Description=SC4S Container -After=network.service -Requires=network.service +Wants=NetworkManager.service network-online.target +After=NetworkManager.service network-online.target + +[Install] +WantedBy=multi-user.target [Service] Environment="SC4S_IMAGE=splunk/scs:latest" +# Required mount point for syslog-ng persist data (including disk buffer) +Environment="SC4S_PERSIST_VOLUME=-v splunk-sc4s-var:/opt/syslog-ng/var" + # Optional mount point for local overrides and configurations; see notes in docs Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Optional mount point for local disk archive (EWMM output) files # Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" -# Mount point for local disk buffer (required) -Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" - # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" TimeoutStartSec=0 Restart=always + ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE ExecStartPre=/usr/bin/podman run \ --env-file=/opt/sc4s/env_file \ @@ -159,8 +167,8 @@ ExecStartPre=/usr/bin/podman run \ --rm $SC4S_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/env_file \ + "$SC4S_PERSIST_VOLUME" \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ "$SC4S_LOCAL_ARCHIVE_MOUNT" \ "$SC4S_TLS_DIR" \ --name SC4S \ @@ -266,3 +274,61 @@ Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', cl If you see http server errors such as 4xx or 5xx responses from the http (HEC) endpoint, one or more of the items above are likely set incorrectly. If validating/fixing the configuration fails to correct the problem, proceed to the "Troubleshooting" section for more information. + +# SC4S non-root operation + +To operate SC4S as a user other than root, follow the instructions above, with these modifications: + +## Prepare sc4s user + +Create a non-root user in which to run SC4S and prepare podman for non-root operation: + +```bash +sudo useradd -m -d /home/sc4s -s /bin/bash sc4s +sudo su - sc4s +mkdir -p /home/sc4s/local +mkdir -p /home/sc4s/archive +mkdir -p /home/sc4s/tls +podman system migrate +``` + +## Initial Setup + +NOTE: Be sure to exectute all instructions below as the SC4S user created above with the exception of changes to the unit file, +which requires sudo access. + +Make the following changes to the unit file(s) configured in the main section: + +* Add the name of the user created above immediately after the Service declaration, as shown in the snippet below: + +``` +[Service] +User=sc4s +``` + +* Replace all references to `/opt/sc4s` in the "Environment" declarations with `/home/sc4s`. Make sure _not_ to change the +right-hand-side of the mount. For example: + +``` +Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /home/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" +``` + +* Replace all references to standard UDP/TCP outside listening ports (typically 514) on the _left hand side only_ of the port pairs +with arbirtrary high-numbered (> 1024) ports so that the container can listen without root privleges. The right hand side of the pairs +(also typically 514) should remain unchanged: + +``` +ExecStart=/usr/bin/podman run -p 2514:514 -p 2514:514/udp -p 6514:6514 +``` + +If not done in the "Prepare SC4S user" above, create the three local mount directories as instructed in the main instructions, +replacing the head of the directory (`/opt/sc4s`) with the sc4s service user's home directory as shown below: +``` +mkdir /home/sc4s/local +mkdir /home/sc4s/archive +mkdir /home/sc4s/tls +``` + +## Remaining Setup + +The remainder of the setup can be followed directly from the main setup instructions. diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md index 7a9e54f..9463ae5 100644 --- a/docs/sources/VMWare/index.md +++ b/docs/sources/VMWare/index.md @@ -1,6 +1,6 @@ -# Vendor - Dell - VMWare +# Vendor - Dell - VMware -## Product - vSphwere - ESX NSX (Controller, Manager, Edge) +## Product - vSphere - ESX NSX (Controller, Manager, Edge) | Ref | Link | diff --git a/docs/upgrade.md b/docs/upgrade.md new file mode 100644 index 0000000..2491f14 --- /dev/null +++ b/docs/upgrade.md @@ -0,0 +1,15 @@ +# Upgrading Splunk Connect for Syslog + +Splunk Connect for Syslog is updated regularly using a CI/CD development process. The notes below outline significant changes that +must be taken into account prior and after an upgrade. Ensure to follow specific instructions below to ensure a smooth transition to +a new version of SC4S in production. + +## Version 1.9.0 + +* Example context files have been added to the local mount `context` directory. These example files will be updated at each release +to outline support for new data sources, which can be added to existing context files (those without the `.example` extension). +Existing context files will _not_ be overwritten on subsequent SC4S starts/upgrades, so ensure that any new content from these example +files is incorporated into existing context files. + +* UNIT FILE CHANGES: Make sure to update the unit file used to start the sc4s service with the changes included in this release. It +includes updates for proper operation with RHEL 8, and is backward-compatible with RHEL 7.7. diff --git a/mkdocs.yml b/mkdocs.yml index 869a54c..8f18791 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,10 +27,12 @@ nav: - Proofpoint: sources/Proofpoint/index.md - Symantec: sources/Symantec/index.md - Ubiquiti: sources/Ubiquiti/index.md + - VMware: sources/VMware/index.md - Zscaler: sources/Zscaler/index.md - 'Demo Lab': 'demo.md' - Performance: 'performance.md' - Troubleshooting: 'troubleshooting.md' + - 'Upgrading SC4S': 'upgrade.md' markdown_extensions: - toc: diff --git a/package/Dockerfile b/package/Dockerfile index 22dbf53..75a9c62 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -77,6 +77,8 @@ COPY sbin/entrypoint.sh / RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V +COPY VERSION / + EXPOSE 514 EXPOSE 601/tcp EXPOSE 6514/tcp diff --git a/package/VERSION b/package/VERSION new file mode 100644 index 0000000..fedd7e5 --- /dev/null +++ b/package/VERSION @@ -0,0 +1 @@ +UNKNOWN \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index 46b63b2..5746a1c 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -61,6 +61,7 @@ template t_JSON_3164 { template('$(format-json --scope rfc3164 --pair PRI="<$PRI>" --key LEGACY_MSGHDR + --key RAWMSG --exclude FACILITY --exclude PRIORITY )'); diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index eb19250..68320bc 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -25,7 +25,6 @@ destination d_hec { reliable(no) {{- end}} disk-buf-size({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "53687091200") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) - dir("/opt/syslog-ng/var/data/disk-buffer/") ) {{- end}} tls(peer-verify({{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}}) diff --git a/package/etc/conf.d/filters/cisco/asa.conf b/package/etc/conf.d/filters/cisco/asa.conf index bbd3157..a7ac9b7 100644 --- a/package/etc/conf.d/filters/cisco/asa.conf +++ b/package/etc/conf.d/filters/cisco/asa.conf @@ -1,3 +1,8 @@ filter f_cisco_asa { - message('^%ASA-\d+-\d{1,10}: '); -}; \ No newline at end of file + message('^%ASA-\d+-\d{1,10}: ') or + match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR")); +}; + +filter f_cisco_asa_nohost { + match('^%ASA-\d+-\d{1,10}:', value("LEGACY_MSGHDR")); +}; diff --git a/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl index 0749e68..a8bf9c5 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl @@ -27,7 +27,11 @@ log { }; parser {p_add_context_splunk(key("cisco_asa")); }; parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + if (filter (f_cisco_asa_nohost)) { + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); }; + } else { + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CISCO_ASA_HEC" "no")) }} destination(d_hec); diff --git a/package/etc/context_templates/compliance_meta_by_source.conf b/package/etc/context_templates/compliance_meta_by_source.conf.example similarity index 100% rename from package/etc/context_templates/compliance_meta_by_source.conf rename to package/etc/context_templates/compliance_meta_by_source.conf.example diff --git a/package/etc/context_templates/compliance_meta_by_source.csv b/package/etc/context_templates/compliance_meta_by_source.csv.example similarity index 100% rename from package/etc/context_templates/compliance_meta_by_source.csv rename to package/etc/context_templates/compliance_meta_by_source.csv.example diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv.example similarity index 100% rename from package/etc/context_templates/splunk_index.csv rename to package/etc/context_templates/splunk_index.csv.example diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf.example similarity index 100% rename from package/etc/context_templates/vendor_product_by_source.conf rename to package/etc/context_templates/vendor_product_by_source.conf.example diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv.example similarity index 100% rename from package/etc/context_templates/vendor_product_by_source.csv rename to package/etc/context_templates/vendor_product_by_source.csv.example diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 7b201eb..a49e89b 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -60,14 +60,14 @@ source s_{{ .port_id }} { }; {{ if eq .parser "rfc3164" }} parser { - syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})); }; rewrite(set_rfc3164); {{ else if eq .parser "rfc3164_version" }} # filter(f_rfc3164_version); rewrite(set_rfc3164_no_version_string); parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})); }; rewrite(set_rfc3164_version); {{ else if eq .parser "rfc5424_strict" }} @@ -104,7 +104,7 @@ source s_{{ .port_id }} { filter(f_rfc3164_version); rewrite(set_rfc3164_no_version_string); parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})); }; rewrite(set_rfc3164_version); } elif { @@ -118,7 +118,7 @@ source s_{{ .port_id }} { rewrite(set_cisco_ios); } else { parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})); }; rewrite(set_rfc3164); }; diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 229a384..66bdb51 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash -source scl_source enable rh-python36 + +# The follwoing will be addressed in a future release +# source scl_source enable rh-python36 # The MICROFOCUS_ARCSIGHT unique port environment variables are currently deprecated # This will be removed when the MICROFOCUS_ARCSIGHT unique port environment variables are removed in version 2.0 @@ -15,11 +17,14 @@ gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl -- mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ -cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ +cp /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ +for file in /opt/syslog-ng/etc/conf.d/local/context/*.example ; do cp --verbose -n $file ${file%.example}; done cp --verbose -R /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ echo syslog-ng checking config -/opt/syslog-ng/sbin/syslog-ng -s >/var/log/syslog-ng.out 2>/var/log/syslog-ng.err +echo sc4s version=$(cat /VERSION) +echo sc4s version=$(cat /VERSION) >/var/log/syslog-ng.out +/opt/syslog-ng/sbin/syslog-ng -s >>/var/log/syslog-ng.out 2>/var/log/syslog-ng.err echo syslog-ng starting -exec /opt/syslog-ng/sbin/syslog-ng $@ \ No newline at end of file +exec /opt/syslog-ng/sbin/syslog-ng $@ diff --git a/tests/test_cisco_asa.py b/tests/test_cisco_asa.py index 1399ac8..6bcd920 100644 --- a/tests/test_cisco_asa.py +++ b/tests/test_cisco_asa.py @@ -36,6 +36,27 @@ def test_cisco_asa_traditional(record_property, setup_wordlist, setup_splunk): assert resultCount == 1 +# <164>Jan 31 2020 17:24:03: %ASA-4-402119: IPSEC: Received an ESP packet (SPI= 0x0C190BF9, sequence number= 0x598243) from 192.0.0.1 (user= 192.0.0.1) to 192.0.0.2 that failed anti-replay checking. +def test_cisco_asa_traditional_nohost(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %}: %ASA-4-402119: IPSEC: Received an ESP packet (SPI= 0x0C190BF9, sequence number= 0x598243) from {host} (user= 192.0.0.1) to 192.0.0.2 that failed anti-replay checking.\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw sourcetype=\"cisco:asa\" \"%ASA-4-402119\" \"{host}\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + # <166>2018-06-27T12:17:46Z asa : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 def test_cisco_asa_rfc5424(record_property, setup_wordlist, setup_splunk): diff --git a/tests/test_common.py b/tests/test_common.py index eecec2d..11912b5 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -172,3 +172,14 @@ def test_check_config_version_multiple(record_property, setup_wordlist, setup_sp record_property("resultCount", resultCount) assert resultCount == 0 + +def test_check_sc4s_version(record_property, setup_wordlist, setup_splunk): + + st = env.from_string("search index=main sourcetype=\"sc4s:events:startup:out\" \"sc4s version=\" NOT \"UNKNOWN\"") + search = st.render() + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("resultCount", resultCount) + + assert resultCount == 0 \ No newline at end of file