From 639f502e6de8c8fea66c1c068b515448fdbf584f Mon Sep 17 00:00:00 2001 From: mbonsack Date: Wed, 27 Nov 2019 20:04:18 -0800 Subject: [PATCH] Docs/fixes nov27 (#207) BYOE fixes add missing python install fix broken wget links Add Docker install guides Add refinements to docs Add manual container launch commands to the troubleshooting guide Fixes issues: #130 #182 #186 #207 #204 #205 #206 #209 --- .circleci/config.yml | 7 +-- CONTRIBUTING.md => docs/CONTRIBUTING.md | 0 LICENSE-BSD2 => docs/LICENSE-BSD2 | 0 LICENSE-CC0 => docs/LICENSE-CC0 | 0 docs/configuration.md | 4 +- docs/gettingstarted.md | 46 +++++++++++++++++-- docs/gettingstarted/byoe-rhel7.md | 22 +++++++-- docs/gettingstarted/docker-swarm-general.md | 21 ++++++++- docs/gettingstarted/docker-swarm-rhel7.md | 2 +- docs/gettingstarted/docker-systemd-general.md | 24 ++++++++-- docs/troubleshooting.md | 16 +++++++ 11 files changed, 123 insertions(+), 19 deletions(-) rename CONTRIBUTING.md => docs/CONTRIBUTING.md (100%) rename LICENSE-BSD2 => docs/LICENSE-BSD2 (100%) rename LICENSE-CC0 => docs/LICENSE-CC0 (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index fda0601..8840fcf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,7 +41,8 @@ jobs: path: package extra_build_args: --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION - - docker/install-goss + - docker/install-goss: + version: v0.3.7 - run: name: Test Docker command: | @@ -49,7 +50,7 @@ jobs: # Don't forget path! export PATH=$PATH:~/bin pushd package - GOSS_FMT_OPTIONS="perfdata verbose" GOSS_FILES_STRATEGY=cp GOSS_OPTS="--max-concurrent=1 --format junit" dgoss run -t $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 3>&1 1>&2 2>&3 > /tmp/goss.tmp || true ; cat /tmp/goss.tmp | grep -v "<" + GOSS_VER=v0.3.7 GOSS_FMT_OPTIONS="perfdata verbose" GOSS_FILES_STRATEGY=cp GOSS_OPTS="--max-concurrent=1 --format junit" dgoss run -t $REGISTRY/$CI_IMAGE:$CIRCLE_SHA1 3>&1 1>&2 2>&3 > /tmp/goss.tmp || true ; cat /tmp/goss.tmp | grep -v "<" popd cat /tmp/goss.tmp | grep "[<\>]" > /tmp/test-results/goss.xml @@ -413,7 +414,7 @@ workflows: requires: - build - test-unit - - test-scan-synk + #- test-scan-synk filters: tags: only: /^\d*\.\d*\.\d*.*$/ diff --git a/CONTRIBUTING.md b/docs/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to docs/CONTRIBUTING.md diff --git a/LICENSE-BSD2 b/docs/LICENSE-BSD2 similarity index 100% rename from LICENSE-BSD2 rename to docs/LICENSE-BSD2 diff --git a/LICENSE-CC0 b/docs/LICENSE-CC0 similarity index 100% rename from LICENSE-CC0 rename to docs/LICENSE-CC0 diff --git a/docs/configuration.md b/docs/configuration.md index 379edf7..5926646 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -95,8 +95,8 @@ which maps to an associated lookup of alternate indexes, sources, or other metad * Get the filter and lookup files ```bash cd /opt/sc4s/default -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/compliance_meta_by_source.conf -sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/compliance_meta_by_source.csv +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context_templates/compliance_meta_by_source.conf +sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context_templates/compliance_meta_by_source.csv ``` * Edit the file ``compliance_meta_by_source.conf`` to supply uniquely named filters to identify events subject to override. * Edit the file ``compliance_meta_by_source.csv`` to supply appropriate the field(s) and values. diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index 887e886..e113881 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -72,12 +72,50 @@ Splunk type. | Container and Orchestration | Notes | |-----------------------------|-------| -| [Podman + systemd single node](gettingstarted/podman-systemd-general.md) | First choice for RedHat 7.x and 8.x, second choice for Debian and Ubuntu (packages provided via PPA) | -| [Docker CE + systemd single node](gettingstarted/docker-systemd-general.md) | First choice for Debian, Ubuntu, and CentOS distributions with limited existing docker experience | -| [Docker CE + Swarm single node](gettingstarted/docker-swarm-general.md) | Option for Debian, Ubuntu, and CentOS desiring swarm orchestration | -| [Docker CE + Swarm single node RHEL 7.7](gettingstarted/docker-swarm-rhel7.md) | Option for RedHat 7.7 desiring swarm orchestration | +| [Podman + systemd single node](gettingstarted/podman-systemd-general.md) | First choice for RedHat 7.x/8.x and CentOS, second choice for Debian and Ubuntu (packages provided via PPA) | +| [Docker CE + systemd single node](gettingstarted/docker-systemd-general.md) | First choice for Debian and Ubuntu; second choice for CentOS for those with limited existing Docker experience | +| [Docker CE + Swarm single node](gettingstarted/docker-swarm-general.md) | Option for Debian, Ubuntu, CentOS, and Desktop Docker desiring Docker Compose or Swarm orchestration | +| [Docker CE + Swarm single node RHEL 7.7](gettingstarted/docker-swarm-rhel7.md) | Option for RedHat 7.7 desiring Docker Compose or Swarm orchestration | | [Bring your own Envionment](gettingstarted/byoe-rhel7.md) | Option for RedHat 7.7 (centos 7) with SC4S configuration without containers | +## Offline Container Installation + +Follow these instructions to "stage" SC4S by downloading the container so that it can be loaded "out of band" on a +host machine, such as an airgapped system, without internet connectivity. + +* Download container image + +``` +sudo wget https://github.com/splunk/splunk-connect-for-syslog/releases/download/latest/oci_container.tar.gz +``` + +* Distribute the container to the airgapped host machine using an appropriate file transfer utility. +* Execute the following command, using docker or podman as appropriate + +``` + load < oci_container.tar.gz +``` + +* Note the container ID of the resultant load + +``` +Loaded image: docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci:90196f77f7525bc55b3b966b5fa1ce74861c0250 +``` + +* Use the container ID to create a local label +``` + tag docker.pkg.github.com/splunk/splunk-connect-for-syslog/ci:90196f77f7525bc55b3b966b5fa1ce74861c0250 sc4slocal:latest +``` + +* Use this local label `sc4slocal:latest` in the relevant unit or yaml file to launch SC4S (see the runtime options +above) by setting the `SC4S_IMAGE` environment variable in the unit file (example below), or the relevant `image:` tag +if using Docker Compose/Swarm. Using this label will cause the runtime to select the locally loaded image, and will not +attempt to obtain the container image via the internet. + +``` +Environment="SC4S_IMAGE=sc4slocal:latest" +``` + # Scale out Additional hosts can be deployed for syslog collection from additional network zones and locations: diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index d970dcd..881f949 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -1,13 +1,19 @@ # SC4S "Bring Your Own Environment" +* FOREWORD: The BYOE SC4S deliverable should be considered as a _secondary_ option for SC4S deployment, and should be +considered only by those with specific needs based on advanced understanding of syslog-ng architectures. The +container deliverable is the preferred deliverable of SC4S for almost all enterprises. If you are simply trying to +"get syslog working", the turnkey, container approach described in the other runtime documents will be the fastest +route to success. + The "Bring Your Own Environment" instructions that follow allow administrators to utilize the SC4S syslog-ng config files directly on the host OS running on a hardware server or virtual machine. Administrators must provide an appropriate host OS as well as an up-to-date syslog-ng installation either built from source (not documented here) or installed from community-built RPMs. Modification of the base configuration will be required for most customer -environments due to enterprise infrastructure variations. +environments due to enterprise infrastructure variations. -* NOTE: Installing or modifying system configurations can have unexpected consequences, and rudimentary linux system -administratrion and syslog-ng configuration experience is assumed. +* NOTE: Installing or modifying system configurations can have unexpected consequences, and advanced linux system +administration and syslog-ng configuration experience is assumed when using the BYOE version of SC4S. * NOTE: Do _not_ depend on the distribution-supplied version of syslog-ng, as it will likely be far too old. Read this [explanation](https://www.syslog-ng.com/community/b/blog/posts/installing-latest-syslog-ng-on-rhel-and-other-rpm-distributions) @@ -68,6 +74,10 @@ sudo chmod 755 /usr/local/bin/gomplate gomplate --help ``` +* Install the latest python + +```scl enable rh-python36 bash``` + * create the sc4s unit file drop in ``/etc/systemd/system/sc4s.service`` and add the following content ```ini @@ -93,7 +103,7 @@ Restart=on-failure WantedBy=multi-user.target ``` -* create the file ``/opt/sc4s/bin/preconfig.sh`` and add the following content +* create the file ``/opt/sc4s/bin/preconfig.sh``. This file should be made executable according to your file permission standards. Add the following content: ```bash #!/usr/bin/env bash @@ -119,7 +129,9 @@ cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/l mkdir -p /opt/syslog-ng/var/data/disk-buffer/ ``` -* Execute the preconfiguration file created above +* (Optional) Execute the preconfiguration shell script created above. You may also optionally execute it as part of the unit +file, which is recommended. If you elect _not_ to execute the script in the unit file, care must be taken to execute it manually "out of band" +when any changes are made. ```bash sudo bash /opt/sc4s/bin/preconfig.sh diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 6fd894f..dfa8b4d 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -1,7 +1,26 @@ # Install Docker CE and Swarm -Refer to [Getting Started](https://docs.docker.com/get-started/) +Refer to relevant installation guides: + +* [CentOS](https://docs.docker.com/install/linux/docker-ce/centos/) +* [Ubuntu](https://docs.docker.com/install/linux/docker-ce/ubuntu/) +* [Debian](https://docs.docker.com/install/linux/docker-ce/debian/) +* [Desktop](https://docs.docker.com/get-started/) + +NOTE: If using a CentOS image provisioned in AWS, IPV4 forwarding is _not_ enabled by default. +This needs to be enabled for container networking to function properly. The following is an example +to set this up; as usual this needs to be vetted with your enterprise security policy: + +```sudo sysctl net.ipv4.ip_forward=1``` + +Then, edit /etc/sysctl.conf, find the text below, and uncomment as shown so that the change made above will survive a +reboot: + +``` +# Uncomment the next line to enable packet forwarding for IPv4 +net.ipv4.ip_forward=1 +``` # SC4S Configuration diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index 5ef0e66..7ed5afe 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -1,7 +1,7 @@ # Install Docker CE and Swarm -*Warning* this method of installing docker on RHEL does not appear to be supported: +* Warning: this method of installing docker on RHEL does not appear to be supported. Consider using podman instead. ## Enable required repositories ```bash diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 94b164f..838cdaa 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -1,7 +1,25 @@ # Install Docker CE -Refer to [Getting Started](https://docs.docker.com/get-started/) +Refer to relevant installation guides: + +* [CentOS](https://docs.docker.com/install/linux/docker-ce/centos/) +* [Ubuntu](https://docs.docker.com/install/linux/docker-ce/ubuntu/) +* [Debian](https://docs.docker.com/install/linux/docker-ce/debian/) + +NOTE: If using a CentOS image provisioned in AWS, IPV4 forwarding is _not_ enabled by default. +This needs to be enabled for container networking to function properly. The following is an example +to set this up; as usual this needs to be vetted with your enterprise security policy: + +```sudo sysctl net.ipv4.ip_forward=1``` + +Then, edit /etc/sysctl.conf, find the text below, and uncomment as shown so that the change made above will survive a +reboot: + +``` +# Uncomment the next line to enable packet forwarding for IPv4 +net.ipv4.ip_forward=1 +``` # Setup @@ -10,8 +28,8 @@ Refer to [Getting Started](https://docs.docker.com/get-started/) ```ini [Unit] Description=SC4S Container -After=network.service -Requires=network.service +Wants=network.target network-online.target +After=network.target network-online.target [Service] Environment="SC4S_IMAGE=splunk/scs:latest" diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 1a78b38..aca10a7 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2,6 +2,22 @@ ## General +To test the container outside of the systemd startup environment, you can run the following to test the syntax +of the container. These commands assume the local mounted directory is set up as shown in the gettingstarted +examples (and omits the disk buffer mount): + +``` +/usr/bin/docker run --env-file=/opt/sc4s/env_file -v "/opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" --name SC4S_preflight --rm splunk/scs:latest -s +``` + +and you can run + +``` +/usr/bin/docker run --env-file=/opt/sc4s/env_file -v "/opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" --name SC4S --rm splunk/scs:latest +``` + +to test the final image. These commands can help with container errors that are hidden in the systemd process. If you +are using podman, substitute "podman" for "docker" for the container runtime command above. ### Verification of TLS Server