diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a82547..d482dff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -168,13 +168,13 @@ jobs: path: /clair-reports publish-common: + machine: + image: ubuntu-1604:201903-01 + docker_layer_caching: true # default - false environment: IMAGE_NAME: rfaircloth/scs - docker: - - image: circleci/buildpack-deps:stretch steps: - - setup_remote_docker: - docker_layer_caching: true + - checkout - run: name: Docker Login command: docker login -u $DOCKER_USER -p $DOCKER_PASS @@ -183,10 +183,14 @@ jobs: command: docker pull $IMAGE_NAME:$CIRCLE_SHA1 - run: name: Docker tag image - command: docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$CIRCLE_BRANCH + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker tag $IMAGE_NAME:$CIRCLE_SHA1 $IMAGE_NAME:$SEMVER - run: name: Docker push tag - command: docker push $IMAGE_NAME:$CIRCLE_BRANCH + command: | + SEMVER=$(docker run --rm -v "$(pwd):/repo" gittools/gitversion:latest-linux-netcoreapp2.1 /repo /showvariable SemVer /nofetch) + docker push $IMAGE_NAME:$SEMVER publish-edge: environment: @@ -273,11 +277,11 @@ workflows: requires: - dgoss - test-unit - filters: - branches: - only: - - master - - develop +# filters: +# branches: +# only: +# - master +# - develop - publish-edge: requires: - dgoss diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index 987fbfa..0c983a3 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -25,8 +25,8 @@ services: - "514" - "601" - "514/udp" - - "5514" - - "5514/udp" + - "5000" + - "5000/udp" stdin_open: true tty: true links: @@ -38,23 +38,8 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - sc4s-juniper: - image: rfaircloth/scs:${CIRCLE_SHA1} - hostname: sc4s-juniper - ports: - - "514" - stdin_open: true - tty: true - links: - - splunk - environment: - - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - - SYSLOG_PRESUME_FILTER=juniper_netscreen + - SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 splunk: image: splunk/splunk:latest hostname: splunk diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml index 55cb296..ecb8f6a 100644 --- a/docker-compose-debug.yml +++ b/docker-compose-debug.yml @@ -38,6 +38,7 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} + - SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no splunk: image: splunk/splunk:latest hostname: splunk diff --git a/docker-compose-demo.yml b/docker-compose-demo.yml index b15d755..b696193 100644 --- a/docker-compose-demo.yml +++ b/docker-compose-demo.yml @@ -40,23 +40,8 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - sc4s-juniper: - image: splunk/scs:latest - hostname: sc4s-juniper - ports: - - "514" - stdin_open: true - tty: true - links: - - splunk - environment: - - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - - SYSLOG_PRESUME_FILTER=juniper_netscreen + - SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 splunk: image: splunk/splunk:latest hostname: splunk diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml index 67f3844..729f51b 100644 --- a/docker-compose-perf.yml +++ b/docker-compose-perf.yml @@ -31,14 +31,9 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} -# logging: -# driver: splunk -# options: -# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94 -# splunk-url: https://splunk:8088/services/collector/event -# splunk-index: main -# splunk-insecureskipverify: true -# splunk-verify-connection: false + - SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 + splunk: image: splunk/splunk:latest hostname: splunk @@ -53,14 +48,6 @@ services: - SPLUNK_APPS_URL=${SPLUNK_APPS_URL} - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME} - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} -# logging: -# driver: splunk -# options: -# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94 -# splunk-url: https://splunk:8088/services/collector/event -# splunk-index: main -# splunk-insecureskipverify: true -# splunk-verify-connection: false egbundles: image: rfaircloth/scs:egb-edge hostname: egbundles diff --git a/docker-compose.yml b/docker-compose.yml index 7493bab..fcf5e48 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,6 @@ services: links: - splunk - sc4s - - sc4s-juniper volumes: - sc4s-results:/work/test-results environment: @@ -34,8 +33,8 @@ services: - "514" - "601" - "514/udp" - - "5514" - - "5514/udp" + - "5000" + - "5000/udp" stdin_open: true tty: true links: @@ -47,28 +46,8 @@ services: - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - sc4s-juniper: - image: splunk/scs:latest - build: - context: ./package - args: - RH_ORG: ${RH_ORG} - RH_ACTIVATION: ${RH_ACTIVATION} - hostname: sc4s-juniper - ports: - - "514" - stdin_open: true - tty: true - links: - - splunk - environment: - - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - - SYSLOG_PRESUME_FILTER=juniper_netscreen + - SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 splunk: image: splunk/splunk:latest hostname: splunk diff --git a/docs/Configuration.md b/docs/Configuration.md new file mode 100644 index 0000000..c4f2903 --- /dev/null +++ b/docs/Configuration.md @@ -0,0 +1,9 @@ + +| Variable | Values | Description | +|----------|---------------|-------------| +| SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list | +| SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token | +| SCS_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | +| SCS_DEST_SPLUNK_HEC_CIPHER_SUITE | comma separated list | Open SSL cipher suite list | +| SCS_DEST_SPLUNK_HEC_SSL_VERSION | comma separated list | Open SSL version list | +| SCS_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index 90917fe..351786f 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -86,32 +86,3 @@ Additional hosts can be deployed for syslog collection from additional network z ![SC4S deployment diagram](SC4S%20deployment.png) -## Single Source Technology instance - Alpha - -For certain source technologies message categorization by content is impossible to support collection -of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using -an alternate port. In the following configration example a dedicated port is opened (6514) for legacy juniper netscreen devices - -This approach is "alpha" and subject to change - -```yaml -version: "3" -services: - sc4s-juniper-netscreen: - image: splunk/scs:latest - hostname: sc4s-juniper-netscreen - ports: - - "6514:514" - - "6514:514/udp" - stdin_open: true - tty: true - environment: - - SPLUNK_HEC_URL=https://foo:8088/services/collector/event - - SPLUNK_HEC_TOKEN= - - SPLUNK_CONNECT_METHOD=hec - - SPLUNK_DEFAULT_INDEX= - - SPLUNK_METRICS_INDEX=em_metrics - - SYSLOG_PRESUME_FILTER=f_juniper_netscreen - volumes: - - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv -``` \ No newline at end of file diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 24861a0..0bad67a 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -25,12 +25,8 @@ services: protocol: udp #Comment the following line out if using docker-compose mode: host - environment: - - SPLUNK_HEC_URL=https://inputs-hec.kops.spl.guru/services/collector/event - - SPLUNK_HEC_TOKEN=02450979-d363-4e6c-b6c9-796d8b546a6e - - SPLUNK_CONNECT_METHOD=hec - - SPLUNK_DEFAULT_INDEX=main - - SPLUNK_METRICS_INDEX=em_metrics + env_file: + - /opt/scs/env_file volumes: #Uncomment the following line if overriding index destinations # - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv @@ -40,6 +36,20 @@ services: ``` +## Configure the SCS environment + +Create the following file ``/opt/scs/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +``` + ## Configure index destinations for Splunk Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow customization of index destinations. @@ -74,38 +84,69 @@ docker stack deploy --compose-file docker-compose.yml sc4s Additional hosts can be deployed for syslog collection from additional network zones and locations. -## Single Source Technology instance - Alpha +# Single Source Technology instance -For certain source technologies message categorization by content is impossible. To support collection -of such legacy nonstandard sources, we provide a means of dedicating a container to a specific source using -an alternate port. In the following configration example a dedicated port is opened (6514) for legacy juniper netscreen devices. +For certain source technologies message categorization by content is impossible to support collection +of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using +an alternate port. +Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. -This approach is "alpha" and subject to change. +In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology specific ports modify the range as appropriate +* Modify the unit file ``/opt/scs/docker-compose.yml`` ```yaml -version: "3" +version: "3.7" services: - sc4s-juniper-netscreen: + sc4s: image: splunk/scs:latest - hostname: sc4s-juniper-netscreen ports: - target: 514 - published: 6514 + published: 514 protocol: tcp #Comment the following line out if using docker-compose mode: host - target: 514 - published: 6514 + published: 514 + protocol: udp +#Comment the following line out if using docker-compose + mode: host + - target: 5000-5021 + published: 5000-5021 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host + - target: 5000-5021 + published: 5000-5021 protocol: udp #Comment the following line out if using docker-compose mode: host - environment: - - SPLUNK_HEC_URL=https://foo:8088/services/collector/event - - SPLUNK_HEC_TOKEN= - - SPLUNK_CONNECT_METHOD=hec - - SPLUNK_DEFAULT_INDEX= - - SPLUNK_METRICS_INDEX=em_metrics - - SYSLOG_PRESUME_FILTER=f_juniper_netscreen + env_file: + - /opt/scs/env_file volumes: - - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv +#Uncomment the following line if overriding index destinations +# - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv +#Uncomment the following lines if using a host or network based filter and log_path +# - ./sc4s-juniper/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv +# - ./sc4s-juniper/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf +``` + +Modify the following file ``/opt/scs/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 +#Uncomment the following line if using untrusted SSL certificates +#SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no +``` + +* Start SC4S. + +```bash +docker stack deploy --compose-file docker-compose.yml sc4s ``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index ebabb18..8575a0f 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -53,12 +53,8 @@ services: protocol: udp #Comment the following line out if using docker-compose mode: host - environment: - - SPLUNK_HEC_URL=https://inputs-hec.kops.spl.guru/services/collector/event - - SPLUNK_HEC_TOKEN=02450979-d363-4e6c-b6c9-796d8b546a6e - - SPLUNK_CONNECT_METHOD=hec - - SPLUNK_DEFAULT_INDEX=main - - SPLUNK_METRICS_INDEX=em_metrics + env_file: + - /opt/scs/env_file volumes: #Uncomment the following line if overriding index destinations # - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv @@ -68,6 +64,21 @@ services: ``` +## Configure the SCS environment + +Create the following file ``/opt/scs/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +``` + + ## Configure index destinations for Splunk Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow customization of index destinations. @@ -101,36 +112,68 @@ sudo docker stack deploy --compose-file docker-compose.yml sc4s Additional hosts can be deployed for syslog collection from additional network zones and locations. -## Single Source Technology instance - Alpha +# Single Source Technology instance -For certain source technologies message categorization by content is impossible. To support collection of such legacy nonstandard sources, we provide a means of dedicating a container to a specific source using an alternate port. In the following configration example a dedicated port is opened (6514) for legacy juniper netscreen devices. +For certain source technologies message categorization by content is impossible to support collection +of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using +an alternate port. +Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. -This approach is "alpha" and subject to change. +In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology specific ports modify the range as appropriate +* Modify the unit file ``/opt/scs/docker-compose.yml`` ```yaml -version: "3" +version: "3.7" services: - sc4s-juniper-netscreen: + sc4s: image: splunk/scs:latest - hostname: sc4s-juniper-netscreen ports: - target: 514 - published: 6514 + published: 514 protocol: tcp #Comment the following line out if using docker-compose mode: host - target: 514 - published: 6514 + published: 514 protocol: udp #Comment the following line out if using docker-compose mode: host - environment: - - SPLUNK_HEC_URL=https://foo:8088/services/collector/event - - SPLUNK_HEC_TOKEN= - - SPLUNK_CONNECT_METHOD=hec - - SPLUNK_DEFAULT_INDEX= - - SPLUNK_METRICS_INDEX=em_metrics - - SYSLOG_PRESUME_FILTER=f_juniper_netscreen + - target: 5000-5021 + published: 5000-5021 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host + - target: 5000-5021 + published: 5000-5021 + protocol: udp +#Comment the following line out if using docker-compose + mode: host + env_file: + - /opt/scs/env_file volumes: - - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv +#Uncomment the following line if overriding index destinations +# - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv +#Uncomment the following lines if using a host or network based filter and log_path +# - ./sc4s-juniper/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv +# - ./sc4s-juniper/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf +``` + +Modify the following file ``/opt/scs/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 +#Uncomment the following line if using untrusted SSL certificates +#SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no +``` +* Start SC4S. + +```bash +docker stack deploy --compose-file docker-compose.yml sc4s ``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index eb63866..cb11a5c 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -13,29 +13,49 @@ Refer to [Getting Started](https://docs.docker.com/get-started/) ```ini [Unit] -Description=Splunk Container -After=docker.service -Requires=docker.service +Description=SCS Container +After=network.service +Requires=network.service [Service] +Environment="SCS_IMAGE=splunk/scs:latest" + +#Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started +Environment="SCS_UNIT_SPLUNK_INDEX=-v /opt/scs/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" +#Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started +#Environment="SCS_UNIT_VP_CSV=-v /opt/scs/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" +#Environment="SCS_UNIT_VP_CONF=-v /opt/scs/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" + TimeoutStartSec=0 Restart=always -ExecStartPre=/usr/bin/docker pull splunk/scs:latest -ExecStart=/usr/bin/docker run -p 514:514\ - -e "SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event" \ - -e "SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94" \ - -e "SPLUNK_CONNECT_METHOD=hec" \ - -e "SPLUNK_DEFAULT_INDEX=main" \ - -e "SPLUNK_METRICS_INDEX=em_metrics" \ +ExecStartPre=/usr/bin/docker pull $SCS_IMAGE +ExecStartPre=/usr/bin/docker run \ + --env-file=/opt/scs/default/env_file \ + "$SCS_UNIT_SPLUNK_INDEX" "$SCS_UNIT_VP_CSV" "$SCS_UNIT_VP_CONF" \ + --name scs_preflight --rm \ + $SCS_IMAGE -s +ExecStart=/usr/bin/docker run -p 514:514 \ + --env-file=/opt/scs/default/env_file \ + "$SCS_UNIT_SPLUNK_INDEX" "$SCS_UNIT_VP_CSV" "$SCS_UNIT_VP_CONF" \ --name scs \ --rm \ - -v /opt/scs/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv \ - -v /opt/scs/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv \ - -v /opt/scs/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf \ -splunk/scs:latest +$SCS_IMAGE +``` + +## Configure the SCS environment -[Install] -WantedBy=multi-user.target +Create the following file ``/opt/scs/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +#Uncomment the following line if using untrusted SSL certificates +#SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` ## Configure index destinations for Splunk @@ -63,7 +83,72 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas * Start SC4S. ```bash +sudo systemctl daemon-reload sudo systemctl enable scs sudo systemctl start scs ``` + +# Single Source Technology instance + +For certain source technologies message categorization by content is impossible to support collection +of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using +an alternate port. + +Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. + +In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology specific ports modify the range as appropriate + +* Modify the unit file ``/opt/scs/default/env_file`` +```ini +[Unit] +Description=SCS Container +After=network.service +Requires=network.service + +[Service] +Environment="SCS_IMAGE=splunk/scs:latest" + +#Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started +Environment="SCS_UNIT_SPLUNK_INDEX=-v /opt/scs/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" +#Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started +#Environment="SCS_UNIT_VP_CSV=-v /opt/scs/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" +#Environment="SCS_UNIT_VP_CONF=-v /opt/scs/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" + +TimeoutStartSec=0 +Restart=always +ExecStartPre=/usr/bin/docker pull $SCS_IMAGE +ExecStartPre=/usr/bin/docker run \ + --env-file=/opt/scs/default/env_file \ + "$SCS_UNIT_SPLUNK_INDEX" "$SCS_UNIT_VP_CSV" "$SCS_UNIT_VP_CONF" \ + --name scs_preflight --rm \ + $SCS_IMAGE -s +ExecStart=/usr/bin/docker run -p 514:514 -p 5000-5020:5000-5020 \ + --env-file=/opt/scs/default/env_file \ + "$SCS_UNIT_SPLUNK_INDEX" "$SCS_UNIT_VP_CSV" "$SCS_UNIT_VP_CONF" \ + --name scs \ + --rm \ +$SCS_IMAGE + +``` + +Modify the following file ``/opt/scs/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 +#Uncomment the following line if using untrusted SSL certificates +#SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no +``` + +* Restart SC4S. + +```bash +sudo systemctl restart scs +``` diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 6b69f03..6884d4d 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -20,26 +20,46 @@ After=network.service Requires=network.service [Service] +Environment="SCS_IMAGE=splunk/scs:latest" + +#Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started +Environment="SCS_UNIT_SPLUNK_INDEX=-v /opt/scs/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" +#Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started +#Environment="SCS_UNIT_VP_CSV=-v /opt/scs/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" +#Environment="SCS_UNIT_VP_CONF=-v /opt/scs/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" + TimeoutStartSec=0 Restart=always -ExecStartPre=/usr/bin/podman pull splunk/scs:latest +ExecStartPre=/usr/bin/podman pull $SCS_IMAGE +ExecStartPre=/usr/bin/podman run \ + --env-file=/opt/scs/default/env_file \ + "$SCS_UNIT_SPLUNK_INDEX" "$SCS_UNIT_VP_CSV" "$SCS_UNIT_VP_CONF" \ + --name scs_preflight --rm \ + $SCS_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 \ - -e "SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event" \ - -e "SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94" \ - -e "SPLUNK_CONNECT_METHOD=hec" \ - -e "SPLUNK_DEFAULT_INDEX=main" \ - -e "SPLUNK_METRICS_INDEX=em_metrics" \ + --env-file=/opt/scs/default/env_file \ + "$SCS_UNIT_SPLUNK_INDEX" "$SCS_UNIT_VP_CSV" "$SCS_UNIT_VP_CONF" \ --name scs \ --rm \ - -v /opt/scs/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv \ - -v /opt/scs/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv \ - -v /opt/scs/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf \ -splunk/scs:latest +$SCS_IMAGE -[Install] -WantedBy=multi-user.target ``` +## Configure the SCS environment + +Create the following file ``/opt/scs/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +#Uncomment the following line if using untrusted SSL certificates +#SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no +``` ## Configure index destinations for Splunk @@ -71,3 +91,66 @@ sudo systemctl enable scs sudo systemctl start scs ``` +# Single Source Technology instance + +For certain source technologies message categorization by content is impossible to support collection +of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using +an alternate port. + +Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. + +In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology specific ports modify the range as appropriate + +* Modify the unit file ``/lib/systemd/system/scs.service`` +```ini +[Unit] +Description=SCS Container +After=network.service +Requires=network.service + +[Service] +Environment="SCS_IMAGE=splunk/scs:latest" + +#Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started +Environment="SCS_UNIT_SPLUNK_INDEX=-v /opt/scs/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" +#Note Uncomment the following two linese for host and ip based source type mapping AND download the two file templates per getting started +#Environment="SCS_UNIT_VP_CSV=-v /opt/scs/default/vendor_product_by_source.csv:/opt/syslog-ng/etc/context-local/vendor_product_by_source.csv" +#Environment="SCS_UNIT_VP_CONF=-v /opt/scs/default/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf" + +TimeoutStartSec=0 +Restart=always +ExecStartPre=/usr/bin/podman pull $SCS_IMAGE +ExecStartPre=/usr/bin/podman run \ + --env-file=/opt/scs/default/env_file \ + "$SCS_UNIT_SPLUNK_INDEX" "$SCS_UNIT_VP_CSV" "$SCS_UNIT_VP_CONF" \ + --name scs_preflight --rm \ + $SCS_IMAGE -s +ExecStart=/usr/bin/podman run -p 514:514 -p 5000-5020:5000-5020 \ + --env-file=/opt/scs/default/env_file \ + "$SCS_UNIT_SPLUNK_INDEX" "$SCS_UNIT_VP_CSV" "$SCS_UNIT_VP_CONF" \ + --name scs \ + --rm \ +$SCS_IMAGE + +``` + +Modify the following file ``/opt/scs/default/env_file`` + +* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment + +```dotenv +SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 +SPLUNK_CONNECT_METHOD=hec +SPLUNK_DEFAULT_INDEX=main +SPLUNK_METRICS_INDEX=em_metrics +SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 +#Uncomment the following line if using untrusted SSL certificates +#SCS_DEST_SPLUNK_HEC_TLS_VERIFY=no +``` + +* Restart SC4S. + +```bash +sudo systemctl restart scs +``` diff --git a/docs/sources.md b/docs/sources.md index 0ff737a..0d16a48 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -38,6 +38,13 @@ MSG Parse: This filter parses message content * device-id is hostname and included * timestamp is included +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SCS_LISTEN_JUNIPER_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SCS_LISTEN_CISCO_ASA_LEGACY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC3164 format | + ### Verification Use the following search to validate events are present @@ -102,6 +109,12 @@ Verify timestamp, and host values match as expected * Ensure proper host names are configured * For security use cases per AP logging is required +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SCS_LISTEN_CISCO_IOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SCS_LISTEN_CISCO_NX_OS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | ### Verification @@ -187,6 +200,12 @@ end ``` +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SCS_LISTEN_FORTINET_FORTIOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + ### Verification An active firewall will generate frequent events, in addition fortigate has the ability to test logging functionality using a built in command @@ -249,6 +268,12 @@ Verify timestamp, and host values match as expected * Review and update the splunk_index.csv file and set the index as required. * Follow vendor configuration steps per referenced Product Manual +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SCS_LISTEN_JUNIPER_JUNOS_LEGACY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined using legacy 3164 format| +| SCS_LISTEN_JUNIPER_JUNOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined using 5424 format | ### Verification @@ -293,6 +318,12 @@ Verify timestamp, and host values match as expected * Review and update the splunk_index.csv file and set the index as required. * Follow vendor configuration steps per Product Manual +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SCS_LISTEN_JUNIPER_NSM_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SCS_LISTEN_JUNIPER_NSM_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | ### Verification @@ -337,6 +368,12 @@ Verify timestamp, and host values match as expected * Review and update the splunk_index.csv file and set the index as required. * Follow vendor configuration steps per Product Manual +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SCS_LISTEN_JUNIPER_NETSCREEN_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | ### Verification @@ -379,6 +416,11 @@ Verify timestamp, and host values match as expected * Review and update the splunk_index.csv file and set the index as required. * Follow vendor configuration steps per referenced Product Manual +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SCS_LISTEN_JUNIPER_JUNOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | ### Verification @@ -437,6 +479,12 @@ MSG Parse: This filter parses message content * Select IETF Format * Ensure the format of the event is not customized +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SCS_LISTEN_PALOALTO_PANOS_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + ### Verification An active firewall will generate frequent events. Use the following search to validate events are present per source device @@ -480,6 +528,12 @@ MSG Parse: This filter parses message content * Select TCP or SSL transport option * Ensure the format of the event is customized per Splunk documentation +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SCS_LISTEN_SYMANTEC_PROXY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + ### Verification An active proxy will generate frequent events. Use the following search to validate events are present per source device diff --git a/package/Dockerfile b/package/Dockerfile index 4e3b4c4..091e2cb 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -67,11 +67,9 @@ RUN cd /tmp ;\ rm epel-release-latest-7.noarch.rpm ;\ rpm --import https://packages.confluent.io/rpm/5.2/archive.key ;\ yum install gcc tzdata libdbi libsecret libxml2 sqlite tcp_wrappers librdkafka \ - rh-python36 rh-python36-python-tools libcurl ivykis scl-utils tcp_wrappers-libs curl -y;\ + rh-python36 rh-python36-python-tools libcurl ivykis scl-utils tcp_wrappers-libs curl wget -y;\ echo source scl_source enable rh-python36 >>/etc/profile.d/enablepython36.sh ;\ - source scl_source enable rh-python36 ;\ - pip install dumb-init - + source scl_source enable rh-python36 ENV DEBCONF_NONINTERACTIVE_SEEN=true ENV SPLUNK_CONNECT_METHOD=hec @@ -81,15 +79,18 @@ RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install COPY goss.yaml /etc/goss.yaml COPY --from=0 /opt/syslog-ng /opt/syslog-ng + +RUN curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.5.0/gomplate_linux-amd64-slim && \ + chmod 755 /usr/local/bin/gomplate + + COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d COPY etc/context-local /opt/syslog-ng/etc/context-local -COPY sbin/entrypoint.sh /sbin/entrypoint.sh RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer -RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V +RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t -RUN mkdir -p /var/log/syslog-ng/data/disk-buffer EXPOSE 514 EXPOSE 601/tcp @@ -97,6 +98,6 @@ EXPOSE 6514/tcp ENV SPLUNK_CONNECT_METHOD=UF -ENTRYPOINT ["/sbin/entrypoint.sh", "--"] +ENTRYPOINT ["/opt/syslog-ng/sbin/syslog-ng", "-F"] -HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file +HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate diff --git a/package/etc/conf.d/conflib/_common/network.conf b/package/etc/conf.d/conflib/_common/network.conf new file mode 100644 index 0000000..ae13635 --- /dev/null +++ b/package/etc/conf.d/conflib/_common/network.conf @@ -0,0 +1,10 @@ +@module confgen context(source) name(gen_source_dedicated_port) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/conflib/_common/network.conf.tmpl") + +block source dedicated_port(udp_port(no) tcp_port(no) tls_port(no) flags("no-parse")) { + gen_source_dedicated_port( + udp_port(`udp_port`) + tcp_port(`tcp_port`) + tls_port(`tls_port`) + flags(`flags`) + ) +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/network.conf.tmpl b/package/etc/conf.d/conflib/_common/network.conf.tmpl new file mode 100644 index 0000000..3a789e9 --- /dev/null +++ b/package/etc/conf.d/conflib/_common/network.conf.tmpl @@ -0,0 +1,93 @@ +# =============================================================================================== +# source definition for remote devices +# =============================================================================================== + +# =============================================================================================== +# Defaults for the default-network-drivers() source: +# 514, both TCP and UDP, for RFC3164 (BSD-syslog) formatted traffic +# 601 TCP, for RFC5424 (IETF-syslog) formatted traffic +# 6514 TCP, for TLS-encrypted traffic +# =============================================================================================== + + channel { + source { +{{ if ne (getenv "confgen_udp_port") "no" }} + syslog ( + transport("udp") + port({{getenv "confgen_udp_port"}}) + ip-protocol(4) + so-rcvbuf({{getenv "SCS_SOURCE_UDP_SO_RCVBUFF" "425984"}}) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); +{{ end }} +{{ if ne (getenv "confgen_tcp_port") "no" }} + network ( + transport("tcp") + port({{getenv "confgen_tcp_port"}}) + ip-protocol(4) + max-connections({{getenv "SCS_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) + log-iw-size({{getenv "SCS_SOURCE_TCP_IW_SIZE" "20000000"}}) + log-fetch-limit({{getenv "SCS_SOURCE_TCP_FETCH_LIMIT" "2000"}}) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); +{{ end }} + }; + #TODO: #60 Remove this function with enhancement + rewrite(set_metadata_presume); + rewrite(set_rfcnonconformant); + rewrite(r_set_splunk_default); + +{{ if eq (getenv "confgen_parser") "rfc5424_strict" }} + filter(f_rfc5424_strict); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_strict); +{{ else if eq (getenv "confgen_parser") "rfc5424_noversion" }} + filter(f_rfc5424_noversion); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_noversion); +{{ else if eq (getenv "confgen_parser") "cisco_parser" }} + parser {cisco-parser()}; + rewrite(set_metadata_vendor_product_cisco_ios); +{{ else if eq (getenv "confgen_parser") "rfc3164" }} + parser { + syslog-parser(time-zone({{getenv "SCS_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); + }; + rewrite(set_rfc3164); +{{ else }} + if { + filter(f_rfc5424_strict); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_strict); + } elif { + filter(f_rfc5424_noversion); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_noversion); + } elif { + parser {cisco-parser()}; + rewrite(set_metadata_vendor_product_cisco_ios); + } else { + parser { + syslog-parser(time-zone({{getenv "SCS_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); + }; + rewrite(set_rfc3164); + }; +{{ end }} + }; diff --git a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf index cb238de..f3789a7 100644 --- a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf +++ b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf @@ -1,6 +1,6 @@ block parser vendor_product_by_source() { add-contextual-data( - selector(filters("/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf")), + selector(filters("`syslog-ng-sysconfdir`/context-local/vendor_product_by_source.conf")), database("context-local/vendor_product_by_source.csv") ignore-case(yes) prefix("fields.") diff --git a/package/etc/conf.d/destinations/splunk_hec.conf b/package/etc/conf.d/destinations/splunk_hec.conf index 0132bfe..81db7ee 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf +++ b/package/etc/conf.d/destinations/splunk_hec.conf @@ -1,76 +1,5 @@ - - -# =============================================================================================== -# Direct connection to Splunk via HEC -# =============================================================================================== - -# =============================================================================================== -# Be sure to adjust batch paramaters below to suit scale/environment -# Set workers to the number of indexers or HWF HEC endpoints -# If validated certs are used, uncomment relevant lines in the tls() block below -# and change peer-verify() to "yes" -# =============================================================================================== +@module confgen context(destination) name(gen_splunk_hec) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/destinations/splunk_hec.conf.tmpl") destination d_hec { - http( - url("`SPLUNK_HEC_URL`") - method("POST") - log-fifo-size(`splunk-log-fifo-size`) - workers(`SYSLOGNG_HEC_WORKERS`) - batch-lines(1000) - batch-bytes(2048Kb) - batch-timeout(3) - timeout(15) - user_agent("syslog-ng User Agent") - user("syslog-ng") - headers("Connection: close") - password("`SPLUNK_HEC_TOKEN`") - persist-name("splunk") - disk-buffer(mem-buf-length(15000) - disk-buf-size(200000) - reliable(no) - dir("/opt/syslog-ng/var/data/disk-buffer/")) - tls(peer-verify(no) -# ca-dir("dir") -# ca-file("ca") -# cert-file("cert") -# cipher-suite("cipher") -# key-file("key") -# peer-verify(yes|no) -# ssl-version() - ) - body('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=${HOST} - source=${.splunk.source} - sourcetype=${.splunk.sourcetype} - index=${.splunk.index} - event=$(template ${.splunk.template} $(template `splunk-template`)) - fields.*)') - ); - }; - -destination d_hecmetrics { - http( - url("`SPLUNK_HEC_URL`") - method("POST") - batch-lines(5) - batch-bytes(512Kb) - batch-timeout(1) - timeout(15) - user_agent("syslog-ng User Agent") - user("syslog-ng") - password("`SPLUNK_HEC_TOKEN`") - persist-name("splunk_metrics") - tls(peer-verify(no) -# ca-dir("dir") -# ca-file("ca") -# cert-file("cert") -# cipher-suite("cipher") -# key-file("key") -# peer-verify(yes|no) -# ssl-version() - ) - body('$MESSAGE') - ); - }; + gen_splunk_hec() +}; diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl new file mode 100644 index 0000000..5da5234 --- /dev/null +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -0,0 +1,42 @@ + http( + url("{{getenv "SPLUNK_HEC_URL"}}") + method("POST") + log-fifo-size({{getenv "SCS_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) + workers({{getenv "SCS_DEST_SPLUNK_HEC_WORKERS" "10"}}) + batch-lines({{getenv "SCS_DEST_SPLUNK_HEC_BATCH_LINES" "1000"}}) + batch-bytes({{getenv "SCS_DEST_SPLUNK_HEC_BATCH_BYTES" "4096kb"}}) + batch-timeout({{getenv "SCS_DEST_SPLUNK_HEC_BATCH_TIMEOUT" "1"}}) + timeout({{getenv "SCS_DEST_SPLUNK_HEC_TIMEOUT" "30"}}) + user_agent("SCS/1.0 (events)") + user("scs") + headers("{{getenv "SCS_DEST_SPLUNK_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}") + password("{{getenv "SPLUNK_HEC_TOKEN"}}") + persist-name("splunk_hec") + +{{ if eq (getenv "SCS_DEST_SPLUNK_HEC_DISKBUFF_ENABLE" "yes") "yes" }} + disk-buffer( +{{ if eq (getenv "SCS_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE" "no") "yes" }} + mem-buf-size({{getenv "SCS_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024"}}) + reliable(yes) +{{else}} + mem-buf-length({{getenv "SCS_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000"}}) + reliable(no) +{{ end }} +{{ end }} + disk-buf-size({{getenv "SCS_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "1048576"}}) + dir("/opt/syslog-ng/var/data/disk-buffer/") + ) + tls(peer-verify({{getenv "SCS_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}}) +{{if ne (getenv "SCS_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}} cipher-suite("{{getenv "SCS_DEST_SPLUNK_HEC_CIPHER_SUITE"}}"){{end}} +{{if ne (getenv "SCS_DEST_SPLUNK_HEC_SSL_VERSION") ""}} ssl-version("{{getenv "SCS_DEST_SPLUNK_HEC_SSL_VERSION"}}"){{end}} + ca-file("{{getenv "SCS_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}") + ) + body('$(format-json + time=$S_UNIXTIME.$S_MSEC + host=${HOST} + source=${.splunk.source} + sourcetype=${.splunk.sourcetype} + index=${.splunk.index} + event=$(template ${fields.sc4s_template} $(template "t_standard")) + fields.*)') + ); diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf b/package/etc/conf.d/destinations/splunk_hec_metrics.conf new file mode 100644 index 0000000..1183f94 --- /dev/null +++ b/package/etc/conf.d/destinations/splunk_hec_metrics.conf @@ -0,0 +1,6 @@ +@module confgen context(destination) name(gen_splunk_hec_metrics) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/destinations/splunk_hec_metrics.conf.tmpl") + + +destination d_hecmetrics { + gen_splunk_hec_metrics() +}; diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl new file mode 100644 index 0000000..119c21b --- /dev/null +++ b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl @@ -0,0 +1,20 @@ + http( + url("{{getenv "SPLUNK_HEC_URL"}}") + method("POST") + batch-lines(50) + batch-bytes(1024Kb) + batch-timeout(1) + timeout(15) + user_agent("SCS/1.0 (internal metrics)") + user("scs") + headers("{{getenv "SCS_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}") + password("{{getenv "SPLUNK_HEC_TOKEN"}}") + persist-name("splunk_metrics") + + tls(peer-verify({{getenv "SCS_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}}) +{{if ne (getenv "SCS_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}} cipher-suite("{{getenv "SCS_DEST_SPLUNK_HEC_CIPHER_SUITE"}}"){{end}} +{{if ne (getenv "SCS_DEST_SPLUNK_HEC_SSL_VERSION") ""}} ssl-version("{{getenv "SCS_DEST_SPLUNK_HEC_SSL_VERSION"}}"){{end}} + ca-file("{{getenv "SCS_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}") + ) + body('$MESSAGE') + ); diff --git a/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf b/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf deleted file mode 100644 index 3e95c30..0000000 --- a/package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf +++ /dev/null @@ -1,14 +0,0 @@ -log { - - source(s_default-ports); - filter(f_is_rfc3164); - filter(f_juniper_nsm_idp); - - rewrite {r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"), template("t_standard"))}; - parser { - p_add_context_splunk(key("juniper_idp")); - }; - - destination(d_hec); #--HEC-- - flags(flow-control); -}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf deleted file mode 100644 index a0c1e8e..0000000 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf +++ /dev/null @@ -1,14 +0,0 @@ -log { - - source(s_default-ports); - filter(f_cisco_nx_os); - - rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))}; - parser { - p_add_context_splunk(key("cisco_nx_os")); - }; - - destination(d_hec); #--HEC-- - - flags(flow-control); -}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf index 866fa14..f97b906 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf @@ -1,20 +1,13 @@ - # =============================================================================================== # Cisco ASA (Firewall) # =============================================================================================== -log { - source(s_default-ports); - - filter(f_is_rfc3164); - filter(f_cisco_asa); - #set the source type based on program field and lookup index from the splunk context csv - #Using the 5424 parser the message content is all we need +@module confgen context(log) name(gen_log_rfc_3164-cisco-asa) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl") - rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; - parser {p_add_context_splunk(key("cisco_asa")); }; - - destination(d_hec); #--HEC-- +log { + #Default source + gen_log_rfc_3164-cisco-asa( + sourcedefault(yes) + ); - flags(flow-control); }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl new file mode 100644 index 0000000..0e34ba1 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -0,0 +1,26 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_cisco_asa); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_CISCO_ASA_LEGACY_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_CISCO_ASA_LEGACY_UDP_PORT" }}) + ) + }; +{{ end }} + + #set the source type based on program field and lookup index from the splunk context csv + #Using the 5424 parser the message content is all we need + + rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("cisco_asa")); }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf index 09848fd..2f7a3bb 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf @@ -3,18 +3,12 @@ # Cisco IOS (Route/Switch) # =============================================================================================== -log { - - source(s_default-ports); - filter(f_cisco_ios); - +@module confgen context(log) name(gen_log_rfc_3164-cisco-ios) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl") - rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_msg_only"))}; - parser { - p_add_context_splunk(key("cisco_ios")); - }; - - destination(d_hec); #--HEC-- +log { + #Default source + gen_log_rfc_3164-cisco-ios( + sourcedefault(yes) + ); - flags(flow-control); -}; +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl new file mode 100644 index 0000000..243621b --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -0,0 +1,25 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_cisco_ios); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_CISCO_IOS_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_CISCO_IOS_UDP_PORT" }}) + + ) + }; +{{ end }} + + rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_msg_only"))}; + parser { + p_add_context_splunk(key("cisco_ios")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf new file mode 100644 index 0000000..04297c6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf @@ -0,0 +1,14 @@ + +# =============================================================================================== +# Cisco IOS (Route/Switch) +# =============================================================================================== + +@module confgen context(log) name(gen_log_rfc_3164-cisco-nx-os) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl") + +log { + #Default source + gen_log_rfc_3164-cisco-nx-os( + sourcedefault(yes) + ); + +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl new file mode 100644 index 0000000..38528fd --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl @@ -0,0 +1,25 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_cisco_nx_os); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_CISCO_NX_OS_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_CISCO_NX_OS_UDP_PORT" }}) + + ) + }; +{{ end }} + + rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))}; + parser { + p_add_context_splunk(key("cisco_nx_os")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf index 0c451ed..c22dceb 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf @@ -1,34 +1,14 @@ -log { - source(s_default-ports); - filter(f_is_rfc3164); - filter(f_fortinet_fortios); - - parser { - kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}")); - date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone(`default-timezone`)); - }; - - rewrite { set("${.kv.devname}", value("HOST")); }; - #set the source type based on program field and lookup index from the splunk context csv +# =============================================================================================== +# Fortigate FortiOS +# =============================================================================================== +@module confgen context(log) name(gen_log_rfc_3164-fortigate-fortios) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl") - if (match("traffic" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"), template("t_standard"))}; - parser {p_add_context_splunk(key("fortinet_fortios_traffic")); }; - } elif (match("utm" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"), template("t_standard"))}; - parser {p_add_context_splunk(key("fortinet_fortios_utm")); }; - } elif (match("event" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"), template("t_standard"))}; - parser {p_add_context_splunk(key("fortinet_fortios_event")); }; - } else { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"), template("t_standard"))}; - parser {p_add_context_splunk(key("fortinet_fortios_log")); }; - }; - - destination(d_hec); #--HEC-- - - flags(flow-control); -}; +log { + #Default source + gen_log_rfc_3164-fortigate-fortios( + sourcedefault(yes) + ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl new file mode 100644 index 0000000..fc6d446 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -0,0 +1,45 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_fortinet_fortios); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_FORTINET_FORTIOS_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_FORTINET_FORTIOS_UDP_PORT" }}) + + ) + }; +{{ end }} + + parser { + kv-parser(prefix(".kv.") template("${MSGHDR} ${MSG}")); + date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{getenv "SCS_DEFAULT_TIMEZONE" "GMT"}})); + }; + + rewrite { set("${.kv.devname}", value("HOST")); }; + + #set the source type based on program field and lookup index from the splunk context csv + + + if (match("traffic" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"), template("t_standard"))}; + parser {p_add_context_splunk(key("fortinet_fortios_traffic")); }; + } elif (match("utm" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"), template("t_standard"))}; + parser {p_add_context_splunk(key("fortinet_fortios_utm")); }; + } elif (match("event" value(".kv.type"))) { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"), template("t_standard"))}; + parser {p_add_context_splunk(key("fortinet_fortios_event")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"), template("t_standard"))}; + parser {p_add_context_splunk(key("fortinet_fortios_log")); }; + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf deleted file mode 100644 index 0f4db26..0000000 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf +++ /dev/null @@ -1,15 +0,0 @@ -log { - - source(s_default-ports); - filter(f_is_rfc5424_strict); - filter(f_juniper_idp); - - rewrite { r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids"), template("t_hdr_sdata_msg"))}; - parser { - p_add_context_splunk(key("juniper_idp")); - }; - - destination(d_hec); #--HEC-- - - flags(flow-control); -}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf new file mode 100644 index 0000000..0509425 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf @@ -0,0 +1,17 @@ +@module confgen context(log) name(gen_log_rfc_3164-juniper-idp) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl") + +log { + #Default source + gen_log_rfc_3164-juniper-idp( + sourcedefault(yes) + ); + +}; + +log { + #Specific source + gen_log_rfc_3164-juniper-idp( + sourcedefault(no) + ); + +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl new file mode 100644 index 0000000..3abcd33 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -0,0 +1,25 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc5424_strict); + filter(f_juniper_idp); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_JUNIPER_IDP_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_JUNIPER_IDP_UDP_PORT" }}) + ); + }; +{{ end }} + + rewrite { r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids"), template("t_hdr_sdata_msg"))}; + parser { + p_add_context_splunk(key("juniper_idp")); + }; + + destination(d_hec); #--HEC-- + + flags(flow-control); \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf new file mode 100644 index 0000000..9c9a9b6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf @@ -0,0 +1,17 @@ +@module confgen context(log) name(gen_log_rfc_3164-juniper-junos) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl") + +log { + #Default source + gen_log_rfc_3164-juniper-junos( + sourcedefault(yes) + ); + +}; + +log { + #Specific source + gen_log_rfc_3164-juniper-junos( + sourcedefault(no) + ); + +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl similarity index 72% rename from package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf rename to package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index f112311..f49af18 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -1,12 +1,20 @@ -# =============================================================================================== -# Juniper Standard and Structured logging -# =============================================================================================== - -log { - +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} source(s_default-ports); filter(f_is_rfc3164); filter(f_juniper_junos_standard); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_JUNIPER_JUNOS_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_JUNIPER_JUNOS_UDP_PORT" }}) + + ) + }; +{{ end }} if (program('RT_IDP')) { rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))}; @@ -30,5 +38,4 @@ log { destination(d_hec); #--HEC-- - flags(flow-control); -}; + flags(flow-control); \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf index f963649..83de2bd 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf @@ -1,14 +1,17 @@ -log { +@module confgen context(log) name(gen_log_rfc_3164-juniper-netscreen) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl") - source(s_default-ports); - filter(f_juniper_netscreen); +log { + #Default source + gen_log_rfc_3164-juniper-netscreen( + sourcedefault(yes) + ); - rewrite { r_set_splunk_dest_default(sourcetype("netscreen:firewall"), index("netfw"), template("t_standard"))}; +}; - parser { - p_add_context_splunk(key("juniper_netscreen")); - }; +log { + #Specific source + gen_log_rfc_3164-juniper-netscreen( + sourcedefault(no) + ); - destination(d_hec); #--HEC-- - flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl new file mode 100644 index 0000000..40ed78c --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -0,0 +1,28 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_juniper_netscreen); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_JUNIPER_NETSCREEN_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_JUNIPER_NETSCREEN_UDP_PORT" }}) + ) + }; +{{ end }} + + rewrite { + r_set_splunk_dest_default(sourcetype("netscreen:firewall"), + index("netfw"), + template("t_standard")) + }; + + parser { + p_add_context_splunk(key("juniper_netscreen")); + }; + + destination(d_hec); #--HEC-- + flags(flow-control); \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf index 645d7ea..b552088 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf @@ -2,18 +2,20 @@ # Juniper NSM # =============================================================================================== -log { +@module confgen context(log) name(gen_log_rfc_3164-juniper-nsm) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl") - source(s_default-ports); - filter(f_is_rfc3164); - filter(f_juniper_nsm); +log { + #Default source + gen_log_rfc_3164-juniper-nsm( + sourcedefault(yes) + ); - rewrite { r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"), template("t_standard"))}; +}; - parser { - p_add_context_splunk(key("juniper_nsm")); - }; +log { + #Specific source + gen_log_rfc_3164-juniper-nsm( + sourcedefault(no) + ); - destination(d_hec); #--HEC-- - flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl new file mode 100644 index 0000000..897f6af --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -0,0 +1,25 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_nsm); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_JUNIPER_NSM_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_JUNIPER_NSM_UDP_PORT" }}) + ) + }; +{{ end }} + + rewrite { r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"), template("t_standard"))}; + + parser { + p_add_context_splunk(key("juniper_nsm")); + }; + + destination(d_hec); #--HEC-- + flags(flow-control); \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf new file mode 100644 index 0000000..ecfa966 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf @@ -0,0 +1,21 @@ +# =============================================================================================== +# Juniper NSM IDP +# =============================================================================================== + +@module confgen context(log) name(gen_log_rfc_3164-juniper-nsm-idp) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl") + +log { + #Default source + gen_log_rfc_3164-juniper-nsm-idp( + sourcedefault(yes) + ); + +}; + +log { + #Specific source + gen_log_rfc_3164-juniper-nsm-idp( + sourcedefault(no) + ); + +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl new file mode 100644 index 0000000..d1b157f --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -0,0 +1,24 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc3164); + filter(f_juniper_nsm_idp); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_JUNIPER_NSM_IDP_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_JUNIPER_NSM_IDP_UDP_PORT" }}) + ) + }; +{{ end }} + + rewrite {r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"), template("t_standard"))}; + parser { + p_add_context_splunk(key("juniper_idp")); + }; + + destination(d_hec); #--HEC-- + flags(flow-control); \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf new file mode 100644 index 0000000..7eccabe --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf @@ -0,0 +1,17 @@ +@module confgen context(log) name(gen_log_rfc_3164-palalto-panos) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl") + +log { + #Default source + gen_log_rfc_3164-palalto-panos( + sourcedefault(yes) + ); + +}; + +log { + #Specific source + gen_log_rfc_3164-palalto-panos( + sourcedefault(no) + ); + +}; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl similarity index 85% rename from package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf rename to package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 9f1bb65..20c5ab6 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -1,7 +1,20 @@ -log { +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} source(s_default-ports); filter(f_is_rfc3164); filter(f_paloalto_panos); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_PALOALTO_PANOS_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_PALOALTO_PANOS_UDP_PORT" }}) + + ) + }; +{{ end }} # the palo message does not include a program value in the header unfortunatly # the use of colon in the message tricks the RFC3164 parser we will re-write the message @@ -61,5 +74,3 @@ log { destination(d_hec); #--HEC-- flags(flow-control); -}; - diff --git a/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf index de558e7..55d4463 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf +++ b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf @@ -1,15 +1,14 @@ -log { - source(s_default-ports); - filter(f_is_rfc5424_noversion); - filter(f_cisco_asa); - #set the source type based on program field and lookup index from the splunk context csv - #Using the 5424 parser the message content is all we need +# =============================================================================================== +# Cisco IOS ASA +# =============================================================================================== - rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; - parser {p_add_context_splunk(key("cisco_asa")); }; +@module confgen context(log) name(gen_log_rfc_5424_noversion-cisco-asa) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl") - destination(d_hec); #--HEC-- +log { + #Default source + gen_log_rfc_5424_noversion-cisco-asa( + sourcedefault(yes) + ); - flags(flow-control); }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl new file mode 100644 index 0000000..b5b9246 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl @@ -0,0 +1,27 @@ +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} + source(s_default-ports); + filter(f_is_rfc5424_noversion); + filter(f_cisco_asa); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_JUNIPER_CISCO_ASA_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_JUNIPER_CISCO_ASA_UDP_PORT" }}) + + ) + }; +{{ end }} + + #set the source type based on program field and lookup index from the splunk context csv + #Using the 5424 parser the message content is all we need + + rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; + parser {p_add_context_splunk(key("cisco_asa")); }; + + destination(d_hec); #--HEC-- + + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf new file mode 100644 index 0000000..d701ba5 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf @@ -0,0 +1,10 @@ + +@module confgen context(log) name(gen_log_rfc_5424_noversion-symantec_proxy) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl") + +log { + #Default source + gen_log_rfc_5424_noversion-symantec_proxy( + sourcedefault(yes) + ); + +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl similarity index 50% rename from package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf rename to package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl index 1d596bf..0f56343 100644 --- a/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf +++ b/package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl @@ -1,7 +1,20 @@ -log { +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} source(s_default-ports); filter(f_is_rfc5424_noversion); filter(f_symantec_bluecoat_proxy); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_SYMANTEC_PROXY_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_SYMANTEC_PROXY_UDP_PORT" }}) + + ) + }; +{{ end }} #set the source type based on program field and lookup index from the splunk context csv. #With the 5424 parser the message is the only portion needed in the template. @@ -13,4 +26,3 @@ log { destination(d_hec); #--HEC-- flags(flow-control); -}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf new file mode 100644 index 0000000..48365ea --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf @@ -0,0 +1,14 @@ +# =============================================================================================== +# Juniper Structured logging +# =============================================================================================== + + +@module confgen context(log) name(gen_log_rfc_5424_strict-juniper_junos) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl") + +log { + #Default source + gen_log_rfc_5424_strict-juniper_junos( + sourcedefault(yes) + ); + +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl similarity index 75% rename from package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf rename to package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl index 56f064c..5142c73 100644 --- a/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf +++ b/package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl @@ -1,12 +1,20 @@ -# =============================================================================================== -# Juniper Structured logging -# =============================================================================================== - -log { - +#The template will be used to produce two log paths +#The first log path is linked to the default (shared) 514 +{{ if eq (getenv "confgen_sourcedefault") "yes" }} source(s_default-ports); filter(f_is_rfc5424_strict); filter(f_juniper_junos_structured); +{{ end }} +#The second log path will only generate if the value of one OR more port variables is set +{{ if eq (getenv "confgen_sourcedefault") "no" }} + source { + dedicated_port( + tcp_port({{ getenv "SCS_LISTEN_JUNIPER_JUNOS_TCP_PORT" }}) + udp_port({{ getenv "SCS_LISTEN_JUNIPER_JUNOS_UDP_PORT" }}) + ) + }; +{{ end }} + if (program('RT_IDP')) { rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) }; @@ -34,5 +42,3 @@ log { destination(d_hec); #--HEC-- - -}; diff --git a/package/etc/conf.d/sources/network.conf b/package/etc/conf.d/sources/network.conf index 2d562b6..599061d 100644 --- a/package/etc/conf.d/sources/network.conf +++ b/package/etc/conf.d/sources/network.conf @@ -1,75 +1,6 @@ -# =============================================================================================== -# source definition for remote devices -# =============================================================================================== +@module confgen context(source) name(gen_s_default_ports) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/sources/network.conf.tmpl") -# =============================================================================================== -# Defaults for the default-network-drivers() source: -# 514, both TCP and UDP, for RFC3164 (BSD-syslog) formatted traffic -# 601 TCP, for RFC5424 (IETF-syslog) formatted traffic -# 6514 TCP, for TLS-encrypted traffic -# =============================================================================================== - -# source s_network-defaults { default-network-drivers() }; source s_default-ports { - channel { - source { - `splunk-udp-driver` ( - transport("udp") - port(`udp-listening-port`) - ip-protocol(`ip-version`) - so-rcvbuf(`splunk-rcvbuf`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - - `splunk-tcp-driver` ( - transport("tcp") - port(`tcp-listening-port`) - ip-protocol(`ip-version`) - max-connections(`splunk-max-connections`) - log-iw-size(`splunk-window-size`) - log-fetch-limit(`splunk-fetch-limit`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - }; - #TODO: #60 Remove this function with enhancement - rewrite(set_metadata_presume); - rewrite(set_rfcnonconformant); - rewrite(r_set_splunk_default); - - if { - filter(f_rfc5424_strict); - parser { - syslog-parser(flags(syslog-protocol store-raw-message)); - }; - rewrite(set_rfc5424_strict); - } elif { - filter(f_rfc5424_noversion); - parser { - syslog-parser(flags(syslog-protocol store-raw-message)); - }; - rewrite(set_rfc5424_noversion); - } elif { - parser {cisco-parser()}; - rewrite(set_metadata_vendor_product_cisco_ios); - } else { - parser { - syslog-parser(time-zone(`default-timezone`) flags(store-raw-message)); - }; - rewrite(set_rfc3164); - }; - - parser { - vendor_product_by_source(); - }; - }; + gen_s_default_ports() }; + diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl new file mode 100644 index 0000000..07a9b8f --- /dev/null +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -0,0 +1,72 @@ +# =============================================================================================== +# source definition for remote devices +# =============================================================================================== + +# =============================================================================================== +# Defaults for the default-network-drivers() source: +# 514, both TCP and UDP, for RFC3164 (BSD-syslog) formatted traffic +# 601 TCP, for RFC5424 (IETF-syslog) formatted traffic +# 6514 TCP, for TLS-encrypted traffic +# =============================================================================================== + + channel { + source { + syslog ( + transport("udp") + port(514) + ip-protocol(4) + so-rcvbuf({{getenv "SCS_SOURCE_UDP_SO_RCVBUFF" "425984"}}) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); + + network ( + transport("tcp") + port(514) + ip-protocol(4) + max-connections({{getenv "SCS_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) + log-iw-size({{getenv "SCS_SOURCE_TCP_IW_SIZE" "20000000"}}) + log-fetch-limit({{getenv "SCS_SOURCE_TCP_FETCH_LIMIT" "2000"}}) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); + }; + #TODO: #60 Remove this function with enhancement + rewrite(set_metadata_presume); + rewrite(set_rfcnonconformant); + rewrite(r_set_splunk_default); + + if { + filter(f_rfc5424_strict); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_strict); + } elif { + filter(f_rfc5424_noversion); + parser { + syslog-parser(flags(syslog-protocol store-raw-message)); + }; + rewrite(set_rfc5424_noversion); + } elif { + parser {cisco-parser()}; + rewrite(set_metadata_vendor_product_cisco_ios); + } else { + parser { + syslog-parser(time-zone({{getenv "SCS_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); + }; + rewrite(set_rfc3164); + }; + + parser { + vendor_product_by_source(); + }; + }; diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index bfb2c10..edee26f 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -33,24 +33,6 @@ options { on-error(fallback-to-string); }; -# =============================================================================================== -# Source driver defaults (which govern the parser used) -# =============================================================================================== -@define splunk-udp-driver "syslog" -@define splunk-tcp-driver "network" - -# =============================================================================================== -# Listening ports -# =============================================================================================== -@define udp-listening-port "514" -@define tcp-listening-port "514" - -# =============================================================================================== -# Splunk metadata (HEC/Kafka transport only) -# =============================================================================================== -@define splunk-sourcetype "syslog-ng:fallback" -@define splunk-index "main" -@define splunk-source "sc4s" # =============================================================================================== # Default message template @@ -60,14 +42,9 @@ options { # =============================================================================================== # Data collection parameters, buffers, and Timezone # =============================================================================================== -@define ip-version 4 -@define splunk-max-connections 1000 -@define splunk-log-fifo-size 180000000 -@define splunk-fetch-limit 20000 -# make sure splunk-window-size >= splunk-max-connections * splunk-fetch-limit -@define splunk-window-size 20000000 -@define splunk-rcvbuf 425984 -@define default-timezone "GMT" +#TODO: Remove once release with this PR is produced by upstream +#https://github.com/balabit/syslog-ng/pull/2932 +@define syslog-ng-sysconfdir "/opt/syslog-ng/etc" # =============================================================================================== # Global modules and includes. All device-specific filters and destinations exist in conf.d @@ -85,6 +62,3 @@ options { @include "conf.d/log_paths/*.conf" @include "conf.d/plugin/*/log_paths/*.conf" - - - diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh deleted file mode 100755 index 5d37dd1..0000000 --- a/package/sbin/entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/opt/rh/rh-python36/root/usr/bin/dumb-init /bin/bash -#Copyright 2019 Splunk, Inc. -# -#Use of this source code is governed by a BSD-2-clause-style -#license that can be found in the LICENSE-BSD2 file or at -#https://opensource.org/licenses/BSD-2-Clause -#Run syslog -mkdir /opt/syslog-ng/var -rm /opt/syslog-ng/var/syslog-ng.ctl || true -/opt/syslog-ng/sbin/syslog-ng -F diff --git a/tests/test_juniper_legacy.py b/tests/test_juniper_legacy.py index b2898af..34de672 100644 --- a/tests/test_juniper_legacy.py +++ b/tests/test_juniper_legacy.py @@ -111,7 +111,7 @@ def test_juniper_netscreen_fw_singleport(record_property, setup_wordlist, get_ho "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} ns204: NetScreen device_id=netscreen2 [Root]system-notification-00257(traffic): start_time=\"2009-03-18 16:07:06\" duration=0 policy_id=320001 service=msrpc Endpoint Mapper(tcp) proto=6 src zone=Null dst zone=self action=Deny sent=0 rcvd=16384 src=21.10.90.125 dst=23.16.1.1\n") message = mt.render(mark="<23>", host=host) - sendsingle(message, host="sc4s-juniper") + sendsingle(message, port=5000) st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"netscreen:firewall\" | head 2") search = st.render(host=host)