From 61bd1072e10a095557f163409db3c618a84c287e Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Sat, 21 Sep 2019 19:12:49 -0400 Subject: [PATCH] Feature/docs (#88) --- docker-compose-demo.yml | 1 - docs/gettingstarted/docker-swarm-general.md | 19 +++++---- docs/gettingstarted/docker-swarm-rhel7.md | 19 ++++----- docs/gettingstarted/docker-systemd-general.md | 31 ++++++-------- docs/gettingstarted/podman-systemd-general.md | 35 +++++++--------- .../conf.d/conflib/_common/network.conf.tmpl | 4 +- .../conf.d/conflib/_splunk/splunkfields.conf | 10 ++--- .../conf.d/destinations/splunk_hec.conf.tmpl | 28 +++++++------ .../destinations/splunk_hec_internal.conf | 5 +++ .../splunk_hec_internal.conf.tmpl | 29 ++++++++++++++ package/etc/conf.d/destinations/stdout.conf | 6 +++ .../etc/conf.d/destinations/stdout.conf.tmpl | 1 + package/etc/conf.d/log_paths/internal.conf | 40 +++---------------- .../etc/conf.d/log_paths/internal.conf.tmpl | 40 +++++++++++++++++++ package/etc/conf.d/log_paths/zfallback.conf | 4 +- package/etc/conf.d/sources/internal.conf | 10 ++++- package/etc/conf.d/sources/network.conf.tmpl | 4 +- package/etc/context-local/splunk_index.csv | 6 +-- tests/test_common.py | 23 ++++++++++- 19 files changed, 190 insertions(+), 125 deletions(-) create mode 100644 package/etc/conf.d/destinations/splunk_hec_internal.conf create mode 100644 package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl create mode 100644 package/etc/conf.d/destinations/stdout.conf create mode 100644 package/etc/conf.d/destinations/stdout.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/internal.conf.tmpl diff --git a/docker-compose-demo.yml b/docker-compose-demo.yml index 5e4c200..c4fb88f 100644 --- a/docker-compose-demo.yml +++ b/docker-compose-demo.yml @@ -13,7 +13,6 @@ services: links: - splunk - sc4s - - sc4s-juniper volumes: - sc4s-tests:/work/tests - sc4s-results:/work/test-results diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index d7beaa7..b5e7306 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -50,12 +50,12 @@ SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics ``` -## Configure index destinations for Splunk +## Configure index destinations for Splunk Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow customization of index destinations. -* Create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above. From this directory, +* Create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/scs/``) created in the first step above. From this directory, execute the command below to download the index context file: ```bash @@ -66,7 +66,7 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas ## Configure sources by source IP or host name Legacy sources and non-standard-compliant source require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. * If not already done in the step immediately above, create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above. From this directory, execute the commands below to download the vendor context files: @@ -84,17 +84,16 @@ docker stack deploy --compose-file docker-compose.yml sc4s ``` -## Scale out +# Scale out Additional hosts can be deployed for syslog collection from additional network zones and locations. -# Single Source Technology instance +# Configure Dedicated Listening Ports -For certain source technologies message categorization by content is impossible to support collection -of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using -an alternate port. -Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. +For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +For collection of such sources we provide a means of dedicating a unique listening port to a specific source. Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. @@ -138,7 +137,7 @@ services: # - ./sc4s-juniper/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf ``` -Modify the following file ``/opt/sc4s/default/env_file`` +* Modify the following file ``/opt/sc4s/default/env_file`` after including the port-specific environment variable(s). * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index e57bdb5..f837969 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -79,7 +79,7 @@ SPLUNK_METRICS_INDEX=em_metrics ``` -## Configure index destinations for Splunk +## Configure index destinations for Splunk Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow customization of index destinations. @@ -95,7 +95,7 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas ## Configure sources by source IP or host name Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. * If not already done in the step immediately above, create a subdirectory called ``context-local`` in the directory (e.g. ``/opt/sc4s/``) created in the first step above. From this directory, execute the commands below to download the vendor context files: @@ -112,17 +112,18 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas sudo docker stack deploy --compose-file docker-compose.yml sc4s ``` -## Scale out +# Scale out Additional hosts can be deployed for syslog collection from additional network zones and locations. -# Single Source Technology instance +# Configure Dedicated Listening Ports -For certain source technologies message categorization by content is impossible to support collection -of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using -an alternate port. -Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. +For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +For collection of such sources we provide a means of dedicating a unique listening port to a specific source. + +Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Modify the individual ports or a range as appropriate for your network. @@ -164,7 +165,7 @@ services: # - ./sc4s-juniper/vendor_product_by_source.conf:/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf ``` -Modify the following file ``/opt/sc4s/default/env_file`` +* Modify the following file ``/opt/sc4s/default/env_file`` * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 1ab3909..73c3046 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -7,9 +7,7 @@ Refer to [Getting Started](https://docs.docker.com/get-started/) * Create a systemd unit file use to start the container with the host os. ``/lib/systemd/system/sc4s.service`` -*NOTE*: In a future release, the mechanism to support mounted volumes (the three -v arguments in the unit file below) will change. For now, do not change these arguments, and ensure that the three files they reference are downloaded to the proper directory per the configuration instructions below. - -*NOTE-2*: Replace the URL and HEC tokens with the appropriate values for our environment +*NOTE*: Replace the URL and HEC tokens with the appropriate values for our environment ```ini [Unit] @@ -28,6 +26,7 @@ Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/s TimeoutStartSec=0 Restart=always + ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE ExecStartPre=/usr/bin/docker run \ --env-file=/opt/sc4s/default/env_file \ @@ -58,15 +57,12 @@ SPLUNK_METRICS_INDEX=em_metrics #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` -*NOTE*: While optional, until the final mechanism for mounted volumes in containers is released, do _not_ skip the download (wget) steps in -the configuration steps below. Ensure these three files are in place in the directory specified, even if there is no intent to modify them. - -## Configure index destinations for Splunk +## Configure index destinations for Splunk Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow customization of index destinations. -* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the unit file above (the ``-v`` arguments) +* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the unit file above (the ``-v`` variables) match the directory you created above. From this directory, execute the following to download the latest index context file: ```bash @@ -77,10 +73,10 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas ## Configure sources by source IP or host name Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. -* If not already done in the step immeidately above, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory -references in the unit file above (the ``-v`` arguments) match the directory you created above. From this directory, execute the following to +* If not already done in the step immediately above, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory +references in the unit file above (the ``-v`` variables) match the directory you created above. From this directory, execute the following to download the latest vendor context files: ```bash @@ -98,13 +94,13 @@ sudo systemctl start sc4s ``` -# Single Source Technology instance +# Configure Dedicated Listening Ports -For certain source technologies message categorization by content is impossible to support collection -of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using -an alternate port. +For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +For collection of such sources we provide a means of dedicating a unique listening port to a specific source. -Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. +Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Modify the individual ports or a range as appropriate for your network. @@ -139,10 +135,9 @@ ExecStart=/usr/bin/docker run -p 514:514 -p 5000-5020:5000-5020 \ --name SC4S \ --rm \ $SC4S_IMAGE - ``` -Modify the following file ``/opt/sc4s/default/env_file`` +* Modify the following file ``/opt/sc4s/default/env_file`` * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 4693995..65f3125 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -7,9 +7,7 @@ Refer to [Installation](https://podman.io/getting-started/installation) * Create a systemd unit file use to start the container with the host os. ``/lib/systemd/system/sc4s.service`` -*NOTE*: In a future release, the mechanism to support mounted volumes (the three -v arguments in the unit file below) will change. For now, do not change these arguments, and ensure that the three files they reference are downloaded to the proper directory per the configuration instructions below. - -*NOTE-2*: Replace the URL and HEC tokens with the appropriate values for our environment +*NOTE*: Replace the URL and HEC tokens with the appropriate values for our environment ```ini [Unit] @@ -18,7 +16,7 @@ After=network.service Requires=network.service [Service] -Environment="SC4S_IMAGE=splunk/sc4s:latest" +Environment="SC4S_IMAGE=splunk/scs:latest" #Note Uncomment this line to use custom index names AND download the splunk_index.csv file template per getting started Environment="SC4S_UNIT_SPLUNK_INDEX=-v /opt/sc4s/default/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv" @@ -43,10 +41,7 @@ $SC4S_IMAGE ``` -*NOTE*: While optional, until the final mechanism for mounted volumes in containers is released, do _not_ skip the download (wget) steps in -the configuration steps below. Ensure these three files are in place in the directory specified, even if there is no intent to modify them. - -## Configure the SC4S environment +## Configure the sc4s environment Create the following file ``/opt/sc4s/default/env_file`` @@ -62,12 +57,12 @@ SPLUNK_METRICS_INDEX=em_metrics #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` -## Configure index destinations for Splunk +## Configure index destinations for Splunk Log paths are preconfigured to utilize a convention of index destinations that is suitable for most customers. This step is optional to allow customization of index destinations. -* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the unit file above (the ``-v`` arguments) +* Create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory references in the unit file above (the ``-v`` variables) match the directory you created above. From this directory, execute the following to download the latest index context file: ```bash @@ -78,10 +73,10 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas ## Configure sources by source IP or host name Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. +apply to support such sources. To identify sources which require this step refer to the "sources" section of this documentation. -* If not already done in the step immeidately above, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory -references in the unit file above (the ``-v`` arguments) match the directory you created above. From this directory, execute the following to +* If not already done in the step immediately above, create a directory (e.g. ``/opt/sc4s/default/`` ). Make sure the local directory +references in the unit file above (the ``-v`` variables) match the directory you created above. From this directory, execute the following to download the latest vendor context files: ```bash @@ -93,18 +88,18 @@ sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/mas # Start SC4S ```bash -sudo systemctl daemon-reload +sudo systemctl daemon-reload sudo systemctl enable sc4s sudo systemctl start sc4s ``` -# Single Source Technology instance +# Configure Dedicated Listening Ports -For certain source technologies message categorization by content is impossible to support collection -of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using -an alternate port. +For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +For collection of such sources we provide a means of dedicating a unique listening port to a specific source. -Refer to the Sources documentation to identify the specific variable used to enable a specific port for the technology in use. +Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Modify the individual ports or a range as appropriate for your network. @@ -142,7 +137,7 @@ $SC4S_IMAGE ``` -Modify the following file ``/opt/sc4s/default/env_file`` +* Modify the following file ``/opt/sc4s/default/env_file`` * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment diff --git a/package/etc/conf.d/conflib/_common/network.conf.tmpl b/package/etc/conf.d/conflib/_common/network.conf.tmpl index 1592248..9b6c3a1 100644 --- a/package/etc/conf.d/conflib/_common/network.conf.tmpl +++ b/package/etc/conf.d/conflib/_common/network.conf.tmpl @@ -43,9 +43,7 @@ {{ end }} }; #TODO: #60 Remove this function with enhancement - rewrite(set_metadata_presume); rewrite(set_rfcnonconformant); - rewrite(r_set_splunk_default); {{ if eq (getenv "confgen_parser") "rfc5424_strict" }} filter(f_rfc5424_strict); @@ -90,4 +88,6 @@ rewrite(set_rfc3164); }; {{ end }} + rewrite(r_set_splunk_default); + }; diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/etc/conf.d/conflib/_splunk/splunkfields.conf index 68e9a01..b535d3b 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf @@ -1,13 +1,9 @@ #Used to set indexed fields we will always use to global defaults rewrite r_set_splunk_default { - set("`index`", value(".splunk.index")); - - set("`splunk-sourcetype`", value(".splunk.sourcetype")); - set("`splunk-source`:$LOGHOST", value(".splunk.source")); - set("`splunk-template`", value(".splunk.template")); + set("SC4S:$SOURCE", value(".splunk.source")); set($FACILITY, value("fields.sc4s_syslog_facility")); set($LEVEL, value("fields.sc4s_syslog_severity")); - set($LOGHOST, value("fields.sc4s_syslog_server")); + set($LOGHOST, value("fields.sc4s_log_host")); set($SOURCEIP, value("fields.sc4s_fromhostip")); }; #used by each log-path to set index and sourcetype which may be @@ -19,7 +15,7 @@ block rewrite r_set_splunk_dest_default( ) { set("`index`", value(".splunk.index")); set("`sourcetype`", value(".splunk.sourcetype")); - set("`template`", value(".splunk.template")); + set("`template`", value("fields.sc4s_template")); }; diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 441c5a9..1f374b9 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -11,21 +11,23 @@ user("sc4s") headers("{{getenv "SC4S_DEST_SPLUNK_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}") password("{{getenv "SPLUNK_HEC_TOKEN"}}") - persist-name("splunk_hec") -{{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE" "yes") "yes" }} + {{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE" "yes") "yes" }} + persist-name("splunk_hec") disk-buffer( -{{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE" "no") "yes" }} - mem-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024"}}) - reliable(yes) -{{else}} - mem-buf-length({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000"}}) - reliable(no) -{{ end }} -{{ end }} - disk-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "1048576"}}) - dir("/opt/syslog-ng/var/data/disk-buffer/") - ) + + {{ if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE" "no") "yes" }} + mem-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024"}}) + reliable(yes) + {{else}} + mem-buf-length({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000"}}) + reliable(no) + {{ end }} + + disk-buf-size({{getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "1048576"}}) + dir("/opt/syslog-ng/var/data/disk-buffer/") + ) + {{ end }} tls(peer-verify({{getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}}) {{if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}} cipher-suite("{{getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE"}}"){{end}} {{if ne (getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION") ""}} ssl-version("{{getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION"}}"){{end}} diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf b/package/etc/conf.d/destinations/splunk_hec_internal.conf new file mode 100644 index 0000000..fee875e --- /dev/null +++ b/package/etc/conf.d/destinations/splunk_hec_internal.conf @@ -0,0 +1,5 @@ +@module confgen context(destination) name(gen_splunk_hec_internal) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/destinations/splunk_hec_internal.conf.tmpl") + +destination d_hec_internal { + gen_splunk_hec_internal() +}; diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl new file mode 100644 index 0000000..32f4969 --- /dev/null +++ b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl @@ -0,0 +1,29 @@ + http( + url("{{getenv "SPLUNK_HEC_URL"}}") + method("POST") + log-fifo-size({{getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) + workers(1) + batch-lines({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_LINES" "1000"}}) + batch-bytes({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_BYTES" "4096kb"}}) + batch-timeout({{getenv "SC4S_DEST_SPLUNK_HEC_BATCH_TIMEOUT" "1"}}) + timeout({{getenv "SC4S_DEST_SPLUNK_HEC_TIMEOUT" "30"}}) + user_agent("sc4s/1.0 (events)") + user("sc4s") + headers("{{getenv "SC4S_DEST_SPLUNK_DEST_SPLUNK_HEC_HEADERS" "Connection: close"}}") + password("{{getenv "SPLUNK_HEC_TOKEN"}}") + + + tls(peer-verify({{getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}}) +{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE") ""}} cipher-suite("{{getenv "SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE"}}"){{end}} +{{if ne (getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION") ""}} ssl-version("{{getenv "SC4S_DEST_SPLUNK_HEC_SSL_VERSION"}}"){{end}} + ca-file("{{getenv "SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}}") + ) + body('$(format-json + time=$S_UNIXTIME.$S_MSEC + host=${HOST} + source=${.splunk.source} + sourcetype=${.splunk.sourcetype} + index=${.splunk.index} + event=$(template ${fields.sc4s_template} $(template "t_standard")) + fields.*)') + ); diff --git a/package/etc/conf.d/destinations/stdout.conf b/package/etc/conf.d/destinations/stdout.conf new file mode 100644 index 0000000..ee9e933 --- /dev/null +++ b/package/etc/conf.d/destinations/stdout.conf @@ -0,0 +1,6 @@ +@module confgen context(destination) name(gen_std_out) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/destinations/stdout.conf.tmpl") + + +destination d_stdout { + gen_std_out() +}; diff --git a/package/etc/conf.d/destinations/stdout.conf.tmpl b/package/etc/conf.d/destinations/stdout.conf.tmpl new file mode 100644 index 0000000..2540674 --- /dev/null +++ b/package/etc/conf.d/destinations/stdout.conf.tmpl @@ -0,0 +1 @@ + file("/dev/stdout"); \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/internal.conf b/package/etc/conf.d/log_paths/internal.conf index 743c15c..2582fd4 100644 --- a/package/etc/conf.d/log_paths/internal.conf +++ b/package/etc/conf.d/log_paths/internal.conf @@ -1,39 +1,9 @@ -log { - source(s_internal); - - if (match("^Log statistics; " value("MESSAGE"))) { +@module confgen context(log) name(gen_log_internal) exec("gomplate --file `syslog-ng-sysconfdir`/conf.d/log_paths/internal.conf.tmpl") - rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:metrics"), index("em_metrics")) }; - - parser {p_add_context_splunk(key("syslog-ng_metrics")); }; - rewrite { - subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?', - ' - {"time": "$S_UNIXTIME.$S_MSEC", - "event": "metric", - "host": "$HOST", - "index": "${.splunk.index}", - "source": "internal", - "sourcetype": "${.splunk.sourcetype}", - "fields": { - "source_name": "${SourceName}", - "source_instance": "${SourceInstance}", - "state": "${State}", - "type": "${Type}", - "_value": ${Number}, - "metric_name": "syslogng.${SourceId}" - } - } - ', - value("MESSAGE") flags("utf8" "global") - ); - }; - destination(d_hecmetrics); #--HEC-- - } else { - rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:events"), index("_internal"), template("t_msg_only"))}; +log { + #Default source + gen_log_internal( - parser {p_add_context_splunk(key("syslog-ng_events")); }; + ); - destination(d_hec); #--HEC-- - }; }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/internal.conf.tmpl b/package/etc/conf.d/log_paths/internal.conf.tmpl new file mode 100644 index 0000000..6d1d7aa --- /dev/null +++ b/package/etc/conf.d/log_paths/internal.conf.tmpl @@ -0,0 +1,40 @@ + source(s_internal); + + if (match("^Log statistics; " value("MESSAGE"))) { + + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:metrics"), index("em_metrics")) }; + + parser {p_add_context_splunk(key("sc4s_metrics")); }; + rewrite { + subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?', + ' + {"time": "$S_UNIXTIME.$S_MSEC", + "event": "metric", + "host": "$HOST", + "index": "${.splunk.index}", + "source": "internal", + "sourcetype": "${.splunk.sourcetype}", + "fields": { + "source_name": "${SourceName}", + "source_instance": "${SourceInstance}", + "state": "${State}", + "type": "${Type}", + "_value": ${Number}, + "metric_name": "syslogng.${SourceId}" + } + } + ', + value("MESSAGE") flags("utf8" "global") + ); + }; + destination(d_hecmetrics); #--HEC-- + } else { + + parser {p_add_context_splunk(key("sc4s_events")); }; + + {{ if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes" }} + destination(d_stdout); + {{ end }} + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"), template("t_msg_only"))}; + destination(d_hec_internal); + }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/zfallback.conf b/package/etc/conf.d/log_paths/zfallback.conf index 3b5b7fc..37d7a86 100644 --- a/package/etc/conf.d/log_paths/zfallback.conf +++ b/package/etc/conf.d/log_paths/zfallback.conf @@ -1,9 +1,9 @@ log { source(s_default-ports); - rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:fallback"), index("main"), template("t_JSON")) }; + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")) }; parser { - p_add_context_splunk(key("syslog-ng_fallback")); + p_add_context_splunk(key("sc4s_fallback")); }; destination(d_hec); #--HEC-- diff --git a/package/etc/conf.d/sources/internal.conf b/package/etc/conf.d/sources/internal.conf index 5ba19a8..3b1f5ba 100644 --- a/package/etc/conf.d/sources/internal.conf +++ b/package/etc/conf.d/sources/internal.conf @@ -3,5 +3,13 @@ # =============================================================================================== source s_internal { - internal(); + channel { + source { + internal(); + + }; + rewrite(r_set_splunk_default); + }; + + }; diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index 0cb943d..d40b914 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -40,9 +40,7 @@ ); }; #TODO: #60 Remove this function with enhancement - rewrite(set_metadata_presume); rewrite(set_rfcnonconformant); - rewrite(r_set_splunk_default); if { filter(f_rfc5424_strict); @@ -66,6 +64,8 @@ rewrite(set_rfc3164); }; + rewrite(r_set_splunk_default); + parser { vendor_product_by_source(); }; diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv index 71bcd53..1df4cc1 100644 --- a/package/etc/context-local/splunk_index.csv +++ b/package/etc/context-local/splunk_index.csv @@ -28,6 +28,6 @@ #pan_correlation,index,main #pan_userid,index,netauth #pan_unknown,index,netops -#syslog-ng_events,index,_internal -#syslog-ng_fallback,index,main -#syslog-ng_metrics,index,em_metrics +#sc4s_events,index,_internal +#sc4s_fallback,index,main +#sc4s_metrics,index,em_metrics diff --git a/tests/test_common.py b/tests/test_common.py index 24cb44a..e026893 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -20,7 +20,26 @@ def test_defaultroute(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"syslog-ng:fallback\" | head 2") + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"sc4s:fallback\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_internal(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} sc4sdefault[0]: test\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main NOT host=\"{{ host }}\" sourcetype=\"sc4s:events\" | head 1") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -39,7 +58,7 @@ def test_tag(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search index=main host=\"testvp-{{ host }}\" sourcetype=\"syslog-ng:fallback\" sc4s_vendor_product=test_test | head 2") + st = env.from_string("search index=main host=\"testvp-{{ host }}\" sourcetype=\"sc4s:fallback\" sc4s_vendor_product=test_test | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search)