diff --git a/docs/configuration.md b/docs/configuration.md index 1e84447..9f38c9d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,6 +21,27 @@ and variables needed to properly configure SC4S for your environment. | SC4S_DEST_SPLUNK_HEC_SSL_VERSION | comma separated list | Open SSL version list | | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | +## SC4S Disk Buffer Configuration + +Disk buffers in SC4S are allocated _per destination_. In the future as more destinations are supported, a separate list of variables +will be used for each. This is why you see the `DEST_SPLUNK_HEC` in the variable names below. +* NOTE: "Reliable" disk buffering offeres little advantage over "normal" disk buffering, at a significant performance penalty. +For this reason, normal disk buffering is recommended. +* NOTE: If you add destinations locally in your configuration, pay attention to the _cumulative_ buffer requirements when allocating local +disk. +* NOTE: The values for the variables below represent the _total_ sizes of the buffers for the destination. These sizes are divded by the +number of workers (threads) when setting the actual syslog-ng buffer options, because the buffer options apply to each worker rather than the +entire destination. Pay careful attention to this when using the "BYOE" version of SC4S, where direct access to the syslog-ng config files +may hide this nuance. + +| Variable | Values/Default | Description | +|----------|---------------|-------------| +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE | yes(default) or no | Enable local disk buffering | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE | yes or no(default) | Enable reliable/normal disk buffering (normal recommended) | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE | bytes (10241024) | Memory buffer size in bytes (used with reliable disk buffering) | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH |messages (15000) | Memory buffer size in message count (used with normal disk buffering) | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE | bytes (53687091200) | size of local disk buffer in bytes (default 50 GB) | + ## Archive File Configuration This feature is designed to support "compliance" archival of all messages. To enable this feature update the Unit file diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 8769667..bc92162 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -18,15 +18,14 @@ destination d_hec { disk-buffer( {{- if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE" "no") "yes"}} - mem-buf-size({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024"}}) + mem-buf-size({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) reliable(yes) {{- else}} - mem-buf-length({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000"}}) + mem-buf-length({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) reliable(no) {{- end}} - - disk-buf-size({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "1048576"}}) - dir("/opt/syslog-ng/var/data/disk-buffer/") + disk-buf-size({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "53687091200") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) + dir("/opt/syslog-ng/var/data/disk-buffer/") ) {{- end}} tls(peer-verify({{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}})