From 41fab070b4a12d8b80328444149c7fde2d0feaf8 Mon Sep 17 00:00:00 2001 From: "William E. T. III" Date: Mon, 6 Apr 2020 19:09:13 -0400 Subject: [PATCH 1/2] Update splunk_hec.conf.tmpl Added dir argument to disk buffer in splunk_hec_conf.tmpl This change adds code to look for the SC4S_DEST_SPLUNK_HEC_DISKBUFF_DIR environmental variable and if present, it uses that directory for the directory of the disk buffer. The dir() option allows you to change the directory used for disk buffers. For 3.16 the feature is document at https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.16/administration-guide/disk-buffer The caveat is if you have already encountered disk buffering, there will be a persist file in the previous location (default /var/lib/syslog-ng) and that persist file will prevent syslog-ng from sucessfully using the new location. --- package/etc/conf.d/destinations/splunk_hec.conf.tmpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 463cba1..65f8dfc 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -26,6 +26,9 @@ destination d_hec { mem-buf-length({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) reliable(no) {{- end}} + {{- if ne (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DIR") ""}} + dir("{{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DIR"}}") + {{- end}} disk-buf-size({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "53687091200") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) ) {{- end}} From 77aa4523b93438b0f3af07c82f703b54477c4a8d Mon Sep 17 00:00:00 2001 From: "William E. Triest III" Date: Tue, 7 Apr 2020 09:41:43 -0400 Subject: [PATCH 2/2] Documented SC4S_DEST_SPLUNK_HEC_DISKBUFF_DIR variable --- docs/configuration.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index d4e8af5..1a3a1bd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -66,6 +66,11 @@ entire destination. Pay careful attention to this when using the "BYOE" version may hide this nuance. Lastly, be sure to factor in the syslog-ng data structure overhead (approx. 2x raw message size) when calculating the total buffer size needed. To determine the proper size of the disk buffer, consult the "Data Resilience" section below. +* NOTE: When changing the disk buffering directory, the new directory must exist. If it doesn't, then syslog-ng will fail to start. + +* NOTE: When changing the disk buffering directory, if buffering has previously occurd on that instance, a persist file may exist which will prevent syslog-ng from changing the directory. + + | Variable | Values/Default | Description | |----------|---------------|-------------| | SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE | yes(default) or no | Enable local disk buffering | @@ -73,6 +78,7 @@ total buffer size needed. To determine the proper size of the disk buffer, consu | SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE | bytes (10241024) | Memory buffer size in bytes (used with reliable disk buffering) | | SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH |messages (15000) | Memory buffer size in message count (used with normal disk buffering) | | SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE | bytes (53687091200) | size of local disk buffer in bytes (default 50 GB) | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_DIR | path | location to store the diskbuffering files | ## Archive File Configuration