diff --git a/.idea/vcs.xml b/.idea/vcs.xml
index 94a25f7..53b20c8 100644
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
@@ -2,5 +2,8 @@
+
+
+
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 4f55352..1e3cebb 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -1,7 +1,36 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -11,46 +40,28 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
+
+
-
+
-
-
+
+
-
+
-
-
+
+
@@ -68,8 +79,12 @@
pip
yum
+ java
+
+
+
-
+
+
-
@@ -102,11 +121,6 @@
-
-
-
-
-
@@ -121,9 +135,11 @@
+
+
@@ -213,7 +229,8 @@
1561601135562
-
+
+
@@ -250,15 +267,15 @@
-
+
-
-
+
-
+
+
@@ -267,10 +284,10 @@
-
+
-
+
@@ -287,45 +304,74 @@
-
+
-
-
+
+
-
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
+
-
-
+
+
-
+
-
-
+
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/docker-compose.yml b/docker-compose.yml
index 6332ab6..3e865c9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,7 +4,51 @@ services:
build: ./tests
links:
- splunk
+ syslog-ng:
+ image: rfaircloth/scs:develop
+ build: ./package
+ hostname: syslog-ng
+ ports:
+ - "514"
+ - "601"
+ - "514/udp"
+ - "5514"
+ - "5514/udp"
+ stdin_open: true
+ tty: true
+ links:
+ - splunk
+ environment:
+ - SPLUNK_HEC_URL=https://splunk:8088/services/collector/event","https://splunk:8088/services/collector/event
+ - SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event
+ - SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
+ - SPLUNK_CONNECT_METHOD=hec
+ - SPLUNK_DEFAULT_INDEX=main
+ - SPLUNK_METRICS_INDEX=metrics
+# logging:
+# driver: splunk
+# options:
+# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
+# splunk-url: https://splunk:8088/services/collector/event
+# splunk-index: main
+# splunk-insecureskipverify: true
+# splunk-verify-connection: false
splunk:
image: splunk/splunk:latest
- syslog:
- build: ./package
\ No newline at end of file
+ hostname: splunk
+ ports:
+ - "8000:8000"
+ - "8088:8088"
+ - "8089:8089"
+ environment:
+ - SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
+ - SPLUNK_PASSWORD=Changed@11
+ - SPLUNK_START_ARGS=--accept-license
+# logging:
+# driver: splunk
+# options:
+# splunk-token: a778f63a-5dff-4e3c-a72c-a03183659e94
+# splunk-url: https://splunk:8088/services/collector/event
+# splunk-index: main
+# splunk-insecureskipverify: true
+# splunk-verify-connection: false
diff --git a/package/Dockerfile b/package/Dockerfile
index 18d17ea..3ef976b 100644
--- a/package/Dockerfile
+++ b/package/Dockerfile
@@ -56,22 +56,28 @@ RUN cd /tmp ;\
yum install epel-release-latest-7.noarch.rpm -y ;\
rm epel-release-latest-7.noarch.rpm ;\
rpm --import https://packages.confluent.io/rpm/5.2/archive.key ;\
- yum install tzdata libdbi libsecret libxml2 sqlite tcp_wrappers librdkafka \
- rh-python36 rh-python36-python-tools libcurl ivykis -y;\
- echo source scl_source enable rh-python36 >>/etc/profile.d/enablepython36.sh
+ yum install gcc tzdata libdbi libsecret libxml2 sqlite tcp_wrappers librdkafka \
+ rh-python36 rh-python36-python-tools libcurl ivykis scl-utils tcp_wrappers-libs -y;\
+ echo source scl_source enable rh-python36 >>/etc/profile.d/enablepython36.sh ;\
+ source scl_source enable rh-python36 ;\
+ pip install dumb-init
+
COPY --from=0 /opt/syslog-ng /opt/syslog-ng
-COPY scripts/* /opt/syslog-ng/sbin/
-COPY etc/* /opt/syslog-ng/etc
-COPY sbin/* /sbin/
+COPY scripts/splunkmetrics.sh /opt/syslog-ng/sbin/splunkmetrics.sh
+COPY scripts/switch_transport.sh /opt/syslog-ng/sbin/switch_transport.sh
+COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf
+COPY etc/conf.d /opt/syslog-ng/etc/conf.d
+COPY sbin/entrypoint.sh /sbin/entrypoint.sh
-RUN /opt/syslog-ng/sbin/syslog-ng -t
+RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V
+RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t
ENV DEBCONF_NONINTERACTIVE_SEEN=true
-ENV SPLUNK_CONNECT_METHOD=uf
+ENV SPLUNK_CONNECT_METHOD=hec
EXPOSE 514
EXPOSE 601/tcp
EXPOSE 6514/tcp
-ENTRYPOINT ["/sbin/entrypoint.sh", "--"]
+#ENTRYPOINT ["/sbin/entrypoint.sh", "--"]
diff --git a/package/etc/conf.d/b_destinations.conf b/package/etc/conf.d/b_destinations.conf
new file mode 100755
index 0000000..3e65461
--- /dev/null
+++ b/package/etc/conf.d/b_destinations.conf
@@ -0,0 +1,251 @@
+# ===============================================================================================
+# Destination Blocks (functions) for Splunk destinations
+# Block argument defaults are defined in main syslog-ng.conf file
+# Additional indexed fields (based on syslog-ng macros) can be added to the "with_metadata" blocks
+# for both HEC and Kafka.
+# Format is "fields.="
+# ===============================================================================================
+
+# ===============================================================================================
+# blocks for file destination (for use with the Splunk Universal Forwarder)
+# ===============================================================================================
+
+# ===============================================================================================
+# Logfile Local Defaults:
+# vendor: unassigned
+# message: suffix for log file names; "messages"
+# ===============================================================================================
+
+block destination d_splunk_logfile(category("")
+ vendor("unassigned")
+ filename("messages")
+ use_template(`splunk-default-template`)
+ ) {
+ file("`splunk-log-root`/`category`/$(lowercase \"${LOGHOST}\")/`vendor`/$(lowercase \"${HOST}\")/${YEAR}.${MONTH}.${DAY}.${HOUR}-`filename`"
+ template(`use_template`)
+ flush-lines(0)
+ create-dirs(yes)
+ dir_owner(`splunk-user`)
+ dir_group(`splunk-group`)
+ dir_perm(0700)
+ owner(`splunk-user`)
+ group(`splunk-group`)
+ perm(0600)
+ );
+ };
+
+# ===============================================================================================
+# blocks for direct connection to Splunk via Kafka and HEC
+# ===============================================================================================
+# ===============================================================================================
+# Kafka with Metadata (Facility and Priority):
+# ===============================================================================================
+
+block destination d_kafka_with_metadata (kafka-libs(`kafka-lib-dir`)
+ kafka-brokers(`splunk-kafka-brokers`)
+ sourcetype(`splunk-sourcetype`)
+ topic(`splunk-topic`)
+ index(`splunk-index`)
+ use_template(`splunk-default-template`)
+ ) {
+ kafka(client-lib-dir("`kafka-libs`")
+ kafka-bootstrap-servers("`kafka-brokers`")
+ persist-name("`kafka_brokers`_`topic`_`sourcetype`_`index`")
+ template('$(format-json
+ time=$S_UNIXTIME.$S_MSEC
+ host=$HOST
+ source=${HOST_FROM}
+ sourcetype=`sourcetype`
+ index=`index`
+ event=$(template `use_template`)
+ fields.facility=$FACILITY
+ fields.severity=$LEVEL)')
+ topic("`topic`")
+ );
+ };
+
+# ===============================================================================================
+# Kafka with Full Metadata; additional fields can be added
+# ===============================================================================================
+
+block destination d_kafka_with_full_metadata (kafka-libs(`kafka-lib-dir`)
+ kafka-brokers(`splunk-kafka-brokers`)
+ sourcetype(`splunk-sourcetype`)
+ topic(`splunk-topic`)
+ index(`splunk-index`)
+ use_template(`splunk-default-template`)
+ ) {
+ kafka(client-lib-dir("`kafka-libs`")
+ kafka-bootstrap-servers("`kafka-brokers`")
+ persist-name("`kafka_brokers`_`topic`_`sourcetype`_`index`")
+ template('$(format-json
+ time=$S_UNIXTIME.$S_MSEC
+ host=$HOST
+ source=${HOST_FROM}
+ sourcetype=`sourcetype`
+ index=`index`
+ event=$(template `use_template`)
+ fields.facility=$FACILITY
+ fields.severity=$LEVEL
+ fields.message_header=$MSGHDR
+ fields.program_name=$PROGRAM
+ fields.pid=$PID
+ fields.full_message=$(template t_everything)
+ )')
+ topic("`topic`")
+ );
+ };
+
+# ===============================================================================================
+# Basic Kafka (no indexed fields)
+# ===============================================================================================
+
+block destination d_kafka_basic (kafka-libs(`kafka-lib-dir`)
+ kafka-brokers(`splunk-kafka-brokers`)
+ sourcetype(`splunk-sourcetype`)
+ topic(`splunk-topic`)
+ index(`splunk-index`)
+ use_template(`splunk-default-template`)
+ ) {
+ kafka(client-lib-dir("`kafka-libs`")
+ kafka-bootstrap-servers("`kafka-brokers`")
+ persist-name("`kafka_brokers`_`topic`_`sourcetype`_`index`")
+ template('$(format-json time=$S_UNIXTIME
+ host=$HOST
+ source=$HOST_FROM
+ sourcetype=`sourcetype`
+ index=`index`
+ event=$(template `use_template`))')
+ topic("`topic`")
+ );
+ };
+
+# ===============================================================================================
+# HEC with Metadata (Facility and Priority):
+# Be sure to adjust batch paramaters below to suit scale/environment
+# If validated certs are used, uncomment relevant lines in the tls() block below
+# and change peer-verify() to "yes"
+# ===============================================================================================
+
+block destination d_hec_with_metadata (hec-url(`splunk-hec-url`)
+ hec-token(`splunk-hec-token`)
+ sourcetype(`splunk-sourcetype`)
+ index(`splunk-index`)
+ use_template(`splunk-default-template`)
+ ) {
+ http(url("`hec-url`")
+ method("POST")
+ batch-lines(10)
+ batch-bytes(512Kb)
+ batch-timeout(5000)
+ user_agent("syslog-ng User Agent")
+ user("syslog-ng")
+ password("`hec-token`")
+ persist-name("`hec-url`_`sourcetype`_`hec-token`_`index`")
+ tls(peer-verify(no)
+# ca-dir("dir")
+# ca-file("ca")
+# cert-file("cert")
+# cipher-suite("cipher")
+# key-file("key")
+# peer-verify(yes|no)
+# ssl-version()
+ )
+ body('$(format-json
+ time=$S_UNIXTIME.$S_MSEC
+ host=$HOST
+ source=${HOST_FROM}
+ sourcetype=`sourcetype`
+ index=`index`
+ event=$(template `use_template`)
+ fields.facility=$FACILITY
+ fields.severity=$LEVEL)')
+ );
+ };
+
+# ===============================================================================================
+# HEC with Full Metadata; additional fields can be added
+# Be sure to adjust batch paramaters below to suit scale/environment
+# If validated certs are used, uncomment relevant lines in the tls() block below
+# and change peer-verify() to "yes"
+# ===============================================================================================
+
+block destination d_hec_with_full_metadata(hec-url(`splunk-hec-url`)
+ hec-token(`splunk-hec-token`)
+ sourcetype(`splunk-sourcetype`)
+ index(`splunk-index`)
+ use_template(`splunk-default-template`)
+ ) {
+ http(url("`hec-url`")
+ method("POST")
+ batch-lines(10)
+ batch-bytes(512Kb)
+ batch-timeout(5000)
+ user_agent("syslog-ng User Agent")
+ user("syslog-ng")
+ password("`hec-token`")
+ persist-name("`hec-url`_`sourcetype`_`hec-token`_`index`")
+ tls(peer-verify(no)
+# ca-dir("dir")
+# ca-file("ca")
+# cert-file("cert")
+# cipher-suite("cipher")
+# key-file("key")
+# peer-verify(yes|no)
+# ssl-version()
+ )
+ body('$(format-json
+ time=$S_UNIXTIME.$S_MSEC
+ host=$HOST
+ source=${HOST_FROM}
+ sourcetype=`sourcetype`
+ index=`index`
+ event=$(template `use_template`)
+ fields.fullhost_from=${FULLHOST_FROM}
+ fields.facility=$FACILITY
+ fields.severity=$LEVEL
+ fields.message_header=$MSGHDR
+ fields.program_name=$PROGRAM
+ fields.pid=$PID
+ fields.full_message=$(template t_everything)
+ )')
+ );
+ };
+
+# ===============================================================================================
+# Basic HEC (no indexed fields)
+# ===============================================================================================
+
+block destination d_hec_basic (hec-url(`splunk-hec-url`)
+ hec-token(`splunk-hec-token`)
+ sourcetype(`splunk-sourcetype`)
+ index(`splunk-index`)
+ use_template(`splunk-default-template`)
+ ) {
+ http(url("`hec-url`")
+ method("POST")
+ batch-lines(10)
+ batch-bytes(512Kb)
+ batch-timeout(5000)
+ user_agent("syslog-ng User Agent")
+ user("syslog-ng")
+ password("`hec-token`")
+ persist-name("`hec-url`_`sourcetype`_`hec-token`_`index`")
+ tls(peer-verify(no)
+# ca-dir("dir")
+# ca-file("ca")
+# cert-file("cert")
+# cipher-suite("cipher")
+# key-file("key")
+# peer-verify(yes|no)
+# ssl-version()
+ )
+ body('$(format-json
+ time=$S_UNIXTIME.$S_MSEC
+ host=$HOST
+ source=${HOST_FROM}
+ sourcetype=`sourcetype`
+ index=`index`
+ event=$(template `use_template`))')
+ );
+ };
diff --git a/package/etc/conf.d/b_parsers.conf b/package/etc/conf.d/b_parsers.conf
new file mode 100755
index 0000000..b68a4e4
--- /dev/null
+++ b/package/etc/conf.d/b_parsers.conf
@@ -0,0 +1,33 @@
+# ===============================================================================================
+# ForcePoint (Websense)
+# ===============================================================================================
+
+# ===============================================================================================
+# Log Format:
+# <159>Dec 19 10:48:57 EST 10.203.28.21 vendor=Websense product=Security product_version=7.7.0 action=permitted severity=1 category=153 user=- src_host=10.64.134.74 src_port=62189 dst_host=mail.google.com dst_ip=74.125.224.53 dst_port=443 bytes_out=197 bytes_in=76 http_response=200 http_method=CONNECT http_content_type=- http_user_agent=Mozilla/5.0_(Windows;_U;_Windows_NT_6.1;_enUS;_rv:1.9.2.23)_Gecko/20110920_Firefox/3.6.23 http_proxy_status_code=200 reason=- disposition=1034 policy=- role=8 duration=0 url=https://mail.google.com
+#
+# Primary reason for non-compliance: %Z (timesone mnemonic) appended after date/time
+# ===============================================================================================
+
+block parser p_websense-parser(prefix(".websense.")) {
+ channel {
+ rewrite {
+ # normal BSD timestamp, plus a timezone code. Remove the
+ # timezone information for now.
+ subst('([A-Za-z]{3} [0-9 ]\d \d{2}:\d{2}:\d{2}) [A-Z]{3,4}' "$1 " flags(global) value("MSG"));
+
+ # add a $PROGRAM field, so that syslog-parser() would extract
+ # that properly
+ subst('(vendor=Websense)' "Websense: $1" flags(global) value("MSG"));
+ };
+ parser {
+ # by this time this message is a properly formatted syslog
+ # message
+ syslog-parser(time-zone(`default-timezone`));
+ # syslog-parser();
+
+ # extract name-value pairs.
+ # kv-parser(prefix("`prefix`"));
+ };
+ };
+};
diff --git a/package/etc/conf.d/blocks/b_sources.conf b/package/etc/conf.d/b_sources.conf
old mode 100644
new mode 100755
similarity index 94%
rename from package/etc/conf.d/blocks/b_sources.conf
rename to package/etc/conf.d/b_sources.conf
index 90a5e2d..5e53849
--- a/package/etc/conf.d/blocks/b_sources.conf
+++ b/package/etc/conf.d/b_sources.conf
@@ -17,6 +17,7 @@ block source s_unique-ports(udp_port() tcp_port() flags("no-parse")) {
use-dns(no)
use-fqdn(no)
chain-hostnames(off)
+ time-zone(`default-timezone`)
flags(`flags`)
);
@@ -36,6 +37,7 @@ block source s_unique-ports(udp_port() tcp_port() flags("no-parse")) {
use-dns(no)
use-fqdn(no)
chain-hostnames(off)
+ time-zone(`default-timezone`)
flags(`flags`)
);
};
@@ -55,6 +57,7 @@ block source s_unique-udp-port(udp_port() flags("no-parse")) {
use-dns(no)
use-fqdn(no)
chain-hostnames(off)
+ time-zone(`default-timezone`)
flags(`flags`)
);
};
@@ -76,6 +79,7 @@ block source s_unique-tcp-port(tcp_port() flags("no-parse")) {
use-dns(no)
use-fqdn(no)
chain-hostnames(off)
+ time-zone(`default-timezone`)
flags(`flags`)
);
};
diff --git a/package/etc/conf.d/blocks/b_parsers.conf b/package/etc/conf.d/blocks/b_parsers.conf
deleted file mode 100644
index c5d9063..0000000
--- a/package/etc/conf.d/blocks/b_parsers.conf
+++ /dev/null
@@ -1,96 +0,0 @@
-# ===============================================================================================
-# ForcePoint (Websense)
-# ===============================================================================================
-
-# ===============================================================================================
-# Log Format:
-# <159>Dec 19 10:48:57 EST 10.203.28.21 vendor=Websense product=Security product_version=7.7.0 action=permitted severity=1 category=153 user=- src_host=10.64.134.74 src_port=62189 dst_host=mail.google.com dst_ip=74.125.224.53 dst_port=443 bytes_out=197 bytes_in=76 http_response=200 http_method=CONNECT http_content_type=- http_user_agent=Mozilla/5.0_(Windows;_U;_Windows_NT_6.1;_enUS;_rv:1.9.2.23)_Gecko/20110920_Firefox/3.6.23 http_proxy_status_code=200 reason=- disposition=1034 policy=- role=8 duration=0 url=https://mail.google.com
-#
-# Primary reason for non-compliance: %Z (timesone mnemonic) appended after date/time
-# ===============================================================================================
-
-block parser p_websense-parser(prefix(".websense.")) {
- channel {
- rewrite {
- # normal BSD timestamp, plus a timezone code. Remove the
- # timezone information for now.
- subst('([A-Za-z]{3} [0-9 ]\d \d{2}:\d{2}:\d{2}) [A-Z]{3,4}' "$1 " flags(global) value("MSG"));
-
- # add a $PROGRAM field, so that syslog-parser() would extract
- # that properly
- subst('(vendor=Websense)' "Websense: $1" flags(global) value("MSG"));
- };
- parser {
- # by this time this message is a properly formatted syslog message
- syslog-parser(time-zone(`default-timezone`));
- # syslog-parser();
-
- # extract name-value pairs.
- # kv-parser(prefix("`prefix`"));
- };
- };
-};
-
-block parser p_RFC5424-lite(prefix(".rfc5424-lite.")) {
- channel {
- rewrite {
- # Remove Version Identifier
- subst('(<\d+>)\d+' "$1" value("MSG"));
-
- };
- parser {
- # by this time this message will parse with the BSD 3164 parser
- syslog-parser(time-zone(`default-timezone`));
- # syslog-parser();
-
- # extract name-value pairs.
- # kv-parser(prefix("`prefix`"));
- };
- };
-};
-
-block parser p_test-parser(prefix(".testparse.")) {
- channel {
- rewrite {
- # Put your rewrite rule here;use "MSG" as no-parse() puts entire payload there
- # subst('(>.{12})\s' "${1}T" value("MSG"));
- };
- parser {
- # Don't initially parse so full output can be seen
- # syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol));
- # syslog-parser();
- # syslog-parser(time-zone(`default-timezone`));
- # date-parser(format("%Y-%m-%d %H:%M:%S") template("$(substr ${MESSAGE} 0 19)") time-zone(`default-timezone`));
- };
- };
-};
-
-block parser p_add_context_data(data-source("unassigned")) {
- add-contextual-data(selector(`data-source`), database("sourcetype-db.csv"), default-selector("unknown"));
-};
-
-block rewrite r_set_splunk(sourcetype(`splunk-sourcetype`)
- index(`splunk-index`)
- template(`splunk-default-template`)
- token(`splunk-hec-token`)
- url(`splunk-hec-url`)
- brokers(`splunk-kafka-brokers`)
- topic(`splunk-kafka-topic`)
- vendor("unassigned")
- category("")
- filename("messages")
- ) {
- set("`index`", value(".splunk.index"));
- set("`sourcetype`", value(".splunk.sourcetype"));
- set("`template`", value(".splunk.template"));
- set('`url`', value(".splunk.url"));
- set("`token`", value(".splunk.token"));
- set("`brokers`", value(".splunk.brokers"));
- set("`topic`", value(".splunk.topic"));
- set("`vendor`", value(".splunk.vendor"));
- set("`category`", value(".splunk.category"));
- set("`filename`", value(".splunk.filename"));
- set($FACILITY, value("fields.facility"));
- set($LEVEL, value("fields.severity"));
- set($LOGHOST, value("fields.syslog-server"));
- };
diff --git a/package/etc/conf.d/destinations/d_UF.conf b/package/etc/conf.d/destinations/d_UF.conf
deleted file mode 100644
index 9c9281a..0000000
--- a/package/etc/conf.d/destinations/d_UF.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# ===============================================================================================
-# File destination (for use with the Splunk Universal Forwarder)
-# ===============================================================================================
-
-# ===============================================================================================
-# Logfile Local Defaults:
-# vendor: unassigned
-# message: suffix for log file names: "messages"
-# ===============================================================================================
-
-destination d_splunk_logfile {
- file("`splunk-log-root`/${.splunk.category}/$(lowercase \"${LOGHOST}\")/${.splunk.vendor}/$(lowercase \"${HOST}\")/${YEAR}.${MONTH}.${DAY}.${HOUR}-${.splunk.filename}"
- template(`splunk-default-template`)
- flush-lines(0)
- create-dirs(yes)
- dir_owner(`splunk-user`)
- dir_group(`splunk-group`)
- dir_perm(0700)
- owner(`splunk-user`)
- group(`splunk-group`)
- perm(0600)
- );
- };
diff --git a/package/etc/conf.d/destinations/d_hec.conf b/package/etc/conf.d/destinations/d_hec.conf
deleted file mode 100644
index 0b808d1..0000000
--- a/package/etc/conf.d/destinations/d_hec.conf
+++ /dev/null
@@ -1,185 +0,0 @@
-# ===============================================================================================
-# Direct connection to Splunk via HEC
-# ===============================================================================================
-
-# ===============================================================================================
-# HEC with standard template
-# Be sure to adjust batch paramaters below to suit scale/environment
-# Set workers to the number of indexers or HWF HEC endpoints
-# If validated certs are used, uncomment relevant lines in the tls() block below
-# and change peer-verify() to "yes"
-# ===============================================================================================
-
-destination d_hec_standard {
- http(url(`splunk-hec-url`)
- method("POST")
- log-fifo-size(`splunk-log-fifo-size`)
- workers(`worker-threads`)
- batch-lines(1000)
- batch-bytes(20Mb)
- batch-timeout(1)
- timeout(5)
- user_agent("syslog-ng User Agent")
- user("syslog-ng")
- password(`splunk-hec-token`)
- persist-name("t_standard")
-# disk-buffer(mem-buf-length(200000)
-# disk-buf-size(107374182400)
-# reliable(no)
-# dir("/usr/local/etc//disk-buffer"))
- tls(peer-verify(no)
-# ca-dir("dir")
-# ca-file("ca")
-# cert-file("cert")
-# cipher-suite("cipher")
-# key-file("key")
-# peer-verify(yes|no)
-# ssl-version()
- )
- body('$(format-json
- time=$S_UNIXTIME.$S_MSEC
- host=$HOST
- source=${HOST_FROM}
- sourcetype=${.splunk.sourcetype}
- index=${.splunk.index}
- event=$(template t_standard)
- fields.*)')
- );
- };
-
-
-# ===============================================================================================
-# HEC with t_hdr_msg template (for Palo Alto)
-# Be sure to adjust batch paramaters below to suit scale/environment
-# Set workers to the number of indexers or HWF HEC endpoints
-# If validated certs are used, uncomment relevant lines in the tls() block below
-# and change peer-verify() to "yes"
-# ===============================================================================================
-
-destination d_hec_hdr_msg {
- http(url(`splunk-hec-url`)
- method("POST")
- log-fifo-size(`splunk-log-fifo-size`)
- workers(`worker-threads`)
- batch-lines(1000)
- batch-bytes(20Mb)
- batch-timeout(1)
- timeout(5)
- user_agent("syslog-ng User Agent")
- user("syslog-ng")
- password(`splunk-hec-token`)
- persist-name("t_hdr_msg")
-# disk-buffer(mem-buf-length(200000)
-# disk-buf-size(107374182400)
-# reliable(no)
-# dir("/usr/local/etc//disk-buffer"))
- tls(peer-verify(no)
-# ca-dir("dir")
-# ca-file("ca")
-# cert-file("cert")
-# cipher-suite("cipher")
-# key-file("key")
-# peer-verify(yes|no)
-# ssl-version()
- )
- body('$(format-json
- time=$S_UNIXTIME.$S_MSEC
- host=$HOST
- source=${HOST_FROM}
- sourcetype=${.splunk.sourcetype}
- index=${.splunk.index}
- event=$(template t_hdr_msg)
- fields.*)')
- );
- };
-
-
-# ===============================================================================================
-# HEC with the "everything" template
-# Be sure to adjust batch paramaters below to suit scale/environment
-# Set workers to the number of indexers or HWF HEC endpoints
-# If validated certs are used, uncomment relevant lines in the tls() block below
-# and change peer-verify() to "yes"
-# ===============================================================================================
-
-destination d_hec_everything {
- http(url(`splunk-hec-url`)
- method("POST")
- log-fifo-size(`splunk-log-fifo-size`)
- workers(`worker-threads`)
- batch-lines(1000)
- batch-bytes(20Mb)
- batch-timeout(1)
- timeout(5)
- user_agent("syslog-ng User Agent")
- user("syslog-ng")
- password(`splunk-hec-token`)
- persist-name("t_everything")
-# disk-buffer(mem-buf-length(200000)
-# disk-buf-size(107374182400)
-# reliable(no)
-# dir("/usr/local/etc//disk-buffer"))
- tls(peer-verify(no)
-# ca-dir("dir")
-# ca-file("ca")
-# cert-file("cert")
-# cipher-suite("cipher")
-# key-file("key")
-# peer-verify(yes|no)
-# ssl-version()
- )
- body('$(format-json
- time=$S_UNIXTIME.$S_MSEC
- host=$HOST
- source=${HOST_FROM}
- sourcetype=${.splunk.sourcetype}
- index=${.splunk.index}
- event=$(template t_everything)
- fields.*)')
- );
- };
-
-# ===============================================================================================
-# HEC with the "JSON" template
-# Be sure to adjust batch paramaters below to suit scale/environment
-# Set workers to the number of indexers or HWF HEC endpoints
-# If validated certs are used, uncomment relevant lines in the tls() block below
-# and change peer-verify() to "yes"
-# ===============================================================================================
-
-destination d_hec_json {
- http(url(`splunk-hec-url`)
- method("POST")
- log-fifo-size(`splunk-log-fifo-size`)
- workers(`worker-threads`)
- batch-lines(1000)
- batch-bytes(20Mb)
- batch-timeout(1)
- timeout(5)
- user_agent("syslog-ng User Agent")
- user("syslog-ng")
- password(`splunk-hec-token`)
- persist-name("t_json")
-# disk-buffer(mem-buf-length(200000)
-# disk-buf-size(107374182400)
-# reliable(no)
-# dir("/usr/local/etc//disk-buffer"))
- tls(peer-verify(no)
-# ca-dir("dir")
-# ca-file("ca")
-# cert-file("cert")
-# cipher-suite("cipher")
-# key-file("key")
-# peer-verify(yes|no)
-# ssl-version()
- )
- body('$(format-json
- time=$S_UNIXTIME.$S_MSEC
- host=$HOST
- source=${HOST_FROM}
- sourcetype=${.splunk.sourcetype}
- index=${.splunk.index}
- event=$(template t_JSON)
- fields.*)')
- );
- };
diff --git a/package/etc/conf.d/destinations/d_kafka.conf b/package/etc/conf.d/destinations/d_kafka.conf
deleted file mode 100644
index 1776f0e..0000000
--- a/package/etc/conf.d/destinations/d_kafka.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# ===============================================================================================
-# Direct connection to Splunk via Kafka
-# ===============================================================================================
-
-# ===============================================================================================
-# Kafka with default template
-# ===============================================================================================
-
-destination d_kafka_default {
- kafka-c(
- bootstrap-servers("`splunk-kafka-brokers`")
- log-fifo-size(`splunk-log-fifo-size`)
- message('$(format-json
- time=$S_UNIXTIME.$S_MSEC
- host=$HOST
- source=${HOST_FROM}
- sourcetype=${.splunk.sourcetype}
- index=${.splunk.index}
- event=$(template `splunk-default-template`)
- fields.*)')
- topic("`splunk-kafka-topic`")
- );
- };
diff --git a/package/etc/conf.d/filters/f_catchall.conf b/package/etc/conf.d/f_catchall.conf
old mode 100644
new mode 100755
similarity index 72%
rename from package/etc/conf.d/filters/f_catchall.conf
rename to package/etc/conf.d/f_catchall.conf
index e46de3c..aa9abbf
--- a/package/etc/conf.d/filters/f_catchall.conf
+++ b/package/etc/conf.d/f_catchall.conf
@@ -30,9 +30,7 @@ source(s_default-ports);
# ===============================================================================================
# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure.
-# parser { syslog-parser(time-zone(`default-timezone`)); };
-# parser { date-parser(format("%Y-%m-%d %H:%M:%S") template("$(substr ${MESSAGE} 8 26)")); };
-# parser { p_test-parser(); };
+parser { syslog-parser(time-zone(`default-timezone`)); };
# For RFC5424-formatted messages, use this instead
# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); };
@@ -48,25 +46,32 @@ source(s_default-ports);
# ===============================================================================================
# Section 4: destination configuration
# Select the desired destination to use in the log path from the three choices below,
-# based on the transfer method (UF/HEC/Kafka) desired. Then, using the rewrite rule below,
-# provide the sourcetype and optional index (if different from the default) for thie data
-# source. Then, uncomment the second rewrite declartion to add any additional indexed fields
-# desired.
+# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string
+# for UF transport, and a unique sourcetype argument and optional index, topic, and message
+# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the
+# note below.
+#
+# IMPORTANT: If modifications are made in this section, ensure that two or more distinct
+# destinations do *not* reference the same destination block call with an idential set of
+# arguments (including no arguments, and arguments passed via destinations declared in other
+# conf.d include files), or you will see this at startup: "Error checking the uniqueness of
+# the persist names, please override it with persist-name option. Shutting down."
+# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new
+# or changed destination declaration.
# ===============================================================================================
# ===============================================================================================
-# This catchall file should be the *only* one where the rewrite rules use defaults (no arguments)!
-# The "everything" destination is used here; check indexed event data in Splunk for full message.
+# This catchall file should be the *only* one where the destinations use defaults for all arguments!
+# Full-metadata destinations used here; check indexed event data in Splunk for full message.
# This can be helpful to determine the compliance of message to standard syslog formats, and as
# an aid for custom parser development if needed.
# ===============================================================================================
-# destination(d_splunk_logfile); #--UF--
- rewrite { r_set_splunk()}; #--HEC--
- destination(d_hec_everything); #--HEC--
-# rewrite { r_set_splunk()}; #--KAFKA--
-# destination(d_kafka_default); #--KAFKA--
-# };
+destination {
+# d_splunk_logfile(); #--UF--
+ d_hec_with_full_metadata(); #--HEC--
+# d_kafka_with_full_metadata(); #--KAFKA--
+ };
# ===============================================================================================
# Fallback flag set here; will catch all unfiltered traffic
diff --git a/package/etc/conf.d/filters/f_cisco-ios.conf b/package/etc/conf.d/f_cisco-ios.conf
old mode 100644
new mode 100755
similarity index 64%
rename from package/etc/conf.d/filters/f_cisco-ios.conf
rename to package/etc/conf.d/f_cisco-ios.conf
index 2f1eaeb..c224a7a
--- a/package/etc/conf.d/filters/f_cisco-ios.conf
+++ b/package/etc/conf.d/f_cisco-ios.conf
@@ -24,8 +24,7 @@ source(s_default-ports);
# ===============================================================================================
# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure.
-# parser { syslog-parser(time-zone(`default-timezone`)); };
- parser { cisco-parser(); };
+parser { syslog-parser(time-zone(`default-timezone`)); };
# For RFC5424-formatted messages, use this instead
# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); };
@@ -35,25 +34,31 @@ source(s_default-ports);
# Filter below may be omitted in the log path if a dedicated source is configured above
# ===============================================================================================
-#filter { match("[%|#][A-Z0-9_]+-[A-Z012_]*-*[0-7]-[A-Z0-9_]+:" value("MESSAGE"))
-# ;};
+filter { match("[%|#][A-Z0-9_]+-[A-Z012_]*-*[0-7]-[A-Z0-9_]+:" value("MESSAGE"))
+ ;};
+
# ===============================================================================================
# Section 4: destination configuration
# Select the desired destination to use in the log path from the three choices below,
-# based on the transfer method (UF/HEC/Kafka) desired. Then, using the rewrite rule below,
-# provide the sourcetype and optional index (if different from the default) for thie data
-# source. Then, uncomment the second rewrite declartion to add any additional indexed fields
-# desired.
+# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string
+# for UF transport, and a unique sourcetype argument and optional index, topic, and message
+# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the
+# note below.
+#
+# IMPORTANT: If modifications are made in this section, ensure that two or more distinct
+# destinations do *not* reference the same destination block call with an idential set of
+# arguments (including no arguments, and arguments passed via destinations declared in other
+# conf.d include files), or you will see this at startup: "Error checking the uniqueness of
+# the persist names, please override it with persist-name option. Shutting down."
+# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new
+# or changed destination declaration.
# ===============================================================================================
-# rewrite { r_set_splunk(vendor("cisco-IOS"))}; #--UF--
-# destination(d_splunk_logfile); #--UF--
- rewrite { r_set_splunk(sourcetype("cisco:ios")) }; #--HEC--
-# rewrite { set("testvalue", value("fields.testfield")); }; #--HEC--
- destination(d_hec_standard); #--HEC--
-# rewrite { r_set_splunk(sourcetype("cisco:ios") index("cisco_infrastructure")) }; #--KAFKA--
-# destination(d_kafka_standard); #--KAFKA--
-
+destination {
+# d_splunk_logfile(vendor("cisco-ios")); #--UF--
+ d_hec_with_metadata(sourcetype("cisco:ios")); #--HEC--
+# d_kafka_with_metadata(sourcetype("cisco:ios")); #--KAFKA--
+ };
flags(final);
};
diff --git a/package/etc/conf.d/filters/f_palo_alto.conf b/package/etc/conf.d/f_palo_alto.conf
old mode 100644
new mode 100755
similarity index 69%
rename from package/etc/conf.d/filters/f_palo_alto.conf
rename to package/etc/conf.d/f_palo_alto.conf
index 61c5b18..868b94c
--- a/package/etc/conf.d/filters/f_palo_alto.conf
+++ b/package/etc/conf.d/f_palo_alto.conf
@@ -44,23 +44,24 @@ filter { match(",THREAT," value("MESSAGE"))
# ===============================================================================================
# Section 4: destination configuration
# Select the desired destination to use in the log path from the three choices below,
-# based on the transfer method (UF/HEC/Kafka) desired. Then, using the rewrite rule below,
-# provide the sourcetype and optional index (if different from the default) for thie data
-# source. Then, uncomment the second rewrite declartion to add any additional indexed fields
-# desired.
-# ===============================================================================================
-
-# rewrite { r_set_splunk(vendor("paloalto"))}; #--UF--
-# destination(d_splunk_logfile); #--UF--
- rewrite { r_set_splunk(sourcetype("pan:log")) }; #--HEC--
-# rewrite { set("testvalue", value("fields.testfield")); }; #--HEC--
- destination(d_hec_everything); #--HEC--
-# rewrite { r_set_splunk(sourcetype("pan:log")) }; #--KAFKA--
-# destination(d_kafka_with_metadata); #--KAFKA--
-
-# ===============================================================================================
-# Final flag set here; further processing stops on filter match
+# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string
+# for UF transport, and a unique sourcetype argument and optional index, topic, and message
+# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the
+# note below.
+#
+# IMPORTANT: If modifications are made in this section, ensure that two or more distinct
+# destinations do *not* reference the same destination block call with an idential set of
+# arguments (including no arguments, and arguments passed via destinations declared in other
+# conf.d include files), or you will see this at startup: "Error checking the uniqueness of
+# the persist names, please override it with persist-name option. Shutting down."
+# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new
+# or changed destination declaration.
# ===============================================================================================
+destination {
+# d_splunk_logfile(vendor("paloalto")); #--UF--
+ d_hec_with_metadata(sourcetype("pan:log") use_template("t_everything")); #--HEC--
+# d_kafka_with_metadata(sourcetype("pan:log") use_template("t_everything")); #--KAFKA--
+ };
flags(final);
};
diff --git a/package/etc/conf.d/filters/f_websense.conf b/package/etc/conf.d/f_websense.conf
old mode 100644
new mode 100755
similarity index 69%
rename from package/etc/conf.d/filters/f_websense.conf
rename to package/etc/conf.d/f_websense.conf
index 855410a..16fb2af
--- a/package/etc/conf.d/filters/f_websense.conf
+++ b/package/etc/conf.d/f_websense.conf
@@ -45,19 +45,24 @@ filter { program(Websense); };
# ===============================================================================================
# Section 4: destination configuration
# Select the desired destination to use in the log path from the three choices below,
-# based on the transfer method (UF/HEC/Kafka) desired. Then, using the rewrite rule below,
-# provide the sourcetype and optional index (if different from the default) for thie data
-# source. Then, uncomment the second rewrite declartion to add any additional indexed fields
-# desired.
+# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string
+# for UF transport, and a unique sourcetype argument and optional index, topic, and message
+# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the
+# note below.
+#
+# IMPORTANT: If modifications are made in this section, ensure that two or more distinct
+# destinations do *not* reference the same destination block call with an idential set of
+# arguments (including no arguments, and arguments passed via destinations declared in other
+# conf.d include files), or you will see this at startup: "Error checking the uniqueness of
+# the persist names, please override it with persist-name option. Shutting down."
+# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new
+# or changed destination declaration.
# ===============================================================================================
-# rewrite { r_set_splunk(vendor("websense"))}; #--UF--
-# destination(d_splunk_logfile); #--UF--
- rewrite { r_set_splunk(sourcetype("websense:cg:kv")) }; #--HEC--
-# rewrite { set("testvalue", value("fields.testfield")); }; #--HEC--
- destination(d_hec_everything); #--HEC--
-# rewrite { r_set_splunk(sourcetype("websense:cg:kv")) }; #--KAFKA--
-# destination(d_kafka_with_metadata); #--KAFKA--
-
+destination {
+# d_splunk_logfile(vendor("websense")); #--UF--
+ d_hec_with_metadata(sourcetype("websense:cg:kv")); #--HEC--
+# d_kafka_with_metadata(sourcetype("websense:cg:kv")); #--KAFKA--
+ };
flags(final);
};
diff --git a/package/etc/conf.d/filters/f_cyberark.conf b/package/etc/conf.d/filters/f_cyberark.conf
deleted file mode 100644
index 94d5c8f..0000000
--- a/package/etc/conf.d/filters/f_cyberark.conf
+++ /dev/null
@@ -1,60 +0,0 @@
-# ===============================================================================================
-# Cisco IOS (Route/Switch)
-# ===============================================================================================
-
-log {
-# ===============================================================================================
-# Section 1: source configuration
-# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf.
-#
-# IMPORTANT: In order to ensure proper startup, select only one source in this section.
-# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed
-# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those
-# in source declarations from other filter files, an "Address already in use" error will occur
-# at startup.
-# ===============================================================================================
-
-# source(s_default-ports);
-
-source { s_unique-ports(udp_port(6514) tcp_port(6514)); };
-
-# ===============================================================================================
-# Section 2: parser configuration
-# Default syslog-parser() should work for most messages
-# ===============================================================================================
-
-# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure.
-# parser { syslog-parser(time-zone(`default-timezone`)); };
-# parser { cisco-parser(); };
-
-# For RFC5424-formatted messages, use this instead
-# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); };
- parser { p_RFC5424-lite(); };
-
-# ===============================================================================================
-# Section 3: filter configuration
-# Filter below may be omitted in the log path if a dedicated source is configured above
-# ===============================================================================================
-
-#filter { match("[%|#][A-Z0-9_]+-[A-Z012_]*-*[0-7]-[A-Z0-9_]+:" value("MESSAGE"))
-# ;};
-
-# ===============================================================================================
-# Section 4: destination configuration
-# Select the desired destination to use in the log path from the three choices below,
-# based on the transfer method (UF/HEC/Kafka) desired. Then, using the rewrite rule below,
-# provide the sourcetype and optional index (if different from the default) for thie data
-# source. Then, uncomment the second rewrite declartion to add any additional indexed fields
-# desired.
-# ===============================================================================================
-
-# rewrite { r_set_splunk(vendor("cisco-IOS"))}; #--UF--
-# destination(d_splunk_logfile); #--UF--
- rewrite { r_set_splunk(sourcetype("cyberark")) }; #--HEC--
-# rewrite { set("testvalue", value("fields.testfield")); }; #--HEC--
- destination(d_hec_hdr_msg); #--HEC--
-# rewrite { r_set_splunk(sourcetype("cyberark") index("cisco_infrastructure")) }; #--KAFKA--
-# destination(d_kafka_standard); #--KAFKA--
-
-flags(final);
-};
diff --git a/package/etc/conf.d/sources/s_defaults.conf b/package/etc/conf.d/s_defaults.conf
old mode 100644
new mode 100755
similarity index 96%
rename from package/etc/conf.d/sources/s_defaults.conf
rename to package/etc/conf.d/s_defaults.conf
index cd7186f..40425dd
--- a/package/etc/conf.d/sources/s_defaults.conf
+++ b/package/etc/conf.d/s_defaults.conf
@@ -25,6 +25,7 @@ source s_default-ports {
use-dns(no)
use-fqdn(no)
chain-hostnames(off)
+ time-zone(`default-timezone`)
flags(no-parse)
);
@@ -44,6 +45,7 @@ source s_default-ports {
use-dns(no)
use-fqdn(no)
chain-hostnames(off)
+ time-zone(`default-timezone`)
flags(no-parse)
);
};
diff --git a/package/etc/conf.d/splunk.conf b/package/etc/conf.d/splunk.conf
old mode 100644
new mode 100755
index eb337ec..fdbb14e
--- a/package/etc/conf.d/splunk.conf
+++ b/package/etc/conf.d/splunk.conf
@@ -26,16 +26,16 @@
# ===============================================================================================
# Kafka transport
# ===============================================================================================
-@define kafka-implementation "kafka-c"
-@define splunk-kafka-brokers "kafka-1:19092,kafka-2:29092,kafka-3:39092"
-@define splunk-kafka-topic "syslog"
+@define kafka-lib-dir "/opt/kafka_2.11-0.11.0.0/libs"
+@define splunk-kafka-brokers "127.0.0.1:9092"
+@define splunk-topic "syslog"
# ===============================================================================================
# HEC transport
# ===============================================================================================
-@define splunk-hec-url '__SPLUNK_HEC_URL__'
-@define splunk-hec-token "__SPLUNK_HEC_TOKEN__"
-@define worker-threads 20
+@define splunk-hec-url "http://127.0.0.1:9088/services/collector/event"
+@define splunk-hec-token "18f31be3-17e6-4e7b-88b0-4bdfbdb7ec88"
+
# ===============================================================================================
# Splunk metadata (HEC/Kafka transport only)
# ===============================================================================================
@@ -43,7 +43,7 @@
@define splunk-index "main"
# ===============================================================================================
-# Default message template
+# Default message template
# ===============================================================================================
@define splunk-default-template "t_standard"
@@ -52,7 +52,6 @@
# ===============================================================================================
@define ip-version 4
@define splunk-max-connections 1000
-@define splunk-log-fifo-size 100000
@define splunk-fetch-limit 100
# make sure splunk-window-size >= splunk-max-connections * splunk-fetch-limit
@define splunk-window-size 100000
@@ -63,8 +62,8 @@
# Global modules and includes. All device-specific filters and destinations exist in conf.d
# Order is important; templates must be loaded first
# ===============================================================================================
-@include "conf.d/templates/*.conf"
-@include "conf.d/sources/*.conf"
-@include "conf.d/blocks/*.conf"
-@include "conf.d/destinations/*.conf"
-@include "conf.d/filters/*.conf"
+@include "scl.conf"
+@include "conf.d/t_*.conf"
+@include "conf.d/s_*.conf"
+@include "conf.d/b_*.conf"
+@include "conf.d/f_*.conf"
diff --git a/package/etc/conf.d/templates/t_templates.conf b/package/etc/conf.d/t_templates.conf
old mode 100644
new mode 100755
similarity index 57%
rename from package/etc/conf.d/templates/t_templates.conf
rename to package/etc/conf.d/t_templates.conf
index e2a595a..b75dbbf
--- a/package/etc/conf.d/templates/t_templates.conf
+++ b/package/etc/conf.d/t_templates.conf
@@ -8,7 +8,7 @@
# ===============================================================================================
template t_standard {
- template("${ISODATE} ${HOST} ${MESSAGE}");
+ template("${ISODATE} ${HOST} ${MESSAGE}\n");
};
# ===============================================================================================
@@ -16,29 +16,13 @@ template t_standard {
# ===============================================================================================
template t_msg_only {
- template("${MESSAGE}");
+ template("${MESSAGE}\n");
};
# ===============================================================================================
-# Everything; useful for bluecoat proxy and others
+# Everything; useful for bluecoat proxy, palo alto, and others
# ===============================================================================================
template t_everything {
- template("${ISODATE} ${HOST} ${MSGHDR}${MESSAGE}");
+ template("${ISODATE} ${HOST} ${MSGHDR}${MESSAGE}\n");
};
-
-# ===============================================================================================
-# Message Header with Message; for Palo Alto
-# ===============================================================================================
-
-template t_hdr_msg {
- template("${MSGHDR}${MESSAGE}");
- };
-
-# ===============================================================================================
-# JSON; for JSON pretty-printing
-# ===============================================================================================
-
-template t_JSON {
- template("$(format-json --scope all-nv-pairs)");
- };
diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf
index 2389d62..612d826 100755
--- a/package/etc/syslog-ng.conf
+++ b/package/etc/syslog-ng.conf
@@ -1,18 +1,14 @@
-#############################################################################
-# Default syslog-ng.conf file which collects all local logs into a
-# single file called /var/log/messages tailored to container usage.
+@version:3.22
+
+# syslog-ng configuration file.
#
-# The changes from the stock, default syslog-ng.conf file is that we've
-# dropped the system() source that is not needed and that we enabled network
-# connections using default-network-drivers(). Customize as needed and
-# override using the -v option to docker, such as:
+# This should behave pretty much like the original syslog on RedHat. But
+# it could be configured a lot smarter.
#
-# docker run ... -v "$PWD/syslog-ng.conf":/etc/syslog-ng/syslog-ng.conf
+# See syslog-ng(8) and syslog-ng.conf(5) for more information.
#
-@version: 3.22
-@include "scl.conf"
-
+#
# ===============================================================================================
# Global Options
# ===============================================================================================
@@ -27,22 +23,64 @@ options {
use_fqdn (no);
create_dirs (no);
keep-hostname (yes);
- stats-level(3);
- stats-max-dynamics(500);
};
-source s_local {
- internal();
-};
+# ===============================================================================================
+# Remote data sources that will be sent to Splunk
+# ===============================================================================================
-destination d_console {
- pipe("/dev/stdout");
-};
+@include "conf.d/splunk.conf"
-@include "conf.d/*.conf"
+# ===============================================================================================
+# Local System sources
+# ===============================================================================================
-log {
- source(s_local);
- destination(d_console);
- flags(final);
+source s_sys {
+ internal();
};
+
+# ===============================================================================================
+# Local System destinations
+# ===============================================================================================
+
+destination d_cons { file("/dev/console"); };
+destination d_mesg { file("/var/log/syslog-ng/messages"); };
+destination d_auth { file("/dev/shm/secure"); };
+destination d_mail { file("/dev/shm/maillog" flush_lines(10)); };
+destination d_spol { file("/dev/shm/spooler"); };
+destination d_boot { file("/dev/shm/boot.log"); };
+destination d_cron { file("/dev/shm/cron"); };
+destination d_kern { file("/dev/shm/kern"); };
+destination d_mlal { usertty("*"); };
+destination d_syslog { file("/var/log/syslog"); };
+
+# ===============================================================================================
+# Local System filters
+# ===============================================================================================
+
+filter f_kernel { facility(kern); };
+filter f_default { level(info..emerg) and
+ not (facility(mail)
+ or facility(authpriv)
+ or facility(cron)); };
+filter f_auth { facility(authpriv); };
+filter f_mail { facility(mail); };
+filter f_emergency { level(emerg); };
+filter f_news { facility(uucp) or
+ (facility(news)
+ and level(crit..emerg)); };
+filter f_boot { facility(local7); };
+filter f_cron { facility(cron); };
+
+# ===============================================================================================
+# Local System Logging
+# ===============================================================================================
+
+log { source(s_sys); filter(f_kernel); destination(d_kern); };
+log { source(s_sys); filter(f_default); destination(d_mesg); };
+log { source(s_sys); filter(f_auth); destination(d_auth); };
+log { source(s_sys); filter(f_mail); destination(d_mail); };
+log { source(s_sys); filter(f_emergency); destination(d_mlal); };
+log { source(s_sys); filter(f_news); destination(d_spol); };
+log { source(s_sys); filter(f_boot); destination(d_boot); };
+log { source(s_sys); filter(f_cron); destination(d_cron); };
diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh
index d323b2e..edd411e 100755
--- a/package/sbin/entrypoint.sh
+++ b/package/sbin/entrypoint.sh
@@ -3,6 +3,6 @@
#Run syslog
mkdir /opt/syslog-ng/var
rm /opt/syslog-ng/var/syslog-ng.ctl
-/opt/syslog-ng/bin/switch_transport.sh hec
+/opt/syslog-ng/sbin/switch_transport.sh hec
/opt/syslog-ng/sbin/syslog-ng --process-mode=background
/opt/syslog-ng/sbin/splunkmetrics.sh