From c59c14e7b5819550ed03f1a6a15a64fc3b7b5855 Mon Sep 17 00:00:00 2001 From: Ryan Faircloth Date: Thu, 27 Jun 2019 22:22:17 -0400 Subject: [PATCH] Functional build --- package/Dockerfile | 16 +- package/etc/conf.d/blocks/b_parsers.conf | 96 +++++++++ package/etc/conf.d/blocks/b_sources.conf | 81 ++++++++ package/etc/conf.d/destinations/d_UF.conf | 23 +++ package/etc/conf.d/destinations/d_hec.conf | 185 ++++++++++++++++++ package/etc/conf.d/destinations/d_kafka.conf | 23 +++ package/etc/conf.d/filters/f_catchall.conf | 75 +++++++ package/etc/conf.d/filters/f_cisco-ios.conf | 59 ++++++ package/etc/conf.d/filters/f_cyberark.conf | 60 ++++++ package/etc/conf.d/filters/f_palo_alto.conf | 66 +++++++ package/etc/conf.d/filters/f_websense.conf | 63 ++++++ package/etc/conf.d/sources/s_defaults.conf | 49 +++++ package/etc/conf.d/splunk.conf | 70 +++++++ package/etc/conf.d/templates/t_templates.conf | 44 +++++ package/etc/syslog-ng.conf | 48 +++++ package/sbin/entrypoint.sh | 8 + package/sbin/goss.yaml | 16 ++ 17 files changed, 981 insertions(+), 1 deletion(-) create mode 100644 package/etc/conf.d/blocks/b_parsers.conf create mode 100644 package/etc/conf.d/blocks/b_sources.conf create mode 100644 package/etc/conf.d/destinations/d_UF.conf create mode 100644 package/etc/conf.d/destinations/d_hec.conf create mode 100644 package/etc/conf.d/destinations/d_kafka.conf create mode 100644 package/etc/conf.d/filters/f_catchall.conf create mode 100644 package/etc/conf.d/filters/f_cisco-ios.conf create mode 100644 package/etc/conf.d/filters/f_cyberark.conf create mode 100644 package/etc/conf.d/filters/f_palo_alto.conf create mode 100644 package/etc/conf.d/filters/f_websense.conf create mode 100644 package/etc/conf.d/sources/s_defaults.conf create mode 100644 package/etc/conf.d/splunk.conf create mode 100644 package/etc/conf.d/templates/t_templates.conf create mode 100755 package/etc/syslog-ng.conf create mode 100755 package/sbin/entrypoint.sh create mode 100644 package/sbin/goss.yaml diff --git a/package/Dockerfile b/package/Dockerfile index 1e8764e..18d17ea 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -60,4 +60,18 @@ RUN cd /tmp ;\ rh-python36 rh-python36-python-tools libcurl ivykis -y;\ echo source scl_source enable rh-python36 >>/etc/profile.d/enablepython36.sh -COPY --from=0 /opt/syslog-ng /opt/syslog-ng \ No newline at end of file +COPY --from=0 /opt/syslog-ng /opt/syslog-ng +COPY scripts/* /opt/syslog-ng/sbin/ +COPY etc/* /opt/syslog-ng/etc +COPY sbin/* /sbin/ + +RUN /opt/syslog-ng/sbin/syslog-ng -t + +ENV DEBCONF_NONINTERACTIVE_SEEN=true +ENV SPLUNK_CONNECT_METHOD=uf + +EXPOSE 514 +EXPOSE 601/tcp +EXPOSE 6514/tcp + +ENTRYPOINT ["/sbin/entrypoint.sh", "--"] diff --git a/package/etc/conf.d/blocks/b_parsers.conf b/package/etc/conf.d/blocks/b_parsers.conf new file mode 100644 index 0000000..c5d9063 --- /dev/null +++ b/package/etc/conf.d/blocks/b_parsers.conf @@ -0,0 +1,96 @@ +# =============================================================================================== +# ForcePoint (Websense) +# =============================================================================================== + +# =============================================================================================== +# Log Format: +# <159>Dec 19 10:48:57 EST 10.203.28.21 vendor=Websense product=Security product_version=7.7.0 action=permitted severity=1 category=153 user=- src_host=10.64.134.74 src_port=62189 dst_host=mail.google.com dst_ip=74.125.224.53 dst_port=443 bytes_out=197 bytes_in=76 http_response=200 http_method=CONNECT http_content_type=- http_user_agent=Mozilla/5.0_(Windows;_U;_Windows_NT_6.1;_enUS;_rv:1.9.2.23)_Gecko/20110920_Firefox/3.6.23 http_proxy_status_code=200 reason=- disposition=1034 policy=- role=8 duration=0 url=https://mail.google.com +# +# Primary reason for non-compliance: %Z (timesone mnemonic) appended after date/time +# =============================================================================================== + +block parser p_websense-parser(prefix(".websense.")) { + channel { + rewrite { + # normal BSD timestamp, plus a timezone code. Remove the + # timezone information for now. + subst('([A-Za-z]{3} [0-9 ]\d \d{2}:\d{2}:\d{2}) [A-Z]{3,4}' "$1 " flags(global) value("MSG")); + + # add a $PROGRAM field, so that syslog-parser() would extract + # that properly + subst('(vendor=Websense)' "Websense: $1" flags(global) value("MSG")); + }; + parser { + # by this time this message is a properly formatted syslog message + syslog-parser(time-zone(`default-timezone`)); + # syslog-parser(); + + # extract name-value pairs. + # kv-parser(prefix("`prefix`")); + }; + }; +}; + +block parser p_RFC5424-lite(prefix(".rfc5424-lite.")) { + channel { + rewrite { + # Remove Version Identifier + subst('(<\d+>)\d+' "$1" value("MSG")); + + }; + parser { + # by this time this message will parse with the BSD 3164 parser + syslog-parser(time-zone(`default-timezone`)); + # syslog-parser(); + + # extract name-value pairs. + # kv-parser(prefix("`prefix`")); + }; + }; +}; + +block parser p_test-parser(prefix(".testparse.")) { + channel { + rewrite { + # Put your rewrite rule here;use "MSG" as no-parse() puts entire payload there + # subst('(>.{12})\s' "${1}T" value("MSG")); + }; + parser { + # Don't initially parse so full output can be seen + # syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); + # syslog-parser(); + # syslog-parser(time-zone(`default-timezone`)); + # date-parser(format("%Y-%m-%d %H:%M:%S") template("$(substr ${MESSAGE} 0 19)") time-zone(`default-timezone`)); + }; + }; +}; + +block parser p_add_context_data(data-source("unassigned")) { + add-contextual-data(selector(`data-source`), database("sourcetype-db.csv"), default-selector("unknown")); +}; + +block rewrite r_set_splunk(sourcetype(`splunk-sourcetype`) + index(`splunk-index`) + template(`splunk-default-template`) + token(`splunk-hec-token`) + url(`splunk-hec-url`) + brokers(`splunk-kafka-brokers`) + topic(`splunk-kafka-topic`) + vendor("unassigned") + category("") + filename("messages") + ) { + set("`index`", value(".splunk.index")); + set("`sourcetype`", value(".splunk.sourcetype")); + set("`template`", value(".splunk.template")); + set('`url`', value(".splunk.url")); + set("`token`", value(".splunk.token")); + set("`brokers`", value(".splunk.brokers")); + set("`topic`", value(".splunk.topic")); + set("`vendor`", value(".splunk.vendor")); + set("`category`", value(".splunk.category")); + set("`filename`", value(".splunk.filename")); + set($FACILITY, value("fields.facility")); + set($LEVEL, value("fields.severity")); + set($LOGHOST, value("fields.syslog-server")); + }; diff --git a/package/etc/conf.d/blocks/b_sources.conf b/package/etc/conf.d/blocks/b_sources.conf new file mode 100644 index 0000000..90a5e2d --- /dev/null +++ b/package/etc/conf.d/blocks/b_sources.conf @@ -0,0 +1,81 @@ +# =============================================================================================== +# Source block for data arriving on unique ports; parsing defaults to "no-parse" +# =============================================================================================== + +block source s_unique-ports(udp_port() tcp_port() flags("no-parse")) { + `splunk-udp-driver` ( + transport("udp") + port(`udp_port`) + ip-protocol(`ip-version`) +# hook-commands( +# setup("firewall-cmd --permanent --add-port=514/udp") +# teardown("firewall-cmd --permanent --remove-port=514/udp") +# ) + so-rcvbuf(`splunk-rcvbuf`) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(`flags`) + ); + + `splunk-tcp-driver` ( + transport("tcp") + port(`tcp_port`) + ip-protocol(`ip-version`) +# hook-commands( +# setup("firewall-cmd --permanent --add-port=514/tcp") +# teardown("firewall-cmd --permanent --remove-port=514/tcp") +# ) + max-connections(`splunk-max-connections`) + log-iw-size(`splunk-window-size`) + log-fetch-limit(`splunk-fetch-limit`) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(`flags`) + ); +}; + +block source s_unique-udp-port(udp_port() flags("no-parse")) { + `splunk-udp-driver` ( + transport("udp") + port(`udp_port`) + ip-protocol(`ip-version`) +# hook-commands( +# setup("firewall-cmd --permanent --add-port=514/udp") +# teardown("firewall-cmd --permanent --remove-port=514/udp") +# ) + so-rcvbuf(`splunk-rcvbuf`) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(`flags`) + ); +}; + +block source s_unique-tcp-port(tcp_port() flags("no-parse")) { + `splunk-tcp-driver` ( + transport("tcp") + port(`tcp_port`) + ip-protocol(`ip-version`) +# hook-commands( +# setup("firewall-cmd --permanent --add-port=514/tcp") +# teardown("firewall-cmd --permanent --remove-port=514/tcp") +# ) + max-connections(`splunk-max-connections`) + log-iw-size(`splunk-window-size`) + log-fetch-limit(`splunk-fetch-limit`) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(`flags`) + ); +}; diff --git a/package/etc/conf.d/destinations/d_UF.conf b/package/etc/conf.d/destinations/d_UF.conf new file mode 100644 index 0000000..9c9281a --- /dev/null +++ b/package/etc/conf.d/destinations/d_UF.conf @@ -0,0 +1,23 @@ +# =============================================================================================== +# File destination (for use with the Splunk Universal Forwarder) +# =============================================================================================== + +# =============================================================================================== +# Logfile Local Defaults: +# vendor: unassigned +# message: suffix for log file names: "messages" +# =============================================================================================== + +destination d_splunk_logfile { + file("`splunk-log-root`/${.splunk.category}/$(lowercase \"${LOGHOST}\")/${.splunk.vendor}/$(lowercase \"${HOST}\")/${YEAR}.${MONTH}.${DAY}.${HOUR}-${.splunk.filename}" + template(`splunk-default-template`) + flush-lines(0) + create-dirs(yes) + dir_owner(`splunk-user`) + dir_group(`splunk-group`) + dir_perm(0700) + owner(`splunk-user`) + group(`splunk-group`) + perm(0600) + ); + }; diff --git a/package/etc/conf.d/destinations/d_hec.conf b/package/etc/conf.d/destinations/d_hec.conf new file mode 100644 index 0000000..0b808d1 --- /dev/null +++ b/package/etc/conf.d/destinations/d_hec.conf @@ -0,0 +1,185 @@ +# =============================================================================================== +# Direct connection to Splunk via HEC +# =============================================================================================== + +# =============================================================================================== +# HEC with standard template +# Be sure to adjust batch paramaters below to suit scale/environment +# Set workers to the number of indexers or HWF HEC endpoints +# If validated certs are used, uncomment relevant lines in the tls() block below +# and change peer-verify() to "yes" +# =============================================================================================== + +destination d_hec_standard { + http(url(`splunk-hec-url`) + method("POST") + log-fifo-size(`splunk-log-fifo-size`) + workers(`worker-threads`) + batch-lines(1000) + batch-bytes(20Mb) + batch-timeout(1) + timeout(5) + user_agent("syslog-ng User Agent") + user("syslog-ng") + password(`splunk-hec-token`) + persist-name("t_standard") +# disk-buffer(mem-buf-length(200000) +# disk-buf-size(107374182400) +# reliable(no) +# dir("/usr/local/etc//disk-buffer")) + tls(peer-verify(no) +# ca-dir("dir") +# ca-file("ca") +# cert-file("cert") +# cipher-suite("cipher") +# key-file("key") +# peer-verify(yes|no) +# ssl-version() + ) + body('$(format-json + time=$S_UNIXTIME.$S_MSEC + host=$HOST + source=${HOST_FROM} + sourcetype=${.splunk.sourcetype} + index=${.splunk.index} + event=$(template t_standard) + fields.*)') + ); + }; + + +# =============================================================================================== +# HEC with t_hdr_msg template (for Palo Alto) +# Be sure to adjust batch paramaters below to suit scale/environment +# Set workers to the number of indexers or HWF HEC endpoints +# If validated certs are used, uncomment relevant lines in the tls() block below +# and change peer-verify() to "yes" +# =============================================================================================== + +destination d_hec_hdr_msg { + http(url(`splunk-hec-url`) + method("POST") + log-fifo-size(`splunk-log-fifo-size`) + workers(`worker-threads`) + batch-lines(1000) + batch-bytes(20Mb) + batch-timeout(1) + timeout(5) + user_agent("syslog-ng User Agent") + user("syslog-ng") + password(`splunk-hec-token`) + persist-name("t_hdr_msg") +# disk-buffer(mem-buf-length(200000) +# disk-buf-size(107374182400) +# reliable(no) +# dir("/usr/local/etc//disk-buffer")) + tls(peer-verify(no) +# ca-dir("dir") +# ca-file("ca") +# cert-file("cert") +# cipher-suite("cipher") +# key-file("key") +# peer-verify(yes|no) +# ssl-version() + ) + body('$(format-json + time=$S_UNIXTIME.$S_MSEC + host=$HOST + source=${HOST_FROM} + sourcetype=${.splunk.sourcetype} + index=${.splunk.index} + event=$(template t_hdr_msg) + fields.*)') + ); + }; + + +# =============================================================================================== +# HEC with the "everything" template +# Be sure to adjust batch paramaters below to suit scale/environment +# Set workers to the number of indexers or HWF HEC endpoints +# If validated certs are used, uncomment relevant lines in the tls() block below +# and change peer-verify() to "yes" +# =============================================================================================== + +destination d_hec_everything { + http(url(`splunk-hec-url`) + method("POST") + log-fifo-size(`splunk-log-fifo-size`) + workers(`worker-threads`) + batch-lines(1000) + batch-bytes(20Mb) + batch-timeout(1) + timeout(5) + user_agent("syslog-ng User Agent") + user("syslog-ng") + password(`splunk-hec-token`) + persist-name("t_everything") +# disk-buffer(mem-buf-length(200000) +# disk-buf-size(107374182400) +# reliable(no) +# dir("/usr/local/etc//disk-buffer")) + tls(peer-verify(no) +# ca-dir("dir") +# ca-file("ca") +# cert-file("cert") +# cipher-suite("cipher") +# key-file("key") +# peer-verify(yes|no) +# ssl-version() + ) + body('$(format-json + time=$S_UNIXTIME.$S_MSEC + host=$HOST + source=${HOST_FROM} + sourcetype=${.splunk.sourcetype} + index=${.splunk.index} + event=$(template t_everything) + fields.*)') + ); + }; + +# =============================================================================================== +# HEC with the "JSON" template +# Be sure to adjust batch paramaters below to suit scale/environment +# Set workers to the number of indexers or HWF HEC endpoints +# If validated certs are used, uncomment relevant lines in the tls() block below +# and change peer-verify() to "yes" +# =============================================================================================== + +destination d_hec_json { + http(url(`splunk-hec-url`) + method("POST") + log-fifo-size(`splunk-log-fifo-size`) + workers(`worker-threads`) + batch-lines(1000) + batch-bytes(20Mb) + batch-timeout(1) + timeout(5) + user_agent("syslog-ng User Agent") + user("syslog-ng") + password(`splunk-hec-token`) + persist-name("t_json") +# disk-buffer(mem-buf-length(200000) +# disk-buf-size(107374182400) +# reliable(no) +# dir("/usr/local/etc//disk-buffer")) + tls(peer-verify(no) +# ca-dir("dir") +# ca-file("ca") +# cert-file("cert") +# cipher-suite("cipher") +# key-file("key") +# peer-verify(yes|no) +# ssl-version() + ) + body('$(format-json + time=$S_UNIXTIME.$S_MSEC + host=$HOST + source=${HOST_FROM} + sourcetype=${.splunk.sourcetype} + index=${.splunk.index} + event=$(template t_JSON) + fields.*)') + ); + }; diff --git a/package/etc/conf.d/destinations/d_kafka.conf b/package/etc/conf.d/destinations/d_kafka.conf new file mode 100644 index 0000000..1776f0e --- /dev/null +++ b/package/etc/conf.d/destinations/d_kafka.conf @@ -0,0 +1,23 @@ +# =============================================================================================== +# Direct connection to Splunk via Kafka +# =============================================================================================== + +# =============================================================================================== +# Kafka with default template +# =============================================================================================== + +destination d_kafka_default { + kafka-c( + bootstrap-servers("`splunk-kafka-brokers`") + log-fifo-size(`splunk-log-fifo-size`) + message('$(format-json + time=$S_UNIXTIME.$S_MSEC + host=$HOST + source=${HOST_FROM} + sourcetype=${.splunk.sourcetype} + index=${.splunk.index} + event=$(template `splunk-default-template`) + fields.*)') + topic("`splunk-kafka-topic`") + ); + }; diff --git a/package/etc/conf.d/filters/f_catchall.conf b/package/etc/conf.d/filters/f_catchall.conf new file mode 100644 index 0000000..e46de3c --- /dev/null +++ b/package/etc/conf.d/filters/f_catchall.conf @@ -0,0 +1,75 @@ +# =============================================================================================== +# "Catch-all" destination; all unfiltered messages end up here +# =============================================================================================== + +log { +# =============================================================================================== +# Section 1: source configuration +# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. +# +# IMPORTANT: In order to ensure proper startup, select only one source in this section. +# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed +# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those +# in source declarations from other filter files, an "Address already in use" error will occur +# at startup. +# +# NOTE: The "Catchall" should not use any source other than the default. Be extremely careful +# when defining an alternate source here. Default ports can be changed in the global declarations +# in the main syslog-ng.conf file. +# =============================================================================================== + +source(s_default-ports); + +# =============================================================================================== +# Section 2: parser configuration +# Default syslog-parser() should work for most messages +# +# If a new data source is not properly parsing using the default, you may wish to comment out +# all parsers to see the raw message as it appeared "on the wire". This can be useful for +# creating custom parsers for non-syslog-compliant meessages. +# =============================================================================================== + +# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. +# parser { syslog-parser(time-zone(`default-timezone`)); }; +# parser { date-parser(format("%Y-%m-%d %H:%M:%S") template("$(substr ${MESSAGE} 8 26)")); }; +# parser { p_test-parser(); }; + +# For RFC5424-formatted messages, use this instead +# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); }; + +# =============================================================================================== +# Section 3: filter configuration +# Filter below may be omitted in the log path if a dedicated source is configured above +# Filter typically not used for a fallback destination +# =============================================================================================== + +# filter { }; + +# =============================================================================================== +# Section 4: destination configuration +# Select the desired destination to use in the log path from the three choices below, +# based on the transfer method (UF/HEC/Kafka) desired. Then, using the rewrite rule below, +# provide the sourcetype and optional index (if different from the default) for thie data +# source. Then, uncomment the second rewrite declartion to add any additional indexed fields +# desired. +# =============================================================================================== + +# =============================================================================================== +# This catchall file should be the *only* one where the rewrite rules use defaults (no arguments)! +# The "everything" destination is used here; check indexed event data in Splunk for full message. +# This can be helpful to determine the compliance of message to standard syslog formats, and as +# an aid for custom parser development if needed. +# =============================================================================================== + +# destination(d_splunk_logfile); #--UF-- + rewrite { r_set_splunk()}; #--HEC-- + destination(d_hec_everything); #--HEC-- +# rewrite { r_set_splunk()}; #--KAFKA-- +# destination(d_kafka_default); #--KAFKA-- +# }; + +# =============================================================================================== +# Fallback flag set here; will catch all unfiltered traffic +# =============================================================================================== +flags(fallback); +}; diff --git a/package/etc/conf.d/filters/f_cisco-ios.conf b/package/etc/conf.d/filters/f_cisco-ios.conf new file mode 100644 index 0000000..2f1eaeb --- /dev/null +++ b/package/etc/conf.d/filters/f_cisco-ios.conf @@ -0,0 +1,59 @@ +# =============================================================================================== +# Cisco IOS (Route/Switch) +# =============================================================================================== + +log { +# =============================================================================================== +# Section 1: source configuration +# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. +# +# IMPORTANT: In order to ensure proper startup, select only one source in this section. +# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed +# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those +# in source declarations from other filter files, an "Address already in use" error will occur +# at startup. +# =============================================================================================== + +source(s_default-ports); + +# source { s_unique-ports(udp_port(814) tcp_port(814)); }; + +# =============================================================================================== +# Section 2: parser configuration +# Default syslog-parser() should work for most messages +# =============================================================================================== + +# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. +# parser { syslog-parser(time-zone(`default-timezone`)); }; + parser { cisco-parser(); }; + +# For RFC5424-formatted messages, use this instead +# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); }; + +# =============================================================================================== +# Section 3: filter configuration +# Filter below may be omitted in the log path if a dedicated source is configured above +# =============================================================================================== + +#filter { match("[%|#][A-Z0-9_]+-[A-Z012_]*-*[0-7]-[A-Z0-9_]+:" value("MESSAGE")) +# ;}; + +# =============================================================================================== +# Section 4: destination configuration +# Select the desired destination to use in the log path from the three choices below, +# based on the transfer method (UF/HEC/Kafka) desired. Then, using the rewrite rule below, +# provide the sourcetype and optional index (if different from the default) for thie data +# source. Then, uncomment the second rewrite declartion to add any additional indexed fields +# desired. +# =============================================================================================== + +# rewrite { r_set_splunk(vendor("cisco-IOS"))}; #--UF-- +# destination(d_splunk_logfile); #--UF-- + rewrite { r_set_splunk(sourcetype("cisco:ios")) }; #--HEC-- +# rewrite { set("testvalue", value("fields.testfield")); }; #--HEC-- + destination(d_hec_standard); #--HEC-- +# rewrite { r_set_splunk(sourcetype("cisco:ios") index("cisco_infrastructure")) }; #--KAFKA-- +# destination(d_kafka_standard); #--KAFKA-- + +flags(final); +}; diff --git a/package/etc/conf.d/filters/f_cyberark.conf b/package/etc/conf.d/filters/f_cyberark.conf new file mode 100644 index 0000000..94d5c8f --- /dev/null +++ b/package/etc/conf.d/filters/f_cyberark.conf @@ -0,0 +1,60 @@ +# =============================================================================================== +# Cisco IOS (Route/Switch) +# =============================================================================================== + +log { +# =============================================================================================== +# Section 1: source configuration +# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. +# +# IMPORTANT: In order to ensure proper startup, select only one source in this section. +# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed +# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those +# in source declarations from other filter files, an "Address already in use" error will occur +# at startup. +# =============================================================================================== + +# source(s_default-ports); + +source { s_unique-ports(udp_port(6514) tcp_port(6514)); }; + +# =============================================================================================== +# Section 2: parser configuration +# Default syslog-parser() should work for most messages +# =============================================================================================== + +# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. +# parser { syslog-parser(time-zone(`default-timezone`)); }; +# parser { cisco-parser(); }; + +# For RFC5424-formatted messages, use this instead +# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); }; + parser { p_RFC5424-lite(); }; + +# =============================================================================================== +# Section 3: filter configuration +# Filter below may be omitted in the log path if a dedicated source is configured above +# =============================================================================================== + +#filter { match("[%|#][A-Z0-9_]+-[A-Z012_]*-*[0-7]-[A-Z0-9_]+:" value("MESSAGE")) +# ;}; + +# =============================================================================================== +# Section 4: destination configuration +# Select the desired destination to use in the log path from the three choices below, +# based on the transfer method (UF/HEC/Kafka) desired. Then, using the rewrite rule below, +# provide the sourcetype and optional index (if different from the default) for thie data +# source. Then, uncomment the second rewrite declartion to add any additional indexed fields +# desired. +# =============================================================================================== + +# rewrite { r_set_splunk(vendor("cisco-IOS"))}; #--UF-- +# destination(d_splunk_logfile); #--UF-- + rewrite { r_set_splunk(sourcetype("cyberark")) }; #--HEC-- +# rewrite { set("testvalue", value("fields.testfield")); }; #--HEC-- + destination(d_hec_hdr_msg); #--HEC-- +# rewrite { r_set_splunk(sourcetype("cyberark") index("cisco_infrastructure")) }; #--KAFKA-- +# destination(d_kafka_standard); #--KAFKA-- + +flags(final); +}; diff --git a/package/etc/conf.d/filters/f_palo_alto.conf b/package/etc/conf.d/filters/f_palo_alto.conf new file mode 100644 index 0000000..61c5b18 --- /dev/null +++ b/package/etc/conf.d/filters/f_palo_alto.conf @@ -0,0 +1,66 @@ +# =============================================================================================== +# Palo Alto Networks Firewalls +# =============================================================================================== + +log { +# =============================================================================================== +# Section 1: source configuration +# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. +# +# IMPORTANT: In order to ensure proper startup, select only one source in this section. +# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed +# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those +# in source declarations from other filter files, an "Address already in use" error will occur +# at startup. +# =============================================================================================== + +source (s_default-ports); + +# source { s_unique-ports(udp_port(714) tcp_port(714)); }; + +# =============================================================================================== +# Section 2: parser configuration +# Default syslog-parser() should work for most messages +# =============================================================================================== + +# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. +parser { syslog-parser(time-zone(`default-timezone`)); }; + +# For RFC5424-formatted messages, use this instead +# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); }; + +# =============================================================================================== +# Section 3: filter configuration +# Filter below may be omitted in the log path if a dedicated source is configured above +# =============================================================================================== + +filter { match(",THREAT," value("MESSAGE")) + or match(",TRAFFIC," value("MESSAGE")) + or match(",SYSTEM," value("MESSAGE")) + or match(",CONFIG," value("MESSAGE")) + or match(",HIPWATCH," value("MESSAGE")) + ;}; + +# =============================================================================================== +# Section 4: destination configuration +# Select the desired destination to use in the log path from the three choices below, +# based on the transfer method (UF/HEC/Kafka) desired. Then, using the rewrite rule below, +# provide the sourcetype and optional index (if different from the default) for thie data +# source. Then, uncomment the second rewrite declartion to add any additional indexed fields +# desired. +# =============================================================================================== + +# rewrite { r_set_splunk(vendor("paloalto"))}; #--UF-- +# destination(d_splunk_logfile); #--UF-- + rewrite { r_set_splunk(sourcetype("pan:log")) }; #--HEC-- +# rewrite { set("testvalue", value("fields.testfield")); }; #--HEC-- + destination(d_hec_everything); #--HEC-- +# rewrite { r_set_splunk(sourcetype("pan:log")) }; #--KAFKA-- +# destination(d_kafka_with_metadata); #--KAFKA-- + +# =============================================================================================== +# Final flag set here; further processing stops on filter match +# =============================================================================================== + +flags(final); +}; diff --git a/package/etc/conf.d/filters/f_websense.conf b/package/etc/conf.d/filters/f_websense.conf new file mode 100644 index 0000000..855410a --- /dev/null +++ b/package/etc/conf.d/filters/f_websense.conf @@ -0,0 +1,63 @@ +# =============================================================================================== +# Websense +# =============================================================================================== + +log { +# =============================================================================================== +# Section 1: source configuration +# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. +# +# IMPORTANT: In order to ensure proper startup, select only one source in this section. +# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed +# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those +# in source declarations from other filter files, an "Address already in use" error will occur +# at startup. +# =============================================================================================== + +source (s_default-ports); # The default port is common in many installations + +# The source block (function) call below can be used to specify unique listening ports for this +# device type + +# source { s_unique-ports(udp_port(714) tcp_port(714)); }; + +# =============================================================================================== +# Section 2: parser configuration +# Default syslog-parser() should work for most messages +# =============================================================================================== + +# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. +# parser { syslog-parser(); }; + +# For RFC5424-formatted messages, use this instead +# parser { syslog-parser(flags(syslog-protocol)); }; + +# For Websense +parser { p_websense-parser(); }; + +# =============================================================================================== +# Section 3: filter configuration +# Filter below may be omitted in the log path if a dedicated source is configured above +# =============================================================================================== + +filter { program(Websense); }; + +# =============================================================================================== +# Section 4: destination configuration +# Select the desired destination to use in the log path from the three choices below, +# based on the transfer method (UF/HEC/Kafka) desired. Then, using the rewrite rule below, +# provide the sourcetype and optional index (if different from the default) for thie data +# source. Then, uncomment the second rewrite declartion to add any additional indexed fields +# desired. +# =============================================================================================== + +# rewrite { r_set_splunk(vendor("websense"))}; #--UF-- +# destination(d_splunk_logfile); #--UF-- + rewrite { r_set_splunk(sourcetype("websense:cg:kv")) }; #--HEC-- +# rewrite { set("testvalue", value("fields.testfield")); }; #--HEC-- + destination(d_hec_everything); #--HEC-- +# rewrite { r_set_splunk(sourcetype("websense:cg:kv")) }; #--KAFKA-- +# destination(d_kafka_with_metadata); #--KAFKA-- + +flags(final); +}; diff --git a/package/etc/conf.d/sources/s_defaults.conf b/package/etc/conf.d/sources/s_defaults.conf new file mode 100644 index 0000000..cd7186f --- /dev/null +++ b/package/etc/conf.d/sources/s_defaults.conf @@ -0,0 +1,49 @@ +# =============================================================================================== +# source definition for remote devices +# =============================================================================================== + +# =============================================================================================== +# Defaults for the default-network-drivers() source: +# 514, both TCP and UDP, for RFC3164 (BSD-syslog) formatted traffic +# 601 TCP, for RFC5424 (IETF-syslog) formatted traffic +# 6514 TCP, for TLS-encrypted traffic +# =============================================================================================== + +# source s_network-defaults { default-network-drivers() }; +source s_default-ports { + `splunk-udp-driver` ( + transport("udp") + port(`udp-listening-port`) + ip-protocol(`ip-version`) +# hook-commands( +# setup("firewall-cmd --permanent --add-port=`udp-listening-port`/udp") +# teardown("firewall-cmd --permanent --remove-port=`udp-listening-port`/udp") +# ) + so-rcvbuf(`splunk-rcvbuf`) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); + + `splunk-tcp-driver` ( + transport("tcp") + port(`tcp-listening-port`) + ip-protocol(`ip-version`) +# hook-commands( +# setup("firewall-cmd --permanent --add-port=`tcp-listening-port`/tcp") +# teardown("firewall-cmd --permanent --remove-port=`tcp-listening-port`/tcp") +# ) + max-connections(`splunk-max-connections`) + log-iw-size(`splunk-window-size`) + log-fetch-limit(`splunk-fetch-limit`) + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(no-parse) + ); + }; diff --git a/package/etc/conf.d/splunk.conf b/package/etc/conf.d/splunk.conf new file mode 100644 index 0000000..eb337ec --- /dev/null +++ b/package/etc/conf.d/splunk.conf @@ -0,0 +1,70 @@ +# =============================================================================================== +# Defaults for remote data collection and transport to Splunk +# Kafka brokers/topics, HEC URL/token, local log directory for UF, sourcetype, and index +# These can be overridden in individual filter (f_*.conf) files +# =============================================================================================== + +# =============================================================================================== +# Source driver defaults (which govern the parser used) +# =============================================================================================== +@define splunk-udp-driver "syslog" +@define splunk-tcp-driver "network" + +# =============================================================================================== +# Listening ports +# =============================================================================================== +@define udp-listening-port "514" +@define tcp-listening-port "514" + +# =============================================================================================== +# UF transport +# =============================================================================================== +@define splunk-log-root "/var/log/syslog-ng/data" +@define splunk-user "splunk" # owner of the local disk files +@define splunk-group "splunk" # group of the local disk files + +# =============================================================================================== +# Kafka transport +# =============================================================================================== +@define kafka-implementation "kafka-c" +@define splunk-kafka-brokers "kafka-1:19092,kafka-2:29092,kafka-3:39092" +@define splunk-kafka-topic "syslog" + +# =============================================================================================== +# HEC transport +# =============================================================================================== +@define splunk-hec-url '__SPLUNK_HEC_URL__' +@define splunk-hec-token "__SPLUNK_HEC_TOKEN__" +@define worker-threads 20 +# =============================================================================================== +# Splunk metadata (HEC/Kafka transport only) +# =============================================================================================== +@define splunk-sourcetype "syslog:unassigned" +@define splunk-index "main" + +# =============================================================================================== +# Default message template +# =============================================================================================== +@define splunk-default-template "t_standard" + +# =============================================================================================== +# Data collection parameters, buffers, and Timezone +# =============================================================================================== +@define ip-version 4 +@define splunk-max-connections 1000 +@define splunk-log-fifo-size 100000 +@define splunk-fetch-limit 100 +# make sure splunk-window-size >= splunk-max-connections * splunk-fetch-limit +@define splunk-window-size 100000 +@define splunk-rcvbuf 425984 +@define default-timezone "GMT" + +# =============================================================================================== +# Global modules and includes. All device-specific filters and destinations exist in conf.d +# Order is important; templates must be loaded first +# =============================================================================================== +@include "conf.d/templates/*.conf" +@include "conf.d/sources/*.conf" +@include "conf.d/blocks/*.conf" +@include "conf.d/destinations/*.conf" +@include "conf.d/filters/*.conf" diff --git a/package/etc/conf.d/templates/t_templates.conf b/package/etc/conf.d/templates/t_templates.conf new file mode 100644 index 0000000..e2a595a --- /dev/null +++ b/package/etc/conf.d/templates/t_templates.conf @@ -0,0 +1,44 @@ +# =============================================================================================== +# Templates for message formatting. Overall JSON formatting for HEC/Kafka is handled in the +# destination blocks. +# =============================================================================================== + +# =============================================================================================== +# Standard format; works for nearly all messages +# =============================================================================================== + +template t_standard { + template("${ISODATE} ${HOST} ${MESSAGE}"); + }; + +# =============================================================================================== +# Message Only; useful for bluecoat proxy, palo alto, and others +# =============================================================================================== + +template t_msg_only { + template("${MESSAGE}"); + }; + +# =============================================================================================== +# Everything; useful for bluecoat proxy and others +# =============================================================================================== + +template t_everything { + template("${ISODATE} ${HOST} ${MSGHDR}${MESSAGE}"); + }; + +# =============================================================================================== +# Message Header with Message; for Palo Alto +# =============================================================================================== + +template t_hdr_msg { + template("${MSGHDR}${MESSAGE}"); + }; + +# =============================================================================================== +# JSON; for JSON pretty-printing +# =============================================================================================== + +template t_JSON { + template("$(format-json --scope all-nv-pairs)"); + }; diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf new file mode 100755 index 0000000..2389d62 --- /dev/null +++ b/package/etc/syslog-ng.conf @@ -0,0 +1,48 @@ +############################################################################# +# Default syslog-ng.conf file which collects all local logs into a +# single file called /var/log/messages tailored to container usage. +# +# The changes from the stock, default syslog-ng.conf file is that we've +# dropped the system() source that is not needed and that we enabled network +# connections using default-network-drivers(). Customize as needed and +# override using the -v option to docker, such as: +# +# docker run ... -v "$PWD/syslog-ng.conf":/etc/syslog-ng/syslog-ng.conf +# + +@version: 3.22 +@include "scl.conf" + +# =============================================================================================== +# Global Options +# =============================================================================================== + +options { + log_msg_size (65536); + flush_lines (0); + time_reopen (10); + log_fifo_size (1000); + chain_hostnames (off); + use_dns (no); + use_fqdn (no); + create_dirs (no); + keep-hostname (yes); + stats-level(3); + stats-max-dynamics(500); +}; + +source s_local { + internal(); +}; + +destination d_console { + pipe("/dev/stdout"); +}; + +@include "conf.d/*.conf" + +log { + source(s_local); + destination(d_console); + flags(final); +}; diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh new file mode 100755 index 0000000..d323b2e --- /dev/null +++ b/package/sbin/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/dumb-init /bin/sh + +#Run syslog +mkdir /opt/syslog-ng/var +rm /opt/syslog-ng/var/syslog-ng.ctl +/opt/syslog-ng/bin/switch_transport.sh hec +/opt/syslog-ng/sbin/syslog-ng --process-mode=background +/opt/syslog-ng/sbin/splunkmetrics.sh diff --git a/package/sbin/goss.yaml b/package/sbin/goss.yaml new file mode 100644 index 0000000..abd12fb --- /dev/null +++ b/package/sbin/goss.yaml @@ -0,0 +1,16 @@ +process: + syslog-ng: + running: true +port: + tcp:514: + listening: true + ip: + - 0.0.0.0 + udp:514: + listening: true + ip: + - 0.0.0.0 + tcp:6514: + listening: true + ip: + - 0.0.0.0