diff --git a/.gitignore b/.gitignore index e0adc68..d96805f 100644 --- a/.gitignore +++ b/.gitignore @@ -306,3 +306,6 @@ tags /test-results/ /.idea/workspace.xml /.idea/tasks.xml +!/package/scripts/switch_transport.sh +!/package/scripts/splunkmetrics.sh +!/package/scripts/ diff --git a/.idea/misc.xml b/.idea/misc.xml index 92749e9..a658ea5 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,8 +3,8 @@ - + - + \ No newline at end of file diff --git a/.idea/splunk-connect-for-syslog.iml b/.idea/splunk-connect-for-syslog.iml index 3ce5619..0ecbb61 100644 --- a/.idea/splunk-connect-for-syslog.iml +++ b/.idea/splunk-connect-for-syslog.iml @@ -2,7 +2,7 @@ - + diff --git a/docker-compose.yml b/docker-compose.yml index cf96c95..94a8320 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,6 +63,11 @@ services: - SPLUNK_HEC_TOKEN - SPLUNK_PASSWORD - SPLUNK_START_ARGS + - SPLUNK_APPS_URL + - SPLUNKBASE_USERNAME + - SPLUNKBASE_PASSWORD + volumes: + - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng # logging: # driver: splunk # options: diff --git a/package/Dockerfile b/package/Dockerfile index f3b9f96..5ecc921 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -76,19 +76,21 @@ RUN cd /tmp ;\ ENV DEBCONF_NONINTERACTIVE_SEEN=true ENV SPLUNK_CONNECT_METHOD=hec +ENV SYSLOGNG_HEC_WORKERS=3 RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh COPY goss.yaml /etc/goss.yaml COPY --from=0 /opt/syslog-ng /opt/syslog-ng COPY scripts/splunkmetrics.sh /opt/syslog-ng/sbin/splunkmetrics.sh -COPY scripts/switch_transport.sh /opt/syslog-ng/sbin/switch_transport.sh COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d +COPY etc/context /opt/syslog-ng/etc/context COPY sbin/entrypoint.sh /sbin/entrypoint.sh RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t +RUN mkdir -p /var/log/syslog-ng/data/disk-buffer EXPOSE 514 EXPOSE 601/tcp diff --git a/package/etc/conf.d/blocks/b_destinations.conf b/package/etc/conf.d/blocks/b_destinations.conf deleted file mode 100755 index 3e65461..0000000 --- a/package/etc/conf.d/blocks/b_destinations.conf +++ /dev/null @@ -1,251 +0,0 @@ -# =============================================================================================== -# Destination Blocks (functions) for Splunk destinations -# Block argument defaults are defined in main syslog-ng.conf file -# Additional indexed fields (based on syslog-ng macros) can be added to the "with_metadata" blocks -# for both HEC and Kafka. -# Format is "fields.=" -# =============================================================================================== - -# =============================================================================================== -# blocks for file destination (for use with the Splunk Universal Forwarder) -# =============================================================================================== - -# =============================================================================================== -# Logfile Local Defaults: -# vendor: unassigned -# message: suffix for log file names; "messages" -# =============================================================================================== - -block destination d_splunk_logfile(category("") - vendor("unassigned") - filename("messages") - use_template(`splunk-default-template`) - ) { - file("`splunk-log-root`/`category`/$(lowercase \"${LOGHOST}\")/`vendor`/$(lowercase \"${HOST}\")/${YEAR}.${MONTH}.${DAY}.${HOUR}-`filename`" - template(`use_template`) - flush-lines(0) - create-dirs(yes) - dir_owner(`splunk-user`) - dir_group(`splunk-group`) - dir_perm(0700) - owner(`splunk-user`) - group(`splunk-group`) - perm(0600) - ); - }; - -# =============================================================================================== -# blocks for direct connection to Splunk via Kafka and HEC -# =============================================================================================== -# =============================================================================================== -# Kafka with Metadata (Facility and Priority): -# =============================================================================================== - -block destination d_kafka_with_metadata (kafka-libs(`kafka-lib-dir`) - kafka-brokers(`splunk-kafka-brokers`) - sourcetype(`splunk-sourcetype`) - topic(`splunk-topic`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - kafka(client-lib-dir("`kafka-libs`") - kafka-bootstrap-servers("`kafka-brokers`") - persist-name("`kafka_brokers`_`topic`_`sourcetype`_`index`") - template('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`) - fields.facility=$FACILITY - fields.severity=$LEVEL)') - topic("`topic`") - ); - }; - -# =============================================================================================== -# Kafka with Full Metadata; additional fields can be added -# =============================================================================================== - -block destination d_kafka_with_full_metadata (kafka-libs(`kafka-lib-dir`) - kafka-brokers(`splunk-kafka-brokers`) - sourcetype(`splunk-sourcetype`) - topic(`splunk-topic`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - kafka(client-lib-dir("`kafka-libs`") - kafka-bootstrap-servers("`kafka-brokers`") - persist-name("`kafka_brokers`_`topic`_`sourcetype`_`index`") - template('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`) - fields.facility=$FACILITY - fields.severity=$LEVEL - fields.message_header=$MSGHDR - fields.program_name=$PROGRAM - fields.pid=$PID - fields.full_message=$(template t_everything) - )') - topic("`topic`") - ); - }; - -# =============================================================================================== -# Basic Kafka (no indexed fields) -# =============================================================================================== - -block destination d_kafka_basic (kafka-libs(`kafka-lib-dir`) - kafka-brokers(`splunk-kafka-brokers`) - sourcetype(`splunk-sourcetype`) - topic(`splunk-topic`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - kafka(client-lib-dir("`kafka-libs`") - kafka-bootstrap-servers("`kafka-brokers`") - persist-name("`kafka_brokers`_`topic`_`sourcetype`_`index`") - template('$(format-json time=$S_UNIXTIME - host=$HOST - source=$HOST_FROM - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`))') - topic("`topic`") - ); - }; - -# =============================================================================================== -# HEC with Metadata (Facility and Priority): -# Be sure to adjust batch paramaters below to suit scale/environment -# If validated certs are used, uncomment relevant lines in the tls() block below -# and change peer-verify() to "yes" -# =============================================================================================== - -block destination d_hec_with_metadata (hec-url(`splunk-hec-url`) - hec-token(`splunk-hec-token`) - sourcetype(`splunk-sourcetype`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - http(url("`hec-url`") - method("POST") - batch-lines(10) - batch-bytes(512Kb) - batch-timeout(5000) - user_agent("syslog-ng User Agent") - user("syslog-ng") - password("`hec-token`") - persist-name("`hec-url`_`sourcetype`_`hec-token`_`index`") - tls(peer-verify(no) -# ca-dir("dir") -# ca-file("ca") -# cert-file("cert") -# cipher-suite("cipher") -# key-file("key") -# peer-verify(yes|no) -# ssl-version() - ) - body('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`) - fields.facility=$FACILITY - fields.severity=$LEVEL)') - ); - }; - -# =============================================================================================== -# HEC with Full Metadata; additional fields can be added -# Be sure to adjust batch paramaters below to suit scale/environment -# If validated certs are used, uncomment relevant lines in the tls() block below -# and change peer-verify() to "yes" -# =============================================================================================== - -block destination d_hec_with_full_metadata(hec-url(`splunk-hec-url`) - hec-token(`splunk-hec-token`) - sourcetype(`splunk-sourcetype`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - http(url("`hec-url`") - method("POST") - batch-lines(10) - batch-bytes(512Kb) - batch-timeout(5000) - user_agent("syslog-ng User Agent") - user("syslog-ng") - password("`hec-token`") - persist-name("`hec-url`_`sourcetype`_`hec-token`_`index`") - tls(peer-verify(no) -# ca-dir("dir") -# ca-file("ca") -# cert-file("cert") -# cipher-suite("cipher") -# key-file("key") -# peer-verify(yes|no) -# ssl-version() - ) - body('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`) - fields.fullhost_from=${FULLHOST_FROM} - fields.facility=$FACILITY - fields.severity=$LEVEL - fields.message_header=$MSGHDR - fields.program_name=$PROGRAM - fields.pid=$PID - fields.full_message=$(template t_everything) - )') - ); - }; - -# =============================================================================================== -# Basic HEC (no indexed fields) -# =============================================================================================== - -block destination d_hec_basic (hec-url(`splunk-hec-url`) - hec-token(`splunk-hec-token`) - sourcetype(`splunk-sourcetype`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - http(url("`hec-url`") - method("POST") - batch-lines(10) - batch-bytes(512Kb) - batch-timeout(5000) - user_agent("syslog-ng User Agent") - user("syslog-ng") - password("`hec-token`") - persist-name("`hec-url`_`sourcetype`_`hec-token`_`index`") - tls(peer-verify(no) -# ca-dir("dir") -# ca-file("ca") -# cert-file("cert") -# cipher-suite("cipher") -# key-file("key") -# peer-verify(yes|no) -# ssl-version() - ) - body('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`))') - ); - }; diff --git a/package/etc/conf.d/blocks/b_parsers.conf b/package/etc/conf.d/blocks/b_parsers.conf old mode 100755 new mode 100644 index b68a4e4..407ea8a --- a/package/etc/conf.d/blocks/b_parsers.conf +++ b/package/etc/conf.d/blocks/b_parsers.conf @@ -1,33 +1,69 @@ -# =============================================================================================== -# ForcePoint (Websense) -# =============================================================================================== - -# =============================================================================================== -# Log Format: -# <159>Dec 19 10:48:57 EST 10.203.28.21 vendor=Websense product=Security product_version=7.7.0 action=permitted severity=1 category=153 user=- src_host=10.64.134.74 src_port=62189 dst_host=mail.google.com dst_ip=74.125.224.53 dst_port=443 bytes_out=197 bytes_in=76 http_response=200 http_method=CONNECT http_content_type=- http_user_agent=Mozilla/5.0_(Windows;_U;_Windows_NT_6.1;_enUS;_rv:1.9.2.23)_Gecko/20110920_Firefox/3.6.23 http_proxy_status_code=200 reason=- disposition=1034 policy=- role=8 duration=0 url=https://mail.google.com -# -# Primary reason for non-compliance: %Z (timesone mnemonic) appended after date/time -# =============================================================================================== - -block parser p_websense-parser(prefix(".websense.")) { +#this parser can be used for "almost compliant" 5424 messages +block parser p_RFC5424-lite(prefix(".rfc5424-lite.")) { channel { rewrite { - # normal BSD timestamp, plus a timezone code. Remove the - # timezone information for now. - subst('([A-Za-z]{3} [0-9 ]\d \d{2}:\d{2}:\d{2}) [A-Z]{3,4}' "$1 " flags(global) value("MSG")); + # Remove Version Identifier + subst('(<\d+>)\d+' "$1" value("MSG")); - # add a $PROGRAM field, so that syslog-parser() would extract - # that properly - subst('(vendor=Websense)' "Websense: $1" flags(global) value("MSG")); }; - parser { - # by this time this message is a properly formatted syslog - # message + parser { + # by this time this message will parse with the BSD 3164 parser syslog-parser(time-zone(`default-timezone`)); # syslog-parser(); # extract name-value pairs. # kv-parser(prefix("`prefix`")); + }; }; +}; + +block parser p_test-parser(prefix(".testparse.")) { + channel { + rewrite { + # Put your rewrite rule here;use "MSG" as no-parse() puts entire payload there + # subst('(>.{12})\s' "${1}T" value("MSG")); + }; + parser { + # Don't initially parse so full output can be seen + # syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); + # syslog-parser(); + # syslog-parser(time-zone(`default-timezone`)); + # date-parser(format("%Y-%m-%d %H:%M:%S") template("$(substr ${MESSAGE} 0 19)") time-zone(`default-timezone`)); + }; }; }; + +#Used by fallback to log everything we can +block rewrite r_set_splunk(sourcetype(`splunk-sourcetype`) + index(`splunk-index`) + template(`splunk-default-template`) + vendor("unassigned") + category("") + filename("messages") + ) { + set("`index`", value(".splunk.index")); + set("`sourcetype`", value(".splunk.sourcetype")); + set("`template`", value(".splunk.template")); + set("`vendor`", value(".splunk.vendor")); + set("`category`", value(".splunk.category")); + set("`filename`", value(".splunk.filename")); + set($FACILITY, value("fields.facility")); + set($LEVEL, value("fields.severity")); + set($LOGHOST, value("fields.syslog-server")); + }; + +#Used when for normal filters index and sourcetype must be set in a prior re-write or context +block rewrite r_set_splunk_basic( + template(`splunk-default-template`) + vendor("unassigned") + category("") + filename("messages") + ) { + set("`template`", value(".splunk.template")); + set("`vendor`", value(".splunk.vendor")); + set("`category`", value(".splunk.category")); + set("`filename`", value(".splunk.filename")); + set($FACILITY, value("fields.facility")); + set($LEVEL, value("fields.severity")); + set($LOGHOST, value("fields.syslog-server")); + }; diff --git a/package/etc/conf.d/blocks/b_sources.conf b/package/etc/conf.d/blocks/b_sources.conf deleted file mode 100755 index 5e53849..0000000 --- a/package/etc/conf.d/blocks/b_sources.conf +++ /dev/null @@ -1,85 +0,0 @@ -# =============================================================================================== -# Source block for data arriving on unique ports; parsing defaults to "no-parse" -# =============================================================================================== - -block source s_unique-ports(udp_port() tcp_port() flags("no-parse")) { - `splunk-udp-driver` ( - transport("udp") - port(`udp_port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=514/udp") -# teardown("firewall-cmd --permanent --remove-port=514/udp") -# ) - so-rcvbuf(`splunk-rcvbuf`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - time-zone(`default-timezone`) - flags(`flags`) - ); - - `splunk-tcp-driver` ( - transport("tcp") - port(`tcp_port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=514/tcp") -# teardown("firewall-cmd --permanent --remove-port=514/tcp") -# ) - max-connections(`splunk-max-connections`) - log-iw-size(`splunk-window-size`) - log-fetch-limit(`splunk-fetch-limit`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - time-zone(`default-timezone`) - flags(`flags`) - ); -}; - -block source s_unique-udp-port(udp_port() flags("no-parse")) { - `splunk-udp-driver` ( - transport("udp") - port(`udp_port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=514/udp") -# teardown("firewall-cmd --permanent --remove-port=514/udp") -# ) - so-rcvbuf(`splunk-rcvbuf`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - time-zone(`default-timezone`) - flags(`flags`) - ); -}; - -block source s_unique-tcp-port(tcp_port() flags("no-parse")) { - `splunk-tcp-driver` ( - transport("tcp") - port(`tcp_port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=514/tcp") -# teardown("firewall-cmd --permanent --remove-port=514/tcp") -# ) - max-connections(`splunk-max-connections`) - log-iw-size(`splunk-window-size`) - log-fetch-limit(`splunk-fetch-limit`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - time-zone(`default-timezone`) - flags(`flags`) - ); -}; diff --git a/package/etc/conf.d/blocks/splunk_context.conf b/package/etc/conf.d/blocks/splunk_context.conf new file mode 100644 index 0000000..e2e0af9 --- /dev/null +++ b/package/etc/conf.d/blocks/splunk_context.conf @@ -0,0 +1,7 @@ +block parser p_add_context_splunk(key("syslogng-fallback")) { + add-contextual-data( + selector("`key`"), + database("context/splunk_index.csv"), + prefix(".splunk.") + ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/d_destinations.conf b/package/etc/conf.d/destinations/d_destinations.conf new file mode 100644 index 0000000..1d373d4 --- /dev/null +++ b/package/etc/conf.d/destinations/d_destinations.conf @@ -0,0 +1,50 @@ + + +# =============================================================================================== +# Direct connection to Splunk via HEC +# =============================================================================================== + +# =============================================================================================== +# Be sure to adjust batch paramaters below to suit scale/environment +# Set workers to the number of indexers or HWF HEC endpoints +# If validated certs are used, uncomment relevant lines in the tls() block below +# and change peer-verify() to "yes" +# =============================================================================================== + +destination d_hec { + http( + url("`SPLUNK_HEC_URL`") + method("POST") + log-fifo-size(`splunk-log-fifo-size`) + workers(`SYSLOGNG_HEC_WORKERS`) + batch-lines(1000) + batch-bytes(512Kb) + batch-timeout(1000) + timeout(5) + user_agent("syslog-ng User Agent") + user("syslog-ng") + password("`SPLUNK_HEC_TOKEN`") +# persist-name("t_standard") +# disk-buffer(mem-buf-length(5000) +# disk-buf-size(107374182400) +# reliable(no) +# dir("/var/log/syslog-ng/data/disk-buffer")) + tls(peer-verify(no) +# ca-dir("dir") +# ca-file("ca") +# cert-file("cert") +# cipher-suite("cipher") +# key-file("key") +# peer-verify(yes|no) +# ssl-version() + ) + body('$(format-json + time=$S_UNIXTIME.$S_MSEC + host=$HOST + source=${HOST_FROM} + sourcetype=${.splunk.sourcetype} + index=${.splunk.index} + event=$(template ${.splunk.template}) + fields.*)') + ); + }; diff --git a/package/etc/conf.d/filters/f_catchall.conf b/package/etc/conf.d/filters/f_catchall.conf deleted file mode 100755 index aa9abbf..0000000 --- a/package/etc/conf.d/filters/f_catchall.conf +++ /dev/null @@ -1,80 +0,0 @@ -# =============================================================================================== -# "Catch-all" destination; all unfiltered messages end up here -# =============================================================================================== - -log { -# =============================================================================================== -# Section 1: source configuration -# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. -# -# IMPORTANT: In order to ensure proper startup, select only one source in this section. -# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed -# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those -# in source declarations from other filter files, an "Address already in use" error will occur -# at startup. -# -# NOTE: The "Catchall" should not use any source other than the default. Be extremely careful -# when defining an alternate source here. Default ports can be changed in the global declarations -# in the main syslog-ng.conf file. -# =============================================================================================== - -source(s_default-ports); - -# =============================================================================================== -# Section 2: parser configuration -# Default syslog-parser() should work for most messages -# -# If a new data source is not properly parsing using the default, you may wish to comment out -# all parsers to see the raw message as it appeared "on the wire". This can be useful for -# creating custom parsers for non-syslog-compliant meessages. -# =============================================================================================== - -# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. -parser { syslog-parser(time-zone(`default-timezone`)); }; - -# For RFC5424-formatted messages, use this instead -# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); }; - -# =============================================================================================== -# Section 3: filter configuration -# Filter below may be omitted in the log path if a dedicated source is configured above -# Filter typically not used for a fallback destination -# =============================================================================================== - -# filter { }; - -# =============================================================================================== -# Section 4: destination configuration -# Select the desired destination to use in the log path from the three choices below, -# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string -# for UF transport, and a unique sourcetype argument and optional index, topic, and message -# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the -# note below. -# -# IMPORTANT: If modifications are made in this section, ensure that two or more distinct -# destinations do *not* reference the same destination block call with an idential set of -# arguments (including no arguments, and arguments passed via destinations declared in other -# conf.d include files), or you will see this at startup: "Error checking the uniqueness of -# the persist names, please override it with persist-name option. Shutting down." -# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new -# or changed destination declaration. -# =============================================================================================== - -# =============================================================================================== -# This catchall file should be the *only* one where the destinations use defaults for all arguments! -# Full-metadata destinations used here; check indexed event data in Splunk for full message. -# This can be helpful to determine the compliance of message to standard syslog formats, and as -# an aid for custom parser development if needed. -# =============================================================================================== - -destination { -# d_splunk_logfile(); #--UF-- - d_hec_with_full_metadata(); #--HEC-- -# d_kafka_with_full_metadata(); #--KAFKA-- - }; - -# =============================================================================================== -# Fallback flag set here; will catch all unfiltered traffic -# =============================================================================================== -flags(fallback); -}; diff --git a/package/etc/conf.d/filters/f_cisco-ios.conf b/package/etc/conf.d/filters/f_cisco-ios.conf deleted file mode 100755 index c224a7a..0000000 --- a/package/etc/conf.d/filters/f_cisco-ios.conf +++ /dev/null @@ -1,64 +0,0 @@ -# =============================================================================================== -# Cisco IOS (Route/Switch) -# =============================================================================================== - -log { -# =============================================================================================== -# Section 1: source configuration -# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. -# -# IMPORTANT: In order to ensure proper startup, select only one source in this section. -# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed -# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those -# in source declarations from other filter files, an "Address already in use" error will occur -# at startup. -# =============================================================================================== - -source(s_default-ports); - -# source { s_unique-ports(udp_port(814) tcp_port(814)); }; - -# =============================================================================================== -# Section 2: parser configuration -# Default syslog-parser() should work for most messages -# =============================================================================================== - -# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. -parser { syslog-parser(time-zone(`default-timezone`)); }; - -# For RFC5424-formatted messages, use this instead -# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); }; - -# =============================================================================================== -# Section 3: filter configuration -# Filter below may be omitted in the log path if a dedicated source is configured above -# =============================================================================================== - -filter { match("[%|#][A-Z0-9_]+-[A-Z012_]*-*[0-7]-[A-Z0-9_]+:" value("MESSAGE")) - ;}; - - -# =============================================================================================== -# Section 4: destination configuration -# Select the desired destination to use in the log path from the three choices below, -# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string -# for UF transport, and a unique sourcetype argument and optional index, topic, and message -# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the -# note below. -# -# IMPORTANT: If modifications are made in this section, ensure that two or more distinct -# destinations do *not* reference the same destination block call with an idential set of -# arguments (including no arguments, and arguments passed via destinations declared in other -# conf.d include files), or you will see this at startup: "Error checking the uniqueness of -# the persist names, please override it with persist-name option. Shutting down." -# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new -# or changed destination declaration. -# =============================================================================================== - -destination { -# d_splunk_logfile(vendor("cisco-ios")); #--UF-- - d_hec_with_metadata(sourcetype("cisco:ios")); #--HEC-- -# d_kafka_with_metadata(sourcetype("cisco:ios")); #--KAFKA-- - }; -flags(final); -}; diff --git a/package/etc/conf.d/filters/f_palo_alto.conf b/package/etc/conf.d/filters/f_palo_alto.conf deleted file mode 100755 index 868b94c..0000000 --- a/package/etc/conf.d/filters/f_palo_alto.conf +++ /dev/null @@ -1,67 +0,0 @@ -# =============================================================================================== -# Palo Alto Networks Firewalls -# =============================================================================================== - -log { -# =============================================================================================== -# Section 1: source configuration -# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. -# -# IMPORTANT: In order to ensure proper startup, select only one source in this section. -# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed -# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those -# in source declarations from other filter files, an "Address already in use" error will occur -# at startup. -# =============================================================================================== - -source (s_default-ports); - -# source { s_unique-ports(udp_port(714) tcp_port(714)); }; - -# =============================================================================================== -# Section 2: parser configuration -# Default syslog-parser() should work for most messages -# =============================================================================================== - -# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. -parser { syslog-parser(time-zone(`default-timezone`)); }; - -# For RFC5424-formatted messages, use this instead -# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); }; - -# =============================================================================================== -# Section 3: filter configuration -# Filter below may be omitted in the log path if a dedicated source is configured above -# =============================================================================================== - -filter { match(",THREAT," value("MESSAGE")) - or match(",TRAFFIC," value("MESSAGE")) - or match(",SYSTEM," value("MESSAGE")) - or match(",CONFIG," value("MESSAGE")) - or match(",HIPWATCH," value("MESSAGE")) - ;}; - -# =============================================================================================== -# Section 4: destination configuration -# Select the desired destination to use in the log path from the three choices below, -# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string -# for UF transport, and a unique sourcetype argument and optional index, topic, and message -# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the -# note below. -# -# IMPORTANT: If modifications are made in this section, ensure that two or more distinct -# destinations do *not* reference the same destination block call with an idential set of -# arguments (including no arguments, and arguments passed via destinations declared in other -# conf.d include files), or you will see this at startup: "Error checking the uniqueness of -# the persist names, please override it with persist-name option. Shutting down." -# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new -# or changed destination declaration. -# =============================================================================================== - -destination { -# d_splunk_logfile(vendor("paloalto")); #--UF-- - d_hec_with_metadata(sourcetype("pan:log") use_template("t_everything")); #--HEC-- -# d_kafka_with_metadata(sourcetype("pan:log") use_template("t_everything")); #--KAFKA-- - }; -flags(final); -}; diff --git a/package/etc/conf.d/filters/f_websense.conf b/package/etc/conf.d/filters/f_websense.conf deleted file mode 100755 index 16fb2af..0000000 --- a/package/etc/conf.d/filters/f_websense.conf +++ /dev/null @@ -1,68 +0,0 @@ -# =============================================================================================== -# Websense -# =============================================================================================== - -log { -# =============================================================================================== -# Section 1: source configuration -# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. -# -# IMPORTANT: In order to ensure proper startup, select only one source in this section. -# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed -# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those -# in source declarations from other filter files, an "Address already in use" error will occur -# at startup. -# =============================================================================================== - -source (s_default-ports); # The default port is common in many installations - -# The source block (function) call below can be used to specify unique listening ports for this -# device type - -# source { s_unique-ports(udp_port(714) tcp_port(714)); }; - -# =============================================================================================== -# Section 2: parser configuration -# Default syslog-parser() should work for most messages -# =============================================================================================== - -# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. -# parser { syslog-parser(); }; - -# For RFC5424-formatted messages, use this instead -# parser { syslog-parser(flags(syslog-protocol)); }; - -# For Websense -parser { p_websense-parser(); }; - -# =============================================================================================== -# Section 3: filter configuration -# Filter below may be omitted in the log path if a dedicated source is configured above -# =============================================================================================== - -filter { program(Websense); }; - -# =============================================================================================== -# Section 4: destination configuration -# Select the desired destination to use in the log path from the three choices below, -# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string -# for UF transport, and a unique sourcetype argument and optional index, topic, and message -# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the -# note below. -# -# IMPORTANT: If modifications are made in this section, ensure that two or more distinct -# destinations do *not* reference the same destination block call with an idential set of -# arguments (including no arguments, and arguments passed via destinations declared in other -# conf.d include files), or you will see this at startup: "Error checking the uniqueness of -# the persist names, please override it with persist-name option. Shutting down." -# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new -# or changed destination declaration. -# =============================================================================================== - -destination { -# d_splunk_logfile(vendor("websense")); #--UF-- - d_hec_with_metadata(sourcetype("websense:cg:kv")); #--HEC-- -# d_kafka_with_metadata(sourcetype("websense:cg:kv")); #--KAFKA-- - }; -flags(final); -}; diff --git a/package/etc/conf.d/filters/paloalto_networks.conf b/package/etc/conf.d/filters/paloalto_networks.conf new file mode 100644 index 0000000..1fbba7b --- /dev/null +++ b/package/etc/conf.d/filters/paloalto_networks.conf @@ -0,0 +1,61 @@ + + +log { + source(s_default-ports); + + filter {tags("pan:log") + or message(',\d+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPWATCH|CORRELATION|USERID),') + ;}; + + # the palo message does not include a program value in the header unfortunatly + # the use of colon in the message tricks the RFC3164 parser we will re-write the message + # so the parser will not incorrectly break it apart. + # while we are at it we will save the mesage type into the program field so parser can find it + rewrite { + subst('^([^ ]+ \w+ \d{2} \d{2}:\d{2}:\d{2} [^ ]+)( \d+,[^,]+,[^,]+,)([^,]+)(.*)', '$1 $3:$2$3$4', value("MESSAGE") ); + }; + parser { + #basic parsing + syslog-parser(time-zone(`default-timezone`)); + #we need to actual even time from the field GeneratedTime use csv parser to get it out + csv-parser( + columns('FUTURE_USE', 'ReceiveTime', 'SerialNumber', 'Type', 'Subtype', 'FUTURE_USE2', 'GeneratedTime') + prefix(".pan.") + delimiters(',') + ); + #2012/04/10 04:39:55 + #parse the date + date-parser( + format("%Y/%m/%d %H:%M:%S") + template("${.pan.GeneratedTime}") + time-zone("Universal") + ); + }; + + #set the source type based on program field and lookup index from the splunk context csv + if (program('THREAT')) { + parser {p_add_context_splunk(key("pan:threat")); }; + } elif (program('TRAFFIC')) { + parser {p_add_context_splunk(key("pan:traffic")); }; + } elif (program('SYSTEM')) { + parser {p_add_context_splunk(key("pan:system")); }; + } elif (program('CONFIG')) { + parser {p_add_context_splunk(key("pan:config")); }; + } elif (program('HIPWATCH')) { + parser {p_add_context_splunk(key("pan:hipwatch")); }; + } elif (program('CORRELATION')) { + parser {p_add_context_splunk(key("pan:correlation")); }; + } elif (program('USERID')) { + parser {p_add_context_splunk(key("pan:userid")); }; + } else { + parser {p_add_context_splunk(key("pan:log")); }; + }; + + #rewrite the final message for splunk json + #sourcetype and index are set in the filter defaults won't be used + rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- + + destination(d_hec); #--HEC-- + + #flags(final); +}; diff --git a/package/etc/conf.d/sources/internal.conf b/package/etc/conf.d/sources/internal.conf new file mode 100644 index 0000000..5ba19a8 --- /dev/null +++ b/package/etc/conf.d/sources/internal.conf @@ -0,0 +1,7 @@ +# =============================================================================================== +# Internal sources +# =============================================================================================== + +source s_internal { + internal(); +}; diff --git a/package/etc/conf.d/sources/s_defaults.conf b/package/etc/conf.d/sources/network.conf old mode 100755 new mode 100644 similarity index 77% rename from package/etc/conf.d/sources/s_defaults.conf rename to package/etc/conf.d/sources/network.conf index 40425dd..e3ed6a9 --- a/package/etc/conf.d/sources/s_defaults.conf +++ b/package/etc/conf.d/sources/network.conf @@ -25,7 +25,6 @@ source s_default-ports { use-dns(no) use-fqdn(no) chain-hostnames(off) - time-zone(`default-timezone`) flags(no-parse) ); @@ -45,7 +44,18 @@ source s_default-ports { use-dns(no) use-fqdn(no) chain-hostnames(off) - time-zone(`default-timezone`) flags(no-parse) ); }; + +source s_all-ports { + `splunk-udp-driver` ( transport("udp") port(514) flags(no-parse) persist-name("example-persist-name1")); + `splunk-udp-driver` ( transport("udp") port(919) flags(no-parse)); + `splunk-udp-driver` ( transport("udp") port(920) flags(no-parse)); + `splunk-udp-driver` ( transport("udp") port(921) flags(no-parse)); + + `splunk-tcp-driver` ( transport("tcp") port(919) flags(no-parse)); + `splunk-tcp-driver` ( transport("tcp") port(920) flags(no-parse)); + `splunk-tcp-driver` ( transport("tcp") port(921) flags(no-parse)); + }; + diff --git a/package/etc/conf.d/splunk.conf b/package/etc/conf.d/splunk.conf deleted file mode 100755 index e8e5306..0000000 --- a/package/etc/conf.d/splunk.conf +++ /dev/null @@ -1,69 +0,0 @@ -# =============================================================================================== -# Defaults for remote data collection and transport to Splunk -# Kafka brokers/topics, HEC URL/token, local log directory for UF, sourcetype, and index -# These can be overridden in individual filter (f_*.conf) files -# =============================================================================================== - -# =============================================================================================== -# Source driver defaults (which govern the parser used) -# =============================================================================================== -@define splunk-udp-driver "syslog" -@define splunk-tcp-driver "network" - -# =============================================================================================== -# Listening ports -# =============================================================================================== -@define udp-listening-port "514" -@define tcp-listening-port "514" - -# =============================================================================================== -# UF transport -# =============================================================================================== -@define splunk-log-root "/var/log/syslog-ng/data" -@define splunk-user "root" # owner of the local disk files -@define splunk-group "root" # group of the local disk files - -# =============================================================================================== -# Kafka transport -# =============================================================================================== -@define kafka-lib-dir "/opt/kafka_2.11-0.11.0.0/libs" -@define splunk-kafka-brokers "127.0.0.1:9092" -@define splunk-topic "syslog" - -# =============================================================================================== -# HEC transport -# =============================================================================================== -@define splunk-hec-url __SPLUNK_HEC_URL__ -@define splunk-hec-token "__SPLUNK_HEC_TOKEN__" - -# =============================================================================================== -# Splunk metadata (HEC/Kafka transport only) -# =============================================================================================== -@define splunk-sourcetype "syslog:unassigned" -@define splunk-index "main" - -# =============================================================================================== -# Default message template -# =============================================================================================== -@define splunk-default-template "t_standard" - -# =============================================================================================== -# Data collection parameters, buffers, and Timezone -# =============================================================================================== -@define ip-version 4 -@define splunk-max-connections 1000 -@define splunk-fetch-limit 100 -# make sure splunk-window-size >= splunk-max-connections * splunk-fetch-limit -@define splunk-window-size 100000 -@define splunk-rcvbuf 425984 -@define default-timezone "GMT" - -# =============================================================================================== -# Global modules and includes. All device-specific filters and destinations exist in conf.d -# Order is important; templates must be loaded first -# =============================================================================================== -@include "scl.conf" -@include "conf.d/templates/*.conf" -@include "conf.d/sources/*.conf" -@include "conf.d/blocks/*.conf" -@include "conf.d/filters/*.conf" diff --git a/package/etc/conf.d/templates/t_templates.conf b/package/etc/conf.d/templates/t_templates.conf old mode 100755 new mode 100644 index b75dbbf..825a705 --- a/package/etc/conf.d/templates/t_templates.conf +++ b/package/etc/conf.d/templates/t_templates.conf @@ -8,7 +8,7 @@ # =============================================================================================== template t_standard { - template("${ISODATE} ${HOST} ${MESSAGE}\n"); + template("${ISODATE} ${HOST} ${MESSAGE}"); }; # =============================================================================================== @@ -16,13 +16,29 @@ template t_standard { # =============================================================================================== template t_msg_only { - template("${MESSAGE}\n"); + template("${MSGONLY}"); }; # =============================================================================================== -# Everything; useful for bluecoat proxy, palo alto, and others +# Everything; useful for bluecoat proxy and others # =============================================================================================== template t_everything { - template("${ISODATE} ${HOST} ${MSGHDR}${MESSAGE}\n"); + template("${ISODATE} ${HOST} ${MSGHDR}${MESSAGE}"); }; + +# =============================================================================================== +# Message Header with Message; for Palo Alto +# =============================================================================================== + +template t_hdr_msg { + template("${MSGHDR}${MESSAGE}"); + }; + +# =============================================================================================== +# JSON; for JSON pretty-printing +# =============================================================================================== + +template t_JSON { + template("$(format-json --scope all-nv-pairs)"); + }; diff --git a/package/etc/context/splunk_index.csv b/package/etc/context/splunk_index.csv new file mode 100644 index 0000000..5d33787 --- /dev/null +++ b/package/etc/context/splunk_index.csv @@ -0,0 +1,18 @@ +pan:traffic,index,main +pan:threat,index,main +pan:system,index,main +pan:config,index,main +pan:hipwatch,index,main +pan:correlation,index,main +pan:userid,index,main +pan:unknown,index,main +pan:traffic,sourcetype,pan:traffic +pan:threat,sourcetype,pan:threat +pan:system,sourcetype,pan:system +pan:config,sourcetype,pan:config +pan:hipwatch,sourcetype,pan:hipwatch +pan:correlation,sourcetype,pan:correlation +pan:userid,sourcetype,pan:userid +pan:unknown,sourcetype,pan:unknown +syslogng-fallback,index,syslogng-fallback +syslogng-fallback,sourcetype,syslogng-fallback diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf old mode 100755 new mode 100644 index e96a65c..ab28af9 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -1,12 +1,4 @@ @version:3.22 -#Splunk Connect for Syslog (SC4S) by Splunk, Inc. -# -#To the extent possible under law, the person who associated CC0 with -#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights -#to Splunk Connect for Syslog (SC4S). -# -#You should have received a copy of the CC0 legalcode along with this -#work. If not, see . # syslog-ng configuration file. # @@ -25,26 +17,71 @@ options { log_msg_size (65536); flush_lines (0); time_reopen (10); - log_fifo_size (1000); + log_fifo_size (10000); chain_hostnames (off); use_dns (no); use_fqdn (no); - dns-cache (no); + dns-cache(no); create_dirs (no); keep-hostname (yes); }; # =============================================================================================== -# Remote data sources that will be sent to Splunk +# Source driver defaults (which govern the parser used) +# =============================================================================================== +@define splunk-udp-driver "syslog" +@define splunk-tcp-driver "network" + +# =============================================================================================== +# Listening ports # =============================================================================================== +@define udp-listening-port "514" +@define tcp-listening-port "514" -@include "conf.d/splunk.conf" +# =============================================================================================== +# Splunk metadata (HEC/Kafka transport only) +# =============================================================================================== +@define splunk-sourcetype "syslog:fallback" +@define splunk-index "main" # =============================================================================================== -# Local System sources +# Default message template # =============================================================================================== +@define splunk-default-template "t_standard" -source s_sys { - internal(); +# =============================================================================================== +# Data collection parameters, buffers, and Timezone +# =============================================================================================== +@define ip-version 4 +@define splunk-max-connections 1000 +@define splunk-log-fifo-size 10000 +@define splunk-fetch-limit 100 +# make sure splunk-window-size >= splunk-max-connections * splunk-fetch-limit +@define splunk-window-size 100000 +@define splunk-rcvbuf 425984 +@define default-timezone "GMT" + +# =============================================================================================== +# Global modules and includes. All device-specific filters and destinations exist in conf.d +# Order is important; templates must be loaded first +# =============================================================================================== +@include "scl.conf" +@include "conf.d/templates/*.conf" +@include "conf.d/sources/*.conf" +@include "conf.d/blocks/*.conf" +@include "conf.d/destinations/*.conf" +@include "conf.d/prefilters/*.conf" +@include "conf.d/filters/*.conf" + +@include "conf.d/plugin/*/blocks/*.conf" +@include "conf.d/plugin/*/prefilters/*.conf" +@include "conf.d/plugin/*/filters/*.conf" + +log { + source(s_default-ports); + rewrite { r_set_splunk(template("t_JSON"))}; #--HEC-- + destination(d_hec); #--HEC-- + flags(fallback); }; + diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index c9d27f7..3578897 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -7,6 +7,5 @@ #Run syslog mkdir /opt/syslog-ng/var rm /opt/syslog-ng/var/syslog-ng.ctl || true -/opt/syslog-ng/sbin/switch_transport.sh /opt/syslog-ng/sbin/syslog-ng --process-mode=background /opt/syslog-ng/sbin/splunkmetrics.sh diff --git a/package/scripts/splunkmetrics.sh b/package/scripts/splunkmetrics.sh index 7017d88..7a497eb 100755 --- a/package/scripts/splunkmetrics.sh +++ b/package/scripts/splunkmetrics.sh @@ -15,7 +15,7 @@ do while read -r SourceName SourceId SourceInstance State Type Number do echo "{ - \"time\":\"$TS.000\", + \"time\":\"$TS\", \"event\":\"metric\", \"source\":\"syslog-ng\", \"host\":\"$HOSTNAME\", @@ -32,4 +32,6 @@ do done < $tmpfilecsv curl -s -S -k $SPLUNK_HEC_STATSURL -H "Authorization: Splunk $SPLUNK_HEC_TOKEN" -d "@$tmpfilejson" + rm $tmpfilejson + rm $tmpfilecsv done diff --git a/package/scripts/switch_transport.sh b/package/scripts/switch_transport.sh deleted file mode 100755 index f942d44..0000000 --- a/package/scripts/switch_transport.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -#Copyright 2019 Splunk, Inc. -# -#Use of this source code is governed by a BSD-2-clause-style -#license that can be found in the LICENSE-BSD2 file or at -#https://opensource.org/licenses/BSD-2-Clause - -# Script to change all filters to use selected transport (UF, HEC, or kafka). -# If one or more filters uses a transport different from all the others, they will need to be -# managed by hand. Unpredictable results can occur if the script is used and -# individual filters are not checked for correctness. - -case $SPLUNK_CONNECT_METHOD in - hec) - echo "Switching transport method for all filters to HEC" - sed '/^# / {/#--HEC--/ s/^#/ /}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--KAFKA--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--UF--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed 's~__SPLUNK_HEC_URL__~"'"$SPLUNK_HEC_URL"'"~' -i /opt/syslog-ng/etc/conf.d/splunk.conf - sed 's/__SPLUNK_HEC_TOKEN__/'"$SPLUNK_HEC_TOKEN"'/' -i /opt/syslog-ng/etc/conf.d/splunk.conf - ;; - kafka) - echo "Switching transport method for all filters to Kafka" - sed '/^# / {/#--KAFKA--/ s/^#/ /}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--HEC--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--UF--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - ;; - UF) - echo "Switching transport method for all filters to filesystem/UF" - sed '/^# / {/#--UF--/ s/^#/ /}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--HEC--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--KAFKA--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - ;; - *) - echo "Usage: $0 hec|kafka|UF" - ;; -esac diff --git a/splunk/SA-syslog-ng/default/indexes.conf b/splunk/SA-syslog-ng/default/indexes.conf new file mode 100644 index 0000000..e795078 --- /dev/null +++ b/splunk/SA-syslog-ng/default/indexes.conf @@ -0,0 +1,21 @@ +[syslogng_metrics] +datatype=metric +homePath = $SPLUNK_DB/syslogng_metrics/db +coldPath = $SPLUNK_DB/syslogng_metrics/colddb +thawedPath = $SPLUNK_DB/syslogng_metrics/thaweddb + +[syslogng_fallback] +homePath = $SPLUNK_DB/syslogng_fallback/db +coldPath = $SPLUNK_DB/syslogng_fallback/colddb +thawedPath = $SPLUNK_DB/syslogng_fallback/thaweddb + + +[test] +homePath = $SPLUNK_DB/test/db +coldPath = $SPLUNK_DB/test/colddb +thawedPath = $SPLUNK_DB/test/thaweddb + +[test2] +homePath = $SPLUNK_DB/test2/db +coldPath = $SPLUNK_DB/test2/colddb +thawedPath = $SPLUNK_DB/test2/thaweddb diff --git a/tests/requirements.txt b/tests/requirements.txt index cc706aa..0ebc865 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,3 +11,4 @@ jinja2 jinja2-time http://dev.splunk.com/goto/sdk-python flake8 +flaky \ No newline at end of file diff --git a/tests/test_poc.py b/tests/test_poc.py index 586d6ba..1128413 100644 --- a/tests/test_poc.py +++ b/tests/test_poc.py @@ -9,6 +9,7 @@ from time import sleep import pytest +from flaky import flaky import splunklib.client as client from jinja2 import Environment @@ -32,7 +33,7 @@ def setup_splunk(): break except ConnectionRefusedError: tried += 1 - if tried > 90: + if tried > 180: raise sleep(1) return c @@ -87,6 +88,7 @@ def splunk_single(service, search): return resultCount, eventCount +@flaky(max_runs=3, min_passes=2) def test_defaultroute(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) @@ -95,7 +97,28 @@ def test_defaultroute(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search \"{{ host }}\" | head 2") + st = env.from_string("search index=main \"{{ host }}\" sourcetype=\"syslog:fallback\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0 +@flaky(max_runs=3, min_passes=2) +def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d 01:02:03' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search) @@ -105,3 +128,25 @@ def test_defaultroute(record_property, setup_wordlist, setup_splunk): record_property("message", message) assert resultCount == 1 + +#<190>Oct 30 09:46:17 1,2012/10/30 09:46:17,01606001116,THREAT,url,1,2012/04/10 04:39:55,192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:57,22860,1,59303,80,0,0,0x208000,tcp,alert,"litetopdetect.cn/index.php",(9999),not-resolved,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,text/html +@flaky(max_runs=3, min_passes=2) +def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d 01:02:03' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},01606001116,THREAT,url,1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:57,22860,1,59303,80,0,0,0x208000,tcp,alert,\"litetopdetect.cn/index.php\",(9999),not-resolved,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,text/html\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"pan:threat\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 +