From 66d75f2f738f0364e040c293421fe3ffd2ebbced Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Thu, 31 Oct 2019 09:23:02 -0400 Subject: [PATCH] Support for archival file destinations (#179) This merge add support for file archival in syslog-ng EWMM format --- docker-compose.yml | 6 + docs/configuration.md | 21 +++- .../conf.d/conflib/_splunk/splunkfields.conf | 1 - .../etc/conf.d/destinations/archive_file.conf | 5 + .../conf.d/destinations/splunk_hec.conf.tmpl | 2 +- .../splunk_hec_internal.conf.tmpl | 2 +- .../local/config/log_paths/example.conf.tmpl | 6 +- .../etc/conf.d/log_paths/internal.conf.tmpl | 3 +- .../p_rfc3164-checkpoint_splunk.conf.tmpl | 50 ++++++--- .../log_paths/p_rfc3164-cisco_asa.conf.tmpl | 30 ++++- .../log_paths/p_rfc3164-cisco_ios.conf.tmpl | 30 ++++- .../log_paths/p_rfc3164-cisco_ise.conf.tmpl | 43 ++++++-- .../log_paths/p_rfc3164-cisco_nxos.conf.tmpl | 32 ++++-- .../p_rfc3164-forcepoint_webprotect.conf.tmpl | 27 ++++- .../p_rfc3164-fortinet_fortios.conf.tmpl | 38 +++++-- .../log_paths/p_rfc3164-juniper_idp.conf.tmpl | 33 ++++-- .../p_rfc3164-juniper_junos.conf.tmpl | 40 +++++-- .../p_rfc3164-juniper_netscreen.conf.tmpl | 30 +++-- .../log_paths/p_rfc3164-juniper_nsm.conf.tmpl | 32 ++++-- .../p_rfc3164-juniper_nsm_idp.conf.tmpl | 32 ++++-- .../p_rfc3164-microfocus_arcsight.conf.tmpl | 36 ++++-- .../p_rfc3164-paloalto_panos.conf.tmpl | 42 ++++--- .../p_rfc3164-proofpoint_pps_filter.conf.tmpl | 29 ++++- ..._rfc3164-proofpoint_pps_sendmail.conf.tmpl | 28 ++++- .../log_paths/p_rfc3164-zscaler_nss.conf.tmpl | 48 +++++--- .../p_rfc5424-noversion_cisco_asa.conf.tmpl | 28 ++++- ...rfc5424-noversion_symantec_proxy.conf.tmpl | 30 ++++- .../p_rfc5424-strict_juniper_junos.conf.tmpl | 38 +++++-- .../p_rfc5424_epoch-cisco_meraki.conf.tmpl | 49 +++++++++ .../p_rfc5424_epoch-cisco_merkai.conf.tmpl | 42 ------- package/etc/conf.d/log_paths/zfallback.conf | 2 +- package/etc/conf.d/sources/network.conf.tmpl | 103 +----------------- package/etc/go_templates/source_network.t | 22 ++-- .../local_config/log_paths/example.conf.tmpl | 6 +- package/sbin/entrypoint.sh | 14 +-- 35 files changed, 628 insertions(+), 352 deletions(-) create mode 100644 package/etc/conf.d/destinations/archive_file.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl diff --git a/docker-compose.yml b/docker-compose.yml index a9e5e74..7be17aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,13 @@ services: - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - SC4S_SOURCE_TLS_ENABLE=no - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SC4S_LISTEN_DEFAULT_TCP_PORT=514 + - SC4S_LISTEN_DEFAULT_UDP_PORT=514 +# - SC4S_LISTEN_DEFAULT_TLS_PORT=6514 - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 + - SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT=6000 +# - SC4S_ARCHIVE_CHECKPOINT=yes + - SC4S_ARCHIVE_GLOBAL=yes volumes: - ./tls:/opt/syslog-ng/tls splunk: diff --git a/docs/configuration.md b/docs/configuration.md index 02c2d18..1e84447 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,11 +21,30 @@ and variables needed to properly configure SC4S for your environment. | SC4S_DEST_SPLUNK_HEC_SSL_VERSION | comma separated list | Open SSL version list | | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | +## Archive File Configuration + +This feature is designed to support "compliance" archival of all messages. To enable this feature update the Unit file +or docker compose to mount an appropriate host folder to the container folder ``/opt/syslog-ng/var/archive``. +The files will be stored in a folder structure using the naming pattern +``${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log"``. +This pattern will create one file per "vendor_product" per minute with records formatted using syslog-ng's EWMM template. + +**WARNING POTENTIAL OUTAGE CAUSING CONSEQUENCE** + +SC4S does not prune the files that are created. The administrator must provide a means of log rotation to prune files +and/or move them to an archival system to avoid disk space failures. + +| Variable | Values | Description | +|----------|---------------|-------------| +| SC4S_ARCHIVE_GLOBAL | yes or undefined | Enable archive of all vendor_products | +| SC4S_ARCHIVE_LISTEN_ | yes(default) or undefined | See sources section of documentation enables selective archival | + + ## Syslog Source Configuration | Variable | Values/Default | Description | |----------|----------------|-------------| -| SC4S_SOURCE_TLS_ENABLE | no(default) or yes | Enable a TLS listener on port 6514 | +| SC4S_LISTEN_DEFAULT_TLS_PORT | undefined or 6514 | Enable a TLS listener on port 6514 | | SC4S_SOURCE_TLS_OPTIONS | See openssl | List of SSl/TLS protocol versions to support | | SC4S_SOURCE_TLS_CIPHER_SUITE | See openssl | List of Ciphers to support | | SC4S_SOURCE_TCP_MAX_CONNECTIONS | 2000 | Max number of TCP Connections | diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/etc/conf.d/conflib/_splunk/splunkfields.conf index c195deb..bf911a6 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf @@ -17,7 +17,6 @@ block rewrite r_set_splunk_dest_default( set("`index`", value(".splunk.index")); set("`source`", value(".splunk.source")); set("`sourcetype`", value(".splunk.sourcetype")); - set("`template`", value("fields.sc4s_template")); }; diff --git a/package/etc/conf.d/destinations/archive_file.conf b/package/etc/conf.d/destinations/archive_file.conf new file mode 100644 index 0000000..efd7990 --- /dev/null +++ b/package/etc/conf.d/destinations/archive_file.conf @@ -0,0 +1,5 @@ +destination d_archive { + file("/opt/syslog-ng/var/archive/${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log" + template('$(format-ewmm)') + ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 6a31372..78a9246 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -43,7 +43,7 @@ destination d_hec { source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$(template ${fields.sc4s_template} $(template "t_standard")) + event=$MSG fields.*)') ); }; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl index 875d3fb..b20109b 100644 --- a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl @@ -28,7 +28,7 @@ destination d_hec_internal { source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$(template ${fields.sc4s_template} $(template "t_standard")) + event=$MSG fields.*)') ); }; \ No newline at end of file diff --git a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl index a8ac264..6a68b45 100644 --- a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl +++ b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl @@ -27,7 +27,7 @@ log { # The first time this template is used the log_path will be linked to the default port {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); # Filters should be updated to use the simplest and most effecient logic possible to discard # the message from this path @@ -40,12 +40,12 @@ log { # In the second pass through the template a link to the dedicated port is used. This # normally does not require additional filters -source (s_dedicated_port_LOCAL_EXAMPLE); +source (s_LOCAL_EXAMPLE); {{- end}} #Set a default sourcetype and index - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"))}; #using the key "local_example" find any cutomized index,source or sourcetype meta values diff --git a/package/etc/conf.d/log_paths/internal.conf.tmpl b/package/etc/conf.d/log_paths/internal.conf.tmpl index 7ed823e..3c210ff 100644 --- a/package/etc/conf.d/log_paths/internal.conf.tmpl +++ b/package/etc/conf.d/log_paths/internal.conf.tmpl @@ -32,11 +32,10 @@ log { destination(d_hecmetrics); #--HEC-- } else { - {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes"}} destination(d_stdout); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))}; parser {p_add_context_splunk(key("sc4s_events")); }; destination(d_hec_internal); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index 0ad2806..ee81c82 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -1,64 +1,80 @@ -# Checkpoint Splunk format -{{- if (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT") "no") "no") }} +# Generate the custom port if defined {{ $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_checkpoint_splunk); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_CHECKPOINT_SPLUNK); + source (s_CHECKPOINT_SPLUNK); {{- end}} parser { kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); - }; - rewrite { set("${.kv.hostname}", value("HOST")); }; + rewrite { + set("${.kv.hostname}", value("HOST")); + set("checkpoint_splunk", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cp_log"), index("netops")) + }; - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), index("netops"), template("t_hdr_msg"))}; parser {p_add_context_splunk(key("checkpoint_splunk")); }; if { filter(f_checkpoint_splunk_NetworkTraffic); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))}; parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); }; } elif { filter(f_checkpoint_splunk_Web); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))}; parser {p_add_context_splunk(key("checkpoint_splunk_web")); }; } elif { filter(f_checkpoint_splunk_NetworkSessions); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))}; parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); }; } elif { filter(f_checkpoint_splunk_IDS_Malware); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))}; parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; } elif { filter(f_checkpoint_splunk_IDS); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))}; parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; } elif { filter(f_checkpoint_splunk_email); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))}; parser {p_add_context_splunk(key("checkpoint_splunk_email")); }; } elif { filter(f_checkpoint_splunk_DLP); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; }; - parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl index 5e56956..e276484 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -1,25 +1,43 @@ # Cisco ASA -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common"}} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_cisco_asa); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_CISCO_ASA_LEGACY); + source (s_CISCO_ASA_LEGACY); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; + rewrite { + set("cisco_asa", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw")) + }; parser {p_add_context_splunk(key("cisco_asa")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ASA_LEGACY") }} + destination(d_archive); +{{- end}} + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index ea17e4e..3c51cf6 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -1,26 +1,44 @@ # Cisco IOS -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_IOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_cisco_ios); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_CISCO_IOS); + source (s_CISCO_IOS); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_msg_only"))}; + rewrite { + set("cisco_ios", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops")) + }; parser { p_add_context_splunk(key("cisco_ios")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".cisco.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_IOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl index 8ef423b..b1d9297 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl @@ -1,3 +1,7 @@ +# Cisco ISE +{{ $context := dict "port_id" "CISCO_ISE" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} + #This filter uses a field we set to prevent the original messages before aggregation from being #sent to Splunk filter f_cisco_ise_complete{ @@ -40,20 +44,16 @@ parser ise_event_time { template("${ISE.DATE} ${ISE.TIME} ${ISE.TZ}") ); }; -# Cisco ISE -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TLS_PORT") "no") "no") }} -{{ $context := dict "port_id" "CISCO_ISE" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_cisco_ise); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_CISCO_ISE); + source (s_CISCO_ISE); {{- end}} parser(ise_grouping); @@ -61,12 +61,35 @@ log { if { filter(f_cisco_ise_complete); parser(ise_event_time); - rewrite { r_set_splunk_dest_default(sourcetype("cisco:ise:syslog"), index("netauth"), template("t_msg_only"))}; - parser {p_add_context_splunk(key("cisco_ise")); }; + rewrite { + set("cisco_ise", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:ise:syslog"), index("netauth")) + }; + parser {p_add_context_splunk(key("cisco_ise")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("PID")); + unset(value("LEGACY_MSGHDR")); + unset(value("EPOCH")); + unset(value("VERSION")); + unset(value("TIMESECFRAC")); + groupunset(values("ISE.*")); + }; + + destination(d_hec); + + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ISE") }} + destination(d_archive); + {{- end}} + flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index fdb592e..f2a4478 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -1,26 +1,44 @@ # Cisco NX_OS -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_cisco_nx_os); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_CISCO_NX_OS); + source (s_CISCO_NX_OS); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))}; + rewrite { + set("cisco_nxos", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg")) + }; + parser { p_add_context_splunk(key("cisco_nx_os")); }; - parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_NXOS") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl index ac0cb48..759f523 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -1,28 +1,43 @@ # Forcepoint Webprotect -{{- if (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_forcepoint_webprotect_kv); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_FORCEPOINT_WEBPROTECT); + source (s_FORCEPOINT_WEBPROTECT); {{- end}} rewrite { subst(" [^ =]+\=\-", "", value("MESSAGE"), flags("global")); + set("forcepoint_webprotect", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg")) }; - rewrite { r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg"))}; parser {p_add_context_splunk(key("forcepoint_webprotect")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FORCEPOINT_WEBPROTECT") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index 094075e..39d2bc9 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -1,17 +1,17 @@ # Fortinet Fortios -{{- if (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "FORTINET_FORTIOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_fortinet_fortios); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_FORTINET_FORTIOS); + source (s_FORTINET_FORTIOS); {{- end}} parser { @@ -19,25 +19,43 @@ log { date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); }; - rewrite { set("${.kv.devname}", value("HOST")); }; + rewrite { + set("${.kv.devname}", value("HOST")); + set("fortigate_fortios", value("fields.sc4s_vendor_product")); + }; if (match("traffic" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"))}; parser {p_add_context_splunk(key("fortinet_fortios_traffic")); }; } elif (match("utm" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"))}; parser {p_add_context_splunk(key("fortinet_fortios_utm")); }; } elif (match("event" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"))}; parser {p_add_context_splunk(key("fortinet_fortios_event")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"))}; parser {p_add_context_splunk(key("fortinet_fortios_log")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FORTINET_FORTIOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl index 65dfb91..7038621 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -1,27 +1,46 @@ # Juniper IDP -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_IDP" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_strict); filter(f_juniper_idp); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_IDP); + source (s_JUNIPER_IDP); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids"), template("t_hdr_sdata_msg"))}; + rewrite { + set("juniper_idp", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids")) + }; parser { - p_add_context_splunk(key("juniper_idp")); + p_add_context_splunk(key("juniper_idp")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + unset(value("PID")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_IDP") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index 5b26695..4e3eea5 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -1,42 +1,60 @@ # Juniper JunOS -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_JUNOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_juniper_junos_standard); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_JUNOS); + source (s_JUNIPER_JUNOS); {{- end}} + rewrite { + set("juniper_junos", value("fields.sc4s_vendor_product")); + }; + if (program('RT_IDP')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"))}; parser {p_add_context_splunk(key("juniper_idp")); }; } elif (program('RT_FLOW') or message('PFE_FW_|DFWD_')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netfw"))}; parser {p_add_context_splunk(key("juniper_junos_fw")); }; } elif (program('RT_IDS')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"))}; parser {p_add_context_splunk(key("juniper_junos_ids")); }; } elif (program('RT_UTM')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netids"))}; parser {p_add_context_splunk(key("juniper_junos_utm")); }; } elif (program('Juniper')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:sslvpn"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:sslvpn"), index("netfw"))}; parser {p_add_context_splunk(key("juniper_sslvpn")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:legacy"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:legacy"), index("netops"))}; parser {p_add_context_splunk(key("juniper_legacy")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_JUNOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl index f530a14..4a9952e 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -1,22 +1,21 @@ # Juniper Netscreen -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_NETSCREEN" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_juniper_netscreen); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_NETSCREEN); + source (s_JUNIPER_NETSCREEN); {{- end}} rewrite { - r_set_splunk_dest_default(sourcetype("netscreen:firewall"), - index("netfw"), - template("t_standard")) + set("juniper_netscreen", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("netscreen:firewall"), index("netfw")) }; parser { @@ -25,7 +24,22 @@ log { parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NETSCREEN") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl index 3a84d12..b21861d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -1,28 +1,46 @@ # Juniper NSM -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_NSM" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_juniper_nsm); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_NSM); + source (s_JUNIPER_NSM); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"), template("t_standard"))}; + rewrite { + set("juniper_nsm", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"))}; parser { - p_add_context_splunk(key("juniper_nsm")); + p_add_context_splunk(key("juniper_nsm")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NSM") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl index e590fd4..9d511c7 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -1,27 +1,43 @@ # Juniper NSM IDP -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_NSM_IDP" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_juniper_nsm_idp); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_NSM_IDP); + source (s_JUNIPER_NSM_IDP); {{- end}} - rewrite {r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"), template("t_standard"))}; + rewrite { + set("juniper_nsm_idp", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"))}; parser { - p_add_context_splunk(key("juniper_idp")); + p_add_context_splunk(key("juniper_nsm_idp")); }; - parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_standard))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NSM_IDP") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl index 4c89731..ba00cc2 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl @@ -1,3 +1,7 @@ +# Microfocus ArcSight +{{ $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} + parser p_microfocus_arcsight_header { csv-parser( columns("fields.sc4s_cef_version", "fields.cef_device_vendor", "fields.cef_device_product", "fields.cef_device_version", "fields.cef_device_event_class", "fields.cef_name", "fields.cef_severity", MESSAGE) @@ -28,30 +32,25 @@ parser p_microfocus_arcsight_source { default-selector("unknown") ); }; - -# Microfocus ArcSight -{{- if (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} -{{ $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_microfocus_arcsight); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_MICROFOCUS_ARCSIGHT); + source (s_MICROFOCUS_ARCSIGHT); {{- end}} rewrite { - r_set_splunk_dest_default(sourcetype("cef"), index("main"), template("t_msg_trim")) + set("microfocus_arcsight", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cef"), index("main")) }; parser { p_add_context_splunk(key("cef_{fields.cef_device_vendor}_${fields.cef_device_product}")); - }; parser (p_microfocus_arcsight_header); @@ -70,7 +69,22 @@ log { parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".cef.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT") }} + destination(d_archive); +{{- end}} flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 8dd5e0f..60acc74 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -1,17 +1,16 @@ # PaloAlto PanOS -{{- if (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "PALOALTO_PANOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_paloalto_panos); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_PALOALTO_PANOS); + source (s_PALOALTO_PANOS); {{- end}} # The palo message does not include a program value in the header, unfortunately. @@ -20,10 +19,10 @@ log { # While we are at it we will save the mesage type into the program field so parser can find it. rewrite { + set("paloalto_panos", value("fields.sc4s_vendor_product")); set("${LEGACY_MSGHDR}${MESSAGE}" value("MESSAGE")); unset(value("LEGACY_MSGHDR")); unset(value("PROGRAM")); - }; parser { #basic parsing @@ -46,34 +45,49 @@ log { #set the source type based on program field and lookup index from the splunk_context csv if (message(',\d+,THREAT')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"))}; parser {p_add_context_splunk(key("pan_threat")); }; } elif (message(',\d+,TRAFFIC')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"))}; parser {p_add_context_splunk(key("pan_traffic")); }; } elif (message(',\d+,SYSTEM')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"))}; parser {p_add_context_splunk(key("pan_system")); }; } elif (message(',\d+,CONFIG')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"))}; parser {p_add_context_splunk(key("pan_config")); }; } elif (message(',\d+,HIPWATCH')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"))}; parser {p_add_context_splunk(key("pan_hipwatch")); }; } elif (message(',\d+,CORRELATION')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"))}; parser {p_add_context_splunk(key("pan_correlation")); }; } elif (message(',\d+,USERID')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"))}; parser {p_add_context_splunk(key("pan_userid")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"))}; parser {p_add_context_splunk(key("pan_log")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".pan.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PALOALTO_PANOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl index e7d7087..60b6736 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl @@ -1,26 +1,43 @@ # Proofpoint -{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "PROOFPOINT_PPS_FILTER" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_proofpoint_pps_filter); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_PROOFPOINT_PPS_FILTER); + source (s_PROOFPOINT_PPS_FILTER); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"), template("t_msg_only"))}; + rewrite { + set("proofpoint_pps_filter", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"))}; parser { p_add_context_splunk(key("proofpoint_pps_filter")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl index 0866ef0..11a8057 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl @@ -1,26 +1,42 @@ # Proofpoint -{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "PROOFPOINT_PPS_SENDMAIL" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_proofpoint_pps_sendmail); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_PROOFPOINT_PPS_SENDMAIL); + source (s_PROOFPOINT_PPS_SENDMAIL); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"), template("t_msg_only"))}; + rewrite { + set("proofpoint_pps_sendmail", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"))}; parser { p_add_context_splunk(key("proofpoint_pps_sendmail")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS_SENDMAIL") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl index 7bd86fc..983e13f 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -1,19 +1,19 @@ # Proofpoint -{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_zscaler_nss); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_ZSCALER_NSS); + source (s_ZSCALER_NSS); {{- end }} rewrite { + set("zscaler_nss", value("fields.sc4s_vendor_product")); subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); }; parser { @@ -22,37 +22,37 @@ log { }; if (match("alerts" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"))}; parser { p_add_context_splunk(key("zscaler_alerts")); }; } elif (match("dns" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"))}; parser { p_add_context_splunk(key("zscaler_dns")); }; } elif (match("fw" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"))}; parser { p_add_context_splunk(key("zscaler_fw")); }; } elif (match("NSS" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_web")); }; } elif (match("audit" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"))}; parser { p_add_context_splunk(key("zscaler_zia_audit")); }; } elif (match("sandbox" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"))}; parser { p_add_context_splunk(key("zscaler_zia_sandbox")); }; } elif (match("zpa" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"))}; parser { p_add_context_splunk(key("zscaler_zpa")); }; } elif (match("zpa_auth" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"))}; parser { p_add_context_splunk(key("zscaler_zpa_auth")); }; } elif (match("zpa_auth_connector" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"))}; parser { p_add_context_splunk(key("zscaler_zpa_connector")); }; } elif (match("zpa_bba" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"))}; parser { p_add_context_splunk(key("zscaler_zpa_bba")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"))}; parser { p_add_context_splunk(key("zscaler_nss")); }; @@ -61,7 +61,23 @@ log { parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_ZSCALER_NSS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl index b041abb..69f4e28 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl @@ -1,25 +1,41 @@ # Cisco ASA RFC5424 -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_ASA" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_noversion); filter(f_cisco_asa); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_CISCO_ASA); + source (s_CISCO_ASA); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; + rewrite { + set("cisco_asa", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))}; parser {p_add_context_splunk(key("cisco_asa")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ASA") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl index e04290b..1ff0958 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl @@ -1,26 +1,44 @@ # Symantec Proxy (Bluecoat) -{{- if (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "SYMANTEC_PROXY" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_noversion); filter(f_symantec_bluecoat_proxy); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_SYMANTEC_PROXY); + source (s_SYMANTEC_PROXY); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy"), template("t_msg_only")) }; + rewrite { + set("bluecoat_proxy", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy")) + }; parser {p_add_context_splunk(key("bluecoat_proxy")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_PROXY") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl index bd473a5..19a26c3 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl @@ -1,29 +1,32 @@ # Juniper JunOS (Structured, RFC5424-compliant) -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_JUNOS_STRUCTURED" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_strict); filter(f_juniper_junos_structured); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_JUNOS_STRUCTURED); + source (s_JUNIPER_JUNOS_STRUCTURED); {{- end}} + + rewrite { + set("juniper_junos", value("fields.sc4s_vendor_product")); + }; if (program('RT_IDP')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids")) }; parser {p_add_context_splunk(key("juniper_idp_structured")); }; } elif (program('RT_FLOW')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw")) }; parser {p_add_context_splunk(key("juniper_junos_fw_structured")); }; } elif (program('RT_IDS')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids")) }; parser {p_add_context_splunk(key("juniper_junos_ids_structured")); }; } elif (program('RT_UTM')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw")) }; parser {p_add_context_splunk(key("juniper_junos_utm_structured")); }; } # Legacy Netscreen IDP is handled in the "p_rfc3164-juniper-idp.conf" log path @@ -33,13 +36,28 @@ log { # parser {p_add_context_splunk(key("juniper_junos_idp")); }; # } else { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:structured"), index("netops"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:structured"), index("netops")) }; parser {p_add_context_splunk(key("juniper_structured")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + unset(value("RAWMSG")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CHECKPOINT") }} + destination(d_archive); +{{- end}} + }; {{- end}} {{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT") "no") "no") }} diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl new file mode 100644 index 0000000..3ff2c86 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl @@ -0,0 +1,49 @@ +# Checkpoint Splunk format +{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_cisco_meraki); +{{- end}} +{{- if eq (.) "no"}} + source (s_CISCO_MERAKI); +{{- end}} + + rewrite { + set("cisco_meraki", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("meraki"), index("netfw")) + }; + parser {p_add_context_splunk(key("cisco_meraki")); }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_MERAKI") }} + destination(d_archive); +{{- end}} + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for CISCO_MERAKI traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for CISCO_MERAKI traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl deleted file mode 100644 index dbbb1f6..0000000 --- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl +++ /dev/null @@ -1,42 +0,0 @@ -# Checkpoint Splunk format -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} -{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} -{{ define "log_path" }} -log { -{{- if eq (.) "yes"}} - source(s_default-ports); - filter(f_cisco_meraki); -{{- end}} -{{- if eq (.) "no"}} - source (s_dedicated_port_CISCO_MERAKI); -{{- end}} - - #parser { - # kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); - # date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); - # - # }; - - #rewrite { set("${.kv.hostname}", value("HOST")); }; - - rewrite { r_set_splunk_dest_default(sourcetype("meraki"), index("netfw"), template("t_hdr_msg"))}; - parser {p_add_context_splunk(key("cisco_meraki")); }; - - - - parser (compliance_meta_by_source); - - destination(d_hec); #--HEC-- - - flags(flow-control); -}; -{{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} -# Listen on the specified dedicated port(s) for CISCO_MERAKI traffic - {{ tmpl.Exec "log_path" "no" }} -{{- end}} - -# Listen on the default port (typically 514) for CISCO_MERAKI traffic -{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/zfallback.conf b/package/etc/conf.d/log_paths/zfallback.conf index 37d7a86..4902bd1 100644 --- a/package/etc/conf.d/log_paths/zfallback.conf +++ b/package/etc/conf.d/log_paths/zfallback.conf @@ -1,5 +1,5 @@ log { - source(s_default-ports); + source(s_DEFAULT); rewrite { r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")) }; parser { diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index 234e87c..be569ff 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -1,101 +1,2 @@ -# =============================================================================================== -# source definition for remote devices -# =============================================================================================== - -# =============================================================================================== -# Defaults for the default-network-drivers() source: -# 514, both TCP and UDP, for RFC3164 (BSD-syslog) formatted traffic -# 601 TCP, for RFC5424 (IETF-syslog) formatted traffic -# 6514 TCP, for TLS-encrypted traffic -# =============================================================================================== -source s_default-ports { - - channel { - source { - syslog ( - transport("udp") - port(514) - ip-protocol(4) - so-rcvbuf({{- getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "425984"}}) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - - network ( - transport("tcp") - port(514) - ip-protocol(4) - max-connections({{- getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) - log-iw-size({{- getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) - log-fetch-limit({{- getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}}) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - - {{- if eq (getenv "SC4S_SOURCE_TLS_ENABLE") "yes"}} - network( - port(6514) - transport("tls") - ip-protocol(4) - max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) - log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) - log-fetch-limit({{getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}}) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - tls(allow-compress(yes) - key-file("/opt/syslog-ng/tls/server.key") - cert-file("/opt/syslog-ng/tls/server.pem") - ssl-options({{- getenv "SC4S_SOURCE_TLS_OPTIONS" "no-sslv2, no-sslv3, no-tlsv1" }}) - cipher-suite("{{- getenv "SC4S_SOURCE_TLS_CIPHER_SUITE" "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH" }}") - ) - ); - - {{- end }} - }; - #TODO: #60 Remove this function with enhancement - rewrite(set_rfcnonconformant); - - if { - filter(f_rfc5424_strict); - parser { - syslog-parser(flags(syslog-protocol store-raw-message)); - }; - rewrite(set_rfc5424_strict); - } elif { - filter(f_rfc5424_noversion); - parser { - syslog-parser(flags(syslog-protocol store-raw-message)); - }; - rewrite(set_rfc5424_noversion); - } elif { - parser {cisco-parser()}; - rewrite(set_cisco_ios); - } elif { - parser (p_cisco_meraki); - rewrite(set_rfc5424_epochtime); - } else { - parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); - }; - rewrite(set_rfc3164); - }; - - rewrite(r_set_splunk_default); - - parser { - vendor_product_by_source(); - }; - }; -}; \ No newline at end of file +{{ $context := dict "port_id" "DEFAULT" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 09ea120..badcbbb 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -1,10 +1,10 @@ -# The following is the dedicated source port declaration for {{ (print .port_id) }} +# The following is the source port declaration for {{ (print .port_id) }} # Two log paths will be created -- one for the dedicated port(s) and one for the default (typically port 514) - -source s_dedicated_port_{{ .port_id}} { +{{- define "T1" }} +source s_{{ .port_id}} { channel { source { -{{- if ne (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT" ) "no") "no" }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT" )) }} syslog ( transport("udp") port({{getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") }}) @@ -18,7 +18,7 @@ source s_dedicated_port_{{ .port_id}} { flags(no-parse) ); {{- end}} -{{- if ne (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "no") "no" }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT" )) }} network ( transport("tcp") port({{getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") }}) @@ -34,7 +34,7 @@ source s_dedicated_port_{{ .port_id}} { flags(no-parse) ); {{- end}} -{{- if ne (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "no") "no" }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT" )) }} network( port({{getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") }}) transport("tls") @@ -111,6 +111,12 @@ source s_dedicated_port_{{ .port_id}} { {{- end }} rewrite(r_set_splunk_default); + parser { + vendor_product_by_source(); + }; }; - -}; \ No newline at end of file +}; +{{- end }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) or (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT")) }} +{{ template "T1" (.) }} +{{- end }} diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl index a8ac264..6a68b45 100644 --- a/package/etc/local_config/log_paths/example.conf.tmpl +++ b/package/etc/local_config/log_paths/example.conf.tmpl @@ -27,7 +27,7 @@ log { # The first time this template is used the log_path will be linked to the default port {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); # Filters should be updated to use the simplest and most effecient logic possible to discard # the message from this path @@ -40,12 +40,12 @@ log { # In the second pass through the template a link to the dedicated port is used. This # normally does not require additional filters -source (s_dedicated_port_LOCAL_EXAMPLE); +source (s_LOCAL_EXAMPLE); {{- end}} #Set a default sourcetype and index - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"))}; #using the key "local_example" find any cutomized index,source or sourcetype meta values diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index ff9a112..a93736d 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -1,17 +1,11 @@ #!/usr/bin/env bash source scl_source enable rh-python36 +export SC4S_LISTEN_DEFAULT_TCP_PORT=514 +export SC4S_LISTEN_DEFAULT_UDP_PORT=514 + cd /opt/syslog-ng -#The following is no longer needed but retained as a comment just in case we run into command line length issues -#for d in $(find /opt/syslog-ng/etc -type d) -#do -# echo Templating conf for $d -# gomplate \ -# --input-dir=$d \ -# --template t=etc/go_templates/ \ -# --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ -# --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" -#done + gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/