From 51b6e6e5cab160c2bdc2ae15cec5cf01935acc6b Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 3 Jan 2020 18:24:47 -0800 Subject: [PATCH 01/14] Add timestamp parsing to zscaler log path * Add missing timestamp parsing to zscaler log path --- package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl index d70f139..79bcf38 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -12,6 +12,7 @@ log { source (s_ZSCALER_NSS); {{- end }} + parser { date-parser(format("%Y-%m-%d %H:%M:%S") template('$(substr "$LEGACY_MSGHDR$MSG" "0" "19")')); }; rewrite { set("zscaler_nss", value("fields.sc4s_vendor_product")); subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); From 230e282b2702a168ee383021b6dbccabd6b0adcb Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sun, 5 Jan 2020 20:44:14 -0800 Subject: [PATCH 02/14] Gomplate update for "soup" port defaults * Add defaults in template for `source_network.t` * Remove default listening ports in entrypoint.sh and docker-compose.yml files * Clean up all log path templates to make final output more readable * Minor cleanup to `network.conf.tmpl` for readability --- docker-compose.yml | 8 ++--- .../p_rfc3164-checkpoint_splunk.conf.tmpl | 12 +++---- .../log_paths/p_rfc3164-cisco_acs.conf.tmpl | 16 +++++---- .../log_paths/p_rfc3164-cisco_asa.conf.tmpl | 9 ++--- .../log_paths/p_rfc3164-cisco_ios.conf.tmpl | 11 ++++--- .../log_paths/p_rfc3164-cisco_ise.conf.tmpl | 15 +++++---- .../log_paths/p_rfc3164-cisco_nxos.conf.tmpl | 11 ++++--- .../p_rfc3164-forcepoint_webprotect.conf.tmpl | 11 ++++--- .../p_rfc3164-fortinet_fortios.conf.tmpl | 11 ++++--- .../log_paths/p_rfc3164-infoblox.conf.tmpl | 11 ++++--- .../log_paths/p_rfc3164-juniper_idp.conf.tmpl | 11 ++++--- .../p_rfc3164-juniper_junos.conf.tmpl | 11 ++++--- .../p_rfc3164-juniper_netscreen.conf.tmpl | 15 +++++---- .../log_paths/p_rfc3164-juniper_nsm.conf.tmpl | 11 ++++--- .../p_rfc3164-juniper_nsm_idp.conf.tmpl | 11 ++++--- .../p_rfc3164-microfocus_arcsight.conf.tmpl | 7 ++-- .../p_rfc3164-paloalto_panos.conf.tmpl | 12 ++++--- .../p_rfc3164-proofpoint_pps.conf.tmpl | 12 ++++--- .../p_rfc3164-symantec_brightmail.conf.tmpl | 15 +++++---- .../p_rfc3164-ubiquiti_unifi.conf.tmpl | 16 ++++----- .../log_paths/p_rfc3164-zscaler_nss.conf.tmpl | 12 ++++--- .../p_rfc5424-noversion_cisco_asa.conf.tmpl | 12 ++++--- ...rfc5424-noversion_symantec_proxy.conf.tmpl | 12 ++++--- .../p_rfc5424-strict_juniper_junos.conf.tmpl | 10 +++--- .../p_rfc5424_epoch-cisco_meraki.conf.tmpl | 12 ++++--- .../log_paths/p_vmware_vsphere.conf.tmpl | 14 ++++---- .../log_paths/p_za_nix_syslog.conf.tmpl | 12 ++++--- .../conf.d/log_paths/p_zz_fallback.conf.tmpl | 1 + .../etc/conf.d/log_paths/startup.conf.tmpl | 4 +-- package/etc/conf.d/sources/network.conf.tmpl | 5 +-- package/etc/go_templates/source_network.t | 33 +++++++++---------- package/sbin/entrypoint.sh | 4 --- 32 files changed, 200 insertions(+), 167 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index dff7a5c..3e3bc3d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,12 +44,12 @@ services: - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SC4S_SOURCE_TLS_ENABLE=no - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no - - SC4S_LISTEN_DEFAULT_TCP_PORT=514 - - SC4S_LISTEN_DEFAULT_UDP_PORT=514 -# - SC4S_LISTEN_DEFAULT_TLS_PORT=6514 +# - SC4S_LISTEN_DEFAULT_TCP_PORT=514 +# - SC4S_LISTEN_DEFAULT_UDP_PORT=514 +# - SC4S_LISTEN_DEFAULT_TLS_PORT=6514 - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 - SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT=6000 -# - SC4S_ARCHIVE_CHECKPOINT=yes +# - SC4S_ARCHIVE_CHECKPOINT=yes - SC4S_ARCHIVE_GLOBAL=yes volumes: - ./tls:/opt/syslog-ng/tls diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index 9c64b3b..0fd2808 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -1,10 +1,10 @@ # Checkpoint -# Generate the custom port if defined -{{ $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -87,7 +87,7 @@ log { {{- end}} {{- if or (or (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CHECKPOINT_SPLUNK traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for CHECKPOINT_SPLUNK traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl index 15d0963..0e3c4c1 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl @@ -1,9 +1,11 @@ # Cisco ACS -{{ $context := dict "port_id" "CISCO_ACS" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +# This filter uses a field we set to prevent the original messages before aggregation from being +# sent to Splunk -#This filter uses a field we set to prevent the original messages before aggregation from being -#sent to Splunk filter f_cisco_acs_complete{ match("yes", value("ACS.COMPLETE") type(glob)); }; @@ -29,8 +31,8 @@ parser acs_grouping { ); }; -#The syslog message includes a date with nano seconds and TZ which is not in the header -#So must reparse the date +# The syslog message includes a date with nano seconds and TZ which is not in the header +# So must reparse the date parser acs_event_time { csv-parser( columns(ACS.DATE, ACS.TIME, ACS.TZ, MESSAGE) @@ -44,7 +46,7 @@ parser acs_event_time { template("${ACS.DATE} ${ACS.TIME} ${ACS.TZ}") ); }; -# The following is an inline template; we will use this to generate the actual log path +{{- /* The following is an inline template to generate the actual log path */}} {{ define "log_path" }} log { {{- if eq (.) "yes"}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl index 8b9ca9a..b794ad8 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -1,9 +1,10 @@ # Cisco ASA -{{ $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index 015f86b..2749bc8 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -1,9 +1,10 @@ # Cisco IOS -{{ $context := dict "port_id" "CISCO_IOS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_IOS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -36,7 +37,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_IOS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for CISCO_IOS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl index 7157f5e..b56dae5 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl @@ -1,15 +1,18 @@ # Cisco ISE -{{ $context := dict "port_id" "CISCO_ISE" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +# This filter uses a field we set to prevent the original messages before aggregation from being +# sent to Splunk -#This filter uses a field we set to prevent the original messages before aggregation from being -#sent to Splunk filter f_cisco_ise_complete{ match("yes", value("ISE.COMPLETE") type(glob)); }; #This parser adds messages from ISE to a context without sending them #forward to Splunk + parser ise_grouping { csv-parser( columns(PID, ISE.num, ISE.seq, MESSAGE) @@ -31,6 +34,7 @@ parser ise_grouping { #The syslog message includes a date with nano seconds and TZ which is not in the header #So must reparse the date + parser ise_event_time { csv-parser( columns(ISE.DATE, ISE.TIME, ISE.TZ, MESSAGE) @@ -44,7 +48,7 @@ parser ise_event_time { template("${ISE.DATE} ${ISE.TIME} ${ISE.TZ}") ); }; -# The following is an inline template; we will use this to generate the actual log path +{{- /* The following is an inline template to generate the actual log path */}} {{ define "log_path" }} log { {{- if eq (.) "yes"}} @@ -81,7 +85,6 @@ log { flags(flow-control,final); }; - }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index 6c40bc0..683dc1f 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -1,9 +1,10 @@ # Cisco NX_OS -{{ $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -38,7 +39,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_NX_OS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for CISCO_NX_OS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl index dc65e02..a3346bd 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -1,9 +1,10 @@ # Forcepoint Webprotect -{{ $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -37,7 +38,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT")) }} # Listen on the specified dedicated port(s) for FORCEPOINT_WEBPROTECT traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for FORCEPOINT_WEBPROTECT traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index a3bfc99..837ce58 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -1,9 +1,10 @@ # Fortinet Fortios -{{ $context := dict "port_id" "FORTINET_FORTIOS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "FORTINET_FORTIOS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -56,7 +57,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for FORTINET_FORTIOS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for FORTINET_FORTIOS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl index 1c047af..a84ecaa 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl @@ -1,9 +1,10 @@ # Infoblox -{{ $context := dict "port_id" "INFOBLOX" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "INFOBLOX" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -70,7 +71,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_INFOBLOX_TCP_PORT")) (getenv (print "SC4S_LISTEN_INFOBLOX_UDP_PORT"))) (getenv (print "SC4S_LISTEN_INFOBLOX_TLS_PORT")) }} # Listen on the specified dedicated port(s) for INFOBLOX traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for INFOBLOX traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl index 5aecfc0..162996d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -1,9 +1,10 @@ # Juniper IDP -{{ $context := dict "port_id" "JUNIPER_IDP" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_IDP" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -36,7 +37,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_IDP_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_IDP traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for JUNIPER_IDP traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index 004e8c8..facaf1c 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -1,9 +1,10 @@ # Juniper JunOS -{{ $context := dict "port_id" "JUNIPER_JUNOS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_JUNOS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -55,7 +56,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_JUNOS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for JUNIPER_JUNOS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl index ece524e..6fca3d1 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -1,9 +1,10 @@ # Juniper Netscreen -{{ $context := dict "port_id" "JUNIPER_NETSCREEN" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_NETSCREEN" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -35,8 +36,8 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NETSCREEN traffic - {{ tmpl.Exec "log_path" "no" }} -{{- end}} +{{ tmpl.Exec "log_path" "no" }} +{{- end }} # Listen on the default port (typically 514) for JUNIPER_NETSCREEN traffic -{{ tmpl.Exec "log_path" "yes" }} +{{ tmpl.Exec "log_path" "yes" }} \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl index 4cac2a7..668f287 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -1,9 +1,10 @@ # Juniper NSM -{{ $context := dict "port_id" "JUNIPER_NSM" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_NSM" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -36,7 +37,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NSM traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for JUNIPER_NSM traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl index f33f3f6..2ac5fa0 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -1,9 +1,10 @@ # Juniper NSM IDP -{{ $context := dict "port_id" "JUNIPER_NSM_IDP" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_NSM_IDP" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -35,7 +36,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NSM_IDP traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for JUNIPER_NSM_IDP traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl index 25ddce5..8a5a386 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl @@ -1,6 +1,7 @@ # Microfocus ArcSight -{{ $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} parser p_microfocus_arcsight_header { csv-parser( @@ -32,7 +33,7 @@ parser p_microfocus_arcsight_source { default-selector("unknown") ); }; -# The following is an inline template; we will use this to generate the actual log path +{{- /* The following is an inline template to generate the actual log path */}} {{ define "log_path" }} log { {{- if eq (.) "yes"}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 8c6f97f..4a797d1 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -1,8 +1,10 @@ # PaloAlto PanOS -{{ $context := dict "port_id" "PALOALTO_PANOS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "PALOALTO_PANOS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -91,7 +93,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for PALOALTO_PANOS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for PALOALTO_PANOS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl index 46c5267..d17a226 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl @@ -1,8 +1,10 @@ # Proofpoint Protection Server -{{ $context := dict "port_id" "PROOFPOINT_PPS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "PROOFPOINT_PPS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -42,7 +44,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_TCP_PORT")) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for PROOFPOINT_PPS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for PROOFPOINT_PPS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl index a2b4c57..95b811b 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl @@ -1,4 +1,5 @@ -#Symantec Brightmail +# Symantec Brightmail + {{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} filter f_symantec_brightmail_complete{ match("yes", value("SMG.COMPLETE") type(glob)); @@ -22,10 +23,12 @@ parser symantec_brightmail_grouping { }; {{- end }} -{{ $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -90,7 +93,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_TCP_PORT")) (getenv (print "SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT"))) (getenv (print "SC4S_SYMANTEC_BRIGHTMAIL_NSS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for SYMANTEC_BRIGHTMAIL traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for SYMANTEC_BRIGHTMAIL traffic diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl index a6ab503..a8b891b 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl @@ -1,9 +1,10 @@ -#Ubiquiti unifi -{{ $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} +# Ubiquiti unifi +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -14,7 +15,6 @@ log { source (s_UBIQUITI_UNIFI); {{- end}} - parser {p_add_context_splunk(key("ubiquiti_unifi")); }; #Firewall @@ -125,8 +125,8 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TCP_PORT")) (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_UDP_PORT"))) (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TLS_PORT")) }} # Listen on the specified dedicated port(s) for UBIQUITI_UNIFI traffic - {{tmpl.Exec "log_path" "no" }} +{{tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for UBIQUITI_UNIFI traffic -{{tmpl.Exec "log_path" "yes" }} +{{tmpl.Exec "log_path" "yes" }} \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl index d70f139..da807c3 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -1,8 +1,10 @@ # Zscaler -{{ $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -75,7 +77,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT")) (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for ZSCALER_NSS traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for ZSCALER_NSS traffic diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl index 1b93b48..59e9c50 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl @@ -1,8 +1,10 @@ # Cisco ASA RFC5424 -{{ $context := dict "port_id" "CISCO_ASA" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_ASA" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -35,7 +37,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_ASA traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for CISCO_ASA traffic diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl index 7a40ebf..64584fd 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl @@ -1,8 +1,10 @@ # Symantec Proxy (Bluecoat) -{{ $context := dict "port_id" "SYMANTEC_PROXY" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "SYMANTEC_PROXY" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -36,7 +38,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TCP_PORT")) (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_UDP_PORT"))) (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TLS_PORT")) }} # Listen on the specified dedicated port(s) for SYMANTEC_PROXY traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for SYMANTEC_PROXY traffic diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl index e808f85..f5ac665 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl @@ -1,8 +1,10 @@ # Juniper JunOS (Structured, RFC5424-compliant) -{{ $context := dict "port_id" "JUNIPER_JUNOS_STRUCTURED" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "JUNIPER_JUNOS_STRUCTURED" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl index f080c1c..4c2ba45 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl @@ -1,8 +1,10 @@ # Cisco Meraki -{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -35,7 +37,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_MERAKI traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for CISCO_MERAKI traffic diff --git a/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl b/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl index 5d31966..171fccc 100644 --- a/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl @@ -1,10 +1,10 @@ -#VMware ESXi and NSX -# Generate the custom port if defined -{{ $context := dict "port_id" "VMWARE" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +# VMware ESXi and NSX +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "VMWARE" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes"}} source(s_DEFAULT); @@ -96,7 +96,7 @@ log { {{- end}} {{- if or (or (getenv (print "SC4S_LISTEN_VMWARE_TCP_PORT")) (getenv (print "SC4S_LISTEN_VMWARE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_VMWARE_TLS_PORT")) }} # Listen on the specified dedicated port(s) for VMWARE traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for VMWARE traffic diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl index 91fa349..e1911f7 100644 --- a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl @@ -1,8 +1,10 @@ # Linux/Unix OS system logs -{{ $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { {{- if eq (.) "yes" }} source(s_DEFAULT); @@ -38,7 +40,7 @@ log { {{- if or (or (getenv (print "SC4S_LISTEN_NIX_SYSLOG_TCP_PORT")) (getenv (print "SC4S_LISTEN_NIX_SYSLOG_UDP_PORT"))) (getenv (print "SC4S_NIX_SYSLOG_NSS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for NIX_SYSLOG traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for NIX_SYSLOG traffic diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl index 44be960..d2057c2 100644 --- a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl @@ -1,4 +1,5 @@ # Fallback for un-parsed sources + log { source(s_DEFAULT); diff --git a/package/etc/conf.d/log_paths/startup.conf.tmpl b/package/etc/conf.d/log_paths/startup.conf.tmpl index 3c5df61..4559544 100644 --- a/package/etc/conf.d/log_paths/startup.conf.tmpl +++ b/package/etc/conf.d/log_paths/startup.conf.tmpl @@ -1,8 +1,9 @@ +# Startup events + {{- define "log_path"}} log { source(s_startup_out); - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))}; parser {p_add_context_splunk(key("sc4s_events:startup:out")); }; @@ -19,7 +20,6 @@ log { log { source(s_startup_err); - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events:startup:err"), index("main"))}; parser {p_add_context_splunk(key("sc4s_events")); }; diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index be569ff..7f6a9e5 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -1,2 +1,3 @@ -{{ $context := dict "port_id" "DEFAULT" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} +# Default "soup" syslog-ng sources, typically UDP/TCP 514; TLS 6514 +{{- $context := dict "port_id" "DEFAULT" "parser" "common" -}} +{{- tmpl.Exec "t/source_network.t" $context -}} \ No newline at end of file diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 8b43665..ff4f766 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -1,13 +1,14 @@ +{{ define "T1" }} + # The following is the source port declaration for {{ (print .port_id) }} -# Two log paths will be created -- one for the dedicated port(s) and one for the default (typically port 514) -{{- define "T1" }} -source s_{{ .port_id}} { + +source s_{{ .port_id }} { channel { source { -{{- if (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT" )) }} +{{- if or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) (eq .port_id "DEFAULT") }} syslog ( transport("udp") - port({{getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") }}) + port({{ getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") "514" }}) ip-protocol(4) so-rcvbuf({{getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "425984"}}) keep-hostname(yes) @@ -18,10 +19,10 @@ source s_{{ .port_id}} { flags(no-parse) ); {{- end}} -{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT" )) }} +{{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (eq .port_id "DEFAULT") }} network ( transport("tcp") - port({{getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") }}) + port({{ getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "514" }}) ip-protocol(4) max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) @@ -34,10 +35,10 @@ source s_{{ .port_id}} { flags(no-parse) ); {{- end}} -{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT" )) }} +{{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT")) (eq .port_id "DEFAULT_TLS") }} network( - port({{getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") }}) transport("tls") + port({{ getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "6514" }}) ip-protocol(4) max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) @@ -59,7 +60,7 @@ source s_{{ .port_id}} { }; #TODO: #60 Remove this function with enhancement rewrite(set_rfcnonconformant); -{{ if eq .parser "rfc5424_strict" }} +{{- if eq .parser "rfc5424_strict" }} filter(f_rfc5424_strict); parser { syslog-parser(flags(syslog-protocol)); @@ -129,11 +130,9 @@ source s_{{ .port_id}} { unset(value("fields.sc4s_time_zone")); }; }; - - - }; + }; }; -{{- end }} -{{- if or (or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT"))) (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT")) }} -{{ template "T1" (.) }} -{{- end }} +{{- end -}} +{{- if or (or (or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT"))) (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT"))) (eq .port_id "DEFAULT") -}} +{{- template "T1" (.) -}} +{{- end -}} \ No newline at end of file diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index aa01301..b7d5b2d 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -1,14 +1,10 @@ #!/usr/bin/env bash source scl_source enable rh-python36 -export SC4S_LISTEN_DEFAULT_TCP_PORT=514 -export SC4S_LISTEN_DEFAULT_UDP_PORT=514 - cd /opt/syslog-ng gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ - mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ From db9219fb7cede627de5518e340e62827c0dd4b03 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Tue, 7 Jan 2020 17:05:17 -0800 Subject: [PATCH 03/14] Added default check for TLS; remove extraneos ToBool declarations * Add check for TLS enable env var in default source * Remove extraneous `conv.ToBool` declarations --- .../etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl | 2 +- .../log_paths/p_rfc3164-symantec_brightmail.conf.tmpl | 6 +++--- package/etc/conf.d/log_paths/startup.conf.tmpl | 2 +- package/etc/go_templates/source_network.t | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 4a797d1..1a54790 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -36,7 +36,7 @@ log { #2012/04/10 04:39:55 #parse the date date-parser( - {{- if ((getenv "SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS") | conv.ToBool) }} + {{- if (conv.ToBool (getenv "SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS")) }} format("%Y/%m/%d %H:%M:%S.%f") {{- else}} format("%Y/%m/%d %H:%M:%S") diff --git a/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl index 95b811b..04ab7d1 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-symantec_brightmail.conf.tmpl @@ -1,6 +1,6 @@ # Symantec Brightmail -{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} +{{- if (conv.ToBool (getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes")) }} filter f_symantec_brightmail_complete{ match("yes", value("SMG.COMPLETE") type(glob)); }; @@ -39,7 +39,7 @@ log { source (s_SYMANTEC_BRIGHTMAIL); {{- end }} -{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} +{{- if (conv.ToBool (getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes")) }} if { filter(f_symantec_brightmail_details); @@ -83,7 +83,7 @@ log { {{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL" "no")) }} destination(d_archive); {{- end}} -{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} +{{- if (conv.ToBool (getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes")) }} }; {{- end}} diff --git a/package/etc/conf.d/log_paths/startup.conf.tmpl b/package/etc/conf.d/log_paths/startup.conf.tmpl index 4559544..8153c55 100644 --- a/package/etc/conf.d/log_paths/startup.conf.tmpl +++ b/package/etc/conf.d/log_paths/startup.conf.tmpl @@ -7,7 +7,7 @@ log { rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))}; parser {p_add_context_splunk(key("sc4s_events:startup:out")); }; - {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no") | conv.ToBool) }} + {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_INTERNAL_EVENTS_HEC" "no")) }} destination(d_hec_internal); {{- end}} diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index ff4f766..eef481a 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -35,7 +35,7 @@ source s_{{ .port_id }} { flags(no-parse) ); {{- end}} -{{- if or (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT")) (eq .port_id "DEFAULT_TLS") }} +{{- if (conv.ToBool (getenv "SC4S_SOURCE_TLS_ENABLE" "no")) }} network( transport("tls") port({{ getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "6514" }}) From 516a7047a789d293b34f41b56480b732bf8ab0d4 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 10 Jan 2020 09:39:21 -0800 Subject: [PATCH 04/14] Fix URL grooming for list of HEC URLs * Fix URL grooming when list of HEC URLs is specified --- package/etc/conf.d/destinations/splunk_hec.conf.tmpl | 2 +- package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl | 2 +- package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 1b2d8fe..90dc794 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -1,6 +1,6 @@ destination d_hec { http( - url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event") + url("{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event") method("POST") log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) workers({{- getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10"}}) diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl index 550063c..3bce5f0 100644 --- a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl @@ -1,6 +1,6 @@ destination d_hec_internal { http( - url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event") + url("{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event") method("POST") log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) workers(10) diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl index 2593b8c..7c97ce8 100644 --- a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl @@ -1,6 +1,6 @@ destination d_hecmetrics { http( - url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector") + url("{{- getenv "SPLUNK_HEC_URL" | strings.ReplaceAll "/services/collector" "" | strings.ReplaceAll "/event" "" | regexp.ReplaceLiteral "[, ]+" "/services/collector/event " }}/services/collector/event") method("POST") batch-lines(50) batch-bytes(1024Kb) From fd77a51bb589af6b631172f87191878226b03d8e Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Fri, 10 Jan 2020 12:31:53 -0800 Subject: [PATCH 05/14] Update docs for default and unique ports * Update all runtime docs for default and unique ports * Add default TLS port (6514) to all docker-compose and unit file examples * Add default env variables to BYOE docs * Runtime docs cleanup for flow/readability --- docs/gettingstarted/byoe-rhel7.md | 20 ++- docs/gettingstarted/docker-swarm-general.md | 126 ++++++++--------- docs/gettingstarted/docker-swarm-rhel7.md | 128 +++++++++--------- docs/gettingstarted/docker-systemd-general.md | 111 ++++++++------- docs/gettingstarted/index.md | 10 +- docs/gettingstarted/podman-systemd-general.md | 111 ++++++++------- 6 files changed, 258 insertions(+), 248 deletions(-) diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 59baff5..437deb8 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -17,7 +17,7 @@ administration and syslog-ng configuration experience is assumed when using the * NOTE: Do _not_ depend on the distribution-supplied version of syslog-ng, as it will likely be far too old. Read this [explanation](https://www.syslog-ng.com/community/b/blog/posts/installing-latest-syslog-ng-on-rhel-and-other-rpm-distributions) -for the reason why syslog-ng builds are so dated in most RHEL/Debian distributions. +for the reason why syslog-ng builds are so dated in almost all RHEL/Debian distributions. # BYOE Installation Instructions @@ -157,3 +157,21 @@ sudo systemctl daemon-reload sudo systemctl enable sc4s sudo systemctl start sc4s ``` +## Configure SC4S Listening Ports + +Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. +The docker compose file and standard SC4S configurations reflect these defaults. These defaults can be changed by adding the following +additional environment variables with appropriate values to the ``env_file`` above: +```dotenv +SC4S_LISTEN_DEFAULT_TCP_PORT=514 +SC4S_LISTEN_DEFAULT_UDP_PORT=514 +SC4S_LISTEN_DEFAULT_TLS_PORT=6514 +``` +### Dedicated (Unique) Listening Ports + +For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +For collection of such sources we provide a means of dedicating a unique listening port to a specific source. + +Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology +in use. \ No newline at end of file diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 44d2255..7f72222 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -22,7 +22,7 @@ reboot: net.ipv4.ip_forward=1 ``` -# SC4S Configuration +# SC4S Initial Configuration * Create a directory on the server for local configurations and disk buffering. This should be available to all administrators, for example: @@ -46,6 +46,11 @@ services: protocol: udp # Comment the following line out if using docker-compose mode: host + - target: 6514 + published: 6514 + protocol: tcp +# Comment the following line out if using docker-compose + mode: host env_file: - /opt/sc4s/env_file volumes: @@ -88,7 +93,7 @@ document for details on the directory structure the archive uses. * IMPORTANT: When creating the directories above, ensure the directories created match the volume mounts specified in the `docker-compose.yml` file. Failure to do this will cause SC4S to abort at startup. -## Configure the SC4S environment +# Configure the SC4S environment Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: @@ -108,58 +113,40 @@ match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to uncomment the last line in the example above. -## Modify index destinations for Splunk - -Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. - -* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your -environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in -this table that pertain to the individual data source filters that are included with SC4S. -* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further -information is covered in the "Log Path overrides" section of the Configuration document. - -## Configure source filtering by source IP or host name - -Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. - -* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. -* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. -* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. - -## Configure compliance index/metadata overrides +## Configure SC4S Listening Ports -In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. -The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to -the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file -lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is -covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. - -## Start/Restart SC4S +Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. +The docker compose file and standard SC4S configurations reflect these defaults. If it desired to change some or all of them, container +port mapping can be used to change the defaults without altering the underlying SC4S configuration. To do this, simply change the +``published`` port(s) in the docker compose file (which represents the actual listening ports on the host machine), like so: -```bash -docker stack deploy --compose-file docker-compose.yml sc4s ``` + ports: + - target: 514 + published: 614 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host +``` +This snippet above instructs the _host_ to listen on TCP port 614 and map that port to the default TCP 514 port on the _container_. +No changes to the underlying SC4S default configuration (environment variables) are needed. -# Scale out - -Additional hosts can be deployed for syslog collection from additional network zones and locations. - - -# Configure Dedicated Listening Ports +### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. For collection of such sources we provide a means of dedicating a unique listening port to a specific source. -Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. +Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology +in use. -In the following example the target port ranges allow for up to 21 technology-specific ports. Modify individual ports or a -range as appropriate for your network. +The docker compose file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured +by the environment variable(s). In the following example, additional ``target`` stanzas are added for the main ``sc4s`` container, where the +``target`` and ``published`` lines provide for 21 additional technology-specific ports. Follow these steps to configure unique ports: -* Modify the unit file ``/opt/sc4s/docker-compose.yml`` +* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" +section for more information on your specific device(s). +* Modify the unit file ``/opt/sc4s/docker-compose.yml`` and add/change port stanzas as appropriate using the example below: ```yaml version: "3.7" services: @@ -176,6 +163,11 @@ services: protocol: udp #Comment the following line out if using docker-compose mode: host + - target: 6514 + published: 6514 + protocol: tcp +# Comment the following line out if using docker-compose + mode: host - target: 5000-5020 published: 5000-5020 protocol: tcp @@ -195,34 +187,44 @@ services: # - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). +## Modify index destinations for Splunk -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. +* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further +information is covered in the "Log Path overrides" section of the Configuration document. -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +## Configure source filtering by source IP or host name -```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088 -SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 -#Uncomment the following line if using untrusted SSL certificates -#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no -``` +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* Restart SC4S (below) +* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. -## Start/Restart SC4S +## Configure compliance index/metadata overrides + +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is +covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. + +# Scale out + +Additional hosts can be deployed for syslog collection from additional network zones and locations. + +# Start/Restart SC4S ```bash docker stack deploy --compose-file docker-compose.yml sc4s ``` - # Stop SC4S Start by obtaining the stack name (ID): @@ -262,7 +264,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index 0af8b6b..09f8179 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -31,7 +31,7 @@ systemctl start docker.service sudo docker swarm init ``` -# SC4S Configuration +# SC4S Initial Configuration * Create a directory on the server for local configurations and disk buffering. This should be available to all administrators, for example: ``/opt/sc4s/`` @@ -52,6 +52,11 @@ services: - target: 514 published: 514 protocol: udp +# Comment the following line out if using docker-compose + mode: host + - target: 6514 + published: 6514 + protocol: tcp # Comment the following line out if using docker-compose mode: host env_file: @@ -96,7 +101,7 @@ document for details on the directory structure the archive uses. * IMPORTANT: When creating the directories above, ensure the directories created match the volume mounts specified in the `docker-compose.yml` file. Failure to do this will cause SC4S to abort at startup. -## Configure the SC4S environment +# Configure the SC4S environment Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: @@ -116,60 +121,40 @@ match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to uncomment the last line in the example below. +## Configure SC4S Listening Ports -## Modify index destinations for Splunk - -Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. - -* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your -environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in -this table that pertain to the individual data source filters that are included with SC4S. -* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further -information is covered in the "Log Path overrides" section of the Configuration document. - - -## Configure source filtering by source IP or host name - -Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. - -* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. -* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. -* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. - -## Configure compliance index/metadata overrides - -In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. -The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to -the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file -lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is -covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. - -## Start/Restart SC4S +Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. +The docker compose file and standard SC4S configurations reflect these defaults. If it desired to change some or all of them, container +port mapping can be used to change the defaults without altering the underlying SC4S configuration. To do this, simply change the +``published`` port(s) in the docker compose file (which represents the actual listening ports on the host machine), like so: -```bash -sudo docker stack deploy --compose-file docker-compose.yml sc4s ``` + ports: + - target: 514 + published: 614 + protocol: tcp +#Comment the following line out if using docker-compose + mode: host +``` +This snippet above instructs the _host_ to listen on TCP port 614 and map that port to the default TCP 514 port on the _container_. +No changes to the underlying SC4S default configuration (environment variables) are needed. -# Scale out - -Additional hosts can be deployed for syslog collection from additional network zones and locations. - - -# Configure Dedicated Listening Ports +### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. For collection of such sources we provide a means of dedicating a unique listening port to a specific source. -Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. +Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology +in use. -In the following example the target port ranges allow for up to 21 technology-specific ports. Modify individual ports or a -range as appropriate for your network. +The docker compose file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured +by the environment variable(s). In the following example, additional ``target`` stanzas are added for the main ``sc4s`` container, where the +``target`` and ``published`` lines provide for 21 additional technology-specific ports. Follow these steps to configure unique ports: -* Modify the unit file ``/opt/sc4s/docker-compose.yml`` +* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). See the "Sources" +section for more information on your specific device(s). +* Modify the unit file ``/opt/sc4s/docker-compose.yml`` and add/change port stanzas as appropriate using the example below: ```yaml version: "3.7" services: @@ -186,6 +171,11 @@ services: protocol: udp #Comment the following line out if using docker-compose mode: host + - target: 6514 + published: 6514 + protocol: tcp +# Comment the following line out if using docker-compose + mode: host - target: 5000-5020 published: 5000-5020 protocol: tcp @@ -205,34 +195,44 @@ services: # - /opt/sc4s/tls:/opt/syslog-ng/tls ``` -* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). +## Modify index destinations for Splunk -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. +* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further +information is covered in the "Log Path overrides" section of the Configuration document. -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +## Configure source filtering by source IP or host name -```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088 -SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 -#Uncomment the following line if using untrusted SSL certificates -#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no -``` +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. + +* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. + +## Configure compliance index/metadata overrides -* Restart SC4S (below) +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is +covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. -## Start/Restart SC4S +# Scale out + +Additional hosts can be deployed for syslog collection from additional network zones and locations. + +# Start/Restart SC4S ```bash docker stack deploy --compose-file docker-compose.yml sc4s ``` - # Stop SC4S Start by obtaining the stack name (ID): @@ -272,7 +272,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 7eae5b0..27834bf 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -21,7 +21,7 @@ reboot: net.ipv4.ip_forward=1 ``` -# Setup +# Initial Setup * Create the systemd unit file `/lib/systemd/system/sc4s.service` based on the following template: @@ -54,7 +54,7 @@ ExecStartPre=/usr/bin/docker run \ "$SC4S_LOCAL_CONFIG_MOUNT" \ --name SC4S_preflight --rm \ $SC4S_IMAGE -s -ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp \ +ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ @@ -95,7 +95,7 @@ document for details on the directory structure the archive uses. * IMPORTANT: When creating the directories above, ensure the directories created match the volume mounts specified in the unit file above. Failure to do this will cause SC4S to abort at startup. -## Configure the SC4S environment +# Configure the SC4S environment Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: @@ -115,55 +115,35 @@ match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to uncomment the last line in the example. -## Modify index destinations for Splunk - -Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. - -* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your -environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in -this table that pertain to the individual data source filters that are included with SC4S. -* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further -information is covered in the "Log Path overrides" section of the Configuration document. +## Configure SC4S Listening Ports -## Configure source filtering by source IP or host name - -Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. - -* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. -* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. -* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. +Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. +The unit file and standard SC4S configurations reflect these defaults. If it desired to change some or all of them, container port mapping +can be used to change the defaults without altering the underlying SC4S configuration. To do this, simply change the initial port in the +`ExecStart` line for the main container (which represents the actual listening port on the host machine), like so: -## Configure compliance index/metadata overrides - -In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. -The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to -the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file -lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is -covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. - -## Configure SC4S for systemd and start SC4S - -```bash -sudo systemctl daemon-reload -sudo systemctl enable sc4s -sudo systemctl start sc4s ``` +-p 614:514 -p 714:514/udp -p 8514:6514 +``` +This instructs the _host_ to listen on TCP port 614, UDP 714, and TCP 8514 (for TLS) and map them to the standard UDP/TCP 514 and 6514 ports +on the _container_. No changes to the underlying SC4S default configuration (environment variables) are needed. -# Configure Dedicated Listening Ports +### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. For collection of such sources we provide a means of dedicating a unique listening port to a specific source. -Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. +Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology +in use. -In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Modify individual ports or a -range as appropriate for your network. +The unit file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured by the +environment variable(s). In the following example, the `ExecStart` line for the main SC4S container is modified, where +``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Follow these steps to configure unique ports: -* Modify the unit file ``/lib/systemd/system/sc4s.service`` +* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). See the "Sources" +section for more information on your specific device(s). +* Modify the unit file ``/lib/systemd/system/sc4s.service`` with the appropriate ``ExecStart`` command line changes using the example below: ```ini [Unit] Description=SC4S Container @@ -190,7 +170,7 @@ ExecStartPre=/usr/bin/docker run \ "$SC4S_LOCAL_CONFIG_MOUNT" \ --name SC4S_preflight --rm \ $SC4S_IMAGE -s -ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ +ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ @@ -199,27 +179,42 @@ ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 - $SC4S_IMAGE ``` -* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). +## Modify index destinations for Splunk -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. +* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further +information is covered in the "Log Path overrides" section of the Configuration document. -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +## Configure source filtering by source IP or host name -```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088 -SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 -#Uncomment the following line if using untrusted SSL certificates -#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no -``` +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. + +* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. + +## Configure compliance index/metadata overrides -* Restart SC4S (below) +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is +covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. + +## Configure SC4S for systemd and start SC4S + +```bash +sudo systemctl daemon-reload +sudo systemctl enable sc4s +sudo systemctl start sc4s +``` # Start SC4S diff --git a/docs/gettingstarted/index.md b/docs/gettingstarted/index.md index f686dc0..b8258f6 100644 --- a/docs/gettingstarted/index.md +++ b/docs/gettingstarted/index.md @@ -76,11 +76,11 @@ Splunk type. | Container and Orchestration | Notes | |-----------------------------|-------| -| [Podman + systemd single node](gettingstarted/podman-systemd-general.md) | First choice for RedHat 7.x/8.x and CentOS, second choice for Debian and Ubuntu (packages provided via PPA) | -| [Docker CE + systemd single node](gettingstarted/docker-systemd-general.md) | First choice for Debian and Ubuntu; second choice for CentOS for those with limited existing Docker experience | -| [Docker CE + Swarm single node](gettingstarted/docker-swarm-general.md) | Option for Debian, Ubuntu, CentOS, and Desktop Docker desiring Docker Compose or Swarm orchestration | -| [Docker CE + Swarm single node RHEL 7.7](gettingstarted/docker-swarm-rhel7.md) | Option for RedHat 7.7 desiring Docker Compose or Swarm orchestration | -| [Bring your own Envionment](gettingstarted/byoe-rhel7.md) | Option for RedHat 7.7 (centos 7) with SC4S configuration without containers | +| [Podman + systemd single node](gettingstarted/podman-systemd-general) | First choice for RedHat 7.x/8.x and CentOS, second choice for Debian and Ubuntu (packages provided via PPA) | +| [Docker CE + systemd single node](gettingstarted/docker-systemd-general) | First choice for Debian and Ubuntu; second choice for CentOS for those with limited existing Docker experience | +| [Docker CE + Swarm single node](gettingstarted/docker-swarm-general) | Option for Debian, Ubuntu, CentOS, and Desktop Docker desiring Docker Compose or Swarm orchestration | +| [Docker CE + Swarm single node RHEL 7.7](gettingstarted/docker-swarm-rhel7) | Option for RedHat 7.7 desiring Docker Compose or Swarm orchestration | +| [Bring your own Envionment](gettingstarted/byoe-rhel7) | Option for RedHat 7.7 (centos 7) with SC4S configuration without containers | ### Offline Container Installation diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 18f8f82..2ecc610 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -3,7 +3,7 @@ Refer to [Installation](https://podman.io/getting-started/installation) -# Setup +# Initial Setup * Create the systemd unit file `/lib/systemd/system/sc4s.service` based on the following template: @@ -36,7 +36,7 @@ ExecStartPre=/usr/bin/podman run \ "$SC4S_LOCAL_CONFIG_MOUNT" \ --name SC4S_preflight --rm \ $SC4S_IMAGE -s -ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp \ +ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ @@ -77,7 +77,7 @@ document for details on the directory structure the archive uses. * IMPORTANT: When creating the directories above, ensure the directories created match the volume mounts specified in the unit file above. Failure to do this will cause SC4S to abort at startup. -## Configure the sc4s environment +# Configure the sc4s environment Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: @@ -97,55 +97,35 @@ match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to uncomment the last line in the example. -## Modify index destinations for Splunk - -Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. - -* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your -environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in -this table that pertain to the individual data source filters that are included with SC4S. -* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further -information is covered in the "Log Path overrides" section of the Configuration document. +## Configure SC4S Listening Ports -## Configure source filtering by source IP or host name - -Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps -apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. - -* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. -* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. -* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. -* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. +Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. +The unit file and standard SC4S configurations reflect these defaults. If it desired to change some or all of them, container port mapping +can be used to change the defaults without altering the underlying SC4S configuration. To do this, simply change the initial port in the +`ExecStart` line for the main container (which represents the actual listening port on the host machine), like so: -## Configure compliance index/metadata overrides - -In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. -The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to -the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file -lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is -covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. - -## Configure SC4S for systemd and start SC4S - -```bash -sudo systemctl daemon-reload -sudo systemctl enable sc4s -sudo systemctl start sc4s ``` +-p 614:514 -p 714:514/udp -p 8514:6514 +``` +This instructs the _host_ to listen on TCP port 614, UDP 714, and TCP 8514 (for TLS) and map them to the standard UDP/TCP 514 and 6514 ports +on the _container_. No changes to the underlying SC4S default configuration (environment variables) are needed. -# Configure Dedicated Listening Ports +### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. For collection of such sources we provide a means of dedicating a unique listening port to a specific source. -Refer to the "Sources" documentation to identify the specific variable used to enable a specific port for the technology in use. +Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology +in use. -In the following example ``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Modify individual ports or a -range as appropriate for your network. +The unit file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured by the +environment variable(s). In the following example, the `ExecStart` line for the main SC4S container is modified, where +``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Follow these steps to configure unique ports: -* Modify the unit file ``/lib/systemd/system/sc4s.service`` +* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). See the "Sources" +section for more information on your specific device(s). +* Modify the unit file ``/lib/systemd/system/sc4s.service`` with the appropriate ``ExecStart`` command line changes using the example below: ```ini [Unit] Description=SC4S Container @@ -172,7 +152,7 @@ ExecStartPre=/usr/bin/podman run \ "$SC4S_LOCAL_CONFIG_MOUNT" \ --name SC4S_preflight --rm \ $SC4S_IMAGE -s -ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ +ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ @@ -181,27 +161,42 @@ ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 5000-5020:5000-5020 - $SC4S_IMAGE ``` -* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). +## Modify index destinations for Splunk -* Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment +Log paths are preconfigured to utilize a convention of index destinations that are suitable for most customers. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* If changes need to be made to index destinations, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Edit `splunk_index.csv` to review or change the index configuration and revise as required for the data sources utilized in your +environment. Simply uncomment the relevant line and enter the desired index. The "Sources" document details the specific entries in +this table that pertain to the individual data source filters that are included with SC4S. +* Other Splunk metadata (e.g. source and sourcetype) can be overriden via this file as well. This is an advanced topic, and further +information is covered in the "Log Path overrides" section of the Configuration document. -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +## Configure source filtering by source IP or host name -```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088 -SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 -SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 -#Uncomment the following line if using untrusted SSL certificates -#SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no -``` +Legacy sources and non-standard-compliant sources require configuration by source IP or hostname as included in the event. The following steps +apply to support such sources. To identify sources that require this step, refer to the "sources" section of this documentation. -* Restart SC4S (below) +* If changes need to be made to source filtering, navigate to the ``/opt/sc4s/local/context`` directory to start. +* Navigate to `vendor_product_by_source.conf` and find the appropriate filter that matches your legacy device type. +* Edit the file to properly identify these products by hostname glob or network mask using syslog-ng filter syntax. Configuration by hostname or source IP is needed only for those devices that cannot be determined via normal syslog-ng parsing or message contents. +* The `vendor_product_by_source.csv` file should not need to be changed unless a local filter is created that is specific to the environment. In this case, a matching filter will also need to be provided in `vendor_product_by_source.conf`. + +## Configure compliance index/metadata overrides + +In some cases, devices that have been properly sourcetyped need to be further categorized by compliance, geography, or other criterion. +The two files `compliance_meta_by_source.conf` and `compliance_meta_by_source.csv` can be used for this purpose. These operate similarly to +the files above, where the `conf` file specifies a filter to uniquely identify the messages that should be overridden, and the `csv` file +lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is +covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. + +## Configure SC4S for systemd and start SC4S + +```bash +sudo systemctl daemon-reload +sudo systemctl enable sc4s +sudo systemctl start sc4s +``` # Start SC4S From f52c5aa45d61715d4f21634b65d6b68d5d986282 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sat, 11 Jan 2020 17:23:26 -0800 Subject: [PATCH 06/14] Runtime docs cleanup * Clean up `docker-compose.yml` and systemd `unit` files to include archive options * Clean up compose/unit file command layout * Clarify instructions for unique port setup --- docs/gettingstarted/docker-swarm-general.md | 29 +++++---- docs/gettingstarted/docker-swarm-rhel7.md | 31 +++++---- docs/gettingstarted/docker-systemd-general.md | 64 +++++++++++-------- docs/gettingstarted/podman-systemd-general.md | 58 ++++++++++------- 4 files changed, 104 insertions(+), 78 deletions(-) diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 7f72222..672f1d8 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -95,7 +95,8 @@ document for details on the directory structure the archive uses. # Configure the SC4S environment -Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: +SC4S is almost entirely controlled through environment variables, which are read from a file at starteup. Create a file named +``/opt/sc4s/env_file`` and add the following environment variables and values: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 @@ -107,8 +108,8 @@ SC4S_DEST_SPLUNK_HEC_WORKERS=6 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints, up to a maxiumum of 32. +If the endpoint is a VIP, match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to uncomment the last line in the example above. @@ -135,18 +136,18 @@ No changes to the underlying SC4S default configuration (environment variables) For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. -For collection of such sources we provide a means of dedicating a unique listening port to a specific source. - -Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology -in use. +For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. The docker compose file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured by the environment variable(s). In the following example, additional ``target`` stanzas are added for the main ``sc4s`` container, where the -``target`` and ``published`` lines provide for 21 additional technology-specific ports. Follow these steps to configure unique ports: +``target`` and ``published`` lines provide for 21 additional technology-specific UDP and TCP ports. + +Follow these steps to configure unique ports: -* Modify the following file ``/opt/sc4s/env_file`` to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). -* Modify the unit file ``/opt/sc4s/docker-compose.yml`` and add/change port stanzas as appropriate using the example below: +* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). Refer to the "Sources" +documentation to identify the specific environment variables that are mapped to each data source vendor/technology. +* Modify the compose file ``/opt/sc4s/docker-compose.yml`` and add/change port stanzas as appropriate using the example below. +* Restart SC4S using the command in the "Start/Restart SC4S" section below. ```yaml version: "3.7" services: @@ -183,8 +184,10 @@ services: volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z -#Uncomment the following line if custom TLS certs are provided -# - /opt/sc4s/tls:/opt/syslog-ng/tls +# Uncomment the following line if local disk archiving is desired +# - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z +# Uncomment the following line if custom TLS certs are provided +# - /opt/sc4s/tls:/opt/syslog-ng/tls:z ``` ## Modify index destinations for Splunk diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index 09f8179..2fbe187 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -103,7 +103,8 @@ document for details on the directory structure the archive uses. # Configure the SC4S environment -Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: +SC4S is almost entirely controlled through environment variables, which are read from a file at starteup. Create a file named +``/opt/sc4s/env_file`` and add the following environment variables and values: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 @@ -115,11 +116,11 @@ SC4S_DEST_SPLUNK_HEC_WORKERS=6 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment. -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints, up to a maxiumum of 32. +If the endpoint is a VIP, match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example below. +uncomment the last line in the example above. ## Configure SC4S Listening Ports @@ -143,18 +144,18 @@ No changes to the underlying SC4S default configuration (environment variables) For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. -For collection of such sources we provide a means of dedicating a unique listening port to a specific source. - -Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology -in use. +For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. The docker compose file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured by the environment variable(s). In the following example, additional ``target`` stanzas are added for the main ``sc4s`` container, where the -``target`` and ``published`` lines provide for 21 additional technology-specific ports. Follow these steps to configure unique ports: +``target`` and ``published`` lines provide for 21 additional technology-specific UDP and TCP ports. + +Follow these steps to configure unique ports: -* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). -* Modify the unit file ``/opt/sc4s/docker-compose.yml`` and add/change port stanzas as appropriate using the example below: +* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). Refer to the "Sources" +documentation to identify the specific environment variables that are mapped to each data source vendor/technology. +* Modify the compose file ``/opt/sc4s/docker-compose.yml`` and add/change port stanzas as appropriate using the example below. +* Restart SC4S using the command in the "Start/Restart SC4S" section below. ```yaml version: "3.7" services: @@ -191,8 +192,10 @@ services: volumes: - /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z - /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z -#Uncomment the following line if custom TLS certs are provided -# - /opt/sc4s/tls:/opt/syslog-ng/tls +# Uncomment the following line if local disk archiving is desired +# - /opt/sc4s/archive:/opt/syslog-ng/var/archive:z +# Uncomment the following line if custom TLS certs are provided +# - /opt/sc4s/tls:/opt/syslog-ng/tls:z ``` ## Modify index destinations for Splunk diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 27834bf..03f3d8f 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -38,10 +38,13 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" +# Optional mount point for local disk archive (EWMM output) files + +# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Mount point for local disk buffer (required) Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" -# Uncomment the following line if local disk archiving is desired -# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls:z" @@ -52,16 +55,16 @@ ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE ExecStartPre=/usr/bin/docker run \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - --name SC4S_preflight --rm \ - $SC4S_IMAGE -s -ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ + --name SC4S_preflight \ + --rm $SC4S_IMAGE -s +ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ "$SC4S_LOCAL_ARCHIVE_MOUNT" \ "$SC4S_TLS_DIR" \ - --name SC4S --rm \ -$SC4S_IMAGE + --name SC4S \ + --rm $SC4S_IMAGE ``` * Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. @@ -95,9 +98,10 @@ document for details on the directory structure the archive uses. * IMPORTANT: When creating the directories above, ensure the directories created match the volume mounts specified in the unit file above. Failure to do this will cause SC4S to abort at startup. -# Configure the SC4S environment +# Configure the sc4s environment -Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: +SC4S is almost entirely controlled through environment variables, which are read from a file at starteup. Create a file named +``/opt/sc4s/env_file`` and add the following environment variables and values: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 @@ -109,18 +113,18 @@ SC4S_DEST_SPLUNK_HEC_WORKERS=6 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints, up to a maxiumum of 32. +If the endpoint is a VIP, match this value to the total number of indexers behind the load balancer. -* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example. +* NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to +uncomment the last line in the example above. ## Configure SC4S Listening Ports Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. The unit file and standard SC4S configurations reflect these defaults. If it desired to change some or all of them, container port mapping can be used to change the defaults without altering the underlying SC4S configuration. To do this, simply change the initial port in the -`ExecStart` line for the main container (which represents the actual listening port on the host machine), like so: +`ExecStart` line in the unit file for the main container (which represents the actual listening port on the host machine), like so: ``` -p 614:514 -p 714:514/udp -p 8514:6514 @@ -132,18 +136,18 @@ on the _container_. No changes to the underlying SC4S default configuration (en For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. -For collection of such sources we provide a means of dedicating a unique listening port to a specific source. - -Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology -in use. +For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. The unit file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured by the -environment variable(s). In the following example, the `ExecStart` line for the main SC4S container is modified, where -``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Follow these steps to configure unique ports: +environment variable(s). In the example below, the `ExecStart` line for the main SC4S container is modified, where +``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. -* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). -* Modify the unit file ``/lib/systemd/system/sc4s.service`` with the appropriate ``ExecStart`` command line changes using the example below: +Follow these steps to configure unique ports: + +* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). Refer to the "Sources" +documentation to identify the specific environment variables that are mapped to each data source vendor/technology. +* Modify the unit file ``/lib/systemd/system/sc4s.service`` with the appropriate ``ExecStart`` command line changes using the example below. +* Ensure that you reload the unit file as well as restarting SC4S. See the "Configure SC4S for systemd and start SC4S" section below. ```ini [Unit] Description=SC4S Container @@ -157,8 +161,13 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" +# Optional mount point for local disk archive (EWMM output) files + +# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Mount point for local disk buffer (required) Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" + # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -168,15 +177,16 @@ ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE ExecStartPre=/usr/bin/docker run \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - --name SC4S_preflight --rm \ - $SC4S_IMAGE -s + --name SC4S_preflight \ + --rm $SC4S_IMAGE -s ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ + "$SC4S_LOCAL_ARCHIVE_MOUNT" \ + "$SC4S_TLS_DIR" \ --name SC4S \ - --rm \ -$SC4S_IMAGE + --rm $SC4S_IMAGE ``` ## Modify index destinations for Splunk diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 2ecc610..51356a8 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -20,10 +20,13 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" +# Optional mount point for local disk archive (EWMM output) files + +# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Mount point for local disk buffer (required) Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" -# Uncomment the following line if local disk archiving is desired -# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls:z" @@ -34,16 +37,16 @@ ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE ExecStartPre=/usr/bin/podman run \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - --name SC4S_preflight --rm \ - $SC4S_IMAGE -s + --name SC4S_preflight \ + --rm $SC4S_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ "$SC4S_LOCAL_ARCHIVE_MOUNT" \ "$SC4S_TLS_DIR" \ - --name SC4S --rm \ -$SC4S_IMAGE + --name SC4S \ + --rm $SC4S_IMAGE ``` * Create the subdirectory ``/opt/sc4s/local``. This will be used as a mount point for local overrides and configurations. @@ -79,7 +82,8 @@ unit file above. Failure to do this will cause SC4S to abort at startup. # Configure the sc4s environment -Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: +SC4S is almost entirely controlled through environment variables, which are read from a file at starteup. Create a file named +``/opt/sc4s/env_file`` and add the following environment variables and values: ```dotenv SPLUNK_HEC_URL=https://splunk.smg.aws:8088 @@ -91,18 +95,18 @@ SC4S_DEST_SPLUNK_HEC_WORKERS=6 * Update ``SPLUNK_HEC_URL`` and ``SPLUNK_HEC_TOKEN`` to reflect the correct values for your environment -* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints. If the endpoint is a VIP, -match this value to the total number of indexers behind the load balancer. +* Set `SC4S_DEST_SPLUNK_HEC_WORKERS` to match the number of indexers and/or HWFs with HEC endpoints, up to a maxiumum of 32. +If the endpoint is a VIP, match this value to the total number of indexers behind the load balancer. * NOTE: Splunk Connect for Syslog defaults to secure configurations. If you are not using trusted SSL certificates, be sure to -uncomment the last line in the example. +uncomment the last line in the example above. ## Configure SC4S Listening Ports Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. The unit file and standard SC4S configurations reflect these defaults. If it desired to change some or all of them, container port mapping can be used to change the defaults without altering the underlying SC4S configuration. To do this, simply change the initial port in the -`ExecStart` line for the main container (which represents the actual listening port on the host machine), like so: +`ExecStart` line in the unit file for the main container (which represents the actual listening port on the host machine), like so: ``` -p 614:514 -p 714:514/udp -p 8514:6514 @@ -114,18 +118,18 @@ on the _container_. No changes to the underlying SC4S default configuration (en For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. -For collection of such sources we provide a means of dedicating a unique listening port to a specific source. - -Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology -in use. +For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. The unit file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured by the -environment variable(s). In the following example, the `ExecStart` line for the main SC4S container is modified, where -``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. Follow these steps to configure unique ports: +environment variable(s). In the example below, the `ExecStart` line for the main SC4S container is modified, where +``-p 5000-5020:5000-5020`` allows for up to 21 technology-specific ports. -* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). See the "Sources" -section for more information on your specific device(s). -* Modify the unit file ``/lib/systemd/system/sc4s.service`` with the appropriate ``ExecStart`` command line changes using the example below: +Follow these steps to configure unique ports: + +* Modify the ``/opt/sc4s/env_file`` file to include the port-specific environment variable(s). Refer to the "Sources" +documentation to identify the specific environment variables that are mapped to each data source vendor/technology. +* Modify the unit file ``/lib/systemd/system/sc4s.service`` with the appropriate ``ExecStart`` command line changes using the example below. +* Ensure that you reload the unit file as well as restarting SC4S. See the "Configure SC4S for systemd and start SC4S" section below. ```ini [Unit] Description=SC4S Container @@ -139,8 +143,13 @@ Environment="SC4S_IMAGE=splunk/scs:latest" Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" +# Optional mount point for local disk archive (EWMM output) files + +# Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" + # Mount point for local disk buffer (required) Environment="SC4S_LOCAL_DISK_BUFFER_MOUNT=-v /opt/sc4s/disk-buffer:/opt/syslog-ng/var/data/disk-buffer:z" + # Uncomment the following line if custom TLS certs are provided # Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/opt/syslog-ng/tls" @@ -150,15 +159,16 @@ ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE ExecStartPre=/usr/bin/podman run \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ - --name SC4S_preflight --rm \ - $SC4S_IMAGE -s + --name SC4S_preflight \ + --rm $SC4S_IMAGE -s ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 -p 5000-5020:5000-5020 -p 5000-5020:5000-5020/udp \ --env-file=/opt/sc4s/env_file \ "$SC4S_LOCAL_CONFIG_MOUNT" \ "$SC4S_LOCAL_DISK_BUFFER_MOUNT" \ + "$SC4S_LOCAL_ARCHIVE_MOUNT" \ + "$SC4S_TLS_DIR" \ --name SC4S \ - --rm \ -$SC4S_IMAGE + --rm $SC4S_IMAGE ``` ## Modify index destinations for Splunk From 0db109e158f1bc3b5c9f56275ec299f4d990ee15 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 13 Jan 2020 09:37:13 -0800 Subject: [PATCH 07/14] Update HEALTHCHECK to 30s interval; remove python source * Remove python source command * Include start time set to 15s * Update interval to 30s --- package/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/Dockerfile b/package/Dockerfile index 77740bb..4e5771b 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -82,4 +82,4 @@ EXPOSE 6514/tcp ENTRYPOINT ["/entrypoint.sh", "-F"] -HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file +HEALTHCHECK --start-period=15s --interval=30s --timeout=6s CMD goss -g /etc/goss.yaml validate \ No newline at end of file From fb082331af588c201beee52ae0997c0256eb79c5 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 13 Jan 2020 10:41:29 -0800 Subject: [PATCH 08/14] Markdown and formatting update * Remove extra spaces at end of sentences; affects md rendering * Remove extra blank lines from Unit file examples for readability * Update all version strings to 3.25.1 * Remove extra "Scale Out" section from runtime docs (covered elsewhere) --- docs/gettingstarted/byoe-rhel7.md | 4 ++-- docs/gettingstarted/docker-swarm-general.md | 8 ++------ docs/gettingstarted/docker-swarm-rhel7.md | 8 ++------ docs/gettingstarted/docker-systemd-general.md | 10 +++------- docs/gettingstarted/podman-systemd-general.md | 10 +++------- 5 files changed, 12 insertions(+), 28 deletions(-) diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 437deb8..e91f5b1 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -160,7 +160,7 @@ sudo systemctl start sc4s ## Configure SC4S Listening Ports Most enterprises use UDP/TCP port 514 as the default as their main listening port for syslog "soup" traffic, and TCP port 6514 for TLS. -The docker compose file and standard SC4S configurations reflect these defaults. These defaults can be changed by adding the following +The standard SC4S configuration reflect these defaults. These defaults can be changed by adding the following additional environment variables with appropriate values to the ``env_file`` above: ```dotenv SC4S_LISTEN_DEFAULT_TCP_PORT=514 @@ -170,7 +170,7 @@ SC4S_LISTEN_DEFAULT_TLS_PORT=6514 ### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in -the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. For collection of such sources we provide a means of dedicating a unique listening port to a specific source. Refer to the "Sources" documentation to identify the specific environment variables used to enable unique listening ports for the technology diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 672f1d8..a26a1c1 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -135,7 +135,7 @@ No changes to the underlying SC4S default configuration (environment variables) ### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in -the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. The docker compose file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured @@ -219,10 +219,6 @@ the files above, where the `conf` file specifies a filter to uniquely identify t lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. -# Scale out - -Additional hosts can be deployed for syslog collection from additional network zones and locations. - # Start/Restart SC4S ```bash @@ -249,7 +245,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.22.1' +syslog-ng starting up; version='3.25.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index 2fbe187..67f9d07 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -143,7 +143,7 @@ No changes to the underlying SC4S default configuration (environment variables) ### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in -the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. The docker compose file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured @@ -227,10 +227,6 @@ the files above, where the `conf` file specifies a filter to uniquely identify t lists one or more metadata items that can be overridden based on the filter name. This is an advanced topic, and further information is covered in the "Override index or metadata based on host, ip, or subnet" section of the Configuration document. -# Scale out - -Additional hosts can be deployed for syslog collection from additional network zones and locations. - # Start/Restart SC4S ```bash @@ -257,7 +253,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.22.1' +syslog-ng starting up; version='3.25.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 03f3d8f..55a4788 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -35,11 +35,9 @@ After=network.target network-online.target Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs - Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Optional mount point for local disk archive (EWMM output) files - # Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" # Mount point for local disk buffer (required) @@ -135,7 +133,7 @@ on the _container_. No changes to the underlying SC4S default configuration (en ### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in -the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. The unit file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured by the @@ -158,11 +156,9 @@ Requires=network.service Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs - Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Optional mount point for local disk archive (EWMM output) files - # Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" # Mount point for local disk buffer (required) @@ -263,7 +259,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.22.1' +syslog-ng starting up; version='3.25.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -281,7 +277,7 @@ docker logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 51356a8..7b49762 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -17,11 +17,9 @@ After=network.target network-online.target Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs - Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Optional mount point for local disk archive (EWMM output) files - # Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" # Mount point for local disk buffer (required) @@ -117,7 +115,7 @@ on the _container_. No changes to the underlying SC4S default configuration (en ### Dedicated (Unique) Listening Ports For certain source technologies, categorization by message content is impossible due to the lack of a unique "fingerprint" in -the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. +the data. In other cases, a unique listening port is required for certain devices due to network requirements in the enterprise. For collection of such sources, we provide a means of dedicating a unique listening port to a specific source. The unit file used to start the SC4S container needs to be modified as well to reflect the additional listening ports configured by the @@ -140,11 +138,9 @@ Requires=network.service Environment="SC4S_IMAGE=splunk/scs:latest" # Optional mount point for local overrides and configurations; see notes in docs - Environment="SC4S_LOCAL_CONFIG_MOUNT=-v /opt/sc4s/local:/opt/syslog-ng/etc/conf.d/local:z" # Optional mount point for local disk archive (EWMM output) files - # Environment="SC4S_LOCAL_ARCHIVE_MOUNT=-v /opt/sc4s/archive:/opt/syslog-ng/var/archive:z" # Mount point for local disk buffer (required) @@ -245,7 +241,7 @@ index=* sourcetype=sc4s:events "starting up" ``` This should yield the following event: ```ini -syslog-ng starting up; version='3.22.1' +syslog-ng starting up; version='3.25.1' ``` when the startup process proceeds normally (without syntax errors). If you do not see this, follow the steps below before proceeding to deeper-level troubleshooting: @@ -263,7 +259,7 @@ podman logs SC4S ``` You should see events similar to those below in the output: ```ini -Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.24.1' +Oct 1 03:13:35 77cd4776af41 syslog-ng[1]: syslog-ng starting up; version='3.25.1' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection accepted; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' Oct 1 05:29:55 77cd4776af41 syslog-ng[1]: Syslog connection closed; fd='49', client='AF_INET(10.0.1.18:55010)', local='AF_INET(0.0.0.0:514)' ``` From 3f50a21ba3af8992ce6a70de682915d4c641f8e0 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 13 Jan 2020 12:25:13 -0800 Subject: [PATCH 09/14] Update local example to reflect latest log path changes * Update local example to reflect latest gomplate changes to log paths. --- .../local/config/log_paths/example.conf.tmpl | 125 +++++++++++------- .../local_config/log_paths/example.conf.tmpl | 117 +++++++++------- 2 files changed, 147 insertions(+), 95 deletions(-) diff --git a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl index 6eae6a3..516f954 100644 --- a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl +++ b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl @@ -1,76 +1,101 @@ # LOCAL_EXAMPLE - -# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique -# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the -# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ - -# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), -# the template generator will build a custom source based on the value of one or more of the set variables. - -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} - -# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name -# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". -# "parser" can be customized on dedicated ports only -# "common" uses the same parser sequence as the default ports and is the most commonly used - -{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} - -# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path -# The ports used are based on the values of one or more of the environment variables set above. - -{{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} -{{ define "log_path" }} +# DO NOT MODIFY THIS EXAMPLE DIRECTLY! It will get overwritten with the shipping example +# version each time SC4S starts. Copy this file to another name for development work. + +{{- /* To start, gomplate comments use the C++ style comment syntax you see here, enclosed by */}} +{{- /* curly braces. They will _not_ appear in the final syslog-ng config files. */}} +{{- /* Comments using this format will be specific to the templating process */}} + +# This comment, on the other hand, _will_ appear in the final syslog-ng config. +# Comments using this style will be relevant to the actual syslog-ng config files, +# independent of the templating process. + +{{- /* When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout */}} +{{- /* this file with a unique string to identify the vendor product. The string should be */}} +{{- /* of the form "VENDOR_PRODUCT" to signify the manufacturer and product type, and must */}} +{{- /* contain only characters matching this regex: [A-Z\_]+ */}} + +{{- /* If any of the "LOCAL_EXAMPLE" variables passed into the environment are set */}} +{{- /* (e.g. TLS, UDP, or TLS), the template generator will build a custom source based */}} +{{- /* on the value of one or more of the set variables. */}} + +{{- /* "port_id" is used to generate the port variable to be used. It should match the */}} +{{- /* "core" of the variable name set in the line above. For example, the "port_id" of */}} +{{- /* "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". "parser" can be customized */}} +{{- /* on dedicated ports only. "common" uses the same parser sequence as the default ports */}} +{{- /* and is the most commonly used */}} + +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} log { -# The first time this template is used the log_path will be linked to the default port +{{- /* The first time this template is used the log_path will be linked to the default port */}} {{- if eq (.) "yes"}} source(s_DEFAULT); - -# Filters should be updated to use the simplest and most effecient logic possible to discard -# the message from this path - filter(f_is_rfc3164); filter(f_local_example); {{- end}} -{{- if eq (.) "no"}} -# In the second pass through the template a link to the dedicated port is used. This -# normally does not require additional filters +{{- /* In the second pass through the template a link to the dedicated port is used. This */}} +{{- /* normally does not require additional filters */}} -source (s_LOCAL_EXAMPLE); +{{- if eq (.) "no"}} + source (s_LOCAL_EXAMPLE); {{- end}} -#Set a default sourcetype and index - - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"))}; - -#using the key "local_example" find any cutomized index,source or sourcetype meta values - - parser {p_add_context_splunk(key("local_example")); }; +# Set a default sourcetype and index, as well as an appropriate value for the field +# "sc4s_vendor_product". This field is sent as an indexed field to Splunk, +# and is useful for downstream analysis. -# Any additional logic needed to process the event before sending to Splunk goes here + rewrite { + set("local_example", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main")); + }; -# Send it to Splunk +# using the key "local_example" find any customized index,source or sourcetype meta values + parser { p_add_context_splunk(key("local_example")); }; +# using any user-supplied filters, override Splunk metadata based on further hostname +# or CIDR block filters. + parser (compliance_meta_by_source); + +# Prepare the payload for sending to Splunk. This step is done here rather than in the +# destination(s) to ensure that it is performed only once. If the template value is not overridden, +# the default value (2nd argument) is used. + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; + +{{- /* Check environment variables (and defaults if unset) for sending to the HEC */}} +{{- /* destination. When more destination options are offered in SC4S, this is where */}} +{{- /* output to them will be configured */}} + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_LOCAL_EXAMPLE_HEC" "no")) }} destination(d_hec); +{{- end}} -# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will -# forward events to another system +{{- /* Check environment variables (and defaults if unset) for sending to the local EWMM-format */}} +{{- /* disk archive */}} - flags(flow-control); +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_LOCAL_EXAMPLE" "no")) }} + destination(d_archive); +{{- end}} +# All passes through any matching log path will be final + flags(flow-control,final); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} -# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic +{{- /* Prepare to run two passes through this template, one for default traffic and another for */}} +{{- /* "unique ports" if they are configured. */}} - {{tmpl.Exec "log_path" "no" }} -{{- end}} +{{- if or (or (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT")) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic +{{ tmpl.Exec "log_path" "no" }} +{{- end }} # Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic - -{{tmpl.Exec "log_path" "yes" }} +{{ tmpl.Exec "log_path" "yes" }} \ No newline at end of file diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl index 6f75c8e..516f954 100644 --- a/package/etc/local_config/log_paths/example.conf.tmpl +++ b/package/etc/local_config/log_paths/example.conf.tmpl @@ -1,74 +1,101 @@ # LOCAL_EXAMPLE +# DO NOT MODIFY THIS EXAMPLE DIRECTLY! It will get overwritten with the shipping example +# version each time SC4S starts. Copy this file to another name for development work. + +{{- /* To start, gomplate comments use the C++ style comment syntax you see here, enclosed by */}} +{{- /* curly braces. They will _not_ appear in the final syslog-ng config files. */}} +{{- /* Comments using this format will be specific to the templating process */}} + +# This comment, on the other hand, _will_ appear in the final syslog-ng config. +# Comments using this style will be relevant to the actual syslog-ng config files, +# independent of the templating process. + +{{- /* When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout */}} +{{- /* this file with a unique string to identify the vendor product. The string should be */}} +{{- /* of the form "VENDOR_PRODUCT" to signify the manufacturer and product type, and must */}} +{{- /* contain only characters matching this regex: [A-Z\_]+ */}} + +{{- /* If any of the "LOCAL_EXAMPLE" variables passed into the environment are set */}} +{{- /* (e.g. TLS, UDP, or TLS), the template generator will build a custom source based */}} +{{- /* on the value of one or more of the set variables. */}} + +{{- /* "port_id" is used to generate the port variable to be used. It should match the */}} +{{- /* "core" of the variable name set in the line above. For example, the "port_id" of */}} +{{- /* "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". "parser" can be customized */}} +{{- /* on dedicated ports only. "common" uses the same parser sequence as the default ports */}} +{{- /* and is the most commonly used */}} + +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +{{- /* The following is an inline template to generate the actual log path */}} +{{- define "log_path"}} +log { -# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique -# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the -# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ - -# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), -# the template generator will build a custom source based on the value of one or more of the set variables. - - -# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name -# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". -# "parser" can be customized on dedicated ports only -# "common" uses the same parser sequence as the default ports and is the most commonly used - - -# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path -# The ports used are based on the values of one or more of the environment variables set above. +{{- /* The first time this template is used the log_path will be linked to the default port */}} -{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} -log { {{- if eq (.) "yes"}} source(s_DEFAULT); filter(f_is_rfc3164); filter(f_local_example); {{- end}} + +{{- /* In the second pass through the template a link to the dedicated port is used. This */}} +{{- /* normally does not require additional filters */}} + {{- if eq (.) "no"}} source (s_LOCAL_EXAMPLE); {{- end}} +# Set a default sourcetype and index, as well as an appropriate value for the field +# "sc4s_vendor_product". This field is sent as an indexed field to Splunk, +# and is useful for downstream analysis. -# The first time this template is used the log_path will be linked to the default port + rewrite { + set("local_example", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main")); + }; -# Filters should be updated to use the simplest and most effecient logic possible to discard -# the message from this path +# using the key "local_example" find any customized index,source or sourcetype meta values + parser { p_add_context_splunk(key("local_example")); }; -# In the second pass through the template a link to the dedicated port is used. This -# normally does not require additional filters +# using any user-supplied filters, override Splunk metadata based on further hostname +# or CIDR block filters. + parser (compliance_meta_by_source); +# Prepare the payload for sending to Splunk. This step is done here rather than in the +# destination(s) to ensure that it is performed only once. If the template value is not overridden, +# the default value (2nd argument) is used. + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); }; -#Set a default sourcetype and index - - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"))}; - -#using the key "local_example" find any cutomized index,source or sourcetype meta values - - parser {p_add_context_splunk(key("local_example")); }; - -# Any additional logic needed to process the event before sending to Splunk goes here - -# Send it to Splunk +{{- /* Check environment variables (and defaults if unset) for sending to the HEC */}} +{{- /* destination. When more destination options are offered in SC4S, this is where */}} +{{- /* output to them will be configured */}} +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_LOCAL_EXAMPLE_HEC" "no")) }} destination(d_hec); +{{- end}} -# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will -# forward events to another system +{{- /* Check environment variables (and defaults if unset) for sending to the local EWMM-format */}} +{{- /* disk archive */}} - flags(flow-control); +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_LOCAL_EXAMPLE" "no")) }} + destination(d_archive); +{{- end}} +# All passes through any matching log path will be final + flags(flow-control,final); }; {{- end}} +{{- /* Prepare to run two passes through this template, one for default traffic and another for */}} +{{- /* "unique ports" if they are configured. */}} + {{- if or (or (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT")) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT")) }} # Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic - -{{tmpl.Exec "log_path" "no" }} -{{- end}} +{{ tmpl.Exec "log_path" "no" }} +{{- end }} # Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic - -{{tmpl.Exec "log_path" "yes" }} +{{ tmpl.Exec "log_path" "yes" }} \ No newline at end of file From b2bb5c423a2779fd3158fc8d0ccce75e47588692 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 13 Jan 2020 14:46:08 -0800 Subject: [PATCH 10/14] Fix documentation navigation * Fix table navigation * Remove "single node" nomenclature; all are "single node" --- docs/gettingstarted/index.md | 10 +++++----- mkdocs.yml | 7 ++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/gettingstarted/index.md b/docs/gettingstarted/index.md index b8258f6..8923b79 100644 --- a/docs/gettingstarted/index.md +++ b/docs/gettingstarted/index.md @@ -76,11 +76,11 @@ Splunk type. | Container and Orchestration | Notes | |-----------------------------|-------| -| [Podman + systemd single node](gettingstarted/podman-systemd-general) | First choice for RedHat 7.x/8.x and CentOS, second choice for Debian and Ubuntu (packages provided via PPA) | -| [Docker CE + systemd single node](gettingstarted/docker-systemd-general) | First choice for Debian and Ubuntu; second choice for CentOS for those with limited existing Docker experience | -| [Docker CE + Swarm single node](gettingstarted/docker-swarm-general) | Option for Debian, Ubuntu, CentOS, and Desktop Docker desiring Docker Compose or Swarm orchestration | -| [Docker CE + Swarm single node RHEL 7.7](gettingstarted/docker-swarm-rhel7) | Option for RedHat 7.7 desiring Docker Compose or Swarm orchestration | -| [Bring your own Envionment](gettingstarted/byoe-rhel7) | Option for RedHat 7.7 (centos 7) with SC4S configuration without containers | +| [Podman + systemd](podman-systemd-general.md) | First choice for RedHat 7.x/8.x and CentOS, second choice for Debian and Ubuntu (packages provided via PPA) | +| [Docker CE + systemd](docker-systemd-general.md) | First choice for Debian and Ubuntu; second choice for CentOS for those with limited existing Docker experience | +| [Docker CE + Swarm](docker-swarm-general.md) | Option for Debian, Ubuntu, CentOS, and Desktop Docker desiring Docker Compose or Swarm orchestration | +| [Docker CE + Swarm RHEL 7.7](docker-swarm-rhel7.md) | Option for RedHat 7.7 desiring Docker Compose or Swarm orchestration | +| [Bring your own Envionment](byoe-rhel7.md) | Option for RedHat 7.7 (centos 7) with SC4S configuration without containers | ### Offline Container Installation diff --git a/mkdocs.yml b/mkdocs.yml index 3848849..4032eae 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,9 +4,10 @@ nav: - Home: 'index.md' - Getting Started: - 'Read First': 'gettingstarted/index.md' - - 'Podman + systemd single node': 'gettingstarted/podman-systemd-general.md' - - 'Docker CE + systemd single node': 'gettingstarted/docker-systemd-general.md' - - 'Docker CE + Swarm single node': 'gettingstarted/docker-swarm-rhel7.md' + - 'Podman + systemd': 'gettingstarted/podman-systemd-general.md' + - 'Docker CE + systemd': 'gettingstarted/docker-systemd-general.md' + - 'Docker CE + Swarm RHEL 7.7': 'gettingstarted/docker-swarm-rhel7.md' + - 'Docker CE + Swarm general': 'gettingstarted/docker-swarm-general.md' - 'Bring your own Envionment': 'gettingstarted/byoe-rhel7.md' - Configuration: 'configuration.md' - Sources: From dff3e3ce9960dbffd697a910cf5f2cfa2e89b931 Mon Sep 17 00:00:00 2001 From: mbonsack Date: Mon, 13 Jan 2020 19:07:05 -0800 Subject: [PATCH 11/14] Update mkdocs.yml * Remove "general" for Docker CE + Swarm menu pick --- mkdocs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 4032eae..097b634 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,7 +7,7 @@ nav: - 'Podman + systemd': 'gettingstarted/podman-systemd-general.md' - 'Docker CE + systemd': 'gettingstarted/docker-systemd-general.md' - 'Docker CE + Swarm RHEL 7.7': 'gettingstarted/docker-swarm-rhel7.md' - - 'Docker CE + Swarm general': 'gettingstarted/docker-swarm-general.md' + - 'Docker CE + Swarm': 'gettingstarted/docker-swarm-general.md' - 'Bring your own Envionment': 'gettingstarted/byoe-rhel7.md' - Configuration: 'configuration.md' - Sources: @@ -43,4 +43,4 @@ theme: primary: 'black' accent: 'orange' favicon: 'logo.png' - logo: 'logo.png' \ No newline at end of file + logo: 'logo.png' From b3f3b612b773f53214fbe758885be528c27221e8 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Tue, 14 Jan 2020 08:17:49 -0800 Subject: [PATCH 12/14] Fix nav ordering in TOC * Fix nav ordering in TOC for gettingstarted --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 097b634..d4497cb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -6,8 +6,8 @@ nav: - 'Read First': 'gettingstarted/index.md' - 'Podman + systemd': 'gettingstarted/podman-systemd-general.md' - 'Docker CE + systemd': 'gettingstarted/docker-systemd-general.md' - - 'Docker CE + Swarm RHEL 7.7': 'gettingstarted/docker-swarm-rhel7.md' - 'Docker CE + Swarm': 'gettingstarted/docker-swarm-general.md' + - 'Docker CE + Swarm RHEL 7.7': 'gettingstarted/docker-swarm-rhel7.md' - 'Bring your own Envionment': 'gettingstarted/byoe-rhel7.md' - Configuration: 'configuration.md' - Sources: From 6a34ced5210bd0c9ec4b3dba9c0dcc2861a6ee53 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Tue, 14 Jan 2020 08:20:18 -0800 Subject: [PATCH 13/14] Update docker-swarm-rhel7.md * Add RHEL 7.7 to title of docker swarm RHEL 7.7 runtime doc --- docs/gettingstarted/docker-swarm-rhel7.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index 67f9d07..251c9fa 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -1,5 +1,5 @@ -# Install Docker CE and Swarm +# Install Docker CE and Swarm (RHEL 7.7) * Warning: this method of installing docker on RHEL does not appear to be supported. Consider using podman instead. From d7a3644c9c73a2623f0f50c86c5070771e00d5ff Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Tue, 14 Jan 2020 08:29:23 -0800 Subject: [PATCH 14/14] Remove "Scale Out" at bottom of "Read First" * Remove "Scale Out" at bottom of "Read First"; a "one-size-fits-all" single sentence and siimple diagram is inappropriate guidance --- docs/gettingstarted/index.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/gettingstarted/index.md b/docs/gettingstarted/index.md index 8923b79..141df9c 100644 --- a/docs/gettingstarted/index.md +++ b/docs/gettingstarted/index.md @@ -120,8 +120,3 @@ attempt to obtain the container image via the internet. Environment="SC4S_IMAGE=sc4slocal:latest" ``` -## Scale out - -Additional hosts can be deployed for syslog collection from additional network zones and locations: - -![SC4S deployment diagram](SC4Sdeployment.png)