diff --git a/docs/configuration.md b/docs/configuration.md index 9f38c9d..379edf7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -123,4 +123,26 @@ redeploy the updated service using the command: - /opt/sc4s/default/compliance_meta_by_source.csv:/opt/syslog-ng/etc/context-local/compliance_meta_by_source.csv - /opt/sc4s/default/compliance_meta_by_source.conf:/opt/syslog-ng/etc/context-local/compliance_meta_by_source.conf `` +## Data Durability - Local Disk Buffer Configuration +SC4S provides capability to minimize the number of lost events if the connection to all the Splunk Indexers goes down. This capability utilizes the disk buffering feature of Syslog-ng. SC4S receives a response from the Splunk HTTP Event Collector (HEC) when a message is received successfully. If a confirmation message from the HEC endpoint is not received (or a “server busy” reply, such as a “503” is sent), the load balancer will try the next HEC endpoint in the pool. If all pool members are exhausted (such as would occur if there were a full network outage to the HEC endpoints), events will queue to the local disk buffer on the SC4S Linux host. SC4S will continue attempting to send the failed events while it buffers all new incoming events to disk. If the disk space allocated to disk buffering fills up then SC4S will stop accepting new events and subsequent events will be lost. Once SC4S gets confirmation that events are again being received by one or more indexers, events will then stream from the buffer using FIFO queueing. The number of events in the disk buffer will reduce as long as the incoming event volume is less than the maximum SC4S (with the disk buffer in the path) can handle. When all events have been emptied from the disk buffer, SC4S will resume streaming events directly to Splunk. + +For more detail on the Syslog-ng behavior the documentation can be found here: https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.22/administration-guide/55#TOPIC-1209280 + +SC4S has disk buffering enabled by default and it is strongly recommended that you keep it on, however this feature does have a performance cost. +Without disk buffering enabled SC4S can handle up to 345K EPS (800 bytes/event avg) +With “Normal” disk buffering enabled SC4S can handle up to 60K EPS (800 bytes/event avg) -- This is still a lot of data! + +To guard against data loss it is important to configure the appropriate type and amount of storage for SC4S disk buffering. To estimate the storage allocation its best to start with your estimated maximum events per second that each SC4S server will experience. Based on the maximum throughput of SC4S with disk buffering enabled, the conservative estimate for maximum events per second is 60K (however, you should use the maximum rate in your environment for this calculation, not the max rate SC4S can handle). Next is your average estimated event size based on your data sources. It is common industry practice to estimate log events as 800 bytes on average. And the final input to the sizing estimation would be the maximum length of connectivity downtime you want disk buffering to be able to handle. This measure is very much dependent on your risk tolerance. For example, to protect against a full day of lost connectivity from SC4S to all your indexers at maximum throughput the calculation would look like the following... + +60,000 EPS * 86400 seconds * 800 bytes = 3.77186 TB of storage + +To configure storage allocation for the SC4S disk buffering, do the following... +Edit the file /opt/sc4s/default/env_file +Add the SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE variable to the file and set the value to the number of bytes based on your estimation (e.g. 4147200000000 in the example above) +Splunk does not recommend reducing the disk allocation below 500 GB +Restart SC4S + +Given that in a connectivity outage to the Indexers events will be saved and read from disk until the buffer is emptied, it is ideal to use the fastest type of storage available. For this reason, NVMe storage is recommended for SC4S disk buffering. + +It is best to design your deployment so that the disk buffer will drain after connectivity is restored to the Splunk Indexers (while incoming data continues at the same general rate). Since "your mileage may vary" with different combinations of data load, instance type, and disk subsystem performance, it is good practice to provision a box that performs twice as well as is required for your max EPS. This headroom will allow for rapid recovery after a connectivity outage. diff --git a/docs/performance.md b/docs/performance.md index ea02c72..14f1b9c 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -34,7 +34,7 @@ and forget protocol making it sensitive to performance. Given this it is highly performance with your hardware and production data samples. The syslog-ng loggen tool available in the SC4S container and the commands above can be utilized in this effort. -Deployment Size | Hardware Spec | Average EPS with average msg size 800 k +Deployment Size | Hardware Spec | Average EPS with average msg size 800 bytes -- | -- | -- Small | 2 X 3.1 ghz cores1 GB of memory | 2K msg/sec Medium | 4 X 3.1 ghz cores2 GB of memory | 4.5K msg/sec diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index cd99a70..2fa996f 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -79,7 +79,7 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT") "no") "no") }} +{{- if or (or (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CHECKPOINT_SPLUNK traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl index e276484..2506ca5 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -43,7 +43,7 @@ log { }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT") "no") "no") }} +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_ASA_LEGACY traffic {{tmpl.Exec "log_path" "no" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index b9d997c..c47fcd9 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -44,7 +44,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_IOS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl index b1d9297..5f1bfb9 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl @@ -96,7 +96,8 @@ log { }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_ISE_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_ISE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_ISE_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_ISE traffic {{tmpl.Exec "log_path" "no" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index e351441..458ada4 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -44,7 +44,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_NX_OS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl index 759f523..5c000ba 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -42,7 +42,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT")) }} # Listen on the specified dedicated port(s) for FORCEPOINT_WEBPROTECT traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index 16b35f4..6fe1189 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -60,7 +60,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for FORTINET_FORTIOS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl index 7038621..c1dc820 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -45,7 +45,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_IDP_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_IDP traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index 4e3eea5..6185a34 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -59,7 +59,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_JUNOS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl index 4a9952e..9d6116f 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -43,7 +43,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NETSCREEN traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl index b21861d..404781d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -44,7 +44,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NSM traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl index 9d511c7..0383731 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -41,7 +41,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_NSM_IDP traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl index ba00cc2..549b939 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl @@ -90,8 +90,8 @@ log { }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} +{{- if or (or (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT")) (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT"))) (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT")) }} # Listen on the specified dedicated port(s) for MICROFOCUS_ARCSIGHT traffic {{tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 73dc1d0..71335e2 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -93,7 +93,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT")) (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for PALOALTO_PANOS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl index 60b6736..09dd410 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl @@ -42,7 +42,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT")) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT")) }} # Listen on the specified dedicated port(s) for PROOFPOINT_PPS_FILTER traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl index 11a8057..694f14e 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl @@ -41,7 +41,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT")) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT")) }} # Listen on the specified dedicated port(s) for PROOFPOINT_PPS_SENDMAIL traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl index 0962cc6..226b310 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl @@ -127,8 +127,8 @@ log { }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TLS_PORT") "no") "no") }} +{{- if or (or (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TCP_PORT")) (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_UDP_PORT"))) (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TLS_PORT")) }} # Listen on the specified dedicated port(s) for UBIQUITI_UNIFI traffic {{tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl index 983e13f..c9de545 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -82,7 +82,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT")) (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT")) }} # Listen on the specified dedicated port(s) for ZSCALER_NSS traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl index 69f4e28..98e3b78 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl @@ -40,7 +40,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_ASA traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl index 1ff0958..a9f881e 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl @@ -43,7 +43,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TCP_PORT")) (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_UDP_PORT"))) (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TLS_PORT")) }} # Listen on the specified dedicated port(s) for SYMANTEC_PROXY traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl index 19a26c3..cc2d052 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl @@ -60,9 +60,10 @@ log { }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT")) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_UDP_PORT"))) (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT")) }} # Listen on the specified dedicated port(s) for JUNIPER_JUNOS_STRUCTURED traffic - {{ tmpl.Exec "log_path" "no" }} +{{ tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for JUNIPER_JUNOS_STRUCTURED traffic diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl index 3ff2c86..149fb4b 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl @@ -40,7 +40,8 @@ log { flags(flow-control); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} + +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT")) }} # Listen on the specified dedicated port(s) for CISCO_MERAKI traffic {{ tmpl.Exec "log_path" "no" }} {{- end}} diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 63fb6d2..ff5be59 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -127,6 +127,6 @@ source s_{{ .port_id}} { }; }; {{- end }} -{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) or (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT")) }} +{{- if or (or (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT"))) (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT")) }} {{ template "T1" (.) }} {{- end }} diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl index 6a68b45..0f12886 100644 --- a/package/etc/local_config/log_paths/example.conf.tmpl +++ b/package/etc/local_config/log_paths/example.conf.tmpl @@ -7,41 +7,39 @@ # If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), # the template generator will build a custom source based on the value of one or more of the set variables. -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} # "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name # set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". # "parser" can be customized on dedicated ports only # "common" uses the same parser sequence as the default ports and is the most commonly used -{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} # The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path # The ports used are based on the values of one or more of the environment variables set above. +{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { - -# The first time this template is used the log_path will be linked to the default port - {{- if eq (.) "yes"}} source(s_DEFAULT); - -# Filters should be updated to use the simplest and most effecient logic possible to discard -# the message from this path - filter(f_is_rfc3164); filter(f_local_example); {{- end}} {{- if eq (.) "no"}} + source (s_LOCAL_EXAMPLE); +{{- end}} + + +# The first time this template is used the log_path will be linked to the default port + +# Filters should be updated to use the simplest and most effecient logic possible to discard +# the message from this path # In the second pass through the template a link to the dedicated port is used. This # normally does not require additional filters -source (s_LOCAL_EXAMPLE); -{{- end}} #Set a default sourcetype and index @@ -64,11 +62,11 @@ source (s_LOCAL_EXAMPLE); }; {{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} +{{- if or (or (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT")) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT")) }} # Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic - {{tmpl.Exec "log_path" "no" }} +{{tmpl.Exec "log_path" "no" }} {{- end}} # Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic