From 368f9d177117e3c155e74d56297b12168eb72f5b Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 24 Feb 2020 08:24:42 -0800 Subject: [PATCH 1/4] Add alternate destination support * Add global and source-specific alternate destination support to all log paths --- docs/configuration.md | 17 +++++++++++++++++ .../log_paths/lp-checkpoint_splunk.conf.tmpl | 10 +++++++++- .../etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl | 10 +++++++++- .../conf.d/log_paths/lp-cisco_apic.conf.tmpl | 8 ++++++++ .../etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl | 8 ++++++++ .../log_paths/lp-cisco_asa_legacy.conf.tmpl | 8 ++++++++ .../etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl | 8 ++++++++ .../etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl | 8 ++++++++ .../conf.d/log_paths/lp-cisco_meraki.conf.tmpl | 8 ++++++++ .../conf.d/log_paths/lp-cisco_nxos.conf.tmpl | 8 ++++++++ .../etc/conf.d/log_paths/lp-cisco_ucm.conf.tmpl | 8 ++++++++ .../log_paths/lp-citrix-netscaler.conf.tmpl | 8 ++++++++ .../log_paths/lp-common_event_format.conf.tmpl | 8 ++++++++ .../lp-forcepoint_webprotect.conf.tmpl | 8 ++++++++ .../etc/conf.d/log_paths/lp-fortinet.conf.tmpl | 8 ++++++++ .../etc/conf.d/log_paths/lp-infoblox.conf.tmpl | 8 ++++++++ .../conf.d/log_paths/lp-juniper_idp.conf.tmpl | 8 ++++++++ .../conf.d/log_paths/lp-juniper_junos.conf.tmpl | 8 ++++++++ .../lp-juniper_junos_structured.conf.tmpl | 8 ++++++++ .../log_paths/lp-juniper_netscreen.conf.tmpl | 10 +++++++++- .../conf.d/log_paths/lp-juniper_nsm.conf.tmpl | 8 ++++++++ .../log_paths/lp-juniper_nsm_idp.conf.tmpl | 8 ++++++++ .../log_paths/lp-paloalto_panos.conf.tmpl | 8 ++++++++ .../etc/conf.d/log_paths/lp-pfsense.conf.tmpl | 8 ++++++++ .../log_paths/lp-proofpoint_pps.conf.tmpl | 8 ++++++++ .../conf.d/log_paths/lp-sc4s_internal.conf.tmpl | 8 ++++++++ .../conf.d/log_paths/lp-sc4s_startup.conf.tmpl | 8 ++++++++ .../log_paths/lp-symantec_brightmail.conf.tmpl | 8 ++++++++ .../log_paths/lp-symantec_proxy.conf.tmpl | 8 ++++++++ .../log_paths/lp-ubiquiti_unifi.conf.tmpl | 10 +++++++++- .../log_paths/lp-vmware_vsphere.conf.tmpl | 8 ++++++++ .../conf.d/log_paths/lp-zscaler_nss.conf.tmpl | 8 ++++++++ .../log_paths/lp-zzy-nix_syslog.conf.tmpl | 8 ++++++++ .../conf.d/log_paths/lp-zzz-fallback.conf.tmpl | 10 +++++++++- 34 files changed, 286 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9860a3c..00908f7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -26,6 +26,23 @@ syslog. | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (threads). Set this to the number of HEC endpoints up to a max of 32. | +## Alternate Destination Configuration + +Alternate destinations other than HEC can be configured in SC4S. Global and/or source-specific forms of the +variables below can be used to send data to alternate destinations. + +* NOTE: The administrator is responsible for ensuring that the alternate destinations are configured in the +local mount tree, and that syslog-ng properly parses them. + +* NOTE: Do not include `d_hec` in any list of alternate destinations. The configuration of the default HEC destination is configured +separately from that of the alternates below. + + +| Variable | Values | Description | +|----------|---------------|-------------| +| SC4S_DEST_GLOBAL_ALTERNATES | Comma or space-separated list of syslog-ng destinations | Send all sources to alternate destinations | +| SC4S_DEST_\_ALTERNATES | Comma or space-separated list of syslog-ng destiinations | Send specific sources to alternate syslog-ng destinations, e.g. SC4S_DEST_CISCO_ASA_ALTERNATES | + ## SC4S Disk Buffer Configuration Disk buffers in SC4S are allocated _per destination_. In the future as more destinations are supported, a separate list of variables diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl index bddcbd3..ed34f40 100644 --- a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl @@ -88,5 +88,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CHECKPOINT_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CHECKPOINT_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); -}; \ No newline at end of file +}; diff --git a/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl index 817df0a..ff1109a 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl @@ -87,7 +87,15 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CISCO_ACS_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CISCO_ACS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; -}; \ No newline at end of file +}; diff --git a/package/etc/conf.d/log_paths/lp-cisco_apic.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_apic.conf.tmpl index cd3d9f9..64c123b 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_apic.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_apic.conf.tmpl @@ -52,5 +52,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CISCO_APIC_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CISCO_APIC_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl index 9045821..76c8a8b 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl @@ -37,5 +37,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CISCO_ASA_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CISCO_ASA_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl index a8bf9c5..f29a551 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl @@ -41,5 +41,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CISCO_ASA_LEGACY_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CISCO_ASA_LEGACY_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl index 80aae01..89186e2 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl @@ -37,5 +37,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CISCO_IOS_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CISCO_IOS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl index 9a5bf0f..a7d47e1 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl @@ -89,6 +89,14 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CISCO_ISE_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CISCO_ISE_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_meraki.conf.tmpl index c0f37ce..3822ee6 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_meraki.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_meraki.conf.tmpl @@ -37,5 +37,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CISCO_MERAKI_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CISCO_MERAKI_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl index 8bf2d2b..ed387d2 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl @@ -38,5 +38,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CISCO_NXOS_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CISCO_NXOS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-cisco_ucm.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_ucm.conf.tmpl index e0823cf..61d0274 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_ucm.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_ucm.conf.tmpl @@ -58,5 +58,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CISCO_UCM_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CISCO_UCM_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-citrix-netscaler.conf.tmpl b/package/etc/conf.d/log_paths/lp-citrix-netscaler.conf.tmpl index ed11613..ed6f197 100644 --- a/package/etc/conf.d/log_paths/lp-citrix-netscaler.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-citrix-netscaler.conf.tmpl @@ -37,5 +37,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CITRIX_NETSCALER_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CITRIX_NETSCALER_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl b/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl index 5c25810..0b6b320 100644 --- a/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-common_event_format.conf.tmpl @@ -97,5 +97,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_CEF_ALTERNATES")) }} + {{ getenv "SC4S_DEST_CEF_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/lp-forcepoint_webprotect.conf.tmpl index 3b55bfa..40c072f 100644 --- a/package/etc/conf.d/log_paths/lp-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-forcepoint_webprotect.conf.tmpl @@ -38,5 +38,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_FORCEPOINT_WEBPROTECT_ALTERNATES")) }} + {{ getenv "SC4S_DEST_FORCEPOINT_WEBPROTECT_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-fortinet.conf.tmpl b/package/etc/conf.d/log_paths/lp-fortinet.conf.tmpl index 7435657..045dbe4 100644 --- a/package/etc/conf.d/log_paths/lp-fortinet.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-fortinet.conf.tmpl @@ -114,5 +114,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_FORTINET_ALTERNATES")) }} + {{ getenv "SC4S_DEST_FORTINET_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-infoblox.conf.tmpl b/package/etc/conf.d/log_paths/lp-infoblox.conf.tmpl index 8b40188..40318ae 100644 --- a/package/etc/conf.d/log_paths/lp-infoblox.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-infoblox.conf.tmpl @@ -70,5 +70,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_INFOBLOX_ALTERNATES")) }} + {{ getenv "SC4S_DEST_INFOBLOX_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_idp.conf.tmpl index 436963a..969c250 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_idp.conf.tmpl @@ -37,5 +37,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_JUNIPER_IDP_ALTERNATES")) }} + {{ getenv "SC4S_DEST_JUNIPER_IDP_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_junos.conf.tmpl index 05d7e5d..c5a2786 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_junos.conf.tmpl @@ -56,5 +56,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_JUNIPER_JUNOS_ALTERNATES")) }} + {{ getenv "SC4S_DEST_JUNIPER_JUNOS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl index d5ae714..e42756a 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl @@ -64,5 +64,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_JUNOS_STRUCTURED_ALTERNATES")) }} + {{ getenv "SC4S_DEST_JUNOS_STRUCTURED_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_netscreen.conf.tmpl index 74c40b0..49cdbb9 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_netscreen.conf.tmpl @@ -36,5 +36,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_JUNIPER_NETSCREEN_ALTERNATES")) }} + {{ getenv "SC4S_DEST_JUNIPER_NETSCREEN_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); -}; \ No newline at end of file +}; diff --git a/package/etc/conf.d/log_paths/lp-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_nsm.conf.tmpl index 650b7c9..9ac7cfd 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_nsm.conf.tmpl @@ -37,5 +37,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_JUNIPER_NSM_ALTERNATES")) }} + {{ getenv "SC4S_DEST_JUNIPER_NSM_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_nsm_idp.conf.tmpl index e76fb0a..e9f58e0 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_nsm_idp.conf.tmpl @@ -36,5 +36,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_JUNIPER_NSM_IDP_ALTERNATES")) }} + {{ getenv "SC4S_DEST_JUNIPER_NSM_IDP_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl index f115db2..2f9cd3d 100644 --- a/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl @@ -93,5 +93,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_PALOALTO_PANOS_ALTERNATES")) }} + {{ getenv "SC4S_DEST_PALOALTO_PANOS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl b/package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl index b9ea159..293f428 100644 --- a/package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-pfsense.conf.tmpl @@ -54,5 +54,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_PFSENSE_ALTERNATES")) }} + {{ getenv "SC4S_DEST_PFSENSE_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-proofpoint_pps.conf.tmpl b/package/etc/conf.d/log_paths/lp-proofpoint_pps.conf.tmpl index dc911ec..8881d4c 100644 --- a/package/etc/conf.d/log_paths/lp-proofpoint_pps.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-proofpoint_pps.conf.tmpl @@ -44,5 +44,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_PROOFPOINT_PPS_ALTERNATES")) }} + {{ getenv "SC4S_DEST_PROOFPOINT_PPS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl index 0756b20..9dc43ae 100644 --- a/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-sc4s_internal.conf.tmpl @@ -33,6 +33,14 @@ log { destination(d_stdout); {{- end}} + {{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} + + {{- if (print (getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES")) }} + {{ getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); + {{- end }} + }; flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-sc4s_startup.conf.tmpl b/package/etc/conf.d/log_paths/lp-sc4s_startup.conf.tmpl index c6607be..ab4c5c7 100644 --- a/package/etc/conf.d/log_paths/lp-sc4s_startup.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-sc4s_startup.conf.tmpl @@ -30,5 +30,13 @@ log { destination(d_stdout); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES")) }} + {{ getenv "SC4S_DEST_INTERNAL_EVENTS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/lp-symantec_brightmail.conf.tmpl index 8ae5329..baa48a9 100644 --- a/package/etc/conf.d/log_paths/lp-symantec_brightmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-symantec_brightmail.conf.tmpl @@ -93,5 +93,13 @@ log { }; {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl index cc3524d..8d5b475 100644 --- a/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl @@ -38,5 +38,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_SYMANTEC_PROXY_ALTERNATES")) }} + {{ getenv "SC4S_DEST_SYMANTEC_PROXY_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/lp-ubiquiti_unifi.conf.tmpl index cee88ad..bccf149 100644 --- a/package/etc/conf.d/log_paths/lp-ubiquiti_unifi.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-ubiquiti_unifi.conf.tmpl @@ -116,7 +116,7 @@ log { rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; -{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_PROOFPOINT_UBIQUITI_UNIFI_HEC" "no")) }} +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_UBIQUITI_UNIFI_HEC" "no")) }} destination(d_hec); {{- end}} @@ -124,5 +124,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_UBIQUITI_UNIFI_ALTERNATES")) }} + {{ getenv "SC4S_DEST_UBIQUITI_UNIFI_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-vmware_vsphere.conf.tmpl b/package/etc/conf.d/log_paths/lp-vmware_vsphere.conf.tmpl index 877eb18..13b77b5 100644 --- a/package/etc/conf.d/log_paths/lp-vmware_vsphere.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-vmware_vsphere.conf.tmpl @@ -97,5 +97,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_VMWARE_VSPHERE_ALTERNATES")) }} + {{ getenv "SC4S_DEST_VMWARE_VSPHERE_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl index 3959dc2..c1c57e3 100644 --- a/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl @@ -78,5 +78,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_ZSCALER_NSS_ALTERNATES")) }} + {{ getenv "SC4S_DEST_ZSCALER_NSS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-zzy-nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/lp-zzy-nix_syslog.conf.tmpl index be65104..9b9f848 100644 --- a/package/etc/conf.d/log_paths/lp-zzy-nix_syslog.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zzy-nix_syslog.conf.tmpl @@ -40,5 +40,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_NIX_SYSLOG_ALTERNATES")) }} + {{ getenv "SC4S_DEST_NIX_SYSLOG_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl b/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl index 1298c67..a3a33d9 100644 --- a/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zzz-fallback.conf.tmpl @@ -28,5 +28,13 @@ log { destination(d_archive); {{- end}} +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_FALLBACK_ALTERNATES")) }} + {{ getenv "SC4S_DEST_FALLBACK_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + flags(flow-control,fallback); -}; \ No newline at end of file +}; From cac25febccc9688f3484c87fbb70c0ecc514c714 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Mon, 24 Feb 2020 15:11:35 -0500 Subject: [PATCH 2/4] Update lp-zscaler_nss.conf.tmpl --- package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl index c1c57e3..3e26753 100644 --- a/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl @@ -1,4 +1,5 @@ # Zscaler + {{- /* The following provides a unique port source configuration if env var(s) are set */}} {{- $context := dict "port_id" "ZSCALER_NSS" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} From e47a3adf032702801c6ed26e8cf62ae996969236 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 5 Mar 2020 13:08:02 -0500 Subject: [PATCH 3/4] Seperate zscaler LSS and NSS provide proper LSS support --- docs/sources/Zscaler/index.md | 70 ++++++++- .../conf.d/conflib/_common/syslog_format.conf | 19 +++ package/etc/conf.d/filters/zscaler/nss.conf | 7 +- .../conf.d/log_paths/lp-zscaler_lss.conf.tmpl | 98 ++++++++++++ .../conf.d/log_paths/lp-zscaler_nss.conf.tmpl | 83 +++++----- package/etc/go_templates/source_network.t | 39 +++++ tests/test_zscaler_proxy.py | 144 +++++++++++++++++- 7 files changed, 404 insertions(+), 56 deletions(-) create mode 100644 package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl diff --git a/docs/sources/Zscaler/index.md b/docs/sources/Zscaler/index.md index 67b70e1..d922ef7 100644 --- a/docs/sources/Zscaler/index.md +++ b/docs/sources/Zscaler/index.md @@ -1,6 +1,6 @@ # Vendor - Zscaler -## Product - All Products +## Product - ZIA The ZScaler product manual includes and extensive section of configuration for multiple Splunk TCP input ports around page 26. When using SC4S these ports are not required and should not be used. Simply configure all outputs from the NSS to utilize @@ -20,9 +20,6 @@ the IP or host name of the SC4S instance and port 514 | zscalernss-alerts | Requires format customization add ``\tvendor=Zscaler\tproduct=alerts`` immediately prior to the ``\n`` in the NSS Alert Web format. See Zscaler manual for more info. | | zscalernss-dns | Requires format customization add ``\tvendor=Zscaler\tproduct=dns`` immediately prior to the ``\n`` in the NSS DNS format. See Zscaler manual for more info. | | zscalernss-web | None | -| zscalernss-zpa-app | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | -| zscalernss-zpa-auth | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa_auth`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | -| zscalernss-zpa-connector | Requires format customization add ``\tvendor=Zscaler\tproduct=zpa_auth_connector`` immediately prior to the ``\n`` in the LSS Connector format. See Zscaler manual for more info. | | zscalernss-fw | Requires format customization add ``\tvendor=Zscaler\tproduct=fw`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. | @@ -34,9 +31,6 @@ the IP or host name of the SC4S instance and port 514 | zscalernss_dns | zscalernss-dns | netdns | none | | zscalernss_fw | zscalernss-fw | netfw | none | | zscalernss_web | zscalernss-web | netproxy | none | -| zscalernss-zpa-app | zscalernss_zpa-app | netids | none | -| zscalernss-zpa-auth | zscalernss_zpa_auth | netauth | none | -| zscalernss-zpa-connector | zscalernss_zpa_connector | netops | none | ### Filter type @@ -67,3 +61,65 @@ An active proxy will generate frequent events. Use the following search to valid ``` index= sourcetype=zscalernss-* | stats count by host ``` + +## Product - LSS + +The ZScaler product manual includes and extensive section of configuration for multiple Splunk TCP input ports around page +26. When using SC4S these ports are not required and should not be used. Simply configure all outputs from the LSS to utilize +the IP or host name of the SC4S instance and port 514 + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/3865/ | +| Product Manual | https://community.zscaler.com/t/zscaler-splunk-app-design-and-installation-documentation/4728 | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| zscalerlss-zpa-app | None | +| zscalerlss-zpa-auth | None | +| zscalerlss-zpa-bba | None | +| zscalerlss-zpa-connector | None | + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| zscalernss-zpa-app | zscalerlss_zpa-app | netproxy | none | +| zscalernss-zpa-auth | zscalerlss_zpa_auth | netauth | none | +| zscalernss-zpa-bba | zscalerlss_zpa_auth | netproxy | none | +| zscalernss-zpa-connector | zscalerlss_zpa_connector | netproxy | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_ZSCALER_LSS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_ZSCALER_LSS_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_ZSCALER_LSS | no | Enable archive to disk for this specific source | +| SC4S_DEST_ZSCALER_LSS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=zscalernss-* | stats count by host +``` diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index 96ab5dc..1a8ed16 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -49,4 +49,23 @@ filter f_is_no_parse{ rewrite set_rfc3164_no_version_string{ subst('(^<\d+>)\d', $1, value("MESSAGE")); +}; +filter f_is_rfc3164_json{ + match("rfc3164_json" value("fields.sc4s_syslog_format")) +}; +rewrite set_rfc3164_json{ + set("rfc3164_json" value("fields.sc4s_syslog_format")); +}; + +filter f_is_tcp_json{ + match("tcp_json" value("fields.sc4s_syslog_format")) +}; +rewrite set_tcp_json{ + set("tcp_json" value("fields.sc4s_syslog_format")); +}; + +filter f_msg_is_tcp_json{ + match("rfc3164_json" value("fields.sc4s_syslog_format")) + or + match("tcp_json" value("fields.sc4s_syslog_format")) }; \ No newline at end of file diff --git a/package/etc/conf.d/filters/zscaler/nss.conf b/package/etc/conf.d/filters/zscaler/nss.conf index 9ee4e1a..543e1ad 100644 --- a/package/etc/conf.d/filters/zscaler/nss.conf +++ b/package/etc/conf.d/filters/zscaler/nss.conf @@ -1,3 +1,8 @@ filter f_zscaler_nss { message('\tvendor=Zscaler\t'); -}; \ No newline at end of file + or message('^ZscalerNSS:'); +}; +# filter f_zscaler_lss { +# match() + +# }; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl new file mode 100644 index 0000000..ca822aa --- /dev/null +++ b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl @@ -0,0 +1,98 @@ +# Zscaler + +{{- /* The following provides a unique port source configuration if env var(s) are set */}} +{{- $context := dict "port_id" "ZSCALER_LSS" "parser" "rfc3164" }} +{{- tmpl.Exec "t/source_network.t" $context }} + +log { + junction { +{{- if or (or (getenv (print "SC4S_LISTEN_ZSCALER_LSS_TCP_PORT")) (getenv (print "SC4S_LISTEN_ZSCALER_LSS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_ZSCALER_LSS_TLS_PORT")) }} + channel { + # Listen on the specified dedicated port(s) for ZSCALER_LSS traffic + source (s_ZSCALER_LSS); + flags (final); + }; +{{- end}} + channel { + # Listen on the default port (typically 514) for ZSCALER_LSS traffic + source (s_DEFAULT); + filter(f_msg_is_tcp_json); + flags(final); + }; + }; + parser { + #.jsonLog.Timestamp Mar 04 20:37:53 2020 + date-parser( + format("%b %d %H:%M:%S %Y", + "%h %d %H:%M:%S %Y", + "%b %d %k:%M:%S %Y", + "%h %d %k:%M:%S %Y") + template("${.json.LogTimestamp}") + time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) + flags(guess-timezone) + ); + + }; + if { + filter { + match('.' value('.json.ClientZEN')) + and match('.' value('.json.AppGroup')) + and match('.' value('.json.Application')) + }; + rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-app"), index("netproxy"))}; + parser { p_add_context_splunk(key("zscaler_lss")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + } elif { + filter { + match('.' value('.json.Exporter')) + and match('.' value('.json.Customer')) + and match('.' value('.json.ConnectionID')) + }; + rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-bba"), index("netproxy"))}; + parser { p_add_context_splunk(key("zscaler_lss")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + } elif { + filter { + match('.' value('.json.Connector')) + and match('.' value('.json.Customer')) + and match('.' value('.json.ConnectorGroup')) + }; + rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-connector"), index("netproxy"))}; + parser { p_add_context_splunk(key("zscaler_lss")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + } elif { + filter { + match('.' value('.json.SAMLAttributes')) + and match('.' value('.json.Customer')) + }; + rewrite { r_set_splunk_dest_default(sourcetype("zscalerlss-zpa-auth"), index("netauth"))}; + parser { p_add_context_splunk(key("zscaler_lss")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + }; + + + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + +{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_ZSCALER_LSS_HEC" "no")) }} + destination(d_hec); +{{- end}} + +{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_ZSCALER_LSS" "no")) }} + destination(d_archive); +{{- end}} + +{{- if (print (getenv "SC4S_DEST_GLOBAL_ALTERNATES")) }} + {{ getenv "SC4S_DEST_GLOBAL_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + +{{- if (print (getenv "SC4S_DEST_ZSCALER_LSS_ALTERNATES")) }} + {{ getenv "SC4S_DEST_ZSCALER_LSS_ALTERNATES" | regexp.ReplaceLiteral "^" "destination(" | regexp.ReplaceLiteral "[, ]+" ");\n destination(" }}); +{{- end }} + + flags(flow-control,final); +}; diff --git a/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl index 3e26753..cc14627 100644 --- a/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl @@ -20,57 +20,48 @@ log { flags(final); }; }; - - parser { date-parser(format("%Y-%m-%d %H:%M:%S") template('$(substr "$LEGACY_MSGHDR$MSG" "0" "19")') flags(guess-timezone)); }; - rewrite { - set("zscaler_nss", value("fields.sc4s_vendor_product")); - subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); - }; - parser { - #basic parsing - kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}")); - }; - - if (match("alerts" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"))}; - parser { p_add_context_splunk(key("zscaler_alerts")); }; - } elif (match("dns" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"))}; - parser { p_add_context_splunk(key("zscaler_dns")); }; - } elif (match("fw" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"))}; - parser { p_add_context_splunk(key("zscaler_fw")); }; - } elif (match("NSS" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"))}; - parser { p_add_context_splunk(key("zscaler_web")); }; - } elif (match("audit" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"))}; - parser { p_add_context_splunk(key("zscaler_zia_audit")); }; - } elif (match("sandbox" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"))}; - parser { p_add_context_splunk(key("zscaler_zia_sandbox")); }; - } elif (match("zpa" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"))}; - parser { p_add_context_splunk(key("zscaler_zpa")); }; - } elif (match("zpa_auth" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"))}; - parser { p_add_context_splunk(key("zscaler_zpa_auth")); }; - } elif (match("zpa_auth_connector" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"))}; - parser { p_add_context_splunk(key("zscaler_zpa_connector")); }; - } elif (match("zpa_bba" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"))}; - parser { p_add_context_splunk(key("zscaler_zpa_bba")); }; + if (message('^ZscalerNSS:')) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("netops"))}; + parser { p_add_context_splunk(key("zscaler_alerts")); }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"))}; + parser { date-parser(format("%Y-%m-%d %H:%M:%S") template('$(substr "$LEGACY_MSGHDR$MSG" "0" "19")') flags(guess-timezone)); }; + rewrite { + set("zscaler_nss", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + }; parser { - p_add_context_splunk(key("zscaler_nss")); + #basic parsing + kv-parser(prefix(".kv.") pair-separator("\t") template("${MSG}")); }; - }; - parser (compliance_meta_by_source); - rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + + if (match("dns" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"))}; + parser { p_add_context_splunk(key("zscaler_dns")); }; + } elif (match("fw" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"))}; + parser { p_add_context_splunk(key("zscaler_fw")); }; + } elif (match("NSS" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"))}; + parser { p_add_context_splunk(key("zscaler_web")); }; + } elif (match("audit" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"))}; + parser { p_add_context_splunk(key("zscaler_zia_audit")); }; + } elif (match("sandbox" value(".kv.product"))) { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"))}; + parser { p_add_context_splunk(key("zscaler_zia_sandbox")); }; + } else { + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"))}; + parser { + p_add_context_splunk(key("zscaler_nss")); + }; + }; + parser (compliance_meta_by_source); + rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); }; + }; {{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_ZSCALER_NSS_HEC" "no")) }} destination(d_hec); {{- end}} diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index ea83f05..8ef31fb 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -96,11 +96,28 @@ source s_{{ .port_id }} { rewrite (r_cisco_ucm_message); {{ else if eq .parser "no_parse" }} rewrite(set_no_parse); +{{ else if eq .parser "tcp_json" }} + filter { message('^{') and message('}$') }; + parser { + json-parser( + prefix('.json.') + ); + }; + rewrite(set_tcp_json); {{ else }} if { filter(f_citrix_netscaler_message); parser(p_citrix_netscaler_date); rewrite(r_citrix_netscaler_message); + } elif { + #JSON over IP its not syslog but it can work + filter { message('^{') and message('}$') }; + parser { + json-parser( + prefix('.json.') + ); + }; + rewrite(set_tcp_json); } elif { filter(f_rfc5424_strict); parser { @@ -135,6 +152,28 @@ source s_{{ .port_id }} { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone {{- if (conv.ToBool (getenv "SC4S_SOURCE_STORE_RAWMSG" "no")) }} store-raw-message {{- end}})); }; rewrite(set_rfc3164); + if { + filter { message('^{') and message('}$') }; + parser { + json-parser( + prefix('.json.') + ); + }; + rewrite(set_rfc3164_json); + } elif { + filter { match('^{' value('LEGACY_MSGHDR')) and message('}$') }; + parser { + json-parser( + prefix('.json.') + template('${LEGACY_MSGHDR}${MSG}') + ); + }; + rewrite { + set('${LEGACY_MSGHDR}${MSG}' value('MSG')); + unset(value('LEGACY_MSGHDR')); + }; + rewrite(set_rfc3164_json); + }; }; {{ end }} rewrite(r_set_splunk_default); diff --git a/tests/test_zscaler_proxy.py b/tests/test_zscaler_proxy.py index 595eb08..3873592 100644 --- a/tests/test_zscaler_proxy.py +++ b/tests/test_zscaler_proxy.py @@ -4,7 +4,7 @@ # license that can be found in the LICENSE-BSD2 file or at # https://opensource.org/licenses/BSD-2-Clause import random - +import pytest from jinja2 import Environment from .sendmessage import * @@ -32,6 +32,26 @@ def test_zscaler_proxy(record_property, setup_wordlist, setup_splunk, setup_sc4s assert resultCount == 1 +#2020-03-02 02:51:56 reason=Allowed event_id=6799437957281873922 protocol=HTTP action=Allowed transactionsize=623 responsesize=512 requestsize=111 urlcategory=Internet Services serverip=13.107.4.52 clienttranstime=3 requestmethod=GET refererURL="None" useragent=Microsoft NCSI product=NSS location=Road Warrior ClientIP=136.35.16.85 status=200 user=mdutta@acme.com url="www.msftconnecttest.com/connecttest.txt" vendor=Zscaler hostname=www.msftconnecttest.com clientpublicIP=136.35.16.85 threatcategory=None threatname=None filetype=None appname=generalbrowsing pagerisk=0 department=Default Department urlsupercategory=Internet Communication appclass=General Browsing dlpengine=None urlclass=Business Use threatclass=None dlpdictionaries=None fileclass=None bwthrottle=NO servertranstime=3 md5=None contenttype=text/plain trafficredirectmethod=Z_APP rulelabel=None ruletype=None mobappname=None mobappcat=None mobdevtype=None bwclassname=General Surfing bwrulename=No Bandwidth Control throttlereqsize=0 throttlerespsize=0 deviceappversion=1.5.1.8 devicemodel=20QF000CUS devicemodel=20QF000CUS devicename=mdutta devicename=mdutta deviceostype=Windows OS deviceostype=Windows OS deviceosversion=Windows 10 Enterprise deviceplatform= clientsslcipher=None clientsslsessreuse=UNKNOWN clienttlsversion=None serversslsessreuse=UNKNOWN servertranstime=3 srvcertchainvalpass=UNKNOWN srvcertvalidationtype=None srvcertvalidityperiod=None srvocspresult=None srvsslcipher=None srvtlsversion=None srvwildcardcert=UNKNOWN serversslsessreuse="UNKNOWN" dlpidentifier="0" dlpmd5="None" epochtime="1583117516" filename="None" filesubtype="None" module="General Browsing" productversion="5.7r.78.218665_84" reqdatasize="0" reqhdrsize="111" respdatasize="22" resphdrsize="490" respsize="512" respversion="1.1" tz="GMT" +def test_zscaler_proxy_new(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{% now 'local', '%Y-%m-%d %H:%M:%S' %}"+ ' reason=Allowed event_id=6799437957281873922 protocol=HTTP action=Allowed transactionsize=623 responsesize=512 requestsize=111 urlcategory=Internet Services serverip=13.107.4.52 clienttranstime=3 requestmethod=GET refererURL="None" useragent=Microsoft NCSI product=NSS location=Road Warrior ClientIP=136.35.16.85 status=200 user=mdutta@acme.com url="www.msftconnecttest.com/connecttest.txt" vendor=Zscaler hostname={{host}}.fls.doubleclick.net clientpublicIP=136.35.16.85 threatcategory=None threatname=None filetype=None appname=generalbrowsing pagerisk=0 department=Default Department urlsupercategory=Internet Communication appclass=General Browsing dlpengine=None urlclass=Business Use threatclass=None dlpdictionaries=None fileclass=None bwthrottle=NO servertranstime=3 md5=None contenttype=text/plain trafficredirectmethod=Z_APP rulelabel=None ruletype=None mobappname=None mobappcat=None mobdevtype=None bwclassname=General Surfing bwrulename=No Bandwidth Control throttlereqsize=0 throttlerespsize=0 deviceappversion=1.5.1.8 devicemodel=20QF000CUS devicemodel=20QF000CUS devicename=mdutta devicename=mdutta deviceostype=Windows OS deviceostype=Windows OS deviceosversion=Windows 10 Enterprise deviceplatform= clientsslcipher=None clientsslsessreuse=UNKNOWN clienttlsversion=None serversslsessreuse=UNKNOWN servertranstime=3 srvcertchainvalpass=UNKNOWN srvcertvalidationtype=None srvcertvalidityperiod=None srvocspresult=None srvsslcipher=None srvtlsversion=None srvwildcardcert=UNKNOWN serversslsessreuse="UNKNOWN" dlpidentifier="0" dlpmd5="None" epochtime="1583117516" filename="None" filesubtype="None" module="General Browsing" productversion="5.7r.78.218665_84" reqdatasize="0" reqhdrsize="111" respdatasize="22" resphdrsize="490" respsize="512" respversion="1.1" tz="GMT"') + message = mt.render(mark="<134>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search earliest=-1m@m latest=+1m@m index=netproxy sourcetype=\"zscalernss-web\" hostname={{host}}.fls.doubleclick.net | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + # def test_zscaler_proxy_pri(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) @@ -52,4 +72,124 @@ def test_zscaler_proxy_pri(record_property, setup_wordlist, setup_splunk, setup_ assert resultCount == 1 -# +#<118>Mar 1 22:05:35 [10.225.64.143] ZscalerNSS: The NSS free memory has decreased to 1.40 GB which is below the recommended 1.55 GB {{host}} +def test_zscaler_nss_alerts(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'local', '%b %d %H:%M:%S' %} [10.0.0.143] ZscalerNSS: The NSS free memory has decreased to 1.40 GB which is below the recommended 1.55 GB {{host}}") + message = mt.render(mark="<134>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search earliest=-1m@m latest=+1m@m index=netops sourcetype=\"zscalernss-alerts\" \"{{host}}\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#{"LogTimestamp": "Mon Mar 2 02:57:01 2020","Customer": "Acme, Inc.","SessionID": "qdLxaTYtMbsCQllNaCZ2","ConnectionID": "qdLxaTYtMbsCQllNaCZ2,aZcOpy7yN8iPncqmSuAv","InternalReason": "","ConnectionStatus": "active","IPProtocol": 6,"DoubleEncryption": 0,"Username": "nlipper@acme.com","ServicePort": 8384,"ClientPublicIP": "73.144.81.255","ClientPrivateIP": "","ClientLatitude": 42.000000,"ClientLongitude": -84.000000,"ClientCountryCode": "US","ClientZEN": "US-OH-8290","Policy": "Any Any Allow","Connector": "DFA Azure-2","ConnectorZEN": "US-OH-8290","ConnectorIP": "10.202.4.68","ConnectorPort": 35992,"Host": "10.26.1.19","Application": "DFA IP SPACE","AppGroup": "Dynamically Discovered Apps","Server": "0","ServerIP": "10.26.1.19","ServerPort": 8384,"PolicyProcessingTime": 120,"CAProcessingTime": 445,"ConnectorZENSetupTime": 46610,"ConnectionSetupTime": 47200,"ServerSetupTime": 22207,"AppLearnTime": 0,"TimestampConnectionStart": "2020-02-29T20:42:01.228Z","TimestampConnectionEnd": "","TimestampCATx": "2020-02-29T20:42:01.228Z","TimestampCARx": "2020-02-29T20:42:01.228Z","TimestampAppLearnStart": "","TimestampZENFirstRxClient": "","TimestampZENFirstTxClient": "","TimestampZENLastRxClient": "","TimestampZENLastTxClient": "","TimestampConnectorZENSetupComplete": "2020-02-29T20:42:01.275Z","TimestampZENFirstRxConnector": "","TimestampZENFirstTxConnector": "","TimestampZENLastRxConnector": "","TimestampZENLastTxConnector": "","ZENTotalBytesRxClient": 0,"ZENBytesRxClient": 0,"ZENTotalBytesTxClient": 0,"ZENBytesTxClient": 0,"ZENTotalBytesRxConnector": 0,"ZENBytesRxConnector": 0,"ZENTotalBytesTxConnector": 0,"ZENBytesTxConnector": 0,"Idp": "IDP Config"} +def test_zscaler_lss_zpa_app(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{\"LogTimestamp\": \"{% now 'local', '%b %d %H:%M:%S %Y' %}" + '","Customer": "Acme, Inc.","SessionID": "qdLxaTYtMbsCQllNaCZ2","ConnectionID": "qdLxaTYtMbsCQllNaCZ2,aZcOpy7yN8iPncqmSuAv","InternalReason": "","ConnectionStatus": "active","IPProtocol": 6,"DoubleEncryption": 0,"Username": "nlipper@acme.com","ServicePort": 8384,"ClientPublicIP": "73.144.81.255","ClientPrivateIP": "","ClientLatitude": 42.000000,"ClientLongitude": -84.000000,"ClientCountryCode": "US","ClientZEN": "US-OH-8290","Policy": "Any Any Allow","Connector": "DFA Azure-2","ConnectorZEN": "US-OH-8290","ConnectorIP": "10.202.4.68","ConnectorPort": 35992,"Host": "10.26.1.19","Application": "DFA IP SPACE","AppGroup": "Dynamically Discovered Apps","Server": "0","ServerIP": "{{host}}","ServerPort": 8384,"PolicyProcessingTime": 120,"CAProcessingTime": 445,"ConnectorZENSetupTime": 46610,"ConnectionSetupTime": 47200,"ServerSetupTime": 22207,"AppLearnTime": 0,"TimestampConnectionStart": "2020-02-29T20:42:01.228Z","TimestampConnectionEnd": "","TimestampCATx": "2020-02-29T20:42:01.228Z","TimestampCARx": "2020-02-29T20:42:01.228Z","TimestampAppLearnStart": "","TimestampZENFirstRxClient": "","TimestampZENFirstTxClient": "","TimestampZENLastRxClient": "","TimestampZENLastTxClient": "","TimestampConnectorZENSetupComplete": "2020-02-29T20:42:01.275Z","TimestampZENFirstRxConnector": "","TimestampZENFirstTxConnector": "","TimestampZENLastRxConnector": "","TimestampZENLastTxConnector": "","ZENTotalBytesRxClient": 0,"ZENBytesRxClient": 0,"ZENTotalBytesTxClient": 0,"ZENBytesTxClient": 0,"ZENTotalBytesRxConnector": 0,"ZENBytesRxConnector": 0,"ZENTotalBytesTxConnector": 0,"ZENBytesTxConnector": 0,"Idp": "IDP Config"}') + message = mt.render(mark="<134>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search earliest=-1m@m latest=+1m@m index=netproxy sourcetype=\"zscalerlss-zpa-app\" \"{{host}}\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<111>{"LogTimestamp": "Mon Mar 2 02:57:01 2020","Customer": "Acme, Inc.","SessionID": "qdLxaTYtMbsCQllNaCZ2","ConnectionID": "qdLxaTYtMbsCQllNaCZ2,aZcOpy7yN8iPncqmSuAv","InternalReason": "","ConnectionStatus": "active","IPProtocol": 6,"DoubleEncryption": 0,"Username": "nlipper@acme.com","ServicePort": 8384,"ClientPublicIP": "73.144.81.255","ClientPrivateIP": "","ClientLatitude": 42.000000,"ClientLongitude": -84.000000,"ClientCountryCode": "US","ClientZEN": "US-OH-8290","Policy": "Any Any Allow","Connector": "DFA Azure-2","ConnectorZEN": "US-OH-8290","ConnectorIP": "10.202.4.68","ConnectorPort": 35992,"Host": "10.26.1.19","Application": "DFA IP SPACE","AppGroup": "Dynamically Discovered Apps","Server": "0","ServerIP": "10.26.1.19","ServerPort": 8384,"PolicyProcessingTime": 120,"CAProcessingTime": 445,"ConnectorZENSetupTime": 46610,"ConnectionSetupTime": 47200,"ServerSetupTime": 22207,"AppLearnTime": 0,"TimestampConnectionStart": "2020-02-29T20:42:01.228Z","TimestampConnectionEnd": "","TimestampCATx": "2020-02-29T20:42:01.228Z","TimestampCARx": "2020-02-29T20:42:01.228Z","TimestampAppLearnStart": "","TimestampZENFirstRxClient": "","TimestampZENFirstTxClient": "","TimestampZENLastRxClient": "","TimestampZENLastTxClient": "","TimestampConnectorZENSetupComplete": "2020-02-29T20:42:01.275Z","TimestampZENFirstRxConnector": "","TimestampZENFirstTxConnector": "","TimestampZENLastRxConnector": "","TimestampZENLastTxConnector": "","ZENTotalBytesRxClient": 0,"ZENBytesRxClient": 0,"ZENTotalBytesTxClient": 0,"ZENBytesTxClient": 0,"ZENTotalBytesRxConnector": 0,"ZENBytesRxConnector": 0,"ZENTotalBytesTxConnector": 0,"ZENBytesTxConnector": 0,"Idp": "IDP Config"} +def test_zscaler_lss_zpa_app_pri(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{\"LogTimestamp\": \"{% now 'local', '%b %d %H:%M:%S %Y' %}" + '","Customer": "Acme, Inc.","SessionID": "qdLxaTYtMbsCQllNaCZ2","ConnectionID": "qdLxaTYtMbsCQllNaCZ2,aZcOpy7yN8iPncqmSuAv","InternalReason": "","ConnectionStatus": "active","IPProtocol": 6,"DoubleEncryption": 0,"Username": "nlipper@acme.com","ServicePort": 8384,"ClientPublicIP": "73.144.81.255","ClientPrivateIP": "","ClientLatitude": 42.000000,"ClientLongitude": -84.000000,"ClientCountryCode": "US","ClientZEN": "US-OH-8290","Policy": "Any Any Allow","Connector": "DFA Azure-2","ConnectorZEN": "US-OH-8290","ConnectorIP": "10.202.4.68","ConnectorPort": 35992,"Host": "10.26.1.19","Application": "DFA IP SPACE","AppGroup": "Dynamically Discovered Apps","Server": "0","ServerIP": "{{host}}","ServerPort": 8384,"PolicyProcessingTime": 120,"CAProcessingTime": 445,"ConnectorZENSetupTime": 46610,"ConnectionSetupTime": 47200,"ServerSetupTime": 22207,"AppLearnTime": 0,"TimestampConnectionStart": "2020-02-29T20:42:01.228Z","TimestampConnectionEnd": "","TimestampCATx": "2020-02-29T20:42:01.228Z","TimestampCARx": "2020-02-29T20:42:01.228Z","TimestampAppLearnStart": "","TimestampZENFirstRxClient": "","TimestampZENFirstTxClient": "","TimestampZENLastRxClient": "","TimestampZENLastTxClient": "","TimestampConnectorZENSetupComplete": "2020-02-29T20:42:01.275Z","TimestampZENFirstRxConnector": "","TimestampZENFirstTxConnector": "","TimestampZENLastRxConnector": "","TimestampZENLastTxConnector": "","ZENTotalBytesRxClient": 0,"ZENBytesRxClient": 0,"ZENTotalBytesTxClient": 0,"ZENBytesTxClient": 0,"ZENTotalBytesRxConnector": 0,"ZENBytesRxConnector": 0,"ZENTotalBytesTxConnector": 0,"ZENBytesTxConnector": 0,"Idp": "IDP Config"}') + message = mt.render(mark="<134>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search earliest=-1m@m latest=+1m@m index=netproxy sourcetype=\"zscalerlss-zpa-app\" \"{{host}}\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#{"LogTimestamp": "Mon Mar 2 02:57:05 2020","Customer": "Acme, Inc.","Username": "chuffma@acme.com","SessionID": "lCINpOrrZl3pGQCVYP+E","SessionStatus": "ZPN_STATUS_AUTHENTICATED","Version": "1.5.1.8.191135","ZEN": "US-IL-8706","CertificateCN": "WJJ26L69Y6bmncPqV/YRQXe17aDzRf6Z0M1n7CU7UaQ=@acme.com","PrivateIP": "","PublicIP": "174.97.166.11","Latitude": 44.000000,"Longitude": -88.000000,"CountryCode": "","TimestampAuthentication": "2020-02-27T13:04:55.000Z","TimestampUnAuthentication": "","TotalBytesRx": 46997613,"TotalBytesTx": 2232391,"Idp": "IDP Config","Hostname": "","Platform": "","ClientType": "zpn_client_type_zapp","TrustedNetworks": ,"TrustedNetworksNames": ,"SAMLAttributes": "{\"FirstName\":[\"Christopher\"],\"LastName\":[\"Huffman\"],\"Email\":[\"chuffma@acme.com\"],\"GroupName\":[\"zScaler_ZPA\"]}","PosturesHit": ,"PosturesMiss": ,"ZENLatitude": 41.000000,"ZENLongitude": -88.000000,"ZENCountryCode": "US"} +def test_zscaler_lss_zpa_bba(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{\"LogTimestamp\": \"{% now 'local', '%b %d %H:%M:%S %Y' %}" + '","ConnectionID":"6N9BHIHZrwGXJXG7q4sn,dUPdoZAgr6vJKlv588GG","Exporter":"unset","TimestampRequestReceiveStart":"2020-03-01T22:39:30.679Z","TimestampRequestReceiveHeaderFinish":"2020-03-01T22:39:30.679Z","TimestampRequestReceiveFinish":"2020-03-01T22:39:30.680Z","TimestampRequestTransmitStart":"2020-03-01T22:39:30.680Z","TimestampRequestTransmitFinish":"2020-03-02T02:28:53.277Z","TimestampResponseReceiveStart":"2020-03-01T22:39:30.707Z","TimestampResponseReceiveFinish":"2020-03-02T02:28:53.309Z","TimestampResponseTransmitStart":"2020-03-01T22:39:30.707Z","TimestampResponseTransmitFinish":"2020-03-02T02:28:51.762Z","TotalTimeRequestReceive":1193,"TotalTimeRequestTransmit":13762597414,"TotalTimeResponseReceive":13762601379,"TotalTimeResponseTransmit":13761054628,"TotalTimeConnectionSetup":1037,"TotalTimeServerResponse":-13762570100,"Method":"GET","Protocol":"HTTPS","Host":"accountman.dfamilk.com","URL":"/remoteDesktopGateway","UserAgent":"","XFF":"","NameID":"carlos.garcia.11@acme.com","StatusCode":101,"RequestSize":2246,"ResponseSize":3823185,"ApplicationPort":443,"ClientPublicIp":"162.205.86.162","ClientPublicPort":49330,"ClientPrivateIp":"","Customer":"{{host}}","ConnectionStatus":"zfce_mt_remote_disconnect","ConnectionReason":"BRK_MT_CLOSED_FROM_ASSISTANT"}') + message = mt.render(mark="<134>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search earliest=-1m@m latest=+1m@m index=netproxy sourcetype=\"zscalerlss-zpa-bba\" \"{{host}}\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +#{"LogTimestamp": "Mon Mar 2 02:51:53 2020","Customer": "Acme, Inc.","SessionID": "NNz9t5AY1Rq5dzyLbNRB","SessionType": "ZPN_ASSISTANT_BROKER_CONTROL","SessionStatus": "ZPN_STATUS_AUTHENTICATED","Version": "19.102.2","Platform": "el7","ZEN": "US-NY-8180","Connector": "St Albans-1","ConnectorGroup": "St Albans Connector","PrivateIP": "192.168.16.15","PublicIP": "184.80.224.186","Latitude": 44.000000,"Longitude": -73.000000,"CountryCode": "","TimestampAuthentication": "2020-02-27T07:03:53.689Z","TimestampUnAuthentication": "","CPUUtilization": 1,"MemUtilization": 16,"ServiceCount": 0,"InterfaceDefRoute": "eth0","DefRouteGW": "192.168.16.1","PrimaryDNSResolver": "192.168.16.16","HostUpTime": "1572630032","ConnectorUpTime": "1579500006","NumOfInterfaces": 2,"BytesRxInterface": 63778867197,"PacketsRxInterface": 669441337,"ErrorsRxInterface": 0,"DiscardsRxInterface": 1181261,"BytesTxInterface": 50473462713,"PacketsTxInterface": 492668679,"ErrorsTxInterface": 0,"DiscardsTxInterface": 0,"TotalBytesRx": 6979022,"TotalBytesTx": 47705494} +def test_zscaler_lss_zpa_connector(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{\"LogTimestamp\": \"{% now 'local', '%b %d %H:%M:%S %Y' %}" + '","Customer": "{{host}}","SessionID": "NNz9t5AY1Rq5dzyLbNRB","SessionType": "ZPN_ASSISTANT_BROKER_CONTROL","SessionStatus": "ZPN_STATUS_AUTHENTICATED","Version": "19.102.2","Platform": "el7","ZEN": "US-NY-8180","Connector": "St Albans-1","ConnectorGroup": "St Albans Connector","PrivateIP": "192.168.16.15","PublicIP": "184.80.224.186","Latitude": 44.000000,"Longitude": -73.000000,"CountryCode": "","TimestampAuthentication": "2020-02-27T07:03:53.689Z","TimestampUnAuthentication": "","CPUUtilization": 1,"MemUtilization": 16,"ServiceCount": 0,"InterfaceDefRoute": "eth0","DefRouteGW": "192.168.16.1","PrimaryDNSResolver": "192.168.16.16","HostUpTime": "1572630032","ConnectorUpTime": "1579500006","NumOfInterfaces": 2,"BytesRxInterface": 63778867197,"PacketsRxInterface": 669441337,"ErrorsRxInterface": 0,"DiscardsRxInterface": 1181261,"BytesTxInterface": 50473462713,"PacketsTxInterface": 492668679,"ErrorsTxInterface": 0,"DiscardsTxInterface": 0,"TotalBytesRx": 6979022,"TotalBytesTx": 47705494}') + message = mt.render(mark="<134>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search earliest=-1m@m latest=+1m@m index=netproxy sourcetype=\"zscalerlss-zpa-connector\" \"{{host}}\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +#{"LogTimestamp": "Fri May 31 17:34:48 2019","Customer": "ANZ Team/zdemo in beta","Username": "ZPA LSS Client","SessionID": "cKgzUERSLl09Y+ytH8v5","SessionStatus": "ZPN_STATUS_AUTHENTICATED","Version": "19.12.0-36-g87dad18","ZEN": "broker1b.pdx2","CertificateCN": "slogger1b.pdx2.zpabeta.net","PrivateIP": "","PublicIP": "34.216.108.5","Latitude": 45.000000,"Longitude": -119.000000,"CountryCode": "US","TimestampAuthentication": "2019-05-29T21:18:38.000Z","TimestampUnAuthentication": "","TotalBytesRx": 31274866,"TotalBytesTx": 25424152,"Idp": "Example IDP Config","Hostname": "DESKTOP-2K299HC","Platform": "windows","ClientType": "zpn_client_type_zapp","TrustedNetworks": "TN1_stc1","TrustedNetworksNames": "145248739466947538","SAMLAttributes": "myname:jdoe,myemail:jdoe@zscaler.com","PosturesHit": "sm-posture1,sm-posture2","PosturesMisses": "sm-posture11,sm-posture12","ZENLatitude": 47.000000,"ZENLongitude": -122.000000,"ZENCountryCode": ""} +def test_zscaler_lss_zpa_auth(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{\"LogTimestamp\": \"{% now 'local', '%b %d %H:%M:%S %Y' %}" + '","Customer": "{{host}}","Username": "ZPA LSS Client","SessionID": "cKgzUERSLl09Y+ytH8v5","SessionStatus": "ZPN_STATUS_AUTHENTICATED","Version": "19.12.0-36-g87dad18","ZEN": "broker1b.pdx2","CertificateCN": "slogger1b.pdx2.zpabeta.net","PrivateIP": "","PublicIP": "34.216.108.5","Latitude": 45.000000,"Longitude": -119.000000,"CountryCode": "US","TimestampAuthentication": "2019-05-29T21:18:38.000Z","TimestampUnAuthentication": "","TotalBytesRx": 31274866,"TotalBytesTx": 25424152,"Idp": "Example IDP Config","Hostname": "DESKTOP-2K299HC","Platform": "windows","ClientType": "zpn_client_type_zapp","TrustedNetworks": "TN1_stc1","TrustedNetworksNames": "145248739466947538","SAMLAttributes": "myname:jdoe,myemail:jdoe@zscaler.com","PosturesHit": "sm-posture1,sm-posture2","PosturesMisses": "sm-posture11,sm-posture12","ZENLatitude": 47.000000,"ZENLongitude": -122.000000,"ZENCountryCode": ""}') + message = mt.render(mark="<134>", host=host) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search earliest=-1m@m latest=+1m@m index=netauth sourcetype=\"zscalerlss-zpa-auth\" \"{{host}}\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From 6aadb8b391866623942155b1de77be8776202ffa Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Thu, 5 Mar 2020 15:53:45 -0500 Subject: [PATCH 4/4] Update lp-zscaler_lss.conf.tmpl --- package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl index ca822aa..3df79f6 100644 --- a/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zscaler_lss.conf.tmpl @@ -1,7 +1,7 @@ # Zscaler {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "ZSCALER_LSS" "parser" "rfc3164" }} +{{- $context := dict "port_id" "ZSCALER_LSS" "parser" "common" }} {{- tmpl.Exec "t/source_network.t" $context }} log {