From c890d6ae9f72d3db83feced5529834befa98819d Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 20 Jan 2020 21:30:43 -0800 Subject: [PATCH 1/2] Update network source template/log path unique port parsers * Cisco ACS: Replace "CISCO_NX_OS" with "CISCO_ACS" in environment variables * Log Paths: specify unique port gomplate dict parser value for network source template (replace "common" with appropriate value) * Network Source Template: reorder "soup" parser tree * Network Source Template: Add `guess-timezone` flag for direct rfc3164 parser * Network Source Template: Comment out format tests in direct parser entries * Rename Meraki log path (remove "epoch" prefix) * Vmware vsphere filter: minor format change * syslog_format.conf: Remove unused "epochtime" filters * docker-compose.yml: Add listen ports to test unique port gomplate templating code --- docker-compose.yml | 7 +++- .../conf.d/conflib/_common/syslog_format.conf | 6 --- .../etc/conf.d/filters/VMware/vsphere.conf | 3 +- .../log_paths/lp-checkpoint_splunk.conf.tmpl | 2 +- .../conf.d/log_paths/lp-cisco_acs.conf.tmpl | 10 ++--- .../conf.d/log_paths/lp-cisco_asa.conf.tmpl | 2 +- .../log_paths/lp-cisco_asa_legacy.conf.tmpl | 2 +- .../conf.d/log_paths/lp-cisco_ios.conf.tmpl | 2 +- .../conf.d/log_paths/lp-cisco_ise.conf.tmpl | 2 +- ...ki.conf.tmpl => lp-cisco_meraki.conf.tmpl} | 2 +- .../conf.d/log_paths/lp-cisco_nxos.conf.tmpl | 2 +- .../lp-forcepoint_webprotect.conf.tmpl | 2 +- .../log_paths/lp-fortinet_fortios.conf.tmpl | 2 +- .../conf.d/log_paths/lp-infoblox.conf.tmpl | 2 +- .../conf.d/log_paths/lp-juniper_idp.conf.tmpl | 2 +- .../log_paths/lp-juniper_junos.conf.tmpl | 2 +- .../lp-juniper_junos_structured.conf.tmpl | 2 +- .../log_paths/lp-juniper_netscreen.conf.tmpl | 2 +- .../conf.d/log_paths/lp-juniper_nsm.conf.tmpl | 2 +- .../log_paths/lp-juniper_nsm_idp.conf.tmpl | 2 +- .../lp-microfocus_arcsight.conf.tmpl | 2 +- .../log_paths/lp-paloalto_panos.conf.tmpl | 2 +- .../log_paths/lp-proofpoint_pps.conf.tmpl | 2 +- .../lp-symantec_brightmail.conf.tmpl | 2 +- .../log_paths/lp-symantec_proxy.conf.tmpl | 2 +- .../log_paths/lp-ubiquiti_unifi.conf.tmpl | 2 +- .../conf.d/log_paths/lp-zscaler_nss.conf.tmpl | 2 +- .../log_paths/lp-zzy-nix_syslog.conf.tmpl | 2 +- package/etc/go_templates/source_network.t | 40 +++++++++++-------- 29 files changed, 60 insertions(+), 54 deletions(-) rename package/etc/conf.d/log_paths/{lp-epoch-cisco_meraki.conf.tmpl => lp-cisco_meraki.conf.tmpl} (94%) diff --git a/docker-compose.yml b/docker-compose.yml index 3e3bc3d..3d936bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,8 +48,11 @@ services: # - SC4S_LISTEN_DEFAULT_UDP_PORT=514 # - SC4S_LISTEN_DEFAULT_TLS_PORT=6514 - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 - - SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT=6000 -# - SC4S_ARCHIVE_CHECKPOINT=yes + - SC4S_LISTEN_CISCO_ASA_TCP_PORT=5001 + - SC4S_LISTEN_CISCO_IOS_TCP_PORT=5002 + - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003 + - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004 + - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005 - SC4S_ARCHIVE_GLOBAL=yes volumes: - ./tls:/opt/syslog-ng/tls diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index b8802a7..0c9fc2f 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -4,9 +4,6 @@ filter f_rfc5424_strict{ filter f_rfc5424_noversion{ message('^(?(?
(?<\d{1,3}>) ?(?(?(?\d{4})-(?\d\d)-(?\d\d))T(?(?(?[0-2]\d):(?[0-5]\d):(?[0-5]\d)(?:.(?\d{1,6}))?)(?Z|(?[+\-][0-2]\d:[0-5]\d))))))'); }; -filter f_rfc5424_epochtime{ - message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) ))'); -}; filter f_rfc3164_version{ message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?[^ ]+) ))'); }; @@ -28,9 +25,6 @@ filter f_is_rfc5424_noversion{ rewrite set_rfc5424_epochtime{ set("rfc5424_epochtime" value("fields.sc4s_syslog_format")); }; -filter f_is_rfc5424_epochtime{ - match("rfc5424_epochtime" value("fields.sc4s_syslog_format")) -}; rewrite set_rfc3164_version{ set("rfc3164_version" value("fields.sc4s_syslog_format")); }; diff --git a/package/etc/conf.d/filters/VMware/vsphere.conf b/package/etc/conf.d/filters/VMware/vsphere.conf index 624c6e0..a2378aa 100644 --- a/package/etc/conf.d/filters/VMware/vsphere.conf +++ b/package/etc/conf.d/filters/VMware/vsphere.conf @@ -23,7 +23,8 @@ filter f_vmware_all { or program("NSX", flags(ignore-case)) or program("NSXV", flags(ignore-case)) or program("dfwpktlogs", flags(ignore-case)) - or program("nsx-.*", flags(ignore-case))}; + or program("nsx-.*", flags(ignore-case)) +}; filter f_vmware_vsphere { program("cimslp", flags(ignore-case)) diff --git a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl index fcfc452..bddcbd3 100644 --- a/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-checkpoint_splunk.conf.tmpl @@ -1,6 +1,6 @@ # Checkpoint {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }} +{{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl index 9397bc0..817df0a 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_acs.conf.tmpl @@ -1,6 +1,6 @@ # Cisco ACS {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} +{{- $context := dict "port_id" "CISCO_ACS" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} # This filter uses a field we set to prevent the original messages before aggregation from being @@ -49,15 +49,15 @@ parser acs_event_time { log { junction { -{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT")) }} +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_ACS_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_ACS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_ACS_TLS_PORT")) }} channel { - # Listen on the specified dedicated port(s) for CISCO_NX_OS traffic - source (s_CISCO_NX_OS); + # Listen on the specified dedicated port(s) for CISCO_ACS traffic + source (s_CISCO_ACS); flags (final); }; {{- end}} channel { - # Listen on the default port (typically 514) for CISCO_NX_OS traffic + # Listen on the default port (typically 514) for CISCO_ACS traffic source (s_DEFAULT); filter(f_is_rfc3164); filter(f_cisco_acs); diff --git a/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl index e09f3cf..9045821 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_asa.conf.tmpl @@ -1,6 +1,6 @@ # Cisco ASA RFC5424 {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "CISCO_ASA" "parser" "common" }} +{{- $context := dict "port_id" "CISCO_ASA" "parser" "rfc5424_noversion" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl index b027c3d..0749e68 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_asa_legacy.conf.tmpl @@ -1,6 +1,6 @@ # Cisco ASA {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common" }} +{{- $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl index 63cde5e..80aae01 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_ios.conf.tmpl @@ -1,6 +1,6 @@ # Cisco IOS {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "CISCO_IOS" "parser" "common" }} +{{- $context := dict "port_id" "CISCO_IOS" "parser" "cisco_parser" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl index 818ada8..9a5bf0f 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_ise.conf.tmpl @@ -1,6 +1,6 @@ # Cisco ISE {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "CISCO_ISE" "parser" "common" }} +{{- $context := dict "port_id" "CISCO_ISE" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} # This filter uses a field we set to prevent the original messages before aggregation from being diff --git a/package/etc/conf.d/log_paths/lp-epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_meraki.conf.tmpl similarity index 94% rename from package/etc/conf.d/log_paths/lp-epoch-cisco_meraki.conf.tmpl rename to package/etc/conf.d/log_paths/lp-cisco_meraki.conf.tmpl index 974ee36..c0f37ce 100644 --- a/package/etc/conf.d/log_paths/lp-epoch-cisco_meraki.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_meraki.conf.tmpl @@ -1,6 +1,6 @@ # Cisco Meraki {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} +{{- $context := dict "port_id" "CISCO_MERAKI" "parser" "cisco_meraki_parser" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl index 66bbc5e..8bf2d2b 100644 --- a/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-cisco_nxos.conf.tmpl @@ -1,6 +1,6 @@ # Cisco NX_OS {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} +{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/lp-forcepoint_webprotect.conf.tmpl index 969a245..3b55bfa 100644 --- a/package/etc/conf.d/log_paths/lp-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-forcepoint_webprotect.conf.tmpl @@ -1,6 +1,6 @@ # Forcepoint Webprotect {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }} +{{- $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/lp-fortinet_fortios.conf.tmpl index 943bb19..aba6936 100644 --- a/package/etc/conf.d/log_paths/lp-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-fortinet_fortios.conf.tmpl @@ -1,6 +1,6 @@ # Fortinet Fortios {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "FORTINET_FORTIOS" "parser" "common" }} +{{- $context := dict "port_id" "FORTINET_FORTIOS" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-infoblox.conf.tmpl b/package/etc/conf.d/log_paths/lp-infoblox.conf.tmpl index 991b679..8b40188 100644 --- a/package/etc/conf.d/log_paths/lp-infoblox.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-infoblox.conf.tmpl @@ -1,6 +1,6 @@ # Infoblox {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "INFOBLOX" "parser" "common" }} +{{- $context := dict "port_id" "INFOBLOX" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_idp.conf.tmpl index 9149f94..436963a 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_idp.conf.tmpl @@ -1,6 +1,6 @@ # Juniper IDP {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "JUNIPER_IDP" "parser" "common" }} +{{- $context := dict "port_id" "JUNIPER_IDP" "parser" "rfc5424_strict" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_junos.conf.tmpl index df12ce7..05d7e5d 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_junos.conf.tmpl @@ -1,6 +1,6 @@ # Juniper JunOS {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "JUNIPER_JUNOS" "parser" "common" }} +{{- $context := dict "port_id" "JUNIPER_JUNOS" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl index 5501976..d5ae714 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_junos_structured.conf.tmpl @@ -1,6 +1,6 @@ # Juniper JunOS (Structured, RFC5424-compliant) {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "JUNIPER_JUNOS_STRUCTURED" "parser" "common" }} +{{- $context := dict "port_id" "JUNIPER_JUNOS_STRUCTURED" "parser" "rfc5424_strict" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_netscreen.conf.tmpl index 27cc1e3..74c40b0 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_netscreen.conf.tmpl @@ -1,6 +1,6 @@ # Juniper Netscreen {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "JUNIPER_NETSCREEN" "parser" "common" }} +{{- $context := dict "port_id" "JUNIPER_NETSCREEN" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_nsm.conf.tmpl index ce99b6c..650b7c9 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_nsm.conf.tmpl @@ -1,6 +1,6 @@ # Juniper NSM {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "JUNIPER_NSM" "parser" "common" }} +{{- $context := dict "port_id" "JUNIPER_NSM" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/lp-juniper_nsm_idp.conf.tmpl index 7a2340c..e76fb0a 100644 --- a/package/etc/conf.d/log_paths/lp-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-juniper_nsm_idp.conf.tmpl @@ -1,6 +1,6 @@ # Juniper NSM IDP {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "JUNIPER_NSM_IDP" "parser" "common" }} +{{- $context := dict "port_id" "JUNIPER_NSM_IDP" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl index 6f22c7b..ae04a47 100644 --- a/package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-microfocus_arcsight.conf.tmpl @@ -1,6 +1,6 @@ # Microfocus ArcSight {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common" }} +{{- $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} parser p_microfocus_arcsight_header { diff --git a/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl index 72b68a6..f115db2 100644 --- a/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-paloalto_panos.conf.tmpl @@ -1,6 +1,6 @@ # PaloAlto PanOS {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "PALOALTO_PANOS" "parser" "common" }} +{{- $context := dict "port_id" "PALOALTO_PANOS" "parser" "rfc3164_version" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-proofpoint_pps.conf.tmpl b/package/etc/conf.d/log_paths/lp-proofpoint_pps.conf.tmpl index e73ada4..dc911ec 100644 --- a/package/etc/conf.d/log_paths/lp-proofpoint_pps.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-proofpoint_pps.conf.tmpl @@ -1,6 +1,6 @@ # Proofpoint Protection Server {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "PROOFPOINT_PPS" "parser" "common" }} +{{- $context := dict "port_id" "PROOFPOINT_PPS" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/lp-symantec_brightmail.conf.tmpl index 8033b87..8ae5329 100644 --- a/package/etc/conf.d/log_paths/lp-symantec_brightmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-symantec_brightmail.conf.tmpl @@ -24,7 +24,7 @@ parser symantec_brightmail_grouping { {{- end }} {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "common" }} +{{- $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl index 6ec7134..cc3524d 100644 --- a/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-symantec_proxy.conf.tmpl @@ -1,6 +1,6 @@ # Symantec Proxy (Bluecoat) {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "SYMANTEC_PROXY" "parser" "common" }} +{{- $context := dict "port_id" "SYMANTEC_PROXY" "parser" "rfc5424_noversion" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/lp-ubiquiti_unifi.conf.tmpl index 903f51b..cee88ad 100644 --- a/package/etc/conf.d/log_paths/lp-ubiquiti_unifi.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-ubiquiti_unifi.conf.tmpl @@ -1,6 +1,6 @@ # Ubiquiti unifi {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "common" }} +{{- $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl index 29c0717..17782bf 100644 --- a/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zscaler_nss.conf.tmpl @@ -1,6 +1,6 @@ # Zscaler {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} +{{- $context := dict "port_id" "ZSCALER_NSS" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/conf.d/log_paths/lp-zzy-nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/lp-zzy-nix_syslog.conf.tmpl index 008a3f6..be65104 100644 --- a/package/etc/conf.d/log_paths/lp-zzy-nix_syslog.conf.tmpl +++ b/package/etc/conf.d/log_paths/lp-zzy-nix_syslog.conf.tmpl @@ -1,6 +1,6 @@ # Linux/Unix OS system logs {{- /* The following provides a unique port source configuration if env var(s) are set */}} -{{- $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }} +{{- $context := dict "port_id" "NIX_SYSLOG" "parser" "rfc3164" }} {{- tmpl.Exec "t/source_network.t" $context }} log { diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index eef481a..6b54feb 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -60,14 +60,26 @@ source s_{{ .port_id }} { }; #TODO: #60 Remove this function with enhancement rewrite(set_rfcnonconformant); -{{- if eq .parser "rfc5424_strict" }} - filter(f_rfc5424_strict); +{{- if eq .parser "rfc3164" }} + parser { + syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + }; + rewrite(set_rfc3164); +{{- else if eq .parser "rfc3164_version" }} +# filter(f_rfc3164_version); + rewrite(set_rfc3164_no_version_string); + parser { + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + }; + rewrite(set_rfc3164_version); +{{- else if eq .parser "rfc5424_strict" }} +# filter(f_rfc5424_strict); parser { syslog-parser(flags(syslog-protocol)); }; rewrite(set_rfc5424_strict); {{- else if eq .parser "rfc5424_noversion" }} - filter(f_rfc5424_noversion); +# filter(f_rfc5424_noversion); parser { syslog-parser(flags(syslog-protocol)); }; @@ -78,15 +90,18 @@ source s_{{ .port_id }} { {{- else if eq .parser "cisco_meraki_parser" }} parser (p_cisco_meraki); rewrite(set_rfc5424_epochtime); -{{- else if eq .parser "rfc3164" }} - parser { - syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) ); - }; - rewrite(set_rfc3164); {{- else if eq .parser "no_parse" }} rewrite(set_no_parse); {{- else }} - if {filter(f_rfc5424_strict); + if { + filter(f_rfc3164_version); + rewrite(set_rfc3164_no_version_string); + parser { + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + }; + rewrite(set_rfc3164_version); + } elif { + filter(f_rfc5424_strict); parser { syslog-parser(flags(syslog-protocol)); }; @@ -100,13 +115,6 @@ source s_{{ .port_id }} { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); - } elif { - filter(f_rfc3164_version); - rewrite(set_rfc3164_no_version_string); - parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); - }; - rewrite(set_rfc3164_version); } elif { parser (p_cisco_meraki); rewrite(set_rfc5424_epochtime); From 78dda6e514597da59694ec4ea4ea86c55239c2e0 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 20 Jan 2020 22:37:30 -0800 Subject: [PATCH 2/2] Remove unneeded `rewrite(set_rfcnonconformant)` function * syslog_format.conf: Remove unneeded `rewrite(set_rfcnonconformant)` funtction. Source template will set an appropriate syslog format field in every case. * source_network.t: Remove comment `#TODO: #60 Remove this function with enhancement`; code has been updated and/or removed and comment no longer relevant. * source_network.t: Adjust gomplate template for line spacing in conf file output --- .../conf.d/conflib/_common/syslog_format.conf | 3 --- package/etc/go_templates/source_network.t | 21 ++++++++----------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index 0c9fc2f..b461e0e 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -7,9 +7,6 @@ filter f_rfc5424_noversion{ filter f_rfc3164_version{ message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?[^ ]+) ))'); }; -rewrite set_rfcnonconformant{ - set("rfc5424_nonconform" value("fields.sc4s_syslog_format")); -}; rewrite set_rfc5424_strict{ set("rfc5424_strict" value("fields.sc4s_syslog_format")); }; diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 6b54feb..480130a 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -58,41 +58,39 @@ source s_{{ .port_id }} { ); {{- end}} }; - #TODO: #60 Remove this function with enhancement - rewrite(set_rfcnonconformant); -{{- if eq .parser "rfc3164" }} +{{ if eq .parser "rfc3164" }} parser { syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); }; rewrite(set_rfc3164); -{{- else if eq .parser "rfc3164_version" }} +{{ else if eq .parser "rfc3164_version" }} # filter(f_rfc3164_version); rewrite(set_rfc3164_no_version_string); parser { syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); }; rewrite(set_rfc3164_version); -{{- else if eq .parser "rfc5424_strict" }} +{{ else if eq .parser "rfc5424_strict" }} # filter(f_rfc5424_strict); parser { syslog-parser(flags(syslog-protocol)); }; rewrite(set_rfc5424_strict); -{{- else if eq .parser "rfc5424_noversion" }} +{{ else if eq .parser "rfc5424_noversion" }} # filter(f_rfc5424_noversion); parser { syslog-parser(flags(syslog-protocol)); }; rewrite(set_rfc5424_noversion); -{{- else if eq .parser "cisco_parser" }} +{{ else if eq .parser "cisco_parser" }} parser {cisco-parser()}; rewrite(set_cisco_ios); -{{- else if eq .parser "cisco_meraki_parser" }} +{{ else if eq .parser "cisco_meraki_parser" }} parser (p_cisco_meraki); rewrite(set_rfc5424_epochtime); -{{- else if eq .parser "no_parse" }} +{{ else if eq .parser "no_parse" }} rewrite(set_no_parse); -{{- else }} +{{ else }} if { filter(f_rfc3164_version); rewrite(set_rfc3164_no_version_string); @@ -124,9 +122,8 @@ source s_{{ .port_id }} { }; rewrite(set_rfc3164); }; -{{- end }} +{{ end }} rewrite(r_set_splunk_default); - parser { vendor_product_by_source(); };