From 086d7d82925459b4534879577207774ec6ce66ff Mon Sep 17 00:00:00 2001 From: mbonsack Date: Fri, 6 Dec 2019 04:46:05 -0800 Subject: [PATCH] Combine proofpoint log paths (#218) * Combine proofpoint log paths * Update sources.md for Proofpoint and fix random notes for UDP unique ports --- docs/sources.md | 11 ++-- .../p_rfc3164-proofpoint_pps.conf.tmpl | 61 +++++++++++++++++++ .../p_rfc3164-proofpoint_pps_filter.conf.tmpl | 52 ---------------- ..._rfc3164-proofpoint_pps_sendmail.conf.tmpl | 51 ---------------- 4 files changed, 66 insertions(+), 109 deletions(-) create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl diff --git a/docs/sources.md b/docs/sources.md index 22c3aab..3d868cc 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -225,7 +225,7 @@ PATTERN MATCH | Variable | default | description | |----------------|----------------|----------------| | SC4S_LISTEN_CISCO_ISE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | -| SC4S_LISTEN_CISCO_ISE_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_ISE_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined expecting RFC5424 format | ### Verification @@ -273,7 +273,7 @@ IP, Netmask, Host or Port | Variable | default | description | |----------------|----------------|----------------| | SC4S_LISTEN_CISCO_MERAKI_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | -| SC4S_LISTEN_CISCO_MERAKI_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_MERAKI_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined expecting RFC5424 format | ### Verification @@ -579,7 +579,7 @@ Verify timestamp, and host values match as expected | Variable | default | description | |----------------|----------------|----------------| | SC4S_LISTEN_JUNIPER_NSM_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | -| SC4S_LISTEN_JUNIPER_NSM_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_JUNIPER_NSM_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | ### Verification @@ -629,7 +629,7 @@ Verify timestamp, and host values match as expected | Variable | default | description | |----------------|----------------|----------------| | SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | -| SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | ### Verification @@ -887,8 +887,7 @@ messages to create meaningful final output. This will require follow-on process | Variable | default | description | |----------------|----------------|----------------| -| SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined. If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT variable immediately below.| -| SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined. If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT variable immediately above. | +| SC4S_PROOFPOINT_PPS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined. | ### Verification diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl new file mode 100644 index 0000000..1081d60 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl @@ -0,0 +1,61 @@ +# Proofpoint Protection Server +{{ $context := dict "port_id" "PROOFPOINT_PPS" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_DEFAULT); + filter { filter(f_proofpoint_pps_filter) or filter(f_proofpoint_pps_sendmail) }; +{{- end }} +{{- if eq (.) "no" }} + source (s_PROOFPOINT_PPS); +{{- end }} + + if (filter(f_proofpoint_pps_filter)) { + rewrite { + set("proofpoint_pps_filter", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"))}; + parser { + p_add_context_splunk(key("proofpoint_pps_filter")); + }; + } else { + rewrite { + set("proofpoint_pps_sendmail", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"))}; + parser { + p_add_context_splunk(key("proofpoint_pps_sendmail")); + }; + }; + + parser (compliance_meta_by_source); + + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS") }} + destination(d_archive); +{{- end}} + + flags(flow-control); +}; +{{- end}} + +{{- if or (or (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_TCP_PORT")) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for PROOFPOINT_PPS traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for PROOFPOINT_PPS traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl deleted file mode 100644 index 09dd410..0000000 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl +++ /dev/null @@ -1,52 +0,0 @@ -# Proofpoint -{{ $context := dict "port_id" "PROOFPOINT_PPS_FILTER" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} -log { -{{- if eq (.) "yes" }} - source(s_DEFAULT); - filter(f_proofpoint_pps_filter); -{{- end }} -{{- if eq (.) "no" }} - source (s_PROOFPOINT_PPS_FILTER); -{{- end }} - - rewrite { - set("proofpoint_pps_filter", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"))}; - parser { - p_add_context_splunk(key("proofpoint_pps_filter")); - }; - - parser (compliance_meta_by_source); - - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; - - destination(d_hec); - -{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS") }} - destination(d_archive); -{{- end}} - - flags(flow-control); -}; -{{- end}} - -{{- if or (or (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT")) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT")) }} -# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_FILTER traffic - {{ tmpl.Exec "log_path" "no" }} -{{- end}} - -# Listen on the default port (typically 514) for PROOFPOINT_PPS_FILTER traffic -{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl deleted file mode 100644 index 694f14e..0000000 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl +++ /dev/null @@ -1,51 +0,0 @@ -# Proofpoint -{{ $context := dict "port_id" "PROOFPOINT_PPS_SENDMAIL" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} -log { -{{- if eq (.) "yes" }} - source(s_DEFAULT); - filter(f_proofpoint_pps_sendmail); -{{- end }} -{{- if eq (.) "no" }} - source (s_PROOFPOINT_PPS_SENDMAIL); -{{- end }} - - rewrite { - set("proofpoint_pps_sendmail", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"))}; - parser { - p_add_context_splunk(key("proofpoint_pps_sendmail")); - }; - - parser (compliance_meta_by_source); - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; - - destination(d_hec); - -{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS_SENDMAIL") }} - destination(d_archive); -{{- end}} - - flags(flow-control); -}; -{{- end}} - -{{- if or (or (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT")) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT")) }} -# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_SENDMAIL traffic - {{ tmpl.Exec "log_path" "no" }} -{{- end}} - -# Listen on the default port (typically 514) for PROOFPOINT_PPS_SENDMAIL traffic -{{ tmpl.Exec "log_path" "yes" }}