Fix/parser metadata (#100)

* Parser/metdata update * Feature/proofpoint (#97) * Add Proofpoint PPS filter support * Fix parser/metdata/IOS filter
CSVD · Sep 28, 2019 · dc6f95a · dc6f95a
1 parent 1a4408c
commit dc6f95a
Show file tree

Hide file tree

Showing 18 changed files with 249 additions and 34 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -47,7 +47,7 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
-      - SC4S_SOURCE_TLS_ENABLE=yes
+      - SC4S_SOURCE_TLS_ENABLE=no
       - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
       - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
     volumes:

diff --git a/docs/sources.md b/docs/sources.md
@@ -493,6 +493,58 @@ An active firewall will generate frequent events. Use the following search to va
 index=<asconfigured> sourcetype=pan:*| stats count by host
 ```
 
+# Vendor - Proofpoint
+
+## Product - Proofpoint Protection Server
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/3080/                                                                 |
+| Product Manual | https://proofpointcommunities.force.com/community/s/article/Remote-Syslog-Forwarding                    |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| pps_filter_log |                                                                               |
+|  pps_mail_log  | This sourcetype will conflict with sendmail itself, so will require that the PPS send syslog on a dedicated port or be uniquely identifiable with a hostname glob or CIDR block if this sourcetype is desired for PPS.   |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| proofpoint_pps_filter        | pps_filter_log       | email          | none          |
+| proofpoint_pps_sendmail      | pps_mail_log       | email          | none          |
+
+
+### Filter type
+
+MSG Parse: This filter parses message content
+* NOTE:  This filter will simply parse the syslog message itself, and will _not_ perform the (required) re-assembly of related
+messages to create meaningful final output.  This will require follow-on processing in Splunk.
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Follow vendor configuration steps per referenced Product Manual
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined.   If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT variable immediately below.|
+| SC4S_PROOFPOINT_PPS_MAIL_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined. If this option is used to ensure PPS sendmail sourcetype uniqueness (see above), set the same port number for this and the SC4S_PROOFPOINT_PPS_FILTER_TCP_PORT variable immediately above. |
+
+### Verification
+
+One or two sourcetypes are included in Proofpoint PPS logs.  The search below will surface both of them:
+
+```
+index=<asconfigured> sourcetype=pps_*_log | stats count by host
+```
+
 # Vendor - Symantec
 
 ## Product - ProxySG/ASG (Bluecoat)

diff --git a/package/etc/conf.d/conflib/_common/network.conf.tmpl b/package/etc/conf.d/conflib/_common/network.conf.tmpl
diff --git a/...tc/conf.d/conflib/_common/rfc_syslog.conf → ...conf.d/conflib/_common/syslog_format.conf b/...tc/conf.d/conflib/_common/rfc_syslog.conf → ...conf.d/conflib/_common/syslog_format.conf
@@ -24,4 +24,16 @@ rewrite set_rfc3164{
 };
 filter f_is_rfc3164{
     match("rfc3164" value("fields.sc4s_syslog_format"))
+};
+rewrite set_cisco_ios{
+    set("cisco_ios" value("fields.sc4s_syslog_format"));
+};
+filter f_is_cisco_ios{
+    match("cisco_ios" value("fields.sc4s_syslog_format"))
+};
+rewrite set_no_parse{
+    set("no_parse" value("fields.sc4s_syslog_format"));
+};
+filter f_is_no_parse{
+    match("no_parse" value("fields.sc4s_syslog_format"))
 };
diff --git a/package/etc/conf.d/conflib/_common/utility.conf b/package/etc/conf.d/conflib/_common/utility.conf
diff --git a/package/etc/conf.d/filters/cisco/ios.conf b/package/etc/conf.d/filters/cisco/ios.conf
@@ -1,8 +1,6 @@
-rewrite set_metadata_vendor_product_cisco_ios{
-    set("cisco_ios" value(".metadata.vendor_product"));
-};
+# In general this will not be used; parser setting will override the need for this
 
 filter f_cisco_ios{
-    match("cisco_ios", value(".metadata.vendor_product") type(glob));
+    match("cisco_ios", value("fields.sc4s_vendor_product") type(glob));
 };
 
diff --git a/package/etc/conf.d/filters/proofpoint/pps.conf b/package/etc/conf.d/filters/proofpoint/pps.conf
@@ -0,0 +1,18 @@
+# Proofpoint
+
+filter f_proofpoint_pps_filter {
+    match("proofpoint_pps_filter", value("fields.sc4s_vendor_product") type(glob)) or
+    (
+        (
+        match('^(background|cvt|filter|pps)_instance\d+$' value("PROGRAM") type("pcre")) or
+        match('^\/opt\/proofpoint\/pps-\d\.\d\.\d\.\d+\/\S' value("PROGRAM") type("pcre")) or
+        match('^queued-(alert|default|reinject|released)$' value("PROGRAM") type("pcre"))
+        ) and
+        match('^rprt\s' value(MESSAGE) type("pcre"))
+    );
+};
+
+filter f_proofpoint_pps_sendmail {
+    match('sendmail' value("PROGRAM") type("pcre")) and
+    match("proofpoint_pps_sendmail", value("fields.sc4s_vendor_product") type(glob));
+};
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl
@@ -7,7 +7,7 @@
 log {
 {{- if eq (.) "yes" }}
     source(s_default-ports);
-    filter(f_cisco_ios);
+    filter(f_is_cisco_ios);
 {{- end }}
 {{- if eq (.) "no" }}
     source (s_dedicated_port_CISCO_IOS);

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl
@@ -5,17 +5,17 @@
 {{- end -}}
 {{ define "log_path" }}
 log {
-{{- if eq (.) "yes"}}
+{{- if eq (.) "yes" }}
     source(s_default-ports);
     filter(f_cisco_nx_os);
-{{- end}}
-{{- if eq (.) "no"}}
+{{- end }}
+{{- if eq (.) "no" }}
     source (s_dedicated_port_CISCO_NX_OS);
-{{- end}}
+{{- end }}
 
     rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))};
     parser {
-            p_add_context_splunk(key("cisco_nx_os"));
+        p_add_context_splunk(key("cisco_nx_os"));
     };
 
     destination(d_hec);  #--HEC--

diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl
@@ -0,0 +1,32 @@
+# Proofpoint
+{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }}
+{{ $context := dict "port_id" "PROOFPOINT_PPS_FILTER" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+{{- end -}}
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes" }}
+    source(s_default-ports);
+    filter(f_proofpoint_pps_filter);
+{{- end }}
+{{- if eq (.) "no" }}
+    source (s_dedicated_port_PROOFPOINT_PPS_FILTER);
+{{- end }}
+
+    rewrite { r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"), template("t_msg_only"))};
+    parser {
+        p_add_context_splunk(key("proofpoint_pps_filter"));
+    };
+
+    destination(d_hec);  #--HEC--
+
+    flags(flow-control);
+};
+{{- end}}
+{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }}
+# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_FILTER traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for PROOFPOINT_PPS_FILTER traffic
+{{ tmpl.Exec "log_path" "yes" }}
diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl
@@ -0,0 +1,32 @@
+# Proofpoint
+{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }}
+{{ $context := dict "port_id" "PROOFPOINT_PPS_SENDMAIL" "parser" "common" }}
+{{ tmpl.Exec "t/source_network.t" $context }}
+{{- end -}}
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes" }}
+    source(s_default-ports);
+    filter(f_proofpoint_pps_sendmail);
+{{- end }}
+{{- if eq (.) "no" }}
+    source (s_dedicated_port_PROOFPOINT_PPS_SENDMAIL);
+{{- end }}
+
+    rewrite { r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"), template("t_msg_only"))};
+    parser {
+        p_add_context_splunk(key("proofpoint_pps_sendmail"));
+    };
+
+    destination(d_hec);  #--HEC--
+
+    flags(flow-control);
+};
+{{- end}}
+{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }}
+# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_SENDMAIL traffic
+    {{ tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for PROOFPOINT_PPS_SENDMAIL traffic
+{{ tmpl.Exec "log_path" "yes" }}
diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl
@@ -81,7 +81,7 @@ source s_default-ports {
             rewrite(set_rfc5424_noversion);
         } elif {
             parser {cisco-parser()};
-            rewrite(set_metadata_vendor_product_cisco_ios);
+            rewrite(set_cisco_ios);
         } else {
             parser {
                 syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message));
@@ -94,5 +94,5 @@ source s_default-ports {
         parser {
             vendor_product_by_source();
         };
-   };
+    };
 };
diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv
@@ -28,6 +28,8 @@
 #pan_correlation,index,main
 #pan_userid,index,netauth
 #pan_unknown,index,netops
-#sc4s_events,index,_internal
+#proofpoint_pps_filter,index,email
+#proofpoint_pps_sendmail,index,email
+#sc4s_events,index,main
 #sc4s_fallback,index,main
 #sc4s_metrics,index,em_metrics
diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/context-local/vendor_product_by_source.conf
@@ -4,26 +4,34 @@
 #filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); };
 
 filter f_test_test {
-    host("testvp-*" type(glob))
-    or match("test_test" value("fields.sc4s_presume"))
+    host("testvp-*" type(glob)) or
+    netmask(192.168.100.1/24)
 };
 filter f_juniper_nsm {
-    host("jnpnsm-*" type(glob))
-    or match("juniper_nsm" value("fields.sc4s_presume"))
+    host("jnpnsm-*" type(glob)) or
+    netmask(192.168.1.0/24)
 };
 filter f_juniper_nsm_idp {
-    host("jnpnsmidp-*" type(glob))
-    or match("juniper_nsm_idp" value("fields.sc4s_presume"))
+    host("jnpnsmidp-*" type(glob)) or
+    netmask(192.168.2.0/24)
 };
 filter f_juniper_idp {
-    host("jnpidp-*" type(glob))
-    or match("juniper_idp" value("fields.sc4s_presume"))
+    host("jnpidp-*" type(glob)) or
+    netmask(192.168.3.0/24)
 };
 filter f_juniper_netscreen {
-    host("jnpns-*" type(glob) )
-    or match("juniper_netscreen" value("fields.sc4s_presume"))
+    host("jnpns-*" type(glob)) or
+    netmask(192.168.4.0/24)
 };
 filter f_cisco_nx_os {
-    host("csconx-*" type(glob) )
-    or match("cisco_nx_os" value("fields.sc4s_presume"))
+    host("csconx-*" type(glob)) or
+    netmask(192.168.5.0/24)
+};    
+filter f_proofpoint_pps_sendmail {
+    host("pps-*" type(glob)) or
+    netmask(192.168.6.0/24)
+};
+filter f_proofpoint_pps_filter {
+    host("pps-*" type(glob)) or
+    netmask(192.168.7.0/24)
 };
diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/context-local/vendor_product_by_source.csv
@@ -3,4 +3,6 @@ f_juniper_nsm,sc4s_vendor_product,"juniper_nsm"
 f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp"
 f_juniper_idp,sc4s_vendor_product,"juniper_idp"
 f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen"
-f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os"
+f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os"
+f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail"
+f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter"
diff --git a/package/etc/templates/source_network.t b/package/etc/templates/source_network.t
@@ -73,12 +73,14 @@ source s_dedicated_port_{{ .port_id}} {
         rewrite(set_rfc5424_noversion);
 {{- else if eq .parser "cisco_parser" }}
         parser {cisco-parser()};
-        rewrite(set_metadata_vendor_product_cisco_ios);
+        rewrite(set_cisco_ios);
 {{- else if eq .parser "rfc3164" }}
         parser {
             syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message));
         };
         rewrite(set_rfc3164);
+{{- else if eq .parser "no_parse" }}
+        rewrite(set_no_parse);
 {{- else }}
         if {filter(f_rfc5424_strict);
             parser {
@@ -93,7 +95,7 @@ source s_dedicated_port_{{ .port_id}} {
             rewrite(set_rfc5424_noversion);
         } elif {
             parser {cisco-parser()};
-            rewrite(set_metadata_vendor_product_cisco_ios);
+            rewrite(set_cisco_ios);
         } else {
             parser {
                 syslog-parser(time-zone({{getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message));

diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf
@@ -39,4 +39,9 @@ thawedPath = $SPLUNK_DB/netops/thaweddb
 [netproxy]
 homePath   = $SPLUNK_DB/netproxy/db
 coldPath   = $SPLUNK_DB/netproxy/colddb
-thawedPath = $SPLUNK_DB/netproxy/thaweddb
+thawedPath = $SPLUNK_DB/netproxy/thaweddb
+
+[email]
+homePath   = $SPLUNK_DB/email/db
+coldPath   = $SPLUNK_DB/email/colddb
+thawedPath = $SPLUNK_DB/email/thaweddb