Add support for CEF syslog (#102)

Support CEF format

.gitmodules

@@ -1,5 +1,4 @@
 [submodule "package/syslog-ng"]
 	path = package/syslog-ng
 	url = https://github.com/balabit/syslog-ng.git
-[submodule "Submod"]
 	branch = syslog-ng-3.23.1

docs/sources.md

@@ -233,6 +233,55 @@ index=<asconfigured> (sourcetype=fgt_log OR sourcetype=fgt_traffic OR sourcetype
 
 Verify timestamp, and host values match as expected
 
+# Vendor - Imperva
+
+## Product - Incapsula
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/                                                              |
+| Splunk Add-on Source Specific | https://bitbucket.org/SPLServices/ta-cef-imperva-incapsula/downloads/                                                               |
+| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm                                                        |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| cef        | Common sourcetype                                                                                                 |
+
+### Source
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Imperva:Incapsula        | Common sourcetype                                                                                                 |
+
+### Index Configuration
+
+| key            | source     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| cef_Incapsula_SIEMintegration      | Imperva:Incapsula      | netwaf          | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+
+### Verification
+
+An active site will generate frequent events use the following search to check for new events
+
+Verify timestamp, and host values match as expected    
+
+```
+index=<asconfigured> (sourcetype=cef source="Imperva:Incapsula")
+```
+
 # Vendor - Juniper
 
 ## Product - Juniper JunOS
@@ -432,6 +481,103 @@ index=<asconfigured> sourcetype=juniper:sslvpn | stats count by host
 
 Verify timestamp, and host values match as expected
 
+# Vendor - Microfocus ArcSight
+
+## Product - Internal Agent Events
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/                                                              |
+| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm                                                        |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| cef        | Common sourcetype                                                                                                 |
+
+### Source
+
+| source     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| ArcSight:ArcSight        | Internal logs                                                                                               |
+
+### Index Configuration
+
+| key            | source     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| cef_ArcSight_ArcSight      | ArcSight:ArcSight      | main          | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+
+### Verification
+
+An active site will generate frequent events use the following search to check for new events
+
+Verify timestamp, and host values match as expected    
+
+```
+index=<asconfigured> (sourcetype=cef source="ArcSight:ArcSight")
+```
+
+## Product - Microsoft Windows
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/                                                              |
+| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-microsoft-windows-for-splunk/downloads/                                                             |
+| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm                                                        |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| cef        | Common sourcetype                                                                                                 |
+
+### Source
+
+| source     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| CEFEventLog:System or Application Event     | Windows Application and System Event Logs                                                                                  |
+| CEFEventLog:Microsoft Windows     | Windows Security Event Logs                                                                                 |
+
+### Index Configuration
+
+| key            | source     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| cef_Microsoft_System or Application Event      | CEFEventLog:System or Application Event      | oswin          | none          |
+| cef_Microsoft_Microsoft Windows      | CEFEventLog:Microsoft Windows      | oswinsec         | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+
+### Verification
+
+An active site will generate frequent events use the following search to check for new events
+
+Verify timestamp, and host values match as expected    
+
+```
+index=<asconfigured> (sourcetype=cef (source="CEFEventLog:Microsoft Windows" OR source="CEFEventLog:System or Application Event"))
+```
+
 # Vendor - PaloAlto
 
 ## Product - NGFW

package/etc/conf.d/conflib/_common/templates.conf

@@ -20,6 +20,13 @@ template t_msg_only {
         };
 
 # ===============================================================================================
+# Message Only trim leading or trailing white space
+# ===============================================================================================
+
+template t_msg_trim {
+        template("$(strip $MSGONLY )");
+        };
+# ===============================================================================================
 # Everything; useful for bluecoat proxy and others
 # ===============================================================================================
 

package/etc/conf.d/filters/microfocus/arcsight.conf

@@ -0,0 +1,4 @@
+
+filter f_microfocus_arcsight {
+    program(CEF);
+};

package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl

@@ -0,0 +1,84 @@
+parser p_microfocus_arcsight_header {
+    csv-parser(
+        columns("fields.sc4s_cef_version", "fields.cef_device_vendor", "fields.cef_device_product", "fields.cef_device_version", "fields.cef_device_event_class", "fields.cef_name", "fields.cef_severity", MESSAGE)
+        delimiters(chars("|"))
+        flags(strip-whitespace, escape-none, greedy)
+    );
+
+    kv-parser(prefix(".cef.") template("t_msg_trim")
+    );
+
+};
+
+parser p_microfocus_arcsight_ts_rt {
+    date-parser(format("%s") template("${.cef.rt}")
+    );
+};
+parser p_microfocus_arcsight_ts_end {
+    date-parser(format("%s") template("${.cef.end}")
+    );
+};
+
+parser p_microfocus_arcsight_source {
+    add-contextual-data(
+        selector("${fields.cef_device_vendor}_${fields.cef_device_product}"),
+        database("context-local/microfocus_arcsight_source.csv")
+        ignore-case(yes)
+        prefix(".splunk.")
+        default-selector("unknown")
+    );
+};
+
+# Microfocus ArcSight
+{{- if (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }}
+{{ $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common"}}
+{{ tmpl.Exec "t/source_network.t" $context }}
+{{- end -}}
+{{ define "log_path" }}
+log {
+{{- if eq (.) "yes"}}
+    source(s_default-ports);
+    filter(f_is_rfc3164);
+    filter(f_microfocus_arcsight);
+{{- end}}
+{{- if eq (.) "no"}}
+    source (s_dedicated_port_MICROFOCUS_ARCSIGHT);
+{{- end}}
+
+    rewrite {
+        r_set_splunk_dest_default(sourcetype("cef"), index("main"), template("t_msg_trim"))
+    };
+
+    parser {
+        p_add_context_splunk(key("cef_{fields.cef_device_vendor}_${fields.cef_device_product}"));
+
+    };
+
+    parser (p_microfocus_arcsight_header);
+
+    # We already have the syslog msg time stamp however that may not be the best one
+    # If we have an rt or end field that is best we use the If trick here so if this parser fails
+    # We don't get sent to fallback.
+    if {
+        parser (p_microfocus_arcsight_ts_rt);
+    } elif {
+        parser (p_microfocus_arcsight_ts_end);
+    } else {}; #Do nothing this is allows for both rt and end to be missing and still pass with the message ts
+
+    #CEF TAs use the source as their bounds in props.conf
+    parser(p_microfocus_arcsight_source);
+
+    destination(d_hec);  #--HEC--
+
+    flags(flow-control);
+
+};
+{{- end}}
+{{- if (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }}
+
+# Listen on the specified dedicated port(s) for MICROFOCUS_ARCSIGHT traffic
+    {{tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for MICROFOCUS_ARCSIGHT traffic
+{{tmpl.Exec "log_path" "yes" }}

package/etc/context-local/microfocus_arcsight_source.csv

@@ -0,0 +1,8 @@
+ArcSight_ArcSight,source,ArcSight:ArcSight
+Microsoft_System or Application Event,source,CEFEventLog:System or Application Event
+Microsoft_System or Application Event,index,oswin
+Microsoft_Microsoft Windows,source,CEFEventLog:Microsoft Windows
+Microsoft_Microsoft Windows,index,oswinsec
+Incapsula_SIEMintegration,source,Imperva:Incapsula
+Incapsula_SIEMintegration,index,netwaf
+unknown,source,ArcSight:unknown

package/etc/context-local/splunk_index.csv

@@ -1,4 +1,8 @@
 #bluecoat_proxy,index,netproxy
+#cef_ArcSight_ArcSight,index,netwaf
+#cef_Incapsula_SIEMintegration,index,netwaf
+#cef_Microsoft_Microsoft Windows,index,oswinsec
+#cef_Microsoft_System or Application Event,index,oswin
 #cisco_asa,index,netfw
 #cisco_ios,index,netops
 #cisco_nx_os,index,netops

splunk/etc/apps/SA-syslog-ng/default/indexes.conf

@@ -20,6 +20,15 @@ homePath   = $SPLUNK_DB/test2/db
 coldPath   = $SPLUNK_DB/test2/colddb
 thawedPath = $SPLUNK_DB/test2/thaweddb
 
+[oswin]
+homePath   = $SPLUNK_DB/oswin/db
+coldPath   = $SPLUNK_DB/oswin/colddb
+thawedPath = $SPLUNK_DB/oswin/thaweddb
+
+[oswinsec]
+homePath   = $SPLUNK_DB/oswinsec/db
+coldPath   = $SPLUNK_DB/oswinsec/colddb
+thawedPath = $SPLUNK_DB/oswinsec/thaweddb
 
 [netfw]
 homePath   = $SPLUNK_DB/netfw/db
@@ -41,6 +50,11 @@ homePath   = $SPLUNK_DB/netproxy/db
 coldPath   = $SPLUNK_DB/netproxy/colddb
 thawedPath = $SPLUNK_DB/netproxy/thaweddb
 
+[netwaf]
+homePath   = $SPLUNK_DB/netwaf/db
+coldPath   = $SPLUNK_DB/netwaf/colddb
+thawedPath = $SPLUNK_DB/netwaf/thaweddb
+
 [email]
 homePath   = $SPLUNK_DB/email/db
 coldPath   = $SPLUNK_DB/email/colddb