Merge pull request #290 from splunk/release/1.8.0

Release/1.8.0
CSVD · Jan 26, 2020 · c788fd6 · c788fd6
2 parents 688b8fc + 6927252
commit c788fd6
Show file tree

Hide file tree

Showing 26 changed files with 423 additions and 85 deletions.
diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml
@@ -45,6 +45,8 @@ services:
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
       - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000
+      - SC4S_LISTEN_PFSENSE_TCP_PORT=5006
+
   splunk:
     image: splunk/splunk:latest
     hostname: splunk

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -53,6 +53,7 @@ services:
       - SC4S_LISTEN_CISCO_MERAKI_TCP_PORT=5003
       - SC4S_LISTEN_JUNIPER_IDP_TCP_PORT=5004
       - SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT=5005
+      - SC4S_LISTEN_PFSENSE_TCP_PORT=5006
       - SC4S_ARCHIVE_GLOBAL=yes
     volumes:
       - ./tls:/opt/syslog-ng/tls

diff --git a/docs/sources/CommonEventFormat/index.md b/docs/sources/CommonEventFormat/index.md
@@ -0,0 +1,70 @@
+# Vendor - Common Event Format Data Sources
+
+## Product - Various products that send CEF-format messages via syslog
+
+Each CEF product should have their own source entry in this documentation set.  In a departure
+from normal configuration, all CEF products should use the "CEF" version of the unique port and
+archive envrionmetn variable settings (rather than a unique one per product), as the CEF log path
+handles all products sending events to SC4S in the CEF format. Examples of this include Arcsight,
+Imperva, and Cyberark.  Therefore, the CEF environment varialbes for unique port, archive, etc.
+should be set only _once_.
+
+If your deployment has multiple CEF devices that send to more than one port,
+set the CEF unique port variable(s) to just one of the ports in use.  Then, map the others with
+container networking to the port chosen.  Example: If you have three CEF devices, sending on TCP
+ports 2000,2001, and 2002, set `SC4S_LISTEN_CEF_TCP_PORT=2000`.  Then, map the other two with
+container networking, e.g. `-p 2000:2000 -p 2001:2000 -p 2002:2000`.  This will route all
+three ports to TCP port 2000 inside the container, and the single CEF log path will properly
+process data from all three devices.
+
+The source documentation included below is a reference baseline for any product that sends data
+using the CEF log path.
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on CEF | https://bitbucket.org/SPLServices/ta-cef-for-splunk/downloads/                                                              |
+| Product Manual | https://docs.imperva.com/bundle/cloud-application-security/page/more/log-configuration.htm                                                        |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| cef        | Common sourcetype                                                                                                 |
+
+### Typical Source
+
+| source     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Varies        | Varies                                                                                               |
+
+### Typical Index Configuration
+
+| key            | source     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| Vendor_Product      | Varies      | main          | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_CEF_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CEF_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CEF_TLS_PORT      | empty string      | Enable a TLS  port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source |
+| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active site will generate frequent events use the following search to check for new events
+
+Verify timestamp, and host values match as expected    
+
+```
+index=<asconfigured> (sourcetype=cef source=<asconfigured>)
+```
diff --git a/docs/sources/CyberArk/index.md b/docs/sources/CyberArk/index.md
@@ -28,7 +28,11 @@ MSG Parse: This filter parses message content
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CEF_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+
+* NOTE:  Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how
+many ports are in use by this CEF source (or any others).  See the "Common Event Format" source
+documentation for more information.
 
 ### Verification
 
@@ -68,7 +72,11 @@ MSG Parse: This filter parses message content
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CEF_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+
+* NOTE:  Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how
+many ports are in use by this CEF source (or any others).  See the "Common Event Format" source
+documentation for more information.
 
 ### Verification
 

diff --git a/docs/sources/Imperva/index.md b/docs/sources/Imperva/index.md
@@ -25,7 +25,7 @@
 
 | key            | source     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| cef_Incapsula_SIEMintegration      | Imperva:Incapsula      | netwaf          | none          |
+| Incapsula_SIEMintegration      | Imperva:Incapsula      | netwaf          | none          |
 
 ### Filter type
 
@@ -37,10 +37,14 @@ Note listed for reference processing utilizes the Microsoft ArcSight log path as
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
-| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
-| SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT | no | Enable archive to disk for this specific source |
-| SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_LISTEN_CEF_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CEF_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source |
+| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+* NOTE:  Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how
+many ports are in use by this CEF source (or any others).  See the "Common Event Format" source
+documentation for more information.
 
 ### Verification
 
@@ -50,4 +54,4 @@ Verify timestamp, and host values match as expected
 
 ```
 index=<asconfigured> (sourcetype=cef source="Imperva:Incapsula")
-```
+```
diff --git a/docs/sources/Microfocus/index.md b/docs/sources/Microfocus/index.md
@@ -1,6 +1,6 @@
-# Vendor - Microfocus ArcSight
+# Vendor - MicroFocus Arcsight
 
-## Product - Internal Agent Events
+## Product - Arcsight Internal Agent
 
 | Ref            | Link                                                                                                    |
 |----------------|---------------------------------------------------------------------------------------------------------|
@@ -24,7 +24,7 @@
 
 | key            | source     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| cef_ArcSight_ArcSight      | ArcSight:ArcSight      | main          | none          |
+| ArcSight_ArcSight      | ArcSight:ArcSight      | main          | none          |
 
 ### Filter type
 
@@ -34,7 +34,12 @@ MSG Parse: This filter parses message content
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CEF_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT      | empty string      | Deprecated equivalent of above variable.  This is included for backward compatibility and will be removed in a future version. _Do not use_ in new installations. |
+
+* NOTE:  Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how
+many ports are in use by this CEF source (or any others).  See the "Common Event Format" source
+documentation for more information.
 
 ### Verification
 
@@ -46,7 +51,7 @@ Verify timestamp, and host values match as expected
 index=<asconfigured> (sourcetype=cef source="ArcSight:ArcSight")
 ```
 
-## Product - Microsoft Windows
+## Product - Arcsight Microsoft Windows (CEF)
 
 | Ref            | Link                                                                                                    |
 |----------------|---------------------------------------------------------------------------------------------------------|
@@ -72,8 +77,8 @@ index=<asconfigured> (sourcetype=cef source="ArcSight:ArcSight")
 
 | key            | source     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| cef_Microsoft_System or Application Event      | CEFEventLog:System or Application Event      | oswin          | none          |
-| cef_Microsoft_Microsoft Windows      | CEFEventLog:Microsoft Windows      | oswinsec         | none          |
+| Microsoft_System or Application Event      | CEFEventLog:System or Application Event      | oswin          | none          |
+| Microsoft_Microsoft Windows      | CEFEventLog:Microsoft Windows      | oswinsec         | none          |
 
 ### Filter type
 
@@ -83,10 +88,15 @@ MSG Parse: This filter parses message content
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
-| SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
-| SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT | no | Enable archive to disk for this specific source |
-| SC4S_DEST_MICROFOCUS_ARCSIGHT_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_LISTEN_CEF_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CEF_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_CEF | no | Enable archive to disk for this specific source |
+| SC4S_DEST_CEF_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_WWW_XXX_MICROFOCUS_ARCSIGHT_YYY_ZZZ | no | Deprecated equivalents of the above variables.  These are included for backward compatibility, and will be removed in a future version. _Do not use_ in new installations. | 
+
+* NOTE:  Set only _one_ set of CEF variables for the entire SC4S deployment, regardless of how
+many ports are in use by this CEF source (or any others).  See the "Common Event Format" source
+documentation for more information.
 
 ### Verification
 
@@ -96,4 +106,4 @@ Verify timestamp, and host values match as expected
 
 ```
 index=<asconfigured> (sourcetype=cef (source="CEFEventLog:Microsoft Windows" OR source="CEFEventLog:System or Application Event"))
-```
+```
diff --git a/docs/sources/Pfsense/index.md b/docs/sources/Pfsense/index.md
@@ -0,0 +1,57 @@
+# Vendor - pfSense
+
+All pfSense based firewalls
+
+
+## Product
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/1527/                                                                 |
+| Product Manual | https://docs.netgate.com/pfsense/en/latest/monitoring/copying-logs-to-a-remote-host-with-syslog.html?highlight=syslog |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| pfsense:filterlog  | None |
+| pfsense:* | All programs other than filterlog |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| pfsense      | pfsense     | netops          | none          |
+| pfsense_filterlog      | pfsense:filterlog      | netfw          | none          |
+
+### Filter type
+
+Source does not provide a hostname, port or IP based filter is required
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Configure a dedicated SC4S port OR configure IP filter 
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_PFSENSE_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_PFSENSE_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_PFSENSE | no | Enable archive to disk for this specific source |
+| SC4S_DEST_PFSENSE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active proxy will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=pfsense:filterlog | stats count by host
+```
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -14,14 +14,16 @@ nav:
       - About: sources/index.md
       - Checkpoint: sources/Checkpoint/index.md
       - Cisco: sources/Cisco/index.md
+      - 'Common Event Format': sources/CommonEventFormat/index.md
       - CyberArk: sources/CyberArk/index.md
       - Forcepoint: sources/Forcepoint/index.md
       - Fortinet: sources/Fortinet/index.md
       - Imperva: sources/Imperva/index.md
       - Juniper: sources/Juniper/index.md
-      - Nix: sources/nix/index.md
       - Microfocus: sources/Microfocus/index.md
-      - 'Paloalto Networks': sources/PaloaltoNetworks/index.md
+      - Nix: sources/nix/index.md
+      - 'Palo Alto Networks': sources/PaloaltoNetworks/index.md
+      - 'pfSense': sources/pfSense/index.md
       - Proofpoint: sources/Proofpoint/index.md
       - Symantec: sources/Symantec/index.md
       - Ubiquiti: sources/Ubiquiti/index.md

diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf
@@ -1,11 +1,11 @@
 filter f_rfc5424_strict{
-    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>(?<FULLDATE>(?<FULLDATEYEAR>\d{4})-(?<FULLDATEMONTH>\d\d)-(?<FULLDATEDAY>\d\d))T(?<FULLTIME>(?<PARTIALTIME>(?<TIMEHOUR>[0-2]\d):(?<TIMEMINUTE>[0-5]\d):(?<TIMESECOND>[0-5]\d)(?:.(?<TIMESECFRAC>\d{1,6}))?)(?<TIMEOFFSET>Z|(?<TIMENUMOFFSET>[+\-][0-2]\d:[0-5]\d))))))');
-};
+    message('^\<(?<PRI>\d+)\>(?<VERSION>\d{1,2})? (?<YEAR>\d+)-(?<MONTH>\d+)-(?<DAY>\d+)T(?<HOUR>\d+):(?<MINUTE>\d+):(?<SECOND>\d+)(?:\.(?<MILLISECONDS>\d+))?(?<OFFSET>Z|[\+-] *\d+:\d+) (?<HOSTNAME>(-)|[^ ]+) (?<APPNAME>(?:-)|\b\w+\b) (?<PROCID>(?:-)|\b\w+\b) (?<MSGID>(?:-)|\b\w+\b) *(?<STRUCDATA>(?:-)|\[.*?\]) *(?<MSG>(?:-)|\b.*)?$');
+    };
 filter f_rfc5424_noversion{
     message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>) ?(?<TIMESTAMP>(?<FULLDATE>(?<FULLDATEYEAR>\d{4})-(?<FULLDATEMONTH>\d\d)-(?<FULLDATEDAY>\d\d))T(?<FULLTIME>(?<PARTIALTIME>(?<TIMEHOUR>[0-2]\d):(?<TIMEMINUTE>[0-5]\d):(?<TIMESECOND>[0-5]\d)(?:.(?<TIMESECFRAC>\d{1,6}))?)(?<TIMEOFFSET>Z|(?<TIMENUMOFFSET>[+\-][0-2]\d:[0-5]\d))))))');
 };
 filter f_rfc3164_version{
-    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?<FROMHOST>[^ ]+) ))');
+    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) ))');
 };
 rewrite set_rfc5424_strict{
     set("rfc5424_strict" value("fields.sc4s_syslog_format"));

diff --git a/....d/context/microfocus_arcsight_source.csv → ....d/context/common_event_format_source.csv b/....d/context/microfocus_arcsight_source.csv → ....d/context/common_event_format_source.csv
diff --git a/package/etc/conf.d/filters/common_event_format/cef.conf b/package/etc/conf.d/filters/common_event_format/cef.conf
@@ -0,0 +1,4 @@
+
+filter f_cef {
+    program(CEF);
+};
diff --git a/...e/etc/conf.d/filters/infoblox/syslog.conf → .../etc/conf.d/filters/infoblox/pfsense.conf b/...e/etc/conf.d/filters/infoblox/syslog.conf → .../etc/conf.d/filters/infoblox/pfsense.conf
diff --git a/package/etc/conf.d/filters/microfocus/arcsight.conf b/package/etc/conf.d/filters/microfocus/arcsight.conf
diff --git a/package/etc/conf.d/filters/pfsense/syslog.conf b/package/etc/conf.d/filters/pfsense/syslog.conf
@@ -0,0 +1,4 @@
+filter f_pfsense {
+    match("^pfsense", value("fields.sc4s_vendor_product"));
+
+};