Merge branch 'release/1.4.0'

Add support for 1. VMware NSX 2. VMware ESX and VSphere (Appliance) 3. InfoBlox 4. Symantec MG (Bright Mail) 5. Linux OS events generated by the OS of common appliances 6. Misc fixes Update container to RHEL UBI8 Temporary removal of Librdkafka Update syslog-ng to 3.25.1 Improve format for RFC5424 in fallback
CSVD · Dec 19, 2019 · c0b3e03 · c0b3e03
2 parents 0f9605c + c930a66
commit c0b3e03
Show file tree

Hide file tree

Showing 78 changed files with 1,655 additions and 267 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -39,7 +39,6 @@ jobs:
           image: $CI_IMAGE
           registry: $REGISTRY
           path: package
-          extra_build_args: --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION
 
       - docker/install-goss:
           version: v0.3.7

diff --git a/.env.template b/.env.template
@@ -6,8 +6,6 @@
 #
 #You should have received a copy of the CC0 legalcode along with this
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
-RH_ORG=xxxx
-RH_ACTIVATION=xxxxx
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SPLUNK_PASSWORD=Changed@11
 SPLUNK_START_ARGS=--accept-license

diff --git a/.gitignore b/.gitignore
@@ -384,3 +384,4 @@ fabric.properties
 tests/test_plugin_*.py
 # package/etc/conf.d/local/
 !package/etc/conf.d/local
+replay
diff --git a/.gitmodules b/.gitmodules
@@ -1,6 +1,6 @@
 [submodule "package/syslog-ng"]
 	path = package/syslog-ng
 	url = https://github.com/balabit/syslog-ng.git
-	branch = syslog-ng-3.24.1
+	branch = syslog-ng-3.25.1
 #
 
diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml
@@ -13,9 +13,6 @@ services:
     image: splunk/scs:latest
     build:
       context: ./package
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
     entrypoint:
       - "tail"
       - "-f"

diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml
@@ -12,9 +12,6 @@ services:
     image: rfaircloth/scs:edge
     build:
       context: ./package
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
     hostname: sc4s
     ports:
       - "514"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -25,9 +25,6 @@ services:
     image: splunk/scs:latest
     build:
       context: ./package
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
     hostname: sc4s
 #When this is enabled test_common will fail
 #    command: -det
@@ -72,6 +69,16 @@ services:
       - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
     volumes:
       - splunk-etc:/opt/splunk/etc
+  pcapreplay:
+    build:
+      context: ./utility/pcapreplay
+    entrypoint: tail -f /dev/null
+    links:
+      - splunk
+      - sc4s
+    volumes:
+      - ./replay:/work
+
 volumes:
   sc4s-results:
     external: true

diff --git a/docs/gettingstarted/index.md b/docs/gettingstarted/index.md
@@ -42,6 +42,8 @@ using the SC4S defaults. SC4S can be easily customized to use different indexes
 * netops
 * netproxy
 * netipam
+* oswinsec
+* osnix
 * em_metrics (ensure this is created as a metrics index)
 
 #### Install Related Splunk Apps

diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md
@@ -1,5 +1,54 @@
 # Vendor - Cisco
 
+## Product - ACS
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/1811/                                                                 |
+| Product Manual | https://community.cisco.com/t5/security-documents/acs-5-x-configuring-the-external-syslog-server/ta-p/3143143 |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| cisco:acs     | Aggregation used                                                                                                    |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| cisco_acs    | cisco:acs    | netauth          | None     |
+
+
+### Filter type
+
+PATTERN MATCH
+
+### Setup and Configuration
+
+* No special steps required
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_CISCO_ACS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CISCO_ACS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_CISCO_ACS | no | Enable archive to disk for this specific source |
+| SC4S_DEST_CISCO_ACS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+Use the following search to validate events are present
+
+```
+index=<asconfigured> sourcetype=cisco:acs
+```
+
+Verify timestamp, and host values match as expected    
+
+
 ## Product - ASA (Pre Firepower)
 
 | Ref            | Link                                                                                                    |
@@ -42,8 +91,8 @@ MSG Parse: This filter parses message content
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_CISCO_ASA_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format |
-| SC4S_LISTEN_CISCO_ASA_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined expecting RFC5424 format |
+| SC4S_LISTEN_CISCO_ASA_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CISCO_ASA_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_CISCO_ASA | no | Enable archive to disk for this specific source |
 | SC4S_DEST_CISCO_ASA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 | SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined expecting RFC3164 format |

diff --git a/docs/sources/InfoBlox/index.md b/docs/sources/InfoBlox/index.md
@@ -0,0 +1,56 @@
+# Vendor - Infoblox
+
+Warning: Despite the TA indication this data source is CIM compliant the all versions of NIOS including the most recent available as of 2019-12-17 do not support the DNS data model correctly. For DNS security use cases use Splunk Stream instead.
+
+## Product - NIOS
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/2934/                                                                 |
+| Product Manual | https://docs.infoblox.com/display/ILP/NIOS?preview=/8945695/43728387/NIOS_8.4_Admin_Guide.pdf   |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| infoblox:dns        | None                                                                                                    |
+| infoblox:dhcp    | None                                                                                         |
+| infoblox:threat     | None                                                                                          |
+| nix:syslog     | None                                                                                          |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| infoblox_dns      | infoblox:dns       | netdns          | none          |
+| infoblox_dhcp    | infoblox:dhcp      | netipam          | none          |
+| infoblox_threat    | infoblox:threat      | netids          | none          |
+| nix_syslog    | nix:syslog      | osnix          | none          |
+
+### Filter type
+
+Must be identified by host or ip assignment. Update the filter `f_infoblox` or configure a dedicated port as required
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the admin manual for specific details of configuration
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_INFOBLOX_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_INFOBLOX_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_INFOBLOX | no | Enable archive to disk for this specific source |
+| SC4S_DEST_INFOBLOX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active device will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=infoblox:*| stats count by host
+```
diff --git a/docs/sources/Juniper/index.md b/docs/sources/Juniper/index.md
@@ -37,9 +37,11 @@
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_JUNIPER_JUNOS_LEGACY_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using legacy 3164 format|
-| SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using 5424 format |
-| SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source |
+| SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using legacy 3164 format|
+| SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined using legacy 3164 format|
+| SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined using legacy 3164 format|
+| SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using 5424 format |
+| SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined using 5424 format || SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_JUNOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 
 ### Verification
@@ -90,6 +92,7 @@ Verify timestamp, and host values match as expected
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
 | SC4S_LISTEN_JUNIPER_NSM_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_JUNIPER_NSM_TLS_PORT      | empty string      | Enable at TLS port for this specific vendor product using the number defined |
 | SC4S_LISTEN_JUNIPER_NSM_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_JUNIPER_NSM | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_NSM_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
@@ -142,6 +145,7 @@ Verify timestamp, and host values match as expected
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
 | SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
 | SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_JUNIPER_NETSCREEN | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_NETSCREEN_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
@@ -192,6 +196,7 @@ Verify timestamp, and host values match as expected
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
 | SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
 | SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_JUNOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
@@ -204,4 +209,4 @@ Use the following search to validate events are present; for Juniper SSL VPN ens
 index=<asconfigured> sourcetype=juniper:sslvpn | stats count by host
 ```
 
-Verify timestamp, and host values match as expected
+Verify timestamp, and host values match as expected
diff --git a/docs/sources/PaloaltoNetworks/index.md b/docs/sources/PaloaltoNetworks/index.md
@@ -53,6 +53,7 @@ MSG Parse: This filter parses message content
 | SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_PALOALTO_PANOS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_PALOALTO_PANOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS | no | Use custom time stamp parsing with ms added |
 
 ### Verification
 

diff --git a/docs/sources/Symantec/index.md b/docs/sources/Symantec/index.md
@@ -49,3 +49,53 @@ An active proxy will generate frequent events. Use the following search to valid
 ```
 index=<asconfigured> sourcetype=bluecoat:proxysg:access:kv | stats count by host
 ```
+
+## Product - Mail Gateway (Brightmail)
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | TBD                                                            |
+| Product Manual | https://support.symantec.com/us/en/article.howto38250.html                                                       |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| symantec:smg        | Requires version TA 3.6                                                                                                    |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| symantec_brightmail      | symantec:smg     | email          | none          |
+
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Setup and Configuration
+
+* No TA available
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL | no | Enable archive to disk for this specific source |
+| SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG | yes | Email processing events generated by the bmserver process will be grouped by host+program+pid+msg ID into a single event |
+### Verification
+
+An active mail server will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=symantec:smg | stats count by host
+```
diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md
@@ -0,0 +1,53 @@
+# Vendor - Dell - VMWare
+
+## Product - vSphwere - ESX NSX (Controller, Manager, Edge)
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | None                                                                |
+| Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html |
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| vmware:vsphere:nsx | None |
+| vmware:vsphere:esx | None |
+| nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx  |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| vmware_nsx      | vmware:vsphere:nsx | main          | none          |
+| vmware_esx      | vmware:vsphere:esx | main          | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content when using the default configuration
+
+### Setup and Configuration
+
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_VMWARE_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_VMWARE | no | Enable archive to disk for this specific source |
+| SC4S_DEST_VMWARE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active proxy will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype="vmware:*:vsphere:*" | stats count by host
+```