Merge pull request #249 from splunk/release/1.4.0

Release/1.4.0

.circleci/config.yml

@@ -39,7 +39,6 @@ jobs:
           image: $CI_IMAGE
           registry: $REGISTRY
           path: package
-          extra_build_args: --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION
 
       - docker/install-goss:
           version: v0.3.7

.env.template

@@ -6,8 +6,6 @@
 #
 #You should have received a copy of the CC0 legalcode along with this
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
-RH_ORG=xxxx
-RH_ACTIVATION=xxxxx
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 SPLUNK_PASSWORD=Changed@11
 SPLUNK_START_ARGS=--accept-license

.gitignore

@@ -384,3 +384,4 @@ fabric.properties
 tests/test_plugin_*.py
 # package/etc/conf.d/local/
 !package/etc/conf.d/local
+replay

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "package/syslog-ng"]
 	path = package/syslog-ng
 	url = https://github.com/balabit/syslog-ng.git
-	branch = syslog-ng-3.24.1
+	branch = syslog-ng-3.25.1
 #
 

docker-compose-debug.yml

@@ -13,9 +13,6 @@ services:
     image: splunk/scs:latest
     build:
       context: ./package
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
     entrypoint:
       - "tail"
       - "-f"

docker-compose-perf.yml

@@ -12,9 +12,6 @@ services:
     image: rfaircloth/scs:edge
     build:
       context: ./package
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
     hostname: sc4s
     ports:
       - "514"

docker-compose.yml

@@ -25,9 +25,6 @@ services:
     image: splunk/scs:latest
     build:
       context: ./package
-      args:
-        RH_ORG: ${RH_ORG}
-        RH_ACTIVATION: ${RH_ACTIVATION}
     hostname: sc4s
 #When this is enabled test_common will fail
 #    command: -det
@@ -72,6 +69,16 @@ services:
       - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
     volumes:
       - splunk-etc:/opt/splunk/etc
+  pcapreplay:
+    build:
+      context: ./utility/pcapreplay
+    entrypoint: tail -f /dev/null
+    links:
+      - splunk
+      - sc4s
+    volumes:
+      - ./replay:/work
+
 volumes:
   sc4s-results:
     external: true

docs/gettingstarted/index.md

@@ -42,6 +42,8 @@ using the SC4S defaults. SC4S can be easily customized to use different indexes
 * netops
 * netproxy
 * netipam
+* oswinsec
+* osnix
 * em_metrics (ensure this is created as a metrics index)
 
 #### Install Related Splunk Apps

docs/sources/Cisco/index.md

@@ -1,5 +1,54 @@
 # Vendor - Cisco
 
+## Product - ACS
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/1811/                                                                 |
+| Product Manual | https://community.cisco.com/t5/security-documents/acs-5-x-configuring-the-external-syslog-server/ta-p/3143143 |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| cisco:acs     | Aggregation used                                                                                                    |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| cisco_acs    | cisco:acs    | netauth          | None     |
+
+
+### Filter type
+
+PATTERN MATCH
+
+### Setup and Configuration
+
+* No special steps required
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_CISCO_ACS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CISCO_ACS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_CISCO_ACS | no | Enable archive to disk for this specific source |
+| SC4S_DEST_CISCO_ACS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+Use the following search to validate events are present
+
+```
+index=<asconfigured> sourcetype=cisco:acs
+```
+
+Verify timestamp, and host values match as expected    
+
+
 ## Product - ASA (Pre Firepower)
 
 | Ref            | Link                                                                                                    |
@@ -42,8 +91,8 @@ MSG Parse: This filter parses message content
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_CISCO_ASA_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format |
-| SC4S_LISTEN_CISCO_ASA_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined expecting RFC5424 format |
+| SC4S_LISTEN_CISCO_ASA_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CISCO_ASA_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_CISCO_ASA | no | Enable archive to disk for this specific source |
 | SC4S_DEST_CISCO_ASA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 | SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined expecting RFC3164 format |

docs/sources/InfoBlox/index.md

@@ -0,0 +1,56 @@
+# Vendor - Infoblox
+
+Warning: Despite the TA indication this data source is CIM compliant the all versions of NIOS including the most recent available as of 2019-12-17 do not support the DNS data model correctly. For DNS security use cases use Splunk Stream instead.
+
+## Product - NIOS
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/2934/                                                                 |
+| Product Manual | https://docs.infoblox.com/display/ILP/NIOS?preview=/8945695/43728387/NIOS_8.4_Admin_Guide.pdf   |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| infoblox:dns        | None                                                                                                    |
+| infoblox:dhcp    | None                                                                                         |
+| infoblox:threat     | None                                                                                          |
+| nix:syslog     | None                                                                                          |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| infoblox_dns      | infoblox:dns       | netdns          | none          |
+| infoblox_dhcp    | infoblox:dhcp      | netipam          | none          |
+| infoblox_threat    | infoblox:threat      | netids          | none          |
+| nix_syslog    | nix:syslog      | osnix          | none          |
+
+### Filter type
+
+Must be identified by host or ip assignment. Update the filter `f_infoblox` or configure a dedicated port as required
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the admin manual for specific details of configuration
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_INFOBLOX_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_INFOBLOX_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_INFOBLOX | no | Enable archive to disk for this specific source |
+| SC4S_DEST_INFOBLOX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active device will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=infoblox:*| stats count by host
+```

docs/sources/Juniper/index.md

@@ -37,9 +37,11 @@
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_JUNIPER_JUNOS_LEGACY_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using legacy 3164 format|
-| SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using 5424 format |
-| SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source |
+| SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using legacy 3164 format|
+| SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined using legacy 3164 format|
+| SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined using legacy 3164 format|
+| SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined using 5424 format |
+| SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined using 5424 format || SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_JUNOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 
 ### Verification
@@ -90,6 +92,7 @@ Verify timestamp, and host values match as expected
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
 | SC4S_LISTEN_JUNIPER_NSM_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_JUNIPER_NSM_TLS_PORT      | empty string      | Enable at TLS port for this specific vendor product using the number defined |
 | SC4S_LISTEN_JUNIPER_NSM_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_JUNIPER_NSM | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_NSM_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
@@ -142,6 +145,7 @@ Verify timestamp, and host values match as expected
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
 | SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
 | SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_JUNIPER_NETSCREEN | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_NETSCREEN_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
@@ -192,6 +196,7 @@ Verify timestamp, and host values match as expected
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
 | SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
 | SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_JUNIPER_JUNOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
@@ -204,4 +209,4 @@ Use the following search to validate events are present; for Juniper SSL VPN ens
 index=<asconfigured> sourcetype=juniper:sslvpn | stats count by host
 ```
 
-Verify timestamp, and host values match as expected
+Verify timestamp, and host values match as expected

docs/sources/PaloaltoNetworks/index.md

@@ -53,6 +53,7 @@ MSG Parse: This filter parses message content
 | SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
 | SC4S_ARCHIVE_PALOALTO_PANOS | no | Enable archive to disk for this specific source |
 | SC4S_DEST_PALOALTO_PANOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS | no | Use custom time stamp parsing with ms added |
 
 ### Verification
 

docs/sources/Symantec/index.md

@@ -49,3 +49,53 @@ An active proxy will generate frequent events. Use the following search to valid
 ```
 index=<asconfigured> sourcetype=bluecoat:proxysg:access:kv | stats count by host
 ```
+
+## Product - Mail Gateway (Brightmail)
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | TBD                                                            |
+| Product Manual | https://support.symantec.com/us/en/article.howto38250.html                                                       |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| symantec:smg        | Requires version TA 3.6                                                                                                    |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| symantec_brightmail      | symantec:smg     | email          | none          |
+
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Setup and Configuration
+
+* No TA available
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL | no | Enable archive to disk for this specific source |
+| SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG | yes | Email processing events generated by the bmserver process will be grouped by host+program+pid+msg ID into a single event |
+### Verification
+
+An active mail server will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype=symantec:smg | stats count by host
+```

docs/sources/VMWare/index.md

@@ -0,0 +1,53 @@
+# Vendor - Dell - VMWare
+
+## Product - vSphwere - ESX NSX (Controller, Manager, Edge)
+
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | None                                                                |
+| Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html |
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| vmware:vsphere:nsx | None |
+| vmware:vsphere:esx | None |
+| nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx  |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| vmware_nsx      | vmware:vsphere:nsx | main          | none          |
+| vmware_esx      | vmware:vsphere:esx | main          | none          |
+
+### Filter type
+
+MSG Parse: This filter parses message content when using the default configuration
+
+### Setup and Configuration
+
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration
+    * Select TCP or SSL transport option
+    * Ensure the format of the event is customized per Splunk documentation
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_VMWARE_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_VMWARE_TLS_PORT      | empty string      | Enable a TLS port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_VMWARE | no | Enable archive to disk for this specific source |
+| SC4S_DEST_VMWARE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+An active proxy will generate frequent events. Use the following search to validate events are present per source device
+
+```
+index=<asconfigured> sourcetype="vmware:*:vsphere:*" | stats count by host
+```