Merge branch 'develop' into mchavda-splunk/imperva-waf-test-cases-424

CSVD · May 20, 2020 · e71b4c7 · e71b4c7
2 parents 27e67a1 + 2ad58c9
commit e71b4c7
Show file tree

Hide file tree

Showing 60 changed files with 1,787 additions and 251 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -90,7 +90,7 @@ jobs:
     docker:
       - image: circleci/python:3.7
     environment:
-      SYSLOG: "syslog-ng-3.26.1"
+      SYSLOG: "syslog-ng-3.27.1"
       SPLUNK_VERSION: "8.0.2"
     <<: *test
   test-sc4s-next-splunk-8-0:
@@ -104,14 +104,14 @@ jobs:
     docker:
       - image: circleci/python:3.7
     environment:
-      SYSLOG: "syslog-ng-3.26.1"
+      SYSLOG: "syslog-ng-3.27.1"
       SPLUNK_VERSION: "7.3.4"
     <<: *test
   test-sc4s-current-splunk-7-2:
     docker:
       - image: circleci/python:3.7
     environment:
-      SYSLOG: "syslog-ng-3.26.1"
+      SYSLOG: "syslog-ng-3.27.1"
       SPLUNK_VERSION: "7.2.9"
     <<: *test
 
@@ -195,7 +195,7 @@ jobs:
     docker:
       - image: circleci/python:3.7
     environment:
-      SYSLOG: "syslog-ng-3.26.1"
+      SYSLOG: "syslog-ng-3.27.1"
     steps:
       - setup_remote_docker:
           docker_layer_caching: true
@@ -215,15 +215,18 @@ jobs:
           name: Build Docker
           command: |
             if [ -n "${CIRCLE_TAG}" ]; then VERSION=${CIRCLE_TAG}; else VERSION="$(./semtag getcurrent)"; fi
+            VERSION_DOCKER_MMP=$(echo  $VERSION | sed -n 's/v\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p')
             echo ${VERSION} >package/VERSION
             echo ${CIRCLE_SHA1}=${VERSION}
             docker build --build-arg BRANCH=${SYSLOG} package -t splunk/scs:${CIRCLE_SHA1}
             docker tag splunk/scs:${CIRCLE_SHA1} splunk/scs:${VERSION}
+            docker tag splunk/scs:${CIRCLE_SHA1} splunk/scs:${VERSION_DOCKER_MMP}
             docker tag splunk/scs:${CIRCLE_SHA1} splunk/scs:${CIRCLE_SHA1:0:7}
             docker push splunk/scs:${CIRCLE_SHA1}
             docker push splunk/scs:${VERSION}
+            docker push splunk/scs:${VERSION_DOCKER_MMP}
             docker push splunk/scs:${CIRCLE_SHA1:0:7}
-            docker save splunk/scs:${VERSION} | gzip -c > /tmp/workspace/oci_container.tar.gz
+            docker save splunk/scs:${VERSION_DOCKER_MMP} | gzip -c > /tmp/workspace/oci_container.tar.gz
       - go/install
       - run:
           name: "Publish on GitHub"

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,2 +1,2 @@
 * @rfaircloth-splunk
-package/etc/ @rfaircloth @mbonsack
+package/etc/ @rfaircloth @mbonsack @nandinivij
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -9,6 +9,8 @@ and variables needed to properly configure SC4S for your environment.
 |----------|---------------|-------------|
 | SPLUNK_HEC_URL | url | URL(s) of the Splunk endpoint, can be a single URL space seperated list |
 | SPLUNK_HEC_TOKEN | string | Splunk HTTP Event Collector Token |
+| SC4S_GLOBAL_DNS_USE | yes or no(default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header |
+| SC4S_CONTAINER_HOST | string | variable passed to the container to identify the actual log host for container implementations |
 
 * NOTE:  Do _not_ configure HEC Acknowledgement when deploying the HEC token on the Splunk side; the underlying syslog-ng http
 destination does not support this feature.  Moreover, HEC Ack would significantly degrade performance for streaming data such as
@@ -25,6 +27,7 @@ syslog.
 | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file |
 | SC4S_DEST_SPLUNK_HEC_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate |
 | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads).  This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. |
+| SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,<br>severity,<br>container,<br>loghost,<br>destport,<br>fromhostip,<br>proto<br><br>none | List of sc4s indexed fields that will be included with each event in Splunk (default is the entire list except "none").  Two other indexed fields, `sc4s_vendor_product` and `sc4s_syslog_format`, will also appear along with the fields selected via the list and cannot be turned on or off individually.  If no indexed fields are desired (including the two internal ones), set the value to the single value of "none".  When setting this variable, separate multiple entries with commas and do not include extra spaces.<br><br>This list maps to the following indexed fields that will appear in all Splunk events:<br>facility: sc4s_syslog_facility<br>severity: sc4s_syslog_severity<br>container: sc4s_container<br>loghost: sc4s_loghost<br>dport: sc4s_destport<br>fromhostip: sc4s_fromhostip<br>proto: sc4s_proto
 
 ## Alternate Destination Configuration
 

diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md
@@ -159,7 +159,6 @@ sudo bash /opt/sc4s/bin/preconfig.sh
 SYSLOGNG_OPTS=-f /etc/syslog-ng/syslog-ng.conf 
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
-SC4S_DEST_SPLUNK_HEC_WORKERS=6
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```

diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md
@@ -107,7 +107,6 @@ SC4S is almost entirely controlled through environment variables, which are read
 ```dotenv
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
-SC4S_DEST_SPLUNK_HEC_WORKERS=6
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```

diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md
@@ -115,7 +115,6 @@ SC4S is almost entirely controlled through environment variables, which are read
 ```dotenv
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
-SC4S_DEST_SPLUNK_HEC_WORKERS=6
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```

diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md
@@ -53,12 +53,14 @@ TimeoutStartSec=0
 Restart=always
 
 ExecStartPre=/usr/bin/docker pull $SC4S_IMAGE
+ExecStartPre=/usr/bin/bash -c "/usr/bin/systemctl set-environment SC4SHOST=$(hostname -s)"
 ExecStartPre=/usr/bin/docker run \
         --env-file=/opt/sc4s/env_file \
         "$SC4S_LOCAL_CONFIG_MOUNT" \
         --name SC4S_preflight \
         --rm $SC4S_IMAGE -s
 ExecStart=/usr/bin/docker run -p 514:514 -p 514:514/udp -p 6514:6514 \
+        -e "SC4S_CONTAINER_HOST=${SC4SHOST}" \
         --env-file=/opt/sc4s/env_file \
         "$SC4S_PERSIST_VOLUME" \
         "$SC4S_LOCAL_CONFIG_MOUNT" \
@@ -113,7 +115,6 @@ SC4S is almost entirely controlled through environment variables, which are read
 ```dotenv
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
-SC4S_DEST_SPLUNK_HEC_WORKERS=6
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```

diff --git a/docs/gettingstarted/index.md b/docs/gettingstarted/index.md
@@ -40,6 +40,7 @@ using the SC4S defaults. SC4S can be easily customized to use different indexes
 * netfw
 * netids
 * netops
+* netwaf
 * netproxy
 * netipam
 * oswinsec

diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md
@@ -1,8 +1,39 @@
 
-# WARNING:  Do _not_ use Podman with RHEL/CentOS 7.x or earlier!
+# UPDATE:  Podman/RHEL UDP data block issue: netfilter connection table
 
-There have been cases where UDP packet loss is noted when Podman is used with RHEL/CentOS 7.x versions.  Stay tuned; the cause is
-currently unkown.
+We have determined the root cause for the issue with UDP data blocking and Podman/RHEL.  The crux of the issue is that the netfilter
+connection tables are _not_ udpdated when a new container starts _and_ there is a constant stream of UDP traffic from a given IP destined
+for a given port.  The table is _only_ updated if the trafffic pauses for the length of the connection table timeout (30 seconds by default).  
+
+Therefore, if you attempt to start up sc4s on a server to which, for example, a firewall is sending a steady stream of UDP events, the kernel
+will mistakenly keep trying to route the packets to the server itself rather than through the virtual network created by the new container.
+Until the firewall pauses its output stream (unlikely) _or_ the workaround provided below is applied, traffic from that particular firewall
+will never been seen by the container (and hence sc4s).
+
+## WORKAROUND
+
+There is a utility called `conntrack` that allows you to view/manipulate the netfilter connection tables in real time. Follow the steps below
+to install and run it each time sc4s starts.  It should be available in all RHEL 7/8 subscriptions.
+
+```
+<dnf or yum> install conntrack
+```
+
+After this is done, add the following entry to the unit file (and/or use the command when starting sc4s manually).
+Note that the space on either side of the semicolon in the `ExecStartPost` entry is _required_ and systemd
+will error out if it is missing.
+
+```
+ExecStartPost=sleep 2 ; conntrack -D -p udp
+```
+
+This command will delete the old (stale) UDP entries two seconds after the container starts and allow the system to build a new table that
+will properly route to the container when it sees UDP traffic.  Note that this command resets the table for _all_ UDP
+ports; for a purpose-built sc4s server this should not cause issues.  If for any reason more granular control over _which_ UPD ports are
+reset is desired, there are additional arguments to `conntrack` that can be used to select the specific UDP ports that are deleted in the
+table.  See the man page for `conntrack` for more information.
+
+The unit file entry above has been added to the example below for completeness.
 
 # Install podman
 
@@ -40,19 +71,22 @@ TimeoutStartSec=0
 Restart=always
 
 ExecStartPre=/usr/bin/podman pull $SC4S_IMAGE
+ExecStartPre=/usr/bin/bash -c "/usr/bin/systemctl set-environment SC4SHOST=$(hostname -s)"
 ExecStartPre=/usr/bin/podman run \
         --env-file=/opt/sc4s/env_file \
         "$SC4S_LOCAL_CONFIG_MOUNT" \
         --name SC4S_preflight \
         --rm $SC4S_IMAGE -s
 ExecStart=/usr/bin/podman run -p 514:514 -p 514:514/udp -p 6514:6514 \
+        -e "SC4S_CONTAINER_HOST=${SC4SHOST}" \
         --env-file=/opt/sc4s/env_file \
         "$SC4S_PERSIST_VOLUME" \
         "$SC4S_LOCAL_CONFIG_MOUNT" \
         "$SC4S_LOCAL_ARCHIVE_MOUNT" \
         "$SC4S_TLS_DIR" \
         --name SC4S \
         --rm $SC4S_IMAGE
+ExecStartPost=sleep 2 ; conntrack -D -p udp
 ```
 
 * Execute the following command to create a local volume that will contain the disk buffer files in the event of a communication
@@ -100,7 +134,6 @@ SC4S is almost entirely controlled through environment variables, which are read
 ```dotenv
 SPLUNK_HEC_URL=https://splunk.smg.aws:8088
 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
-SC4S_DEST_SPLUNK_HEC_WORKERS=6
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no
 ```

diff --git a/docs/sources/Checkpoint/index.md b/docs/sources/Checkpoint/index.md
@@ -51,6 +51,7 @@ MSG Parse: This filter parses message content
 | SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the port number defined |
 | SC4S_ARCHIVE_CHECKPOINT_SPLUNK | no | Enable archive to disk for this specific source |
 | SC4S_DEST_CHECKPOINT_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL | no | Suppress any duplicate product+loguid pairs processed within 2 seconds of the last matching event |
 
 ### Verification
 

diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md
@@ -95,27 +95,30 @@ Use the following search to validate events are present
 index=<asconfigured> sourcetype=cisco:apic:*
 ```
 
-Verify timestamp, and host values match as expected    
+Verify timestamp, and host values match as expected 
 
 ## Product - ASA AND FTD (Firepower)
 
 | Ref            | Link                                                                                                    |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| Splunk Add-on  | https://splunkbase.splunk.com/app/1620/                                                                 |
+| Splunk Add-on for ASA | https://splunkbase.splunk.com/app/1620/                                                          |
+| Cisco eStreamer for Splunk | https://splunkbase.splunk.com/app/1629/                                                     |
 | Product Manual | https://www.cisco.com/c/en/us/td/docs/security/asa/asa82/configuration/guide/config/monitor_syslog.html |
 
 
 ### Sourcetypes
 
 | sourcetype     | notes                                                                                                   |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| cisco:asa      | cisco FTD Firepower will also use this source type                                                      |
+| cisco:asa      | cisco FTD Firepower will also use this source type except those noted below                                                      |
+| cisco:firepower:syslog | FTD Unified events see https://www.cisco.com/c/en/us/td/docs/security/firepower/Syslogs/b_fptd_syslog_guide.pdf |
 
 ### Sourcetype and Index Configuration
 
 | key            | sourcetype     | index          | notes          |
 |----------------|----------------|----------------|----------------|
 | cisco_asa      | cisco:asa      | netfw          | none           |
+| cisco_ftd      | cisco:firepower:syslog      | netfw          | none           |
 
 
 ### Filter type
@@ -375,4 +378,57 @@ Use the following search to validate events are present
 index=<asconfigured> sourcetype=cisco:ucm
 ```
 
-Verify timestamp, and host values match as expected    
+Verify timestamp, and host values match as expected
+
+## Product - WSA
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/1747/                                                                 |
+| Product Manual | https://www.cisco.com/c/en/us/td/docs/security/wsa/wsa11-7/user_guide/b_WSA_UserGuide_11_7.html |
+
+* Update ``vi /opt/sc4s/local/context/vendor_product_by_source.conf `` update the host or ip mask for ``f_cisco_wsa`` to identiy the wsa events prior to WSA v11.7 and ``f_cisco_wsa11_7`` to identify the events since WSA v11.7.
+
+
+### Sourcetypes
+
+| cisco:wsa:l4tm      | The L4TM logs of Cisco IronPort WSA record sites added to the L4TM block and allow lists.                                                                                                    |
+| cisco:wsa:squid      | The access logs of Cisco IronPort WSA version prior to 11.7 record Web Proxy client history in squid.                                                                                           |
+| cisco:wsa:squid:new     | The access logs of Cisco IronPort WSA version since 11.7 record Web Proxy client history in squid.                                                                                           |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| cisco_wsa_l4tm    | cisco:wsa:l4tm    | netops          | None     |
+| cisco_wsa_squid    | cisco:wsa:squid    | netops          | None     |
+| cisco_wsa_squid_new    | cisco:wsa:squid:new    | netops          | None     |
+
+### Filter type
+
+IP, Netmask or Host
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* WSA Follow vendor configuration steps per Product Manual.
+* Ensure host and timestamp are included.
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_CISCO_WSA_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the number defined |
+| SC4S_LISTEN_CISCO_WSA_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the number defined |
+| SC4S_ARCHIVE_CISCO_WSA | no | Enable archive to disk for this specific source |
+| SC4S_DEST_CISCO_WSA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+Use the following search to validate events are present
+
+```
+index=netops sourcetype=cisco:wsa:*
+```
+
+Verify timestamp, and host values match as expected
diff --git a/docs/sources/F5/index.md b/docs/sources/F5/index.md
@@ -15,15 +15,21 @@
 |----------------|---------------------------------------------------------------------------------------------------------|
 | f5:bigip:syslog        | None                                                                                                    |
 | f5:bigip:irule    | None                                                                                         |
+| f5:bigip:ltm:http:irule | None |
+| f5:bigip:gtm:dns:request:irule | None |
+| f5:bigip:gtm:dns:response:irule | None |
+| f5:bigip:ltm:failed:irule | None |
+| f5:bigip:asm:syslog | None |
 | nix:syslog     | None                                                                                          |
 
-### Sourcetype and Index Configuration
+### Index Configuration
 
-| key            | sourcetype     | index          | notes          |
-|----------------|----------------|----------------|----------------|
-| f5_bigip      | f5:bigip:syslog       | netops          | none          |
-| f5_bigip_irule    | f5:bigip:syslog      | netops          | none          |
-| f5_bigip_nix    | nix:syslog      | netops          | if `f_f5_bigip` is not set the index osnix will be used          |
+| key            | index          | notes          |
+|----------------|----------------|----------------|
+| f5_bigip       | netops          | none          |
+| f5_bigip_irule | netops          | none          |
+| f5_bigip_asm   | netwaf          | none          |
+| f5_bigip_nix   | netops          | if `f_f5_bigip` is not set the index osnix will be used          |
 
 ### Filter type
 

diff --git a/docs/sources/Juniper/index.md b/docs/sources/Juniper/index.md
@@ -12,7 +12,11 @@
 | sourcetype               | notes                                                            |
 |--------------------------|------------------------------------------------------------------|
 | juniper:junos:firewall   | None                                                             |
-| juniper:junos:idp        | None                                                             |
+| juniper:junos:firewall:structured   | None                                                             |
+| juniper:junos:idp   | None                                                             |
+| juniper:junos:idp:structured   | None                                                             |
+| juniper:junos:aamw:structured   | None                                                             |
+| juniper:junos:secintel:structured   | None                                                             |
 
 ### Sourcetype and Index Configuration
 

diff --git a/docs/sources/PaloaltoNetworks/index.md b/docs/sources/PaloaltoNetworks/index.md
@@ -17,7 +17,7 @@
 | pan:threat     | None                                                                                          |
 | pan:system     | None                                                                                          |
 | pan:config     | None                                                                                          |
-| pan:hipwatch   | None                                                                                          |
+| pan:hipmatch   | None                                                                                          |
 | pan:correlation | None                                                                                          |
 
 ### Sourcetype and Index Configuration
@@ -29,7 +29,7 @@
 | pan_threat    | pan:threat      | netproxy          | none          |
 | pan_system    | pan:system      | netops          | none          |
 | pan_config    | pan:config      | netops          | none          |
-| pan_hipwatch    | pan:hipwatch      | netops          | none          |
+| hipmatch    | pan:hipmatch      | netops          | none          |
 | pan_correlation    | pan:correlation      | netops          | none          |
 
 ### Filter type