From 8d007ebb57f5a348829a2a4fcc729e14233ee5ae Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Tue, 12 Nov 2019 09:55:38 -0500 Subject: [PATCH] Release/1.1.0 (#189) * Feature/cisco ise (#178) This merge aggregates Cisco ISE messages from multiple segments into a single event in Splunk * Support for archival file destinations (#179) This merge add support for file archival in syslog-ng EWMM format * Feature/improvedmetrics (#185) * This change correct an incorrect URL issue preventing metrics flow and cleans up related documentation and filters anon metrics * Update Metrics for Splunk 8 * Resolve splunk sdk for CI unit tests move * Add disk buffer envrionment variables table (#184) * Add disk buffer envrionment variables table Add env var table to docs Adjust default disk buffer to 50 GB * Disk Buffer refinement Perform internal math to convert specified buffer sizes to values appropriate for internal (syslog-ng) destination options, which are set per worker. * Add Ubiquiti Networks Unifi product range (#188) * Resolve fallback events in msg rather than JSON mode * Add support for Ubiquiti networks unifi product line AP, Switch, USG (firewalls) --- .env.template | 4 - docker-compose-ci.yml | 1 - docker-compose-debug.yml | 1 - docker-compose-demo.yml | 1 - docker-compose-perf.yml | 1 - docker-compose.yml | 13 +- docs/configuration.md | 42 +++++- docs/gettingstarted/byoe-rhel7.md | 5 +- docs/gettingstarted/docker-swarm-general.md | 10 +- docs/gettingstarted/docker-swarm-rhel7.md | 10 +- docs/gettingstarted/docker-systemd-general.md | 10 +- docs/gettingstarted/podman-systemd-general.md | 10 +- docs/sources.md | 123 +++++++++++++++- package/Dockerfile | 4 - .../conf.d/conflib/_splunk/splunkfields.conf | 1 - .../etc/conf.d/destinations/archive_file.conf | 5 + .../conf.d/destinations/splunk_hec.conf.tmpl | 13 +- .../splunk_hec_internal.conf.tmpl | 4 +- .../destinations/splunk_hec_metrics.conf.tmpl | 2 +- .../etc/conf.d/filters/Ubiquiti/unifi.conf | 7 + package/etc/conf.d/filters/cisco/ise.conf | 4 + .../local/config/log_paths/example.conf.tmpl | 6 +- .../etc/conf.d/log_paths/internal.conf.tmpl | 28 +--- .../p_rfc3164-checkpoint_splunk.conf.tmpl | 50 ++++--- .../log_paths/p_rfc3164-cisco_asa.conf.tmpl | 30 +++- .../log_paths/p_rfc3164-cisco_ios.conf.tmpl | 30 +++- .../log_paths/p_rfc3164-cisco_ise.conf.tmpl | 106 ++++++++++++++ .../log_paths/p_rfc3164-cisco_nxos.conf.tmpl | 32 +++- .../p_rfc3164-forcepoint_webprotect.conf.tmpl | 27 +++- .../p_rfc3164-fortinet_fortios.conf.tmpl | 38 +++-- .../log_paths/p_rfc3164-juniper_idp.conf.tmpl | 33 ++++- .../p_rfc3164-juniper_junos.conf.tmpl | 40 +++-- .../p_rfc3164-juniper_netscreen.conf.tmpl | 30 +++- .../log_paths/p_rfc3164-juniper_nsm.conf.tmpl | 32 +++- .../p_rfc3164-juniper_nsm_idp.conf.tmpl | 32 +++- .../p_rfc3164-microfocus_arcsight.conf.tmpl | 36 +++-- .../p_rfc3164-paloalto_panos.conf.tmpl | 42 ++++-- .../p_rfc3164-proofpoint_pps_filter.conf.tmpl | 29 +++- ..._rfc3164-proofpoint_pps_sendmail.conf.tmpl | 28 +++- .../p_rfc3164-ubiquiti_unifi.conf.tmpl | 137 ++++++++++++++++++ .../log_paths/p_rfc3164-zscaler_nss.conf.tmpl | 48 ++++-- .../p_rfc5424-noversion_cisco_asa.conf.tmpl | 28 +++- ...rfc5424-noversion_symantec_proxy.conf.tmpl | 30 +++- .../p_rfc5424-strict_juniper_junos.conf.tmpl | 38 +++-- .../p_rfc5424_epoch-cisco_meraki.conf.tmpl | 49 +++++++ .../p_rfc5424_epoch-cisco_merkai.conf.tmpl | 42 ------ package/etc/conf.d/log_paths/zfallback.conf | 7 +- package/etc/conf.d/sources/network.conf.tmpl | 103 +------------ .../etc/context_templates/splunk_index.csv | 1 + .../vendor_product_by_source.conf | 4 + .../vendor_product_by_source.csv | 3 +- package/etc/go_templates/source_network.t | 22 ++- .../local_config/log_paths/example.conf.tmpl | 6 +- package/etc/syslog-ng.conf | 8 +- package/sbin/entrypoint.sh | 14 +- .../apps/SA-syslog-ng/default/indexes.conf | 11 ++ tests/requirements.txt | 2 +- tests/test_cisco_ise.py | 52 +++++++ tests/test_common.py | 12 ++ tests/test_ubiquiti_unifi.py | 93 ++++++++++++ 60 files changed, 1195 insertions(+), 435 deletions(-) create mode 100644 package/etc/conf.d/destinations/archive_file.conf create mode 100644 package/etc/conf.d/filters/Ubiquiti/unifi.conf create mode 100644 package/etc/conf.d/filters/cisco/ise.conf create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl create mode 100644 package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl delete mode 100644 package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl create mode 100644 tests/test_cisco_ise.py create mode 100644 tests/test_ubiquiti_unifi.py diff --git a/.env.template b/.env.template index c1f7af2..f360cd6 100644 --- a/.env.template +++ b/.env.template @@ -12,10 +12,6 @@ SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SPLUNK_PASSWORD=Changed@11 SPLUNK_START_ARGS=--accept-license SPLUNK_HEC_URL=https://splunk:8088/services/collector/event -SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download #SPLUNKBASE_USERNAME=username #SPLUNKBASE_PASSWORD=password diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index 6028e6e..a699498 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -33,7 +33,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml index 9b885f3..bb22763 100644 --- a/docker-compose-debug.yml +++ b/docker-compose-debug.yml @@ -33,7 +33,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose-demo.yml b/docker-compose-demo.yml index c4fb88f..3934fe1 100644 --- a/docker-compose-demo.yml +++ b/docker-compose-demo.yml @@ -34,7 +34,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml index 9831278..e1bd795 100644 --- a/docker-compose-perf.yml +++ b/docker-compose-perf.yml @@ -26,7 +26,6 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} diff --git a/docker-compose.yml b/docker-compose.yml index a9e5e74..082040f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,8 @@ services: RH_ORG: ${RH_ORG} RH_ACTIVATION: ${RH_ACTIVATION} hostname: sc4s - command: -det +#When this is enabled test_common will fail +# command: -det ports: - "514:514" - "601:601" @@ -43,14 +44,16 @@ services: - splunk environment: - SPLUNK_HEC_URL=${SPLUNK_HEC_URL} - - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL} - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD} - - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX} - - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX} - SC4S_SOURCE_TLS_ENABLE=no - SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no + - SC4S_LISTEN_DEFAULT_TCP_PORT=514 + - SC4S_LISTEN_DEFAULT_UDP_PORT=514 +# - SC4S_LISTEN_DEFAULT_TLS_PORT=6514 - SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 + - SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT=6000 +# - SC4S_ARCHIVE_CHECKPOINT=yes + - SC4S_ARCHIVE_GLOBAL=yes volumes: - ./tls:/opt/syslog-ng/tls splunk: diff --git a/docs/configuration.md b/docs/configuration.md index aa91a1a..379edf7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,11 +21,51 @@ and variables needed to properly configure SC4S for your environment. | SC4S_DEST_SPLUNK_HEC_SSL_VERSION | comma separated list | Open SSL version list | | SC4S_DEST_SPLUNK_HEC_TLS_CA_FILE | path | Custom trusted cert file | +## SC4S Disk Buffer Configuration + +Disk buffers in SC4S are allocated _per destination_. In the future as more destinations are supported, a separate list of variables +will be used for each. This is why you see the `DEST_SPLUNK_HEC` in the variable names below. +* NOTE: "Reliable" disk buffering offeres little advantage over "normal" disk buffering, at a significant performance penalty. +For this reason, normal disk buffering is recommended. +* NOTE: If you add destinations locally in your configuration, pay attention to the _cumulative_ buffer requirements when allocating local +disk. +* NOTE: The values for the variables below represent the _total_ sizes of the buffers for the destination. These sizes are divded by the +number of workers (threads) when setting the actual syslog-ng buffer options, because the buffer options apply to each worker rather than the +entire destination. Pay careful attention to this when using the "BYOE" version of SC4S, where direct access to the syslog-ng config files +may hide this nuance. + +| Variable | Values/Default | Description | +|----------|---------------|-------------| +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_ENABLE | yes(default) or no | Enable local disk buffering | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE | yes or no(default) | Enable reliable/normal disk buffering (normal recommended) | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE | bytes (10241024) | Memory buffer size in bytes (used with reliable disk buffering) | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH |messages (15000) | Memory buffer size in message count (used with normal disk buffering) | +| SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE | bytes (53687091200) | size of local disk buffer in bytes (default 50 GB) | + +## Archive File Configuration + +This feature is designed to support "compliance" archival of all messages. To enable this feature update the Unit file +or docker compose to mount an appropriate host folder to the container folder ``/opt/syslog-ng/var/archive``. +The files will be stored in a folder structure using the naming pattern +``${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log"``. +This pattern will create one file per "vendor_product" per minute with records formatted using syslog-ng's EWMM template. + +**WARNING POTENTIAL OUTAGE CAUSING CONSEQUENCE** + +SC4S does not prune the files that are created. The administrator must provide a means of log rotation to prune files +and/or move them to an archival system to avoid disk space failures. + +| Variable | Values | Description | +|----------|---------------|-------------| +| SC4S_ARCHIVE_GLOBAL | yes or undefined | Enable archive of all vendor_products | +| SC4S_ARCHIVE_LISTEN_ | yes(default) or undefined | See sources section of documentation enables selective archival | + + ## Syslog Source Configuration | Variable | Values/Default | Description | |----------|----------------|-------------| -| SC4S_SOURCE_TLS_ENABLE | no(default) or yes | Enable a TLS listener on port 6514 | +| SC4S_LISTEN_DEFAULT_TLS_PORT | undefined or 6514 | Enable a TLS listener on port 6514 | | SC4S_SOURCE_TLS_OPTIONS | See openssl | List of SSl/TLS protocol versions to support | | SC4S_SOURCE_TLS_CIPHER_SUITE | See openssl | List of Ciphers to support | | SC4S_SOURCE_TCP_MAX_CONNECTIONS | 2000 | Max number of TCP Connections | diff --git a/docs/gettingstarted/byoe-rhel7.md b/docs/gettingstarted/byoe-rhel7.md index 73b7b33..d970dcd 100644 --- a/docs/gettingstarted/byoe-rhel7.md +++ b/docs/gettingstarted/byoe-rhel7.md @@ -129,12 +129,9 @@ sudo bash /opt/sc4s/bin/preconfig.sh ```dotenv SYSLOGNG_OPTS=-f /opt/syslog-ng/etc/syslog-ng.conf -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` diff --git a/docs/gettingstarted/docker-swarm-general.md b/docs/gettingstarted/docker-swarm-general.md index 8f536d1..6fd894f 100644 --- a/docs/gettingstarted/docker-swarm-general.md +++ b/docs/gettingstarted/docker-swarm-general.md @@ -65,12 +65,9 @@ of events in the event of network failure to the Splunk infrastructure. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -179,12 +176,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/docs/gettingstarted/docker-swarm-rhel7.md b/docs/gettingstarted/docker-swarm-rhel7.md index ec93e82..5ef0e66 100644 --- a/docs/gettingstarted/docker-swarm-rhel7.md +++ b/docs/gettingstarted/docker-swarm-rhel7.md @@ -92,12 +92,9 @@ again upon restart. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -208,12 +205,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md index 27fa688..94b164f 100644 --- a/docs/gettingstarted/docker-systemd-general.md +++ b/docs/gettingstarted/docker-systemd-general.md @@ -71,12 +71,9 @@ unit file above. Failure to do this will cause SC4S to abort at startup. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -182,12 +179,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md index 1dea7bd..1d0ddc3 100644 --- a/docs/gettingstarted/podman-systemd-general.md +++ b/docs/gettingstarted/podman-systemd-general.md @@ -71,12 +71,9 @@ unit file above. Failure to do this will cause SC4S to abort at startup. Create a file named ``/opt/sc4s/env_file`` and add the following environment variables: ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no ``` @@ -182,12 +179,9 @@ match this value to the total number of indexers behind the load balancer. uncomment the last line in the example below. ```dotenv -SPLUNK_HEC_URL=https://splunk.smg.aws:8088/services/collector/event +SPLUNK_HEC_URL=https://splunk.smg.aws:8088 SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SC4S_DEST_SPLUNK_HEC_WORKERS=6 -SPLUNK_CONNECT_METHOD=hec -SPLUNK_DEFAULT_INDEX=main -SPLUNK_METRICS_INDEX=em_metrics SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT=5000 #Uncomment the following line if using untrusted SSL certificates #SC4S_DEST_SPLUNK_HEC_TLS_VERIFY=no diff --git a/docs/sources.md b/docs/sources.md index 5b912bc..22c3aab 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -191,6 +191,52 @@ Use the following search to validate events are present, for NX-OS, WLC and ACI index= sourcetype=cisco:ios | stats count by host ``` +## Product - ISE + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/1915/ | +| Product Manual | https://www.cisco.com/c/en/us/td/docs/security/ise/2-6/Cisco_ISE_Syslogs/Cisco_ISE_Syslogs/Cisco_ISE_Syslogs_chapter_00.html | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cisco:ise:syslog | Aggregation used | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_ise | cisco:ise:syslog | netauth | None | + + +### Filter type + +PATTERN MATCH + +### Setup and Configuration + +* No special steps required + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CISCO_ISE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_ISE_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | + +### Verification + +Use the following search to validate events are present + +``` +index= sourcetype=cisco:ise:syslog +``` + +Verify timestamp, and host values match as expected + ## Product - Meraki Product Line MR, MS, MX, MV | Ref | Link | @@ -237,9 +283,6 @@ Use the following search to validate events are present index= sourcetype=merkai ``` -Verify timestamp, and host values match as expected - - Verify timestamp, and host values match as expected # Vendor - Forcepoint @@ -905,6 +948,80 @@ index= sourcetype=bluecoat:proxysg:access:kv | stats count by host ``` +# Vendor - Ubiquiti - Unifi + +All Ubiquity Unfi firewalls, switches, and access points share a common syslog configuration via the NMS. + + +* Login to NMS +* Navigate to settings +* Navigate to Site +* Enable Remote syslog server +* Enter hostname and port +* Update ``vi /opt/sc4s/local/context/vendor_product_by_source.conf `` update the host or ip mask for ``f_ubiquiti_unifi_fw`` to identify USG firewalls + +## Product - Unifi Switch and Access Points + +Unifi devices are managed using the Network Management Controller + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/4107/ | +| Product Manual | https://https://help.ubnt.com/ | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| ubnt | Used when no sub source type is required by add on | +| ubnt:fw | USG events | +| ubnt:threat | USG IDS events | +| ubnt:switch | Unifi Switches | +| ubnt:wireless | Access Point logs | + + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| ubiquiti_unifi | ubnt | netops | none | +| ubiquiti_unifi_fw | ubnt:fw | netfw | none | +| ubiquiti_unifi_link | ubnt:link | netops | none | +| ubiquiti_unifi_sudo | ubnt:sudo | netops | none | +| ubiquiti_unifi_switch | ubnt:switch | netops | none | +| ubiquiti_unifi_threat | ubnt:threat | netids | none | +| ubiquiti_unifi_wireless | ubnt:wireless | netops | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_ZSCALER_NSS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=zscalernss-* | stats count by host +``` + + # Vendor - Zscaler ## Product - All Products diff --git a/package/Dockerfile b/package/Dockerfile index 7008a31..a5cbb4b 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -72,8 +72,6 @@ RUN cd /tmp ;\ source scl_source enable rh-python36 ENV DEBCONF_NONINTERACTIVE_SEEN=true -ENV SPLUNK_CONNECT_METHOD=hec -ENV SYSLOGNG_HEC_WORKERS=10 RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh COPY goss.yaml /etc/goss.yaml @@ -98,8 +96,6 @@ EXPOSE 514 EXPOSE 601/tcp EXPOSE 6514/tcp -ENV SPLUNK_CONNECT_METHOD=UF - ENTRYPOINT ["/entrypoint.sh", "-F"] HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/etc/conf.d/conflib/_splunk/splunkfields.conf index c195deb..bf911a6 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf @@ -17,7 +17,6 @@ block rewrite r_set_splunk_dest_default( set("`index`", value(".splunk.index")); set("`source`", value(".splunk.source")); set("`sourcetype`", value(".splunk.sourcetype")); - set("`template`", value("fields.sc4s_template")); }; diff --git a/package/etc/conf.d/destinations/archive_file.conf b/package/etc/conf.d/destinations/archive_file.conf new file mode 100644 index 0000000..efd7990 --- /dev/null +++ b/package/etc/conf.d/destinations/archive_file.conf @@ -0,0 +1,5 @@ +destination d_archive { + file("/opt/syslog-ng/var/archive/${YEAR}/${MONTH}/${DAY}/${fields.sc4s_vendor_product}_${YEAR}${MONTH}${DAY}${HOUR}${MIN}.log" + template('$(format-ewmm)') + ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index 6a31372..bc92162 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -1,6 +1,6 @@ destination d_hec { http( - url("{{- getenv "SPLUNK_HEC_URL"}}") + url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event") method("POST") log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) workers({{- getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10"}}) @@ -18,15 +18,14 @@ destination d_hec { disk-buffer( {{- if eq (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_RELIABLE" "no") "yes"}} - mem-buf-size({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024"}}) + mem-buf-size({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFSIZE" "10241024") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) reliable(yes) {{- else}} - mem-buf-length({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000"}}) + mem-buf-length({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_MEMBUFLENGTH" "15000") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) reliable(no) {{- end}} - - disk-buf-size({{- getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "1048576"}}) - dir("/opt/syslog-ng/var/data/disk-buffer/") + disk-buf-size({{conv.ToInt64 (math.Round ( math.Div (getenv "SC4S_DEST_SPLUNK_HEC_DISKBUFF_DISKBUFSIZE" "53687091200") (getenv "SC4S_DEST_SPLUNK_HEC_WORKERS" "10")))}}) + dir("/opt/syslog-ng/var/data/disk-buffer/") ) {{- end}} tls(peer-verify({{- getenv "SC4S_DEST_SPLUNK_HEC_TLS_VERIFY" "yes"}}) @@ -43,7 +42,7 @@ destination d_hec { source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$(template ${fields.sc4s_template} $(template "t_standard")) + event=$MSG fields.*)') ); }; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl index 875d3fb..550063c 100644 --- a/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_internal.conf.tmpl @@ -1,6 +1,6 @@ destination d_hec_internal { http( - url("{{- getenv "SPLUNK_HEC_URL"}}") + url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector/event") method("POST") log-fifo-size({{- getenv "SC4S_DEST_SPLUNK_HEC_LOG_FIFO_SIZE" "180000000"}}) workers(10) @@ -28,7 +28,7 @@ destination d_hec_internal { source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$(template ${fields.sc4s_template} $(template "t_standard")) + event=$MSG fields.*)') ); }; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl index a473a04..2593b8c 100644 --- a/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec_metrics.conf.tmpl @@ -1,6 +1,6 @@ destination d_hecmetrics { http( - url("{{- getenv "SPLUNK_HEC_URL"}}") + url("{{- getenv "SPLUNK_HEC_URL" | strings.TrimSuffix "/services/collector/event" | strings.TrimSuffix "/services/collector" }}/services/collector") method("POST") batch-lines(50) batch-bytes(1024Kb) diff --git a/package/etc/conf.d/filters/Ubiquiti/unifi.conf b/package/etc/conf.d/filters/Ubiquiti/unifi.conf new file mode 100644 index 0000000..49740eb --- /dev/null +++ b/package/etc/conf.d/filters/Ubiquiti/unifi.conf @@ -0,0 +1,7 @@ +filter f_ubiquiti_unifi { + host('^U[^,]{1,10},[a-z0-9]{9,16},v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6}') + or + program('^U[^,]{1,10},[a-z0-9]{9,16},v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6}') + or + match("ubiquiti_unifi_*", value("fields.sc4s_vendor_product") type(glob) ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/cisco/ise.conf b/package/etc/conf.d/filters/cisco/ise.conf new file mode 100644 index 0000000..e9a2c8a --- /dev/null +++ b/package/etc/conf.d/filters/cisco/ise.conf @@ -0,0 +1,4 @@ + +filter f_cisco_ise { + program("CISE_.*"); +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl index a8ac264..6a68b45 100644 --- a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl +++ b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl @@ -27,7 +27,7 @@ log { # The first time this template is used the log_path will be linked to the default port {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); # Filters should be updated to use the simplest and most effecient logic possible to discard # the message from this path @@ -40,12 +40,12 @@ log { # In the second pass through the template a link to the dedicated port is used. This # normally does not require additional filters -source (s_dedicated_port_LOCAL_EXAMPLE); +source (s_LOCAL_EXAMPLE); {{- end}} #Set a default sourcetype and index - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"))}; #using the key "local_example" find any cutomized index,source or sourcetype meta values diff --git a/package/etc/conf.d/log_paths/internal.conf.tmpl b/package/etc/conf.d/log_paths/internal.conf.tmpl index 7ed823e..c751b9e 100644 --- a/package/etc/conf.d/log_paths/internal.conf.tmpl +++ b/package/etc/conf.d/log_paths/internal.conf.tmpl @@ -8,35 +8,21 @@ log { parser {p_add_context_splunk(key("sc4s_metrics")); }; rewrite { - subst('(?:Log statistics; )?(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27(?:, )?', - ' - {"time": "$S_UNIXTIME.$S_MSEC", - "event": "metric", - "host": "$HOST", - "index": "${.splunk.index}", - "source": "internal", - "sourcetype": "${.splunk.sourcetype}", - "fields": { - "source_name": "${SourceName}", - "source_instance": "${SourceInstance}", - "state": "${State}", - "type": "${Type}", - "_value": ${Number}, - "metric_name": "syslogng.${SourceId}" - } - } - ', + subst('Log statistics; ', '', value("MESSAGE"), flags("utf8" "global")); + subst('([^= ]+=\x27[^\(]+\(#anon[^,\)]+(?:,[^,]+,[^\)]+)?\)\=\d+\x27(?:, )?)', '', value("MESSAGE"), flags("utf8" "global")); + subst('(?[^= ]+)=\x27(?[^\(]+)\((?[^,\)]+)(?:,(?[^,]+),(?[^\)]+))?\)\=(?\d+)\x27,? ?', +'{"time": "$S_UNIXTIME.$S_MSEC","event": "metric","host": "$HOST","index": "${.splunk.index}","source": "internal","sourcetype": "${.splunk.sourcetype}","fields": {"source_name": "${SourceName}","source_instance": "${SourceInstance}","state": "${State}","type": "${Type}","_value": ${Number},"metric_name": "syslogng.${SourceId}"}} +', value("MESSAGE") flags("utf8" "global") - ); + ); }; destination(d_hecmetrics); #--HEC-- } else { - {{- if eq (getenv "SC4S_DEBUG_STDOUT" "yes") "yes"}} destination(d_stdout); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:events"), index("main"))}; parser {p_add_context_splunk(key("sc4s_events")); }; destination(d_hec_internal); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index 0ad2806..ee81c82 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -1,64 +1,80 @@ -# Checkpoint Splunk format -{{- if (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT") "no") "no") }} +# Generate the custom port if defined {{ $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_checkpoint_splunk); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_CHECKPOINT_SPLUNK); + source (s_CHECKPOINT_SPLUNK); {{- end}} parser { kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); - }; - rewrite { set("${.kv.hostname}", value("HOST")); }; + rewrite { + set("${.kv.hostname}", value("HOST")); + set("checkpoint_splunk", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cp_log"), index("netops")) + }; - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), index("netops"), template("t_hdr_msg"))}; parser {p_add_context_splunk(key("checkpoint_splunk")); }; if { filter(f_checkpoint_splunk_NetworkTraffic); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))}; parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); }; } elif { filter(f_checkpoint_splunk_Web); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))}; parser {p_add_context_splunk(key("checkpoint_splunk_web")); }; } elif { filter(f_checkpoint_splunk_NetworkSessions); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))}; parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); }; } elif { filter(f_checkpoint_splunk_IDS_Malware); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))}; parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; } elif { filter(f_checkpoint_splunk_IDS); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))}; parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; } elif { filter(f_checkpoint_splunk_email); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))}; parser {p_add_context_splunk(key("checkpoint_splunk_email")); }; } elif { filter(f_checkpoint_splunk_DLP); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; }; - parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CHECKPOINT_SPLUNK") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl index 5e56956..e276484 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -1,25 +1,43 @@ # Cisco ASA -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common"}} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_cisco_asa); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_CISCO_ASA_LEGACY); + source (s_CISCO_ASA_LEGACY); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; + rewrite { + set("cisco_asa", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw")) + }; parser {p_add_context_splunk(key("cisco_asa")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ASA_LEGACY") }} + destination(d_archive); +{{- end}} + flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index ea17e4e..3c51cf6 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -1,26 +1,44 @@ # Cisco IOS -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_IOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_cisco_ios); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_CISCO_IOS); + source (s_CISCO_IOS); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_msg_only"))}; + rewrite { + set("cisco_ios", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops")) + }; parser { p_add_context_splunk(key("cisco_ios")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".cisco.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_IOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl new file mode 100644 index 0000000..b1d9297 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl @@ -0,0 +1,106 @@ +# Cisco ISE +{{ $context := dict "port_id" "CISCO_ISE" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} + +#This filter uses a field we set to prevent the original messages before aggregation from being +#sent to Splunk +filter f_cisco_ise_complete{ + match("yes", value("ISE.COMPLETE") type(glob)); +}; + +#This parser adds messages from ISE to a context without sending them +#forward to Splunk +parser ise_grouping { + csv-parser( + columns(PID, ISE.num, ISE.seq, MESSAGE) + delimiters(chars(" ")) + flags(greedy) + ); + grouping-by( + scope(program) + key("$PID") + trigger("$(+ ${ISE.seq} 1)" == "${ISE.num}") + sort-key("${ISE.seq}") + aggregate( + value("MESSAGE" "$(implode '' $(context-values ${MESSAGE}))") + value("ISE.COMPLETE" "yes") + ) + timeout(10) + ); +}; + +#The syslog message includes a date with nano seconds and TZ which is not in the header +#So must reparse the date +parser ise_event_time { + csv-parser( + columns(ISE.DATE, ISE.TIME, ISE.TZ, MESSAGE) + delimiters(chars(" ")) + flags(greedy) + ); + + date-parser( + #YYYY- MM-DD hh:mm:ss:xxx +/-zh:zm + format("%Y-%m-%d %H:%M:%S.%f %z" ) + template("${ISE.DATE} ${ISE.TIME} ${ISE.TZ}") + ); +}; +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_is_rfc3164); + filter(f_cisco_ise); +{{- end}} +{{- if eq (.) "no"}} + source (s_CISCO_ISE); +{{- end}} + + parser(ise_grouping); + + if { + filter(f_cisco_ise_complete); + parser(ise_event_time); + rewrite { + set("cisco_ise", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:ise:syslog"), index("netauth")) + }; + + parser {p_add_context_splunk(key("cisco_ise")); }; + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("PID")); + unset(value("LEGACY_MSGHDR")); + unset(value("EPOCH")); + unset(value("VERSION")); + unset(value("TIMESECFRAC")); + groupunset(values("ISE.*")); + }; + + destination(d_hec); + + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ISE") }} + destination(d_archive); + {{- end}} + + flags(flow-control); + }; + + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ISE_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for CISCO_ISE traffic + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for CISCO_ISE traffic +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index fdb592e..f2a4478 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -1,26 +1,44 @@ # Cisco NX_OS -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_cisco_nx_os); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_CISCO_NX_OS); + source (s_CISCO_NX_OS); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg"))}; + rewrite { + set("cisco_nxos", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"), template("t_hdr_msg")) + }; + parser { p_add_context_splunk(key("cisco_nx_os")); }; - parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_NXOS") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl index ac0cb48..759f523 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -1,28 +1,43 @@ # Forcepoint Webprotect -{{- if (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORCEPOINT_WEBPROTECT_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "FORCEPOINT_WEBPROTECT" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_forcepoint_webprotect_kv); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_FORCEPOINT_WEBPROTECT); + source (s_FORCEPOINT_WEBPROTECT); {{- end}} rewrite { subst(" [^ =]+\=\-", "", value("MESSAGE"), flags("global")); + set("forcepoint_webprotect", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg")) }; - rewrite { r_set_splunk_dest_default(sourcetype("websense:cg:kv"), index("netproxy"), template("t_hdr_msg"))}; parser {p_add_context_splunk(key("forcepoint_webprotect")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FORCEPOINT_WEBPROTECT") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index 094075e..39d2bc9 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -1,17 +1,17 @@ # Fortinet Fortios -{{- if (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_FORTINET_FORTIOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "FORTINET_FORTIOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_fortinet_fortios); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_FORTINET_FORTIOS); + source (s_FORTINET_FORTIOS); {{- end}} parser { @@ -19,25 +19,43 @@ log { date-parser(format("%Y-%m-%d:%H:%M:%S") template("${.kv.date}:${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); }; - rewrite { set("${.kv.devname}", value("HOST")); }; + rewrite { + set("${.kv.devname}", value("HOST")); + set("fortigate_fortios", value("fields.sc4s_vendor_product")); + }; if (match("traffic" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"))}; parser {p_add_context_splunk(key("fortinet_fortios_traffic")); }; } elif (match("utm" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"))}; parser {p_add_context_splunk(key("fortinet_fortios_utm")); }; } elif (match("event" value(".kv.type"))) { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"))}; parser {p_add_context_splunk(key("fortinet_fortios_event")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"))}; parser {p_add_context_splunk(key("fortinet_fortios_log")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FORTINET_FORTIOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl index 65dfb91..7038621 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -1,27 +1,46 @@ # Juniper IDP -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_IDP_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_IDP" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_strict); filter(f_juniper_idp); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_IDP); + source (s_JUNIPER_IDP); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids"), template("t_hdr_sdata_msg"))}; + rewrite { + set("juniper_idp", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids")) + }; parser { - p_add_context_splunk(key("juniper_idp")); + p_add_context_splunk(key("juniper_idp")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + unset(value("PID")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_IDP") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index 5b26695..4e3eea5 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -1,42 +1,60 @@ # Juniper JunOS -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_JUNOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_juniper_junos_standard); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_JUNOS); + source (s_JUNIPER_JUNOS); {{- end}} + rewrite { + set("juniper_junos", value("fields.sc4s_vendor_product")); + }; + if (program('RT_IDP')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"))}; parser {p_add_context_splunk(key("juniper_idp")); }; } elif (program('RT_FLOW') or message('PFE_FW_|DFWD_')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netfw"))}; parser {p_add_context_splunk(key("juniper_junos_fw")); }; } elif (program('RT_IDS')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"))}; parser {p_add_context_splunk(key("juniper_junos_ids")); }; } elif (program('RT_UTM')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netids"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netids"))}; parser {p_add_context_splunk(key("juniper_junos_utm")); }; } elif (program('Juniper')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:sslvpn"), index("netfw"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:sslvpn"), index("netfw"))}; parser {p_add_context_splunk(key("juniper_sslvpn")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:legacy"), index("netops"), template("t_standard"))}; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:legacy"), index("netops"))}; parser {p_add_context_splunk(key("juniper_legacy")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_JUNOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl index f530a14..4a9952e 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -1,22 +1,21 @@ # Juniper Netscreen -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_NETSCREEN" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_juniper_netscreen); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_NETSCREEN); + source (s_JUNIPER_NETSCREEN); {{- end}} rewrite { - r_set_splunk_dest_default(sourcetype("netscreen:firewall"), - index("netfw"), - template("t_standard")) + set("juniper_netscreen", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("netscreen:firewall"), index("netfw")) }; parser { @@ -25,7 +24,22 @@ log { parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NETSCREEN") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl index 3a84d12..b21861d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -1,28 +1,46 @@ # Juniper NSM -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_NSM" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_juniper_nsm); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_NSM); + source (s_JUNIPER_NSM); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"), template("t_standard"))}; + rewrite { + set("juniper_nsm", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"))}; parser { - p_add_context_splunk(key("juniper_nsm")); + p_add_context_splunk(key("juniper_nsm")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NSM") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl index e590fd4..9d511c7 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -1,27 +1,43 @@ # Juniper NSM IDP -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_NSM_IDP_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_NSM_IDP" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} + +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_juniper_nsm_idp); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_NSM_IDP); + source (s_JUNIPER_NSM_IDP); {{- end}} - rewrite {r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"), template("t_standard"))}; + rewrite { + set("juniper_nsm_idp", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"))}; parser { - p_add_context_splunk(key("juniper_idp")); + p_add_context_splunk(key("juniper_nsm_idp")); }; - parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_standard))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_JUNIPER_NSM_IDP") }} + destination(d_archive); +{{- end}} + flags(flow-control); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl index 4c89731..ba00cc2 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl @@ -1,3 +1,7 @@ +# Microfocus ArcSight +{{ $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} + parser p_microfocus_arcsight_header { csv-parser( columns("fields.sc4s_cef_version", "fields.cef_device_vendor", "fields.cef_device_product", "fields.cef_device_version", "fields.cef_device_event_class", "fields.cef_name", "fields.cef_severity", MESSAGE) @@ -28,30 +32,25 @@ parser p_microfocus_arcsight_source { default-selector("unknown") ); }; - -# Microfocus ArcSight -{{- if (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} -{{ $context := dict "port_id" "MICROFOCUS_ARCSIGHT" "parser" "common"}} -{{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_microfocus_arcsight); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_MICROFOCUS_ARCSIGHT); + source (s_MICROFOCUS_ARCSIGHT); {{- end}} rewrite { - r_set_splunk_dest_default(sourcetype("cef"), index("main"), template("t_msg_trim")) + set("microfocus_arcsight", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cef"), index("main")) }; parser { p_add_context_splunk(key("cef_{fields.cef_device_vendor}_${fields.cef_device_product}")); - }; parser (p_microfocus_arcsight_header); @@ -70,7 +69,22 @@ log { parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".cef.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_MICROFOCUS_ARCSIGHT") }} + destination(d_archive); +{{- end}} flags(flow-control); diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 8dd5e0f..60acc74 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -1,17 +1,16 @@ # PaloAlto PanOS -{{- if (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PALOALTO_PANOS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "PALOALTO_PANOS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc3164); filter(f_paloalto_panos); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_PALOALTO_PANOS); + source (s_PALOALTO_PANOS); {{- end}} # The palo message does not include a program value in the header, unfortunately. @@ -20,10 +19,10 @@ log { # While we are at it we will save the mesage type into the program field so parser can find it. rewrite { + set("paloalto_panos", value("fields.sc4s_vendor_product")); set("${LEGACY_MSGHDR}${MESSAGE}" value("MESSAGE")); unset(value("LEGACY_MSGHDR")); unset(value("PROGRAM")); - }; parser { #basic parsing @@ -46,34 +45,49 @@ log { #set the source type based on program field and lookup index from the splunk_context csv if (message(',\d+,THREAT')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"))}; parser {p_add_context_splunk(key("pan_threat")); }; } elif (message(',\d+,TRAFFIC')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"))}; parser {p_add_context_splunk(key("pan_traffic")); }; } elif (message(',\d+,SYSTEM')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"))}; parser {p_add_context_splunk(key("pan_system")); }; } elif (message(',\d+,CONFIG')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"))}; parser {p_add_context_splunk(key("pan_config")); }; } elif (message(',\d+,HIPWATCH')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"))}; parser {p_add_context_splunk(key("pan_hipwatch")); }; } elif (message(',\d+,CORRELATION')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"))}; parser {p_add_context_splunk(key("pan_correlation")); }; } elif (message(',\d+,USERID')) { - rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"))}; parser {p_add_context_splunk(key("pan_userid")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"))}; parser {p_add_context_splunk(key("pan_log")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".pan.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PALOALTO_PANOS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl index e7d7087..60b6736 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl @@ -1,26 +1,43 @@ # Proofpoint -{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "PROOFPOINT_PPS_FILTER" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_proofpoint_pps_filter); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_PROOFPOINT_PPS_FILTER); + source (s_PROOFPOINT_PPS_FILTER); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"), template("t_msg_only"))}; + rewrite { + set("proofpoint_pps_filter", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"))}; parser { p_add_context_splunk(key("proofpoint_pps_filter")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl index 0866ef0..11a8057 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl @@ -1,26 +1,42 @@ # Proofpoint -{{- if (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_SENDMAIL_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "PROOFPOINT_PPS_SENDMAIL" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_proofpoint_pps_sendmail); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_PROOFPOINT_PPS_SENDMAIL); + source (s_PROOFPOINT_PPS_SENDMAIL); {{- end }} - rewrite { r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"), template("t_msg_only"))}; + rewrite { + set("proofpoint_pps_sendmail", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("pps_mail_log"), index("email"))}; parser { p_add_context_splunk(key("proofpoint_pps_sendmail")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS_SENDMAIL") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl new file mode 100644 index 0000000..0962cc6 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl @@ -0,0 +1,137 @@ +#Ubiquiti unifi +{{ $context := dict "port_id" "UBIQUITI_UNIFI" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} + +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_is_rfc3164); + filter(f_ubiquiti_unifi); +{{- end}} +{{- if eq (.) "no"}} + source (s_UBIQUITI_UNIFI); +{{- end}} + + + parser {p_add_context_splunk(key("ubiquiti_unifi")); }; + + #Fiewall + if { + filter { + match("ubiquiti_unifi_fw", value("fields.sc4s_vendor_product") type(glob) ); + }; + + if (match("[^)]\s\S+\skernel:\s[^ll\sheader][^\[\d+.\d+\]]\S+\s\w+:" value("RAWMSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:threat"), index("netids")) + set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + parser {p_add_context_splunk(key("ubiquiti_unifi_threat")); }; + } elif (match("\S+\slinkcheck:" value("RAWMSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:link"), index("netops")) + set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + parser {p_add_context_splunk(key("ubiquiti_unifi_link")); }; + } elif (match("\d+:\d+:\d+\s\S+\ssudo:" value("RAWMSG"))) { + rewrite { r_set_splunk_dest_default(sourcetype("ubnt:sudo"), index("netops")) + set("${LEGACY_MSGHDR}${MSG}" value("MSG"));}; + parser {p_add_context_splunk(key("ubiquiti_unifi_sudo")); }; + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:fw"), index("netfw")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_fw")); }; + }; + #Switch + } elif { + filter { + host('^(?US[^,]{1,10}),(?[a-z0-9]{9,16}),(?v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6})', flags("store-matches")); + }; + if (match("hostapd:\s+ath" value("RAWMSG"))) { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:hostapd"), index("netops")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + set("${HOST_FROM}", value("HOST")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; + } elif (match("\d+:\d+:\d+\s\S+\smcad:" value("RAWMSG"))) { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:mcad"), index("netops")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + set("${HOST_FROM}", value("HOST")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:switch"), index("netops")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + set("${FROM_HOST}",value("HOST")); + set("${model}", value("fields.model")); + set("${serial}", value("fields.serial")); + set("${firmware}", value("fields.firmware")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_switch")); }; + + }; + + } elif { + filter { + program('^(?U\d[^,]{1,10}),(?[a-z0-9]{9,16}),(?v\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,6})', flags("store-matches")); + }; + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:wireless"), index("netops")); + set("${FROM_HOST}",value("HOST")); + set("${model}", value("fields.model")); + set("${serial}", value("fields.serial")); + set("${firmware}", value("fields.firmware")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_wireless")); }; + + } elif (match("traputil.c\(696\) " value("RAWMSG"))) { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt:edgeswitch"), index("netops")); + set("${HOST_FROM}", value("HOST")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi_edge_switch")); }; + + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("ubnt"), index("netops")); + set("${HOST_FROM}", value("HOST")); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + }; + parser {p_add_context_splunk(key("ubiquiti_unifi")); }; + }; + + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_UBIQUITI_UNIFI") }} + destination(d_archive); +{{- end}} + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_UBIQUITI_UNIFI_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for UBIQUITI_UNIFI traffic + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for UBIQUITI_UNIFI traffic +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl index 7bd86fc..983e13f 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -1,19 +1,19 @@ # Proofpoint -{{- if (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_ZSCALER_NSS_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "ZSCALER_NSS" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes" }} - source(s_default-ports); + source(s_DEFAULT); filter(f_zscaler_nss); {{- end }} {{- if eq (.) "no" }} - source (s_dedicated_port_ZSCALER_NSS); + source (s_ZSCALER_NSS); {{- end }} rewrite { + set("zscaler_nss", value("fields.sc4s_vendor_product")); subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); }; parser { @@ -22,37 +22,37 @@ log { }; if (match("alerts" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-alerts"), index("main"))}; parser { p_add_context_splunk(key("zscaler_alerts")); }; } elif (match("dns" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-dns"), index("netdns"))}; parser { p_add_context_splunk(key("zscaler_dns")); }; } elif (match("fw" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-fw"), index("netfw"))}; parser { p_add_context_splunk(key("zscaler_fw")); }; } elif (match("NSS" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-web"), index("netproxy"))}; parser { p_add_context_splunk(key("zscaler_web")); }; } elif (match("audit" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-audit"), index("netops"))}; parser { p_add_context_splunk(key("zscaler_zia_audit")); }; } elif (match("sandbox" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zia-sandbox"), index("main"))}; parser { p_add_context_splunk(key("zscaler_zia_sandbox")); }; } elif (match("zpa" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-app"), index("netids"))}; parser { p_add_context_splunk(key("zscaler_zpa")); }; } elif (match("zpa_auth" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpaauth"), index("netauth"))}; parser { p_add_context_splunk(key("zscaler_zpa_auth")); }; } elif (match("zpa_auth_connector" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-zpa-connector"), index("netops"))}; parser { p_add_context_splunk(key("zscaler_zpa_connector")); }; } elif (match("zpa_bba" value(".kv.product"))) { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-bba"), index("main"))}; parser { p_add_context_splunk(key("zscaler_zpa_bba")); }; } else { - rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("zscalernss-unknown"), index("main"))}; parser { p_add_context_splunk(key("zscaler_nss")); }; @@ -61,7 +61,23 @@ log { parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_ZSCALER_NSS") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl index b041abb..69f4e28 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl @@ -1,25 +1,41 @@ # Cisco ASA RFC5424 -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "CISCO_ASA" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_noversion); filter(f_cisco_asa); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_CISCO_ASA); + source (s_CISCO_ASA); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))}; + rewrite { + set("cisco_asa", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))}; parser {p_add_context_splunk(key("cisco_asa")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ASA") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl index e04290b..1ff0958 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl @@ -1,26 +1,44 @@ # Symantec Proxy (Bluecoat) -{{- if (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_SYMANTEC_PROXY_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "SYMANTEC_PROXY" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_noversion); filter(f_symantec_bluecoat_proxy); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_SYMANTEC_PROXY); + source (s_SYMANTEC_PROXY); {{- end}} - rewrite { r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy"), template("t_msg_only")) }; + rewrite { + set("bluecoat_proxy", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy")) + }; parser {p_add_context_splunk(key("bluecoat_proxy")); }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_PROXY") }} + destination(d_archive); +{{- end}} flags(flow-control); }; diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl index bd473a5..19a26c3 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl @@ -1,29 +1,32 @@ # Juniper JunOS (Structured, RFC5424-compliant) -{{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT") "no") "no") }} {{ $context := dict "port_id" "JUNIPER_JUNOS_STRUCTURED" "parser" "common" }} {{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} +# The following is an inline template; we will use this to generate the actual log path {{ define "log_path" }} log { {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); filter(f_is_rfc5424_strict); filter(f_juniper_junos_structured); {{- end}} {{- if eq (.) "no"}} - source (s_dedicated_port_JUNIPER_JUNOS_STRUCTURED); + source (s_JUNIPER_JUNOS_STRUCTURED); {{- end}} + + rewrite { + set("juniper_junos", value("fields.sc4s_vendor_product")); + }; if (program('RT_IDP')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids")) }; parser {p_add_context_splunk(key("juniper_idp_structured")); }; } elif (program('RT_FLOW')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw")) }; parser {p_add_context_splunk(key("juniper_junos_fw_structured")); }; } elif (program('RT_IDS')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids")) }; parser {p_add_context_splunk(key("juniper_junos_ids_structured")); }; } elif (program('RT_UTM')) { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw")) }; parser {p_add_context_splunk(key("juniper_junos_utm_structured")); }; } # Legacy Netscreen IDP is handled in the "p_rfc3164-juniper-idp.conf" log path @@ -33,13 +36,28 @@ log { # parser {p_add_context_splunk(key("juniper_junos_idp")); }; # } else { - rewrite { r_set_splunk_dest_default(sourcetype("juniper:structured"), index("netops"), template("t_JSON_5424")) }; + rewrite { r_set_splunk_dest_default(sourcetype("juniper:structured"), index("netops")) }; parser {p_add_context_splunk(key("juniper_structured")); }; }; parser (compliance_meta_by_source); - destination(d_hec); #--HEC-- + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + unset(value("RAWMSG")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CHECKPOINT") }} + destination(d_archive); +{{- end}} + }; {{- end}} {{- if (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT") "no") "no") }} diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl new file mode 100644 index 0000000..3ff2c86 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl @@ -0,0 +1,49 @@ +# Checkpoint Splunk format +{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_cisco_meraki); +{{- end}} +{{- if eq (.) "no"}} + source (s_CISCO_MERAKI); +{{- end}} + + rewrite { + set("cisco_meraki", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("meraki"), index("netfw")) + }; + parser {p_add_context_splunk(key("cisco_meraki")); }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + destination(d_hec); + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_MERAKI") }} + destination(d_archive); +{{- end}} + + flags(flow-control); +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} +# Listen on the specified dedicated port(s) for CISCO_MERAKI traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for CISCO_MERAKI traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl deleted file mode 100644 index dbbb1f6..0000000 --- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_merkai.conf.tmpl +++ /dev/null @@ -1,42 +0,0 @@ -# Checkpoint Splunk format -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} -{{ $context := dict "port_id" "CISCO_MERAKI" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} -{{ define "log_path" }} -log { -{{- if eq (.) "yes"}} - source(s_default-ports); - filter(f_cisco_meraki); -{{- end}} -{{- if eq (.) "no"}} - source (s_dedicated_port_CISCO_MERAKI); -{{- end}} - - #parser { - # kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); - # date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}})); - # - # }; - - #rewrite { set("${.kv.hostname}", value("HOST")); }; - - rewrite { r_set_splunk_dest_default(sourcetype("meraki"), index("netfw"), template("t_hdr_msg"))}; - parser {p_add_context_splunk(key("cisco_meraki")); }; - - - - parser (compliance_meta_by_source); - - destination(d_hec); #--HEC-- - - flags(flow-control); -}; -{{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_CISCO_MERAKI_TLS_PORT") "no") "no") }} -# Listen on the specified dedicated port(s) for CISCO_MERAKI traffic - {{ tmpl.Exec "log_path" "no" }} -{{- end}} - -# Listen on the default port (typically 514) for CISCO_MERAKI traffic -{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/zfallback.conf b/package/etc/conf.d/log_paths/zfallback.conf index 37d7a86..478ff31 100644 --- a/package/etc/conf.d/log_paths/zfallback.conf +++ b/package/etc/conf.d/log_paths/zfallback.conf @@ -1,7 +1,10 @@ log { - source(s_default-ports); + source(s_DEFAULT); - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")) }; + rewrite { + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); + set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG")); + }; parser { p_add_context_splunk(key("sc4s_fallback")); }; diff --git a/package/etc/conf.d/sources/network.conf.tmpl b/package/etc/conf.d/sources/network.conf.tmpl index 234e87c..be569ff 100644 --- a/package/etc/conf.d/sources/network.conf.tmpl +++ b/package/etc/conf.d/sources/network.conf.tmpl @@ -1,101 +1,2 @@ -# =============================================================================================== -# source definition for remote devices -# =============================================================================================== - -# =============================================================================================== -# Defaults for the default-network-drivers() source: -# 514, both TCP and UDP, for RFC3164 (BSD-syslog) formatted traffic -# 601 TCP, for RFC5424 (IETF-syslog) formatted traffic -# 6514 TCP, for TLS-encrypted traffic -# =============================================================================================== -source s_default-ports { - - channel { - source { - syslog ( - transport("udp") - port(514) - ip-protocol(4) - so-rcvbuf({{- getenv "SC4S_SOURCE_UDP_SO_RCVBUFF" "425984"}}) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - - network ( - transport("tcp") - port(514) - ip-protocol(4) - max-connections({{- getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) - log-iw-size({{- getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) - log-fetch-limit({{- getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}}) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - ); - - {{- if eq (getenv "SC4S_SOURCE_TLS_ENABLE") "yes"}} - network( - port(6514) - transport("tls") - ip-protocol(4) - max-connections({{getenv "SC4S_SOURCE_TCP_MAX_CONNECTIONS" "2000"}}) - log-iw-size({{getenv "SC4S_SOURCE_TCP_IW_SIZE" "20000000"}}) - log-fetch-limit({{getenv "SC4S_SOURCE_TCP_FETCH_LIMIT" "2000"}}) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - flags(no-parse) - tls(allow-compress(yes) - key-file("/opt/syslog-ng/tls/server.key") - cert-file("/opt/syslog-ng/tls/server.pem") - ssl-options({{- getenv "SC4S_SOURCE_TLS_OPTIONS" "no-sslv2, no-sslv3, no-tlsv1" }}) - cipher-suite("{{- getenv "SC4S_SOURCE_TLS_CIPHER_SUITE" "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH" }}") - ) - ); - - {{- end }} - }; - #TODO: #60 Remove this function with enhancement - rewrite(set_rfcnonconformant); - - if { - filter(f_rfc5424_strict); - parser { - syslog-parser(flags(syslog-protocol store-raw-message)); - }; - rewrite(set_rfc5424_strict); - } elif { - filter(f_rfc5424_noversion); - parser { - syslog-parser(flags(syslog-protocol store-raw-message)); - }; - rewrite(set_rfc5424_noversion); - } elif { - parser {cisco-parser()}; - rewrite(set_cisco_ios); - } elif { - parser (p_cisco_meraki); - rewrite(set_rfc5424_epochtime); - } else { - parser { - syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message)); - }; - rewrite(set_rfc3164); - }; - - rewrite(r_set_splunk_default); - - parser { - vendor_product_by_source(); - }; - }; -}; \ No newline at end of file +{{ $context := dict "port_id" "DEFAULT" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index 7a418e3..d4c05b9 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -13,6 +13,7 @@ #checkpoint_splunk,index,netops #cisco_asa,index,netfw #cisco_ios,index,netops +#cisco_ise,index,netauth #cisco_nx_os,index,netops #local_example,index,main #forcepoint_webprotect,index,netproxy diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 57c73ac..ec20e82 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -36,3 +36,7 @@ filter f_proofpoint_pps_sendmail { host("pps-*" type(glob)) or netmask(192.168.6.0/24) }; +filter f_ubiquiti_unifi_fw { + host("usg-*" type(glob)) or + netmask(192.168.6.0/24) +}; \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 2f85bc4..698f672 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -6,4 +6,5 @@ f_juniper_idp,sc4s_vendor_product,"juniper_idp" f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" -f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file +f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" +f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" \ No newline at end of file diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index 09ea120..badcbbb 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -1,10 +1,10 @@ -# The following is the dedicated source port declaration for {{ (print .port_id) }} +# The following is the source port declaration for {{ (print .port_id) }} # Two log paths will be created -- one for the dedicated port(s) and one for the default (typically port 514) - -source s_dedicated_port_{{ .port_id}} { +{{- define "T1" }} +source s_{{ .port_id}} { channel { source { -{{- if ne (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT" ) "no") "no" }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT" )) }} syslog ( transport("udp") port({{getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT") }}) @@ -18,7 +18,7 @@ source s_dedicated_port_{{ .port_id}} { flags(no-parse) ); {{- end}} -{{- if ne (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") "no") "no" }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT" )) }} network ( transport("tcp") port({{getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT") }}) @@ -34,7 +34,7 @@ source s_dedicated_port_{{ .port_id}} { flags(no-parse) ); {{- end}} -{{- if ne (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") "no") "no" }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT" )) }} network( port({{getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT") }}) transport("tls") @@ -111,6 +111,12 @@ source s_dedicated_port_{{ .port_id}} { {{- end }} rewrite(r_set_splunk_default); + parser { + vendor_product_by_source(); + }; }; - -}; \ No newline at end of file +}; +{{- end }} +{{- if (getenv (print "SC4S_LISTEN_" .port_id "_TCP_PORT")) or (getenv (print "SC4S_LISTEN_" .port_id "_UDP_PORT")) or (getenv (print "SC4S_LISTEN_" .port_id "_TLS_PORT")) }} +{{ template "T1" (.) }} +{{- end }} diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl index a8ac264..6a68b45 100644 --- a/package/etc/local_config/log_paths/example.conf.tmpl +++ b/package/etc/local_config/log_paths/example.conf.tmpl @@ -27,7 +27,7 @@ log { # The first time this template is used the log_path will be linked to the default port {{- if eq (.) "yes"}} - source(s_default-ports); + source(s_DEFAULT); # Filters should be updated to use the simplest and most effecient logic possible to discard # the message from this path @@ -40,12 +40,12 @@ log { # In the second pass through the template a link to the dedicated port is used. This # normally does not require additional filters -source (s_dedicated_port_LOCAL_EXAMPLE); +source (s_LOCAL_EXAMPLE); {{- end}} #Set a default sourcetype and index - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"))}; #using the key "local_example" find any cutomized index,source or sourcetype meta values diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index e3e33e9..88897f1 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -21,7 +21,7 @@ options { chain_hostnames (off); use_dns (no); use_fqdn (no); - dns-cache(no); + dns-cache(yes); create_dirs (no); keep-hostname (yes); create_dirs(yes); @@ -39,12 +39,6 @@ options { # =============================================================================================== @define splunk-template "t_standard" -# =============================================================================================== -# Data collection parameters, buffers, and Timezone -# =============================================================================================== -#TODO: Remove once release with this PR is produced by upstream -#https://github.com/balabit/syslog-ng/pull/2932 -@define syslog-ng-sysconfdir "/opt/syslog-ng/etc" # =============================================================================================== # Global modules and includes. All device-specific filters and destinations exist in conf.d diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index ff9a112..a93736d 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -1,17 +1,11 @@ #!/usr/bin/env bash source scl_source enable rh-python36 +export SC4S_LISTEN_DEFAULT_TCP_PORT=514 +export SC4S_LISTEN_DEFAULT_UDP_PORT=514 + cd /opt/syslog-ng -#The following is no longer needed but retained as a comment just in case we run into command line length issues -#for d in $(find /opt/syslog-ng/etc -type d) -#do -# echo Templating conf for $d -# gomplate \ -# --input-dir=$d \ -# --template t=etc/go_templates/ \ -# --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ -# --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" -#done + gomplate $(find . -name *.tmpl | sed -E 's/^(\/.*\/)*(.*)\..*$/--file=\2.tmpl --out=\2/') --template t=etc/go_templates/ diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf index 612fe0b..04c2df6 100644 --- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -4,6 +4,12 @@ homePath = $SPLUNK_DB/syslogng_metrics/db coldPath = $SPLUNK_DB/syslogng_metrics/colddb thawedPath = $SPLUNK_DB/syslogng_metrics/thaweddb +[em_metrics] +datatype=metric +homePath = $SPLUNK_DB/em_metrics/db +coldPath = $SPLUNK_DB/em_metrics/colddb +thawedPath = $SPLUNK_DB/em_metrics/thaweddb + [syslogng_fallback] homePath = $SPLUNK_DB/syslogng_fallback/db coldPath = $SPLUNK_DB/syslogng_fallback/colddb @@ -30,6 +36,11 @@ homePath = $SPLUNK_DB/oswinsec/db coldPath = $SPLUNK_DB/oswinsec/colddb thawedPath = $SPLUNK_DB/oswinsec/thaweddb +[netauth] +homePath = $SPLUNK_DB/netauth/db +coldPath = $SPLUNK_DB/netauth/colddb +thawedPath = $SPLUNK_DB/netauth/thaweddb + [netdlp] homePath = $SPLUNK_DB/netdlp/db coldPath = $SPLUNK_DB/netdlp/colddb diff --git a/tests/requirements.txt b/tests/requirements.txt index cc706aa..758c531 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,5 +9,5 @@ pytest jinja2 jinja2-time -http://dev.splunk.com/goto/sdk-python +splunk-sdk flake8 diff --git a/tests/test_cisco_ise.py b/tests/test_cisco_ise.py new file mode 100644 index 0000000..db1e493 --- /dev/null +++ b/tests/test_cisco_ise.py @@ -0,0 +1,52 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<165>Apr 24 15:00:48 ICDC-ISE03 CISE_Passed_Authentications 0001939187 4 0 2019-04-24 15:00:48.610 +00:00 0042009748 5200 NOTICE Passed-Authentication: Authentication succeeded, ConfigVersionId=128, Device IP Address=10.6.64.15, DestinationIPAddress=10.16.20.23, DestinationPort=1812, UserName=90-1B-0E-34-EA-92, Protocol=Radius, RequestLatency=8, NetworkDeviceName=ICPAV2-SW15, User-Name=901b0e34ea92, NAS-IP-Address=10.6.64.15, NAS-Port=50104, Service-Type=Call Check, Framed-IP-Address=10.6.226.138, Framed-MTU=1500, Called-Station-ID=B0-FA-EB-11-70-04, Calling-Station-ID=90-1B-0E-34-EA-92, NAS-Port-Type=Ethernet, NAS-Port-Id=GigabitEthernet0/4, EAP-Key-Name=, cisco-av-pair=service-type=Call Check, cisco-av-pair=audit-session-id=0A06400F000006AA6F83C371, cisco-av-pair=method=mab, OriginalUserName=901b0e34ea92, NetworkDeviceProfileName=Cisco, NetworkDeviceProfileId=b2652f13-5b3f-41ba-ada2-8385c8870809, IsThirdPartyDeviceFlow=false, RadiusFlowType=WiredMAB, SSID=B0-FA-EB-11-70-04, +#<165>Apr 24 15:00:48 ICDC-ISE03 CISE_Passed_Authentications 0001939187 4 1 AcsSessionID=ICDC-ISE03/341048949/1407358, AuthenticationIdentityStore=Internal Endpoints, AuthenticationMethod=Lookup, SelectedAccessService=Default Network Access, SelectedAuthorizationProfiles=WIRED_GUEST_REDIRECT, UseCase=Host Lookup, IdentityGroup=Endpoint Identity Groups:Unknown, Step=11001, Step=11017, Step=11027, Step=15049, Step=15008, Step=15048, Step=15048, Step=15041, Step=15013, Step=24209, Step=24211, Step=22037, Step=24715, Step=15036, Step=15048, Step=15048, Step=15016, Step=11002, SelectedAuthenticationIdentityStores=Internal Endpoints, AuthenticationStatus=AuthenticationPassed, NetworkDeviceGroups=Location#All Locations#Provo, NetworkDeviceGroups=Device Type#All Device Types#Switches, NetworkDeviceGroups=Migrated NDGs#All Migrated NDGs, IdentityPolicyMatchedRule=Default, AuthorizationPolicyMatchedRule=Guest Web Auth, UserType=Host, CPMSessionID=0A06400F000006AA6F83C371, EndPointMACAddress=90-1B-0E-34-EA-92, +#<165>Apr 24 15:00:48 ICDC-ISE03 CISE_Passed_Authentications 0001939187 4 2 PostureAssessmentStatus=NotApplicable, EndPointMatchedProfile=Unknown, DeviceRegistrationStatus=notRegistered, ISEPolicySetName=Wired MAB, IdentitySelectionMatchedRule=Default, StepData=5= Aruba.Aruba-Essid-Name, StepData=6= DEVICE.Device Type, StepData=8=Internal Endpoints, StepData=14= EndPoints.LogicalProfile, StepData=15= Network Access.Protocol, allowEasyWiredSession=false, DTLSSupport=Unknown, HostIdentityGroup=Endpoint Identity Groups:Unknown, Network Device Profile=Cisco, Location=Location#All Locations#Provo, Device Type=Device Type#All Device Types#Switches, Migrated NDGs=Migrated NDGs#All Migrated NDGs, Name=Endpoint Identity Groups:Unknown, +#<165>Apr 24 15:00:48 ICDC-ISE03 CISE_Passed_Authentications 0001939187 4 3 Response={UserName=90:1B:0E:34:EA:92; User-Name=90-1B-0E-34-EA-92; State=ReauthSession:0A06400F000006AA6F83C371; Class=CACS:0A06400F000006AA6F83C371:ICDC-ISE03/341048949/1407358; Session-Timeout=300; Idle-Timeout=240; Termination-Action=RADIUS-Request; cisco-av-pair=url-redirect-acl=ACL-WEBAUTH-REDIRECT; cisco-av-pair=url-redirect=https://ICDC-ISE03.example.com:8443/portal/gateway?sessionId=0A06400F000006AA6F83C371&portal=c5a76cb0-6150-11e5-b062-0050568d954e&action=cwa&type=drw&token=6ded1943789b345f7afdd09e91549047; cisco-av-pair=profile-name=Unknown; LicenseTypes=1; }, + +def test_cisco_ise(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 0 {% now 'utc', '%Y-%m-%d %H:%M:%S' %}.610 +00:00 0042009748 5200 NOTICE Passed-Authentication: Authentication succeeded, ConfigVersionId=128, Device IP Address=10.6.64.15, DestinationIPAddress=10.16.20.23, DestinationPort=1812, UserName=90-1B-0E-34-EA-92, Protocol=Radius, RequestLatency=8, NetworkDeviceName=ICPAV2-SW15, User-Name=901b0e34ea92, NAS-IP-Address=10.6.64.15, NAS-Port=50104, Service-Type=Call Check, Framed-IP-Address=10.6.226.138, Framed-MTU=1500, Called-Station-ID=B0-FA-EB-11-70-04, Calling-Station-ID=90-1B-0E-34-EA-92, NAS-Port-Type=Ethernet, NAS-Port-Id=GigabitEthernet0/4, EAP-Key-Name=, cisco-av-pair=service-type=Call Check, cisco-av-pair=audit-session-id=0A06400F000006AA6F83C371, cisco-av-pair=method=mab, OriginalUserName=901b0e34ea92, NetworkDeviceProfileName=Cisco, NetworkDeviceProfileId=b2652f13-5b3f-41ba-ada2-8385c8870809, IsThirdPartyDeviceFlow=false, RadiusFlowType=WiredMAB, SSID=B0-FA-EB-11-70-04,\n") + message = mt.render(mark="<165>", host=host) + sendsingle(message) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 1 AcsSessionID=ICDC-ISE03/341048949/1407358, AuthenticationIdentityStore=Internal Endpoints, AuthenticationMethod=Lookup, SelectedAccessService=Default Network Access, SelectedAuthorizationProfiles=WIRED_GUEST_REDIRECT, UseCase=Host Lookup, IdentityGroup=Endpoint Identity Groups:Unknown, Step=11001, Step=11017, Step=11027, Step=15049, Step=15008, Step=15048, Step=15048, Step=15041, Step=15013, Step=24209, Step=24211, Step=22037, Step=24715, Step=15036, Step=15048, Step=15048, Step=15016, Step=11002, SelectedAuthenticationIdentityStores=Internal Endpoints, AuthenticationStatus=AuthenticationPassed, NetworkDeviceGroups=Location#All Locations#Provo, NetworkDeviceGroups=Device Type#All Device Types#Switches, NetworkDeviceGroups=Migrated NDGs#All Migrated NDGs, IdentityPolicyMatchedRule=Default, AuthorizationPolicyMatchedRule=Guest Web Auth, UserType=Host, CPMSessionID=0A06400F000006AA6F83C371, EndPointMACAddress=90-1B-0E-34-EA-92,\n") + message = mt.render(mark="<111>", host=host) + sendsingle(message) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 2 PostureAssessmentStatus=NotApplicable, EndPointMatchedProfile=Unknown, DeviceRegistrationStatus=notRegistered, ISEPolicySetName=Wired MAB, IdentitySelectionMatchedRule=Default, StepData=5= Aruba.Aruba-Essid-Name, StepData=6= DEVICE.Device Type, StepData=8=Internal Endpoints, StepData=14= EndPoints.LogicalProfile, StepData=15= Network Access.Protocol, allowEasyWiredSession=false, DTLSSupport=Unknown, HostIdentityGroup=Endpoint Identity Groups:Unknown, Network Device Profile=Cisco, Location=Location#All Locations#Provo, Device Type=Device Type#All Device Types#Switches, Migrated NDGs=Migrated NDGs#All Migrated NDGs, Name=Endpoint Identity Groups:Unknown,\n") + message = mt.render(mark="<111>", host=host) + sendsingle(message) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_Passed_Authentications 0001939187 4 3 Response={UserName=90:1B:0E:34:EA:92; User-Name=90-1B-0E-34-EA-92; State=ReauthSession:0A06400F000006AA6F83C371; Class=CACS:0A06400F000006AA6F83C371:ICDC-ISE03/341048949/1407358; Session-Timeout=300; Idle-Timeout=240; Termination-Action=RADIUS-Request; cisco-av-pair=url-redirect-acl=ACL-WEBAUTH-REDIRECT; cisco-av-pair=url-redirect=https://ICDC-ISE03.example.com:8443/portal/gateway?sessionId=0A06400F000006AA6F83C371&portal=c5a76cb0-6150-11e5-b062-0050568d954e&action=cwa&type=drw&token=6ded1943789b345f7afdd09e91549047; cisco-av-pair=profile-name=Unknown; LicenseTypes=1; },\n") + message = mt.render(mark="<111>", host=host) + sendsingle(message) + + st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:ise:syslog\" | head 11") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_common.py b/tests/test_common.py index e026893..8325b52 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -67,4 +67,16 @@ def test_tag(record_property, setup_wordlist, setup_splunk): record_property("resultCount", resultCount) record_property("message", message) + assert resultCount == 1 + +# +def test_metrics(record_property, setup_wordlist, setup_splunk): + + st = env.from_string('mcatalog values(metric_name) WHERE metric_name="syslogng.d_*#0" AND ("index"="*" OR "index"="_*") BY index | fields index') + search = st.render() + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("resultCount", resultCount) + assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_ubiquiti_unifi.py b/tests/test_ubiquiti_unifi.py new file mode 100644 index 0000000..4eed2b2 --- /dev/null +++ b/tests/test_ubiquiti_unifi.py @@ -0,0 +1,93 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +#<27>Nov 8 17:28:43 US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type + +def test_ubiquiti_unifi_us8p60(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netops sourcetype=ubnt:switch earliest=-2m | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<29>Nov 10 20:46:02 US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: 0 +def test_ubiquiti_unifi_switch_us24p250(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: 0") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netops sourcetype=ubnt:switch earliest=-2m | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<30>Nov 10 11:49:46 U7PG2,788a2056b181,v4.0.66.10832: logread[5495]: Logread connected to 10.2.0.9:514 +def test_ubiquiti_unifi_ap_u7pg2(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} U7PG2,788a2056b181,v4.0.66.10832: logread[5495]: Logread connected to 10.2.0.9:514") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netops sourcetype=ubnt:wireless earliest=-2m | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<4>Nov 10 23:04:06 USG kernel: [LAN_LOCAL-default-A]IN=eth0.2004 OUT= MAC= SRC=10.254.3.1 DST=224.0.0.251 LEN=348 TOS=0x00 PREC=0x00 TTL=255 ID=32463 DF PROTO=UDP SPT=5353 DPT=5353 LEN=328 +def test_ubiquiti_unifi_usg(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} usg-{{host}} kernel: [LAN_LOCAL-default-A]IN=eth0.2004 OUT= MAC= SRC=10.254.3.1 DST=224.0.0.251 LEN=348 TOS=0x00 PREC=0x00 TTL=255 ID=32463 DF PROTO=UDP SPT=5353 DPT=5353 LEN=328") + message = mt.render(mark="<27>", host=host) + sendsingle(message) + + st = env.from_string("search index=netfw sourcetype=ubnt:fw host=usg-{{host}} | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1