diff --git a/.circleci/config.yml b/.circleci/config.yml index 8840fcf..a455417 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,6 @@ jobs: image: $CI_IMAGE registry: $REGISTRY path: package - extra_build_args: --build-arg RH_ORG=$RH_ORG --build-arg RH_ACTIVATION=$RH_ACTIVATION - docker/install-goss: version: v0.3.7 diff --git a/.env.template b/.env.template index f360cd6..b795fde 100644 --- a/.env.template +++ b/.env.template @@ -6,8 +6,6 @@ # #You should have received a copy of the CC0 legalcode along with this #work. If not, see . -RH_ORG=xxxx -RH_ACTIVATION=xxxxx SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 SPLUNK_PASSWORD=Changed@11 SPLUNK_START_ARGS=--accept-license diff --git a/.gitignore b/.gitignore index 4d40b52..921f415 100644 --- a/.gitignore +++ b/.gitignore @@ -384,3 +384,4 @@ fabric.properties tests/test_plugin_*.py # package/etc/conf.d/local/ !package/etc/conf.d/local +replay \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index b460d99..fd4c8a0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "package/syslog-ng"] path = package/syslog-ng url = https://github.com/balabit/syslog-ng.git - branch = syslog-ng-3.24.1 + branch = syslog-ng-3.25.1 # diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml index bb22763..ee8d0e0 100644 --- a/docker-compose-debug.yml +++ b/docker-compose-debug.yml @@ -13,9 +13,6 @@ services: image: splunk/scs:latest build: context: ./package - args: - RH_ORG: ${RH_ORG} - RH_ACTIVATION: ${RH_ACTIVATION} entrypoint: - "tail" - "-f" diff --git a/docker-compose-perf.yml b/docker-compose-perf.yml index e1bd795..b552c66 100644 --- a/docker-compose-perf.yml +++ b/docker-compose-perf.yml @@ -12,9 +12,6 @@ services: image: rfaircloth/scs:edge build: context: ./package - args: - RH_ORG: ${RH_ORG} - RH_ACTIVATION: ${RH_ACTIVATION} hostname: sc4s ports: - "514" diff --git a/docker-compose.yml b/docker-compose.yml index d1e1558..dff7a5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,9 +25,6 @@ services: image: splunk/scs:latest build: context: ./package - args: - RH_ORG: ${RH_ORG} - RH_ACTIVATION: ${RH_ACTIVATION} hostname: sc4s #When this is enabled test_common will fail # command: -det @@ -72,6 +69,16 @@ services: - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD} volumes: - splunk-etc:/opt/splunk/etc + pcapreplay: + build: + context: ./utility/pcapreplay + entrypoint: tail -f /dev/null + links: + - splunk + - sc4s + volumes: + - ./replay:/work + volumes: sc4s-results: external: true diff --git a/docs/gettingstarted/index.md b/docs/gettingstarted/index.md index c3822dc..f686dc0 100644 --- a/docs/gettingstarted/index.md +++ b/docs/gettingstarted/index.md @@ -42,6 +42,8 @@ using the SC4S defaults. SC4S can be easily customized to use different indexes * netops * netproxy * netipam +* oswinsec +* osnix * em_metrics (ensure this is created as a metrics index) #### Install Related Splunk Apps diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md index 9d6cad4..b0234dd 100644 --- a/docs/sources/Cisco/index.md +++ b/docs/sources/Cisco/index.md @@ -1,5 +1,54 @@ # Vendor - Cisco +## Product - ACS + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/1811/ | +| Product Manual | https://community.cisco.com/t5/security-documents/acs-5-x-configuring-the-external-syslog-server/ta-p/3143143 | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| cisco:acs | Aggregation used | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| cisco_acs | cisco:acs | netauth | None | + + +### Filter type + +PATTERN MATCH + +### Setup and Configuration + +* No special steps required + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_CISCO_ACS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CISCO_ACS_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_CISCO_ACS | no | Enable archive to disk for this specific source | +| SC4S_DEST_CISCO_ACS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +Use the following search to validate events are present + +``` +index= sourcetype=cisco:acs +``` + +Verify timestamp, and host values match as expected + + ## Product - ASA (Pre Firepower) | Ref | Link | @@ -42,8 +91,8 @@ MSG Parse: This filter parses message content | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC5424 format | -| SC4S_LISTEN_CISCO_ASA_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined expecting RFC5424 format | +| SC4S_LISTEN_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_CISCO_ASA_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | | SC4S_ARCHIVE_CISCO_ASA | no | Enable archive to disk for this specific source | | SC4S_DEST_CISCO_ASA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | | SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined expecting RFC3164 format | diff --git a/docs/sources/InfoBlox/index.md b/docs/sources/InfoBlox/index.md new file mode 100644 index 0000000..627d8fc --- /dev/null +++ b/docs/sources/InfoBlox/index.md @@ -0,0 +1,56 @@ +# Vendor - Infoblox + +Warning: Despite the TA indication this data source is CIM compliant the all versions of NIOS including the most recent available as of 2019-12-17 do not support the DNS data model correctly. For DNS security use cases use Splunk Stream instead. + +## Product - NIOS + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/2934/ | +| Product Manual | https://docs.infoblox.com/display/ILP/NIOS?preview=/8945695/43728387/NIOS_8.4_Admin_Guide.pdf | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| infoblox:dns | None | +| infoblox:dhcp | None | +| infoblox:threat | None | +| nix:syslog | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| infoblox_dns | infoblox:dns | netdns | none | +| infoblox_dhcp | infoblox:dhcp | netipam | none | +| infoblox_threat | infoblox:threat | netids | none | +| nix_syslog | nix:syslog | osnix | none | + +### Filter type + +Must be identified by host or ip assignment. Update the filter `f_infoblox` or configure a dedicated port as required + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the admin manual for specific details of configuration + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_INFOBLOX_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_INFOBLOX_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_INFOBLOX | no | Enable archive to disk for this specific source | +| SC4S_DEST_INFOBLOX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active device will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=infoblox:*| stats count by host +``` diff --git a/docs/sources/Juniper/index.md b/docs/sources/Juniper/index.md index 98cd701..e497d0c 100644 --- a/docs/sources/Juniper/index.md +++ b/docs/sources/Juniper/index.md @@ -37,9 +37,11 @@ | Variable | default | description | |----------------|----------------|----------------| -| SC4S_LISTEN_JUNIPER_JUNOS_LEGACY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined using legacy 3164 format| -| SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined using 5424 format | -| SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source | +| SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined using legacy 3164 format| +| SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined using legacy 3164 format| +| SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined using legacy 3164 format| +| SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined using 5424 format | +| SC4S_LISTEN_JUNIPER_JUNOS_STRUCTURED_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined using 5424 format || SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source | | SC4S_DEST_JUNIPER_JUNOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -90,6 +92,7 @@ Verify timestamp, and host values match as expected | Variable | default | description | |----------------|----------------|----------------| | SC4S_LISTEN_JUNIPER_NSM_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_JUNIPER_NSM_TLS_PORT | empty string | Enable at TLS port for this specific vendor product using the number defined | | SC4S_LISTEN_JUNIPER_NSM_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | | SC4S_ARCHIVE_JUNIPER_NSM | no | Enable archive to disk for this specific source | | SC4S_DEST_JUNIPER_NSM_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | @@ -142,6 +145,7 @@ Verify timestamp, and host values match as expected | Variable | default | description | |----------------|----------------|----------------| | SC4S_LISTEN_JUNIPER_NETSCREEN_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_JUNIPER_NETSCREEN_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | | SC4S_LISTEN_JUNIPER_NETSCREEN_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | | SC4S_ARCHIVE_JUNIPER_NETSCREEN | no | Enable archive to disk for this specific source | | SC4S_DEST_JUNIPER_NETSCREEN_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | @@ -192,6 +196,7 @@ Verify timestamp, and host values match as expected | Variable | default | description | |----------------|----------------|----------------| | SC4S_LISTEN_JUNIPER_JUNOS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_JUNIPER_JUNOS_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | | SC4S_LISTEN_JUNIPER_JUNOS_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | | SC4S_ARCHIVE_JUNIPER_JUNOS | no | Enable archive to disk for this specific source | | SC4S_DEST_JUNIPER_JUNOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | @@ -204,4 +209,4 @@ Use the following search to validate events are present; for Juniper SSL VPN ens index= sourcetype=juniper:sslvpn | stats count by host ``` -Verify timestamp, and host values match as expected \ No newline at end of file +Verify timestamp, and host values match as expected diff --git a/docs/sources/PaloaltoNetworks/index.md b/docs/sources/PaloaltoNetworks/index.md index bc27602..814344b 100644 --- a/docs/sources/PaloaltoNetworks/index.md +++ b/docs/sources/PaloaltoNetworks/index.md @@ -53,6 +53,7 @@ MSG Parse: This filter parses message content | SC4S_LISTEN_PALOALTO_PANOS_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | | SC4S_ARCHIVE_PALOALTO_PANOS | no | Enable archive to disk for this specific source | | SC4S_DEST_PALOALTO_PANOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS | no | Use custom time stamp parsing with ms added | ### Verification diff --git a/docs/sources/Symantec/index.md b/docs/sources/Symantec/index.md index e169c78..58e79a5 100644 --- a/docs/sources/Symantec/index.md +++ b/docs/sources/Symantec/index.md @@ -49,3 +49,53 @@ An active proxy will generate frequent events. Use the following search to valid ``` index= sourcetype=bluecoat:proxysg:access:kv | stats count by host ``` + +## Product - Mail Gateway (Brightmail) + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | TBD | +| Product Manual | https://support.symantec.com/us/en/article.howto38250.html | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| symantec:smg | Requires version TA 3.6 | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| symantec_brightmail | symantec:smg | email | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* No TA available +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL | no | Enable archive to disk for this specific source | +| SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG | yes | Email processing events generated by the bmserver process will be grouped by host+program+pid+msg ID into a single event | +### Verification + +An active mail server will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=symantec:smg | stats count by host +``` diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md new file mode 100644 index 0000000..7a9e54f --- /dev/null +++ b/docs/sources/VMWare/index.md @@ -0,0 +1,53 @@ +# Vendor - Dell - VMWare + +## Product - vSphwere - ESX NSX (Controller, Manager, Edge) + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | None | +| Manual | https://docs.vmware.com/en/VMware-NSX-Data-Center-for-vSphere/6.4/com.vmware.nsx.logging.doc/GUID-0674A29A-9D61-4E36-A302-E4192A3DA1A5.html | + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| vmware:vsphere:nsx | None | +| vmware:vsphere:esx | None | +| nix:syslog | When used with a default port this will follow the generic NIX configuration when using a dedicated port, IP or host rules events will follow the index configuration for vmware nsx | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| vmware_nsx | vmware:vsphere:nsx | main | none | +| vmware_esx | vmware:vsphere:esx | main | none | + +### Filter type + +MSG Parse: This filter parses message content when using the default configuration + +### Setup and Configuration + +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. +* Refer to the Splunk TA documentation for the specific customer format required for proxy configuration + * Select TCP or SSL transport option + * Ensure the format of the event is customized per Splunk documentation + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_VMWARE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using the number defined | +| SC4S_LISTEN_VMWARE_TLS_PORT | empty string | Enable a TLS port for this specific vendor product using the number defined | +| SC4S_ARCHIVE_VMWARE | no | Enable archive to disk for this specific source | +| SC4S_DEST_VMWARE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype="vmware:*:vsphere:*" | stats count by host +``` diff --git a/docs/sources/nix/index.md b/docs/sources/nix/index.md new file mode 100644 index 0000000..7be21b0 --- /dev/null +++ b/docs/sources/nix/index.md @@ -0,0 +1,55 @@ +# Vendor - Nix Generic + +## Product - All Products + +Many appliance vendor utilize Linux and BSD distributions as the foundation of the solution. When configured to log via +syslog, these devices' OS logs (from a security perspective) can be monitored using the common Splunk Nix TA. + +Note: This is NOT a replacement for or alternative to the Splunk Universal forwarder on Linux and Unix. For general-purpose +server applications, the Universal Forwarder offers more comprehensive collection of events and metrics appropriate for both +security and operations use cases. + + + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/833/ | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| nix:syslog | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| nix_syslog | nix:syslog | osnix | none | + + +### Filter type + +MSG Parse: This filter parses message content + +### Setup and Configuration + +* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. +* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source. + + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_ARCHIVE_NIX_SYSLOG | no | Enable archive to disk for this specific source | +| SC4S_DEST_NIX_SYSLOG_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +An active proxy will generate frequent events. Use the following search to validate events are present per source device + +``` +index=osnix sourcetype=nix:syslog | stats count by host +``` diff --git a/mkdocs.yml b/mkdocs.yml index 6b82a92..3848849 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -17,6 +17,7 @@ nav: - Fortinet: sources/Fortinet/index.md - Imperva: sources/Imperva/index.md - Juniper: sources/Juniper/index.md + - Nix: sources/nix/index.md - Microfocus: sources/Microfocus/index.md - 'Paloalto Networks': sources/PaloaltoNetworks/index.md - Proofpoint: sources/Proofpoint/index.md diff --git a/package/Dockerfile b/package/Dockerfile index a5cbb4b..7ea7def 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -6,35 +6,27 @@ # #You should have received a copy of the CC0 legalcode along with this #work. If not, see . -FROM registry.access.redhat.com/rhel7/rhel +FROM centos:centos8 -ARG RH_ORG -ARG RH_ACTIVATION -ENV CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --enable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --enable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking" +ENV CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --disable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --disable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking" -ENV DISTCHECK_CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --enable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --enable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking" -COPY confluent.repo /etc/yum.repos.d/confluent.repo -RUN subscription-manager register --org=$RH_ORG --activationkey=$RH_ACTIVATION --force -RUN subscription-manager repos --enable=rhel-7-server-extras-rpms --enable=rhel-7-server-optional-rpms --enable=rhel-server-rhscl-7-rpms -RUN yum update -y; yum install wget -y -RUN rpm --import https://packages.confluent.io/rpm/5.2/archive.key -RUN cd /tmp ;\ - wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm ;\ - yum install epel-release-latest-7.noarch.rpm -y ;\ - rm epel-release-latest-7.noarch.rpm -RUN yum install findutils autoconf \ - autoconf-archive automake ca-certificates git libtool pkgconfig bison byacc file \ - flex pcre-devel glib2-devel openssl-devel librdkafka-devel libcurl-devel \ - rh-python36 rh-python36-python-tools rh-python36-scldevel\ - net-snmp-devel \ - libuuid-devel cmake make libxslt docbook-style-xsl gcc-c++ tzdata libxml2 sqlite \ - json-c-devel gnupg wget curl which bzip2 doxygen libsecret ivykis-devel -y +ENV DISTCHECK_CONFIGURE_FLAGS="--prefix=/opt/syslog-ng --with-ivykis=system --with-jsonc=system --disable-env-wrapper --disable-memtrace --disable-tcp-wrapper --disable-linux-caps --disable-man-pages --enable-all-modules --enable-force-gnu99 --enable-json --enable-native --enable-python --enable-http --disable-kafka --disable-java --disable-java-modules --disable-spoof_source --disable-sun_streams --disable-sql --disable-pacct --disable-mongodb --disable-amqp --disable-stomp --disable-redis --disable-systemd --disable-geoip --disable-geoip2 --disable-riemann --disable-smtp --disable-snmp_dest --with-python=3 --enable-dynamic-linking" -RUN subscription-manager unregister +RUN dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y ;\ + dnf install 'dnf-command(config-manager)' -y ;\ + dnf config-manager --set-enabled PowerTools -y; \ + dnf update -y + +RUN dnf group install "Development Tools" ;\ + dnf install findutils autoconf \ + autoconf automake ca-certificates git libtool pkgconfig bison byacc file \ + flex pcre-devel glib2-devel openssl-devel libcurl-devel \ + python3 python3-devel \ + net-snmp-devel \ + libuuid-devel cmake make libxslt gcc-c++ tzdata libxml2 sqlite \ + gnupg wget curl which bzip2 libsecret ivykis-devel autoconf-archive json-c-devel -y -RUN echo source scl_source enable rh-python36 >>/etc/profile.d/enablepython36.sh ;\ - source scl_source enable rh-python36 RUN CRITERION_VERSION=2.3.3 ;\ cd /tmp/;\ @@ -48,32 +40,24 @@ RUN CRITERION_VERSION=2.3.3 ;\ COPY syslog-ng /work -RUN source scl_source enable rh-python36 ;\ - cd /work;\ - pip install -r requirements.txt ;\ +RUN cd /work;\ + pip3 install -r requirements.txt ;\ ./autogen.sh ;\ ./configure $CONFIGURE_FLAGS ;\ make -j -l 2.5 install -FROM registry.access.redhat.com/ubi7/ubi - -COPY confluent.repo /etc/yum.repos.d/confluent.repo +FROM registry.access.redhat.com/ubi8/ubi RUN cd /tmp ;\ - yum install -y wget ;\ - wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm ;\ - yum install epel-release-latest-7.noarch.rpm -y ;\ - rm epel-release-latest-7.noarch.rpm ;\ - rpm --import https://packages.confluent.io/rpm/5.2/archive.key ;\ - yum install gcc tzdata libdbi libsecret libxml2 sqlite tcp_wrappers librdkafka \ - rh-python36 rh-python36-python-tools libcurl ivykis scl-utils tcp_wrappers-libs curl wget openssl -y;\ - echo source scl_source enable rh-python36 >>/etc/profile.d/enablepython36.sh ;\ - source scl_source enable rh-python36 + dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y; \ + dnf update -y ;\ + dnf install wget gcc tzdata libdbi libsecret libxml2 sqlite \ + python3 libcurl ivykis scl-utils curl wget openssl -y ENV DEBCONF_NONINTERACTIVE_SEEN=true -RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh +RUN curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh COPY goss.yaml /etc/goss.yaml COPY --from=0 /opt/syslog-ng /opt/syslog-ng diff --git a/package/confluent.repo b/package/confluent.repo index 26ecf54..ff4016a 100644 --- a/package/confluent.repo +++ b/package/confluent.repo @@ -8,14 +8,14 @@ #work. If not, see . [Confluent.dist] name=Confluent repository (dist) -baseurl=https://packages.confluent.io/rpm/5.2/7 +baseurl=https://packages.confluent.io/rpm/5.3/7 gpgcheck=1 -gpgkey=https://packages.confluent.io/rpm/5.2/archive.key +gpgkey=https://packages.confluent.io/rpm/5.3/archive.key enabled=1 [Confluent] name=Confluent repository -baseurl=https://packages.confluent.io/rpm/5.2 +baseurl=https://packages.confluent.io/rpm/5.3 gpgcheck=1 -gpgkey=https://packages.confluent.io/rpm/5.2/archive.key +gpgkey=https://packages.confluent.io/rpm/5.3/archive.key enabled=1 \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/syslog_format.conf b/package/etc/conf.d/conflib/_common/syslog_format.conf index 5b69e71..49e6529 100644 --- a/package/etc/conf.d/conflib/_common/syslog_format.conf +++ b/package/etc/conf.d/conflib/_common/syslog_format.conf @@ -7,6 +7,9 @@ filter f_rfc5424_noversion{ filter f_rfc5424_epochtime{ message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?(?\d{10})(?:.(?\d{1,9})?)) (?[^ ]+) ))'); }; +filter f_rfc5424_bsdtime{ + message('^(?(?
(?<\d{1,3}>)(?[1-9][0-9]?) (?[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?[^ ]+) ))'); +}; rewrite set_rfcnonconformant{ set("rfc5424_nonconform" value("fields.sc4s_syslog_format")); }; @@ -28,6 +31,12 @@ rewrite set_rfc5424_epochtime{ filter f_is_rfc5424_epochtime{ match("rfc5424_epochtime" value("fields.sc4s_syslog_format")) }; +rewrite set_rfc5424_bsdtime{ + set("rfc5424_bsdtime" value("fields.sc4s_syslog_format")); +}; +filter f_is_rfc5424_bsdtime{ + match("rfc5424_bsdtime" value("fields.sc4s_syslog_format")) +}; rewrite set_rfc3164{ set("rfc3164" value("fields.sc4s_syslog_format")); }; @@ -45,4 +54,8 @@ rewrite set_no_parse{ }; filter f_is_no_parse{ match("no_parse" value("fields.sc4s_syslog_format")) +}; + +rewrite set_rfc3894_bsdtime{ + subst('^<\d+>(\d)', "", value("MESSAGE")); }; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/templates.conf b/package/etc/conf.d/conflib/_common/templates.conf index a08f36d..97db7ef 100644 --- a/package/etc/conf.d/conflib/_common/templates.conf +++ b/package/etc/conf.d/conflib/_common/templates.conf @@ -42,6 +42,9 @@ template t_hdr_msg { template("${MSGHDR}${MESSAGE}"); }; +template t_legacy_hdr_msg { + template("${LEGACY_MSGHDR}${MESSAGE}"); + }; # =============================================================================================== # Message Header, Structured Data (from RFC5424 parse) and Message; for Juniper # =============================================================================================== @@ -60,6 +63,7 @@ template t_JSON { --exclude .splunk.* --exclude HOST --exclude HOST_FROM + --exclude SOURCE )"); }; @@ -74,5 +78,6 @@ template t_JSON_5424 { --exclude HOST --exclude HOST_FROM --exclude RAWMSG + --exclude SOURCE )"); }; diff --git a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl index bc92162..1b2d8fe 100644 --- a/package/etc/conf.d/destinations/splunk_hec.conf.tmpl +++ b/package/etc/conf.d/destinations/splunk_hec.conf.tmpl @@ -42,7 +42,7 @@ destination d_hec { source=${.splunk.source} sourcetype=${.splunk.sourcetype} index=${.splunk.index} - event=$MSG + event="$MSG" fields.*)') ); }; \ No newline at end of file diff --git a/package/etc/conf.d/filters/VMware/vsphere.conf b/package/etc/conf.d/filters/VMware/vsphere.conf new file mode 100644 index 0000000..624c6e0 --- /dev/null +++ b/package/etc/conf.d/filters/VMware/vsphere.conf @@ -0,0 +1,58 @@ +filter f_vmware_all { + #begin base vmware + program("cimslp", flags(ignore-case)) + or program("Fdm", flags(ignore-case)) + or program("Hostd", flags(ignore-case)) + or program("hostd-probe", flags(ignore-case)) + or program("indcfg", flags(ignore-case)) + or program("lwsmd", flags(ignore-case)) + or program("netcpa", flags(ignore-case)) + or program("pktcap-agent", flags(ignore-case)) + or program("Rhttpproxy", flags(ignore-case)) + or program("sdrsInjector", flags(ignore-case)) + or program("sfcb-.*", flags(ignore-case)) + or program("storageRM", flags(ignore-case)) + or program("vmkernel", flags(ignore-case)) + or program("vmkwarning", flags(ignore-case)) + or program("vobd", flags(ignore-case)) + or program("Vpxa", flags(ignore-case)) + or program("Vpxd", flags(ignore-case)) + or program("VSANMGMTSVC", flags(ignore-case)) + or program("vsfwd", flags(ignore-case)) + #begin nsx + or program("NSX", flags(ignore-case)) + or program("NSXV", flags(ignore-case)) + or program("dfwpktlogs", flags(ignore-case)) + or program("nsx-.*", flags(ignore-case))}; + +filter f_vmware_vsphere { + program("cimslp", flags(ignore-case)) + or program("Fdm", flags(ignore-case)) + or program("Hostd", flags(ignore-case)) + or program("hostd-probe", flags(ignore-case)) + or program("indcfg", flags(ignore-case)) + or program("lwsmd", flags(ignore-case)) + or program("netcpa", flags(ignore-case)) + or program("pktcap-agent", flags(ignore-case)) + or program("Rhttpproxy", flags(ignore-case)) + or program("sdrsInjector", flags(ignore-case)) + or program("sfcb-.*", flags(ignore-case)) + or program("storageRM", flags(ignore-case)) + or program("vmkernel", flags(ignore-case)) + or program("vmkwarning", flags(ignore-case)) + or program("vobd", flags(ignore-case)) + or program("Vpxa", flags(ignore-case)) + or program("Vpxd", flags(ignore-case)) + or program("VSANMGMTSVC", flags(ignore-case)) + or program("vsfwd", flags(ignore-case)) +}; + +filter f_vmware_nsx { + program("NSX", flags(ignore-case)) + or + program("NSXV", flags(ignore-case)) + or + program("dfwpktlogs", flags(ignore-case)) + or + program("nsx-.*", flags(ignore-case)) +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/cisco/acs.conf b/package/etc/conf.d/filters/cisco/acs.conf new file mode 100644 index 0000000..42e9938 --- /dev/null +++ b/package/etc/conf.d/filters/cisco/acs.conf @@ -0,0 +1,4 @@ + +filter f_cisco_acs { + program("CSCOacs.*"); +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/infoblox/syslog.conf b/package/etc/conf.d/filters/infoblox/syslog.conf new file mode 100644 index 0000000..f131781 --- /dev/null +++ b/package/etc/conf.d/filters/infoblox/syslog.conf @@ -0,0 +1,4 @@ +filter f_infoblox { + match("^infoblox", value("fields.sc4s_vendor_product")); + +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/juniper/legacy.conf b/package/etc/conf.d/filters/juniper/legacy.conf index 27b3f52..50383ab 100644 --- a/package/etc/conf.d/filters/juniper/legacy.conf +++ b/package/etc/conf.d/filters/juniper/legacy.conf @@ -14,7 +14,7 @@ filter f_juniper_netscreen { }; - filter f_juniper_idp { +filter f_juniper_idp { match("juniper_idp", value("fields.sc4s_vendor_product") type(glob)) or match('^\[syslog@juniper' value("SDATA")) }; diff --git a/package/etc/conf.d/filters/nix/syslog.conf b/package/etc/conf.d/filters/nix/syslog.conf new file mode 100644 index 0000000..bfeeb2a --- /dev/null +++ b/package/etc/conf.d/filters/nix/syslog.conf @@ -0,0 +1,5 @@ +filter f_nix_syslog { + program("[a-zA-Z0-9\/]+") + and + match('[a-zA-Z\]]: $' value("LEGACY_MSGHDR")) +}; \ No newline at end of file diff --git a/package/etc/conf.d/filters/paloalto/panos.conf b/package/etc/conf.d/filters/paloalto/panos.conf index da0cb04..6b645b1 100644 --- a/package/etc/conf.d/filters/paloalto/panos.conf +++ b/package/etc/conf.d/filters/paloalto/panos.conf @@ -1,3 +1,9 @@ +filter f_is_palalto_format{ + match("rfc5424_bsdtime" value("fields.sc4s_syslog_format")) + or + match("rfc3164" value("fields.sc4s_syslog_format")) +}; + filter f_paloalto_panos { message(',\d+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPWATCH|CORRELATION|USERID),'); }; \ No newline at end of file diff --git a/package/etc/conf.d/filters/symantec/brightmail.conf b/package/etc/conf.d/filters/symantec/brightmail.conf new file mode 100644 index 0000000..7d78535 --- /dev/null +++ b/package/etc/conf.d/filters/symantec/brightmail.conf @@ -0,0 +1,6 @@ +filter f_symantec_brightmail { + message('\[Brightmail\]') or program("bmserver") +}; +filter f_symantec_brightmail_details { + program("bmserver") and not message('\[Brightmail\]') +}; \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl index 1f5c620..da7ffeb 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-checkpoint_splunk.conf.tmpl @@ -14,47 +14,63 @@ log { source (s_CHECKPOINT_SPLUNK); {{- end}} - parser { - kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); - date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); - }; + if { + filter(f_nix_syslog); + rewrite { + set("checkpoint_splunk", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + rewrite { + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("netops"), source("program:${.PROGRAM}") ) + }; + parser { p_add_context_splunk(key("checkpoint_os")); }; - rewrite { - set("${.kv.hostname}", value("HOST")); - set("checkpoint_splunk", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("cp_log"), index("netops")) - }; - parser {p_add_context_splunk(key("checkpoint_splunk")); }; + } else { + parser { + kv-parser(prefix(".kv.") pair-separator("|") template("${MSGHDR} ${MSG}")); + date-parser(format("%s") template("${.kv.time}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone)); + }; + + rewrite { + set("${.kv.hostname}", value("HOST")); + set("checkpoint_splunk", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cp_log"), index("netops")) + }; + + parser {p_add_context_splunk(key("checkpoint_splunk")); }; - if { - filter(f_checkpoint_splunk_NetworkTraffic); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); }; - } elif { - filter(f_checkpoint_splunk_Web); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_web")); }; - } elif { - filter(f_checkpoint_splunk_NetworkSessions); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); }; - } elif { - filter(f_checkpoint_splunk_IDS_Malware); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; - } elif { - filter(f_checkpoint_splunk_IDS); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; - } elif { - filter(f_checkpoint_splunk_email); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_email")); }; - } elif { - filter(f_checkpoint_splunk_DLP); - rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; - parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; + if { + filter(f_checkpoint_splunk_NetworkTraffic); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netfw"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_firewall")); }; + } elif { + filter(f_checkpoint_splunk_Web); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("web"), index("netproxy"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_web")); }; + } elif { + filter(f_checkpoint_splunk_NetworkSessions); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("sessions"), index("netops"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_sessions")); }; + } elif { + filter(f_checkpoint_splunk_IDS_Malware); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids_malware"), index("netids"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; + } elif { + filter(f_checkpoint_splunk_IDS); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("ids"), index("netids"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_ids")); }; + } elif { + filter(f_checkpoint_splunk_email); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("email"), index("email"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_email")); }; + } elif { + filter(f_checkpoint_splunk_DLP); + rewrite { r_set_splunk_dest_default(sourcetype("cp_log"), source("firewall"), index("netdlp"))}; + parser {p_add_context_splunk(key("checkpoint_splunk_dlp")); }; + }; }; parser (compliance_meta_by_source); @@ -63,7 +79,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -78,7 +94,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} {{- if or (or (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl new file mode 100644 index 0000000..0b9ccfd --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_acs.conf.tmpl @@ -0,0 +1,109 @@ +# Cisco ACS +{{ $context := dict "port_id" "CISCO_ACS" "parser" "common"}} +{{ tmpl.Exec "t/source_network.t" $context }} + +#This filter uses a field we set to prevent the original messages before aggregation from being +#sent to Splunk +filter f_cisco_acs_complete{ + match("yes", value("ACS.COMPLETE") type(glob)); +}; + +#This parser adds messages from ACS to a context without sending them +#forward to Splunk +parser acs_grouping { + csv-parser( + columns(PID, ACS.num, ACS.seq, MESSAGE) + delimiters(chars(" ")) + flags(greedy) + ); + grouping-by( + scope(program) + key("$PID") + trigger("$(+ ${ACS.seq} 1)" == "${ACS.num}") + sort-key("${ACS.seq}") + aggregate( + value("MESSAGE" "$(implode '' $(context-values ${MESSAGE}))") + value("ACS.COMPLETE" "yes") + ) + timeout(10) + ); +}; + +#The syslog message includes a date with nano seconds and TZ which is not in the header +#So must reparse the date +parser acs_event_time { + csv-parser( + columns(ACS.DATE, ACS.TIME, ACS.TZ, MESSAGE) + delimiters(chars(" ")) + flags(greedy) + ); + + date-parser( + #YYYY- MM-DD hh:mm:ss:xxx +/-zh:zm + format("%Y-%m-%d %H:%M:%S.%f %z" ) + template("${ACS.DATE} ${ACS.TIME} ${ACS.TZ}") + ); +}; +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_is_rfc3164); + filter(f_cisco_acs); +{{- end}} +{{- if eq (.) "no"}} + source (s_CISCO_ACS); +{{- end}} + + parser(acs_grouping); + + if { + filter(f_cisco_acs_complete); + parser(acs_event_time); + rewrite { + set("cisco_acs", value("fields.sc4s_vendor_product")); + r_set_splunk_dest_default(sourcetype("cisco:acs"), index("netauth")) + }; + + parser {p_add_context_splunk(key("cisco_acs")); }; + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("PID")); + unset(value("LEGACY_MSGHDR")); + unset(value("EPOCH")); + unset(value("VERSION")); + unset(value("TIMESECFRAC")); + groupunset(values("ACS.*")); + }; + +{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_CISCO_ACS_HEC" "no") | conv.ToBool) }} + destination(d_hec); +{{- end}} + + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_CISCO_ACS") }} + destination(d_archive); + {{- end}} + + flags(flow-control,final); + }; + + +}; +{{- end}} + +{{- if or (or (getenv (print "SC4S_LISTEN_CISCO_ACS_TCP_PORT")) (getenv (print "SC4S_LISTEN_CISCO_ACS_UDP_PORT"))) (getenv (print "SC4S_LISTEN_CISCO_ACS_TLS_PORT")) }} + +# Listen on the specified dedicated port(s) for CISCO_ACS traffic + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for CISCO_ACS traffic +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl index a07d6a1..172445d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl @@ -26,7 +26,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -41,7 +41,7 @@ log { {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl index 554277b..cb90f70 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl @@ -28,7 +28,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -43,7 +43,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl index cca9afd..544c0a3 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_ise.conf.tmpl @@ -73,7 +73,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("PID")); @@ -92,7 +92,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; diff --git a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl index fd7a8b4..59db061 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-cisco_nxos.conf.tmpl @@ -28,7 +28,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -44,7 +44,7 @@ log { {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl index b35e857..72d4de7 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-forcepoint_webprotect.conf.tmpl @@ -27,7 +27,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -41,7 +41,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl index 1be63f1..29f17fc 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl @@ -44,7 +44,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -60,7 +60,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl new file mode 100644 index 0000000..ecfad6e --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3164-infoblox.conf.tmpl @@ -0,0 +1,104 @@ +# Juniper IDP +{{ $context := dict "port_id" "INFOBLOX" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} + +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + filter(f_is_rfc3164); + filter(f_infoblox); +{{- end}} +{{- if eq (.) "no"}} + source (s_INFOBLOX); +{{- end}} + + + if { + filter{program("named")}; + rewrite { + set("infoblox_dns", value("fields.sc4s_vendor_product")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("infoblox:dns"), index("netdns"), source("program:${.PROGRAM}")) + set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); + }; + parser { + p_add_context_splunk(key("infoblox_dns")); + }; + } elif { + filter{program("dhcpd")}; + rewrite { + set("infoblox_dhcp", value("fields.sc4s_vendor_product")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("infoblox:dhcp"), index("netipam"), source("program:${.PROGRAM}")) + set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); + }; + parser { + p_add_context_splunk(key("infoblox_dhcp")); + }; + } elif { + filter{program("threat-protect-log")}; + rewrite { + set("infoblox_dns", value("fields.sc4s_vendor_product")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("infoblox:threat"), index("netids"), source("program:${.PROGRAM}")) + set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); + }; + parser { + p_add_context_splunk(key("infoblox_threat")); + }; + } else { + rewrite { + set("nix_syslog", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + + rewrite { + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") ) + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + + }; + + parser { p_add_context_splunk(key("nix_syslog")); }; + }; + + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + unset(value("PID")); + groupunset(values(".kv.*")); + }; + + +{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_INFOBLOX_HEC" "no") | conv.ToBool) }} + destination(d_hec); +{{- end}} + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_INFOBLOX") }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; +{{- end}} + +{{- if or (or (getenv (print "SC4S_LISTEN_INFOBLOX_TCP_PORT")) (getenv (print "SC4S_LISTEN_INFOBLOX_UDP_PORT"))) (getenv (print "SC4S_LISTEN_INFOBLOX_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for INFOBLOX traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for INFOBLOX traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl index 761a8c0..16f2c0c 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl @@ -28,7 +28,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_hdr_sdata_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -45,7 +45,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl index d461e5e..f0c8a1d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl @@ -44,7 +44,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -58,7 +58,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl index b735a73..dde7d54 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl @@ -28,7 +28,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -42,7 +42,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl index 5571ba0..8d8d999 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl @@ -29,7 +29,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -43,7 +43,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl index bb717e0..c1196c5 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl @@ -26,7 +26,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_standard))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_standard))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -40,7 +40,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl index db6cd6a..8d26d45 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-microfocus_arcsight.conf.tmpl @@ -73,7 +73,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -88,7 +88,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl index 05c45ad..ee4caaf 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl @@ -6,7 +6,7 @@ log { {{- if eq (.) "yes"}} source(s_DEFAULT); - filter(f_is_rfc3164); + filter(f_is_palalto_format); filter(f_paloalto_panos); {{- end}} {{- if eq (.) "no"}} @@ -36,7 +36,12 @@ log { #2012/04/10 04:39:55 #parse the date date-parser( + {{- if ((getenv "SC4S_SOURCE_FF_PALOALTO_PANOS_TIME_MS") | conv.ToBool) }} + format("%Y/%m/%d %H:%M:%S.%f") + {{- else}} format("%Y/%m/%d %H:%M:%S") + {{- end}} + template("${.pan.GeneratedTime}") time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone) @@ -77,7 +82,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -92,7 +97,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl index 78f234c..0c805e7 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps.conf.tmpl @@ -35,7 +35,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -50,7 +50,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl deleted file mode 100644 index 09dd410..0000000 --- a/package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl +++ /dev/null @@ -1,52 +0,0 @@ -# Proofpoint -{{ $context := dict "port_id" "PROOFPOINT_PPS_FILTER" "parser" "common" }} -{{ tmpl.Exec "t/source_network.t" $context }} -# The following is an inline template; we will use this to generate the actual log path -{{ define "log_path" }} -log { -{{- if eq (.) "yes" }} - source(s_DEFAULT); - filter(f_proofpoint_pps_filter); -{{- end }} -{{- if eq (.) "no" }} - source (s_PROOFPOINT_PPS_FILTER); -{{- end }} - - rewrite { - set("proofpoint_pps_filter", value("fields.sc4s_vendor_product")); - r_set_splunk_dest_default(sourcetype("pps_filter_log"), index("email"))}; - parser { - p_add_context_splunk(key("proofpoint_pps_filter")); - }; - - parser (compliance_meta_by_source); - - - #We want to unset the fields we won't need, as this is copied into the - #disk queue for network destinations. This can be very disk expensive - #if we don't - rewrite { - set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; - - destination(d_hec); - -{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_PROOFPOINT_PPS") }} - destination(d_archive); -{{- end}} - - flags(flow-control); -}; -{{- end}} - -{{- if or (or (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TCP_PORT")) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_UDP_PORT"))) (getenv (print "SC4S_LISTEN_PROOFPOINT_PPS_FILTER_TLS_PORT")) }} -# Listen on the specified dedicated port(s) for PROOFPOINT_PPS_FILTER traffic - {{ tmpl.Exec "log_path" "no" }} -{{- end}} - -# Listen on the default port (typically 514) for PROOFPOINT_PPS_FILTER traffic -{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl index 55bd6a7..7ca3176 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-ubiquiti_unifi.conf.tmpl @@ -126,7 +126,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl index 47a77c1..0bcdf3d 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164-zscaler_nss.conf.tmpl @@ -66,7 +66,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -81,7 +81,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl new file mode 100644 index 0000000..70c1298 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_rfc3165-symantec_brightmail.conf.tmpl @@ -0,0 +1,127 @@ +{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} +filter f_symantec_brightmail_complete{ + match("yes", value("SMG.COMPLETE") type(glob)); +}; + +parser symantec_brightmail_grouping { + csv-parser( + columns(PID, SMG.seq, MESSAGE) + delimiters(chars("|")) + flags(greedy) + ); + grouping-by( + scope(program) + key("${SMG.seq}") + timeout(2) + aggregate( + value("MESSAGE" "${PID}|${SMG.seq}|$(implode ';' $(context-values ${MESSAGE}))") + value("SMG.COMPLETE" "yes") + ) + ); +}; +{{- end }} + +{{ $context := dict "port_id" "SYMANTEC_BRIGHTMAIL" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_DEFAULT); + filter(f_is_rfc3164); + filter(f_symantec_brightmail); +{{- end }} +{{- if eq (.) "no" }} + source (s_SYMANTEC_BRIGHTMAIL); +{{- end }} + +{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} + if { + + filter(f_symantec_brightmail_details); + parser(symantec_brightmail_grouping); + + if { + filter(f_symantec_brightmail_complete); + + rewrite { + set("symantec_brightmail", value("fields.sc4s_vendor_product")); + }; + + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:smg:mail"), index("email"), source("program:${.PROGRAM}") ) + + }; + + parser { p_add_context_splunk(key("symantec_brightmail")); }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + {{- if ((getenv "SC4S_SYMANTEC_BRIGHTMAIL_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC" "no") | conv.ToBool) }} + destination(d_hec); + {{- end}} + + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL") }} + destination(d_archive); + {{- end}} + + }; + } else { +{{- end }} + + rewrite { + set("symantec_brightmail", value("fields.sc4s_vendor_product")); + }; + + rewrite { + r_set_splunk_dest_default(sourcetype("symantec:smg"), index("email"), source("program:${.PROGRAM}") ) + + }; + + parser { p_add_context_splunk(key("symantec_brightmail")); }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + + {{- if ((getenv "SC4S_SYMANTEC_BRIGHTMAIL_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_SYMANTEC_BRIGHTMAIL_HEC" "no") | conv.ToBool) }} + destination(d_hec); + {{- end}} + + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_SYMANTEC_BRIGHTMAIL") }} + destination(d_archive); + {{- end}} +{{- if ((getenv "SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG" "yes") | conv.ToBool) }} + }; +{{- end}} + + + flags(flow-control,final); +}; +{{- end}} + +{{- if or (or (getenv (print "SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_TCP_PORT")) (getenv (print "SC4S_LISTEN_SYMANTEC_BRIGHTMAIL_UDP_PORT"))) (getenv (print "SC4S_SYMANTEC_BRIGHTMAIL_NSS_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for SYMANTEC_BRIGHTMAIL traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for SYMANTEC_BRIGHTMAIL traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl index a7f069c..065c824 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_cisco_asa.conf.tmpl @@ -24,7 +24,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -39,7 +39,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl index 855d390..c455a9d 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-noversion_symantec_proxy.conf.tmpl @@ -27,7 +27,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_msg_only))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -42,7 +42,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl index 40bce61..0d103a8 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424-strict_juniper_junos.conf.tmpl @@ -47,7 +47,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_JSON_5424))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); unset(value("RAWMSG")); groupunset(values(".kv.*")); }; @@ -61,6 +61,8 @@ log { destination(d_archive); {{- end}} + + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl index 18a37b4..793b356 100644 --- a/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc5424_epoch-cisco_meraki.conf.tmpl @@ -24,7 +24,7 @@ log { #disk queue for network destinations. This can be very disk expensive #if we don't rewrite { - set("$(template ${fields.sc4s_template} $(template t_hdr_msg))" value("MSG")); + set("$(template ${.splunk.sc4s_template} $(template t_hdr_msg))" value("MSG")); unset(value("RAWMSG")); unset(value("PROGRAM")); unset(value("LEGACY_MSGHDR")); @@ -39,7 +39,7 @@ log { destination(d_archive); {{- end}} - flags(flow-control); + flags(flow-control,final); }; {{- end}} diff --git a/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl b/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl new file mode 100644 index 0000000..5396355 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_vmware_vsphere.conf.tmpl @@ -0,0 +1,117 @@ +# Generate the custom port if defined +{{ $context := dict "port_id" "VMWARE" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} + +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes"}} + source(s_DEFAULT); + + filter(f_vmware_all); +{{- end}} +{{- if eq (.) "no"}} + source (s_VMWARE); +{{- end}} + + + #NSX first because its the cheapest check + if { + filter(f_is_rfc5424_strict); + filter(f_vmware_nsx); + + rewrite { + r_set_splunk_dest_default(sourcetype("vmware:vsphere:nsx"), index("main"), source("program:${PROGRAM}")); + set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_nsx")); + }; + } elif { + + filter(f_vmware_nsx); + + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("vmware:vsphere:nsx"), index("main"), source("program:${.PROGRAM}")); + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_nsx")); + }; + #esx things + } elif { + filter(f_is_rfc5424_strict); + filter(f_vmware_vsphere); + + rewrite { + r_set_splunk_dest_default(sourcetype("vmware:vsphere:esx"), index("main"), source("program:${PROGRAM}")); + set("$(template ${.splunk.sc4s_template} $(template t_JSON_5424))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_esx")); + }; + } elif { + + filter(f_vmware_vsphere); + + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default(sourcetype("vmware:vsphere:esx"), index("main"), source("program:${.PROGRAM}")); + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + }; + parser { + p_add_context_splunk(key("vmware_esx")); + }; + } else { + + rewrite { + set("nix_syslog", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + + rewrite { + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") ) + + }; + + parser { p_add_context_splunk(key("nix_syslog")); }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + }; + + parser (compliance_meta_by_source); + +{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_VMWARE_HEC" "no") | conv.ToBool) }} + destination(d_hec); +{{- end}} + + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_VMWARE") }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; +{{- end}} +{{- if or (or (getenv (print "SC4S_LISTEN_VMWARE_TCP_PORT")) (getenv (print "SC4S_LISTEN_VMWARE_UDP_PORT"))) (getenv (print "SC4S_LISTEN_VMWARE_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for VMWARE traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for VMWARE traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl new file mode 100644 index 0000000..c8f4e2a --- /dev/null +++ b/package/etc/conf.d/log_paths/p_za_nix_syslog.conf.tmpl @@ -0,0 +1,59 @@ +# Proofpoint +{{ $context := dict "port_id" "NIX_SYSLOG" "parser" "common" }} +{{ tmpl.Exec "t/source_network.t" $context }} +# The following is an inline template; we will use this to generate the actual log path +{{ define "log_path" }} +log { +{{- if eq (.) "yes" }} + source(s_DEFAULT); + filter(f_nix_syslog); +{{- end }} +{{- if eq (.) "no" }} + source (s_NIX_SYSLOG); +{{- end }} + + rewrite { + set("nix_syslog", value("fields.sc4s_vendor_product")); + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + + rewrite { + r_set_splunk_dest_default(sourcetype("nix:syslog"), index("osnix"), source("program:${.PROGRAM}") ) + + }; + + parser { p_add_context_splunk(key("nix_syslog")); }; + + parser (compliance_meta_by_source); + + #We want to unset the fields we won't need, as this is copied into the + #disk queue for network destinations. This can be very disk expensive + #if we don't + rewrite { + set("$(template ${.splunk.sc4s_template} $(template t_legacy_hdr_msg))" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + }; + +{{- if ((getenv "SC4S_NIX_SYSLOG_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_NIX_SYSLOG_HEC" "no") | conv.ToBool) }} + destination(d_hec); +{{- end}} + +{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_NIX_SYSLOG") }} + destination(d_archive); +{{- end}} + + flags(flow-control,final); +}; +{{- end}} + +{{- if or (or (getenv (print "SC4S_LISTEN_NIX_SYSLOG_TCP_PORT")) (getenv (print "SC4S_LISTEN_NIX_SYSLOG_UDP_PORT"))) (getenv (print "SC4S_NIX_SYSLOG_NSS_TLS_PORT")) }} +# Listen on the specified dedicated port(s) for NIX_SYSLOG traffic + {{ tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for NIX_SYSLOG traffic +{{ tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl new file mode 100644 index 0000000..cb9ed87 --- /dev/null +++ b/package/etc/conf.d/log_paths/p_zz_fallback.conf.tmpl @@ -0,0 +1,59 @@ +log { + source(s_DEFAULT); + + if { + filter(f_is_rfc5424_strict); + rewrite { + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main")); + set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG")); + }; + parser { + p_add_context_splunk(key("sc4s_fallback")); + }; + {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} + destination(d_hec); + {{- end}} + + + #in fallback archive only write rawmsg as msg + rewrite { + unset(value("RAWMSG")); + groupunset(values(".kv.*")); + }; + + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} + destination(d_archive); + {{- end}} + + } else { + rewrite { + r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main") ); + set("$(template ${.splunk.sc4s_template} $(template t_JSON))" value("MSG")); + }; + parser { + p_add_context_splunk(key("sc4s_fallback")); + }; + + {{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} + destination(d_hec); + {{- end}} + + + #in fallback archive only write rawmsg as msg + rewrite { + set("$RAWMSG" value("MSG")); + unset(value("RAWMSG")); + unset(value("PROGRAM")); + unset(value("LEGACY_MSGHDR")); + groupunset(values(".kv.*")); + }; + + {{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} + destination(d_archive); + {{- end}} + }; + + + + flags(flow-control,fallback); +}; diff --git a/package/etc/conf.d/log_paths/zfallback.conf.tmpl b/package/etc/conf.d/log_paths/zfallback.conf.tmpl deleted file mode 100644 index 0f36b1f..0000000 --- a/package/etc/conf.d/log_paths/zfallback.conf.tmpl +++ /dev/null @@ -1,31 +0,0 @@ -log { - source(s_DEFAULT); - - rewrite { - r_set_splunk_dest_default(sourcetype("sc4s:fallback"), index("main"), template("t_JSON")); - set("$(template ${fields.sc4s_template} $(template t_JSON))" value("MSG")); - }; - parser { - p_add_context_splunk(key("sc4s_fallback")); - }; - -{{- if ((getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes") | conv.ToBool) or (conv.ToBool (getenv "SC4S_DEST_ARCHIVE_HEC" "no") | conv.ToBool) }} - destination(d_hec); -{{- end}} - - - #in fallback archive only write rawmsg as msg - rewrite { - set("value(RAWMSG)" value("MSG")); - unset(value("RAWMSG")); - unset(value("PROGRAM")); - unset(value("LEGACY_MSGHDR")); - groupunset(values(".kv.*")); - }; - -{{- if (getenv "SC4S_ARCHIVE_GLOBAL") or (getenv "SC4S_ARCHIVE_FALLBACK") }} - destination(d_archive); -{{- end}} - - flags(flow-control,fallback); -}; diff --git a/package/etc/context_templates/compliance_meta_by_source.conf b/package/etc/context_templates/compliance_meta_by_source.conf index 322b938..90b67f4 100644 --- a/package/etc/context_templates/compliance_meta_by_source.conf +++ b/package/etc/context_templates/compliance_meta_by_source.conf @@ -1,5 +1,6 @@ @version: 3.24 -#filter f_test_test { -# host("something-*" type(glob)) or -# netmask(192.168.100.1/24) -#}; +filter f_test_test { +# host("something-*" type(glob)) or +# netmask(169.254.100.0/24) + host("cannot_ever_happen") +}; diff --git a/package/etc/context_templates/compliance_meta_by_source.csv b/package/etc/context_templates/compliance_meta_by_source.csv index 6608db0..4ecf53a 100644 --- a/package/etc/context_templates/compliance_meta_by_source.csv +++ b/package/etc/context_templates/compliance_meta_by_source.csv @@ -1,2 +1,2 @@ -#f_test_test,.splunk.index,"badindex" -#f_test_test,fields.compliance,"pci" +f_test_test,.splunk.index,"will_never_happen_index" +f_test_test,fields.compliance,"pci" diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv index d4c05b9..51b71c0 100644 --- a/package/etc/context_templates/splunk_index.csv +++ b/package/etc/context_templates/splunk_index.csv @@ -11,6 +11,7 @@ #checkpoint_splunk_web,index,netproxy #checkpoint_splunk,index,netops #checkpoint_splunk,index,netops +#cisco_acs,index,netauth #cisco_asa,index,netfw #cisco_ios,index,netops #cisco_ise,index,netauth @@ -21,6 +22,9 @@ #fortinet_fortios_log,index,netops #fortinet_fortios_traffic,index,netfw #fortinet_fortios_utm,index,netids +#infoblox_dns,index,netdns +#infoblox_dhcp,index,netipam +#infoblox_threat,index,netids #juniper_idp,index,netids #juniper_structured,index,netops #juniper_idp_structured,index,netids @@ -35,6 +39,7 @@ #juniper_nsm,index,netfw #juniper_nsm_idp,index,netids #juniper_legacy,index,netops +#nix_syslog,index,osnix #pan_traffic,index,netfw #pan_threat,index,netproxy #pan_system,index,netops @@ -48,3 +53,4 @@ #sc4s_events,index,main #sc4s_fallback,index,main #sc4s_metrics,index,em_metrics +#vmware_nsx,index,main \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index b0609de..ac87c2e 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -1,51 +1,54 @@ @version: 3.24 filter f_test_test { - host("testvp-*" type(glob)) or - netmask(192.168.100.1/24) + host("testvp-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_juniper_idp { - host("jnpidp-*" type(glob)) or - netmask(192.168.3.0/24) + host("jnpidp-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_juniper_netscreen { - host("jnpns-*" type(glob)) or - netmask(192.168.4.0/24) + host("jnpns-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_juniper_nsm { - host("jnpnsm-*" type(glob)) or - netmask(192.168.1.0/24) + host("jnpnsm-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_juniper_nsm_idp { - host("jnpnsmidp-*" type(glob)) or - netmask(192.168.2.0/24) + host("jnpnsmidp-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_cisco_meraki { - host("testcm-*" type(glob)) or - netmask(192.168.4.0/24) + host("testcm-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_cisco_nx_os { - host("csconx-*" type(glob)) or - netmask(192.168.5.0/24) + host("csconx-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) +}; +filter f_infoblox { + host("vib-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_proofpoint_pps_filter { - host("pps-*" type(glob)) or - netmask(192.168.7.0/24) + host("pps-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_proofpoint_pps_sendmail { - host("pps-*" type(glob)) or - netmask(192.168.6.0/24) + host("pps-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_ubiquiti_unifi_fw { - host("usg-*" type(glob)) or - netmask(192.168.6.0/24) + host("usg-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; filter f_tzfixhst { - host("tzfhst-*" type(glob)) or - netmask(192.168.6.0/24) + host("tzfhst-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) }; - filter f_tzfixny { - host("tzfny-*" type(glob)) or - netmask(192.168.6.0/24) -}; \ No newline at end of file + host("tzfny-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) +}; diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv index 975d558..510cc19 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -1,5 +1,6 @@ f_test_test,sc4s_vendor_product,"test_test" f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" +f_infoblox,sc4s_vendor_product,"infoblox" f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" f_juniper_idp,sc4s_vendor_product,"juniper_idp" diff --git a/package/etc/go_templates/source_network.t b/package/etc/go_templates/source_network.t index ff5be59..8a71d29 100644 --- a/package/etc/go_templates/source_network.t +++ b/package/etc/go_templates/source_network.t @@ -99,6 +99,13 @@ source s_{{ .port_id}} { } elif { parser {cisco-parser()}; rewrite(set_cisco_ios); + } elif { + filter(f_rfc5424_bsdtime); + rewrite(set_rfc3894_bsdtime); + parser { + syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(store-raw-message, guess-timezone)); + }; + rewrite(set_rfc5424_bsdtime); } elif { parser (p_cisco_meraki); rewrite(set_rfc5424_epochtime); diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 4f19083..84c177b 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -52,11 +52,14 @@ options { @include "conf.d/conflib/blocks/*/*.conf" @include "conf.d/filters/*/*.conf" -@include "conf.d/sources/*.conf" -@include "conf.d/destinations/*.conf" -@include "conf.d/log_paths/*.conf" - @include "conf.d/local/config/filters/*.conf" + +@include "conf.d/sources/*.conf" @include "conf.d/local/config/sources/*.conf" +@include "conf.d/destinations/*.conf" @include "conf.d/local/config/destinations/*.conf" + @include "conf.d/local/config/log_paths/*.conf" +@include "conf.d/log_paths/*.conf" + + diff --git a/package/syslog-ng b/package/syslog-ng index f219fbb..b880087 160000 --- a/package/syslog-ng +++ b/package/syslog-ng @@ -1 +1 @@ -Subproject commit f219fbbb12dad0b0b6cb458a0bb4415b2cd94d8a +Subproject commit b880087c95ce44e3d33722a825e895b9351aca58 diff --git a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf index 04c2df6..1a37448 100644 --- a/splunk/etc/apps/SA-syslog-ng/default/indexes.conf +++ b/splunk/etc/apps/SA-syslog-ng/default/indexes.conf @@ -4,6 +4,11 @@ homePath = $SPLUNK_DB/syslogng_metrics/db coldPath = $SPLUNK_DB/syslogng_metrics/colddb thawedPath = $SPLUNK_DB/syslogng_metrics/thaweddb +[email] +homePath = $SPLUNK_DB/email/db +coldPath = $SPLUNK_DB/email/colddb +thawedPath = $SPLUNK_DB/email/thaweddb + [em_metrics] datatype=metric homePath = $SPLUNK_DB/em_metrics/db @@ -26,6 +31,11 @@ homePath = $SPLUNK_DB/test2/db coldPath = $SPLUNK_DB/test2/colddb thawedPath = $SPLUNK_DB/test2/thaweddb +[osnix] +homePath = $SPLUNK_DB/osnix/db +coldPath = $SPLUNK_DB/osnix/colddb +thawedPath = $SPLUNK_DB/osnix/thaweddb + [oswin] homePath = $SPLUNK_DB/oswin/db coldPath = $SPLUNK_DB/oswin/colddb @@ -46,6 +56,11 @@ homePath = $SPLUNK_DB/netdlp/db coldPath = $SPLUNK_DB/netdlp/colddb thawedPath = $SPLUNK_DB/netdlp/thaweddb +[netdns] +homePath = $SPLUNK_DB/netdns/db +coldPath = $SPLUNK_DB/netdns/colddb +thawedPath = $SPLUNK_DB/netdns/thaweddb + [netfw] homePath = $SPLUNK_DB/netfw/db coldPath = $SPLUNK_DB/netfw/colddb @@ -56,6 +71,11 @@ homePath = $SPLUNK_DB/netids/db coldPath = $SPLUNK_DB/netids/colddb thawedPath = $SPLUNK_DB/netids/thaweddb +[netipam] +homePath = $SPLUNK_DB/netipam/db +coldPath = $SPLUNK_DB/netipam/colddb +thawedPath = $SPLUNK_DB/netipam/thaweddb + [netops] homePath = $SPLUNK_DB/netops/db coldPath = $SPLUNK_DB/netops/colddb diff --git a/tests/pytest.ini b/tests/pytest.ini index 00b64d9..ce9083e 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,5 +1,5 @@ [pytest] addopts = - --force-flaky --max-runs=3 --min-passes=1 +# --force-flaky --max-runs=3 --min-passes=1 filterwarnings = ignore::DeprecationWarning diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 92d1717..c9adef0 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -138,3 +138,27 @@ def test_checkpoint_splunk_smartconsole(record_property, setup_wordlist, setup_s record_property("message", message) assert resultCount == 1 + + +#<6>kernel: sd 2:0:0:0: SCSI error: return code = 0x00040000 +def test_checkpoint_splunk_os(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + + mt = env.from_string( + "{{ mark }}kernel: sd 2:0:0:0: SCSI error: return code = 0x{{pid}}\n") + message = mt.render(mark="<6>", pid=pid) + + sendsingle(message) + + st = env.from_string("search index=osnix \"0x{{ pid }}\" sourcetype=\"nix:syslog\" | head 2") + search = st.render(pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_cisco_acs.py b/tests/test_cisco_acs.py new file mode 100644 index 0000000..29bf81f --- /dev/null +++ b/tests/test_cisco_acs.py @@ -0,0 +1,57 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + + +def test_cisco_acs_single(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0765855540 1 0 2019-10-24 21:01:05.028 +00:00 0178632943 5202 NOTICE Device-Administration: Command Authorization succeeded, ACSVersion=acs-5.8.1.4-B.462.x86_64, ConfigVersionId=16489, Device IP Address=10.0.0.93, DestinationIPAddress=10.0.0.10, DestinationPort=49, UserName=nsdevman, CmdSet=[ CmdAV=show CmdArgAV=vpn-sessiondb CmdArgAV=full CmdArgAV=ra-ikev2-ipsec ], Protocol=Tacacs, MatchedCommandSet=fw3, RequestLatency=11, Type=Authorization, Privilege-Level=15, Authen-Type=ASCII, Service=None, User=nsdevman, Port=443, Remote-Address=10.0.0.15, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=mnsvdcfpiuac03/359448835/9871764, AuthenticationIdentityStore=AD1, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedCommandSet=fw3, IdentityGroup=IdentityGroup:All Groups:SystemID, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15004 , Step=15013 , Step=24210 , Step=24212 , Step=24432 , Step=24325 , Step=24313 , Step=24319 , Step=24323 , Step=24420 , Step=24355 , Step=24416 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15018 , Step=13024 , Step=13034 , SelectedAuthenticationIdentityStores=Internal Users, NetworkDeviceName=devicenamehere, NetworkDeviceGroups=Device Type:All Device Types:Firewall:Cisco Systems:Firewall:ASA5545, NetworkDeviceGroups=Location:All Locations:MN, ServiceSelectionMatchedRule=TACACS, IdentityPolicyMatchedRule=Firewall, AuthorizationPolicyMatchedRule=nsdevman, AD-User-Candidate-Identities=nsdevman@ent.example.corp, AD-User-DNS-Domain=ent.example.corp, AD-User-NetBios-Name=AD-ENT, AD-User-Resolved-Identities=nsdevman@ent.example.corp, AD-User-Join-Point=ENT.example.CORP, AD-User-Resolved-DNs=CN=nsdevman\,OU=Service Accounts\,OU=CAO\,OU=ENT\,DC=ent\,DC=wfb\,DC=example\,DC=corp, StepData=10=nsdevman, StepData=11=ent.example.corp, StepData=12=example.corp, StepData=15=ent.example.corp, AD-Domain=ent.example.corp, IdentityAccessRestricted=false, UserIdentityGroup=IdentityGroup:All Groups:SystemID, Cisco-Firewall=Superuser, Firewall=Superuser, NetSec-CSM=User, NetSec-Logging=Engineer, Response={Type=Authorization; Author-Reply-Status=PassAdd; ExternalIdentityStoreName=AD1; }\n") + message = mt.render(mark="<165>", host=host) + sendsingle(message) + + st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\" | head 11") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_cisco_acs_multi(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0000000002 2 0 2011-08-01 22:32:53.032 +00:00 0000008450 5203 NOTICE Device-Administration: Session Authorization succeeded, ACSVersion=acs-5.2.0.26-B.3075, ConfigVersionId=117, Device IP Address=192.168.26.137, UserName=edward, CmdSet=[ CmdAV= ], Protocol=Tacacs, RequestLatency=10, NetworkDeviceName=switch, Type=Authorization, Privilege-Level=1, Authen-Type=ASCII, Service=Login, User=edward, Port=tty2, Remote-Address=10.78.167.190, Authen-Method=TacacsPlus, Service-Argument=shell, AcsSessionID=ACS41/101085887/112, AuthenticationIdentityStore=Internal Users, AuthenticationMethod=Lookup, SelectedAccessService=Default Device Admin, SelectedShellProfile=Permit Access, IdentityGroup=IdentityGroup:All Groups, Step=13005 , Step=15008 , Step=15004 , Step=15012 , Step=15041 , Step=15006 , Step=15013 , Step=24210 , Step=24212 , Step=22037 , Step=15044 , Step=15035 , Step=15042 , Step=15036 , Step=15004 , Step=15017 , Step=13034 ,\n") + message = mt.render(mark="<165>", host=host) + sendsingle(message) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CSCOacs_Passed_Authentications 0000000002 2 1 Step=13015 , SelectedAuthenticationIdentityStores=Internal Users, NetworkDeviceGroups=s1Migrated_NDGs:All s1Migrated_NDGs, NetworkDeviceGroups=Device Type:All Device Types, NetworkDeviceGroups=Location:All Locations, ServiceSelectionMatchedRule=Rule-2, IdentityPolicyMatchedRule=Default, AuthorizationPolicyMatchedRule=Rule-0, Action=Login, Privilege-Level=1, Authen-Type=ASCII, Service=Login, Remote-Address=10.78.167.190, UserIdentityGroup=IdentityGroup:All\n") + message = mt.render(mark="<165>", host=host) + sendsingle(message) + + st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:acs\" \"Step=13015\" | head 11") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_cisco_ise.py b/tests/test_cisco_ise.py index db1e493..0093686 100644 --- a/tests/test_cisco_ise.py +++ b/tests/test_cisco_ise.py @@ -50,3 +50,23 @@ def test_cisco_ise(record_property, setup_wordlist, setup_splunk): record_property("message", message) assert resultCount == 1 + +#<181>Oct 24 21:00:02 ciscohost CISE_RADIUS_Accounting 0006028545 1 0 2019-10-24 21:00:02.879 +00:00 0088472694 3002 NOTICE Radius-Accounting: RADIUS Accounting watchdog update, ConfigVersionId=336, Device IP Address=10.0.0.3, RequestLatency=3, NetworkDeviceName=nc-aaa-aaa1, User-Name=U100000.ent.corp, NAS-IP-Address=10.0.0.3, NAS-Port=50047, Service-Type=Framed, Framed-IP-Address=10.0.0.80, Class=CACS:0AEF12345677832097B3F362:ncsilsepsuie212/356139633/9969901, Called-Station-ID=00-08-00-00-1B-AF, Calling-Station-ID=00-00-00-00-A0-7E, Acct-Status-Type=Interim-Update, Acct-Delay-Time=0, Acct-Input-Octets=653293631, Acct-Output-Octets=1497972244, Acct-Session-Id=00000B68, Acct-Authentic=RADIUS, Acct-Session-Time=241598, Acct-Input-Packets=2656224, Acct-Output-Packets=7614179, Acct-Input-Gigawords=0, Acct-Output-Gigawords=1, NAS-Port-Type=Ethernet, NAS-Port-Id=FastEthernet0/47, undefined-151=31D7AADD, cisco-av-pair=audit-session-id=0AEF10030000032097B3F362, cisco-av-pair=connect-progress=Auth Open, AcsSessionID=ncsilsepsuie205/359238109/4017186, SelectedAccessService=Default Network Access, Step=11004, Step=11017, Step=15049, Step=15008, Step=15048, Step=15048, Step=15048, Step=15004, Step=22094, Step=11005, NetworkDeviceGroups=Location#All Locations#NC, NetworkDeviceGroups=Device Type#All Device Types#Switch#2960-Switches, NetworkDeviceGroups=All Network Device Groups#All Network Device Groups, CPMSessionID=0AEF10030000032097B3F362, AllowedProtocolMatchedRule=EAP-TLS, All Network Device Groups=All Network Device Groups#All Network Device Groups, Location=Location#All Locations#NC, Device Type=Device Type#All Device Types#Switch#2960-Switches, Network Device Profile=Cisco, +def test_cisco_ise_single(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} CISE_RADIUS_Accounting 0006028545 1 0 2019-10-24 21:00:02.879 +00:00 0088472694 3002 NOTICE Radius-Accounting: RADIUS Accounting watchdog update, ConfigVersionId=336, Device IP Address=10.0.0.3, RequestLatency=3, NetworkDeviceName=nc-aaa-aaa1, User-Name=U100000.ent.corp, NAS-IP-Address=10.0.0.3, NAS-Port=50047, Service-Type=Framed, Framed-IP-Address=10.0.0.80, Class=CACS:0AEF12345677832097B3F362:ncsilsepsuie212/356139633/9969901, Called-Station-ID=00-08-00-00-1B-AF, Calling-Station-ID=00-00-00-00-A0-7E, Acct-Status-Type=Interim-Update, Acct-Delay-Time=0, Acct-Input-Octets=653293631, Acct-Output-Octets=1497972244, Acct-Session-Id=00000B68, Acct-Authentic=RADIUS, Acct-Session-Time=241598, Acct-Input-Packets=2656224, Acct-Output-Packets=7614179, Acct-Input-Gigawords=0, Acct-Output-Gigawords=1, NAS-Port-Type=Ethernet, NAS-Port-Id=FastEthernet0/47, undefined-151=31D7AADD, cisco-av-pair=audit-session-id=0AEF10030000032097B3F362, cisco-av-pair=connect-progress=Auth Open, AcsSessionID=ncsilsepsuie205/359238109/4017186, SelectedAccessService=Default Network Access, Step=11004, Step=11017, Step=15049, Step=15008, Step=15048, Step=15048, Step=15048, Step=15004, Step=22094, Step=11005, NetworkDeviceGroups=Location#All Locations#NC, NetworkDeviceGroups=Device Type#All Device Types#Switch#2960-Switches, NetworkDeviceGroups=All Network Device Groups#All Network Device Groups, CPMSessionID=0AEF10030000032097B3F362, AllowedProtocolMatchedRule=EAP-TLS, All Network Device Groups=All Network Device Groups#All Network Device Groups, Location=Location#All Locations#NC, Device Type=Device Type#All Device Types#Switch#2960-Switches, Network Device Profile=Cisco,\n") + message = mt.render(mark="<165>", host=host) + sendsingle(message) + + st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cisco:ise:syslog\" | head 11") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_common.py b/tests/test_common.py index ce279a6..9855cc9 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -17,7 +17,7 @@ def test_defaultroute(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} sc4sdefault[0]: test\n") + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} test something else\n") message = mt.render(mark="<111>", host=host) sendsingle(message) @@ -55,7 +55,7 @@ def test_internal(record_property, setup_wordlist, setup_splunk): def test_tag(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) - mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} sc4sdefault[0]: test\n") + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} testvp-{{ host }} test\n") message = mt.render(mark="<111>", host=host) sendsingle(message) diff --git a/tests/test_infoblox.py b/tests/test_infoblox.py new file mode 100644 index 0000000..f7da9c7 --- /dev/null +++ b/tests/test_infoblox.py @@ -0,0 +1,58 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import datetime +import random +import pytz + +from jinja2 import Environment, environment + +from .sendmessage import * +from .splunkutils import * +import random + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<142>Oct 25 13:08:00 161.231.218.156 named[6597]: FORMERR resolving 'www.google.com/AAAA/IN': 209.2.230.6#53 +def test_infoblox_dns(record_property, setup_wordlist, setup_splunk): + host = "vib-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} named[{{ pid }}]: FORMERR resolving 'www.google.com/AAAA/IN': 209.2.230.6#53\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=netdns host={{ host }} sourcetype=\"infoblox:dns\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<150>Oct 24 21:09:00 162.101.157.246 dhcpd[28922]: DHCPREQUEST for 10.130.151.62 from 80:ce:62:9c:0e:70 (DTCCE0826E00C97) via eth2 TransID 802c562c uid 01:80:ce:62:9c:0e:70 (RENEW) +def test_infoblox_dhcp(record_property, setup_wordlist, setup_splunk): + host = "vib-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dhcpd[{{ pri }}]: DHCPREQUEST for 10.00.00.62 from 80:00:00:00:0e:70 (EXAMPLE) via eth2 TransID 802c562c uid 01:80:00:00:00:00:70 (RENEW)\n") + message = mt.render(mark="<150>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=netipam host={{ host }} sourcetype=\"infoblox:dhcp\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_linux_syslog.py b/tests/test_linux_syslog.py new file mode 100644 index 0000000..9297929 --- /dev/null +++ b/tests/test_linux_syslog.py @@ -0,0 +1,97 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import datetime +import random +import pytz + +from jinja2 import Environment, environment + +from .sendmessage import * +from .splunkutils import * +import random + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#<78>Oct 25 09:10:00 /usr/sbin/cron[54928]: (root) CMD (/usr/libexec/atrun) +def test_linux__nohost_program_as_path(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} /usr/sbin/cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_linux__host_program_as_path(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} /usr/sbin/cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_linux__nohost_program_conforms(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=osnix \"[{{ pid }}]\" sourcetype=\"nix:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_linux__host_program_conforms(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} cron[{{ pid }}]: (root) CMD (/usr/libexec/atrun)\n") + message = mt.render(mark="<111>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=osnix \"[{{ pid }}]\" host={{ host }} sourcetype=\"nix:syslog\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 \ No newline at end of file diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py index 2d40c8f..6211590 100644 --- a/tests/test_palo_alto.py +++ b/tests/test_palo_alto.py @@ -6,6 +6,7 @@ import random from jinja2 import Environment +from pytest import mark from .sendmessage import * from .splunkutils import * @@ -54,3 +55,45 @@ def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk): record_property("message", message) assert resultCount == 1 + +def test_palo_alto_traffic_badietf(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}1 {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +@mark.skip() +def test_palo_alto_traffic_mstime(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,{% now 'utc', '%Y/%m/%d %H:%M:%S.%f' %},2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=netfw host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_symantec_brightmail.py b/tests/test_symantec_brightmail.py new file mode 100644 index 0000000..6e877f9 --- /dev/null +++ b/tests/test_symantec_brightmail.py @@ -0,0 +1,71 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random +import uuid + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) +# <141>Oct 24 21:05:43 smg-1 conduit: [Brightmail] (NOTICE:7500.3119331456): [12066] 'BrightSig3 Newsletter Rules' were updated successfully. +def test_symantec_brightmail(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} conduit: [Brightmail] (NOTICE:7500.3119331456): [12066] 'BrightSig3 Newsletter Rules' were updated successfully.") + message = mt.render(mark="<134>", host=host) + sendsingle(message) + + st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"symantec:smg\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +def test_symantec_brightmail_msg(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + msgid = uuid.uuid4() + + mt = env.from_string("""{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|VERDICT|someone@example.com|none|default|default\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|FIRED|someone@example.com|none\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|UNTESTED|someone@example.com|safe|opl|content_1574820902092|content_1574820956288|content_1574821059194|content_1574821017042|sys_deny_ip|sys_allow_ip|sys_deny_email|dns_allow|dns_deny|user_allow|user_deny|freq_va|freq_dha|freq_sa|connection_class_0|connection_class_1|connection_class_2|connection_class_3|connection_class_4|connection_class_5|connection_class_6|connection_class_7|connection_class_8|connection_class_9|blockedlang|knownlang\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|LOGICAL_IP|200.200.200.154\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|google-play_111-33.png\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|mac_appstore_136_33.png\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|6f0c8ad8-e0da-4bcc-aac9-8fa71ba43bb6.jpg\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|b340ec99-9c66-4a19-a2f4-ee467e0d63e1.jpg\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|product-logo.update.png\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|header-research.png\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195989|{{ MSGID }}|ATTACHFILTER|ms-logo-138.png\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|ATTACH|ms-logo-138.png|header-research.png|product-logo.update.png|b340ec99-9c66-4a19-a2f4-ee467e0d63e1.jpg|6f0c8ad8-e0da-4bcc-aac9-8fa71ba43bb6.jpg|mac_appstore_136_33.png|google-play_111-33.png\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|EHLO|mail6.bemta23.messagelabs.com\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|MSG_SIZE|94239\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|MSGID| <7jszytr60wmja@example.com>\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|SUBJECT|pulse: this is a subject\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195988|{{ MSGID }}|SOURCE|external\n +{{ mark }}{% now 'utc', '%b %d %H:%M:%S' %} {{host}} bmserver: 1576195987|{{ MSGID }}|VERDICT||connection_class_1|default|static connection class 1\n""") + message = mt.render(mark="<1>", host=host, MSGID=msgid) + sendsingle(message) + + st = env.from_string("search index=email host=\"{{ host }}\" sourcetype=\"symantec:smg:mail\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# diff --git a/tests/test_vmware.py b/tests/test_vmware.py new file mode 100644 index 0000000..4ed57ca --- /dev/null +++ b/tests/test_vmware.py @@ -0,0 +1,79 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import datetime +import random +import pytz + +from jinja2 import Environment, environment + +from .sendmessage import * +from .splunkutils import * +import random + +env = Environment(extensions=['jinja2_time.TimeExtension']) + +#vpxd 123 - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)] +def test_linux_vmware(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} vpxd {{ pid }} - - Event [3481177] [1-1] [2019-05-23T09:03:36.213922Z] [vim.event.UserLoginSessionEvent] [info] [VSPHERE.LOCAL\svc-vcenter-user] [] [3481177] [User VSPHERE.LOCAL\svc-vcenter-user@192.168.10.10 logged in as pyvmomi Python/2.7.13 (Linux; 4.9.0-7-amd64; x86_64)]\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main {{ pid }} sourcetype=\"vmware:vsphere:esx\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +#<46>1 2019-10-24T21:00:02.403Z {{ host }} NSXV 5996 - [nsxv@6876 comp="nsx-manager" subcomp="manager"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704 +def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSXV {{ pid }} - [nsxv@6876 comp=\"nsx-manager\" subcomp=\"manager\"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main host={{ host }} PID={{ pid }} sourcetype=\"vmware:vsphere:nsx\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + +# +def test_linux_vmware_nsx_fw(record_property, setup_wordlist, setup_splunk): + host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + mt = env.from_string("{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} dfwpktlogs: {{ pid }} INET match PASS domain-c7/1001 IN 60 TCP 10.33.24.50/45926->10.33.24.9/8140 S\n") + message = mt.render(mark="<144>", host=host, pid=pid) + + sendsingle(message) + + st = env.from_string("search index=main host={{ host }} {{ pid }} sourcetype=\"vmware:vsphere:nsx\" | head 2") + search = st.render(host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/utility/pcapreplay/Dockerfile b/utility/pcapreplay/Dockerfile new file mode 100644 index 0000000..322eb6a --- /dev/null +++ b/utility/pcapreplay/Dockerfile @@ -0,0 +1,24 @@ +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . +FROM ubuntu:latest + +RUN apt-get update ;apt-get install -y build-essential autoconf wget git libboost-all-dev libpcap-dev libtool autogen + + +RUN cd ~ ; git clone https://github.com/ska-sa/udpreplay.git +RUN cd ~/udpreplay && \ + ./bootstrap.sh && \ + ./configure && \ + make && \ + make install && \ + cd ~ ; rm -Rf udpreplay + +RUN cd ~ ; git clone https://github.com/GabrielGanne/tcpreplay.git +RUN cd ~/tcpreplay && \ + ./autogen.sh; ./configure; make install +COPY entrypoint.sh / +ENTRYPOINT ["/entrypoint.sh"] diff --git a/utility/pcapreplay/entrypoint.sh b/utility/pcapreplay/entrypoint.sh new file mode 100755 index 0000000..fb099fc --- /dev/null +++ b/utility/pcapreplay/entrypoint.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +tail -f /dev/null