From 82cccc7debb0285bec31b5805089d9aaf5e4bb5b Mon Sep 17 00:00:00 2001 From: Ryan Faircloth <35384120+rfaircloth-splunk@users.noreply.github.com> Date: Mon, 15 Jul 2019 13:21:19 -0500 Subject: [PATCH] Release alpha checkpoint (#22) Initial release pre-alpha Supports Palo Alto Cisco ASA Cisco IOS (non NX) --- .circleci/config.yml | 40 +- .env.template | 11 + .gitignore | 13 +- .idea/misc.xml | 4 +- .idea/modules.xml | 8 + .idea/splunk-connect-for-syslog.iml | 19 + LICENSE-2.0 | 13 - LICENSE-BSD2 | 25 + LICENSE-CC0 | 121 ++ README.md | 48 +- clair-whitelist.yml | 20 + clair_to_junit_parser.py | 80 + demo-with-compose.sh | 22 + docker-compose.yml | 15 +- package/Dockerfile | 12 +- package/confluent.repo | 8 + package/etc/conf.d/blocks/b_destinations.conf | 251 --- package/etc/conf.d/blocks/b_parsers.conf | 78 +- package/etc/conf.d/blocks/b_sources.conf | 85 - package/etc/conf.d/blocks/splunk_context.conf | 7 + .../conf.d/destinations/d_destinations.conf | 50 + .../filters/cisco_asa_tradditional.conf | 24 + package/etc/conf.d/filters/cisco_ios.conf | 16 + package/etc/conf.d/filters/f_catchall.conf | 80 - package/etc/conf.d/filters/f_cisco-ios.conf | 64 - package/etc/conf.d/filters/f_palo_alto.conf | 67 - package/etc/conf.d/filters/f_websense.conf | 68 - .../etc/conf.d/filters/paloalto_networks.conf | 61 + package/etc/conf.d/sources/internal.conf | 7 + .../sources/{s_defaults.conf => network.conf} | 14 +- package/etc/conf.d/splunk.conf | 69 - package/etc/conf.d/templates/t_templates.conf | 24 +- package/etc/context/splunk_index.csv | 22 + package/etc/syslog-ng.conf | 90 +- package/goss.yaml | 8 + package/sbin/entrypoint.sh | 7 +- package/sbin/goss.yaml | 8 + package/scripts/splunkmetrics.sh | 9 +- package/scripts/switch_transport.sh | 32 - requirements.txt | 11 +- splunk/SA-syslog-ng/default/indexes.conf | 21 + test-with-compose.sh | 7 +- tests/Dockerfile | 8 + tests/__init__.py | 5 + tests/conftest.py | 45 + tests/data/wordlist.txt | 1633 +++++++++++++++++ tests/requirements.txt | 9 + tests/sendmessage.py | 26 + tests/splunkutils.py | 37 + tests/test_cisco_asa.py | 61 + tests/test_cisco_ios.py | 35 + tests/test_palo_alto.py | 60 + tests/test_poc.py | 89 +- 53 files changed, 2755 insertions(+), 892 deletions(-) create mode 100644 .idea/modules.xml create mode 100644 .idea/splunk-connect-for-syslog.iml delete mode 100644 LICENSE-2.0 create mode 100644 LICENSE-BSD2 create mode 100644 LICENSE-CC0 create mode 100644 clair-whitelist.yml create mode 100644 clair_to_junit_parser.py create mode 100755 demo-with-compose.sh delete mode 100755 package/etc/conf.d/blocks/b_destinations.conf mode change 100755 => 100644 package/etc/conf.d/blocks/b_parsers.conf delete mode 100755 package/etc/conf.d/blocks/b_sources.conf create mode 100644 package/etc/conf.d/blocks/splunk_context.conf create mode 100644 package/etc/conf.d/destinations/d_destinations.conf create mode 100644 package/etc/conf.d/filters/cisco_asa_tradditional.conf create mode 100644 package/etc/conf.d/filters/cisco_ios.conf delete mode 100755 package/etc/conf.d/filters/f_catchall.conf delete mode 100755 package/etc/conf.d/filters/f_cisco-ios.conf delete mode 100755 package/etc/conf.d/filters/f_palo_alto.conf delete mode 100755 package/etc/conf.d/filters/f_websense.conf create mode 100644 package/etc/conf.d/filters/paloalto_networks.conf create mode 100644 package/etc/conf.d/sources/internal.conf rename package/etc/conf.d/sources/{s_defaults.conf => network.conf} (77%) mode change 100755 => 100644 delete mode 100755 package/etc/conf.d/splunk.conf mode change 100755 => 100644 package/etc/conf.d/templates/t_templates.conf create mode 100644 package/etc/context/splunk_index.csv mode change 100755 => 100644 package/etc/syslog-ng.conf delete mode 100755 package/scripts/switch_transport.sh create mode 100644 splunk/SA-syslog-ng/default/indexes.conf create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/data/wordlist.txt create mode 100644 tests/sendmessage.py create mode 100644 tests/splunkutils.py create mode 100644 tests/test_cisco_asa.py create mode 100644 tests/test_cisco_ios.py create mode 100644 tests/test_palo_alto.py diff --git a/.circleci/config.yml b/.circleci/config.yml index d6d5b9b..4786b41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,16 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . version: 2.1 + +orbs: + clair_scanner: ovotech/clair-scanner@1 + jobs: build: environment: @@ -110,6 +122,27 @@ jobs: when: always - store_test_results: path: test-results + test-scan_images: + environment: + IMAGE_NAME: rfaircloth/scs + executor: clair_scanner/default + steps: + - clair_scanner/scan: + image: $IMAGE_NAME:$CIRCLE_SHA1 + whitelist: clair-whitelist.yml + - run: + command: | + mkdir -p /root/project/test-results + pip install -r requirements.txt + python clair_to_junit_parser.py "/clair-reports/$IMAGE_NAME:$CIRCLE_SHA1.json" --output test-results/results.xml + when: on_fail +# - store_test_results: +# path: test-results/results.xml + - store_artifacts: + path: test-results + - store_artifacts: + path: /clair-reports + publish: environment: IMAGE_NAME: rfaircloth/scs @@ -138,7 +171,12 @@ workflows: - dgoss: requires: - build - - test-unit + - test-unit: + requires: + - build + - test-scan_images: + requires: + - build - publish: requires: - dgoss diff --git a/.env.template b/.env.template index dd14966..8950cae 100644 --- a/.env.template +++ b/.env.template @@ -1,3 +1,11 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . RH_ORG=xxxx RH_ACTIVATION=xxxxx SPLUNK_HEC_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94 @@ -8,3 +16,6 @@ SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=metrics +SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download +SPLUNKBASE_USERNAME=username +SPLUNKBASE_PASSWORD=password diff --git a/.gitignore b/.gitignore index 36345c7..d96805f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,11 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . # Created by .ignore support plugin (hsz.mobi) ### GitBook template # Node rules: @@ -295,6 +303,9 @@ tags # Persistent undo [._]*.un~ -/test-results +/test-results/ /.idea/workspace.xml /.idea/tasks.xml +!/package/scripts/switch_transport.sh +!/package/scripts/splunkmetrics.sh +!/package/scripts/ diff --git a/.idea/misc.xml b/.idea/misc.xml index 87f2df8..86b631d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,8 +3,8 @@ - + - \ No newline at end of file + diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..4e16e75 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/splunk-connect-for-syslog.iml b/.idea/splunk-connect-for-syslog.iml new file mode 100644 index 0000000..0ecbb61 --- /dev/null +++ b/.idea/splunk-connect-for-syslog.iml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/LICENSE-2.0 b/LICENSE-2.0 deleted file mode 100644 index 913f595..0000000 --- a/LICENSE-2.0 +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2019 Splunk Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file diff --git a/LICENSE-BSD2 b/LICENSE-BSD2 new file mode 100644 index 0000000..904f2fe --- /dev/null +++ b/LICENSE-BSD2 @@ -0,0 +1,25 @@ +BSD 2-Clause License + +Copyright (c) 2019, Splunk, Inc +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENSE-CC0 b/LICENSE-CC0 new file mode 100644 index 0000000..1625c17 --- /dev/null +++ b/LICENSE-CC0 @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. \ No newline at end of file diff --git a/README.md b/README.md index 785eacf..02b59dd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,48 @@ # splunk-connect-for-syslog -Splunk Connect for Syslog + +Splunk Connect for Syslog is an open source packaged solution for +getting data into Splunk using syslog-ng (OSE) and the Splunk +HTTP event Collector. + +# Use the demo + +The Splunk Connect for syslog demo uses docker and docker compose +to configure a instance of Splunk along with syslog-ng and a test +harness to simulate a mix of events. Ensure git, docker and docker-compose +are pre-installed and working prior to continuing. + + +- Clone the repository and cd into directory + +```bash +git clone git@github.com:splunk/splunk-connect-for-syslog.git +cd splunk-connect-for-syslog +``` + +- Create a working .env file * Note for demo purposes this file does not need to be modified + +```bash +cp .env.template .env +``` + +- Start the demo environment + +```bash +./demo-with-compose.sh +``` + +- Login to splunk by browsing to http://127.0.0.1:8000 user name admin password "Changed@11" + +- Search the main index to see indexed events + +```spl +index = main +``` + +# License + + + +Configuration and documentation licensed subject to [CC0](LICENSE-CC0) + +Code and scripts licensed subject to [BSD-2-Clause](LICENSE-BSD2) \ No newline at end of file diff --git a/clair-whitelist.yml b/clair-whitelist.yml new file mode 100644 index 0000000..979a352 --- /dev/null +++ b/clair-whitelist.yml @@ -0,0 +1,20 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . +generalwhitelist: + RHSA-2019:1619: False Positive + RHSA-2018:0654: False Positive + RHSA-2018:1967: False Positive + RHSA-2017:0372: False Positive + RHSA-2018:0502: False Positive + RHSA-2018:2772: False Positive + RHSA-2018:1374: False Positive + RHSA-2018:0180: False Positive + +images: + scs: diff --git a/clair_to_junit_parser.py b/clair_to_junit_parser.py new file mode 100644 index 0000000..7dbc220 --- /dev/null +++ b/clair_to_junit_parser.py @@ -0,0 +1,80 @@ +#Copyright 2019 Splunk, Inc. +# +#Use of this source code is governed by a BSD-2-clause-style +#license that can be found in the LICENSE-BSD2 file or at +#https://opensource.org/licenses/BSD-2-Clause +import json +from junit_xml import TestSuite, TestCase +import os +import argparse +import logging + +logger = logging.getLogger('clair_scanner_converter') +logger.setLevel(logging.WARN) +console_logger = logging.StreamHandler() +console_logger.setLevel(logging.WARN) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +console_logger.setFormatter(formatter) +logger.addHandler(console_logger) + +def parse_args(): + parser = argparse.ArgumentParser(description="Process Json File") + parser.add_argument("clairfile", type=str, default=None, help="Location of clair scanner ouptut file to convert to cucumber.json") + parser.add_argument("--output", type=str, default=None, help="name of output file to store in new format. Defaults to clair inputfile") + args = parser.parse_args() + if not args.output: + logger.warning("No output file specified, replacing input file.") + args.output = args.clairfile + return args + +def main(): + cwd = os.getcwd() + args = parse_args() + try: + if os.path.exists(args.clairfile): + with open(args.clairfile) as clairfile: + clair_parsed_file = json.load(clairfile) + if os.path.exists(os.path.join("clair-scanner-logs", "/clair_setup_errors.log")): + with open(os.path.join("clair-scanner-logs", "/clair_setup_errors.log"), 'r') as clairfile_errors: + clair_parsed_error_file = clairfile_errors.readlines() + else: + clair_parsed_error_file = None + except: + logger.exception("Failed to parse clair / clair_error file. Exiting.") + + current_sorted_level = None + current_suite = None + test_suites = [] + if clair_parsed_error_file: + current_suite = TestSuite("SetupError") + new_step = TestCase(name="SetupError", classname="SetupError", status="unapproved", stderr=clair_parsed_error_file) + new_step.log = clair_parsed_error_file + new_step.category = "SetupError" + new_step.failure_type = "unapproved" + new_step.failure_message = "Please have the following security issue reviewed by Splunk: {}".format(vuln["link"]) + new_step.failure_output = clair_parsed_error_file + current_suite.test_cases.append(new_step) + test_suites.append(current_suite) + for vuln in clair_parsed_file["vulnerabilities"]: + if current_sorted_level != vuln["severity"]: + if current_suite: + test_suites.append(current_suite) + current_suite = TestSuite(name=vuln["severity"]) + current_sorted_level = vuln["severity"] + new_step = TestCase(name=vuln["vulnerability"], classname=vuln["severity"], status="unapproved", url=vuln["link"], stderr=vuln["description"]) + new_step.log = vuln + new_step.category = vuln["severity"] + new_step.failure_type = "unapproved" + new_step.failure_message = "Please have the following security issue reviewed by Splunk: {}".format(vuln["link"]) + new_step.failure_output = vuln["description"] + current_suite.test_cases.append(new_step) + # try to write new file + try: + with open(args.output, 'w') as outfile: + outfile.write(TestSuite.to_xml_string(test_suites)) + except: + logger.exception("Filed saving file.") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/demo-with-compose.sh b/demo-with-compose.sh new file mode 100755 index 0000000..86c0fbf --- /dev/null +++ b/demo-with-compose.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +#Copyright 2019 Splunk, Inc. +# +#Use of this source code is governed by a BSD-2-clause-style +#license that can be found in the LICENSE-BSD2 file or at +#https://opensource.org/licenses/BSD-2-Clause + +mkdir test-results +docker volume create sc4s-tests + +docker container create --name dummy \ + -v sc4s-tests:/work/tests \ + registry.access.redhat.com/ubi7/ubi +docker cp tests/ dummy:/work/tests/ +docker rm dummy + +docker-compose pull +docker-compose up + +EXIT=$0 + + diff --git a/docker-compose.yml b/docker-compose.yml index 829b864..94a8320 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,11 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . version: "3" services: test: @@ -12,7 +20,7 @@ services: - SPLUNK_PASSWORD sc4s: -# image: rfaircloth/scs:latest + image: rfaircloth/scs:latest build: context: ./package args: @@ -55,6 +63,11 @@ services: - SPLUNK_HEC_TOKEN - SPLUNK_PASSWORD - SPLUNK_START_ARGS + - SPLUNK_APPS_URL + - SPLUNKBASE_USERNAME + - SPLUNKBASE_PASSWORD + volumes: + - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng # logging: # driver: splunk # options: diff --git a/package/Dockerfile b/package/Dockerfile index 2c622a1..5ecc921 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -1,3 +1,11 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . FROM registry.access.redhat.com/rhel7/rhel ARG RH_ORG @@ -68,19 +76,21 @@ RUN cd /tmp ;\ ENV DEBCONF_NONINTERACTIVE_SEEN=true ENV SPLUNK_CONNECT_METHOD=hec +ENV SYSLOGNG_HEC_WORKERS=3 RUN source scl_source enable rh-python36 ; curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.7 sh COPY goss.yaml /etc/goss.yaml COPY --from=0 /opt/syslog-ng /opt/syslog-ng COPY scripts/splunkmetrics.sh /opt/syslog-ng/sbin/splunkmetrics.sh -COPY scripts/switch_transport.sh /opt/syslog-ng/sbin/switch_transport.sh COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d +COPY etc/context /opt/syslog-ng/etc/context COPY sbin/entrypoint.sh /sbin/entrypoint.sh RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t +RUN mkdir -p /var/log/syslog-ng/data/disk-buffer EXPOSE 514 EXPOSE 601/tcp diff --git a/package/confluent.repo b/package/confluent.repo index 1627e8d..26ecf54 100644 --- a/package/confluent.repo +++ b/package/confluent.repo @@ -1,3 +1,11 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . [Confluent.dist] name=Confluent repository (dist) baseurl=https://packages.confluent.io/rpm/5.2/7 diff --git a/package/etc/conf.d/blocks/b_destinations.conf b/package/etc/conf.d/blocks/b_destinations.conf deleted file mode 100755 index 3e65461..0000000 --- a/package/etc/conf.d/blocks/b_destinations.conf +++ /dev/null @@ -1,251 +0,0 @@ -# =============================================================================================== -# Destination Blocks (functions) for Splunk destinations -# Block argument defaults are defined in main syslog-ng.conf file -# Additional indexed fields (based on syslog-ng macros) can be added to the "with_metadata" blocks -# for both HEC and Kafka. -# Format is "fields.=" -# =============================================================================================== - -# =============================================================================================== -# blocks for file destination (for use with the Splunk Universal Forwarder) -# =============================================================================================== - -# =============================================================================================== -# Logfile Local Defaults: -# vendor: unassigned -# message: suffix for log file names; "messages" -# =============================================================================================== - -block destination d_splunk_logfile(category("") - vendor("unassigned") - filename("messages") - use_template(`splunk-default-template`) - ) { - file("`splunk-log-root`/`category`/$(lowercase \"${LOGHOST}\")/`vendor`/$(lowercase \"${HOST}\")/${YEAR}.${MONTH}.${DAY}.${HOUR}-`filename`" - template(`use_template`) - flush-lines(0) - create-dirs(yes) - dir_owner(`splunk-user`) - dir_group(`splunk-group`) - dir_perm(0700) - owner(`splunk-user`) - group(`splunk-group`) - perm(0600) - ); - }; - -# =============================================================================================== -# blocks for direct connection to Splunk via Kafka and HEC -# =============================================================================================== -# =============================================================================================== -# Kafka with Metadata (Facility and Priority): -# =============================================================================================== - -block destination d_kafka_with_metadata (kafka-libs(`kafka-lib-dir`) - kafka-brokers(`splunk-kafka-brokers`) - sourcetype(`splunk-sourcetype`) - topic(`splunk-topic`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - kafka(client-lib-dir("`kafka-libs`") - kafka-bootstrap-servers("`kafka-brokers`") - persist-name("`kafka_brokers`_`topic`_`sourcetype`_`index`") - template('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`) - fields.facility=$FACILITY - fields.severity=$LEVEL)') - topic("`topic`") - ); - }; - -# =============================================================================================== -# Kafka with Full Metadata; additional fields can be added -# =============================================================================================== - -block destination d_kafka_with_full_metadata (kafka-libs(`kafka-lib-dir`) - kafka-brokers(`splunk-kafka-brokers`) - sourcetype(`splunk-sourcetype`) - topic(`splunk-topic`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - kafka(client-lib-dir("`kafka-libs`") - kafka-bootstrap-servers("`kafka-brokers`") - persist-name("`kafka_brokers`_`topic`_`sourcetype`_`index`") - template('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`) - fields.facility=$FACILITY - fields.severity=$LEVEL - fields.message_header=$MSGHDR - fields.program_name=$PROGRAM - fields.pid=$PID - fields.full_message=$(template t_everything) - )') - topic("`topic`") - ); - }; - -# =============================================================================================== -# Basic Kafka (no indexed fields) -# =============================================================================================== - -block destination d_kafka_basic (kafka-libs(`kafka-lib-dir`) - kafka-brokers(`splunk-kafka-brokers`) - sourcetype(`splunk-sourcetype`) - topic(`splunk-topic`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - kafka(client-lib-dir("`kafka-libs`") - kafka-bootstrap-servers("`kafka-brokers`") - persist-name("`kafka_brokers`_`topic`_`sourcetype`_`index`") - template('$(format-json time=$S_UNIXTIME - host=$HOST - source=$HOST_FROM - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`))') - topic("`topic`") - ); - }; - -# =============================================================================================== -# HEC with Metadata (Facility and Priority): -# Be sure to adjust batch paramaters below to suit scale/environment -# If validated certs are used, uncomment relevant lines in the tls() block below -# and change peer-verify() to "yes" -# =============================================================================================== - -block destination d_hec_with_metadata (hec-url(`splunk-hec-url`) - hec-token(`splunk-hec-token`) - sourcetype(`splunk-sourcetype`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - http(url("`hec-url`") - method("POST") - batch-lines(10) - batch-bytes(512Kb) - batch-timeout(5000) - user_agent("syslog-ng User Agent") - user("syslog-ng") - password("`hec-token`") - persist-name("`hec-url`_`sourcetype`_`hec-token`_`index`") - tls(peer-verify(no) -# ca-dir("dir") -# ca-file("ca") -# cert-file("cert") -# cipher-suite("cipher") -# key-file("key") -# peer-verify(yes|no) -# ssl-version() - ) - body('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`) - fields.facility=$FACILITY - fields.severity=$LEVEL)') - ); - }; - -# =============================================================================================== -# HEC with Full Metadata; additional fields can be added -# Be sure to adjust batch paramaters below to suit scale/environment -# If validated certs are used, uncomment relevant lines in the tls() block below -# and change peer-verify() to "yes" -# =============================================================================================== - -block destination d_hec_with_full_metadata(hec-url(`splunk-hec-url`) - hec-token(`splunk-hec-token`) - sourcetype(`splunk-sourcetype`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - http(url("`hec-url`") - method("POST") - batch-lines(10) - batch-bytes(512Kb) - batch-timeout(5000) - user_agent("syslog-ng User Agent") - user("syslog-ng") - password("`hec-token`") - persist-name("`hec-url`_`sourcetype`_`hec-token`_`index`") - tls(peer-verify(no) -# ca-dir("dir") -# ca-file("ca") -# cert-file("cert") -# cipher-suite("cipher") -# key-file("key") -# peer-verify(yes|no) -# ssl-version() - ) - body('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`) - fields.fullhost_from=${FULLHOST_FROM} - fields.facility=$FACILITY - fields.severity=$LEVEL - fields.message_header=$MSGHDR - fields.program_name=$PROGRAM - fields.pid=$PID - fields.full_message=$(template t_everything) - )') - ); - }; - -# =============================================================================================== -# Basic HEC (no indexed fields) -# =============================================================================================== - -block destination d_hec_basic (hec-url(`splunk-hec-url`) - hec-token(`splunk-hec-token`) - sourcetype(`splunk-sourcetype`) - index(`splunk-index`) - use_template(`splunk-default-template`) - ) { - http(url("`hec-url`") - method("POST") - batch-lines(10) - batch-bytes(512Kb) - batch-timeout(5000) - user_agent("syslog-ng User Agent") - user("syslog-ng") - password("`hec-token`") - persist-name("`hec-url`_`sourcetype`_`hec-token`_`index`") - tls(peer-verify(no) -# ca-dir("dir") -# ca-file("ca") -# cert-file("cert") -# cipher-suite("cipher") -# key-file("key") -# peer-verify(yes|no) -# ssl-version() - ) - body('$(format-json - time=$S_UNIXTIME.$S_MSEC - host=$HOST - source=${HOST_FROM} - sourcetype=`sourcetype` - index=`index` - event=$(template `use_template`))') - ); - }; diff --git a/package/etc/conf.d/blocks/b_parsers.conf b/package/etc/conf.d/blocks/b_parsers.conf old mode 100755 new mode 100644 index b68a4e4..407ea8a --- a/package/etc/conf.d/blocks/b_parsers.conf +++ b/package/etc/conf.d/blocks/b_parsers.conf @@ -1,33 +1,69 @@ -# =============================================================================================== -# ForcePoint (Websense) -# =============================================================================================== - -# =============================================================================================== -# Log Format: -# <159>Dec 19 10:48:57 EST 10.203.28.21 vendor=Websense product=Security product_version=7.7.0 action=permitted severity=1 category=153 user=- src_host=10.64.134.74 src_port=62189 dst_host=mail.google.com dst_ip=74.125.224.53 dst_port=443 bytes_out=197 bytes_in=76 http_response=200 http_method=CONNECT http_content_type=- http_user_agent=Mozilla/5.0_(Windows;_U;_Windows_NT_6.1;_enUS;_rv:1.9.2.23)_Gecko/20110920_Firefox/3.6.23 http_proxy_status_code=200 reason=- disposition=1034 policy=- role=8 duration=0 url=https://mail.google.com -# -# Primary reason for non-compliance: %Z (timesone mnemonic) appended after date/time -# =============================================================================================== - -block parser p_websense-parser(prefix(".websense.")) { +#this parser can be used for "almost compliant" 5424 messages +block parser p_RFC5424-lite(prefix(".rfc5424-lite.")) { channel { rewrite { - # normal BSD timestamp, plus a timezone code. Remove the - # timezone information for now. - subst('([A-Za-z]{3} [0-9 ]\d \d{2}:\d{2}:\d{2}) [A-Z]{3,4}' "$1 " flags(global) value("MSG")); + # Remove Version Identifier + subst('(<\d+>)\d+' "$1" value("MSG")); - # add a $PROGRAM field, so that syslog-parser() would extract - # that properly - subst('(vendor=Websense)' "Websense: $1" flags(global) value("MSG")); }; - parser { - # by this time this message is a properly formatted syslog - # message + parser { + # by this time this message will parse with the BSD 3164 parser syslog-parser(time-zone(`default-timezone`)); # syslog-parser(); # extract name-value pairs. # kv-parser(prefix("`prefix`")); + }; }; +}; + +block parser p_test-parser(prefix(".testparse.")) { + channel { + rewrite { + # Put your rewrite rule here;use "MSG" as no-parse() puts entire payload there + # subst('(>.{12})\s' "${1}T" value("MSG")); + }; + parser { + # Don't initially parse so full output can be seen + # syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); + # syslog-parser(); + # syslog-parser(time-zone(`default-timezone`)); + # date-parser(format("%Y-%m-%d %H:%M:%S") template("$(substr ${MESSAGE} 0 19)") time-zone(`default-timezone`)); + }; }; }; + +#Used by fallback to log everything we can +block rewrite r_set_splunk(sourcetype(`splunk-sourcetype`) + index(`splunk-index`) + template(`splunk-default-template`) + vendor("unassigned") + category("") + filename("messages") + ) { + set("`index`", value(".splunk.index")); + set("`sourcetype`", value(".splunk.sourcetype")); + set("`template`", value(".splunk.template")); + set("`vendor`", value(".splunk.vendor")); + set("`category`", value(".splunk.category")); + set("`filename`", value(".splunk.filename")); + set($FACILITY, value("fields.facility")); + set($LEVEL, value("fields.severity")); + set($LOGHOST, value("fields.syslog-server")); + }; + +#Used when for normal filters index and sourcetype must be set in a prior re-write or context +block rewrite r_set_splunk_basic( + template(`splunk-default-template`) + vendor("unassigned") + category("") + filename("messages") + ) { + set("`template`", value(".splunk.template")); + set("`vendor`", value(".splunk.vendor")); + set("`category`", value(".splunk.category")); + set("`filename`", value(".splunk.filename")); + set($FACILITY, value("fields.facility")); + set($LEVEL, value("fields.severity")); + set($LOGHOST, value("fields.syslog-server")); + }; diff --git a/package/etc/conf.d/blocks/b_sources.conf b/package/etc/conf.d/blocks/b_sources.conf deleted file mode 100755 index 5e53849..0000000 --- a/package/etc/conf.d/blocks/b_sources.conf +++ /dev/null @@ -1,85 +0,0 @@ -# =============================================================================================== -# Source block for data arriving on unique ports; parsing defaults to "no-parse" -# =============================================================================================== - -block source s_unique-ports(udp_port() tcp_port() flags("no-parse")) { - `splunk-udp-driver` ( - transport("udp") - port(`udp_port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=514/udp") -# teardown("firewall-cmd --permanent --remove-port=514/udp") -# ) - so-rcvbuf(`splunk-rcvbuf`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - time-zone(`default-timezone`) - flags(`flags`) - ); - - `splunk-tcp-driver` ( - transport("tcp") - port(`tcp_port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=514/tcp") -# teardown("firewall-cmd --permanent --remove-port=514/tcp") -# ) - max-connections(`splunk-max-connections`) - log-iw-size(`splunk-window-size`) - log-fetch-limit(`splunk-fetch-limit`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - time-zone(`default-timezone`) - flags(`flags`) - ); -}; - -block source s_unique-udp-port(udp_port() flags("no-parse")) { - `splunk-udp-driver` ( - transport("udp") - port(`udp_port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=514/udp") -# teardown("firewall-cmd --permanent --remove-port=514/udp") -# ) - so-rcvbuf(`splunk-rcvbuf`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - time-zone(`default-timezone`) - flags(`flags`) - ); -}; - -block source s_unique-tcp-port(tcp_port() flags("no-parse")) { - `splunk-tcp-driver` ( - transport("tcp") - port(`tcp_port`) - ip-protocol(`ip-version`) -# hook-commands( -# setup("firewall-cmd --permanent --add-port=514/tcp") -# teardown("firewall-cmd --permanent --remove-port=514/tcp") -# ) - max-connections(`splunk-max-connections`) - log-iw-size(`splunk-window-size`) - log-fetch-limit(`splunk-fetch-limit`) - keep-hostname(yes) - keep-timestamp(yes) - use-dns(no) - use-fqdn(no) - chain-hostnames(off) - time-zone(`default-timezone`) - flags(`flags`) - ); -}; diff --git a/package/etc/conf.d/blocks/splunk_context.conf b/package/etc/conf.d/blocks/splunk_context.conf new file mode 100644 index 0000000..e2e0af9 --- /dev/null +++ b/package/etc/conf.d/blocks/splunk_context.conf @@ -0,0 +1,7 @@ +block parser p_add_context_splunk(key("syslogng-fallback")) { + add-contextual-data( + selector("`key`"), + database("context/splunk_index.csv"), + prefix(".splunk.") + ); +}; \ No newline at end of file diff --git a/package/etc/conf.d/destinations/d_destinations.conf b/package/etc/conf.d/destinations/d_destinations.conf new file mode 100644 index 0000000..1d373d4 --- /dev/null +++ b/package/etc/conf.d/destinations/d_destinations.conf @@ -0,0 +1,50 @@ + + +# =============================================================================================== +# Direct connection to Splunk via HEC +# =============================================================================================== + +# =============================================================================================== +# Be sure to adjust batch paramaters below to suit scale/environment +# Set workers to the number of indexers or HWF HEC endpoints +# If validated certs are used, uncomment relevant lines in the tls() block below +# and change peer-verify() to "yes" +# =============================================================================================== + +destination d_hec { + http( + url("`SPLUNK_HEC_URL`") + method("POST") + log-fifo-size(`splunk-log-fifo-size`) + workers(`SYSLOGNG_HEC_WORKERS`) + batch-lines(1000) + batch-bytes(512Kb) + batch-timeout(1000) + timeout(5) + user_agent("syslog-ng User Agent") + user("syslog-ng") + password("`SPLUNK_HEC_TOKEN`") +# persist-name("t_standard") +# disk-buffer(mem-buf-length(5000) +# disk-buf-size(107374182400) +# reliable(no) +# dir("/var/log/syslog-ng/data/disk-buffer")) + tls(peer-verify(no) +# ca-dir("dir") +# ca-file("ca") +# cert-file("cert") +# cipher-suite("cipher") +# key-file("key") +# peer-verify(yes|no) +# ssl-version() + ) + body('$(format-json + time=$S_UNIXTIME.$S_MSEC + host=$HOST + source=${HOST_FROM} + sourcetype=${.splunk.sourcetype} + index=${.splunk.index} + event=$(template ${.splunk.template}) + fields.*)') + ); + }; diff --git a/package/etc/conf.d/filters/cisco_asa_tradditional.conf b/package/etc/conf.d/filters/cisco_asa_tradditional.conf new file mode 100644 index 0000000..a912baf --- /dev/null +++ b/package/etc/conf.d/filters/cisco_asa_tradditional.conf @@ -0,0 +1,24 @@ +log { + source(s_default-ports); + + filter {tags("cisco:asa") + or message('^.{10,60} : %ASA-\d+-\d{1,10}: ') + ;}; + + #Cisco ASA can send 3264 or 5424 either way we are good with basic parsing + parser { + #basic parsing + syslog-parser(time-zone(`default-timezone`)); + }; + + #set the source type based on program field and lookup index from the splunk context csv + parser {p_add_context_splunk(key("cisco:asa")); }; + + #rewrite the final message for splunk json + #sourcetype and index are set in the filter defaults won't be used + rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- + + destination(d_hec); #--HEC-- + + flags(final); +}; diff --git a/package/etc/conf.d/filters/cisco_ios.conf b/package/etc/conf.d/filters/cisco_ios.conf new file mode 100644 index 0000000..98427f1 --- /dev/null +++ b/package/etc/conf.d/filters/cisco_ios.conf @@ -0,0 +1,16 @@ +# =============================================================================================== +# Cisco IOS (Route/Switch) +# =============================================================================================== + +log { + + source(s_default-ports); + + parser { cisco-parser(); }; + + parser {p_add_context_splunk(key("cisco:ios")); }; + rewrite { r_set_splunk_basic(template("t_msg_only"))}; #--HEC-- + destination(d_hec); #--HEC-- + + flags(final); +}; diff --git a/package/etc/conf.d/filters/f_catchall.conf b/package/etc/conf.d/filters/f_catchall.conf deleted file mode 100755 index aa9abbf..0000000 --- a/package/etc/conf.d/filters/f_catchall.conf +++ /dev/null @@ -1,80 +0,0 @@ -# =============================================================================================== -# "Catch-all" destination; all unfiltered messages end up here -# =============================================================================================== - -log { -# =============================================================================================== -# Section 1: source configuration -# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. -# -# IMPORTANT: In order to ensure proper startup, select only one source in this section. -# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed -# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those -# in source declarations from other filter files, an "Address already in use" error will occur -# at startup. -# -# NOTE: The "Catchall" should not use any source other than the default. Be extremely careful -# when defining an alternate source here. Default ports can be changed in the global declarations -# in the main syslog-ng.conf file. -# =============================================================================================== - -source(s_default-ports); - -# =============================================================================================== -# Section 2: parser configuration -# Default syslog-parser() should work for most messages -# -# If a new data source is not properly parsing using the default, you may wish to comment out -# all parsers to see the raw message as it appeared "on the wire". This can be useful for -# creating custom parsers for non-syslog-compliant meessages. -# =============================================================================================== - -# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. -parser { syslog-parser(time-zone(`default-timezone`)); }; - -# For RFC5424-formatted messages, use this instead -# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); }; - -# =============================================================================================== -# Section 3: filter configuration -# Filter below may be omitted in the log path if a dedicated source is configured above -# Filter typically not used for a fallback destination -# =============================================================================================== - -# filter { }; - -# =============================================================================================== -# Section 4: destination configuration -# Select the desired destination to use in the log path from the three choices below, -# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string -# for UF transport, and a unique sourcetype argument and optional index, topic, and message -# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the -# note below. -# -# IMPORTANT: If modifications are made in this section, ensure that two or more distinct -# destinations do *not* reference the same destination block call with an idential set of -# arguments (including no arguments, and arguments passed via destinations declared in other -# conf.d include files), or you will see this at startup: "Error checking the uniqueness of -# the persist names, please override it with persist-name option. Shutting down." -# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new -# or changed destination declaration. -# =============================================================================================== - -# =============================================================================================== -# This catchall file should be the *only* one where the destinations use defaults for all arguments! -# Full-metadata destinations used here; check indexed event data in Splunk for full message. -# This can be helpful to determine the compliance of message to standard syslog formats, and as -# an aid for custom parser development if needed. -# =============================================================================================== - -destination { -# d_splunk_logfile(); #--UF-- - d_hec_with_full_metadata(); #--HEC-- -# d_kafka_with_full_metadata(); #--KAFKA-- - }; - -# =============================================================================================== -# Fallback flag set here; will catch all unfiltered traffic -# =============================================================================================== -flags(fallback); -}; diff --git a/package/etc/conf.d/filters/f_cisco-ios.conf b/package/etc/conf.d/filters/f_cisco-ios.conf deleted file mode 100755 index c224a7a..0000000 --- a/package/etc/conf.d/filters/f_cisco-ios.conf +++ /dev/null @@ -1,64 +0,0 @@ -# =============================================================================================== -# Cisco IOS (Route/Switch) -# =============================================================================================== - -log { -# =============================================================================================== -# Section 1: source configuration -# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. -# -# IMPORTANT: In order to ensure proper startup, select only one source in this section. -# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed -# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those -# in source declarations from other filter files, an "Address already in use" error will occur -# at startup. -# =============================================================================================== - -source(s_default-ports); - -# source { s_unique-ports(udp_port(814) tcp_port(814)); }; - -# =============================================================================================== -# Section 2: parser configuration -# Default syslog-parser() should work for most messages -# =============================================================================================== - -# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. -parser { syslog-parser(time-zone(`default-timezone`)); }; - -# For RFC5424-formatted messages, use this instead -# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); }; - -# =============================================================================================== -# Section 3: filter configuration -# Filter below may be omitted in the log path if a dedicated source is configured above -# =============================================================================================== - -filter { match("[%|#][A-Z0-9_]+-[A-Z012_]*-*[0-7]-[A-Z0-9_]+:" value("MESSAGE")) - ;}; - - -# =============================================================================================== -# Section 4: destination configuration -# Select the desired destination to use in the log path from the three choices below, -# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string -# for UF transport, and a unique sourcetype argument and optional index, topic, and message -# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the -# note below. -# -# IMPORTANT: If modifications are made in this section, ensure that two or more distinct -# destinations do *not* reference the same destination block call with an idential set of -# arguments (including no arguments, and arguments passed via destinations declared in other -# conf.d include files), or you will see this at startup: "Error checking the uniqueness of -# the persist names, please override it with persist-name option. Shutting down." -# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new -# or changed destination declaration. -# =============================================================================================== - -destination { -# d_splunk_logfile(vendor("cisco-ios")); #--UF-- - d_hec_with_metadata(sourcetype("cisco:ios")); #--HEC-- -# d_kafka_with_metadata(sourcetype("cisco:ios")); #--KAFKA-- - }; -flags(final); -}; diff --git a/package/etc/conf.d/filters/f_palo_alto.conf b/package/etc/conf.d/filters/f_palo_alto.conf deleted file mode 100755 index 868b94c..0000000 --- a/package/etc/conf.d/filters/f_palo_alto.conf +++ /dev/null @@ -1,67 +0,0 @@ -# =============================================================================================== -# Palo Alto Networks Firewalls -# =============================================================================================== - -log { -# =============================================================================================== -# Section 1: source configuration -# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. -# -# IMPORTANT: In order to ensure proper startup, select only one source in this section. -# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed -# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those -# in source declarations from other filter files, an "Address already in use" error will occur -# at startup. -# =============================================================================================== - -source (s_default-ports); - -# source { s_unique-ports(udp_port(714) tcp_port(714)); }; - -# =============================================================================================== -# Section 2: parser configuration -# Default syslog-parser() should work for most messages -# =============================================================================================== - -# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. -parser { syslog-parser(time-zone(`default-timezone`)); }; - -# For RFC5424-formatted messages, use this instead -# parser { syslog-parser(time-zone(`default-timezone`) flags(syslog-protocol)); }; - -# =============================================================================================== -# Section 3: filter configuration -# Filter below may be omitted in the log path if a dedicated source is configured above -# =============================================================================================== - -filter { match(",THREAT," value("MESSAGE")) - or match(",TRAFFIC," value("MESSAGE")) - or match(",SYSTEM," value("MESSAGE")) - or match(",CONFIG," value("MESSAGE")) - or match(",HIPWATCH," value("MESSAGE")) - ;}; - -# =============================================================================================== -# Section 4: destination configuration -# Select the desired destination to use in the log path from the three choices below, -# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string -# for UF transport, and a unique sourcetype argument and optional index, topic, and message -# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the -# note below. -# -# IMPORTANT: If modifications are made in this section, ensure that two or more distinct -# destinations do *not* reference the same destination block call with an idential set of -# arguments (including no arguments, and arguments passed via destinations declared in other -# conf.d include files), or you will see this at startup: "Error checking the uniqueness of -# the persist names, please override it with persist-name option. Shutting down." -# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new -# or changed destination declaration. -# =============================================================================================== - -destination { -# d_splunk_logfile(vendor("paloalto")); #--UF-- - d_hec_with_metadata(sourcetype("pan:log") use_template("t_everything")); #--HEC-- -# d_kafka_with_metadata(sourcetype("pan:log") use_template("t_everything")); #--KAFKA-- - }; -flags(final); -}; diff --git a/package/etc/conf.d/filters/f_websense.conf b/package/etc/conf.d/filters/f_websense.conf deleted file mode 100755 index 16fb2af..0000000 --- a/package/etc/conf.d/filters/f_websense.conf +++ /dev/null @@ -1,68 +0,0 @@ -# =============================================================================================== -# Websense -# =============================================================================================== - -log { -# =============================================================================================== -# Section 1: source configuration -# Defaults: default source and ports as defined in s_sources.conf and syslog-ng.conf. -# -# IMPORTANT: In order to ensure proper startup, select only one source in this section. -# If a source other than the default is used, ensure that any UDP/TCP protocol/port pairs passed -# are unique throughout the *entire* sylog-ng config. If there are any port overlaps with those -# in source declarations from other filter files, an "Address already in use" error will occur -# at startup. -# =============================================================================================== - -source (s_default-ports); # The default port is common in many installations - -# The source block (function) call below can be used to specify unique listening ports for this -# device type - -# source { s_unique-ports(udp_port(714) tcp_port(714)); }; - -# =============================================================================================== -# Section 2: parser configuration -# Default syslog-parser() should work for most messages -# =============================================================================================== - -# This is "normal" RFC3164-formatted (BSD-syslog) messages. Always try this one first if unsure. -# parser { syslog-parser(); }; - -# For RFC5424-formatted messages, use this instead -# parser { syslog-parser(flags(syslog-protocol)); }; - -# For Websense -parser { p_websense-parser(); }; - -# =============================================================================================== -# Section 3: filter configuration -# Filter below may be omitted in the log path if a dedicated source is configured above -# =============================================================================================== - -filter { program(Websense); }; - -# =============================================================================================== -# Section 4: destination configuration -# Select the desired destination to use in the log path from the three choices below, -# based on the transfer method (UF/HEC/Kafka) desired. Then, provide a unique vendor string -# for UF transport, and a unique sourcetype argument and optional index, topic, and message -# template arguments (as appropriate) for HEC/Kafka transport, paying careful attention to the -# note below. -# -# IMPORTANT: If modifications are made in this section, ensure that two or more distinct -# destinations do *not* reference the same destination block call with an idential set of -# arguments (including no arguments, and arguments passed via destinations declared in other -# conf.d include files), or you will see this at startup: "Error checking the uniqueness of -# the persist names, please override it with persist-name option. Shutting down." -# The easiest way to satisfy this is to always pass a unique sourcetype argument for any new -# or changed destination declaration. -# =============================================================================================== - -destination { -# d_splunk_logfile(vendor("websense")); #--UF-- - d_hec_with_metadata(sourcetype("websense:cg:kv")); #--HEC-- -# d_kafka_with_metadata(sourcetype("websense:cg:kv")); #--KAFKA-- - }; -flags(final); -}; diff --git a/package/etc/conf.d/filters/paloalto_networks.conf b/package/etc/conf.d/filters/paloalto_networks.conf new file mode 100644 index 0000000..1ef160c --- /dev/null +++ b/package/etc/conf.d/filters/paloalto_networks.conf @@ -0,0 +1,61 @@ + + +log { + source(s_default-ports); + + filter {tags("pan:log") + or message(',\d+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPWATCH|CORRELATION|USERID),') + ;}; + + # the palo message does not include a program value in the header unfortunatly + # the use of colon in the message tricks the RFC3164 parser we will re-write the message + # so the parser will not incorrectly break it apart. + # while we are at it we will save the mesage type into the program field so parser can find it + rewrite { + subst('^([^ ]+ \w+ \d{2} \d{2}:\d{2}:\d{2} [^ ]+)( \d+,[^,]+,[^,]+,)([^,]+)(.*)', '$1 $3:$2$3$4', value("MESSAGE") ); + }; + parser { + #basic parsing + syslog-parser(time-zone(`default-timezone`)); + #we need to actual even time from the field GeneratedTime use csv parser to get it out + csv-parser( + columns('FUTURE_USE', 'ReceiveTime', 'SerialNumber', 'Type', 'Subtype', 'FUTURE_USE2', 'GeneratedTime') + prefix(".pan.") + delimiters(',') + ); + #2012/04/10 04:39:55 + #parse the date + date-parser( + format("%Y/%m/%d %H:%M:%S") + template("${.pan.GeneratedTime}") + time-zone("Universal") + ); + }; + + #set the source type based on program field and lookup index from the splunk context csv + if (program('THREAT')) { + parser {p_add_context_splunk(key("pan:threat")); }; + } elif (program('TRAFFIC')) { + parser {p_add_context_splunk(key("pan:traffic")); }; + } elif (program('SYSTEM')) { + parser {p_add_context_splunk(key("pan:system")); }; + } elif (program('CONFIG')) { + parser {p_add_context_splunk(key("pan:config")); }; + } elif (program('HIPWATCH')) { + parser {p_add_context_splunk(key("pan:hipwatch")); }; + } elif (program('CORRELATION')) { + parser {p_add_context_splunk(key("pan:correlation")); }; + } elif (program('USERID')) { + parser {p_add_context_splunk(key("pan:userid")); }; + } else { + parser {p_add_context_splunk(key("pan:log")); }; + }; + + #rewrite the final message for splunk json + #sourcetype and index are set in the filter defaults won't be used + rewrite {r_set_splunk_basic(template("t_msg_only")) }; #--HEC-- + + destination(d_hec); #--HEC-- + + flags(final); +}; diff --git a/package/etc/conf.d/sources/internal.conf b/package/etc/conf.d/sources/internal.conf new file mode 100644 index 0000000..5ba19a8 --- /dev/null +++ b/package/etc/conf.d/sources/internal.conf @@ -0,0 +1,7 @@ +# =============================================================================================== +# Internal sources +# =============================================================================================== + +source s_internal { + internal(); +}; diff --git a/package/etc/conf.d/sources/s_defaults.conf b/package/etc/conf.d/sources/network.conf old mode 100755 new mode 100644 similarity index 77% rename from package/etc/conf.d/sources/s_defaults.conf rename to package/etc/conf.d/sources/network.conf index 40425dd..e3ed6a9 --- a/package/etc/conf.d/sources/s_defaults.conf +++ b/package/etc/conf.d/sources/network.conf @@ -25,7 +25,6 @@ source s_default-ports { use-dns(no) use-fqdn(no) chain-hostnames(off) - time-zone(`default-timezone`) flags(no-parse) ); @@ -45,7 +44,18 @@ source s_default-ports { use-dns(no) use-fqdn(no) chain-hostnames(off) - time-zone(`default-timezone`) flags(no-parse) ); }; + +source s_all-ports { + `splunk-udp-driver` ( transport("udp") port(514) flags(no-parse) persist-name("example-persist-name1")); + `splunk-udp-driver` ( transport("udp") port(919) flags(no-parse)); + `splunk-udp-driver` ( transport("udp") port(920) flags(no-parse)); + `splunk-udp-driver` ( transport("udp") port(921) flags(no-parse)); + + `splunk-tcp-driver` ( transport("tcp") port(919) flags(no-parse)); + `splunk-tcp-driver` ( transport("tcp") port(920) flags(no-parse)); + `splunk-tcp-driver` ( transport("tcp") port(921) flags(no-parse)); + }; + diff --git a/package/etc/conf.d/splunk.conf b/package/etc/conf.d/splunk.conf deleted file mode 100755 index e8e5306..0000000 --- a/package/etc/conf.d/splunk.conf +++ /dev/null @@ -1,69 +0,0 @@ -# =============================================================================================== -# Defaults for remote data collection and transport to Splunk -# Kafka brokers/topics, HEC URL/token, local log directory for UF, sourcetype, and index -# These can be overridden in individual filter (f_*.conf) files -# =============================================================================================== - -# =============================================================================================== -# Source driver defaults (which govern the parser used) -# =============================================================================================== -@define splunk-udp-driver "syslog" -@define splunk-tcp-driver "network" - -# =============================================================================================== -# Listening ports -# =============================================================================================== -@define udp-listening-port "514" -@define tcp-listening-port "514" - -# =============================================================================================== -# UF transport -# =============================================================================================== -@define splunk-log-root "/var/log/syslog-ng/data" -@define splunk-user "root" # owner of the local disk files -@define splunk-group "root" # group of the local disk files - -# =============================================================================================== -# Kafka transport -# =============================================================================================== -@define kafka-lib-dir "/opt/kafka_2.11-0.11.0.0/libs" -@define splunk-kafka-brokers "127.0.0.1:9092" -@define splunk-topic "syslog" - -# =============================================================================================== -# HEC transport -# =============================================================================================== -@define splunk-hec-url __SPLUNK_HEC_URL__ -@define splunk-hec-token "__SPLUNK_HEC_TOKEN__" - -# =============================================================================================== -# Splunk metadata (HEC/Kafka transport only) -# =============================================================================================== -@define splunk-sourcetype "syslog:unassigned" -@define splunk-index "main" - -# =============================================================================================== -# Default message template -# =============================================================================================== -@define splunk-default-template "t_standard" - -# =============================================================================================== -# Data collection parameters, buffers, and Timezone -# =============================================================================================== -@define ip-version 4 -@define splunk-max-connections 1000 -@define splunk-fetch-limit 100 -# make sure splunk-window-size >= splunk-max-connections * splunk-fetch-limit -@define splunk-window-size 100000 -@define splunk-rcvbuf 425984 -@define default-timezone "GMT" - -# =============================================================================================== -# Global modules and includes. All device-specific filters and destinations exist in conf.d -# Order is important; templates must be loaded first -# =============================================================================================== -@include "scl.conf" -@include "conf.d/templates/*.conf" -@include "conf.d/sources/*.conf" -@include "conf.d/blocks/*.conf" -@include "conf.d/filters/*.conf" diff --git a/package/etc/conf.d/templates/t_templates.conf b/package/etc/conf.d/templates/t_templates.conf old mode 100755 new mode 100644 index b75dbbf..825a705 --- a/package/etc/conf.d/templates/t_templates.conf +++ b/package/etc/conf.d/templates/t_templates.conf @@ -8,7 +8,7 @@ # =============================================================================================== template t_standard { - template("${ISODATE} ${HOST} ${MESSAGE}\n"); + template("${ISODATE} ${HOST} ${MESSAGE}"); }; # =============================================================================================== @@ -16,13 +16,29 @@ template t_standard { # =============================================================================================== template t_msg_only { - template("${MESSAGE}\n"); + template("${MSGONLY}"); }; # =============================================================================================== -# Everything; useful for bluecoat proxy, palo alto, and others +# Everything; useful for bluecoat proxy and others # =============================================================================================== template t_everything { - template("${ISODATE} ${HOST} ${MSGHDR}${MESSAGE}\n"); + template("${ISODATE} ${HOST} ${MSGHDR}${MESSAGE}"); }; + +# =============================================================================================== +# Message Header with Message; for Palo Alto +# =============================================================================================== + +template t_hdr_msg { + template("${MSGHDR}${MESSAGE}"); + }; + +# =============================================================================================== +# JSON; for JSON pretty-printing +# =============================================================================================== + +template t_JSON { + template("$(format-json --scope all-nv-pairs)"); + }; diff --git a/package/etc/context/splunk_index.csv b/package/etc/context/splunk_index.csv new file mode 100644 index 0000000..75b98d6 --- /dev/null +++ b/package/etc/context/splunk_index.csv @@ -0,0 +1,22 @@ +cisco:asa,index,main +cisco:asa,sourcetype,cisco:asa +cisco:ios,index,main +cisco:ios,sourcetype,cisco:ios +pan:traffic,index,main +pan:threat,index,main +pan:system,index,main +pan:config,index,main +pan:hipwatch,index,main +pan:correlation,index,main +pan:userid,index,main +pan:unknown,index,main +pan:traffic,sourcetype,pan:traffic +pan:threat,sourcetype,pan:threat +pan:system,sourcetype,pan:system +pan:config,sourcetype,pan:config +pan:hipwatch,sourcetype,pan:hipwatch +pan:correlation,sourcetype,pan:correlation +pan:userid,sourcetype,pan:userid +pan:unknown,sourcetype,pan:unknown +syslogng-fallback,index,syslogng-fallback +syslogng-fallback,sourcetype,syslogng-fallback diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf old mode 100755 new mode 100644 index 68c8b17..ab28af9 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -17,71 +17,71 @@ options { log_msg_size (65536); flush_lines (0); time_reopen (10); - log_fifo_size (1000); + log_fifo_size (10000); chain_hostnames (off); use_dns (no); use_fqdn (no); - dns-cache (no); + dns-cache(no); create_dirs (no); keep-hostname (yes); }; # =============================================================================================== -# Remote data sources that will be sent to Splunk +# Source driver defaults (which govern the parser used) # =============================================================================================== - -@include "conf.d/splunk.conf" +@define splunk-udp-driver "syslog" +@define splunk-tcp-driver "network" # =============================================================================================== -# Local System sources +# Listening ports # =============================================================================================== - -source s_sys { - internal(); -}; +@define udp-listening-port "514" +@define tcp-listening-port "514" # =============================================================================================== -# Local System destinations +# Splunk metadata (HEC/Kafka transport only) # =============================================================================================== - -destination d_cons { file("/dev/console"); }; -destination d_mesg { file("/var/log/syslog-ng/messages"); }; -destination d_auth { file("/dev/shm/secure"); }; -destination d_mail { file("/dev/shm/maillog" flush_lines(10)); }; -destination d_spol { file("/dev/shm/spooler"); }; -destination d_boot { file("/dev/shm/boot.log"); }; -destination d_cron { file("/dev/shm/cron"); }; -destination d_kern { file("/dev/shm/kern"); }; -destination d_mlal { usertty("*"); }; -destination d_syslog { file("/var/log/syslog"); }; +@define splunk-sourcetype "syslog:fallback" +@define splunk-index "main" # =============================================================================================== -# Local System filters +# Default message template # =============================================================================================== +@define splunk-default-template "t_standard" -filter f_kernel { facility(kern); }; -filter f_default { level(info..emerg) and - not (facility(mail) - or facility(authpriv) - or facility(cron)); }; -filter f_auth { facility(authpriv); }; -filter f_mail { facility(mail); }; -filter f_emergency { level(emerg); }; -filter f_news { facility(uucp) or - (facility(news) - and level(crit..emerg)); }; -filter f_boot { facility(local7); }; -filter f_cron { facility(cron); }; +# =============================================================================================== +# Data collection parameters, buffers, and Timezone +# =============================================================================================== +@define ip-version 4 +@define splunk-max-connections 1000 +@define splunk-log-fifo-size 10000 +@define splunk-fetch-limit 100 +# make sure splunk-window-size >= splunk-max-connections * splunk-fetch-limit +@define splunk-window-size 100000 +@define splunk-rcvbuf 425984 +@define default-timezone "GMT" # =============================================================================================== -# Local System Logging +# Global modules and includes. All device-specific filters and destinations exist in conf.d +# Order is important; templates must be loaded first # =============================================================================================== +@include "scl.conf" +@include "conf.d/templates/*.conf" +@include "conf.d/sources/*.conf" +@include "conf.d/blocks/*.conf" +@include "conf.d/destinations/*.conf" +@include "conf.d/prefilters/*.conf" +@include "conf.d/filters/*.conf" + +@include "conf.d/plugin/*/blocks/*.conf" +@include "conf.d/plugin/*/prefilters/*.conf" +@include "conf.d/plugin/*/filters/*.conf" + +log { + source(s_default-ports); + rewrite { r_set_splunk(template("t_JSON"))}; #--HEC-- + destination(d_hec); #--HEC-- + flags(fallback); +}; + -log { source(s_sys); filter(f_kernel); destination(d_kern); }; -log { source(s_sys); filter(f_default); destination(d_mesg); }; -log { source(s_sys); filter(f_auth); destination(d_auth); }; -log { source(s_sys); filter(f_mail); destination(d_mail); }; -log { source(s_sys); filter(f_emergency); destination(d_mlal); }; -log { source(s_sys); filter(f_news); destination(d_spol); }; -log { source(s_sys); filter(f_boot); destination(d_boot); }; -log { source(s_sys); filter(f_cron); destination(d_cron); }; diff --git a/package/goss.yaml b/package/goss.yaml index 5c1893e..83c913b 100644 --- a/package/goss.yaml +++ b/package/goss.yaml @@ -1,3 +1,11 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . process: syslog-ng: running: true diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 7a7b0aa..3578897 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -1,8 +1,11 @@ #!/opt/rh/rh-python36/root/usr/bin/dumb-init /bin/bash - +#Copyright 2019 Splunk, Inc. +# +#Use of this source code is governed by a BSD-2-clause-style +#license that can be found in the LICENSE-BSD2 file or at +#https://opensource.org/licenses/BSD-2-Clause #Run syslog mkdir /opt/syslog-ng/var rm /opt/syslog-ng/var/syslog-ng.ctl || true -/opt/syslog-ng/sbin/switch_transport.sh /opt/syslog-ng/sbin/syslog-ng --process-mode=background /opt/syslog-ng/sbin/splunkmetrics.sh diff --git a/package/sbin/goss.yaml b/package/sbin/goss.yaml index abd12fb..7d1cd7d 100644 --- a/package/sbin/goss.yaml +++ b/package/sbin/goss.yaml @@ -1,3 +1,11 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . process: syslog-ng: running: true diff --git a/package/scripts/splunkmetrics.sh b/package/scripts/splunkmetrics.sh index d62014d..7a497eb 100755 --- a/package/scripts/splunkmetrics.sh +++ b/package/scripts/splunkmetrics.sh @@ -1,4 +1,9 @@ #!/bin/bash +#Copyright 2019 Splunk, Inc. +# +#Use of this source code is governed by a BSD-2-clause-style +#license that can be found in the LICENSE-BSD2 file or at +#https://opensource.org/licenses/BSD-2-Clause sleep 30 while sleep 10 do @@ -10,7 +15,7 @@ do while read -r SourceName SourceId SourceInstance State Type Number do echo "{ - \"time\":\"$TS.000\", + \"time\":\"$TS\", \"event\":\"metric\", \"source\":\"syslog-ng\", \"host\":\"$HOSTNAME\", @@ -27,4 +32,6 @@ do done < $tmpfilecsv curl -s -S -k $SPLUNK_HEC_STATSURL -H "Authorization: Splunk $SPLUNK_HEC_TOKEN" -d "@$tmpfilejson" + rm $tmpfilejson + rm $tmpfilecsv done diff --git a/package/scripts/switch_transport.sh b/package/scripts/switch_transport.sh deleted file mode 100755 index 15863c3..0000000 --- a/package/scripts/switch_transport.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Script to change all filters to use selected transport (UF, HEC, or kafka). -# If one or more filters uses a transport different from all the others, they will need to be -# managed by hand. Unpredictable results can occur if the script is used and -# individual filters are not checked for correctness. - -case $SPLUNK_CONNECT_METHOD in - hec) - echo "Switching transport method for all filters to HEC" - sed '/^# / {/#--HEC--/ s/^#/ /}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--KAFKA--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--UF--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed 's~__SPLUNK_HEC_URL__~"'"$SPLUNK_HEC_URL"'"~' -i /opt/syslog-ng/etc/conf.d/splunk.conf - sed 's/__SPLUNK_HEC_TOKEN__/'"$SPLUNK_HEC_TOKEN"'/' -i /opt/syslog-ng/etc/conf.d/splunk.conf - ;; - kafka) - echo "Switching transport method for all filters to Kafka" - sed '/^# / {/#--KAFKA--/ s/^#/ /}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--HEC--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--UF--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - ;; - UF) - echo "Switching transport method for all filters to filesystem/UF" - sed '/^# / {/#--UF--/ s/^#/ /}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--HEC--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - sed '/^#/! {/#--KAFKA--/ s/^ /#/}' -i /opt/syslog-ng/etc/conf.d/filters/*.conf; - ;; - *) - echo "Usage: $0 hec|kafka|UF" - ;; -esac diff --git a/requirements.txt b/requirements.txt index 7b11f5e..345c370 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,11 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . -r ./tests/requirements.txt - +junit_xml +argparse diff --git a/splunk/SA-syslog-ng/default/indexes.conf b/splunk/SA-syslog-ng/default/indexes.conf new file mode 100644 index 0000000..e795078 --- /dev/null +++ b/splunk/SA-syslog-ng/default/indexes.conf @@ -0,0 +1,21 @@ +[syslogng_metrics] +datatype=metric +homePath = $SPLUNK_DB/syslogng_metrics/db +coldPath = $SPLUNK_DB/syslogng_metrics/colddb +thawedPath = $SPLUNK_DB/syslogng_metrics/thaweddb + +[syslogng_fallback] +homePath = $SPLUNK_DB/syslogng_fallback/db +coldPath = $SPLUNK_DB/syslogng_fallback/colddb +thawedPath = $SPLUNK_DB/syslogng_fallback/thaweddb + + +[test] +homePath = $SPLUNK_DB/test/db +coldPath = $SPLUNK_DB/test/colddb +thawedPath = $SPLUNK_DB/test/thaweddb + +[test2] +homePath = $SPLUNK_DB/test2/db +coldPath = $SPLUNK_DB/test2/colddb +thawedPath = $SPLUNK_DB/test2/thaweddb diff --git a/test-with-compose.sh b/test-with-compose.sh index f426220..400c3c8 100755 --- a/test-with-compose.sh +++ b/test-with-compose.sh @@ -1,4 +1,9 @@ #!/usr/bin/env bash +#Copyright 2019 Splunk, Inc. +# +#Use of this source code is governed by a BSD-2-clause-style +#license that can be found in the LICENSE-BSD2 file or at +#https://opensource.org/licenses/BSD-2-Clause mkdir test-results docker volume create sc4s-tests @@ -22,5 +27,3 @@ docker container create --name dummy \ docker cp dummy:/work/test-results/functional test-results docker rm dummy EXIT=$0 - - diff --git a/tests/Dockerfile b/tests/Dockerfile index 36fa35e..cb630f3 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,3 +1,11 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . FROM python:3.7 COPY requirements.txt / diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..4913c30 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,5 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..cc16563 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,45 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import os +import random +from time import sleep + +import pytest +import splunklib.client as client + + +@pytest.fixture(scope="module") +def setup_wordlist(): + path_to_current_file = os.path.realpath(__file__) + current_directory = os.path.split(path_to_current_file)[0] + path_to_file = os.path.join(current_directory, "data/wordlist.txt") + + wordlist = [line.rstrip('\n') for line in open(path_to_file)] + return wordlist + + +@pytest.fixture +def get_host_key(setup_wordlist): + part1 = random.choice(setup_wordlist) + part2 = random.choice(setup_wordlist) + host = "{}-{}".format(part1, part2) + + return host + + +@pytest.fixture +def setup_splunk(): + tried = 0 + while True: + try: + c = client.connect(username="admin", password="Changed@11", host="splunk", port="8089") + break + except ConnectionRefusedError: + tried += 1 + if tried > 180: + raise + sleep(1) + return c diff --git a/tests/data/wordlist.txt b/tests/data/wordlist.txt new file mode 100644 index 0000000..d6a73fe --- /dev/null +++ b/tests/data/wordlist.txt @@ -0,0 +1,1633 @@ +acrobat +africa +alaska +albert +albino +album +alcohol +alex +alpha +amadeus +amanda +amazon +america +analog +animal +antenna +antonio +apollo +april +aroma +artist +aspirin +athlete +atlas +banana +bandit +banjo +bikini +bingo +bonus +camera +canada +carbon +casino +catalog +cinema +citizen +cobra +comet +compact +complex +context +credit +critic +crystal +culture +david +delta +dialog +diploma +doctor +domino +dragon +drama +extra +fabric +final +focus +forum +galaxy +gallery +global +harmony +hotel +humor +index +japan +kilo +lemon +liter +lotus +mango +melon +menu +meter +metro +mineral +model +music +object +piano +pirate +plastic +radio +report +signal +sport +studio +subject +super +tango +taxi +tempo +tennis +textile +tokyo +total +tourist +video +visa +academy +alfred +atlanta +atomic +barbara +bazaar +brother +budget +cabaret +cadet +candle +capsule +caviar +channel +chapter +circle +cobalt +comrade +condor +crimson +cyclone +darwin +declare +denver +desert +divide +dolby +domain +double +eagle +echo +eclipse +editor +educate +edward +effect +electra +emerald +emotion +empire +eternal +evening +exhibit +expand +explore +extreme +ferrari +forget +freedom +friday +fuji +galileo +genesis +gravity +habitat +hamlet +harlem +helium +holiday +hunter +ibiza +iceberg +imagine +infant +isotope +jackson +jamaica +jasmine +java +jessica +kitchen +lazarus +letter +license +lithium +loyal +lucky +magenta +manual +marble +maxwell +mayor +monarch +monday +money +morning +mother +mystery +native +nectar +nelson +network +nikita +nobel +nobody +nominal +norway +nothing +number +october +office +oliver +opinion +option +order +outside +package +pandora +panther +papa +pattern +pedro +pencil +people +phantom +philips +pioneer +pluto +podium +portal +potato +process +proxy +pupil +python +quality +quarter +quiet +rabbit +radical +radius +rainbow +ramirez +ravioli +raymond +respect +respond +result +resume +richard +river +roger +roman +rondo +sabrina +salary +salsa +sample +samuel +saturn +savage +scarlet +scorpio +sector +serpent +shampoo +sharon +silence +simple +society +sonar +sonata +soprano +sparta +spider +sponsor +abraham +action +active +actor +adam +address +admiral +adrian +agenda +agent +airline +airport +alabama +aladdin +alarm +algebra +alibi +alice +alien +almond +alpine +amber +amigo +ammonia +analyze +anatomy +angel +annual +answer +apple +archive +arctic +arena +arizona +armada +arnold +arsenal +arthur +asia +aspect +athena +audio +august +austria +avenue +average +axiom +aztec +bagel +baker +balance +ballad +ballet +bambino +bamboo +baron +basic +basket +battery +belgium +benefit +berlin +bermuda +bernard +bicycle +binary +biology +bishop +blitz +block +blonde +bonjour +boris +boston +bottle +boxer +brandy +bravo +brazil +bridge +british +bronze +brown +bruce +bruno +brush +burger +burma +cabinet +cactus +cafe +cairo +calypso +camel +campus +canal +cannon +canoe +cantina +canvas +canyon +capital +caramel +caravan +career +cargo +carlo +carol +carpet +cartel +cartoon +castle +castro +cecilia +cement +center +century +ceramic +chamber +chance +change +chaos +charlie +charm +charter +cheese +chef +chemist +cherry +chess +chicago +chicken +chief +china +cigar +circus +city +clara +classic +claudia +clean +client +climax +clinic +clock +club +cockpit +coconut +cola +collect +colombo +colony +color +combat +comedy +command +company +concert +connect +consul +contact +contour +control +convert +copy +corner +corona +correct +cosmos +couple +courage +cowboy +craft +crash +cricket +crown +cuba +dallas +dance +daniel +decade +decimal +degree +delete +deliver +delphi +deluxe +demand +demo +denmark +derby +design +detect +develop +diagram +diamond +diana +diego +diesel +diet +digital +dilemma +direct +disco +disney +distant +dollar +dolphin +donald +drink +driver +dublin +duet +dynamic +earth +east +ecology +economy +edgar +egypt +elastic +elegant +element +elite +elvis +email +empty +energy +engine +english +episode +equator +escape +escort +ethnic +europe +everest +evident +exact +example +exit +exotic +export +express +factor +falcon +family +fantasy +fashion +fiber +fiction +fidel +fiesta +figure +film +filter +finance +finish +finland +first +flag +flash +florida +flower +fluid +flute +folio +ford +forest +formal +formula +fortune +forward +fragile +france +frank +fresh +friend +frozen +future +gabriel +gamma +garage +garcia +garden +garlic +gemini +general +genetic +genius +germany +gloria +gold +golf +gondola +gong +good +gordon +gorilla +grand +granite +graph +green +group +guide +guitar +guru +hand +happy +harbor +harvard +havana +hawaii +helena +hello +henry +hilton +history +horizon +house +human +icon +idea +igloo +igor +image +impact +import +india +indigo +input +insect +instant +iris +italian +jacket +jacob +jaguar +janet +jargon +jazz +jeep +john +joker +jordan +judo +jumbo +june +jungle +junior +jupiter +karate +karma +kayak +kermit +king +koala +korea +labor +lady +lagoon +laptop +laser +latin +lava +lecture +left +legal +level +lexicon +liberal +libra +lily +limbo +limit +linda +linear +lion +liquid +little +llama +lobby +lobster +local +logic +logo +lola +london +lucas +lunar +machine +macro +madam +madonna +madrid +maestro +magic +magnet +magnum +mailbox +major +mama +mambo +manager +manila +marco +marina +market +mars +martin +marvin +mary +master +matrix +maximum +media +medical +mega +melody +memo +mental +mentor +mercury +message +metal +meteor +method +mexico +miami +micro +milk +million +minimum +minus +minute +miracle +mirage +miranda +mister +mixer +mobile +modem +modern +modular +moment +monaco +monica +monitor +mono +monster +montana +morgan +motel +motif +motor +mozart +multi +museum +mustang +natural +neon +nepal +neptune +nerve +neutral +nevada +news +next +ninja +nirvana +normal +nova +novel +nuclear +numeric +nylon +oasis +observe +ocean +octopus +olivia +olympic +omega +opera +optic +optimal +orange +orbit +organic +orient +origin +orlando +oscar +oxford +oxygen +ozone +pablo +pacific +pagoda +palace +pamela +panama +pancake +panda +panel +panic +paradox +pardon +paris +parker +parking +parody +partner +passage +passive +pasta +pastel +patent +patient +patriot +patrol +pegasus +pelican +penguin +pepper +percent +perfect +perfume +period +permit +person +peru +phone +photo +picasso +picnic +picture +pigment +pilgrim +pilot +pixel +pizza +planet +plasma +plaza +pocket +poem +poetic +poker +polaris +police +politic +polo +polygon +pony +popcorn +popular +postage +precise +prefix +premium +present +price +prince +printer +prism +private +prize +product +profile +program +project +protect +proton +public +pulse +puma +pump +pyramid +queen +radar +ralph +random +rapid +rebel +record +recycle +reflex +reform +regard +regular +relax +reptile +reverse +ricardo +right +ringo +risk +ritual +robert +robot +rocket +rodeo +romeo +royal +russian +safari +salad +salami +salmon +salon +salute +samba +sandra +santana +sardine +school +scoop +scratch +screen +script +scroll +second +secret +section +segment +select +seminar +senator +senior +sensor +serial +service +shadow +sharp +sheriff +shock +short +shrink +sierra +silicon +silk +silver +similar +simon +single +siren +slang +slogan +smart +smoke +snake +social +soda +solar +solid +solo +sonic +source +soviet +special +speed +sphere +spiral +spirit +spring +static +status +stereo +stone +stop +street +strong +student +style +sultan +susan +sushi +suzuki +switch +symbol +system +tactic +tahiti +talent +tarzan +telex +texas +theory +thermos +tiger +titanic +tomato +topic +tornado +toronto +torpedo +totem +tractor +traffic +transit +trapeze +travel +tribal +trick +trident +trilogy +tripod +tropic +trumpet +tulip +tuna +turbo +twist +ultra +uniform +union +uranium +vacuum +valid +vampire +vanilla +vatican +velvet +ventura +venus +vertigo +veteran +victor +vienna +viking +village +vincent +violet +violin +virtual +virus +vision +visitor +visual +vitamin +viva +vocal +vodka +volcano +voltage +volume +voyage +water +weekend +welcome +western +window +winter +wizard +wolf +world +xray +yankee +yoga +yogurt +yoyo +zebra +zero +zigzag +zipper +zodiac +zoom +acid +adios +agatha +alamo +alert +almanac +aloha +andrea +anita +arcade +aurora +avalon +baby +baggage +balloon +bank +basil +begin +biscuit +blue +bombay +botanic +brain +brenda +brigade +cable +calibre +carmen +cello +celtic +chariot +chrome +citrus +civil +cloud +combine +common +cool +copper +coral +crater +cubic +cupid +cycle +depend +door +dream +dynasty +edison +edition +enigma +equal +eric +event +evita +exodus +extend +famous +farmer +food +fossil +frog +fruit +geneva +gentle +george +giant +gilbert +gossip +gram +greek +grille +hammer +harvest +hazard +heaven +herbert +heroic +hexagon +husband +immune +inca +inch +initial +isabel +ivory +jason +jerome +joel +joshua +journal +judge +juliet +jump +justice +kimono +kinetic +leonid +leopard +lima +maze +medusa +member +memphis +michael +miguel +milan +mile +miller +mimic +mimosa +mission +monkey +moral +moses +mouse +nancy +natasha +nebula +nickel +nina +noise +orchid +oregano +origami +orinoco +orion +othello +paper +paprika +prelude +prepare +pretend +promise +prosper +provide +puzzle +remote +repair +reply +rival +riviera +robin +rose +rover +rudolf +saga +sahara +scholar +shelter +ship +shoe +sigma +sister +sleep +smile +spain +spark +split +spray +square +stadium +star +storm +story +strange +stretch +stuart +subway +sugar +sulfur +summer +survive +sweet +swim +table +taboo +target +teacher +telecom +temple +tibet +ticket +tina +today +toga +tommy +tower +trivial +tunnel +turtle +twin +uncle +unicorn +unique +update +valery +vega +version +voodoo +warning +william +wonder +year +yellow +young +absent +absorb +absurd +accent +alfonso +alias +ambient +anagram +andy +anvil +appear +apropos +archer +ariel +armor +arrow +austin +avatar +axis +baboon +bahama +bali +balsa +barcode +bazooka +beach +beast +beatles +beauty +before +benny +betty +between +beyond +billy +bison +blast +bless +bogart +bonanza +book +border +brave +bread +break +broken +bucket +buenos +buffalo +bundle +button +buzzer +byte +caesar +camilla +canary +candid +carrot +cave +chant +child +choice +chris +cipher +clarion +clark +clever +cliff +clone +conan +conduct +congo +costume +cotton +cover +crack +current +danube +data +decide +deposit +desire +detail +dexter +dinner +donor +druid +drum +easy +eddie +enjoy +enrico +epoxy +erosion +except +exile +explain +fame +fast +father +felix +field +fiona +fire +fish +flame +flex +flipper +float +flood +floor +forbid +forever +fractal +frame +freddie +front +fuel +gallop +game +garbo +gate +gelatin +gibson +ginger +giraffe +gizmo +glass +goblin +gopher +grace +gray +gregory +grid +griffin +ground +guest +gustav +gyro +hair +halt +harris +heart +heavy +herman +hippie +hobby +honey +hope +horse +hostel +hydro +imitate +info +ingrid +inside +invent +invest +invite +ivan +james +jester +jimmy +join +joseph +juice +julius +july +kansas +karl +kevin +kiwi +ladder +lake +laura +learn +legacy +legend +lesson +life +light +list +locate +lopez +lorenzo +love +lunch +malta +mammal +margin +margo +marion +mask +match +mayday +meaning +mercy +middle +mike +mirror +modest +morph +morris +mystic +nadia +nato +navy +needle +neuron +never +newton +nice +night +nissan +nitro +nixon +north +oberon +octavia +ohio +olga +open +opus +orca +oval +owner +page +paint +palma +parent +parlor +parole +paul +peace +pearl +perform +phoenix +phrase +pierre +pinball +place +plate +plato +plume +pogo +point +polka +poncho +powder +prague +press +presto +pretty +prime +promo +quest +quick +quiz +quota +race +rachel +raja +ranger +region +remark +rent +reward +rhino +ribbon +rider +road +rodent +round +rubber +ruby +rufus +sabine +saddle +sailor +saint +salt +scale +scuba +season +secure +shake +shallow +shannon +shave +shelf +sherman +shine +shirt +side +sinatra +sincere +size +slalom +slow +small +snow +sofia +song +sound +south +speech +spell +spend +spoon +stage +stamp +stand +state +stella +stick +sting +stock +store +sunday +sunset +support +supreme +sweden +swing +tape +tavern +think +thomas +tictac +time +toast +tobacco +tonight +torch +torso +touch +toyota +trade +tribune +trinity +triton +truck +trust +type +under +unit +urban +urgent +user +value +vendor +venice +verona +vibrate +virgo +visible +vista +vital +voice +vortex +waiter +watch +wave +weather +wedding +wheel +whiskey +wisdom +android +annex +armani +cake +confide +deal +define +dispute +genuine +idiom +impress +include +ironic +null +nurse +obscure +prefer +prodigy +ego +fax +jet +job +rio +ski +yes \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index 199f49b..149b7a8 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,14 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . pytest jinja2 jinja2-time http://dev.splunk.com/goto/sdk-python flake8 +flaky diff --git a/tests/sendmessage.py b/tests/sendmessage.py new file mode 100644 index 0000000..6388c78 --- /dev/null +++ b/tests/sendmessage.py @@ -0,0 +1,26 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +import socket +from time import sleep + + +def sendsingle(message): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_address = ('sc4s', 514) + + tried = 0 + while True: + try: + sock.connect(server_address) + break + except socket: + tried += 1 + if tried > 90: + raise + sleep(1) + sock.sendall(str.encode(message)) + sock.close() diff --git a/tests/splunkutils.py b/tests/splunkutils.py new file mode 100644 index 0000000..7227ae9 --- /dev/null +++ b/tests/splunkutils.py @@ -0,0 +1,37 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +from time import sleep + + +def splunk_single(service, search): + kwargs_normalsearch = {"exec_mode": "normal"} + tried = 0 + while True: + job = service.jobs.create(search, **kwargs_normalsearch) + + # A normal search returns the job's SID right away, so we need to poll for completion + while True: + while not job.is_ready(): + pass + stats = {"isDone": job["isDone"], + "doneProgress": float(job["doneProgress"]) * 100, + "scanCount": int(job["scanCount"]), + "eventCount": int(job["eventCount"]), + "resultCount": int(job["resultCount"])} + + if stats["isDone"] == "1": + break + sleep(2) + + # Get the results and display them + resultCount = stats["resultCount"] + eventCount = stats["eventCount"] + if resultCount > 0 or tried > 15: + break + else: + tried += 1 + sleep(1) + return resultCount, eventCount diff --git a/tests/test_cisco_asa.py b/tests/test_cisco_asa.py new file mode 100644 index 0000000..a908fc4 --- /dev/null +++ b/tests/test_cisco_asa.py @@ -0,0 +1,61 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from flaky import flaky +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + + +# Apr 15 2017 00:21:14 192.168.12.1 : %ASA-5-111010: User 'john', running 'CLI' from IP 0.0.0.0, executed 'dir disk0:/dap.xml' +# Apr 15 2017 00:22:27 192.168.12.1 : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:81.24.28.226 dst inside:72.142.17.10 (type 3, code 0) on outside interface. Original IP payload: udp src 72.142.17.10/40998 dst 194.153.237.66/53. +# Apr 15 2017 00:22:42 192.168.12.1 : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 +@flaky(max_runs=3, min_passes=2) +def test_cisco_asa_tradditional(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:asa\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +# <166>2018-06-27T12:17:46Z asa : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/8949 to outside:72.142.18.38/23 +def test_cisco_asa_rfc5424(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} : %ASA-3-710003: TCP access denied by ACL from 179.236.133.160/5424 to outside:72.142.18.38/23\n") + message = mt.render(mark="<166>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:asa\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_cisco_ios.py b/tests/test_cisco_ios.py new file mode 100644 index 0000000..19746af --- /dev/null +++ b/tests/test_cisco_ios.py @@ -0,0 +1,35 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + + +# <190>30: foo: *Apr 29 13:58:46.411: %SYS-6-LOGGINGHOST_STARTSTOP: Logging to host 192.168.1.239 stopped - CLI initiated +def test_cisco_ios(record_property, setup_wordlist, get_host_key, setup_splunk): + host = get_host_key + + mt = env.from_string( + "{{ mark }}{{ seq }}: {{ host }}: *{% now 'utc', '%b %d %H:%M:%S' %}.100: CET: %SEC-6-IPACCESSLOGP: list 110 denied tcp 54.122.123.124(8932) -> 10.1.0.1(22), 1 packet\n") + message = mt.render(mark="<166>", seq=20, host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"cisco:ios\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py new file mode 100644 index 0000000..ec8744e --- /dev/null +++ b/tests/test_palo_alto.py @@ -0,0 +1,60 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause +import random + +from flaky import flaky +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * + +env = Environment(extensions=['jinja2_time.TimeExtension']) + + +# <190>Jan 28 01:28:35 PA-VM300-goran1 1,2014/01/28 01:28:35,007200001056,TRAFFIC,end,1,2014/01/28 01:28:34,192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0 +@flaky(max_runs=3, min_passes=2) +def test_palo_alto_traffic(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},007200001056,TRAFFIC,end,1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},192.168.41.30,192.168.41.255,10.193.16.193,192.168.41.255,allow-all,,,netbios-ns,vsys1,Trust,Untrust,ethernet1/1,ethernet1/2,To-Panorama,2014/01/28 01:28:34,8720,1,137,137,11637,137,0x400000,udp,allow,276,276,0,3,2014/01/28 01:28:02,2,any,0,2076326,0x0,192.168.0.0-192.168.255.255,192.168.0.0-192.168.255.255,0,3,0\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"pan:traffic\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + + +# <190>Oct 30 09:46:17 1,2012/10/30 09:46:17,01606001116,THREAT,url,1,2012/04/10 04:39:55,192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:57,22860,1,59303,80,0,0,0x208000,tcp,alert,"litetopdetect.cn/index.php",(9999),not-resolved,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,text/html +@flaky(max_runs=3, min_passes=2) +def test_palo_alto_threat(record_property, setup_wordlist, setup_splunk): + host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + + mt = env.from_string( + "{{ mark }} {% now 'utc', '%b %d %H:%M:%S' %} {{ host }} 1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},01606001116,THREAT,url,1,{% now 'utc', '%Y/%m/%d %H:%M:%S' %},192.168.0.2,204.232.231.46,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:39:57,22860,1,59303,80,0,0,0x208000,tcp,alert,\"litetopdetect.cn/index.php\",(9999),not-resolved,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,text/html\n") + message = mt.render(mark="<111>", host=host) + + sendsingle(message) + + st = env.from_string("search index=main host=\"{{ host }}\" sourcetype=\"pan:threat\" | head 2") + search = st.render(host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 diff --git a/tests/test_poc.py b/tests/test_poc.py index e04aa01..37dcdef 100644 --- a/tests/test_poc.py +++ b/tests/test_poc.py @@ -1,87 +1,20 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause import random -import socket -import urllib.request -from time import sleep -import pytest -import splunklib.client as client +from flaky import flaky from jinja2 import Environment -env = Environment(extensions=['jinja2_time.TimeExtension']) - - -@pytest.fixture -def setup_wordlist(): - word_url = "http://svnweb.freebsd.org/csrg/share/dict/words?view=co&content-type=text/plain" - response = urllib.request.urlopen(word_url) - long_txt = response.read().decode() - return long_txt.splitlines() - - -@pytest.fixture -def setup_splunk(): - tried = 0 - while True: - try: - c = client.connect(username="admin", password="Changed@11", host="splunk", port="8089") - break - except ConnectionRefusedError: - tried += 1 - if tried > 90: - raise - sleep(1) - return c - - -def sendsingle(message): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_address = ('sc4s', 514) +from .sendmessage import * +from .splunkutils import * - tried = 0 - while True: - try: - sock.connect(server_address) - break - except socket: - tried += 1 - if tried > 90: - raise - sleep(1) - sock.sendall(str.encode(message)) - sock.close() - - -def splunk_single(service, search): - kwargs_normalsearch = {"exec_mode": "normal"} - tried = 0 - while True: - job = service.jobs.create(search, **kwargs_normalsearch) - - # A normal search returns the job's SID right away, so we need to poll for completion - while True: - while not job.is_ready(): - pass - stats = {"isDone": job["isDone"], - "doneProgress": float(job["doneProgress"]) * 100, - "scanCount": int(job["scanCount"]), - "eventCount": int(job["eventCount"]), - "resultCount": int(job["resultCount"])} - - if stats["isDone"] == "1": - break - sleep(2) - - # Get the results and display them - resultCount = stats["resultCount"] - eventCount = stats["eventCount"] - if resultCount > 0 or tried > 15: - break - else: - tried += 1 - sleep(1) - return resultCount, eventCount +env = Environment(extensions=['jinja2_time.TimeExtension']) +@flaky(max_runs=3, min_passes=2) def test_defaultroute(record_property, setup_wordlist, setup_splunk): host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) @@ -90,7 +23,7 @@ def test_defaultroute(record_property, setup_wordlist, setup_splunk): sendsingle(message) - st = env.from_string("search \"{{ host }}\" | head 2") + st = env.from_string("search index=main \"{{ host }}\" sourcetype=\"syslog:fallback\" | head 2") search = st.render(host=host) resultCount, eventCount = splunk_single(setup_splunk, search)