Feature/hostcontext (#61)

RHINO-221 Remove use of FLAKY 
RHINO-228 Allow explicit mapping to log-path by hostname (parsed) or IP Fixes #45

.circleci/config.yml

@@ -123,13 +123,16 @@ jobs:
                     registry.access.redhat.com/ubi7/ubi
             docker cp tests/ dummy:/work/tests/
             docker rm dummy
-#      - run:
-#          name: Docker Compose build
-#          command: |
-#            docker-compose build
+      - run:
+          name: Docker Compose build
+          command: |
+            docker-compose build test
       - run:
           name: Docker Compose up
-          command: docker-compose  -f ./docker-compose-ci.yml up --abort-on-container-exit
+          command: |
+            docker-compose  -f ./docker-compose-ci.yml up -d splunk
+            sleep 70
+            docker-compose  -f ./docker-compose-ci.yml up --abort-on-container-exit
       - run:
           name: Collect test results from Docker Compose
           command: |

docker-compose-ci.yml

@@ -39,6 +39,23 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
+  sc4s-juniper:
+    image: rfaircloth/scs:${CIRCLE_SHA1}
+    hostname: sc4s-juniper
+    ports:
+      - "514"
+    stdin_open: true
+    tty: true
+    links:
+      - splunk
+    environment:
+      - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
+      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
+      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+      - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
+      - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
+      - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
+      - SYSLOG_PRESUME_FILTER=juniper_netscreen
 #    logging:
 #      driver: splunk
 #      options:

docker-compose-debug.yml

@@ -0,0 +1,56 @@
+#Splunk Connect for Syslog (SC4S) by Splunk, Inc.
+#
+#To the extent possible under law, the person who associated CC0 with
+#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights
+#to Splunk Connect for Syslog (SC4S).
+#
+#You should have received a copy of the CC0 legalcode along with this
+#work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
+version: "3.2"
+services:
+
+  sc4s:
+    image: splunk/scs:latest
+    build:
+      context: ./package
+      args:
+        RH_ORG: ${RH_ORG}
+        RH_ACTIVATION: ${RH_ACTIVATION}
+    entrypoint:
+      - "tail"
+      - "-f"
+      - "/dev/null"
+    hostname: sc4s
+    ports:
+      - "514"
+      - "601"
+      - "514/udp"
+      - "5514"
+      - "5514/udp"
+    stdin_open: true
+    tty: true
+    links:
+      - splunk
+    environment:
+      - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
+      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
+      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+      - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
+      - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
+      - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
+  splunk:
+    image: splunk/splunk:latest
+    hostname: splunk
+    ports:
+      - "8000:8000"
+      - "8088:8088"
+      - "8089:8089"
+    environment:
+      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+      - SPLUNK_PASSWORD=${SPLUNK_PASSWORD}
+      - SPLUNK_START_ARGS=${SPLUNK_START_ARGS}
+      - SPLUNK_APPS_URL=${SPLUNK_APPS_URL}
+      - SPLUNKBASE_USERNAME=${SPLUNKBASE_USERNAME}
+      - SPLUNKBASE_PASSWORD=${SPLUNKBASE_PASSWORD}
+    volumes:
+      - ./splunk/SA-syslog-ng:/opt/splunk/etc/apps/SA-syslog-ng

docker-compose.yml

@@ -13,6 +13,7 @@ services:
     links:
       - splunk
       - sc4s
+      - sc4s-juniper
     volumes:
       - sc4s-tests:/work/tests
       - sc4s-results:/work/test-results
@@ -44,6 +45,32 @@ services:
       - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
       - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
       - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
+  sc4s-juniper:
+    image: splunk/scs:latest
+    build:
+      context: ./package
+#      cache_from:
+#        - registry.access.redhat.com/ubi7/ubi
+#        - registry.access.redhat.com/rhel7/rhel
+#        - rfaircloth/scs:edge
+      args:
+        RH_ORG: ${RH_ORG}
+        RH_ACTIVATION: ${RH_ACTIVATION}
+    hostname: sc4s-juniper
+    ports:
+      - "514"
+    stdin_open: true
+    tty: true
+    links:
+      - splunk
+    environment:
+      - SPLUNK_HEC_URL=${SPLUNK_HEC_URL}
+      - SPLUNK_HEC_STATSURL=${SPLUNK_HEC_STATSURL}
+      - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+      - SPLUNK_CONNECT_METHOD=${SPLUNK_CONNECT_METHOD}
+      - SPLUNK_DEFAULT_INDEX=${SPLUNK_DEFAULT_INDEX}
+      - SPLUNK_METRICS_INDEX=${SPLUNK_DEFAULT_INDEX}
+      - SYSLOG_PRESUME_FILTER=juniper_netscreen
 #    logging:
 #      driver: splunk
 #      options:

docs/gettingstarted.md

@@ -22,11 +22,11 @@ services:
     image: splunk/scs:latest
     hostname: sc4s
     ports:
-      - "514"
-      - "601"
-      - "514/udp"
-      - "5514"
-      - "5514/udp"
+      - "514:514"
+      - "601:601"
+      - "514:514/udp"
+      - "5514:5514"
+      - "5514:5514/udp"
     stdin_open: true
     tty: true
     environment:
@@ -36,16 +36,25 @@ services:
       - SPLUNK_DEFAULT_INDEX=<defaultindex>
       - SPLUNK_METRICS_INDEX=em_metrics
     volumes:
-    - splunk_index.csv:/opt/syslog-ng/etc/context/splunk_index.csv
+    - ./sc4s/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
+    - ./sc4s/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
 ```
 
-* Download the latest context.csv file to the current directory
+## Configure index destinations for Splunk 
+* Download the latest context.csv file to a subdirectory sc4s below the docker-compose.yml file created above
 ```bash
-wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context/splunk_index.csv
+wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/splunk_index.csv
 ```
-
 * Edit splunk_index.csv review the index configuration and revise as required for sourcertypes utilized in your environment.
 
+## Configure sources by source IP or host name
+* This step is required even if not used
+* Download the latest vendor_product_by_source.conf file to a subdirectory sc4s below the docker-compose.yml file created above
+```bash
+wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/vendor_product_by_source.conf
+```
+* Edit the file to identify appropriate vendor products by host glob or network mask using syslog-ng filter syntax.
+
 * Start SC4S
 
 ```bash
@@ -56,3 +65,34 @@ docker stack deploy --compose-file docker-compose.yml sc4s
 ## Scale out
 
 Additional hosts can be deployed for syslog collection from additional network zones and locations
+
+
+## Single Source Technology instance - Alpha
+
+For certain source technologies message categorization by content is impossible to support collection 
+of such legacy nonstandard sources we provide a means of dedicating a container to a specific source using
+an alternate port. In the following configration example a dedicated port is opened (6514) for legacy juniper netscreen devices
+
+This approach is "alpha" and subject to change
+
+```yaml
+version: "3"
+services:
+  sc4s-juniper-netscreen:
+    image: splunk/scs:latest
+    hostname: sc4s-juniper-netscreen
+    ports:
+      - "6514:514"
+      - "6514:514/udp"
+    stdin_open: true
+    tty: true
+    environment:
+      - SPLUNK_HEC_URL=https://foo:8088/services/collector/event
+      - SPLUNK_HEC_TOKEN=<token>
+      - SPLUNK_CONNECT_METHOD=hec
+      - SPLUNK_DEFAULT_INDEX=<defaultindex>
+      - SPLUNK_METRICS_INDEX=em_metrics
+      - SYSLOG_PRESUME_FILTER=f_juniper_netscreen
+    volumes:
+    - ./sc4s-juniper/splunk_index.csv:/opt/syslog-ng/etc/context-local/splunk_index.csv
+```

package/Dockerfile

@@ -83,7 +83,7 @@ COPY goss.yaml /etc/goss.yaml
 COPY --from=0 /opt/syslog-ng /opt/syslog-ng
 COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf
 COPY etc/conf.d /opt/syslog-ng/etc/conf.d
-COPY etc/context /opt/syslog-ng/etc/context
+COPY etc/context-local /opt/syslog-ng/etc/context-local
 COPY sbin/entrypoint.sh /sbin/entrypoint.sh
 
 RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer

package/etc/conf.d/conflib/_common/rfc_syslog.conf

@@ -0,0 +1,27 @@
+filter f_rfc5424_strict{
+    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>(?<FULLDATE>(?<FULLDATEYEAR>\d{4})-(?<FULLDATEMONTH>\d\d)-(?<FULLDATEDAY>\d\d))T(?<FULLTIME>(?<PARTIALTIME>(?<TIMEHOUR>[0-2]\d):(?<TIMEMINUTE>[0-5]\d):(?<TIMESECOND>[0-5]\d)(?:.(?<TIMESECFRAC>\d{1,6}))?)(?<TIMEOFFSET>Z|(?<TIMENUMOFFSET>[+\-][0-2]\d:[0-5]\d))))))');
+};
+filter f_rfc5424_noversion{
+    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>) ?(?<TIMESTAMP>(?<FULLDATE>(?<FULLDATEYEAR>\d{4})-(?<FULLDATEMONTH>\d\d)-(?<FULLDATEDAY>\d\d))T(?<FULLTIME>(?<PARTIALTIME>(?<TIMEHOUR>[0-2]\d):(?<TIMEMINUTE>[0-5]\d):(?<TIMESECOND>[0-5]\d)(?:.(?<TIMESECFRAC>\d{1,6}))?)(?<TIMEOFFSET>Z|(?<TIMENUMOFFSET>[+\-][0-2]\d:[0-5]\d))))))');
+};
+rewrite set_rfcnonconformant{
+    set("rfc5424_nonconform" value("fields.sc4s_syslog_format"));
+};
+rewrite set_rfc5424_strict{
+    set("rfc5424_strict" value("fields.sc4s_syslog_format"));
+};
+filter f_is_rfc5424_strict{
+    match("rfc5424_strict" value("fields.sc4s_syslog_format"))
+};
+rewrite set_rfc5424_noversion{
+    set("rfc5424_noversion" value("fields.sc4s_syslog_format"));
+};
+filter f_is_rfc5424_noversion{
+    match("rfc5424_noversion" value("fields.sc4s_syslog_format"))
+};
+rewrite set_rfc3164{
+    set("rfc3164" value("fields.sc4s_syslog_format"));
+};
+filter f_is_rfc3164{
+    match("rfc3164" value("fields.sc4s_syslog_format"))
+};

package/etc/conf.d/templates/t_templates.conf → package/etc/conf.d/conflib/_common/templates.conf

@@ -48,5 +48,9 @@ template t_hdr_sdata_msg {
 # ===============================================================================================
 
 template t_JSON {
-	template("$(format-json --scope all-nv-pairs)");
+	template("$(format-json --scope all-nv-pairs
+	            --exclude fields.*
+	            --exclude .splunk.*
+	            --exclude HOST --exclude HOSTFROMgit m
+	            )");
 	};

package/etc/conf.d/conflib/_common/utility.conf

@@ -0,0 +1,4 @@
+
+rewrite set_metadata_presume {
+    set("$(env SYSLOG_PRESUME_FILTER)" value("fields.sc4s_presume"));
+};

package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf

@@ -0,0 +1,9 @@
+block parser vendor_product_by_source() {
+    add-contextual-data(
+        selector(filters("/opt/syslog-ng/etc/context-local/vendor_product_by_source.conf")),
+        database("context-local/vendor_product_by_source.csv")
+        ignore-case(yes)
+        prefix("fields.")
+    );
+};
+

package/etc/conf.d/blocks/splunk_context.conf → package/etc/conf.d/conflib/_splunk/splunk_context.conf

@@ -1,7 +1,7 @@
 block parser p_add_context_splunk(key("syslogng-fallback")) {
     add-contextual-data(
         selector("`key`"),
-        database("context/splunk_index.csv"),
+        database("context-local/splunk_index.csv"),
         prefix(".splunk.")
     );
 };