Merge pull request #278 from splunk/update/log_paths

Rename log paths and update with junction/channel architecture
CSVD · Jan 20, 2020 · e10b49b · e10b49b
2 parents c86a6d5 + 9ae33b5
commit e10b49b
Show file tree

Hide file tree

Showing 32 changed files with 435 additions and 550 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -8,7 +8,7 @@
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
 version: 2.1
 orbs:
-  docker: circleci/docker@0.5.13
+  docker: circleci/docker@0.5.20
   go: circleci/go@0.2.0
   snyk: snyk/snyk@0.0.8
   versioning: kollex/versioning@1.0.0
@@ -94,7 +94,7 @@ jobs:
       - run: git submodule sync
       - run: git submodule update --init --recursive
 
-      - docker/install-docker-tools
+      #- docker/install-docker-tools
       - docker/check:
           registry: $REGISTRY
           docker-username: GITHUB_USER
@@ -418,4 +418,4 @@ workflows:
             tags:
               only: /^\d*\.\d*\.\d*.*$/
             branches:
-              ignore: /.*/
+              ignore: /.*/
diff --git a/.../local/config/log_paths/example.conf.tmpl → ...cal/config/log_paths/lp-example.conf.tmpl b/.../local/config/log_paths/example.conf.tmpl → ...cal/config/log_paths/lp-example.conf.tmpl
@@ -29,24 +29,23 @@
 {{- $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
-{{- /* The following is an inline template to generate the actual log path */}}
-{{- define "log_path"}}
 log {
-
-{{- /* The first time this template is used the log_path will be linked to the default port */}}
-
-{{- if eq (.) "yes"}}
-    source(s_DEFAULT);
-    filter(f_is_rfc3164);
-    filter(f_local_example);
-{{- end}}
-
-{{- /* In the second pass through the template a link to the dedicated port is used. This */}}
-{{- /* normally does not require additional filters */}}
-
-{{- if eq (.) "no"}}
-    source (s_LOCAL_EXAMPLE);
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT")) (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic
+            source (s_LOCAL_EXAMPLE);
+            flags (final);
+	    };
 {{- end}}
+        channel {
+        # Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic
+            source (s_DEFAULT);
+            filter(f_is_rfc3164);
+            filter(f_local_example);
+            flags(final);
+        };
+    };
 
 # Set a default sourcetype and index, as well as an appropriate value for the field
 # "sc4s_vendor_product".  This field is sent as an indexed field to Splunk,
@@ -86,16 +85,4 @@ log {
 
 # All passes through any matching log path will be final
     flags(flow-control,final);
-};
-{{- end}}
-
-{{- /* Prepare to run two passes through this template, one for default traffic and another for */}}
-{{- /* "unique ports" if they are configured. */}}
-
-{{- if or (or (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT")) (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT")) }}
-# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic
-{{ tmpl.Exec "log_path" "no" }}
-{{- end }}
-
-# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic
-{{ tmpl.Exec "log_path" "yes" }}
+};
diff --git a/...ths/p_rfc3164-checkpoint_splunk.conf.tmpl → .../log_paths/lp-checkpoint_splunk.conf.tmpl b/...ths/p_rfc3164-checkpoint_splunk.conf.tmpl → .../log_paths/lp-checkpoint_splunk.conf.tmpl
@@ -3,17 +3,23 @@
 {{- $context := dict "port_id" "CHECKPOINT_SPLUNK" "parser" "common" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
-{{- /* The following is an inline template to generate the actual log path */}}
-{{- define "log_path"}}
 log {
-{{- if eq (.) "yes"}}
-    source(s_DEFAULT);
-    filter(f_is_rfc3164);
-    filter(f_checkpoint_splunk);
-{{- end}}
-{{- if eq (.) "no"}}
-    source (s_CHECKPOINT_SPLUNK);
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for CHECKPOINT_SPLUNK traffic
+            source (s_CHECKPOINT_SPLUNK);
+            flags (final);
+	    };
 {{- end}}
+        channel {
+        # Listen on the default port (typically 514) for CHECKPOINT_SPLUNK traffic
+            source(s_DEFAULT);
+            filter(f_is_rfc3164);
+            filter(f_checkpoint_splunk);
+            flags(final);
+        };
+    };
 
     if {
         filter(f_nix_syslog);
@@ -83,12 +89,4 @@ log {
 {{- end}}
 
     flags(flow-control,final);
-};
-{{- end}}
-{{- if or (or (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CHECKPOINT_SPLUNK_TLS_PORT")) }}
-# Listen on the specified dedicated port(s) for CHECKPOINT_SPLUNK traffic
-{{ tmpl.Exec "log_path" "no" }}
-{{- end}}
-
-# Listen on the default port (typically 514) for CHECKPOINT_SPLUNK traffic
-{{ tmpl.Exec "log_path" "yes" }}
+};
diff --git a/...d/log_paths/p_rfc3164-cisco_acs.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_acs.conf.tmpl b/...d/log_paths/p_rfc3164-cisco_acs.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_acs.conf.tmpl
@@ -46,17 +46,24 @@ parser acs_event_time {
             template("${ACS.DATE} ${ACS.TIME} ${ACS.TZ}")
     );
 };
-{{- /* The following is an inline template to generate the actual log path */}}
-{{ define "log_path" }}
+
 log {
-{{- if eq (.) "yes"}}
-    source(s_DEFAULT);
-    filter(f_is_rfc3164);
-    filter(f_cisco_acs);
-{{- end}}
-{{- if eq (.) "no"}}
-    source (s_CISCO_ACS);
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_NX_OS_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_NX_OS_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_NX_OS_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for CISCO_NX_OS traffic
+            source (s_CISCO_NX_OS);
+            flags (final);
+	    };
 {{- end}}
+        channel {
+        # Listen on the default port (typically 514) for CISCO_NX_OS traffic
+            source (s_DEFAULT);
+            filter(f_is_rfc3164);
+            filter(f_cisco_acs);
+            flags(final);
+        };
+    };
 
     parser(acs_grouping);
 
@@ -83,15 +90,4 @@ log {
         flags(flow-control,final);
     };
 
-
-};
-{{- end}}
-
-{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ACS_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ACS_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ACS_TLS_PORT")) }}
-
-# Listen on the specified dedicated port(s) for CISCO_ACS traffic
-    {{tmpl.Exec "log_path" "no" }}
-{{- end}}
-
-# Listen on the default port (typically 514) for CISCO_ACS traffic
-{{tmpl.Exec "log_path" "yes" }}
+};
diff --git a/...s/p_rfc5424-noversion_cisco_asa.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_asa.conf.tmpl b/...s/p_rfc5424-noversion_cisco_asa.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_asa.conf.tmpl
@@ -3,17 +3,23 @@
 {{- $context := dict "port_id" "CISCO_ASA" "parser" "common" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
-{{- /* The following is an inline template to generate the actual log path */}}
-{{- define "log_path"}}
 log {
-{{- if eq (.) "yes"}}
-    source(s_DEFAULT);
-    filter(f_is_rfc5424_noversion);
-    filter(f_cisco_asa);
-{{- end}}
-{{- if eq (.) "no"}}
-    source (s_CISCO_ASA);
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for CISCO_ASA traffic
+            source (s_CISCO_ASA);
+            flags (final);
+	    };
 {{- end}}
+        channel {
+        # Listen on the default port (typically 514) for CISCO_ASA traffic
+            source (s_DEFAULT);
+            filter(f_is_rfc5424_noversion);
+            filter(f_cisco_asa);
+            flags(final);
+        };
+    };
 
     rewrite {
         set("cisco_asa", value("fields.sc4s_vendor_product"));
@@ -33,12 +39,3 @@ log {
 
     flags(flow-control,final);
 };
-{{- end}}
-
-{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ASA_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ASA_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ASA_TLS_PORT")) }}
-# Listen on the specified dedicated port(s) for CISCO_ASA traffic
-{{ tmpl.Exec "log_path" "no" }}
-{{- end}}
-
-# Listen on the default port (typically 514) for CISCO_ASA traffic
-{{ tmpl.Exec "log_path" "yes" }}
diff --git a/...d/log_paths/p_rfc3164-cisco_asa.conf.tmpl → ...d/log_paths/lp-cisco_asa_legacy.conf.tmpl b/...d/log_paths/p_rfc3164-cisco_asa.conf.tmpl → ...d/log_paths/lp-cisco_asa_legacy.conf.tmpl
@@ -3,17 +3,23 @@
 {{- $context := dict "port_id" "CISCO_ASA_LEGACY" "parser" "common" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
-{{- /* The following is an inline template to generate the actual log path */}}
-{{- define "log_path"}}
 log {
-{{- if eq (.) "yes"}}
-    source(s_DEFAULT);
-    filter(f_is_rfc3164);
-    filter(f_cisco_asa);
-{{- end}}
-{{- if eq (.) "no"}}
-    source (s_CISCO_ASA_LEGACY);
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for CISCO_ASA_LEGACY traffic
+            source (s_CISCO_ASA_LEGACY);
+            flags (final);
+	    };
 {{- end}}
+        channel {
+        # Listen on the default port (typically 514) for CISCO_ASA_LEGACY traffic
+            source (s_DEFAULT);
+            filter(f_is_rfc3164);
+            filter(f_cisco_asa);
+            flags(final);
+        };
+    };
 
     rewrite {
         set("cisco_asa", value("fields.sc4s_vendor_product"));
@@ -31,16 +37,5 @@ log {
     destination(d_archive);
 {{- end}}
 
-
     flags(flow-control,final);
-
 };
-{{- end}}
-{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ASA_LEGACY_TLS_PORT")) }}
-
-# Listen on the specified dedicated port(s) for CISCO_ASA_LEGACY traffic
-    {{tmpl.Exec "log_path" "no" }}
-{{- end}}
-
-# Listen on the default port (typically 514) for CISCO_ASA_LEGACY traffic
-{{tmpl.Exec "log_path" "yes" }}
diff --git a/...d/log_paths/p_rfc3164-cisco_ios.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_ios.conf.tmpl b/...d/log_paths/p_rfc3164-cisco_ios.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_ios.conf.tmpl
@@ -3,16 +3,22 @@
 {{- $context := dict "port_id" "CISCO_IOS" "parser" "common" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
-{{- /* The following is an inline template to generate the actual log path */}}
-{{- define "log_path"}}
 log {
-{{- if eq (.) "yes" }}
-    source(s_DEFAULT);
-    filter(f_is_cisco_ios);
-{{- end }}
-{{- if eq (.) "no" }}
-    source (s_CISCO_IOS);
-{{- end }}
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for CISCO_IOS traffic
+            source (s_CISCO_IOS);
+            flags (final);
+	    };
+{{- end}}
+        channel {
+        # Listen on the default port (typically 514) for CISCO_IOS traffic
+            source (s_DEFAULT);
+            filter(f_is_cisco_ios);
+            flags(final);
+        };
+    };
 
     rewrite {
         set("cisco_ios", value("fields.sc4s_vendor_product"));
@@ -33,12 +39,3 @@ log {
 
     flags(flow-control,final);
 };
-{{- end}}
-
-{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_IOS_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_IOS_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_IOS_TLS_PORT")) }}
-# Listen on the specified dedicated port(s) for CISCO_IOS traffic
-{{ tmpl.Exec "log_path" "no" }}
-{{- end}}
-
-# Listen on the default port (typically 514) for CISCO_IOS traffic
-{{ tmpl.Exec "log_path" "yes" }}
diff --git a/...d/log_paths/p_rfc3164-cisco_ise.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_ise.conf.tmpl b/...d/log_paths/p_rfc3164-cisco_ise.conf.tmpl → ...c/conf.d/log_paths/lp-cisco_ise.conf.tmpl
@@ -1,6 +1,6 @@
 # Cisco ISE
 {{- /* The following provides a unique port source configuration if env var(s) are set */}}
-{{- $context := dict "port_id" "CISCO_NX_OS" "parser" "common" }}
+{{- $context := dict "port_id" "CISCO_ISE" "parser" "common" }}
 {{- tmpl.Exec "t/source_network.t" $context }}
 
 # This filter uses a field we set to prevent the original messages before aggregation from being
@@ -48,17 +48,24 @@ parser ise_event_time {
             template("${ISE.DATE} ${ISE.TIME} ${ISE.TZ}")
     );
 };
-{{- /* The following is an inline template to generate the actual log path */}}
-{{ define "log_path" }}
+
 log {
-{{- if eq (.) "yes"}}
-    source(s_DEFAULT);
-    filter(f_is_rfc3164);
-    filter(f_cisco_ise);
-{{- end}}
-{{- if eq (.) "no"}}
-    source (s_CISCO_ISE);
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ISE_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ISE_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ISE_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for CISCO_ISE traffic
+            source (s_CISCO_ISE);
+            flags (final);
+	    };
 {{- end}}
+        channel {
+        # Listen on the default port (typically 514) for CISCO_ISE traffic
+            source (s_DEFAULT);
+            filter(f_is_rfc3164);
+            filter(f_cisco_ise);
+            flags(final);
+        };
+    };
 
     parser(ise_grouping);
 
@@ -86,13 +93,3 @@ log {
     };
 
 };
-{{- end}}
-
-{{- if or (or (getenv  (print "SC4S_LISTEN_CISCO_ISE_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CISCO_ISE_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CISCO_ISE_TLS_PORT")) }}
-
-# Listen on the specified dedicated port(s) for CISCO_ISE traffic
-    {{tmpl.Exec "log_path" "no" }}
-{{- end}}
-
-# Listen on the default port (typically 514) for CISCO_ISE traffic
-{{tmpl.Exec "log_path" "yes" }}