Merge pull request #118 from splunk/update/mount_point

update to single mount point

.env.template

@@ -16,6 +16,6 @@ SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event
 SPLUNK_CONNECT_METHOD=hec
 SPLUNK_DEFAULT_INDEX=main
 SPLUNK_METRICS_INDEX=em_metrics
-SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download
-SPLUNKBASE_USERNAME=username
-SPLUNKBASE_PASSWORD=password
+#SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download
+#SPLUNKBASE_USERNAME=username
+#SPLUNKBASE_PASSWORD=password

.gitignore

@@ -382,4 +382,5 @@ fabric.properties
 /.idea/
 
 tests/test_plugin_*.py
-package/etc/conf.d/local/
+# package/etc/conf.d/local/
+!package/etc/conf.d/local

package/Dockerfile

@@ -86,12 +86,13 @@ RUN curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gompl
 
 COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf
 COPY etc/conf.d /opt/syslog-ng/etc/conf.d
-COPY etc/templates /opt/syslog-ng/etc/templates
-COPY etc/context-local /opt/syslog-ng/etc/context-local
+COPY etc/go_templates /opt/syslog-ng/etc/go_templates
+COPY etc/context_templates /opt/syslog-ng/etc/context_templates
+COPY etc/local_config /opt/syslog-ng/etc/local_config
+
 COPY sbin/entrypoint.sh /
 RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer
 RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V
-#RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t
 
 EXPOSE 514
 EXPOSE 601/tcp
@@ -101,4 +102,4 @@ ENV SPLUNK_CONNECT_METHOD=UF
 
 ENTRYPOINT ["/entrypoint.sh", "-F"]
 
-HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate
+HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate

package/etc/conf.d/conflib/_common/compliance_meta.conf

@@ -1,7 +1,7 @@
 parser compliance_meta_by_source {
     add-contextual-data(
-        selector(filters("`syslog-ng-sysconfdir`/context-local/compliance_meta_by_source.conf")),
-        database("context-local/compliance_meta_by_source.csv")
+        selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/compliance_meta_by_source.conf")),
+        database("conf.d/local/context/compliance_meta_by_source.csv")
         ignore-case(yes)
     );
 };

package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf

@@ -1,7 +1,7 @@
 block parser vendor_product_by_source() {
     add-contextual-data(
-        selector(filters("`syslog-ng-sysconfdir`/context-local/vendor_product_by_source.conf")),
-        database("context-local/vendor_product_by_source.csv")
+        selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.conf")),
+        database("conf.d/local/context/vendor_product_by_source.csv")
         ignore-case(yes)
         prefix("fields.")
     );

package/etc/conf.d/conflib/_splunk/splunk_context.conf

@@ -1,7 +1,7 @@
 block parser p_add_context_splunk(key("syslogng-fallback")) {
     add-contextual-data(
         selector("`key`"),
-        database("context-local/splunk_index.csv"),
+        database("conf.d/local/context/splunk_index.csv"),
         prefix(".splunk.")
     );
-};
+};

package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl

@@ -22,7 +22,7 @@ parser p_microfocus_arcsight_ts_end {
 parser p_microfocus_arcsight_source {
     add-contextual-data(
         selector("${fields.cef_device_vendor}_${fields.cef_device_product}"),
-        database("context-local/microfocus_arcsight_source.csv")
+        database("conf.d/context/microfocus_arcsight_source.csv")
         ignore-case(yes)
         prefix(".splunk.")
         default-selector("unknown")

package/etc/context_templates/compliance_meta_by_source.conf

@@ -0,0 +1,5 @@
+@version: 3.23
+filter f_test_test {
+    host("something-*" type(glob)) or
+    netmask(192.168.100.1/24)
+};

package/etc/context_templates/compliance_meta_by_source.csv

@@ -0,0 +1,2 @@
+#f_test_test,.splunk.index,"badindex"
+#f_test_test,fields.compliance,"pci"

package/etc/context_templates/splunk_index.csv

@@ -0,0 +1,40 @@
+#bluecoat_proxy,index,netproxy
+#cef_ArcSight_ArcSight,index,netwaf
+#cef_Incapsula_SIEMintegration,index,netwaf
+#cef_Microsoft_Microsoft Windows,index,oswinsec
+#cef_Microsoft_System or Application Event,index,oswin
+#cisco_asa,index,netfw
+#cisco_ios,index,netops
+#cisco_nx_os,index,netops
+#local_example,index,main
+#fortinet_fortios_event,index,netops
+#fortinet_fortios_log,index,netops
+#fortinet_fortios_traffic,index,netfw
+#fortinet_fortios_utm,index,netids
+#juniper_idp,index,netids
+#juniper_structured,index,netops
+#juniper_idp_structured,index,netids
+#juniper_junos_fw_structured,index,netfw
+#juniper_junos_ids_structured,index,netids
+#juniper_junos_utm_structured,index,netfw
+#juniper_junos_fw,index,netfw
+#juniper_junos_ids,index,netids
+#juniper_junos_utm,index,netfw
+#juniper_sslvpn,index,netfw
+#juniper_netscreen,index,netfw
+#juniper_nsm,index,netfw
+#juniper_nsm_idp,index,netids
+#juniper_legacy,index,netops
+#pan_traffic,index,netfw
+#pan_threat,index,netproxy
+#pan_system,index,netops
+#pan_config,index,netops
+#pan_hipwatch,index,main
+#pan_correlation,index,main
+#pan_userid,index,netauth
+#pan_unknown,index,netops
+#proofpoint_pps_filter,index,email
+#proofpoint_pps_sendmail,index,email
+#sc4s_events,index,main
+#sc4s_fallback,index,main
+#sc4s_metrics,index,em_metrics

package/etc/context_templates/vendor_product_by_source.conf

@@ -0,0 +1,34 @@
+@version: 3.23
+
+filter f_test_test {
+    host("testvp-*" type(glob)) or
+    netmask(192.168.100.1/24)
+};
+filter f_juniper_nsm {
+    host("jnpnsm-*" type(glob)) or
+    netmask(192.168.1.0/24)
+};
+filter f_juniper_nsm_idp {
+    host("jnpnsmidp-*" type(glob)) or
+    netmask(192.168.2.0/24)
+};
+filter f_juniper_idp {
+    host("jnpidp-*" type(glob)) or
+    netmask(192.168.3.0/24)
+};
+filter f_juniper_netscreen {
+    host("jnpns-*" type(glob)) or
+    netmask(192.168.4.0/24)
+};
+filter f_cisco_nx_os {
+    host("csconx-*" type(glob)) or
+    netmask(192.168.5.0/24)
+};    
+filter f_proofpoint_pps_sendmail {
+    host("pps-*" type(glob)) or
+    netmask(192.168.6.0/24)
+};
+filter f_proofpoint_pps_filter {
+    host("pps-*" type(glob)) or
+    netmask(192.168.7.0/24)
+};

package/etc/context_templates/vendor_product_by_source.csv

@@ -0,0 +1,8 @@
+f_test_test,sc4s_vendor_product,"test_test"
+f_juniper_nsm,sc4s_vendor_product,"juniper_nsm"
+f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp"
+f_juniper_idp,sc4s_vendor_product,"juniper_idp"
+f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen"
+f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os"
+f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail"
+f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter"

package/etc/local_config/destinations/README.md

@@ -0,0 +1 @@
+This file exists to preserve the path for plugin use

package/etc/local_config/filters/README.md

@@ -0,0 +1 @@
+This file exists to preserve the path for plugin use

package/etc/local_config/filters/example.conf

@@ -0,0 +1,4 @@
+
+filter f_local_example {
+    program(sc4splugin);
+};

package/etc/local_config/log_paths/README.md

@@ -0,0 +1 @@
+This file exists to preserve the path for plugin use

package/etc/local_config/log_paths/example.conf.tmpl

@@ -0,0 +1,76 @@
+# LOCAL_EXAMPLE
+
+# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique 
+# string to identify the vendor product.  The string should be of the form "VENDOR_PRODUCT" to signify the
+# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+
+
+# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS),
+# the template generator will build a custom source based on the value of one or more of the set variables.
+
+{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }}
+
+# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name
+# set in the line above.  For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE".
+# "parser" can be customized on dedicated ports only
+# "common" uses the same parser sequence as the default ports and is the most commonly used
+
+{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}}
+
+# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path
+# The ports used are based on the values of one or more of the environment variables set above.
+
+{{ tmpl.Exec "t/source_network.t" $context }}
+{{- end -}}
+{{ define "log_path" }}
+log {
+
+# The first time this template is used the log_path will be linked to the default port
+
+{{- if eq (.) "yes"}}
+    source(s_default-ports);
+
+# Filters should be updated to use the simplest and most effecient logic possible to discard
+# the message from this path
+
+    filter(f_is_rfc3164);
+    filter(f_local_example);
+{{- end}}
+{{- if eq (.) "no"}}
+
+# In the second pass through the template a link to the dedicated port is used. This
+# normally does not require additional filters
+
+source (s_dedicated_port_LOCAL_EXAMPLE);
+{{- end}}
+
+#Set a default sourcetype and index
+
+    rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))};
+
+#using the key "local_example" find any cutomized index,source or sourcetype meta values
+
+    parser {p_add_context_splunk(key("local_example")); };
+
+# Any additional logic needed to process the event before sending to Splunk goes here
+
+# Send it to Splunk
+
+    destination(d_hec);  #--HEC--
+
+# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will
+# forward events to another system
+
+    flags(flow-control);
+
+};
+{{- end}}
+{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }}
+
+# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic
+
+    {{tmpl.Exec "log_path" "no" }}
+{{- end}}
+
+# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic
+
+{{tmpl.Exec "log_path" "yes" }}

package/etc/local_config/sources/README.md

@@ -0,0 +1 @@
+This file exists to preserve the path for plugin use

package/etc/syslog-ng.conf

@@ -62,8 +62,7 @@ options {
 @include "conf.d/destinations/*.conf"
 @include "conf.d/log_paths/*.conf"
 
-@include "conf.d/local/filters/*.conf"
-@include "conf.d/local/filters/*/*.conf"
-@include "conf.d/local/sources/*.conf"
-@include "conf.d/local/destinations/*.conf"
-@include "conf.d/local/log_paths/*.conf"
+@include "conf.d/local/config/filters/*.conf"
+@include "conf.d/local/config/sources/*.conf"
+@include "conf.d/local/config/destinations/*.conf"
+@include "conf.d/local/config/log_paths/*.conf"

package/sbin/entrypoint.sh

@@ -7,10 +7,15 @@ do
   echo Templating conf for $d
   gomplate \
     --input-dir=$d \
-    --template t=etc/templates/  \
+    --template t=etc/go_templates/  \
     --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\
     --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}"
 done
 
-echo syslog-ng started
+mkdir -p /opt/syslog-ng/etc/conf.d/local/context/
+mkdir -p /opt/syslog-ng/etc/conf.d/local/config/
+cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/
+cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/
+
+echo syslog-ng starting
 exec /opt/syslog-ng/sbin/syslog-ng $@