From 928350baa111f517b06d3b04442b0593018a15a7 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sat, 5 Oct 2019 22:35:27 -0700 Subject: [PATCH] update to single mount point --- package/Dockerfile | 4 +- .../conflib/_common/compliance_meta.conf | 4 +- .../vendor_product_by_source_context.conf | 4 +- .../conflib/_splunk/splunk_context.conf | 4 +- .../context}/microfocus_arcsight_source.csv | 0 .../etc/conf.d/local/destinations/README.md | 1 - package/etc/conf.d/local/filters/README.md | 1 - package/etc/conf.d/local/filters/example.conf | 4 - package/etc/conf.d/local/log_paths/README.md | 1 - .../conf.d/local/log_paths/example.conf.tmpl | 76 ------------------- package/etc/conf.d/local/sources/README.md | 1 - .../p_rfc3164_microfocus_arcsight.conf.tmpl | 2 +- .../compliance_meta_by_source.conf | 5 -- .../compliance_meta_by_source.csv | 2 - package/etc/context-local/splunk_index.csv | 40 ---------- .../vendor_product_by_source.conf | 37 --------- .../vendor_product_by_source.csv | 8 -- .../source_network.t | 0 package/etc/syslog-ng.conf | 9 +-- package/sbin/entrypoint.sh | 2 +- 20 files changed, 14 insertions(+), 191 deletions(-) rename package/etc/{context-local => conf.d/context}/microfocus_arcsight_source.csv (100%) delete mode 100644 package/etc/conf.d/local/destinations/README.md delete mode 100644 package/etc/conf.d/local/filters/README.md delete mode 100644 package/etc/conf.d/local/filters/example.conf delete mode 100644 package/etc/conf.d/local/log_paths/README.md delete mode 100644 package/etc/conf.d/local/log_paths/example.conf.tmpl delete mode 100644 package/etc/conf.d/local/sources/README.md delete mode 100644 package/etc/context-local/compliance_meta_by_source.conf delete mode 100644 package/etc/context-local/compliance_meta_by_source.csv delete mode 100644 package/etc/context-local/splunk_index.csv delete mode 100644 package/etc/context-local/vendor_product_by_source.conf delete mode 100644 package/etc/context-local/vendor_product_by_source.csv rename package/etc/{templates => go_templates}/source_network.t (100%) diff --git a/package/Dockerfile b/package/Dockerfile index a5f67cc..570a16a 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -86,8 +86,8 @@ RUN curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gompl COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d -COPY etc/templates /opt/syslog-ng/etc/templates -COPY etc/context-local /opt/syslog-ng/etc/context-local +COPY etc/go_templates /opt/syslog-ng/etc/go_templates +COPY etc/conf.d/local/context /opt/syslog-ng/etc/conf.d/local/context COPY sbin/entrypoint.sh / RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V diff --git a/package/etc/conf.d/conflib/_common/compliance_meta.conf b/package/etc/conf.d/conflib/_common/compliance_meta.conf index 3529ae5..75479c6 100644 --- a/package/etc/conf.d/conflib/_common/compliance_meta.conf +++ b/package/etc/conf.d/conflib/_common/compliance_meta.conf @@ -1,7 +1,7 @@ parser compliance_meta_by_source { add-contextual-data( - selector(filters("`syslog-ng-sysconfdir`/context-local/compliance_meta_by_source.conf")), - database("context-local/compliance_meta_by_source.csv") + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/compliance_meta_by_source.conf")), + database("conf.d/local/context/compliance_meta_by_source.csv") ignore-case(yes) ); }; diff --git a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf index f3789a7..44d8ff1 100644 --- a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf +++ b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf @@ -1,7 +1,7 @@ block parser vendor_product_by_source() { add-contextual-data( - selector(filters("`syslog-ng-sysconfdir`/context-local/vendor_product_by_source.conf")), - database("context-local/vendor_product_by_source.csv") + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.conf")), + database("conf.d/local/context/vendor_product_by_source.csv") ignore-case(yes) prefix("fields.") ); diff --git a/package/etc/conf.d/conflib/_splunk/splunk_context.conf b/package/etc/conf.d/conflib/_splunk/splunk_context.conf index e2d1fbb..6fb181d 100644 --- a/package/etc/conf.d/conflib/_splunk/splunk_context.conf +++ b/package/etc/conf.d/conflib/_splunk/splunk_context.conf @@ -1,7 +1,7 @@ block parser p_add_context_splunk(key("syslogng-fallback")) { add-contextual-data( selector("`key`"), - database("context-local/splunk_index.csv"), + database("conf.d/local/context/splunk_index.csv"), prefix(".splunk.") ); -}; \ No newline at end of file +}; diff --git a/package/etc/context-local/microfocus_arcsight_source.csv b/package/etc/conf.d/context/microfocus_arcsight_source.csv similarity index 100% rename from package/etc/context-local/microfocus_arcsight_source.csv rename to package/etc/conf.d/context/microfocus_arcsight_source.csv diff --git a/package/etc/conf.d/local/destinations/README.md b/package/etc/conf.d/local/destinations/README.md deleted file mode 100644 index ee6571d..0000000 --- a/package/etc/conf.d/local/destinations/README.md +++ /dev/null @@ -1 +0,0 @@ -This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/filters/README.md b/package/etc/conf.d/local/filters/README.md deleted file mode 100644 index ee6571d..0000000 --- a/package/etc/conf.d/local/filters/README.md +++ /dev/null @@ -1 +0,0 @@ -This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/filters/example.conf b/package/etc/conf.d/local/filters/example.conf deleted file mode 100644 index 047fdc8..0000000 --- a/package/etc/conf.d/local/filters/example.conf +++ /dev/null @@ -1,4 +0,0 @@ - -filter f_local_example { - program(sc4splugin); -}; \ No newline at end of file diff --git a/package/etc/conf.d/local/log_paths/README.md b/package/etc/conf.d/local/log_paths/README.md deleted file mode 100644 index ee6571d..0000000 --- a/package/etc/conf.d/local/log_paths/README.md +++ /dev/null @@ -1 +0,0 @@ -This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/log_paths/example.conf.tmpl b/package/etc/conf.d/local/log_paths/example.conf.tmpl deleted file mode 100644 index a8ac264..0000000 --- a/package/etc/conf.d/local/log_paths/example.conf.tmpl +++ /dev/null @@ -1,76 +0,0 @@ -# LOCAL_EXAMPLE - -# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique -# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the -# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ - -# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), -# the template generator will build a custom source based on the value of one or more of the set variables. - -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} - -# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name -# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". -# "parser" can be customized on dedicated ports only -# "common" uses the same parser sequence as the default ports and is the most commonly used - -{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} - -# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path -# The ports used are based on the values of one or more of the environment variables set above. - -{{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} -{{ define "log_path" }} -log { - -# The first time this template is used the log_path will be linked to the default port - -{{- if eq (.) "yes"}} - source(s_default-ports); - -# Filters should be updated to use the simplest and most effecient logic possible to discard -# the message from this path - - filter(f_is_rfc3164); - filter(f_local_example); -{{- end}} -{{- if eq (.) "no"}} - -# In the second pass through the template a link to the dedicated port is used. This -# normally does not require additional filters - -source (s_dedicated_port_LOCAL_EXAMPLE); -{{- end}} - -#Set a default sourcetype and index - - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; - -#using the key "local_example" find any cutomized index,source or sourcetype meta values - - parser {p_add_context_splunk(key("local_example")); }; - -# Any additional logic needed to process the event before sending to Splunk goes here - -# Send it to Splunk - - destination(d_hec); #--HEC-- - -# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will -# forward events to another system - - flags(flow-control); - -}; -{{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} - -# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic - - {{tmpl.Exec "log_path" "no" }} -{{- end}} - -# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic - -{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/local/sources/README.md b/package/etc/conf.d/local/sources/README.md deleted file mode 100644 index ee6571d..0000000 --- a/package/etc/conf.d/local/sources/README.md +++ /dev/null @@ -1 +0,0 @@ -This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl index 6170b56..4c89731 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl @@ -22,7 +22,7 @@ parser p_microfocus_arcsight_ts_end { parser p_microfocus_arcsight_source { add-contextual-data( selector("${fields.cef_device_vendor}_${fields.cef_device_product}"), - database("context-local/microfocus_arcsight_source.csv") + database("conf.d/context/microfocus_arcsight_source.csv") ignore-case(yes) prefix(".splunk.") default-selector("unknown") diff --git a/package/etc/context-local/compliance_meta_by_source.conf b/package/etc/context-local/compliance_meta_by_source.conf deleted file mode 100644 index 1d5acae..0000000 --- a/package/etc/context-local/compliance_meta_by_source.conf +++ /dev/null @@ -1,5 +0,0 @@ -@version: 3.23 -filter f_test_test { - host("something-*" type(glob)) or - netmask(192.168.100.1/24) -}; diff --git a/package/etc/context-local/compliance_meta_by_source.csv b/package/etc/context-local/compliance_meta_by_source.csv deleted file mode 100644 index 6608db0..0000000 --- a/package/etc/context-local/compliance_meta_by_source.csv +++ /dev/null @@ -1,2 +0,0 @@ -#f_test_test,.splunk.index,"badindex" -#f_test_test,fields.compliance,"pci" diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv deleted file mode 100644 index e93911a..0000000 --- a/package/etc/context-local/splunk_index.csv +++ /dev/null @@ -1,40 +0,0 @@ -#bluecoat_proxy,index,netproxy -#cef_ArcSight_ArcSight,index,netwaf -#cef_Incapsula_SIEMintegration,index,netwaf -#cef_Microsoft_Microsoft Windows,index,oswinsec -#cef_Microsoft_System or Application Event,index,oswin -#cisco_asa,index,netfw -#cisco_ios,index,netops -#cisco_nx_os,index,netops -#local_example,index,main -#fortinet_fortios_event,index,netops -#fortinet_fortios_log,index,netops -#fortinet_fortios_traffic,index,netfw -#fortinet_fortios_utm,index,netids -#juniper_idp,index,netids -#juniper_structured,index,netops -#juniper_idp_structured,index,netids -#juniper_junos_fw_structured,index,netfw -#juniper_junos_ids_structured,index,netids -#juniper_junos_utm_structured,index,netfw -#juniper_junos_fw,index,netfw -#juniper_junos_ids,index,netids -#juniper_junos_utm,index,netfw -#juniper_sslvpn,index,netfw -#juniper_netscreen,index,netfw -#juniper_nsm,index,netfw -#juniper_nsm_idp,index,netids -#juniper_legacy,index,netops -#pan_traffic,index,netfw -#pan_threat,index,netproxy -#pan_system,index,netops -#pan_config,index,netops -#pan_hipwatch,index,main -#pan_correlation,index,main -#pan_userid,index,netauth -#pan_unknown,index,netops -#proofpoint_pps_filter,index,email -#proofpoint_pps_sendmail,index,email -#sc4s_events,index,main -#sc4s_fallback,index,main -#sc4s_metrics,index,em_metrics diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/context-local/vendor_product_by_source.conf deleted file mode 100644 index 37e3412..0000000 --- a/package/etc/context-local/vendor_product_by_source.conf +++ /dev/null @@ -1,37 +0,0 @@ -@version: 3.22 -#TODO: #60 The syntax below uses regex and an indirect reference to a variable due to a -#bug/limitation of selector files. The better syntax should be as follows -#filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; - -filter f_test_test { - host("testvp-*" type(glob)) or - netmask(192.168.100.1/24) -}; -filter f_juniper_nsm { - host("jnpnsm-*" type(glob)) or - netmask(192.168.1.0/24) -}; -filter f_juniper_nsm_idp { - host("jnpnsmidp-*" type(glob)) or - netmask(192.168.2.0/24) -}; -filter f_juniper_idp { - host("jnpidp-*" type(glob)) or - netmask(192.168.3.0/24) -}; -filter f_juniper_netscreen { - host("jnpns-*" type(glob)) or - netmask(192.168.4.0/24) -}; -filter f_cisco_nx_os { - host("csconx-*" type(glob)) or - netmask(192.168.5.0/24) -}; -filter f_proofpoint_pps_sendmail { - host("pps-*" type(glob)) or - netmask(192.168.6.0/24) -}; -filter f_proofpoint_pps_filter { - host("pps-*" type(glob)) or - netmask(192.168.7.0/24) -}; \ No newline at end of file diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/context-local/vendor_product_by_source.csv deleted file mode 100644 index 3f90603..0000000 --- a/package/etc/context-local/vendor_product_by_source.csv +++ /dev/null @@ -1,8 +0,0 @@ -f_test_test,sc4s_vendor_product,"test_test" -f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" -f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" -f_juniper_idp,sc4s_vendor_product,"juniper_idp" -f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" -f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" -f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" -f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/package/etc/templates/source_network.t b/package/etc/go_templates/source_network.t similarity index 100% rename from package/etc/templates/source_network.t rename to package/etc/go_templates/source_network.t diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index e4a0a78..d0422fb 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -62,8 +62,7 @@ options { @include "conf.d/destinations/*.conf" @include "conf.d/log_paths/*.conf" -@include "conf.d/local/filters/*.conf" -@include "conf.d/local/filters/*/*.conf" -@include "conf.d/local/sources/*.conf" -@include "conf.d/local/destinations/*.conf" -@include "conf.d/local/log_paths/*.conf" +@include "conf.d/local/config/filters/*.conf" +@include "conf.d/local/config/sources/*.conf" +@include "conf.d/local/config/destinations/*.conf" +@include "conf.d/local/config/log_paths/*.conf" diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index d100fc0..d93862b 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -7,7 +7,7 @@ do echo Templating conf for $d gomplate \ --input-dir=$d \ - --template t=etc/templates/ \ + --template t=etc/go_templates/ \ --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" done