From 928350baa111f517b06d3b04442b0593018a15a7 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sat, 5 Oct 2019 22:35:27 -0700 Subject: [PATCH 01/12] update to single mount point --- package/Dockerfile | 4 +- .../conflib/_common/compliance_meta.conf | 4 +- .../vendor_product_by_source_context.conf | 4 +- .../conflib/_splunk/splunk_context.conf | 4 +- .../context}/microfocus_arcsight_source.csv | 0 .../etc/conf.d/local/destinations/README.md | 1 - package/etc/conf.d/local/filters/README.md | 1 - package/etc/conf.d/local/filters/example.conf | 4 - package/etc/conf.d/local/log_paths/README.md | 1 - .../conf.d/local/log_paths/example.conf.tmpl | 76 ------------------- package/etc/conf.d/local/sources/README.md | 1 - .../p_rfc3164_microfocus_arcsight.conf.tmpl | 2 +- .../compliance_meta_by_source.conf | 5 -- .../compliance_meta_by_source.csv | 2 - package/etc/context-local/splunk_index.csv | 40 ---------- .../vendor_product_by_source.conf | 37 --------- .../vendor_product_by_source.csv | 8 -- .../source_network.t | 0 package/etc/syslog-ng.conf | 9 +-- package/sbin/entrypoint.sh | 2 +- 20 files changed, 14 insertions(+), 191 deletions(-) rename package/etc/{context-local => conf.d/context}/microfocus_arcsight_source.csv (100%) delete mode 100644 package/etc/conf.d/local/destinations/README.md delete mode 100644 package/etc/conf.d/local/filters/README.md delete mode 100644 package/etc/conf.d/local/filters/example.conf delete mode 100644 package/etc/conf.d/local/log_paths/README.md delete mode 100644 package/etc/conf.d/local/log_paths/example.conf.tmpl delete mode 100644 package/etc/conf.d/local/sources/README.md delete mode 100644 package/etc/context-local/compliance_meta_by_source.conf delete mode 100644 package/etc/context-local/compliance_meta_by_source.csv delete mode 100644 package/etc/context-local/splunk_index.csv delete mode 100644 package/etc/context-local/vendor_product_by_source.conf delete mode 100644 package/etc/context-local/vendor_product_by_source.csv rename package/etc/{templates => go_templates}/source_network.t (100%) diff --git a/package/Dockerfile b/package/Dockerfile index a5f67cc..570a16a 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -86,8 +86,8 @@ RUN curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gompl COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d -COPY etc/templates /opt/syslog-ng/etc/templates -COPY etc/context-local /opt/syslog-ng/etc/context-local +COPY etc/go_templates /opt/syslog-ng/etc/go_templates +COPY etc/conf.d/local/context /opt/syslog-ng/etc/conf.d/local/context COPY sbin/entrypoint.sh / RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V diff --git a/package/etc/conf.d/conflib/_common/compliance_meta.conf b/package/etc/conf.d/conflib/_common/compliance_meta.conf index 3529ae5..75479c6 100644 --- a/package/etc/conf.d/conflib/_common/compliance_meta.conf +++ b/package/etc/conf.d/conflib/_common/compliance_meta.conf @@ -1,7 +1,7 @@ parser compliance_meta_by_source { add-contextual-data( - selector(filters("`syslog-ng-sysconfdir`/context-local/compliance_meta_by_source.conf")), - database("context-local/compliance_meta_by_source.csv") + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/compliance_meta_by_source.conf")), + database("conf.d/local/context/compliance_meta_by_source.csv") ignore-case(yes) ); }; diff --git a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf index f3789a7..44d8ff1 100644 --- a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf +++ b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf @@ -1,7 +1,7 @@ block parser vendor_product_by_source() { add-contextual-data( - selector(filters("`syslog-ng-sysconfdir`/context-local/vendor_product_by_source.conf")), - database("context-local/vendor_product_by_source.csv") + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.conf")), + database("conf.d/local/context/vendor_product_by_source.csv") ignore-case(yes) prefix("fields.") ); diff --git a/package/etc/conf.d/conflib/_splunk/splunk_context.conf b/package/etc/conf.d/conflib/_splunk/splunk_context.conf index e2d1fbb..6fb181d 100644 --- a/package/etc/conf.d/conflib/_splunk/splunk_context.conf +++ b/package/etc/conf.d/conflib/_splunk/splunk_context.conf @@ -1,7 +1,7 @@ block parser p_add_context_splunk(key("syslogng-fallback")) { add-contextual-data( selector("`key`"), - database("context-local/splunk_index.csv"), + database("conf.d/local/context/splunk_index.csv"), prefix(".splunk.") ); -}; \ No newline at end of file +}; diff --git a/package/etc/context-local/microfocus_arcsight_source.csv b/package/etc/conf.d/context/microfocus_arcsight_source.csv similarity index 100% rename from package/etc/context-local/microfocus_arcsight_source.csv rename to package/etc/conf.d/context/microfocus_arcsight_source.csv diff --git a/package/etc/conf.d/local/destinations/README.md b/package/etc/conf.d/local/destinations/README.md deleted file mode 100644 index ee6571d..0000000 --- a/package/etc/conf.d/local/destinations/README.md +++ /dev/null @@ -1 +0,0 @@ -This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/filters/README.md b/package/etc/conf.d/local/filters/README.md deleted file mode 100644 index ee6571d..0000000 --- a/package/etc/conf.d/local/filters/README.md +++ /dev/null @@ -1 +0,0 @@ -This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/filters/example.conf b/package/etc/conf.d/local/filters/example.conf deleted file mode 100644 index 047fdc8..0000000 --- a/package/etc/conf.d/local/filters/example.conf +++ /dev/null @@ -1,4 +0,0 @@ - -filter f_local_example { - program(sc4splugin); -}; \ No newline at end of file diff --git a/package/etc/conf.d/local/log_paths/README.md b/package/etc/conf.d/local/log_paths/README.md deleted file mode 100644 index ee6571d..0000000 --- a/package/etc/conf.d/local/log_paths/README.md +++ /dev/null @@ -1 +0,0 @@ -This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/log_paths/example.conf.tmpl b/package/etc/conf.d/local/log_paths/example.conf.tmpl deleted file mode 100644 index a8ac264..0000000 --- a/package/etc/conf.d/local/log_paths/example.conf.tmpl +++ /dev/null @@ -1,76 +0,0 @@ -# LOCAL_EXAMPLE - -# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique -# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the -# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ - -# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), -# the template generator will build a custom source based on the value of one or more of the set variables. - -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} - -# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name -# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". -# "parser" can be customized on dedicated ports only -# "common" uses the same parser sequence as the default ports and is the most commonly used - -{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} - -# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path -# The ports used are based on the values of one or more of the environment variables set above. - -{{ tmpl.Exec "t/source_network.t" $context }} -{{- end -}} -{{ define "log_path" }} -log { - -# The first time this template is used the log_path will be linked to the default port - -{{- if eq (.) "yes"}} - source(s_default-ports); - -# Filters should be updated to use the simplest and most effecient logic possible to discard -# the message from this path - - filter(f_is_rfc3164); - filter(f_local_example); -{{- end}} -{{- if eq (.) "no"}} - -# In the second pass through the template a link to the dedicated port is used. This -# normally does not require additional filters - -source (s_dedicated_port_LOCAL_EXAMPLE); -{{- end}} - -#Set a default sourcetype and index - - rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; - -#using the key "local_example" find any cutomized index,source or sourcetype meta values - - parser {p_add_context_splunk(key("local_example")); }; - -# Any additional logic needed to process the event before sending to Splunk goes here - -# Send it to Splunk - - destination(d_hec); #--HEC-- - -# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will -# forward events to another system - - flags(flow-control); - -}; -{{- end}} -{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} - -# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic - - {{tmpl.Exec "log_path" "no" }} -{{- end}} - -# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic - -{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/local/sources/README.md b/package/etc/conf.d/local/sources/README.md deleted file mode 100644 index ee6571d..0000000 --- a/package/etc/conf.d/local/sources/README.md +++ /dev/null @@ -1 +0,0 @@ -This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl index 6170b56..4c89731 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl @@ -22,7 +22,7 @@ parser p_microfocus_arcsight_ts_end { parser p_microfocus_arcsight_source { add-contextual-data( selector("${fields.cef_device_vendor}_${fields.cef_device_product}"), - database("context-local/microfocus_arcsight_source.csv") + database("conf.d/context/microfocus_arcsight_source.csv") ignore-case(yes) prefix(".splunk.") default-selector("unknown") diff --git a/package/etc/context-local/compliance_meta_by_source.conf b/package/etc/context-local/compliance_meta_by_source.conf deleted file mode 100644 index 1d5acae..0000000 --- a/package/etc/context-local/compliance_meta_by_source.conf +++ /dev/null @@ -1,5 +0,0 @@ -@version: 3.23 -filter f_test_test { - host("something-*" type(glob)) or - netmask(192.168.100.1/24) -}; diff --git a/package/etc/context-local/compliance_meta_by_source.csv b/package/etc/context-local/compliance_meta_by_source.csv deleted file mode 100644 index 6608db0..0000000 --- a/package/etc/context-local/compliance_meta_by_source.csv +++ /dev/null @@ -1,2 +0,0 @@ -#f_test_test,.splunk.index,"badindex" -#f_test_test,fields.compliance,"pci" diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/context-local/splunk_index.csv deleted file mode 100644 index e93911a..0000000 --- a/package/etc/context-local/splunk_index.csv +++ /dev/null @@ -1,40 +0,0 @@ -#bluecoat_proxy,index,netproxy -#cef_ArcSight_ArcSight,index,netwaf -#cef_Incapsula_SIEMintegration,index,netwaf -#cef_Microsoft_Microsoft Windows,index,oswinsec -#cef_Microsoft_System or Application Event,index,oswin -#cisco_asa,index,netfw -#cisco_ios,index,netops -#cisco_nx_os,index,netops -#local_example,index,main -#fortinet_fortios_event,index,netops -#fortinet_fortios_log,index,netops -#fortinet_fortios_traffic,index,netfw -#fortinet_fortios_utm,index,netids -#juniper_idp,index,netids -#juniper_structured,index,netops -#juniper_idp_structured,index,netids -#juniper_junos_fw_structured,index,netfw -#juniper_junos_ids_structured,index,netids -#juniper_junos_utm_structured,index,netfw -#juniper_junos_fw,index,netfw -#juniper_junos_ids,index,netids -#juniper_junos_utm,index,netfw -#juniper_sslvpn,index,netfw -#juniper_netscreen,index,netfw -#juniper_nsm,index,netfw -#juniper_nsm_idp,index,netids -#juniper_legacy,index,netops -#pan_traffic,index,netfw -#pan_threat,index,netproxy -#pan_system,index,netops -#pan_config,index,netops -#pan_hipwatch,index,main -#pan_correlation,index,main -#pan_userid,index,netauth -#pan_unknown,index,netops -#proofpoint_pps_filter,index,email -#proofpoint_pps_sendmail,index,email -#sc4s_events,index,main -#sc4s_fallback,index,main -#sc4s_metrics,index,em_metrics diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/context-local/vendor_product_by_source.conf deleted file mode 100644 index 37e3412..0000000 --- a/package/etc/context-local/vendor_product_by_source.conf +++ /dev/null @@ -1,37 +0,0 @@ -@version: 3.22 -#TODO: #60 The syntax below uses regex and an indirect reference to a variable due to a -#bug/limitation of selector files. The better syntax should be as follows -#filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; - -filter f_test_test { - host("testvp-*" type(glob)) or - netmask(192.168.100.1/24) -}; -filter f_juniper_nsm { - host("jnpnsm-*" type(glob)) or - netmask(192.168.1.0/24) -}; -filter f_juniper_nsm_idp { - host("jnpnsmidp-*" type(glob)) or - netmask(192.168.2.0/24) -}; -filter f_juniper_idp { - host("jnpidp-*" type(glob)) or - netmask(192.168.3.0/24) -}; -filter f_juniper_netscreen { - host("jnpns-*" type(glob)) or - netmask(192.168.4.0/24) -}; -filter f_cisco_nx_os { - host("csconx-*" type(glob)) or - netmask(192.168.5.0/24) -}; -filter f_proofpoint_pps_sendmail { - host("pps-*" type(glob)) or - netmask(192.168.6.0/24) -}; -filter f_proofpoint_pps_filter { - host("pps-*" type(glob)) or - netmask(192.168.7.0/24) -}; \ No newline at end of file diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/context-local/vendor_product_by_source.csv deleted file mode 100644 index 3f90603..0000000 --- a/package/etc/context-local/vendor_product_by_source.csv +++ /dev/null @@ -1,8 +0,0 @@ -f_test_test,sc4s_vendor_product,"test_test" -f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" -f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" -f_juniper_idp,sc4s_vendor_product,"juniper_idp" -f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" -f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" -f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" -f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/package/etc/templates/source_network.t b/package/etc/go_templates/source_network.t similarity index 100% rename from package/etc/templates/source_network.t rename to package/etc/go_templates/source_network.t diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index e4a0a78..d0422fb 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -62,8 +62,7 @@ options { @include "conf.d/destinations/*.conf" @include "conf.d/log_paths/*.conf" -@include "conf.d/local/filters/*.conf" -@include "conf.d/local/filters/*/*.conf" -@include "conf.d/local/sources/*.conf" -@include "conf.d/local/destinations/*.conf" -@include "conf.d/local/log_paths/*.conf" +@include "conf.d/local/config/filters/*.conf" +@include "conf.d/local/config/sources/*.conf" +@include "conf.d/local/config/destinations/*.conf" +@include "conf.d/local/config/log_paths/*.conf" diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index d100fc0..d93862b 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -7,7 +7,7 @@ do echo Templating conf for $d gomplate \ --input-dir=$d \ - --template t=etc/templates/ \ + --template t=etc/go_templates/ \ --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" done From d926c345711a067803a753d6556c00c96b15cb04 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sun, 6 Oct 2019 08:42:51 -0700 Subject: [PATCH 02/12] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3ac4e78..87b76b1 100644 --- a/.gitignore +++ b/.gitignore @@ -382,4 +382,4 @@ fabric.properties /.idea/ tests/test_plugin_*.py -package/etc/conf.d/local/ \ No newline at end of file +# package/etc/conf.d/local/ From 2f4b8ebc7356b462f051568ceaa60c401e3b1a0d Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sun, 6 Oct 2019 10:59:39 -0700 Subject: [PATCH 03/12] Add local directory --- .../local/config/destinations/README.md | 1 + .../etc/conf.d/local/config/filters/README.md | 1 + .../conf.d/local/config/filters/example.conf | 4 + .../conf.d/local/config/log_paths/README.md | 1 + .../local/config/log_paths/example.conf.tmpl | 76 +++++++++++++++++++ .../etc/conf.d/local/config/sources/README.md | 1 + .../context/compliance_meta_by_source.conf | 5 ++ .../context/compliance_meta_by_source.csv | 2 + .../etc/conf.d/local/context/splunk_index.csv | 40 ++++++++++ .../context/vendor_product_by_source.conf | 37 +++++++++ .../context/vendor_product_by_source.csv | 8 ++ 11 files changed, 176 insertions(+) create mode 100644 package/etc/conf.d/local/config/destinations/README.md create mode 100644 package/etc/conf.d/local/config/filters/README.md create mode 100644 package/etc/conf.d/local/config/filters/example.conf create mode 100644 package/etc/conf.d/local/config/log_paths/README.md create mode 100644 package/etc/conf.d/local/config/log_paths/example.conf.tmpl create mode 100644 package/etc/conf.d/local/config/sources/README.md create mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.conf create mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.csv create mode 100644 package/etc/conf.d/local/context/splunk_index.csv create mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.conf create mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.csv diff --git a/package/etc/conf.d/local/config/destinations/README.md b/package/etc/conf.d/local/config/destinations/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/config/destinations/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/config/filters/README.md b/package/etc/conf.d/local/config/filters/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/config/filters/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/config/filters/example.conf b/package/etc/conf.d/local/config/filters/example.conf new file mode 100644 index 0000000..047fdc8 --- /dev/null +++ b/package/etc/conf.d/local/config/filters/example.conf @@ -0,0 +1,4 @@ + +filter f_local_example { + program(sc4splugin); +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/config/log_paths/README.md b/package/etc/conf.d/local/config/log_paths/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/config/log_paths/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/config/log_paths/example.conf.tmpl b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl new file mode 100644 index 0000000..a8ac264 --- /dev/null +++ b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl @@ -0,0 +1,76 @@ +# LOCAL_EXAMPLE + +# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique +# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the +# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ + +# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), +# the template generator will build a custom source based on the value of one or more of the set variables. + +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} + +# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name +# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". +# "parser" can be customized on dedicated ports only +# "common" uses the same parser sequence as the default ports and is the most commonly used + +{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} + +# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path +# The ports used are based on the values of one or more of the environment variables set above. + +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { + +# The first time this template is used the log_path will be linked to the default port + +{{- if eq (.) "yes"}} + source(s_default-ports); + +# Filters should be updated to use the simplest and most effecient logic possible to discard +# the message from this path + + filter(f_is_rfc3164); + filter(f_local_example); +{{- end}} +{{- if eq (.) "no"}} + +# In the second pass through the template a link to the dedicated port is used. This +# normally does not require additional filters + +source (s_dedicated_port_LOCAL_EXAMPLE); +{{- end}} + +#Set a default sourcetype and index + + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + +#using the key "local_example" find any cutomized index,source or sourcetype meta values + + parser {p_add_context_splunk(key("local_example")); }; + +# Any additional logic needed to process the event before sending to Splunk goes here + +# Send it to Splunk + + destination(d_hec); #--HEC-- + +# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will +# forward events to another system + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic + + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic + +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/conf.d/local/config/sources/README.md b/package/etc/conf.d/local/config/sources/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/conf.d/local/config/sources/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.conf b/package/etc/conf.d/local/context/compliance_meta_by_source.conf new file mode 100644 index 0000000..1d5acae --- /dev/null +++ b/package/etc/conf.d/local/context/compliance_meta_by_source.conf @@ -0,0 +1,5 @@ +@version: 3.23 +filter f_test_test { + host("something-*" type(glob)) or + netmask(192.168.100.1/24) +}; diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.csv b/package/etc/conf.d/local/context/compliance_meta_by_source.csv new file mode 100644 index 0000000..6608db0 --- /dev/null +++ b/package/etc/conf.d/local/context/compliance_meta_by_source.csv @@ -0,0 +1,2 @@ +#f_test_test,.splunk.index,"badindex" +#f_test_test,fields.compliance,"pci" diff --git a/package/etc/conf.d/local/context/splunk_index.csv b/package/etc/conf.d/local/context/splunk_index.csv new file mode 100644 index 0000000..e93911a --- /dev/null +++ b/package/etc/conf.d/local/context/splunk_index.csv @@ -0,0 +1,40 @@ +#bluecoat_proxy,index,netproxy +#cef_ArcSight_ArcSight,index,netwaf +#cef_Incapsula_SIEMintegration,index,netwaf +#cef_Microsoft_Microsoft Windows,index,oswinsec +#cef_Microsoft_System or Application Event,index,oswin +#cisco_asa,index,netfw +#cisco_ios,index,netops +#cisco_nx_os,index,netops +#local_example,index,main +#fortinet_fortios_event,index,netops +#fortinet_fortios_log,index,netops +#fortinet_fortios_traffic,index,netfw +#fortinet_fortios_utm,index,netids +#juniper_idp,index,netids +#juniper_structured,index,netops +#juniper_idp_structured,index,netids +#juniper_junos_fw_structured,index,netfw +#juniper_junos_ids_structured,index,netids +#juniper_junos_utm_structured,index,netfw +#juniper_junos_fw,index,netfw +#juniper_junos_ids,index,netids +#juniper_junos_utm,index,netfw +#juniper_sslvpn,index,netfw +#juniper_netscreen,index,netfw +#juniper_nsm,index,netfw +#juniper_nsm_idp,index,netids +#juniper_legacy,index,netops +#pan_traffic,index,netfw +#pan_threat,index,netproxy +#pan_system,index,netops +#pan_config,index,netops +#pan_hipwatch,index,main +#pan_correlation,index,main +#pan_userid,index,netauth +#pan_unknown,index,netops +#proofpoint_pps_filter,index,email +#proofpoint_pps_sendmail,index,email +#sc4s_events,index,main +#sc4s_fallback,index,main +#sc4s_metrics,index,em_metrics diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.conf b/package/etc/conf.d/local/context/vendor_product_by_source.conf new file mode 100644 index 0000000..37e3412 --- /dev/null +++ b/package/etc/conf.d/local/context/vendor_product_by_source.conf @@ -0,0 +1,37 @@ +@version: 3.22 +#TODO: #60 The syntax below uses regex and an indirect reference to a variable due to a +#bug/limitation of selector files. The better syntax should be as follows +#filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; + +filter f_test_test { + host("testvp-*" type(glob)) or + netmask(192.168.100.1/24) +}; +filter f_juniper_nsm { + host("jnpnsm-*" type(glob)) or + netmask(192.168.1.0/24) +}; +filter f_juniper_nsm_idp { + host("jnpnsmidp-*" type(glob)) or + netmask(192.168.2.0/24) +}; +filter f_juniper_idp { + host("jnpidp-*" type(glob)) or + netmask(192.168.3.0/24) +}; +filter f_juniper_netscreen { + host("jnpns-*" type(glob)) or + netmask(192.168.4.0/24) +}; +filter f_cisco_nx_os { + host("csconx-*" type(glob)) or + netmask(192.168.5.0/24) +}; +filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) or + netmask(192.168.6.0/24) +}; +filter f_proofpoint_pps_filter { + host("pps-*" type(glob)) or + netmask(192.168.7.0/24) +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.csv b/package/etc/conf.d/local/context/vendor_product_by_source.csv new file mode 100644 index 0000000..3f90603 --- /dev/null +++ b/package/etc/conf.d/local/context/vendor_product_by_source.csv @@ -0,0 +1,8 @@ +f_test_test,sc4s_vendor_product,"test_test" +f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" +f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" +f_juniper_idp,sc4s_vendor_product,"juniper_idp" +f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" +f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" +f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" +f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file From 63d69a91af586823b3b0fedf160b9db62efdaee5 Mon Sep 17 00:00:00 2001 From: ryan faircloth Date: Sun, 6 Oct 2019 16:18:20 -0400 Subject: [PATCH 04/12] seed the context directory in local if it does not exist --- .env.template | 6 +++--- package/Dockerfile | 5 +++-- .../compliance_meta_by_source.conf | 0 .../compliance_meta_by_source.csv | 0 .../local/context => context_templates}/splunk_index.csv | 0 .../vendor_product_by_source.conf | 0 .../vendor_product_by_source.csv | 0 package/sbin/entrypoint.sh | 4 +++- 8 files changed, 9 insertions(+), 6 deletions(-) rename package/etc/{conf.d/local/context => context_templates}/compliance_meta_by_source.conf (100%) rename package/etc/{conf.d/local/context => context_templates}/compliance_meta_by_source.csv (100%) rename package/etc/{conf.d/local/context => context_templates}/splunk_index.csv (100%) rename package/etc/{conf.d/local/context => context_templates}/vendor_product_by_source.conf (100%) rename package/etc/{conf.d/local/context => context_templates}/vendor_product_by_source.csv (100%) diff --git a/.env.template b/.env.template index 8d0cafa..c1f7af2 100644 --- a/.env.template +++ b/.env.template @@ -16,6 +16,6 @@ SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics -SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download -SPLUNKBASE_USERNAME=username -SPLUNKBASE_PASSWORD=password +#SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download +#SPLUNKBASE_USERNAME=username +#SPLUNKBASE_PASSWORD=password diff --git a/package/Dockerfile b/package/Dockerfile index 570a16a..a0c05f4 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -87,11 +87,12 @@ RUN curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gompl COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d COPY etc/go_templates /opt/syslog-ng/etc/go_templates +COPY etc/context_templates /opt/syslog-ng/etc/context_templates + COPY etc/conf.d/local/context /opt/syslog-ng/etc/conf.d/local/context COPY sbin/entrypoint.sh / RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V -#RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t EXPOSE 514 EXPOSE 601/tcp @@ -101,4 +102,4 @@ ENV SPLUNK_CONNECT_METHOD=UF ENTRYPOINT ["/entrypoint.sh", "-F"] -HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate +HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc \ No newline at end of file diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.conf b/package/etc/context_templates/compliance_meta_by_source.conf similarity index 100% rename from package/etc/conf.d/local/context/compliance_meta_by_source.conf rename to package/etc/context_templates/compliance_meta_by_source.conf diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.csv b/package/etc/context_templates/compliance_meta_by_source.csv similarity index 100% rename from package/etc/conf.d/local/context/compliance_meta_by_source.csv rename to package/etc/context_templates/compliance_meta_by_source.csv diff --git a/package/etc/conf.d/local/context/splunk_index.csv b/package/etc/context_templates/splunk_index.csv similarity index 100% rename from package/etc/conf.d/local/context/splunk_index.csv rename to package/etc/context_templates/splunk_index.csv diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf similarity index 100% rename from package/etc/conf.d/local/context/vendor_product_by_source.conf rename to package/etc/context_templates/vendor_product_by_source.conf diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv similarity index 100% rename from package/etc/conf.d/local/context/vendor_product_by_source.csv rename to package/etc/context_templates/vendor_product_by_source.csv diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index d93862b..0af0742 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -12,5 +12,7 @@ do --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" done -echo syslog-ng started +cp -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context + +echo syslog-ng starting exec /opt/syslog-ng/sbin/syslog-ng $@ \ No newline at end of file From 49c014a65efb11e76c0404dd67f692f036c2f432 Mon Sep 17 00:00:00 2001 From: ryan faircloth Date: Sun, 6 Oct 2019 16:38:49 -0400 Subject: [PATCH 05/12] Update Dockerfile --- package/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/package/Dockerfile b/package/Dockerfile index a0c05f4..91da569 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -89,7 +89,6 @@ COPY etc/conf.d /opt/syslog-ng/etc/conf.d COPY etc/go_templates /opt/syslog-ng/etc/go_templates COPY etc/context_templates /opt/syslog-ng/etc/context_templates -COPY etc/conf.d/local/context /opt/syslog-ng/etc/conf.d/local/context COPY sbin/entrypoint.sh / RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V From db72d9f00475f1eddc04002376081fa90ec748fb Mon Sep 17 00:00:00 2001 From: ryan faircloth Date: Sun, 6 Oct 2019 16:47:01 -0400 Subject: [PATCH 06/12] Update entrypoint.sh --- package/sbin/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 0af0742..5b7d6a1 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -12,7 +12,7 @@ do --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" done -cp -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context +cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ echo syslog-ng starting exec /opt/syslog-ng/sbin/syslog-ng $@ \ No newline at end of file From 694ae594c05572efbd75a5763e4e85c0ded0b59d Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sun, 6 Oct 2019 16:19:30 -0700 Subject: [PATCH 07/12] entrypoint.sh add mkdir Add mkdir for subdirectories of the local directory to prepare for copy. --- package/sbin/entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 5b7d6a1..13a2464 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -12,6 +12,8 @@ do --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" done +mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ +mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ echo syslog-ng starting From 281266544d162a16d77daf2c86cf8b05dae1af4e Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sun, 6 Oct 2019 17:22:01 -0700 Subject: [PATCH 08/12] Update entrypoint.sh to copy config local dir --- .../etc/local_config/destinations/README.md | 1 + package/etc/local_config/filters/README.md | 1 + package/etc/local_config/filters/example.conf | 4 + package/etc/local_config/log_paths/README.md | 1 + .../local_config/log_paths/example.conf.tmpl | 76 +++++++++++++++++++ package/etc/local_config/sources/README.md | 1 + package/sbin/entrypoint.sh | 1 + 7 files changed, 85 insertions(+) create mode 100644 package/etc/local_config/destinations/README.md create mode 100644 package/etc/local_config/filters/README.md create mode 100644 package/etc/local_config/filters/example.conf create mode 100644 package/etc/local_config/log_paths/README.md create mode 100644 package/etc/local_config/log_paths/example.conf.tmpl create mode 100644 package/etc/local_config/sources/README.md diff --git a/package/etc/local_config/destinations/README.md b/package/etc/local_config/destinations/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/destinations/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/filters/README.md b/package/etc/local_config/filters/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/filters/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/filters/example.conf b/package/etc/local_config/filters/example.conf new file mode 100644 index 0000000..047fdc8 --- /dev/null +++ b/package/etc/local_config/filters/example.conf @@ -0,0 +1,4 @@ + +filter f_local_example { + program(sc4splugin); +}; \ No newline at end of file diff --git a/package/etc/local_config/log_paths/README.md b/package/etc/local_config/log_paths/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/log_paths/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl new file mode 100644 index 0000000..a8ac264 --- /dev/null +++ b/package/etc/local_config/log_paths/example.conf.tmpl @@ -0,0 +1,76 @@ +# LOCAL_EXAMPLE + +# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique +# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the +# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ + +# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), +# the template generator will build a custom source based on the value of one or more of the set variables. + +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} + +# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name +# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". +# "parser" can be customized on dedicated ports only +# "common" uses the same parser sequence as the default ports and is the most commonly used + +{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} + +# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path +# The ports used are based on the values of one or more of the environment variables set above. + +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { + +# The first time this template is used the log_path will be linked to the default port + +{{- if eq (.) "yes"}} + source(s_default-ports); + +# Filters should be updated to use the simplest and most effecient logic possible to discard +# the message from this path + + filter(f_is_rfc3164); + filter(f_local_example); +{{- end}} +{{- if eq (.) "no"}} + +# In the second pass through the template a link to the dedicated port is used. This +# normally does not require additional filters + +source (s_dedicated_port_LOCAL_EXAMPLE); +{{- end}} + +#Set a default sourcetype and index + + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + +#using the key "local_example" find any cutomized index,source or sourcetype meta values + + parser {p_add_context_splunk(key("local_example")); }; + +# Any additional logic needed to process the event before sending to Splunk goes here + +# Send it to Splunk + + destination(d_hec); #--HEC-- + +# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will +# forward events to another system + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic + + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic + +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/local_config/sources/README.md b/package/etc/local_config/sources/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/sources/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 13a2464..30537c7 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -15,6 +15,7 @@ done mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ +cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ echo syslog-ng starting exec /opt/syslog-ng/sbin/syslog-ng $@ \ No newline at end of file From 2d5c178c6c19faf72106930019477425b139c105 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Sun, 6 Oct 2019 20:33:59 -0400 Subject: [PATCH 09/12] Update Dockerfile --- package/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/Dockerfile b/package/Dockerfile index 91da569..92dd2cf 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -101,4 +101,4 @@ ENV SPLUNK_CONNECT_METHOD=UF ENTRYPOINT ["/entrypoint.sh", "-F"] -HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc \ No newline at end of file +HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file From 67faacfee8c52a83df233a6b54f890478fa5e8a1 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sun, 6 Oct 2019 17:47:36 -0700 Subject: [PATCH 10/12] Update Dockerfile to include local_config tree --- package/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/package/Dockerfile b/package/Dockerfile index 91da569..74f7bb8 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -88,6 +88,7 @@ COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d COPY etc/go_templates /opt/syslog-ng/etc/go_templates COPY etc/context_templates /opt/syslog-ng/etc/context_templates +COPY etc/local_config /opt/syslog-ng/etc/local_config COPY sbin/entrypoint.sh / RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer From ed700e4bbf13529183f9df55f623eae2aede3981 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Sun, 6 Oct 2019 18:06:17 -0700 Subject: [PATCH 11/12] gitignore -- ignore local mount directory --- .gitignore | 1 + .../context/compliance_meta_by_source.conf | 5 +++ .../context/compliance_meta_by_source.csv | 2 + .../etc/conf.d/local/context/splunk_index.csv | 40 +++++++++++++++++++ .../context/vendor_product_by_source.conf | 37 +++++++++++++++++ .../context/vendor_product_by_source.csv | 8 ++++ 6 files changed, 93 insertions(+) create mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.conf create mode 100644 package/etc/conf.d/local/context/compliance_meta_by_source.csv create mode 100644 package/etc/conf.d/local/context/splunk_index.csv create mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.conf create mode 100644 package/etc/conf.d/local/context/vendor_product_by_source.csv diff --git a/.gitignore b/.gitignore index 87b76b1..4d40b52 100644 --- a/.gitignore +++ b/.gitignore @@ -383,3 +383,4 @@ fabric.properties tests/test_plugin_*.py # package/etc/conf.d/local/ +!package/etc/conf.d/local diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.conf b/package/etc/conf.d/local/context/compliance_meta_by_source.conf new file mode 100644 index 0000000..1d5acae --- /dev/null +++ b/package/etc/conf.d/local/context/compliance_meta_by_source.conf @@ -0,0 +1,5 @@ +@version: 3.23 +filter f_test_test { + host("something-*" type(glob)) or + netmask(192.168.100.1/24) +}; diff --git a/package/etc/conf.d/local/context/compliance_meta_by_source.csv b/package/etc/conf.d/local/context/compliance_meta_by_source.csv new file mode 100644 index 0000000..6608db0 --- /dev/null +++ b/package/etc/conf.d/local/context/compliance_meta_by_source.csv @@ -0,0 +1,2 @@ +#f_test_test,.splunk.index,"badindex" +#f_test_test,fields.compliance,"pci" diff --git a/package/etc/conf.d/local/context/splunk_index.csv b/package/etc/conf.d/local/context/splunk_index.csv new file mode 100644 index 0000000..e93911a --- /dev/null +++ b/package/etc/conf.d/local/context/splunk_index.csv @@ -0,0 +1,40 @@ +#bluecoat_proxy,index,netproxy +#cef_ArcSight_ArcSight,index,netwaf +#cef_Incapsula_SIEMintegration,index,netwaf +#cef_Microsoft_Microsoft Windows,index,oswinsec +#cef_Microsoft_System or Application Event,index,oswin +#cisco_asa,index,netfw +#cisco_ios,index,netops +#cisco_nx_os,index,netops +#local_example,index,main +#fortinet_fortios_event,index,netops +#fortinet_fortios_log,index,netops +#fortinet_fortios_traffic,index,netfw +#fortinet_fortios_utm,index,netids +#juniper_idp,index,netids +#juniper_structured,index,netops +#juniper_idp_structured,index,netids +#juniper_junos_fw_structured,index,netfw +#juniper_junos_ids_structured,index,netids +#juniper_junos_utm_structured,index,netfw +#juniper_junos_fw,index,netfw +#juniper_junos_ids,index,netids +#juniper_junos_utm,index,netfw +#juniper_sslvpn,index,netfw +#juniper_netscreen,index,netfw +#juniper_nsm,index,netfw +#juniper_nsm_idp,index,netids +#juniper_legacy,index,netops +#pan_traffic,index,netfw +#pan_threat,index,netproxy +#pan_system,index,netops +#pan_config,index,netops +#pan_hipwatch,index,main +#pan_correlation,index,main +#pan_userid,index,netauth +#pan_unknown,index,netops +#proofpoint_pps_filter,index,email +#proofpoint_pps_sendmail,index,email +#sc4s_events,index,main +#sc4s_fallback,index,main +#sc4s_metrics,index,em_metrics diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.conf b/package/etc/conf.d/local/context/vendor_product_by_source.conf new file mode 100644 index 0000000..37e3412 --- /dev/null +++ b/package/etc/conf.d/local/context/vendor_product_by_source.conf @@ -0,0 +1,37 @@ +@version: 3.22 +#TODO: #60 The syntax below uses regex and an indirect reference to a variable due to a +#bug/limitation of selector files. The better syntax should be as follows +#filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; + +filter f_test_test { + host("testvp-*" type(glob)) or + netmask(192.168.100.1/24) +}; +filter f_juniper_nsm { + host("jnpnsm-*" type(glob)) or + netmask(192.168.1.0/24) +}; +filter f_juniper_nsm_idp { + host("jnpnsmidp-*" type(glob)) or + netmask(192.168.2.0/24) +}; +filter f_juniper_idp { + host("jnpidp-*" type(glob)) or + netmask(192.168.3.0/24) +}; +filter f_juniper_netscreen { + host("jnpns-*" type(glob)) or + netmask(192.168.4.0/24) +}; +filter f_cisco_nx_os { + host("csconx-*" type(glob)) or + netmask(192.168.5.0/24) +}; +filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) or + netmask(192.168.6.0/24) +}; +filter f_proofpoint_pps_filter { + host("pps-*" type(glob)) or + netmask(192.168.7.0/24) +}; \ No newline at end of file diff --git a/package/etc/conf.d/local/context/vendor_product_by_source.csv b/package/etc/conf.d/local/context/vendor_product_by_source.csv new file mode 100644 index 0000000..3f90603 --- /dev/null +++ b/package/etc/conf.d/local/context/vendor_product_by_source.csv @@ -0,0 +1,8 @@ +f_test_test,sc4s_vendor_product,"test_test" +f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" +f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" +f_juniper_idp,sc4s_vendor_product,"juniper_idp" +f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" +f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" +f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" +f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file From d54bf2af14ec2526343b34cca23be2f9b237a3f3 Mon Sep 17 00:00:00 2001 From: Mark Bonsack Date: Mon, 7 Oct 2019 07:57:19 -0700 Subject: [PATCH 12/12] update vendor_product_by_source.conf Remove bogus comment and update version string --- package/etc/context_templates/vendor_product_by_source.conf | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf index 37e3412..342c9f4 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -1,7 +1,4 @@ -@version: 3.22 -#TODO: #60 The syntax below uses regex and an indirect reference to a variable due to a -#bug/limitation of selector files. The better syntax should be as follows -#filter {match("f5_test" template("$(env PRESUME_SYSLOG)")); }; +@version: 3.23 filter f_test_test { host("testvp-*" type(glob)) or