diff --git a/.env.template b/.env.template index 8d0cafa..c1f7af2 100644 --- a/.env.template +++ b/.env.template @@ -16,6 +16,6 @@ SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event SPLUNK_CONNECT_METHOD=hec SPLUNK_DEFAULT_INDEX=main SPLUNK_METRICS_INDEX=em_metrics -SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download -SPLUNKBASE_USERNAME=username -SPLUNKBASE_PASSWORD=password +#SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download +#SPLUNKBASE_USERNAME=username +#SPLUNKBASE_PASSWORD=password diff --git a/.gitignore b/.gitignore index 3ac4e78..4d40b52 100644 --- a/.gitignore +++ b/.gitignore @@ -382,4 +382,5 @@ fabric.properties /.idea/ tests/test_plugin_*.py -package/etc/conf.d/local/ \ No newline at end of file +# package/etc/conf.d/local/ +!package/etc/conf.d/local diff --git a/package/Dockerfile b/package/Dockerfile index a5f67cc..7008a31 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -86,12 +86,13 @@ RUN curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gompl COPY etc/syslog-ng.conf /opt/syslog-ng/etc/syslog-ng.conf COPY etc/conf.d /opt/syslog-ng/etc/conf.d -COPY etc/templates /opt/syslog-ng/etc/templates -COPY etc/context-local /opt/syslog-ng/etc/context-local +COPY etc/go_templates /opt/syslog-ng/etc/go_templates +COPY etc/context_templates /opt/syslog-ng/etc/context_templates +COPY etc/local_config /opt/syslog-ng/etc/local_config + COPY sbin/entrypoint.sh / RUN mkdir -p /opt/syslog-ng/var/data/disk-buffer RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -V -#RUN source scl_source enable rh-python36 ;/opt/syslog-ng/sbin/syslog-ng -t EXPOSE 514 EXPOSE 601/tcp @@ -101,4 +102,4 @@ ENV SPLUNK_CONNECT_METHOD=UF ENTRYPOINT ["/entrypoint.sh", "-F"] -HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate +HEALTHCHECK --interval=1s --timeout=6s CMD source scl_source enable rh-python36 ;goss -g /etc/goss.yaml validate \ No newline at end of file diff --git a/package/etc/conf.d/conflib/_common/compliance_meta.conf b/package/etc/conf.d/conflib/_common/compliance_meta.conf index 3529ae5..75479c6 100644 --- a/package/etc/conf.d/conflib/_common/compliance_meta.conf +++ b/package/etc/conf.d/conflib/_common/compliance_meta.conf @@ -1,7 +1,7 @@ parser compliance_meta_by_source { add-contextual-data( - selector(filters("`syslog-ng-sysconfdir`/context-local/compliance_meta_by_source.conf")), - database("context-local/compliance_meta_by_source.csv") + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/compliance_meta_by_source.conf")), + database("conf.d/local/context/compliance_meta_by_source.csv") ignore-case(yes) ); }; diff --git a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf index f3789a7..44d8ff1 100644 --- a/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf +++ b/package/etc/conf.d/conflib/_common/vendor_product_by_source_context.conf @@ -1,7 +1,7 @@ block parser vendor_product_by_source() { add-contextual-data( - selector(filters("`syslog-ng-sysconfdir`/context-local/vendor_product_by_source.conf")), - database("context-local/vendor_product_by_source.csv") + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.conf")), + database("conf.d/local/context/vendor_product_by_source.csv") ignore-case(yes) prefix("fields.") ); diff --git a/package/etc/conf.d/conflib/_splunk/splunk_context.conf b/package/etc/conf.d/conflib/_splunk/splunk_context.conf index e2d1fbb..6fb181d 100644 --- a/package/etc/conf.d/conflib/_splunk/splunk_context.conf +++ b/package/etc/conf.d/conflib/_splunk/splunk_context.conf @@ -1,7 +1,7 @@ block parser p_add_context_splunk(key("syslogng-fallback")) { add-contextual-data( selector("`key`"), - database("context-local/splunk_index.csv"), + database("conf.d/local/context/splunk_index.csv"), prefix(".splunk.") ); -}; \ No newline at end of file +}; diff --git a/package/etc/context-local/microfocus_arcsight_source.csv b/package/etc/conf.d/context/microfocus_arcsight_source.csv similarity index 100% rename from package/etc/context-local/microfocus_arcsight_source.csv rename to package/etc/conf.d/context/microfocus_arcsight_source.csv diff --git a/package/etc/conf.d/local/destinations/README.md b/package/etc/conf.d/local/config/destinations/README.md similarity index 100% rename from package/etc/conf.d/local/destinations/README.md rename to package/etc/conf.d/local/config/destinations/README.md diff --git a/package/etc/conf.d/local/filters/README.md b/package/etc/conf.d/local/config/filters/README.md similarity index 100% rename from package/etc/conf.d/local/filters/README.md rename to package/etc/conf.d/local/config/filters/README.md diff --git a/package/etc/conf.d/local/filters/example.conf b/package/etc/conf.d/local/config/filters/example.conf similarity index 100% rename from package/etc/conf.d/local/filters/example.conf rename to package/etc/conf.d/local/config/filters/example.conf diff --git a/package/etc/conf.d/local/log_paths/README.md b/package/etc/conf.d/local/config/log_paths/README.md similarity index 100% rename from package/etc/conf.d/local/log_paths/README.md rename to package/etc/conf.d/local/config/log_paths/README.md diff --git a/package/etc/conf.d/local/log_paths/example.conf.tmpl b/package/etc/conf.d/local/config/log_paths/example.conf.tmpl similarity index 100% rename from package/etc/conf.d/local/log_paths/example.conf.tmpl rename to package/etc/conf.d/local/config/log_paths/example.conf.tmpl diff --git a/package/etc/conf.d/local/sources/README.md b/package/etc/conf.d/local/config/sources/README.md similarity index 100% rename from package/etc/conf.d/local/sources/README.md rename to package/etc/conf.d/local/config/sources/README.md diff --git a/package/etc/context-local/compliance_meta_by_source.conf b/package/etc/conf.d/local/context/compliance_meta_by_source.conf similarity index 100% rename from package/etc/context-local/compliance_meta_by_source.conf rename to package/etc/conf.d/local/context/compliance_meta_by_source.conf diff --git a/package/etc/context-local/compliance_meta_by_source.csv b/package/etc/conf.d/local/context/compliance_meta_by_source.csv similarity index 100% rename from package/etc/context-local/compliance_meta_by_source.csv rename to package/etc/conf.d/local/context/compliance_meta_by_source.csv diff --git a/package/etc/context-local/splunk_index.csv b/package/etc/conf.d/local/context/splunk_index.csv similarity index 100% rename from package/etc/context-local/splunk_index.csv rename to package/etc/conf.d/local/context/splunk_index.csv diff --git a/package/etc/context-local/vendor_product_by_source.conf b/package/etc/conf.d/local/context/vendor_product_by_source.conf similarity index 100% rename from package/etc/context-local/vendor_product_by_source.conf rename to package/etc/conf.d/local/context/vendor_product_by_source.conf diff --git a/package/etc/context-local/vendor_product_by_source.csv b/package/etc/conf.d/local/context/vendor_product_by_source.csv similarity index 100% rename from package/etc/context-local/vendor_product_by_source.csv rename to package/etc/conf.d/local/context/vendor_product_by_source.csv diff --git a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl index 6170b56..4c89731 100644 --- a/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl +++ b/package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl @@ -22,7 +22,7 @@ parser p_microfocus_arcsight_ts_end { parser p_microfocus_arcsight_source { add-contextual-data( selector("${fields.cef_device_vendor}_${fields.cef_device_product}"), - database("context-local/microfocus_arcsight_source.csv") + database("conf.d/context/microfocus_arcsight_source.csv") ignore-case(yes) prefix(".splunk.") default-selector("unknown") diff --git a/package/etc/context_templates/compliance_meta_by_source.conf b/package/etc/context_templates/compliance_meta_by_source.conf new file mode 100644 index 0000000..1d5acae --- /dev/null +++ b/package/etc/context_templates/compliance_meta_by_source.conf @@ -0,0 +1,5 @@ +@version: 3.23 +filter f_test_test { + host("something-*" type(glob)) or + netmask(192.168.100.1/24) +}; diff --git a/package/etc/context_templates/compliance_meta_by_source.csv b/package/etc/context_templates/compliance_meta_by_source.csv new file mode 100644 index 0000000..6608db0 --- /dev/null +++ b/package/etc/context_templates/compliance_meta_by_source.csv @@ -0,0 +1,2 @@ +#f_test_test,.splunk.index,"badindex" +#f_test_test,fields.compliance,"pci" diff --git a/package/etc/context_templates/splunk_index.csv b/package/etc/context_templates/splunk_index.csv new file mode 100644 index 0000000..e93911a --- /dev/null +++ b/package/etc/context_templates/splunk_index.csv @@ -0,0 +1,40 @@ +#bluecoat_proxy,index,netproxy +#cef_ArcSight_ArcSight,index,netwaf +#cef_Incapsula_SIEMintegration,index,netwaf +#cef_Microsoft_Microsoft Windows,index,oswinsec +#cef_Microsoft_System or Application Event,index,oswin +#cisco_asa,index,netfw +#cisco_ios,index,netops +#cisco_nx_os,index,netops +#local_example,index,main +#fortinet_fortios_event,index,netops +#fortinet_fortios_log,index,netops +#fortinet_fortios_traffic,index,netfw +#fortinet_fortios_utm,index,netids +#juniper_idp,index,netids +#juniper_structured,index,netops +#juniper_idp_structured,index,netids +#juniper_junos_fw_structured,index,netfw +#juniper_junos_ids_structured,index,netids +#juniper_junos_utm_structured,index,netfw +#juniper_junos_fw,index,netfw +#juniper_junos_ids,index,netids +#juniper_junos_utm,index,netfw +#juniper_sslvpn,index,netfw +#juniper_netscreen,index,netfw +#juniper_nsm,index,netfw +#juniper_nsm_idp,index,netids +#juniper_legacy,index,netops +#pan_traffic,index,netfw +#pan_threat,index,netproxy +#pan_system,index,netops +#pan_config,index,netops +#pan_hipwatch,index,main +#pan_correlation,index,main +#pan_userid,index,netauth +#pan_unknown,index,netops +#proofpoint_pps_filter,index,email +#proofpoint_pps_sendmail,index,email +#sc4s_events,index,main +#sc4s_fallback,index,main +#sc4s_metrics,index,em_metrics diff --git a/package/etc/context_templates/vendor_product_by_source.conf b/package/etc/context_templates/vendor_product_by_source.conf new file mode 100644 index 0000000..342c9f4 --- /dev/null +++ b/package/etc/context_templates/vendor_product_by_source.conf @@ -0,0 +1,34 @@ +@version: 3.23 + +filter f_test_test { + host("testvp-*" type(glob)) or + netmask(192.168.100.1/24) +}; +filter f_juniper_nsm { + host("jnpnsm-*" type(glob)) or + netmask(192.168.1.0/24) +}; +filter f_juniper_nsm_idp { + host("jnpnsmidp-*" type(glob)) or + netmask(192.168.2.0/24) +}; +filter f_juniper_idp { + host("jnpidp-*" type(glob)) or + netmask(192.168.3.0/24) +}; +filter f_juniper_netscreen { + host("jnpns-*" type(glob)) or + netmask(192.168.4.0/24) +}; +filter f_cisco_nx_os { + host("csconx-*" type(glob)) or + netmask(192.168.5.0/24) +}; +filter f_proofpoint_pps_sendmail { + host("pps-*" type(glob)) or + netmask(192.168.6.0/24) +}; +filter f_proofpoint_pps_filter { + host("pps-*" type(glob)) or + netmask(192.168.7.0/24) +}; \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.csv b/package/etc/context_templates/vendor_product_by_source.csv new file mode 100644 index 0000000..3f90603 --- /dev/null +++ b/package/etc/context_templates/vendor_product_by_source.csv @@ -0,0 +1,8 @@ +f_test_test,sc4s_vendor_product,"test_test" +f_juniper_nsm,sc4s_vendor_product,"juniper_nsm" +f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp" +f_juniper_idp,sc4s_vendor_product,"juniper_idp" +f_juniper_netscreen,sc4s_vendor_product,"juniper_netscreen" +f_cisco_nx_os,sc4s_vendor_product,"cisco_nx_os" +f_proofpoint_pps_sendmail,sc4s_vendor_product,"proofpoint_pps_sendmail" +f_proofpoint_pps_filter,sc4s_vendor_product,"proofpoint_pps_filter" \ No newline at end of file diff --git a/package/etc/templates/source_network.t b/package/etc/go_templates/source_network.t similarity index 100% rename from package/etc/templates/source_network.t rename to package/etc/go_templates/source_network.t diff --git a/package/etc/local_config/destinations/README.md b/package/etc/local_config/destinations/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/destinations/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/filters/README.md b/package/etc/local_config/filters/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/filters/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/filters/example.conf b/package/etc/local_config/filters/example.conf new file mode 100644 index 0000000..047fdc8 --- /dev/null +++ b/package/etc/local_config/filters/example.conf @@ -0,0 +1,4 @@ + +filter f_local_example { + program(sc4splugin); +}; \ No newline at end of file diff --git a/package/etc/local_config/log_paths/README.md b/package/etc/local_config/log_paths/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/log_paths/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/local_config/log_paths/example.conf.tmpl b/package/etc/local_config/log_paths/example.conf.tmpl new file mode 100644 index 0000000..a8ac264 --- /dev/null +++ b/package/etc/local_config/log_paths/example.conf.tmpl @@ -0,0 +1,76 @@ +# LOCAL_EXAMPLE + +# When creating a real plugin, replace the upper case text "LOCAL_EXAMPLE" throughout this file with a unique +# string to identify the vendor product. The string should be of the form "VENDOR_PRODUCT" to signify the +# manufacturer and product type, and must contain only characters matching this regex: [A-Z\_]+ + +# If any of the "LOCAL_EXAMPLE" variables passed into the environment are set (e.g. TLS, UDP, or TLS), +# the template generator will build a custom source based on the value of one or more of the set variables. + +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TLS_PORT") "no") "no") }} + +# "port_id" is used to generate the port variable to be used. It should match the "core" of the variable name +# set in the line above. For example, the "port_id" of "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT" is "LOCAL_EXAMPLE". +# "parser" can be customized on dedicated ports only +# "common" uses the same parser sequence as the default ports and is the most commonly used + +{{ $context := dict "port_id" "LOCAL_EXAMPLE" "parser" "common"}} + +# The following template execution creates a syslog-ng source with one or more dedicated ports for use with this log_path +# The ports used are based on the values of one or more of the environment variables set above. + +{{ tmpl.Exec "t/source_network.t" $context }} +{{- end -}} +{{ define "log_path" }} +log { + +# The first time this template is used the log_path will be linked to the default port + +{{- if eq (.) "yes"}} + source(s_default-ports); + +# Filters should be updated to use the simplest and most effecient logic possible to discard +# the message from this path + + filter(f_is_rfc3164); + filter(f_local_example); +{{- end}} +{{- if eq (.) "no"}} + +# In the second pass through the template a link to the dedicated port is used. This +# normally does not require additional filters + +source (s_dedicated_port_LOCAL_EXAMPLE); +{{- end}} + +#Set a default sourcetype and index + + rewrite { r_set_splunk_dest_default(sourcetype("sc4s:local_example"), index("main"), template("t_msg_only"))}; + +#using the key "local_example" find any cutomized index,source or sourcetype meta values + + parser {p_add_context_splunk(key("local_example")); }; + +# Any additional logic needed to process the event before sending to Splunk goes here + +# Send it to Splunk + + destination(d_hec); #--HEC-- + +# Note: We normally do not use the "final" flag; this will allow another plugin to be created that will +# forward events to another system + + flags(flow-control); + +}; +{{- end}} +{{- if (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_TCP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_LOCAL_EXAMPLE_UDP_PORT") "no") "no") or (ne (getenv (print "SC4S_LISTEN_MICROFOCUS_ARCSIGHT_TLS_PORT") "no") "no") }} + +# Listen on the specified dedicated port(s) for LOCAL_EXAMPLE traffic + + {{tmpl.Exec "log_path" "no" }} +{{- end}} + +# Listen on the default port (typically 514) for LOCAL_EXAMPLE traffic + +{{tmpl.Exec "log_path" "yes" }} diff --git a/package/etc/local_config/sources/README.md b/package/etc/local_config/sources/README.md new file mode 100644 index 0000000..ee6571d --- /dev/null +++ b/package/etc/local_config/sources/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index e4a0a78..d0422fb 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -62,8 +62,7 @@ options { @include "conf.d/destinations/*.conf" @include "conf.d/log_paths/*.conf" -@include "conf.d/local/filters/*.conf" -@include "conf.d/local/filters/*/*.conf" -@include "conf.d/local/sources/*.conf" -@include "conf.d/local/destinations/*.conf" -@include "conf.d/local/log_paths/*.conf" +@include "conf.d/local/config/filters/*.conf" +@include "conf.d/local/config/sources/*.conf" +@include "conf.d/local/config/destinations/*.conf" +@include "conf.d/local/config/log_paths/*.conf" diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index d100fc0..30537c7 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -7,10 +7,15 @@ do echo Templating conf for $d gomplate \ --input-dir=$d \ - --template t=etc/templates/ \ + --template t=etc/go_templates/ \ --exclude=*.conf --exclude=*.csv --exclude=*.t --exclude=.*\ --output-map="$d/{{ .in | strings.ReplaceAll \".conf.tmpl\" \".conf\" }}" done -echo syslog-ng started +mkdir -p /opt/syslog-ng/etc/conf.d/local/context/ +mkdir -p /opt/syslog-ng/etc/conf.d/local/config/ +cp --verbose -n /opt/syslog-ng/etc/context_templates/* /opt/syslog-ng/etc/conf.d/local/context/ +cp --verbose -R -n /opt/syslog-ng/etc/local_config/* /opt/syslog-ng/etc/conf.d/local/config/ + +echo syslog-ng starting exec /opt/syslog-ng/sbin/syslog-ng $@ \ No newline at end of file