Setup meta data for compliance needs by filter after source type (#111)

* Setup meta data for compliance needs by filter after source type

* Finish support for featuree and docs

* Update configuration.md

docs/configuration.md

@@ -45,3 +45,42 @@ and variables needed to properly configure SC4S for your environment.
 SC4S_SOURCE_TLS_ENABLE=yes
 ```
 
+## Override index or metadata based on host, ip, or subnet
+
+In some cases it is appropriate to re-direct events to an alternate index or append metadata (such as an
+indexed field) based on PCI scope, geography, or other criterion.  This is accomplished by the use
+of a file that uniquely identifies these source exceptions via syslog-ng filters,
+which maps to an associated lookup of alternate indexes, sources, or other metadata.
+
+* Get the filter and lookup files
+```bash
+cd /opt/sc4s/default
+sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/compliance_meta_by_source.conf
+sudo wget https://raw.githubusercontent.com/splunk/splunk-connect-for-syslog/master/package/etc/context-local/compliance_meta_by_source.csv
+```
+* Edit the file ``compliance_meta_by_source.conf`` to supply uniquely named filters to identify events subject to override.
+* Edit the file ``compliance_meta_by_source.csv``  to supply appropriate the field(s) and values.
+The three columns in the table are `filter name`, `field name`, and `value`.  `field name` obeys the following convention:
+    * ``fields.fieldname`` where `fieldname` will become the name of an indexed field with the supplied value
+    * ``.splunk.index`` to specify an alternate value for index
+    * ``.splunk.source`` to specify an alternate value for source 
+
+* For the Docker/Podman runtimes, update the docker/podman run command in the systemd unit file or the docker-compose to
+include volumes mapping the files above.
+* In the Unit file, add the following lines to the `ExecStart` command prior to `$SC4SIMAGE` then restart using the command
+``sudo systemctl daemon-reload; sudo systemctl restart sc4s``
+
+``
+SC4S_UNIT_VP_CSV=-v /opt/sc4s/default/compliance_meta_by_source.csv:/opt/syslog-ng/etc/context-local/compliance_meta_by_source.csv \
+SC4S_UNIT_VP_CONF=-v /opt/sc4s/default/compliance_meta_by_source.conf:/opt/syslog-ng/etc/context-local/compliance_meta_by_source.conf \
+``
+
+* For the Docker Swarm runtime, update the docker compose yml to add the following volume mounts to thee sc4s service and
+redeploy the updated service using the command:
+``docker stack deploy --compose-file docker-compose.yml sc4s``
+
+``
+      - /opt/sc4s/default/compliance_meta_by_source.csv:/opt/syslog-ng/etc/context-local/compliance_meta_by_source.csv
+      - /opt/sc4s/default/compliance_meta_by_source.conf:/opt/syslog-ng/etc/context-local/compliance_meta_by_source.conf
+``
+

package/etc/conf.d/conflib/_common/compliance_meta.conf

@@ -0,0 +1,8 @@
+parser compliance_meta_by_source {
+    add-contextual-data(
+        selector(filters("`syslog-ng-sysconfdir`/context-local/compliance_meta_by_source.conf")),
+        database("context-local/compliance_meta_by_source.csv")
+        ignore-case(yes)
+    );
+};
+

package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf.tmpl

@@ -17,6 +17,8 @@ log {
     rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))};
     parser {p_add_context_splunk(key("cisco_asa")); };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf.tmpl

@@ -18,6 +18,8 @@ log {
         p_add_context_splunk(key("cisco_ios"));
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164-cisco_nx-os.conf.tmpl

@@ -18,6 +18,8 @@ log {
         p_add_context_splunk(key("cisco_nx_os"));
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf.tmpl

@@ -35,6 +35,8 @@ log {
        parser {p_add_context_splunk(key("fortinet_fortios_log")); };
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164-juniper_idp.conf.tmpl

@@ -19,6 +19,8 @@ log {
                 p_add_context_splunk(key("juniper_idp"));
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164-juniper_junos.conf.tmpl

@@ -34,6 +34,8 @@ log {
         parser {p_add_context_splunk(key("juniper_legacy")); };
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf.tmpl

@@ -23,6 +23,8 @@ log {
             p_add_context_splunk(key("juniper_netscreen"));
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
     flags(flow-control);
 };

package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf.tmpl

@@ -20,6 +20,8 @@ log {
                 p_add_context_splunk(key("juniper_nsm"));
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
     flags(flow-control);
 };

package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm_idp.conf.tmpl

@@ -19,6 +19,8 @@ log {
                 p_add_context_splunk(key("juniper_idp"));
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
     flags(flow-control);
 };

package/etc/conf.d/log_paths/p_rfc3164-paloalto_panos.conf.tmpl

@@ -71,6 +71,8 @@ log {
         parser {p_add_context_splunk(key("pan_log")); };
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_filter.conf.tmpl

@@ -18,6 +18,8 @@ log {
         p_add_context_splunk(key("proofpoint_pps_filter"));
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164-proofpoint_pps_sendmail.conf.tmpl

@@ -18,6 +18,8 @@ log {
         p_add_context_splunk(key("proofpoint_pps_sendmail"));
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc3164_microfocus_arcsight.conf.tmpl

@@ -68,6 +68,8 @@ log {
     #CEF TAs use the source as their bounds in props.conf
     parser(p_microfocus_arcsight_source);
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf.tmpl

@@ -17,6 +17,8 @@ log {
     rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"), template("t_msg_only"))};
     parser {p_add_context_splunk(key("cisco_asa")); };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec_proxy.conf.tmpl

@@ -18,6 +18,8 @@ log {
 
     parser {p_add_context_splunk(key("bluecoat_proxy")); };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 
     flags(flow-control);

package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper_junos.conf.tmpl

@@ -37,6 +37,8 @@ log {
         parser {p_add_context_splunk(key("juniper_structured")); };
     };
 
+    parser (compliance_meta_by_source);
+
     destination(d_hec);  #--HEC--
 };
 {{- end}}

package/etc/context-local/compliance_meta_by_source.conf

@@ -0,0 +1,5 @@
+@version: 3.23
+filter f_test_test {
+    host("something-*" type(glob)) or
+    netmask(192.168.100.1/24)
+};

package/etc/context-local/compliance_meta_by_source.csv

@@ -0,0 +1,2 @@
+#f_test_test,.splunk.index,"badindex"
+#f_test_test,fields.compliance,"pci"