Merge pull request #310 from splunk/feature/netscaler2

Add Citrix Netscaler support

docs/sources/Citrix/index.md

@@ -0,0 +1,53 @@
+# Vendor - Citrix
+
+## Product - Netscaler ADC
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/2770/                                                                 |
+| Product Manual | https://docs.citrix.com/en-us/citrix-adc/12-1/system/audit-logging/configuring-audit-logging.html |
+
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| citrix:netscaler:syslog         | None                                                                                                    |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| citrix_netscaler         | citrix:netscaler:syslog         | netfw          | none           |
+
+
+### Filter type
+
+MSG Parse: This filter parses message content
+
+### Setup and Configuration
+
+* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
+* Review and update the splunk_index.csv file and set the index and sourcetype as required for the data source.
+* Follow vendor configuration steps per Product Manual above. Ensure the data format selected is "MMDDYYYY" 
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_CITRIX_NETSCALER_SPLUNK_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using the port number defined |
+| SC4S_LISTEN_CITRIX_NETSCALERSPLUNK_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using the port number defined |
+| SC4S_ARCHIVE_CITRIX_NETSCALER_SPLUNK | no | Enable archive to disk for this specific source |
+| SC4S_DEST_CITRIX_NETSCALER_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_DEST_CITRIX_NETSCALER_SPLUNK_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_SOURCE_CITRIX_NETSCALER_USEALT_DATE_FORMAT | no | Use "DDMMYYYY" format rather than "MMDDYYYY" |
+
+### Verification
+
+Use the following search to validate events are present
+
+```
+index=<asconfigured> sourcetype=cp_log
+```
+
+Verify timestamp, and host values match as expected   

mkdocs.yml

@@ -1,52 +1,53 @@
 site_name: Splunk Connect for Syslog
 
 nav:
-  - Home: 'index.md'
+  - Home: "index.md"
   - Getting Started:
-      - 'Read First': 'gettingstarted/index.md'
-      - 'Podman + systemd': 'gettingstarted/podman-systemd-general.md'
-      - 'Docker CE + systemd': 'gettingstarted/docker-systemd-general.md'
-      - 'Docker CE + Swarm': 'gettingstarted/docker-swarm-general.md'
-      - 'Docker CE + Swarm RHEL 7.7': 'gettingstarted/docker-swarm-rhel7.md'
-      - 'Bring your own Envionment': 'gettingstarted/byoe-rhel7.md'
-  - Configuration: 'configuration.md'
-  - Developing: 'docs/developing/index.md'
+      - "Read First": "gettingstarted/index.md"
+      - "Podman + systemd": "gettingstarted/podman-systemd-general.md"
+      - "Docker CE + systemd": "gettingstarted/docker-systemd-general.md"
+      - "Docker CE + Swarm": "gettingstarted/docker-swarm-general.md"
+      - "Docker CE + Swarm RHEL 7.7": "gettingstarted/docker-swarm-rhel7.md"
+      - "Bring your own Envionment": "gettingstarted/byoe-rhel7.md"
+  - Configuration: "configuration.md"
+  - Developing: "docs/developing/index.md"
   - Sources:
       - About: sources/index.md
       - Checkpoint: sources/Checkpoint/index.md
       - Cisco: sources/Cisco/index.md
-      - 'Common Event Format': sources/CommonEventFormat/index.md
+      - Citrix: sources/Citrix/index.md
+      - "Common Event Format": sources/CommonEventFormat/index.md
       - CyberArk: sources/CyberArk/index.md
       - Forcepoint: sources/Forcepoint/index.md
       - Fortinet: sources/Fortinet/index.md
       - Imperva: sources/Imperva/index.md
       - Juniper: sources/Juniper/index.md
       - Microfocus: sources/Microfocus/index.md
       - Nix: sources/nix/index.md
-      - 'Palo Alto Networks': sources/PaloaltoNetworks/index.md
-      - 'pfSense': sources/pfSense/index.md
+      - "Palo Alto Networks": sources/PaloaltoNetworks/index.md
+      - "pfSense": sources/pfSense/index.md
       - Proofpoint: sources/Proofpoint/index.md
       - Symantec: sources/Symantec/index.md
       - Ubiquiti: sources/Ubiquiti/index.md
       - VMware: sources/VMWare/index.md
       - Zscaler: sources/Zscaler/index.md
-  - 'Demo Lab': 'demo.md'
-  - Performance: 'performance.md'
-  - Troubleshooting: 'troubleshooting.md'
-  - 'Upgrading SC4S': 'upgrade.md'
+  - "Demo Lab": "demo.md"
+  - Performance: "performance.md"
+  - Troubleshooting: "troubleshooting.md"
+  - "Upgrading SC4S": "upgrade.md"
 
 markdown_extensions:
-    - toc:
-        permalink: True
-    - smarty
-    - fenced_code
-    - sane_lists
-    - codehilite
+  - toc:
+      permalink: True
+  - smarty
+  - fenced_code
+  - sane_lists
+  - codehilite
 
 theme:
-  name: 'material'
+  name: "material"
   palette:
-    primary: 'black'
-    accent: 'orange'
-  favicon: 'logo.png'
-  logo: 'logo.png'
+    primary: "black"
+    accent: "orange"
+  favicon: "logo.png"
+  logo: "logo.png"

package/etc/conf.d/filters/citrix/netscaler.conf

@@ -0,0 +1,28 @@
+filter f_citrix_netscaler {
+    match("^citrix_netscaler", value("fields.sc4s_vendor_product"));
+};
+filter f_citrix_netscaler_message {
+    message(
+        '^(<\d{1,3}>) (\d\d\/\d\d\/\d\d\d\d\:\d\d:\d\d:\d\d [^ ]{3}+) ([^ ]+) (.*)' 
+        flags(store-matches)
+    );
+};
+
+parser p_citrix_netscaler_date {
+{{- if (conv.ToBool (getenv "SC4S_SOURCE_CITRIX_NETSCALER_USEALT_DATE_FORMAT" "yes")) }}
+    #01/10/2001:01:01:01 GMT
+    date-parser(format('%d/%m/%Y:%H:%M:%S %Z')
+                template("$2"));
+{{ else }}
+    #10/01/2001:01:01:01 GMT
+    date-parser(format('%m/%d/%Y:%H:%M:%S %Z')
+                template("$2"));
+{{- end}}
+};
+
+rewrite r_citrix_netscaler_message {
+    set("citrix_netscaler" value("fields.sc4s_syslog_format"));
+    set("citrix_netscaler" value("fields.sc4s_vendor_product"));
+    set("$3" value("HOST"));
+    set("$4" value("MESSAGE"));
+};

package/etc/conf.d/log_paths/lp-citrix-netscaler.conf.tmpl

@@ -0,0 +1,41 @@
+# Citrix Netscaler ADC
+{{- /* The following provides a unique port source configuration if env var(s) are set */}}
+{{- $context := dict "port_id" "CITRIX_NETSCALER" "parser" "citrix_netscaler" }}
+{{- tmpl.Exec "t/source_network.t" $context }}
+
+log {
+    junction {
+{{- if or (or (getenv  (print "SC4S_LISTEN_CITRIX_NETSCALER_TCP_PORT")) (getenv  (print "SC4S_LISTEN_CITRIX_NETSCALER_UDP_PORT"))) (getenv  (print "SC4S_LISTEN_CITRIX_NETSCALER_TLS_PORT")) }}
+        channel {
+        # Listen on the specified dedicated port(s) for CITRIX_NETSCALER traffic
+            source (s_CITRIX_NETSCALER);
+            flags (final);
+	    };
+{{- end}}
+        channel {
+        # Listen on the default port (typically 514) for CITRIX_NETSCALER traffic
+            source (s_DEFAULT);
+            filter(f_citrix_netscaler);
+            flags(final);
+        };
+    };
+
+    rewrite {
+        set("citrix_netscaler", value("fields.sc4s_vendor_product"));
+        r_set_splunk_dest_default(sourcetype("citrix:netscaler:syslog"), index("netfw"))
+    };
+
+    parser {p_add_context_splunk(key("citrix_netscaler")); };
+    parser (compliance_meta_by_source);
+    rewrite { set("$(template ${.splunk.sc4s_template} $(template t_msg_only))" value("MSG")); };
+
+{{- if or (conv.ToBool (getenv "SC4S_DEST_SPLUNK_HEC_GLOBAL" "yes")) (conv.ToBool (getenv "SC4S_DEST_CITRIX_NETSCALER_HEC" "no")) }}
+    destination(d_hec);
+{{- end}}
+
+{{- if or (conv.ToBool (getenv "SC4S_ARCHIVE_GLOBAL" "no")) (conv.ToBool (getenv "SC4S_ARCHIVE_CITRIX_NETSCALER" "no")) }}
+    destination(d_archive);
+{{- end}}
+
+    flags(flow-control,final);
+};

package/etc/context_templates/splunk_index.csv.example

@@ -21,6 +21,7 @@
 #cisco_ise,index,netauth
 #cisco_nx_os,index,netops
 #cisco_ucm,index,main
+#citrix_netscaler,index,netfw
 #local_example,index,main
 #forcepoint_webprotect,index,netproxy
 #fortinet_fortios_event,index,netops

package/etc/context_templates/vendor_product_by_source.conf.example

@@ -2,6 +2,10 @@ filter f_test_test {
     host("testvp-*" type(glob))
     #or netmask(xxx.xxx.xxx.xxx/xx)
 };
+filter f_citrix_netscaler {
+    host("test_ctitrixns-*" type(glob))
+    #or netmask(xxx.xxx.xxx.xxx/xx)
+};
 filter f_juniper_idp {
     host("jnpidp-*" type(glob))
     #or netmask(xxx.xxx.xxx.xxx/xx)

package/etc/context_templates/vendor_product_by_source.csv.example

@@ -1,5 +1,6 @@
 f_test_test,sc4s_vendor_product,"test_test"
 f_cisco_meraki,sc4s_vendor_product,"cisco_meraki"
+f_citrix_netscaler,sc4s_vendor_product,"citrix_netscaler"
 f_infoblox,sc4s_vendor_product,"infoblox"
 f_juniper_nsm,sc4s_vendor_product,"juniper_nsm"
 f_juniper_nsm_idp,sc4s_vendor_product,"juniper_nsm_idp"

package/etc/go_templates/source_network.t

@@ -88,13 +88,20 @@ source s_{{ .port_id }} {
 {{ else if eq .parser "cisco_meraki_parser" }}
         parser (p_cisco_meraki);
         rewrite(set_rfc5424_epochtime);
+{{ else if eq .parser "citrix_netscaler" }}
+        parser(p_citrix_netscaler_date);
+        rewrite(r_citrix_netscaler_message);
 {{ else if eq .parser "cisco_ucm" }}
         parser (p_cisco_ucm_date);
         rewrite (r_cisco_ucm_message);
 {{ else if eq .parser "no_parse" }}
         rewrite(set_no_parse);
 {{ else }}
         if {
+            filter(f_citrix_netscaler_message);
+            parser(p_citrix_netscaler_date);
+            rewrite(r_citrix_netscaler_message);
+        } elif {
             filter(f_rfc5424_strict);
             parser {
                     syslog-parser(flags(syslog-protocol));

tests/test_citrix_netscaler.py

@@ -0,0 +1,37 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import datetime
+import random
+import pytz
+
+from jinja2 import Environment, environment
+
+from .sendmessage import *
+from .splunkutils import *
+import random
+
+env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+#<12> 01/10/2001:01:01:01 GMT netscaler ABC-D : SSLVPN HTTPREQUEST 1234567 : Context username@192.0.2.1 - SessionId: 12345- example.com User username : Group(s) groupname : Vserver a1b2:c3d4:e5f6:a7b8:c9d0:e1f2:a3b4:c5d6:123 - 01/01/2001:01:01:01 GMT GET file/path.gif - -
+def test_citrix_netscaler(record_property, setup_wordlist, setup_splunk, setup_sc4s):
+    host = "test-ctitrixns-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+    pid = random.randint(1000, 32000)
+
+    mt = env.from_string("{{ mark }} {% now 'utc', '%m/%d/%Y:%H:%M:%S' %} GMT {{ host }} ABC-D : SSLVPN HTTPREQUEST 1234567 : Context username@192.0.2.1 - SessionId: 12345- example.com User username : Group(s) groupname : Vserver a1b2:c3d4:e5f6:a7b8:c9d0:e1f2:a3b4:c5d6:123 - 01/01/2001:01:01:01 GMT GET file/path.gif - -\n")
+    message = mt.render(mark="<12>", host=host, pid=pid)
+
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    st = env.from_string("search index=netfw host={{ host }} sourcetype=\"citrix:netscaler:syslog\" | head 2")
+    search = st.render(host=host, pid=pid)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1