Merge pull request #288 from splunk/fix/5424check

Fixes #287

package/etc/conf.d/conflib/_common/syslog_format.conf

@@ -1,11 +1,11 @@
 filter f_rfc5424_strict{
-    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>(?<FULLDATE>(?<FULLDATEYEAR>\d{4})-(?<FULLDATEMONTH>\d\d)-(?<FULLDATEDAY>\d\d))T(?<FULLTIME>(?<PARTIALTIME>(?<TIMEHOUR>[0-2]\d):(?<TIMEMINUTE>[0-5]\d):(?<TIMESECOND>[0-5]\d)(?:.(?<TIMESECFRAC>\d{1,6}))?)(?<TIMEOFFSET>Z|(?<TIMENUMOFFSET>[+\-][0-2]\d:[0-5]\d))))))');
-};
+    message('^\<(?<PRI>\d+)\>(?<VERSION>\d{1,2})? (?<YEAR>\d+)-(?<MONTH>\d+)-(?<DAY>\d+)T(?<HOUR>\d+):(?<MINUTE>\d+):(?<SECOND>\d+)(?:\.(?<MILLISECONDS>\d+))?(?<OFFSET>Z|[\+-] *\d+:\d+) (?<HOSTNAME>(-)|[^ ]+) (?<APPNAME>(?:-)|\b\w+\b) (?<PROCID>(?:-)|\b\w+\b) (?<MSGID>(?:-)|\b\w+\b) *(?<STRUCDATA>(?:-)|\[.*?\]) *(?<MSG>(?:-)|\b.*)?$');
+    };
 filter f_rfc5424_noversion{
     message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>) ?(?<TIMESTAMP>(?<FULLDATE>(?<FULLDATEYEAR>\d{4})-(?<FULLDATEMONTH>\d\d)-(?<FULLDATEDAY>\d\d))T(?<FULLTIME>(?<PARTIALTIME>(?<TIMEHOUR>[0-2]\d):(?<TIMEMINUTE>[0-5]\d):(?<TIMESECOND>[0-5]\d)(?:.(?<TIMESECFRAC>\d{1,6}))?)(?<TIMEOFFSET>Z|(?<TIMENUMOFFSET>[+\-][0-2]\d:[0-5]\d))))))');
 };
 filter f_rfc3164_version{
-    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) (?<TIMESTAMP>[A-Za-z]{3} \d\d \d\d:\d\d:\d\d) (?<FROMHOST>[^ ]+) ))');
+    message('^(?<SYSLOGMSG>(?<HEADER>(?<PRI><\d{1,3}>)(?<VERSION>[1-9][0-9]?) ))');
 };
 rewrite set_rfc5424_strict{
     set("rfc5424_strict" value("fields.sc4s_syslog_format"));

package/etc/go_templates/source_network.t

@@ -92,18 +92,21 @@ source s_{{ .port_id }} {
         rewrite(set_no_parse);
 {{ else }}
         if {
+            filter(f_rfc5424_strict);
+            parser {
+                    syslog-parser(flags(syslog-protocol));
+                };
+            rewrite(set_rfc5424_strict);
+        } elif {
+            parser (p_cisco_meraki);
+            rewrite(set_rfc5424_epochtime);
+        } elif {
             filter(f_rfc3164_version);
             rewrite(set_rfc3164_no_version_string);
             parser {
                     syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));
                 };
             rewrite(set_rfc3164_version);
-        } elif {
-            filter(f_rfc5424_strict);
-            parser {
-                    syslog-parser(flags(syslog-protocol));
-                };
-            rewrite(set_rfc5424_strict);
         } elif {
             filter(f_rfc5424_noversion);
             parser {
@@ -113,9 +116,6 @@ source s_{{ .port_id }} {
         } elif {
             parser {cisco-parser()};
             rewrite(set_cisco_ios);
-        } elif {
-            parser (p_cisco_meraki);
-            rewrite(set_rfc5424_epochtime);
         } else {
             parser {
                 syslog-parser(time-zone({{- getenv "SC4S_DEFAULT_TIMEZONE" "GMT"}}) flags(guess-timezone));

tests/test_cyberark.py

@@ -12,6 +12,26 @@
 
 env = Environment(extensions=['jinja2_time.TimeExtension'])
 
+#<5>1 2020-01-24T22:53:03Z REDACTEDHOSTNAME CEF:0|Cyber-Ark|Vault|10.9.0000|22|CPM Verify Password|5|act="CPM Verify Password" suser=PasswordManager fname=Root\Operating System-OBO-ISSO-Windows-Domain-Account-redacted dvc= shost=10.0.0.10 dhost= duser=redacted externalId= app= reason= cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2="re-dact-ted" cs3Label="Device Type" cs3="Operating System" cs4Label="Database" cs4= cs5Label="Other info" cs5= cn1Label="Request Id" cn1= cn2Label="Ticket Id" cn2="VerificationPeriod"  msg="VerificationPeriod"
+def test_cyberark_epv_5424(record_property, setup_wordlist, setup_splunk):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    mt = env.from_string(
+        "{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%S' %}Z {{ host }} CEF:0|Cyber-Ark|Vault|9.20.0000|7|Logon|5|act=\"Logon\" suser=##USER_NAME## fname= dvc= shost=##SOURCE_IP## dhost= duser= externalId= app= reason= cs1Label=\"Affected User Name\" cs1= cs2Label=\"Safe Name\" cs2= cs3Label=\"Device Type\" cs3=11111 cs4Label=\"Database\" cs4=222222 cs5Label=\"Other info\" cs5= cn1Label=\"Request Id\" cn1= cn2Label=\"Ticket Id\" cn2=  msg=\n")
+    message = mt.render(mark="<111>", host=host)
+
+    sendsingle(message)
+
+    st = env.from_string("search index=netauth host=\"{{ host }}\" sourcetype=\"cyberark:epv:cef\"| head 2")
+    search = st.render(host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
 
 #<190>Jul 27 23:31:58 VAULT CEF:0|Cyber-Ark|Vault|9.20.0000|7|Logon|5|act="Logon" suser=##USER_NAME## fname= dvc= shost=##SOURCE_IP## dhost= duser= externalId= app= reason= cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2= cs3Label="Device Type" cs3=11111 cs4Label="Database" cs4=222222 cs5Label="Other info" cs5= cn1Label="Request Id" cn1= cn2Label="Ticket Id" cn2=  msg=
 def test_cyberark_epv(record_property, setup_wordlist, setup_splunk):

tests/test_vmware.py

@@ -41,12 +41,12 @@ def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk):
     host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
     pid = random.randint(1000, 32000)
 
-    mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSXV {{ pid }} - [nsxv@6876 comp=\"nsx-manager\" subcomp=\"manager\"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704\n")
+    mt = env.from_string("{{ mark }}1 {% now 'utc', '%Y-%m-%dT%H:%M:%SZ' %} {{ host }} NSX - SYSTEM [nsx@6876 comp=\"nsx-manager\" errorCode=\"MP4039\" subcomp=\"manager\"] Connection verification failed for broker '10.160.108.196'. Marking broker unhealthy.\n")
     message = mt.render(mark="<144>", host=host, pid=pid)
 
     sendsingle(message)
 
-    st = env.from_string("search index=main host={{ host }} PID={{ pid }} sourcetype=\"vmware:vsphere:nsx\" | head 2")
+    st = env.from_string("search index=main host={{ host }} sourcetype=\"vmware:vsphere:nsx\" | head 2")
     search = st.render(host=host, pid=pid)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)