Skip to content

Commit

Permalink
Fixes #70 (#71)
Browse files Browse the repository at this point in the history
  • Loading branch information
Ryan Faircloth authored and GitHub committed Aug 31, 2019
1 parent c294fcb commit ae7d5c2
Show file tree
Hide file tree
Showing 22 changed files with 180 additions and 109 deletions.
54 changes: 52 additions & 2 deletions docs/sources.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,13 @@
| cisco:pix | Not supported |
| cisco:fwsm | Not supported |

### Sourcetype and Index Configuration

| key | sourcetype | index | notes |
|----------------|----------------|----------------|----------------|
| cisco_asa | cisco:asa | netfw | none |


### Filter type

MSG Parse: This filter parses message content
Expand Down Expand Up @@ -50,17 +57,26 @@ Verify timestamp, and host values match as expected
| NX-OS Manual | https://www.cisco.com/c/en/us/td/docs/switches/datacenter/nexus9000/sw/6-x/system_management/configuration/guide/b_Cisco_Nexus_9000_Series_NX-OS_System_Management_Configuration_Guide/sm_5syslog.html|
| Cisco ACI | https://community.cisco.com/legacyfs/online/attachments/document/technote-aci-syslog_external-v1.pdf |
| Cisco WLC & AP | https://www.cisco.com/c/en/us/support/docs/wireless/4100-series-wireless-lan-controllers/107252-WLC-Syslog-Server.html#anc8 |

### Sourcetypes

| sourcetype | notes |
|----------------|---------------------------------------------------------------------------------------------------------|
| cisco:ios | This source type is also used for NX-OS, ACI and WLC product lines |

### Sourcetype and Index Configuration

| key | sourcetype | index | notes |
|----------------|----------------|----------------|----------------|
| cisco_ios | cisco:ios | netops | none |
| cisco_nx_os | cisco:ios | netops | none |

### Filter type

* Cisco IOS products can be identified by message parsing alone
* Cisco NX OS, WLC, and ACI products must be identified by host or ip assignment update the filter `f_cisco_nx_os` as required


### Setup and Configuration

* Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer.
Expand Down Expand Up @@ -112,11 +128,22 @@ Verify timestamp, and host values match as expected

| sourcetype | notes |
|----------------|---------------------------------------------------------------------------------------------------------|
| fgt_log | The catch all sourcetype is not used |
| fgt_log | The catch all sourcetype is not used |
| fgt_traffic | None |
| fgt_utm | None |
| fgt_event | None


### Sourcetype and Index Configuration

| key | sourcetype | index | notes |
|----------------|----------------|----------------|----------------|
| fortinet_fortios_traffic | fgt_traffic | netops | none |
| fortinet_fortios_utm | fgt_utm | netids | none |
| fortinet_fortios_event | fgt_event | netops | none |
| fortinet_fortios_log | fgt_log | netops | none |


### Filter type

MSG Parse: This filter parses message content
Expand Down Expand Up @@ -204,6 +231,22 @@ Verify timestamp, and host values match as expected
| pan:log | None |
| pan:traffic | None |
| pan:threat | None |
| pan:system | None |
| pan:config | None |
| pan:hipwatch | None |
| pan:correlation | None |

### Sourcetype and Index Configuration

| key | sourcetype | index | notes |
|----------------|----------------|----------------|----------------|
| pan_log | pan:log | netops | none |
| pan_traffic | pan:traffic | netfw | none |
| pan_threat | pan:threat | netproxy | none |
| pan_system | pan:system | netops | none |
| pan_config | pan:config | netops | none |
| pan_hipwatch | pan:hipwatch | netops | none |
| pan_correlation | pan:correlation | netops | none |

### Filter type

Expand Down Expand Up @@ -240,7 +283,14 @@ index=<asconfigured> sourcetype=pan:*| stats count by host

| sourcetype | notes |
|----------------|---------------------------------------------------------------------------------------------------------|
| bluecoat:proxysg:access:kv | Requires version 3.6 |
| bluecoat:proxysg:access:kv | Requires version TA 3.6 |

### Sourcetype and Index Configuration

| key | sourcetype | index | notes |
|----------------|----------------|----------------|----------------|
| bluecoat_proxy | bluecoat:proxysg:access:kv | netops | none |


### Filter type

Expand Down
16 changes: 11 additions & 5 deletions package/etc/conf.d/conflib/_splunk/splunkfields.conf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#Used to set index fields we will always use
rewrite r_set_splunk_basic {
#Used to set indexed fields we will always use to global defaults
rewrite r_set_splunk_default {
set("`index`", value(".splunk.index"));

set("`sourcetype`", value(".splunk.sourcetype"));
Expand All @@ -9,10 +9,16 @@ rewrite r_set_splunk_basic {
set($LOGHOST, value("fields.sc4s_syslog_server"));
set($SOURCEIP, value("fields.sc4s_fromhostip"));
};
#used by each log-path to set index and sourcetype which may be
#overridden by user defined values
block rewrite r_set_splunk_dest_default(
index()
sourcetype()
) {
set("`index`", value(".splunk.index"));
set("`sourcetype`", value(".splunk.sourcetype"));
};

#rewrite

#Used by fallback to log everything we can
block rewrite r_set_splunk(
template(`splunk-default-template`)
) {
Expand Down
3 changes: 2 additions & 1 deletion package/etc/conf.d/log_paths/P_rfc3164-juniper_nsm_idp.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ log {
filter(f_is_rfc3164);
filter(f_juniper_nsm_idp);

rewrite {r_set_splunk_dest_default(sourcetype("juniper:nsm:idp"), index("netids"))};
parser {
p_add_context_splunk(key("juniper:nsm:idp"));
p_add_context_splunk(key("juniper_idp"));
};
rewrite {
r_set_splunk(template("t_standard"))
Expand Down
8 changes: 6 additions & 2 deletions package/etc/conf.d/log_paths/internal.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ log {

if (match("^Log statistics; " value("MESSAGE"))) {

parser {p_add_context_splunk(key("syslog-ng:metrics")); };
rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:metrics"), index("em_metrics")) };

parser {p_add_context_splunk(key("syslog-ng_metrics")); };
rewrite {
subst('(?:Log statistics; )?(?<Type>[^= ]+)=\x27(?<SourceName>[^\(]+)\((?<SourceId>[^,\)]+)(?:,(?<SourceInstance>[^,]+),(?<State>[^\)]+))?\)\=(?<Number>\d+)\x27(?:, )?',
'
Expand All @@ -28,7 +30,9 @@ log {
};
destination(d_hecmetrics); #--HEC--
} else {
parser {p_add_context_splunk(key("syslog-ng:events")); };
rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:events"), index("_internal"))};

parser {p_add_context_splunk(key("syslog-ng_events")); };
rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC--
destination(d_hec); #--HEC--
};
Expand Down
3 changes: 2 additions & 1 deletion package/etc/conf.d/log_paths/p_rfc3164-cisco-nx-os.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ log {
source(s_default-ports);
filter(f_cisco_nx_os);

rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"))};
parser {
p_add_context_splunk(key("cisco:nx-os"));
p_add_context_splunk(key("cisco_nx_os"));
};
rewrite {
r_set_splunk(template("t_hdr_msg"))
Expand Down
3 changes: 2 additions & 1 deletion package/etc/conf.d/log_paths/p_rfc3164-cisco_asa.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ log {
filter(f_cisco_asa);
#set the source type based on program field and lookup index from the splunk context csv

parser {p_add_context_splunk(key("cisco:asa")); };
rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))};
parser {p_add_context_splunk(key("cisco_asa")); };

#Using the 5424 parser the message content is all we need
rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC--
Expand Down
3 changes: 2 additions & 1 deletion package/etc/conf.d/log_paths/p_rfc3164-cisco_ios.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@ log {
filter(f_cisco_ios);


rewrite { r_set_splunk_dest_default(sourcetype("cisco:ios"), index("netops"))};
parser {
p_add_context_splunk(key("cisco:ios"));
p_add_context_splunk(key("cisco_ios"));
};
rewrite {
r_set_splunk(template("t_msg_only")) #--HEC--
Expand Down
14 changes: 10 additions & 4 deletions package/etc/conf.d/log_paths/p_rfc3164-fortinet_fortios.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,20 @@ log {
rewrite { set("${.kv.devname}", value("HOST")); };

#set the source type based on program field and lookup index from the splunk context csv


if (match("traffic" value(".kv.type"))) {
parser {p_add_context_splunk(key("fgt_traffic")); };
rewrite { r_set_splunk_dest_default(sourcetype("fgt_traffic"), index("netfw"))};
parser {p_add_context_splunk(key("fortinet_fortios_traffic")); };
} elif (match("utm" value(".kv.type"))) {
parser {p_add_context_splunk(key("fgt_utm")); };
rewrite { r_set_splunk_dest_default(sourcetype("fgt_utm"), index("netids"))};
parser {p_add_context_splunk(key("fortinet_fortios_utm")); };
} elif (match("event" value(".kv.type"))) {
parser {p_add_context_splunk(key("fgt_event")); };
rewrite { r_set_splunk_dest_default(sourcetype("fgt_event"), index("netops"))};
parser {p_add_context_splunk(key("fortinet_fortios_event")); };
} else {
parser {p_add_context_splunk(key("fgt_log")); };
rewrite { r_set_splunk_dest_default(sourcetype("fgt_log"), index("netops"))};
parser {p_add_context_splunk(key("fortinet_fortios_log")); };
};

#rewrite the final message for splunk json
Expand Down
3 changes: 2 additions & 1 deletion package/etc/conf.d/log_paths/p_rfc3164-juniper-idp.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ log {
filter(f_is_rfc3164);
filter(f_juniper_idp);

rewrite { r_set_splunk_dest_default(sourcetype("juniper:idp"), index("netids"))};
parser {
p_add_context_splunk(key("juniper:idp"));
p_add_context_splunk(key("juniper_idp"));
};
rewrite {
r_set_splunk(template("t_standard"))
Expand Down
20 changes: 15 additions & 5 deletions package/etc/conf.d/log_paths/p_rfc3164-juniper-junos.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,23 @@ log {
filter(f_juniper_junos_legacy);

if (program('RT_IDP')) {
parser {p_add_context_splunk(key("juniper:junos:idp")); };
} elif (program('RT_FLOW|RT_IDS|RT_UTM')) {
parser {p_add_context_splunk(key("juniper:junos:firewall")); };
rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp"), index("netids"))};
parser {p_add_context_splunk(key("juniper_idp")); };
} elif (program('RT_FLOW')) {
rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netfw"))};
parser {p_add_context_splunk(key("juniper_junos_flow")); };
} elif (program('RT_IDS')) {
rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netids"))};
parser {p_add_context_splunk(key("juniper_junos_ids")); };
} elif (program('RT_UTM')) {
rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall"), index("netids"))};
parser {p_add_context_splunk(key("juniper_junos_utm")); };
} elif (program('Juniper')) {
parser {p_add_context_splunk(key("juniper:sslvpn")); };
rewrite { r_set_splunk_dest_default(sourcetype("juniper:sslvpn"), index("netfw"))};
parser {p_add_context_splunk(key("juniper_sslvpn")); };
} else {
parser {p_add_context_splunk(key("juniper:legacy")); };
rewrite { r_set_splunk_dest_default(sourcetype("juniper:legacy"), index("netops"))};
parser {p_add_context_splunk(key("juniper_legacy")); };
};

rewrite { r_set_splunk(template("t_standard")) };
Expand Down
4 changes: 3 additions & 1 deletion package/etc/conf.d/log_paths/p_rfc3164-juniper_netscreen.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ log {
source(s_default-ports);
filter(f_juniper_netscreen);

rewrite { r_set_splunk_dest_default(sourcetype("netscreen:firewall"), index("netfw"))};

parser {
p_add_context_splunk(key("juniper:netscreen"));
p_add_context_splunk(key("juniper_netscreen"));
};
rewrite {
r_set_splunk(template("t_standard"))
Expand Down
4 changes: 3 additions & 1 deletion package/etc/conf.d/log_paths/p_rfc3164-juniper_nsm.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ log {
filter(f_is_rfc3164);
filter(f_juniper_nsm);

rewrite { r_set_splunk_dest_default(sourcetype("juniper:nsm"), index("netfw"))};

parser {
p_add_context_splunk(key("juniper:nsm"));
p_add_context_splunk(key("juniper_nsm"));
};
rewrite {
r_set_splunk(template("t_standard"))
Expand Down
24 changes: 16 additions & 8 deletions package/etc/conf.d/log_paths/p_rfc3164-paloalto-panos.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,21 +33,29 @@ log {

#set the source type based on program field and lookup index from the splunk context csv
if (message(',\d+,THREAT')) {
parser {p_add_context_splunk(key("pan:threat")); };
rewrite { r_set_splunk_dest_default(sourcetype("pan:threat"), index("netproxy"))};
parser {p_add_context_splunk(key("pan_threat")); };
} elif (message(',\d+,TRAFFIC')) {
parser {p_add_context_splunk(key("pan:traffic")); };
rewrite { r_set_splunk_dest_default(sourcetype("pan:traffic"), index("netfw"))};
parser {p_add_context_splunk(key("pan_traffic")); };
} elif (message(',\d+,SYSTEM')) {
parser {p_add_context_splunk(key("pan:system")); };
rewrite { r_set_splunk_dest_default(sourcetype("pan:system"), index("netops"))};
parser {p_add_context_splunk(key("pan_system")); };
} elif (message(',\d+,CONFIG')) {
parser {p_add_context_splunk(key("pan:config")); };
rewrite { r_set_splunk_dest_default(sourcetype("pan:config"), index("netops"))};
parser {p_add_context_splunk(key("pan_config")); };
} elif (message(',\d+,HIPWATCH')) {
parser {p_add_context_splunk(key("pan:hipwatch")); };
rewrite { r_set_splunk_dest_default(sourcetype("pan:hipwatch"), index("main"))};
parser {p_add_context_splunk(key("pan_hipwatch")); };
} elif (message(',\d+,CORRELATION')) {
parser {p_add_context_splunk(key("pan:correlation")); };
rewrite { r_set_splunk_dest_default(sourcetype("pan:correlation"), index("main"))};
parser {p_add_context_splunk(key("pan_correlation")); };
} elif (message(',\d+,USERID')) {
parser {p_add_context_splunk(key("pan:userid")); };
rewrite { r_set_splunk_dest_default(sourcetype("pan:userid"), index("netauth"))};
parser {p_add_context_splunk(key("pan_userid")); };
} else {
parser {p_add_context_splunk(key("pan:log")); };
rewrite { r_set_splunk_dest_default(sourcetype("pan:log"), index("netops"))};
parser {p_add_context_splunk(key("pan_log")); };
};

#rewrite the final message for splunk json
Expand Down
3 changes: 2 additions & 1 deletion package/etc/conf.d/log_paths/p_rfc5424_noversion-cisco_asa.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ log {

#set the source type based on program field and lookup index from the splunk context csv

parser {p_add_context_splunk(key("cisco:asa")); };
rewrite { r_set_splunk_dest_default(sourcetype("cisco:asa"), index("netfw"))};
parser {p_add_context_splunk(key("cisco_asa")); };

#Using the 5424 parser the message content is all we need
rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC--
Expand Down
4 changes: 3 additions & 1 deletion package/etc/conf.d/log_paths/p_rfc_5424_noversion-symantec-proxy.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ log {

#set the source type based on program field and lookup index from the splunk context csv

parser {p_add_context_splunk(key("bluecoat:proxysg:access:kv")); };
rewrite { r_set_splunk_dest_default(sourcetype("bluecoat:proxysg:access:kv"), index("netproxy"))};

parser {p_add_context_splunk(key("bluecoat_proxy")); };

#Using the 5424 parser the message content is all we need
rewrite {r_set_splunk(template("t_msg_only")) }; #--HEC--
Expand Down
23 changes: 16 additions & 7 deletions package/etc/conf.d/log_paths/p_rfc_5424_strict-juniper-junos.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,24 @@ log {
filter(f_juniper_junos);

if (program('RT_IDP')) {
parser {p_add_context_splunk(key("juniper:junos:idp:structured")); };
} elif (program('RT_FLOW|RT_IDS|RT_UTM')) {
parser {p_add_context_splunk(key("juniper:junos:firewall:structured")); };
}
elif (program('Jnpr')) {
parser {p_add_context_splunk(key("juniper:idp:structured")); };
rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:idp:structured"), index("netids"))};
parser {p_add_context_splunk(key("juniper_idp")); };
} elif (program('RT_FLOW')) {
rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netfw"))};
parser {p_add_context_splunk(key("juniper_junos_flow")); };
} elif (program('RT_IDS')) {
rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netids"))};
parser {p_add_context_splunk(key("juniper_junos_ids")); };
} elif (program('RT_UTM')) {
rewrite { r_set_splunk_dest_default(sourcetype("juniper:junos:firewall:structured"), index("netids"))};
parser {p_add_context_splunk(key("juniper_junos_utm")); };
} elif (program('Jnpr')) {
rewrite { r_set_splunk_dest_default(sourcetype("juniper:idp:structured"), index("netids"))};
parser {p_add_context_splunk(key("juniper_junos_idp")); };
}
else {
parser {p_add_context_splunk(key("juniper:structured")); };
rewrite { r_set_splunk_dest_default(sourcetype("juniper:structured"), index("netops"))};
parser {p_add_context_splunk(key("juniper_structured")); };
};

rewrite { r_set_splunk(template("t_hdr_sdata_msg"))};
Expand Down
3 changes: 2 additions & 1 deletion package/etc/conf.d/log_paths/zfallback.conf
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
log {
source(s_default-ports);

rewrite { r_set_splunk_dest_default(sourcetype("syslog-ng:fallback"), index("main"))};
parser {
p_add_context_splunk(key("syslog-ng:fallback"));
p_add_context_splunk(key("syslog-ng_fallback"));
};

rewrite { r_set_splunk(template("t_JSON"))}; #--HEC--
Expand Down
2 changes: 1 addition & 1 deletion package/etc/conf.d/sources/network.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ source s_default-ports {
#TODO: #60 Remove this function with enhancement
rewrite(set_metadata_presume);
rewrite(set_rfcnonconformant);
rewrite(r_set_splunk_basic);
rewrite(r_set_splunk_default);

if {
filter(f_rfc5424_strict);
Expand Down

0 comments on commit ae7d5c2

Please sign in to comment.