Skip to content

Commit

Permalink
Filter for Juniper devices (#37)
Browse files Browse the repository at this point in the history
Also changed:
1.  default standard template to include $MSGHDR
2.  Added template for RFC5424 structured data
3. Added fallback template in hec destination
  • Loading branch information
mbonsack authored and Ryan Faircloth committed Aug 6, 2019
1 parent b6b0b05 commit f472608
Show file tree
Hide file tree
Showing 9 changed files with 385 additions and 12 deletions.
2 changes: 1 addition & 1 deletion .env.template
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ SPLUNK_HEC_STATSURL=https://splunk:8088/services/collector/event
SPLUNK_CONNECT_METHOD=hec
SPLUNK_DEFAULT_INDEX=main
SPLUNK_METRICS_INDEX=metrics
SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download
SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/2757/release/6.1.1/download,https://splunkbase.splunk.com/app/3245/release/1.0/download,https://splunkbase.splunk.com/app/1620/release/3.4.0/download,https://splunkbase.splunk.com/app/1467/release/2.5.8/download,https://splunkbase.splunk.com/app/2846/release/1.6.0/download,https://splunkbase.splunk.com/app/2847/release/1.2.0/download
SPLUNKBASE_USERNAME=username
SPLUNKBASE_PASSWORD=password
4 changes: 2 additions & 2 deletions .idea/misc.xml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion .idea/splunk-connect-for-syslog.iml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 6 additions & 6 deletions package/etc/conf.d/blocks/b_parsers.conf
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ block rewrite r_set_splunk(sourcetype(`splunk-sourcetype`)
set("`vendor`", value(".splunk.vendor"));
set("`category`", value(".splunk.category"));
set("`filename`", value(".splunk.filename"));
set($FACILITY, value("fields.facility"));
set($LEVEL, value("fields.severity"));
set($LOGHOST, value("fields.syslog-server"));
set($FACILITY, value("fields.syslog_facility"));
set($LEVEL, value("fields.syslog_severity"));
set($LOGHOST, value("fields.syslog_server"));
};

#Used when for normal filters index and sourcetype must be set in a prior re-write or context
Expand All @@ -63,7 +63,7 @@ block rewrite r_set_splunk_basic(
set("`vendor`", value(".splunk.vendor"));
set("`category`", value(".splunk.category"));
set("`filename`", value(".splunk.filename"));
set($FACILITY, value("fields.facility"));
set($LEVEL, value("fields.severity"));
set($LOGHOST, value("fields.syslog-server"));
set($FACILITY, value("fields.syslog_facility"));
set($LEVEL, value("fields.syslog_severity"));
set($LOGHOST, value("fields.syslog_server"));
};
2 changes: 1 addition & 1 deletion package/etc/conf.d/destinations/d_destinations.conf
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ destination d_hec {
source=${HOST_FROM}
sourcetype=${.splunk.sourcetype}
index=${.splunk.index}
event=$(template ${.splunk.template})
event=$(template ${.splunk.template} $(template `splunk-default-template`))
fields.*)')
);
};
Expand Down
65 changes: 65 additions & 0 deletions package/etc/conf.d/filters/juniper_junos.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# ===============================================================================================
# Juniper Standard and Structured logging
# ===============================================================================================

log {

source(s_default-ports);

# For newer (structured) format; RFC 5424 compliant. The existing TA will not work for this format; these filters
# are included for future TA development. The exception is NSM IDP, for which we're awaiting a data
# sample to confirm that it follows the RFC 5424 format.
# If it is determined that NSM IDP ("syslog@juniper.net") messages are indeed 3164 format, remove the check for
# the "syslog@juniper.net" message below. It will then be caught by the standard section following.

junction {
channel {
filter { tags("juniper_structured") or message('junos@2636') or message('syslog@juniper.net'); };
parser { syslog-parser(flags(syslog-protocol)); };
if (tags("juniper_JunOS_IDP") or program('RT_IDP')) {
parser {p_add_context_splunk(key("juniper:junos:idp:structured")); };
} elif (tags("juniper_JunOS_Firewall") or program('RT_FLOW|RT_IDS|RT_UTM')) {
parser {p_add_context_splunk(key("juniper:junos:firewall:structured")); };
}
elif (tags("juniper_idp") or program('Jnpr')) {
parser {p_add_context_splunk(key("juniper:idp:structured")); };
}
else {
parser {p_add_context_splunk(key("juniper:structured")); };
};
rewrite { r_set_splunk_basic(template("t_hdr_sdata_msg"))};
destination(d_hec); #--HEC--
flags(final);
};

# For legacy "standard" format. The existing Juniper TA will work for these data sources.
# Remove the message filter below when tag support is implemented.

channel {
filter { tags("juniper_standard") or message('RT_IDP:|RT_FLOW:|RT_IDS:|RT_UTM:|Juniper:|traffic\,\s+traffic\s+log|syslog@juniper.net|predefined\,\s+|NetScreen\s+device_id\=') };
# filter { tags("juniper_standard") };
parser { syslog-parser(time-zone(`default-timezone`)); };
if (program('RT_IDP')) {
parser {p_add_context_splunk(key("juniper:junos:idp")); };
} elif (program('RT_FLOW|RT_IDS|RT_UTM')) {
parser {p_add_context_splunk(key("juniper:junos:firewall")); };
} elif (program('Juniper')) {
parser {p_add_context_splunk(key("juniper:sslvpn")); };
} elif (message('traffic\,\s+traffic\s+log')) {
parser {p_add_context_splunk(key("juniper:nsm")); };
} elif (message('syslog@juniper.net')) {
parser {p_add_context_splunk(key("juniper:idp")); };
} elif (message('predefined\,\s+')) {
parser {p_add_context_splunk(key("juniper:nsm:idp")); };
} elif (message('NetScreen\s+device_id\=')) {
parser {p_add_context_splunk(key("juniper:netscreen")); };
}
else {
parser {p_add_context_splunk(key("juniper:legacy")); };
};
rewrite { r_set_splunk_basic(template("t_standard")) };
destination(d_hec); #--HEC--
flags(final);
};
};
};
10 changes: 9 additions & 1 deletion package/etc/conf.d/templates/t_templates.conf
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
# ===============================================================================================

template t_standard {
template("${ISODATE} ${HOST} ${MESSAGE}");
template("${DATE} ${HOST} ${MSGHDR}${MESSAGE}");
};

# ===============================================================================================
Expand All @@ -35,6 +35,14 @@ template t_hdr_msg {
template("${MSGHDR}${MESSAGE}");
};

# ===============================================================================================
# Message Header, Structured Data (from RFC5424 parse) and Message; for Juniper
# ===============================================================================================

template t_hdr_sdata_msg {
template("${MSGHDR}${MSGID} ${SDATA} ${MESSAGE}");
};

# ===============================================================================================
# JSON; for JSON pretty-printing
# ===============================================================================================
Expand Down
24 changes: 24 additions & 0 deletions package/etc/context/splunk_index.csv
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,30 @@ fgt_traffic,sourcetype,fgt_traffic
fgt_traffic,index,main
fgt_utm,sourcetype,fgt_utm
fgt_utm,index,main
juniper:structured,index,main
juniper:structured,sourcetype,juniper:structured
juniper:junos:firewall:structured,index,main
juniper:junos:firewall:structured,sourcetype,juniper:junos:firewall:structured
juniper:junos:idp:structured,index,main
juniper:junos:idp:structured,sourcetype,juniper:junos:idp:structured
juniper:junos:firewall,index,main
juniper:junos:firewall,sourcetype,juniper:junos:firewall
juniper:junos:idp,index,main
juniper:junos:idp,sourcetype,juniper:junos:idp
juniper:nsm:idp,index,main
juniper:nsm:idp,sourcetype,juniper:nsm:idp
juniper:sslvpn,index,main
juniper:sslvpn,sourcetype,juniper:sslvpn
juniper:netscreen,index,main
juniper:netscreen,sourcetype,netscreen:firewall
juniper:idp,index,main
juniper:idp,sourcetype,juniper:idp
juniper:idp:structured,index,main
juniper:idp:structured,sourcetype,juniper:idp:structured
juniper:nsm,index,main
juniper:nsm,sourcetype,juniper:nsm
juniper:legacy,index,main
juniper:legacy,sourcetype,juniper:legacy
pan:traffic,index,main
pan:threat,index,main
pan:system,index,main
Expand Down
Loading

0 comments on commit f472608

Please sign in to comment.