Skip to content

Commit

Permalink
Dyanamic reverse DNS
Browse files Browse the repository at this point in the history
  • Loading branch information
rfaircloth-splunk committed May 28, 2020
1 parent 789650b commit 3132063
Show file tree
Hide file tree
Showing 4 changed files with 136 additions and 51 deletions.
39 changes: 39 additions & 0 deletions package/etc/conf.d/conflib/_splunk/fix_dns.conf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
python {

"""
simple syslog-ng Python parser example
resolves IP to hostname
value pair names are hard-coded
"""
import re
import socket

class FixHostResolver(object):

def parse(self, log_message):
"""
Resolves IP to hostname
"""


# try to resolve the IP address
try:
ipaddr = log_message['HOST'].decode('utf-8')

resolved = socket.gethostbyaddr(ipaddr)
hostname = resolved[0]
log_message['HOST'] = hostname
except:
pass

# return True, other way message is dropped
return True

};


parser p_fix_host_resolver {
python(
class("FixHostResolver")
);
};
10 changes: 8 additions & 2 deletions package/etc/go_templates/source_network.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,8 +207,14 @@ source s_{{ .port_id }} {
};
{{ end }}
rewrite(r_set_splunk_default);
parser {
vendor_product_by_source();
if {
filter {
host('((^\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))\s*$)|(^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$))')
};
parser(p_fix_host_resolver);
};
parser {
vendor_product_by_source();
};

if {
Expand Down
6 changes: 3 additions & 3 deletions package/etc/syslog-ng.conf.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ options {
time_reopen (10);
log_fifo_size (10000);
chain_hostnames (yes);
use_dns ({{getenv "SC4S_GLOBAL_DNS_USE" "no"}});
use_dns (no);
use_fqdn (no);
dns-cache({{getenv "SC4S_GLOBAL_DNS_CACHE" "yes"}});
dns-cache(no);
create_dirs (no);
keep-hostname (no);
keep-hostname (yes);
create_dirs(yes);
dir_perm(0750);
stats-freq(30);
Expand Down
132 changes: 86 additions & 46 deletions tests/test_common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,22 @@


def test_defaultroute(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

dt = datetime.datetime.now()
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)

# Tune time functions
epoch = epoch[:-7]

mt = env.from_string(
"{{ mark }} {{ bsd }} {{ host }} test something else\n")
mt = env.from_string("{{ mark }} {{ bsd }} {{ host }} test something else\n")
message = mt.render(mark="<111>", bsd=bsd, host=host)

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string(
"search _time={{ epoch }} index=main host=\"{{ host }}\" sourcetype=\"sc4s:fallback\" PROGRAM=\"test\"")
'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="sc4s:fallback" PROGRAM="test"'
)
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -48,23 +47,22 @@ def test_defaultroute(record_property, setup_wordlist, setup_splunk, setup_sc4s)

@mark.skip()
def test_internal(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

dt = datetime.datetime.now()
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)

# Tune time functions
epoch = epoch[:-7]

mt = env.from_string(
"{{ mark }} {{ bsd }} {{ host }} sc4sdefault[0]: test\n")
mt = env.from_string("{{ mark }} {{ bsd }} {{ host }} sc4sdefault[0]: test\n")
message = mt.render(mark="<111>", bsd=bsd, host=host)

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string(
"search _time={{ epoch }} index=main NOT host=\"{{ host }}\" sourcetype=\"sc4s:events\"")
'search _time={{ epoch }} index=main NOT host="{{ host }}" sourcetype="sc4s:events"'
)
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -77,8 +75,7 @@ def test_internal(record_property, setup_wordlist, setup_splunk, setup_sc4s):


def test_fallback(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

dt = datetime.datetime.now()
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
Expand All @@ -92,7 +89,8 @@ def test_fallback(record_property, setup_wordlist, setup_splunk, setup_sc4s):
sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string(
"search _time={{ epoch }} index=main host=\"testvp-{{ host }}\" sourcetype=\"sc4s:fallback\"")
'search _time={{ epoch }} index=main host="testvp-{{ host }}" sourcetype="sc4s:fallback"'
)
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -103,26 +101,54 @@ def test_fallback(record_property, setup_wordlist, setup_splunk, setup_sc4s):

assert resultCount == 1


#


def test_fix_dns(record_property, setup_wordlist, setup_splunk, setup_sc4s):
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
pid = random.randint(1000, 32000)

dt = datetime.datetime.now()
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)

# Tune time functions
epoch = epoch[:-7]

mt = env.from_string("{{ mark }} {{ bsd }} 8.8.4.4 dnstest[{{ pid }}]: {{ host }}\n")
message = mt.render(mark="<111>", bsd=bsd, host=host, pid=pid)

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string("search _time={{ epoch }} host=dns.google index=osnix \"[{{ pid }}]\" {{ host }} sourcetype=\"nix:syslog\"")
search = st.render(epoch=epoch, pid=pid, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)

record_property("host", host)
record_property("resultCount", resultCount)
record_property("message", message)

assert resultCount == 1


def test_metrics(record_property, setup_wordlist, setup_splunk, setup_sc4s):

st = env.from_string(
'mcatalog values(metric_name) WHERE metric_name="syslogng.*" AND ("index"="*" OR "index"="_*") BY metric_name | fields metric_name')
'mcatalog values(metric_name) WHERE metric_name="syslogng.*" AND ("index"="*" OR "index"="_*") BY metric_name | fields metric_name'
)
search = st.render()

resultCount, eventCount = splunk_single(setup_splunk, search)

record_property("resultCount", resultCount)

assert resultCount != 0
assert resultCount != 0


def test_tz_guess(record_property, setup_wordlist, setup_splunk, setup_sc4s):

host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

dt = datetime.datetime.now()
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
Expand All @@ -131,14 +157,17 @@ def test_tz_guess(record_property, setup_wordlist, setup_splunk, setup_sc4s):
epoch = epoch[:-7]

mt = env.from_string(
"{{ mark }} {{ bsd }} {{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n")
message = mt.render(mark="<111>", bsd=bsd, host=host,
date=date, time=time, tzoffset=tzoffset)
"{{ mark }} {{ bsd }} {{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n"
)
message = mt.render(
mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset
)

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string(
"search _time={{ epoch }} index=netfw host=\"{{ host }}\" sourcetype=\"cisco:asa\" \"%ASA-3-003164\"")
'search _time={{ epoch }} index=netfw host="{{ host }}" sourcetype="cisco:asa" "%ASA-3-003164"'
)
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -152,33 +181,33 @@ def test_tz_guess(record_property, setup_wordlist, setup_splunk, setup_sc4s):

def test_tz_fix_hst(record_property, setup_wordlist, setup_splunk, setup_sc4s):

host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

# 10 minute offset (reserved for future use)
# dt = datetime.datetime.utcnow() - datetime.timedelta(hours=10, minutes=10)
# 10 minute offset (reserved for future use)
# dt = datetime.datetime.utcnow() - datetime.timedelta(hours=10, minutes=10)

# dt = datetime.datetime.utcnow() - datetime.timedelta(hours=10)
# dt = datetime.datetime.utcnow() - datetime.timedelta(hours=10)

# Set the date to Hawaii time
dt = datetime.datetime.now(datetime.timezone.utc) - \
datetime.timedelta(hours=10)
# Set the date to Hawaii time
dt = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=10)
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)

mt = env.from_string(
"{{ mark }} {{ bsd }} tzfhst-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n")
"{{ mark }} {{ bsd }} tzfhst-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n"
)
message = mt.render(mark="<111>", bsd=bsd, host=host)

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

# Add the 10 hours back to search for current time
# Add the 10 hours back to search for current time
dt = dt + datetime.timedelta(hours=10)
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)

epoch = epoch[:-7]

st = env.from_string(
"search _time={{ epoch }} index=netfw host=\"tzfhst-{{ host }}\" sourcetype=\"cisco:asa\"")
'search _time={{ epoch }} index=netfw host="tzfhst-{{ host }}" sourcetype="cisco:asa"'
)
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -192,26 +221,27 @@ def test_tz_fix_hst(record_property, setup_wordlist, setup_splunk, setup_sc4s):

def test_tz_fix_ny(record_property, setup_wordlist, setup_splunk, setup_sc4s):

host = "{}-{}".format(random.choice(setup_wordlist),
random.choice(setup_wordlist))
host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))

# 10 minute offset (reserved for future use)
# dt = datetime.datetime.now(pytz.timezone('America/New_York')) - datetime.timedelta(minutes=10)
# 10 minute offset (reserved for future use)
# dt = datetime.datetime.now(pytz.timezone('America/New_York')) - datetime.timedelta(minutes=10)

dt = datetime.datetime.now(pytz.timezone('America/New_York'))
dt = datetime.datetime.now(pytz.timezone("America/New_York"))
iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)

# Tune time functions
epoch = epoch[:-7]

mt = env.from_string(
"{{ mark }} {{ bsd }} tzfny-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n")
"{{ mark }} {{ bsd }} tzfny-{{ host }} : %ASA-3-003164: TCP access denied by ACL from 179.236.133.160/3624 to outside:72.142.18.38/23\n"
)
message = mt.render(mark="<111>", bsd=bsd, host=host)

sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])

st = env.from_string(
"search _time={{ epoch }} index=netfw host=\"tzfny-{{ host }}\" sourcetype=\"cisco:asa\"")
'search _time={{ epoch }} index=netfw host="tzfny-{{ host }}" sourcetype="cisco:asa"'
)
search = st.render(epoch=epoch, host=host)

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -223,10 +253,13 @@ def test_tz_fix_ny(record_property, setup_wordlist, setup_splunk, setup_sc4s):
assert resultCount == 1


def test_check_config_version(record_property, setup_wordlist, setup_splunk, setup_sc4s):

def test_check_config_version(
record_property, setup_wordlist, setup_splunk, setup_sc4s
):

st = env.from_string(
"search earliest=-50m@m latest=+1m@m index=main sourcetype=\"sc4s:events:startup:err\" \"Configuration file format is too old\" ")
'search earliest=-50m@m latest=+1m@m index=main sourcetype="sc4s:events:startup:err" "Configuration file format is too old" '
)
search = st.render()

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -236,10 +269,13 @@ def test_check_config_version(record_property, setup_wordlist, setup_splunk, set
assert resultCount == 0


def test_check_config_version_multiple(record_property, setup_wordlist, setup_splunk, setup_sc4s):
def test_check_config_version_multiple(
record_property, setup_wordlist, setup_splunk, setup_sc4s
):

st = env.from_string(
"search earliest=-50m@m latest=+1m@m index=main sourcetype=\"sc4s:events:startup:err\" \"you have multiple @version directives\" ")
'search earliest=-50m@m latest=+1m@m index=main sourcetype="sc4s:events:startup:err" "you have multiple @version directives" '
)
search = st.render()

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -248,11 +284,13 @@ def test_check_config_version_multiple(record_property, setup_wordlist, setup_sp

assert resultCount == 0


# This test fails on circle; Cisco ACS single test seems to trigger a utf8 error.
@mark.skip()
def test_check_utf8(record_property, setup_wordlist, setup_splunk, setup_sc4s):
st = env.from_string(
"search earliest=-50m@m latest=+1m@m index=main sourcetype=\"sc4s:events\" \"Input is valid utf8\"")
'search earliest=-50m@m latest=+1m@m index=main sourcetype="sc4s:events" "Input is valid utf8"'
)
search = st.render()

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand All @@ -261,10 +299,12 @@ def test_check_utf8(record_property, setup_wordlist, setup_splunk, setup_sc4s):

assert resultCount == 0


def test_check_sc4s_version(record_property, setup_wordlist, setup_splunk, setup_sc4s):

st = env.from_string(
"search earliest=-50m@m latest=+1m@m index=main sourcetype=\"sc4s:events:startup:out\" \"sc4s version=\" NOT \"UNKNOWN\"")
'search earliest=-50m@m latest=+1m@m index=main sourcetype="sc4s:events:startup:out" "sc4s version=" NOT "UNKNOWN"'
)
search = st.render()

resultCount, eventCount = splunk_single(setup_splunk, search)
Expand Down

0 comments on commit 3132063

Please sign in to comment.