Improve CI performance and reliability (#219)

Due to container network occasionally a test will fail that will work on retry this change adds flaky support with minimum 1 max 3 and improves test run time by adding pytest-parallel

docker-compose-ci.yml

@@ -6,10 +6,16 @@
 #
 #You should have received a copy of the CC0 legalcode along with this
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
-version: "3.2"
+version: "3.7"
 services:
   test:
     build: ./tests
+    entrypoint:
+      - /entrypoint.sh
+      - --workers
+      - auto
+      - --tests-per-worker
+      - auto
     links:
       - splunk
       - sc4s

tests/Dockerfile

@@ -9,13 +9,14 @@
 FROM python:3.7
 
 COPY requirements.txt /
-
 RUN pip3 install -r /requirements.txt
 RUN mkdir -p /work/tests
 RUN mkdir -p /work/test-results/functional
 COPY entrypoint.sh /
 COPY wait-for /bin/
 COPY ./* /work/tests/
+COPY pytest.ini /work
 COPY ./data /work/tests/data
 #WORKDIR /work
-CMD /entrypoint.sh
+CMD /entrypoint.sh
+

tests/entrypoint.sh

@@ -20,4 +20,4 @@ echo check for splunk hec
 wait-for splunk:8088 -t 0 -- echo splunkhec is up
 
 
-cd /work;python -m pytest --junitxml=/work/test-results/functional/functional.xml
+cd /work;python -m pytest --junitxml=/work/test-results/functional/functional.xml $@

tests/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+addopts =
+    --force-flaky --max-runs=3 --min-passes=1
+filterwarnings =
+    ignore::DeprecationWarning

tests/requirements.txt

@@ -11,4 +11,7 @@ jinja2
 jinja2-time
 splunk-sdk
 flake8
-pytz
+pytz
+flaky
+#pytest-randomly
+pytest-parallel

tests/test_ubiquiti_unifi.py

@@ -17,12 +17,12 @@ def test_ubiquiti_unifi_us8p60(record_property, setup_wordlist, setup_splunk):
     host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
 
     mt = env.from_string(
-        "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type")
-    message = mt.render(mark="<27>", host=host)
+        "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US8P60,18e8294876c3,v4.0.66.10832 switch: DOT1S: dot1sBpduReceive(): Discarding the BPDU on port 0/7, since it is an invalid BPDU type {{key}}")
+    message = mt.render(mark="<27>", key=host)
     sendsingle(message)
 
-    st = env.from_string("search index=netops sourcetype=ubnt:switch earliest=-2m | head 2")
-    search = st.render(host=host)
+    st = env.from_string("search index=netops sourcetype=ubnt:switch \"{{key}}\" earliest=-2m | head 2")
+    search = st.render(key=host)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)
 
@@ -37,12 +37,12 @@ def test_ubiquiti_unifi_switch_us24p250(record_property, setup_wordlist, setup_s
     host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
 
     mt = env.from_string(
-        "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: 0")
-    message = mt.render(mark="<27>", host=host)
+        "{{mark}}{% now 'utc', '%b %d %H:%M:%S' %} US24P250,f09fc26f4419,v4.0.54.10625 switch: TRAPMGR: Cold Start: Unit: {{key}}")
+    message = mt.render(mark="<27>", key=host)
     sendsingle(message)
 
-    st = env.from_string("search index=netops sourcetype=ubnt:switch earliest=-2m | head 2")
-    search = st.render(host=host)
+    st = env.from_string("search index=netops sourcetype=ubnt:switch \"{{key}}\" earliest=-2m | head 2")
+    search = st.render(key=host)
 
     resultCount, eventCount = splunk_single(setup_splunk, search)