Fix the python test runner and add a test for it

.ci/travis/linux/script.sh

@@ -34,6 +34,18 @@ if [[ ${DOCKER_BUILD_QGIS_IMAGE} =~ true ]]; then
   docker push "qgis/qgis:${DOCKER_TAG}"
   popd
 else
-  # running tests
-  docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
+  # running QGIS tests
+  #docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
+  # running tests for the python test runner
+  docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}:/tests/src/python -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
+  docker exec -it qgis-testing-environment sh -c "qgis_testrunner.sh test_testrunner"
+  [ $? -ne 0 ]   # expected failure
+  docker exec -it qgis-testing-environment sh -c "qgis_testrunner.sh test_testrunner.run_all"
+  [ $? -ne 0 ]   # expected failure
+  docker exec -it qgis-testing-environment sh -c "qgis_testrunner.sh test_testrunner.run_failing"
+  [ $? -ne 0 ]   # expected failure
+  docker exec -it qgis-testing-environment sh -c "qgis_testrunner.sh test_testrunner.run_passing"
+  [ $? -eq 0 ]   # expected pass
+  docker exec -it qgis-testing-environment sh -c "qgis_testrunner.sh test_testrunner.run_skipped_and_passing"
+  [ $? -eq 0 ]   # expected pass
 fi

.docker/qgis_resources/test_runner/qgis_testrunner.sh

@@ -1,11 +1,15 @@
 #!/bin/bash
-# Run a test inside QGIS
+# Run a python test inside QGIS
+# Note: the test module and function are specified in dotted notation
+# for example, to run the test function run_all (which is the default anyway)
+# $ qgis_testrunner.sh tests_folder.test_module.run_all
+# tests_folder must be in PYTHONPATH (plugins main folders are automatically added to path)
+
 ### Turn on debug mode ###
 #set -x
 
 TEST_NAME=$1
 
-cd /tests_directory || exit
 echo "Running test $1 ..."
 OUTPUT=$(QGIS_TEST_MODULE=${TEST_NAME} unbuffer qgis --version-migration --nologo --code /usr/bin/qgis_testrunner.py "$TEST_NAME"  2>/dev/null | tee /dev/tty)
 EXIT_CODE="$?"
@@ -13,13 +17,13 @@ if [ -z "$OUTPUT" ]; then
     echo "ERROR: no output from the test runner! (exit code: ${EXIT_CODE})"
     exit 1
 fi
-echo "$OUTPUT" | grep -q FAILED
+echo "$OUTPUT" | grep -q 'FAILED'
 IS_FAILED="$?"
-echo "$OUTPUT" | grep OK | grep -q 'Ran'
+echo "$OUTPUT" | grep -q 'OK' && echo "$OUTPUT" | grep -q 'Ran'
 IS_PASSED="$?"
 echo "$OUTPUT" | grep "QGIS died on signal"
 IS_DEAD="$?"
-echo "Finished running test $1."
+echo "Finished running test $1 (codes: IS_DEAD=$IS_DEAD IS_FAILED=$IS_FAILED IS_PASSED=$IS_PASSED)."
 if [ "$IS_PASSED" -eq "0" ] && [ "$IS_FAILED" -eq "1" ] && [ "$IS_DEAD" -eq "1" ]; then
     exit 0;
 fi

tests/src/python/test_testrunner.py

@@ -0,0 +1,72 @@
+
+# -*- coding: utf-8 -*-
+"""QGIS Unit tests for the docker python test runner
+
+.. note:: This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+"""
+__author__ = 'Alessandro Pasotti'
+__date__ = '19.11.2018'
+__copyright__ = 'Copyright 2018, The QGIS Project'
+# This will get replaced with a git SHA1 when you do a git archive
+__revision__ = '$Format:%H$'
+
+import qgis  # NOQA
+import sys
+
+from qgis.testing import unittest, start_app
+from console import console
+from qgis.core import Qgis
+from qgis.PyQt.QtCore import QCoreApplication
+
+start_app()
+
+
+class TestTestRunner(unittest.TestCase):
+
+    def test_fails(self):
+        self.assertTrue(False)
+
+    def test_passes(self):
+        self.assertTrue(Qgis.QGIS_VERSION_INT > 0)
+
+    @unittest.skip('Skipped!')
+    def test_skipped(self):
+        self.assertTrue(False)
+
+
+def _make_runner(tests=[]):
+    suite = unittest.TestSuite()
+    for t in tests:
+        suite.addTest(TestTestRunner(t))
+    runner = unittest.TextTestRunner(verbosity=2)
+    return runner.run(suite)
+
+
+# Test functions to be called by the runner
+
+def run_all():
+    """Default function that is called by the runner if nothing else is specified"""
+    return _make_runner(['test_fails', 'test_skipped', 'test_passes'])
+
+
+def run_failing():
+    """Run failing test only"""
+    return _make_runner(['test_fails'])
+
+
+def run_passes():
+    """Run passing test only"""
+    return _make_runner(['test_passes'])
+
+
+def run_skipped():
+    """Run skipped test only"""
+    return _make_runner(['test_skipped'])
+
+
+def run_skipped_and_passing():
+    """Run skipped and passing test only"""
+    return _make_runner(['test_skipped', 'test_passes'])