From e8e66d086b3a4f8464de890b07cb980743152b3f Mon Sep 17 00:00:00 2001 From: Alessandro Pasotti Date: Mon, 19 Nov 2018 15:40:27 +0100 Subject: [PATCH] Fix the python test runner and add a test for it --- .ci/travis/linux/script.sh | 16 ++++- .../test_runner/qgis_testrunner.sh | 14 ++-- tests/src/python/test_testrunner.py | 72 +++++++++++++++++++ 3 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 tests/src/python/test_testrunner.py diff --git a/.ci/travis/linux/script.sh b/.ci/travis/linux/script.sh index 767c3a6e5f5..e455d91300c 100755 --- a/.ci/travis/linux/script.sh +++ b/.ci/travis/linux/script.sh @@ -34,6 +34,18 @@ if [[ ${DOCKER_BUILD_QGIS_IMAGE} =~ true ]]; then docker push "qgis/qgis:${DOCKER_TAG}" popd else - # running tests - docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps + # running QGIS tests + #docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps + # running tests for the python test runner + docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}:/tests/src/python -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}" + docker exec -it qgis-testing-environment sh -c "qgis_testrunner.sh test_testrunner" + [ $? -ne 0 ] # expected failure + docker exec -it qgis-testing-environment sh -c "qgis_testrunner.sh test_testrunner.run_all" + [ $? -ne 0 ] # expected failure + docker exec -it qgis-testing-environment sh -c "qgis_testrunner.sh test_testrunner.run_failing" + [ $? -ne 0 ] # expected failure + docker exec -it qgis-testing-environment sh -c "qgis_testrunner.sh test_testrunner.run_passing" + [ $? -eq 0 ] # expected pass + docker exec -it qgis-testing-environment sh -c "qgis_testrunner.sh test_testrunner.run_skipped_and_passing" + [ $? -eq 0 ] # expected pass fi diff --git a/.docker/qgis_resources/test_runner/qgis_testrunner.sh b/.docker/qgis_resources/test_runner/qgis_testrunner.sh index fc6317d69b8..d60b346a355 100755 --- a/.docker/qgis_resources/test_runner/qgis_testrunner.sh +++ b/.docker/qgis_resources/test_runner/qgis_testrunner.sh @@ -1,11 +1,15 @@ #!/bin/bash -# Run a test inside QGIS +# Run a python test inside QGIS +# Note: the test module and function are specified in dotted notation +# for example, to run the test function run_all (which is the default anyway) +# $ qgis_testrunner.sh tests_folder.test_module.run_all +# tests_folder must be in PYTHONPATH (plugins main folders are automatically added to path) + ### Turn on debug mode ### #set -x TEST_NAME=$1 -cd /tests_directory || exit echo "Running test $1 ..." OUTPUT=$(QGIS_TEST_MODULE=${TEST_NAME} unbuffer qgis --version-migration --nologo --code /usr/bin/qgis_testrunner.py "$TEST_NAME" 2>/dev/null | tee /dev/tty) EXIT_CODE="$?" @@ -13,13 +17,13 @@ if [ -z "$OUTPUT" ]; then echo "ERROR: no output from the test runner! (exit code: ${EXIT_CODE})" exit 1 fi -echo "$OUTPUT" | grep -q FAILED +echo "$OUTPUT" | grep -q 'FAILED' IS_FAILED="$?" -echo "$OUTPUT" | grep OK | grep -q 'Ran' +echo "$OUTPUT" | grep -q 'OK' && echo "$OUTPUT" | grep -q 'Ran' IS_PASSED="$?" echo "$OUTPUT" | grep "QGIS died on signal" IS_DEAD="$?" -echo "Finished running test $1." +echo "Finished running test $1 (codes: IS_DEAD=$IS_DEAD IS_FAILED=$IS_FAILED IS_PASSED=$IS_PASSED)." if [ "$IS_PASSED" -eq "0" ] && [ "$IS_FAILED" -eq "1" ] && [ "$IS_DEAD" -eq "1" ]; then exit 0; fi diff --git a/tests/src/python/test_testrunner.py b/tests/src/python/test_testrunner.py new file mode 100644 index 00000000000..d355837725a --- /dev/null +++ b/tests/src/python/test_testrunner.py @@ -0,0 +1,72 @@ + +# -*- coding: utf-8 -*- +"""QGIS Unit tests for the docker python test runner + +.. note:: This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. +""" +__author__ = 'Alessandro Pasotti' +__date__ = '19.11.2018' +__copyright__ = 'Copyright 2018, The QGIS Project' +# This will get replaced with a git SHA1 when you do a git archive +__revision__ = '$Format:%H$' + +import qgis # NOQA +import sys + +from qgis.testing import unittest, start_app +from console import console +from qgis.core import Qgis +from qgis.PyQt.QtCore import QCoreApplication + +start_app() + + +class TestTestRunner(unittest.TestCase): + + def test_fails(self): + self.assertTrue(False) + + def test_passes(self): + self.assertTrue(Qgis.QGIS_VERSION_INT > 0) + + @unittest.skip('Skipped!') + def test_skipped(self): + self.assertTrue(False) + + +def _make_runner(tests=[]): + suite = unittest.TestSuite() + for t in tests: + suite.addTest(TestTestRunner(t)) + runner = unittest.TextTestRunner(verbosity=2) + return runner.run(suite) + + +# Test functions to be called by the runner + +def run_all(): + """Default function that is called by the runner if nothing else is specified""" + return _make_runner(['test_fails', 'test_skipped', 'test_passes']) + + +def run_failing(): + """Run failing test only""" + return _make_runner(['test_fails']) + + +def run_passes(): + """Run passing test only""" + return _make_runner(['test_passes']) + + +def run_skipped(): + """Run skipped test only""" + return _make_runner(['test_skipped']) + + +def run_skipped_and_passing(): + """Run skipped and passing test only""" + return _make_runner(['test_skipped', 'test_passes'])