Merge pull request #8512 from elpaso/docker-testing-env-tests

Docker testing env tests

.ci/travis/linux/docker_test.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+###########################################################################
+#    docker_test.sh
+#
+#    Run a particular test on docker testing env and return its exit code
+#
+#    Arguments:
+#
+#        $1:  test name in dotted notation
+#
+#    ---------------------
+#    Date                 : November 2018
+#    Copyright            : (C) 2018 by Alessandro Pasotti
+#    Email                : elpaso at itopen dot it
+###########################################################################
+#                                                                         #
+#   This program is free software; you can redistribute it and/or modify  #
+#   it under the terms of the GNU General Public License as published by  #
+#   the Free Software Foundation; either version 2 of the License, or     #
+#   (at your option) any later version.                                   #
+#                                                                         #
+###########################################################################
+
+TEST_NAME=$1
+
+docker exec -it qgis-testing-environment sh -c "cd /tests_directory && qgis_testrunner.sh ${TEST_NAME}" &>/dev/null
+
+echo $?

.ci/travis/linux/script.sh

@@ -34,6 +34,21 @@ if [[ ${DOCKER_BUILD_QGIS_IMAGE} =~ true ]]; then
   docker push "qgis/qgis:${DOCKER_TAG}"
   popd
 else
-  # running tests
+  # running QGIS tests
   docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
+
+  # running tests for the python test runner
+  docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
+  sleep 10  # Wait for xvfb to finish starting
+  # Temporary workaround until docker images are built
+  docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
+  # Run tests in the docker
+  # Passing cases:
+  TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
+  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
+  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
+  # Failing cases:
+  [[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
+  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
+  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
 fi

.docker/qgis_resources/test_runner/qgis_testrunner.sh

@@ -1,11 +1,15 @@
 #!/bin/bash
-# Run a test inside QGIS
+# Run a python test inside QGIS
+# Note: the test module and function are specified in dotted notation
+# for example, to run the test function run_all (which is the default anyway)
+# $ qgis_testrunner.sh tests_folder.test_module.run_all
+# tests_folder must be in PYTHONPATH (plugins main folders are automatically added to path)
+
 ### Turn on debug mode ###
 #set -x
 
 TEST_NAME=$1
 
-cd /tests_directory || exit
 echo "Running test $1 ..."
 OUTPUT=$(QGIS_TEST_MODULE=${TEST_NAME} unbuffer qgis --version-migration --nologo --code /usr/bin/qgis_testrunner.py "$TEST_NAME"  2>/dev/null | tee /dev/tty)
 EXIT_CODE="$?"
@@ -13,13 +17,13 @@ if [ -z "$OUTPUT" ]; then
     echo "ERROR: no output from the test runner! (exit code: ${EXIT_CODE})"
     exit 1
 fi
-echo "$OUTPUT" | grep -q FAILED
+echo "$OUTPUT" | grep -q 'FAILED'
 IS_FAILED="$?"
-echo "$OUTPUT" | grep OK | grep -q 'Ran'
+echo "$OUTPUT" | grep -q 'OK' && echo "$OUTPUT" | grep -q 'Ran'
 IS_PASSED="$?"
 echo "$OUTPUT" | grep "QGIS died on signal"
 IS_DEAD="$?"
-echo "Finished running test $1."
+echo "Finished running test $1 (codes: IS_DEAD=$IS_DEAD IS_FAILED=$IS_FAILED IS_PASSED=$IS_PASSED)."
 if [ "$IS_PASSED" -eq "0" ] && [ "$IS_FAILED" -eq "1" ] && [ "$IS_DEAD" -eq "1" ]; then
     exit 0;
 fi

python/testing/__init__.py

@@ -394,6 +394,11 @@ def start_app(cleanup=True):
     except NameError:
         myGuiFlag = True  # All test will run qgis in gui mode
 
+        try:
+            sys.argv
+        except:
+            sys.argv = ['']
+
         # In python3 we need to convert to a bytes object (or should
         # QgsApplication accept a QString instead of const char* ?)
         try:

tests/src/python/test_testrunner.py

@@ -0,0 +1,69 @@
+
+# -*- coding: utf-8 -*-
+"""QGIS Unit tests for the docker python test runner
+
+.. note:: This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+"""
+__author__ = 'Alessandro Pasotti'
+__date__ = '19.11.2018'
+__copyright__ = 'Copyright 2018, The QGIS Project'
+# This will get replaced with a git SHA1 when you do a git archive
+__revision__ = '$Format:%H$'
+
+import qgis  # NOQA
+import sys
+
+from qgis.testing import unittest
+from console import console
+from qgis.core import Qgis
+
+
+class TestTestRunner(unittest.TestCase):
+
+    def test_fails(self):
+        self.assertTrue(False)
+
+    def test_passes(self):
+        self.assertTrue(Qgis.QGIS_VERSION_INT > 0)
+
+    @unittest.skip('Skipped!')
+    def test_skipped(self):
+        self.assertTrue(False)
+
+
+def _make_runner(tests=[]):
+    suite = unittest.TestSuite()
+    for t in tests:
+        suite.addTest(TestTestRunner(t))
+    runner = unittest.TextTestRunner(verbosity=2)
+    return runner.run(suite)
+
+
+# Test functions to be called by the runner
+
+def run_all():
+    """Default function that is called by the runner if nothing else is specified"""
+    return _make_runner(['test_fails', 'test_skipped', 'test_passes'])
+
+
+def run_failing():
+    """Run failing test only"""
+    return _make_runner(['test_fails'])
+
+
+def run_passing():
+    """Run passing test only"""
+    return _make_runner(['test_passes'])
+
+
+def run_skipped():
+    """Run skipped test only"""
+    return _make_runner(['test_skipped'])
+
+
+def run_skipped_and_passing():
+    """Run skipped and passing test only"""
+    return _make_runner(['test_skipped', 'test_passes'])