[TRAVIS] run flaky test when label is defined (#9509)

to declare a test as flaky:

* for cpp, use

```
if ( !QgsTest::runFlakyTests() )
    QSKIP( "This test is disabled on Travis CI environment" );
```

* for Python, you can use `RUN_FLAKY_TEST` environment variable
This commit is contained in:
Denis Rouzaud 2019-03-27 09:45:34 -05:00 committed by GitHub
parent b9fdd02db6
commit 7fb752e0ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 46 additions and 3 deletions

View File

@ -1,6 +1,6 @@
# Env variables for Docker
# These without assignment are taken from the host env variabbles
# These without assignment are taken from the host env variables
# TRAVIS variables
TRAVIS_AVAILABLE_TIME
@ -11,6 +11,7 @@ TRAVIS_PULL_REQUEST
TRAVIS_OS_NAME
TRAVIS_CONFIG
TRAVIS
RUN_FLAKY_TESTS
# CTEST
LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so

View File

@ -16,6 +16,8 @@
set -e
echo "Running flaky test: ${RUN_FLAKY_TESTS}"
# build QGIS in docker
echo "travis_fold:start:docker_build_qgis"
echo "${bold}Docker build QGIS${endbold}"

View File

@ -0,0 +1,33 @@
#!/usr/bin/env python3
import sys
import json
from urllib.request import urlopen # using urllib since it is a standard module (vs. requests)
from urllib.error import URLError
import argparse
parser = argparse.ArgumentParser(description='Determines if a pull request has a defined label')
parser.add_argument('pull_request', type=int,
help='pull request id')
parser.add_argument('label', type=int,
help='label ID')
args = parser.parse_args()
url = "https://api.github.com/repos/qgis/QGIS/pulls/{}".format(args.pull_request)
try:
data = urlopen(url).read().decode('utf-8')
except URLError as err:
print("URLError: ".format(err.reason))
sys.exit(1)
obj = json.loads(data)
for label in obj['labels']:
if label["id"] == args.label:
print("true")
sys.exit(0)
print("label not found")
sys.exit(1)

View File

@ -11,7 +11,7 @@ cache:
- ${HOME}/.ccache_docker_build_cosmic
- ${HOME}/.ccache_docker_build_bionic
timeout: 1000
if: NOT branch =~ /^(cherry-pick-)?backport-\d+-on-/ AND NOT branch =~ /-patch-\d+$/
if: NOT branch =~ /^(cherry-pick-)?backport-\d+-on-/ AND NOT branch =~ /-patch-\d+$/
env:
global:
@ -46,6 +46,8 @@ matrix:
- CCACHE_DIR=${HOME}/.ccache_testing
- DOCKER_TAG=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && echo $TRAVIS_BRANCH | sed 's/master/latest/' || echo "latest" )
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile
# Label ID can be found here https://api.github.com/repos/qgis/QGIS/labels
- RUN_FLAKY_TESTS=$(.ci/travis/scripts/pr_has_label.py $TRAVIS_PULL_REQUEST 1271248184)
##########################################################
# CODE LAYOUT

View File

@ -83,6 +83,11 @@ namespace QgsTest
{
return qgetenv( "TRAVIS" ) == QStringLiteral( "true" );
}
bool runFlakyTests()
{
return qgetenv( "RUN_FLAKY_TESTS" ) == QStringLiteral( "true" );
}
}
#endif // QGSTEST_H

View File

@ -187,7 +187,7 @@ void TestQgsOgrProvider::testThread()
// Disabled by @m-kuhn
// This test is flaky
// See https://travis-ci.org/qgis/QGIS/jobs/505008602#L6464-L7108
if ( QgsTest::isTravis() )
if ( !QgsTest::runFlakyTests() )
QSKIP( "This test is disabled on Travis CI environment" );
// After reading a QgsVectorLayer (getFeatures) from another thread the QgsOgrConnPoolGroup starts