sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-10-06 00:07:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9419 from 3nids/docker_tree

Browse Source 
Refactoring of Docker builds
This commit is contained in:
Denis Rouzaud 2019-03-09 15:31:18 +01:00 committed by GitHub
commit 8f0cfda89a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 199 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.ci/travis/linux/before_script.sh
View File

@ -26,7 +26,7 @@ echo "travis_fold:start:docker_build"
echo "${bold}Docker build deps${endbold}"
docker --version
docker-compose --version
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
docker-compose -f ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-compose.travis.yml config
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
echo "travis_fold:end:docker_build"

24
.ci/travis/linux/docker-compose.travis.yml Normal file
View File

@ -0,0 +1,24 @@
version: '3'
services:
postgres:
image: kartoza/postgis:9.5-2.2
environment:
- ALLOW_IP_RANGE="172.18.0.0/16"
mssql:
image: microsoft/mssql-server-linux:2017-latest
environment:
ACCEPT_EULA: Y
SA_PASSWORD: <YourStrong!Passw0rd>
qgis-deps:
tty: true
image: qgis_image
volumes:
- ${TRAVIS_BUILD_DIR}:/root/QGIS
links:
- postgres
- mssql
env_file:
- docker-variables.env

23
.ci/travis/linux/docker-variables.env Normal file
View File

@ -0,0 +1,23 @@
# Env variables for Docker
# These without assignment are taken from the host env variabbles
# TRAVIS variables
TRAVIS_AVAILABLE_TIME
TRAVIS_TIMESTAMP
TRAVIS_UPLOAD_TIME
TRAVIS_BRANCH
TRAVIS_PULL_REQUEST
TRAVIS_OS_NAME
TRAVIS_CONFIG
TRAVIS
# CTEST
LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
SEGFAULT_SIGNALS="abrt segv"
CTEST_BUILD_COMMAND=/usr/bin/ninja
CTEST_PARALLEL_LEVEL=1
CTEST_BUILD_DIR=/root/QGIS
# Other var
QGIS_NO_OVERRIDE_IMPORT=1

28
.ci/travis/linux/docker_test.sh
View File

@ -1,28 +0,0 @@
#!/usr/bin/env bash
###########################################################################
# docker_test.sh
#
# Run a particular test on docker testing env and return its exit code
#
# Arguments:
#
# $1: test name in dotted notation
#
# ---------------------
# Date : November 2018
# Copyright : (C) 2018 by Alessandro Pasotti
# Email : elpaso at itopen dot it
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################
TEST_NAME=$1
docker exec -it qgis-testing-environment sh -c "cd /tests_directory && qgis_testrunner.sh ${TEST_NAME}" &>/dev/null
echo $?

72
.ci/travis/linux/script.sh
View File

@ -16,21 +16,63 @@
set -e
# running QGIS tests
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
# build QGIS in docker
echo "travis_fold:start:docker_build_qgis"
echo "${bold}Docker build QGIS${endbold}"
docker run -t --name qgis_container \
-v ${TRAVIS_BUILD_DIR}:/root/QGIS \
-v ${CCACHE_DIR}:/root/.ccache \
--env-file ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-variables.env \
qgis/qgis3-build-deps:${DOCKER_TAG} \
/root/QGIS/.ci/travis/linux/scripts/docker-qgis-build.sh
# commit container
docker commit qgis_container qgis_image
echo "travis_fold:end:docker_build_qgis"
# running QGIS tests in commited image
echo "travis_fold:start:docker_test_qgis"
echo "${bold}Docker run tests${endbold}"
docker-compose -f ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-compose.travis.yml run qgis-deps /root/QGIS/.ci/travis/linux/scripts/docker-qgis-test.sh
echo "travis_fold:end:docker_test_qgis"
# running tests for the python test runner
docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
sleep 10 # Wait for xvfb to finish starting
# Temporary workaround until docker images are built
docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
# Run tests in the docker
# Passing cases:
TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
# Failing cases:
[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
echo "travis_fold:start:docker_test_runners"
echo "${bold}Docker test QGIS runners${endbold}"
docker run -d --name qgis-testing-environment \
-v ${TRAVIS_BUILD_DIR}:/root/QGIS \
-v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory \
-v ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner:/usr/bin/test_runner \
-v ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/supervisor:/etc/supervisor \
-e QGIS_BUILD_PATH=/root/QGIS/build/output/bin/qgis \
-e TEST_RUNNER_PATH=/usr/bin/test_runner/qgis_testrunner.py \
-e DISPLAY=:99 \
qgis_image \
/usr/bin/supervisord -c /etc/supervisor/supervisord.conf
# Wait for xvfb to finish starting
printf "Waiting for the docker...🐳..."
sleep 10
echo " done 🥩"
declare -A testrunners
# Passing cases:
testrunners["test_testrunner.run_passing"]=0
testrunners["test_testrunner.run_skipped_and_passing"]=0
# Failing cases:
testrunners["test_testrunner"]=1
testrunners["test_testrunner.run_all"]=1
testrunners["test_testrunner.run_failing"]=1
set +e # do not exit on error
# Run tests in the docker
for i in "${!testrunners[@]}"
do
echo "travis_fold:start:docker_test_runner_${i}"
echo "test ${i}..."
docker exec -it qgis-testing-environment sh -c "cd /tests_directory && /usr/bin/test_runner/qgis_testrunner.sh ${i}"
[[ $? -eq "${testrunners[$i]}" ]] && echo "success" || exit 1
echo "travis_fold:end:docker_test_runner_${i}"
done
set -e # switch back
docker stop qgis-testing-environment
echo "travis_fold:end:docker_test_runners"

87
.ci/travis/linux/docker-build-test.sh → .ci/travis/linux/scripts/docker-qgis-build.sh
View File

@ -12,14 +12,6 @@ ccache -M 1G
# export CCACHE_LOGFILE=/tmp/cache.debug
ccache -z
############################
# Setup the (c)test environment
############################
export LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
export SEGFAULT_SIGNALS="abrt segv"
export CTEST_BUILD_COMMAND="/usr/bin/ninja"
export CTEST_PARALLEL_LEVEL=1
##############################
# Variables for output styling
##############################
@ -75,10 +67,9 @@ echo "travis_fold:end:cmake"
# Travis will kill the job after approx 150 minutes, we subtract 5 minutes for
# uploading and subtract the bootstrapping time from that.
# Hopefully clocks are in sync :)
TRAVIS_TIME=150
UPLOAD_TIME=5
CURRENT_TIME=$(date +%s)
TIMEOUT=$((( TRAVIS_TIME - UPLOAD_TIME ) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
TIMEOUT=$((( TRAVIS_AVAILABLE_TIME - TRAVIS_UPLOAD_TIME ) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
TIMEOUT=$(( TIMEOUT < 300 ? 300 : TIMEOUT ))
echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
@ -86,73 +77,7 @@ echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_T
echo "${bold}Building QGIS...${endbold}"
timeout ${TIMEOUT}s ${CTEST_BUILD_COMMAND}
# echo "travis_fold:end:ninja-build.1"
rv=$?
if [ $rv -eq 124 ] ; then
printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
exit #$rv
fi
# Temporarily uncomment to debug ccache issues
# echo "travis_fold:start:ccache-debug"
# cat /tmp/cache.debug
# echo "travis_fold:end:ccache-debug"
############################
# Restore postgres test data
############################
printf "[qgis_test]\nhost=postgres\nport=5432\ndbname=qgis_test\nuser=docker\npassword=docker" > ~/.pg_service.conf
export PGUSER=docker
export PGHOST=postgres
export PGPASSWORD=docker
export PGDATABASE=qgis_test
pushd /root/QGIS > /dev/null
/root/QGIS/tests/testdata/provider/testdata_pg.sh
popd > /dev/null # /root/QGIS
##############################
# Restore SQL Server test data
##############################
echo "Importing SQL Server test data..."
export SQLUSER=sa
export SQLHOST=mssql
export SQLPORT=1433
export SQLPASSWORD='<YourStrong!Passw0rd>'
export SQLDATABASE=qgis_test
export PATH=$PATH:/opt/mssql-tools/bin
pushd /root/QGIS > /dev/null
/root/QGIS/tests/testdata/provider/testdata_mssql.sh
popd > /dev/null # /root/QGIS
echo "Setting up DSN for test SQL Server"
cat <<EOT > /etc/odbc.ini
[ODBC Data Sources]
testsqlserver = ODBC Driver 17 for SQL Server
[testsqlserver]
Driver = ODBC Driver 17 for SQL Server
Description = Test SQL Server
Server = mssql
EOT
###########
# Run tests
###########
CURRENT_TIME=$(date +%s)
TIMEOUT=$((( TRAVIS_TIME - UPLOAD_TIME) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
timeout ${TIMEOUT}s python3 /root/QGIS/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V -E "$(cat /root/QGIS/.ci/travis/linux/blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S /root/QGIS/.ci/travis/travis.ctest --output-on-failure
rv=$?
if [ $rv -eq 124 ] ; then
printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
exit #$rv
fi
########################
# Show ccache statistics
@ -166,3 +91,11 @@ popd > /dev/null # build
popd > /dev/null # /root/QGIS
[ -r /tmp/ctest-important.log ] && cat /tmp/ctest-important.log || true
############################
# Exit with error if timeout
############################
if [ $rv -eq 124 ] ; then
printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
exit #$rv
fi

76
.ci/travis/linux/scripts/docker-qgis-test.sh Executable file
View File

@ -0,0 +1,76 @@
#!/usr/bin/env bash
set -e
# Temporarily uncomment to debug ccache issues
# echo "travis_fold:start:ccache-debug"
# cat /tmp/cache.debug
# echo "travis_fold:end:ccache-debug"
############################
# Restore postgres test data
############################
printf "[qgis_test]\nhost=postgres\nport=5432\ndbname=qgis_test\nuser=docker\npassword=docker" > ~/.pg_service.conf
export PGUSER=docker
export PGHOST=postgres
export PGPASSWORD=docker
export PGDATABASE=qgis_test
# wait for the DB to be available
echo "Wait a moment while loading the database."
while ! PGPASSWORD='docker' psql -h postgres -U docker -p 5432 -l &> /dev/null
do
printf "🐘"
sleep 1
done
echo " done 🥩"
pushd /root/QGIS > /dev/null
/root/QGIS/tests/testdata/provider/testdata_pg.sh
popd > /dev/null # /root/QGIS
##############################
# Restore SQL Server test data
##############################
echo "Importing SQL Server test data..."
export SQLUSER=sa
export SQLHOST=mssql
export SQLPORT=1433
export SQLPASSWORD='<YourStrong!Passw0rd>'
export SQLDATABASE=qgis_test
export PATH=$PATH:/opt/mssql-tools/bin
pushd /root/QGIS > /dev/null
/root/QGIS/tests/testdata/provider/testdata_mssql.sh
popd > /dev/null # /root/QGIS
echo "Setting up DSN for test SQL Server"
cat <<EOT > /etc/odbc.ini
[ODBC Data Sources]
testsqlserver = ODBC Driver 17 for SQL Server
[testsqlserver]
Driver = ODBC Driver 17 for SQL Server
Description = Test SQL Server
Server = mssql
EOT
###########
# Run tests
###########
CURRENT_TIME=$(date +%s)
TIMEOUT=$((( TRAVIS_AVAILABLE_TIME - TRAVIS_UPLOAD_TIME) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
timeout ${TIMEOUT}s python3 /root/QGIS/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V -E "$(cat /root/QGIS/.ci/travis/linux/scripts/test_blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S /root/QGIS/.ci/travis/travis.ctest --output-on-failure
rv=$?
if [ $rv -eq 124 ] ; then
printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
exit #$rv
fi

0
.ci/travis/linux/blacklist.txt → .ci/travis/linux/scripts/test_blacklist.txt
View File

1
.ci/travis/scripts/echo_travis_var.sh
View File

@ -6,7 +6,6 @@ echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
echo "DOCKER_TAG: $DOCKER_TAG"
echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
echo "travis_fold:end:travis_env"

35
.docker/docker-compose.travis.yml
View File

@ -1,35 +0,0 @@
version: '3'
services:
postgres:
image: kartoza/postgis:9.5-2.2
environment:
- ALLOW_IP_RANGE="172.18.0.0/16"
mssql:
image: microsoft/mssql-server-linux:2017-latest
environment:
ACCEPT_EULA: Y
SA_PASSWORD: <YourStrong!Passw0rd>
qgis-deps:
tty: true
image: qgis/qgis3-build-deps:${DOCKER_TAG}
volumes:
- ${TRAVIS_BUILD_DIR}:/root/QGIS
- ${CCACHE_DIR}:/root/.ccache # if changed, also change env var
links:
- postgres
- mssql
environment:
- CCACHE_DIR=/root/.ccache
- CTEST_BUILD_DIR=/root/QGIS
- TRAVIS_BRANCH=${TRAVIS_BRANCH}
- TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST}
- TRAVIS_PULL_REQUEST_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH}
- TRAVIS_COMMIT_RANGE=${TRAVIS_COMMIT_RANGE}
- TRAVIS_OS_NAME=${TRAVIS_OS_NAME}
- TRAVIS_CONFIG=${TRAVIS_CONFIG}
- TRAVIS=${TRAVIS}
- TRAVIS_TIMESTAMP=${TRAVIS_TIMESTAMP}
- QGIS_DISABLE_MESSAGE_HOOKS=1
- QGIS_NO_OVERRIDE_IMPORT=1

2
.docker/qgis.dockerfile
View File

@ -53,7 +53,7 @@ RUN cmake \
# Python testing environment setup
# Add QGIS test runner
COPY .docker/qgis_resources/test_runner/qgis_* /usr/bin/
COPY .docker/qgis_resources/test_runner/qgis_* /usr/bin
# Make all scripts executable
RUN chmod +x /usr/bin/qgis_*

2
.docker/qgis3-build-deps-cosmic.dockerfile
View File

@ -135,5 +135,3 @@ ENV CXX=/usr/lib/ccache/clang++
ENV QT_SELECT=5
ENV LANG=C.UTF-8
ENV PATH="/usr/local/bin:${PATH}"
CMD /root/QGIS/.ci/travis/linux/docker-build-test.sh

1
.docker/qgis3-build-deps.dockerfile
View File

@ -153,4 +153,3 @@ ENV QT_SELECT=5
ENV LANG=C.UTF-8
ENV PATH="/usr/local/bin:${PATH}"
CMD /root/QGIS/.ci/travis/linux/docker-build-test.sh

0
.docker/qgis_resources/supervisor/supervisor.xvfb.conf → .docker/qgis_resources/supervisor/supervisor.d/supervisor.xvfb.conf
View File

6
.docker/qgis_resources/test_runner/qgis_testrunner.sh
View File

@ -10,8 +10,12 @@
TEST_NAME=$1
TEST_RUNNER_PATH=${TEST_RUNNER_PATH:-/usr/bin/qgis_testrunner.py}
QGIS_BUILD_PATH=${QGIS_BUILD_PATH:-qgis}
echo "Running test $1 ..."
OUTPUT=$(QGIS_TEST_MODULE=${TEST_NAME} unbuffer qgis --version-migration --nologo --code /usr/bin/qgis_testrunner.py "$TEST_NAME" 2>/dev/null | tee /dev/tty)
OUTPUT=$(QGIS_TEST_MODULE=${TEST_NAME} unbuffer ${QGIS_BUILD_PATH} --version-migration --nologo --code ${TEST_RUNNER_PATH} "$TEST_NAME" 2>/dev/null | tee /dev/tty)
EXIT_CODE="$?"
if [ -z "$OUTPUT" ]; then
echo "ERROR: no output from the test runner! (exit code: ${EXIT_CODE})"

2
.travis.yml
View File

@ -16,6 +16,8 @@ if: NOT branch =~ /^(cherry-pick-)?backport-\d+-on-/ AND NOT branch =~ /-patch-\
env:
global:
- TRAVIS_TIMESTAMP=$(date +%s)
- TRAVIS_AVAILABLE_TIME=150 # in minutes
- TRAVIS_UPLOAD_TIME=5 # time considered to start the machine and the container (minutes)
# Docker hub username and passowrd
- secure: "b7eMDIolaAnq1voGKC1ez7Kcf+/A0WZDJEHBvNwk2KubBfrGOE83GMDrFNF4NqjIprqIAvVKj+TrX1ckCvs24re3IqUJo71TaF1IgxzDDPwSsmNh5UMmvZkeiJys9bWjqDO9wYR5ietNmIE18qyMc8ToJk8oKm6AXuAG2n6znmM="
- secure: "PHCp7F3nApp38Mz6b4/OLxgfBiikRGzPQDHg3R5LX+SQOll24c/DMtwpPwizNuFEiCFcRmJ9uc1t0HWEerIZe5uqm7AtE/nMXBsvDZ+hj4Tz/fEBF98a1k4WLYheN1exFidVkJgdAeiwMOOUQXw5KuIX62bxBdzsdcd0QGwxiXo="