diff --git a/.ci/travis/docker_image/script.sh b/.ci/travis/docker_image/script.sh index 21995e9ff0c..edcb2fe500a 100755 --- a/.ci/travis/docker_image/script.sh +++ b/.ci/travis/docker_image/script.sh @@ -17,48 +17,79 @@ set -e -mkdir -p "${CCACHE_DIR}" +# test if ccache dir exists (coming from Travis cache) +[[ -d ${CCACHE_DIR} ]] && echo "cache directory (${CCACHE_DIR}) exists" || mkdir -p "${CCACHE_DIR}" # copy ccache dir within QGIS source so it can be accessed from docker -cp -r ${CCACHE_DIR} ${TRAVIS_BUILD_DIR}/.ccache_image_build +cp -r ${CCACHE_DIR}/. ${TRAVIS_BUILD_DIR}/.ccache_image_build + +echo "Cache directory size: "$(du -h --max-depth=0 ${TRAVIS_BUILD_DIR}/.ccache_image_build) + +# calculate timeouts +CURRENT_TIME=$(date +%s) +TIMEOUT=$((( TRAVIS_AVAILABLE_TIME - TRAVIS_UPLOAD_TIME ) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP)) +#TIMEOUT=$(( TIMEOUT < 300 ? 300 : TIMEOUT )) +echo "Timeout: ${TIMEOUT}s" + # building docker images -DIR=$(git rev-parse --show-toplevel)/.docker -pushd "${DIR}" +pushd "${TRAVIS_BUILD_DIR}/.docker" echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}" -docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \ +DOCKER_BUILD_ARGS="--build-arg DOCKER_TAG=${DOCKER_TAG} \ + --build-arg BUILD_TIMEOUT=${TIMEOUT} \ + --build-arg CC --build-arg CXX" +docker build ${DOCKER_BUILD_ARGS} \ --cache-from "qgis/qgis:${DOCKER_TAG}" \ - -t "qgis/qgis:${DOCKER_TAG}" \ - --build-arg CC --build-arg CXX \ + -t "qgis/qgis:BUILDER" \ -f qgis.dockerfile .. -echo "${bold}Pushing image to docker hub...${endbold}" -docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" -docker push "qgis/qgis:${DOCKER_TAG}" + +docker run --name qgis_container qgis/qgis:BUILDER /bin/true + echo "Copy build cache from Docker container to Travis cache directory" rm -rf "${CCACHE_DIR:?}/"* -docker run --name qgis_container qgis/qgis:${DOCKER_TAG} /bin/true -docker cp qgis_container:/usr/src/.ccache_image_build ${CCACHE_DIR} -docker rm qgis_container -popd -echo "Trigger build of PyQGIS Documentation" -if [[ ${TRIGGER_PYQGIS_DOC} =~ ^TRUE$ ]]; then - body='{ - "request": { - "branch":"master", - "message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__", - "config": { - "merge_mode": "deep_merge", - "matrix": { - "include": { - "env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"] +mkdir -p ${CCACHE_DIR} +docker cp qgis_container:/QGIS/.ccache_image_build/. ${CCACHE_DIR} + +docker cp qgis_container:/QGIS/build_exit_value ${HOME}/build_exit_value + +if [[ $(cat ${HOME}/build_exit_value) == "TIMEOUT" ]]; then + echo "Build timeout, not pushing image or triggering PyQGIS docs" + exit 1 +else + echo "${bold}Finalize image…${endbold}" + # enable experimental features in Docker to squash + echo '{ "experimental": true}' | sudo tee /etc/docker/daemon.json + sudo service docker restart + docker build ${DOCKER_BUILD_ARGS} \ + --cache-from "qgis/qgis:BUILDER" \ + --squash \ + -t "qgis/qgis:${DOCKER_TAG}" \ + -f qgis.dockerfile .. + + echo "${bold}Pushing image to docker hub…${endbold}" + docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + docker push "qgis/qgis:${DOCKER_TAG}" + + echo "Trigger build of PyQGIS Documentation…" + if [[ ${TRIGGER_PYQGIS_DOC} =~ ^TRUE$ ]]; then + body='{ + "request": { + "branch":"master", + "message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__", + "config": { + "merge_mode": "deep_merge", + "matrix": { + "include": { + "env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"] + } } } } - } - }' - body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body) - curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \ - -H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \ - https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests -else - echo "skipped from configuration" + }' + body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body) + curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \ + -H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \ + https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests + else + echo "skipped from configuration" + fi fi diff --git a/.docker/qgis.dockerfile b/.docker/qgis.dockerfile index f6607af213f..e71f6b6aebb 100644 --- a/.docker/qgis.dockerfile +++ b/.docker/qgis.dockerfile @@ -1,27 +1,32 @@ -# CACHE_TAG is provided by Docker cloud # see https://docs.docker.com/docker-cloud/builds/advanced/ # using ARG in FROM requires min v17.05.0-ce ARG DOCKER_TAG=latest -FROM qgis/qgis3-build-deps:${DOCKER_TAG} +FROM qgis/qgis3-build-deps:${DOCKER_TAG} AS BUILDER MAINTAINER Denis Rouzaud LABEL Description="Docker container with QGIS" Vendor="QGIS.org" Version="1.1" +# build timeout in seconds, so no timeout by default +ARG BUILD_TIMEOUT=360000 + ARG CC=/usr/lib/ccache/clang ARG CXX=/usr/lib/ccache/clang++ ENV LANG=C.UTF-8 -COPY . /usr/src/QGIS +COPY . /QGIS # If this directory is changed, also adapt script.sh which copies the directory -RUN mkdir -p /usr/src/.ccache_image_build -ENV CCACHE_DIR=/usr/src/.ccache_image_build +# if ccache directory is not provided with the source +RUN mkdir -p /QGIS/.ccache_image_build +ENV CCACHE_DIR=/QGIS/.ccache_image_build RUN ccache -M 1G RUN ccache -s -WORKDIR /usr/src/QGIS/build +RUN echo "ccache_dir: "$(du -h --max-depth=0 ${CCACHE_DIR}) + +WORKDIR /QGIS/build RUN cmake \ -GNinja \ @@ -42,13 +47,18 @@ RUN cmake \ -DWITH_APIDOC=OFF \ -DWITH_ASTYLE=OFF \ -DQT5_3DEXTRA_LIBRARY="/usr/lib/x86_64-linux-gnu/libQt53DExtras.so" \ - -DQT5_3DEXTRA_INCLUDE_DIR="/usr/src/QGIS/external/qt3dextra-headers" \ - -DCMAKE_PREFIX_PATH="/usr/src/QGIS/external/qt3dextra-headers/cmake" \ + -DQT5_3DEXTRA_INCLUDE_DIR="/QGIS/external/qt3dextra-headers" \ + -DCMAKE_PREFIX_PATH="/QGIS/external/qt3dextra-headers/cmake" \ .. \ - && ninja install \ - && rm -rf /usr/src/QGIS + && echo "Timeout: ${BUILD_TIMEOUT}s" \ + && SUCCESS=OK \ + && timeout ${BUILD_TIMEOUT}s ninja install || SUCCESS=TIMEOUT \ + && echo "$SUCCESS" > /QGIS/build_exit_value ################################################################################ +ARG DELETE_CACHE=FALSE +RUN if [[ ${DELETE_CACHE} == TRUE ]]; then rm /QGIS; fi + # Python testing environment setup # Add QGIS test runner diff --git a/.travis.yml b/.travis.yml index 82d3f515d05..4a52250c441 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ env: - TRAVIS_TIMESTAMP=$(date +%s) - TRAVIS_AVAILABLE_TIME=150 # in minutes - TRAVIS_UPLOAD_TIME=5 # time considered to start the machine and the container (minutes) - # Docker hub username and passowrd + # Docker hub username and password - secure: "b7eMDIolaAnq1voGKC1ez7Kcf+/A0WZDJEHBvNwk2KubBfrGOE83GMDrFNF4NqjIprqIAvVKj+TrX1ckCvs24re3IqUJo71TaF1IgxzDDPwSsmNh5UMmvZkeiJys9bWjqDO9wYR5ietNmIE18qyMc8ToJk8oKm6AXuAG2n6znmM=" - secure: "PHCp7F3nApp38Mz6b4/OLxgfBiikRGzPQDHg3R5LX+SQOll24c/DMtwpPwizNuFEiCFcRmJ9uc1t0HWEerIZe5uqm7AtE/nMXBsvDZ+hj4Tz/fEBF98a1k4WLYheN1exFidVkJgdAeiwMOOUQXw5KuIX62bxBdzsdcd0QGwxiXo=" # Travis Token to create PyQGIS Documentation Travis build after Docker push @@ -60,7 +60,7 @@ matrix: addons: apt: sources: - # - sourceline: 'ppa:jonathonf/backports' # silversearcher-ag backport + # - sourceline: 'ppa:jonathonf/backports' # silversearcher-ag backport packages: - doxygen - graphviz @@ -105,6 +105,7 @@ matrix: name: bionic docker build 🐳 if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron) services: docker + sudo: required # required to restart Docker Daemon with experimental features env: - TRAVIS_CONFIG=docker_image - CCACHE_DIR=${HOME}/.ccache_docker_build_bionic @@ -121,6 +122,7 @@ matrix: name: disco docker build 💃 if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron) services: docker + sudo: required # required to restart Docker Daemon with experimental features env: - TRAVIS_CONFIG=docker_image - CCACHE_DIR=${HOME}/.ccache_docker_build_disco