simplify CI scripts (#9387)

by separating the testing and docker image building scripts there was almost no redundany in the scripts, it is much clearer now
2025-04-18 00:03:05 -04:00 · 2019-03-05 17:38:04 +01:00 · 2019-03-05 17:38:04 +01:00 · 1e11e1157a
commit 1e11e1157a
parent 47f1f4c5bb
9 changed files with 164 additions and 89 deletions
--- a/.ci/travis/docker_image/before_install.sh
+++ b/.ci/travis/docker_image/before_install.sh
@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+###########################################################################
+#    before_install.sh
+#    ---------------------
+#    Date                 : March 2019
+#    Copyright            : (C) 2019 by Denis Rouzaud
+#    Email                : denis@opengis.ch
+###########################################################################
+#                                                                         #
+#   This program is free software; you can redistribute it and/or modify  #
+#   it under the terms of the GNU General Public License as published by  #
+#   the Free Software Foundation; either version 2 of the License, or     #
+#   (at your option) any later version.                                   #
+#                                                                         #
+###########################################################################
+
--- a/.ci/travis/docker_image/before_script.sh
+++ b/.ci/travis/docker_image/before_script.sh
@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+###########################################################################
+#    before_install.sh
+#    ---------------------
+#    Date                 : March 2019
+#    Copyright            : (C) 2019 by Denis Rouzaud
+#    Email                : denis@opengis.ch
+###########################################################################
+#                                                                         #
+#   This program is free software; you can redistribute it and/or modify  #
+#   it under the terms of the GNU General Public License as published by  #
+#   the Free Software Foundation; either version 2 of the License, or     #
+#   (at your option) any later version.                                   #
+#                                                                         #
+###########################################################################
+
+
+set -e
+
+.ci/travis/scripts/echo_travis_var.sh
+
+pushd .docker
+
+
+
+echo "travis_fold:start:docker_build"
+echo "${bold}Docker build deps${endbold}"
+docker --version
+
+docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
+docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
+
+echo "travis_fold:end:docker_build"
+
+
+popd
--- a/.ci/travis/docker_image/install.sh
+++ b/.ci/travis/docker_image/install.sh
@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+###########################################################################
+#    before_install.sh
+#    ---------------------
+#    Date                 : March 2019
+#    Copyright            : (C) 2019 by Denis Rouzaud
+#    Email                : denis@opengis.ch
+###########################################################################
+#                                                                         #
+#   This program is free software; you can redistribute it and/or modify  #
+#   it under the terms of the GNU General Public License as published by  #
+#   the Free Software Foundation; either version 2 of the License, or     #
+#   (at your option) any later version.                                   #
+#                                                                         #
+###########################################################################
+
+
--- a/.ci/travis/docker_image/script.sh
+++ b/.ci/travis/docker_image/script.sh
@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+###########################################################################
+#    before_install.sh
+#    ---------------------
+#    Date                 : March 2019
+#    Copyright            : (C) 2019 by Denis Rouzaud
+#    Email                : denis@opengis.ch
+###########################################################################
+#                                                                         #
+#   This program is free software; you can redistribute it and/or modify  #
+#   it under the terms of the GNU General Public License as published by  #
+#   the Free Software Foundation; either version 2 of the License, or     #
+#   (at your option) any later version.                                   #
+#                                                                         #
+###########################################################################
+
+
+set -e
+
+mkdir -p "CCACHE_DIR_IMAGE_BUILD"
+
+# copy ccache dir within QGIS source so it can be accessed from docker
+cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
+# building docker images
+DIR=$(git rev-parse --show-toplevel)/.docker
+pushd "${DIR}"
+echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
+docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
+             --cache-from "qgis/qgis:${DOCKER_TAG}" \
+             -t "qgis/qgis:${DOCKER_TAG}" \
+             -f qgis.dockerfile ..
+echo "${bold}Pushing image to docker hub...${endbold}"
+docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+docker push "qgis/qgis:${DOCKER_TAG}"
+echo "Copy build cache from Docker container to Travis cache directory"
+rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
+container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
+docker start ${container_id}
+docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
+popd
+echo "Trigger build of PyQGIS Documentation"
+body='{
+  "request": {
+    "branch":"master",
+    "message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
+    "config": {
+      "merge_mode": "deep_merge",
+      "matrix": {
+        "include": {
+          "env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
+        }
+      }
+    }
+  }
+}'
+body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
+curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
+  -H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
+  https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests
--- a/.ci/travis/linux/before_install.sh
+++ b/.ci/travis/linux/before_install.sh
@ -14,4 +14,3 @@
 #                                                                         #
 ###########################################################################

-#pip3 install termcolor
--- a/.ci/travis/linux/before_script.sh
+++ b/.ci/travis/linux/before_script.sh
@ -16,39 +16,19 @@

 set -e

-pushd .docker
-
 DOCKER_DEPS_PUSH=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && [[ "${TRAVIS_EVENT_TYPE}" != "pull_request" ]] && echo "true" || echo "false" )
-DOCKER_DEPS_IMAGE_REBUILD=$( [[ $TRAVIS_COMMIT_MESSAGE =~ '[docker] update dependencies' ]] && echo "true" || echo "false" )

-echo "travis_fold:start:travis_env"
-echo "${bold}Travis environment variables${endbold}"
-echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
-echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
-echo "DOCKER_TAG: $DOCKER_TAG"
-echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
-echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
-echo "DOCKER_DEPS_IMAGE_REBUILD: $DOCKER_DEPS_IMAGE_REBUILD"
-echo "DOCKER_BUILD_QGIS_IMAGE: $DOCKER_BUILD_QGIS_IMAGE"
-echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
-echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
-echo "travis_fold:end:travis_env"
+.ci/travis/scripts/echo_travis_var.sh

+pushd .docker

 echo "travis_fold:start:docker_build"
 echo "${bold}Docker build deps${endbold}"
 docker --version
-if [[ $DOCKER_BUILD_QGIS_IMAGE =~ false ]]; then
-  docker-compose --version
-  docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
-fi
-
-if [[ $DOCKER_DEPS_IMAGE_REBUILD =~ true ]]; then
-  docker build --no-cache -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
-else
-  docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
-  docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
-fi
+docker-compose --version
+docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
+docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
+docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
 echo "travis_fold:end:docker_build"

 echo "travis_fold:start:docker_push"
--- a/.ci/travis/linux/script.sh
+++ b/.ci/travis/linux/script.sh
@ -16,62 +16,21 @@

 set -e

-mkdir -p "CCACHE_DIR_IMAGE_BUILD"
+# running QGIS tests
+docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps

-if [[ ${DOCKER_BUILD_QGIS_IMAGE} =~ true ]]; then
-  # copy ccache dir within QGIS source so it can be accessed from docker
-  cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
-  # building docker images
-  DIR=$(git rev-parse --show-toplevel)/.docker
-  pushd "${DIR}"
-  echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
-  docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
-               --cache-from "qgis/qgis:${DOCKER_TAG}" \
-               -t "qgis/qgis:${DOCKER_TAG}" \
-               -f qgis.dockerfile ..
-  echo "${bold}Pushing image to docker hub...${endbold}"
-  docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
-  docker push "qgis/qgis:${DOCKER_TAG}"
-  echo "Copy build cache from Docker container to Travis cache directory"
-  rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
-  container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
-  docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
-  popd
-  echo "Trigger build of PyQGIS Documentation"
-  body='{
-    "request": {
-      "branch":"master",
-      "message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
-      "config": {
-        "merge_mode": "deep_merge",
-        "matrix": {
-          "include": {
-            "env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
-          }
-        }
-      }
-    }
-  }'
-  body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
-  curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
-    -H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
-    https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests
-else
-  # running QGIS tests
-  docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
+# running tests for the python test runner
+docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
+sleep 10  # Wait for xvfb to finish starting
+# Temporary workaround until docker images are built
+docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
+# Run tests in the docker
+# Passing cases:
+TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
+[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
+[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
+# Failing cases:
+[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
+[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
+[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]

-  # running tests for the python test runner
-  docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
-  sleep 10  # Wait for xvfb to finish starting
-  # Temporary workaround until docker images are built
-  docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
-  # Run tests in the docker
-  # Passing cases:
-  TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
-  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
-  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
-  # Failing cases:
-  [[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
-  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
-  [[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
-fi
--- a/.ci/travis/scripts/echo_travis_var.sh
+++ b/.ci/travis/scripts/echo_travis_var.sh
@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+echo "travis_fold:start:travis_env"
+echo "${bold}Travis environment variables${endbold}"
+echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
+echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
+echo "DOCKER_TAG: $DOCKER_TAG"
+echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
+echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
+echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
+echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
+echo "travis_fold:end:travis_env"
--- a/.travis.yml
+++ b/.travis.yml
@ -43,7 +43,6 @@ matrix:
      env:
        - TRAVIS_CONFIG=linux
        - DOCKER_TAG=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && echo $TRAVIS_BRANCH | sed 's/master/latest/' || echo "latest" )
-        - DOCKER_BUILD_QGIS_IMAGE=false
        - DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile

      ##########################################################
@ -104,9 +103,8 @@ matrix:
      if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
      services: docker
      env:
-        - TRAVIS_CONFIG=linux
+        - TRAVIS_CONFIG=docker_image
        - DOCKER_TAG=$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )
-        - DOCKER_BUILD_QGIS_IMAGE=true
        - DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile

      ##########################################################
@ -117,9 +115,8 @@ matrix:
      if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
      services: docker
      env:
-        - TRAVIS_CONFIG=linux
+        - TRAVIS_CONFIG=docker_image
        - DOCKER_TAG="$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )_cosmic"
-        - DOCKER_BUILD_QGIS_IMAGE=true
        - DOCKER_BUILD_DEPS_FILE=qgis3-build-deps-cosmic.dockerfile