From bc080cad16ff546f56299476c8f5960b6d08c46f Mon Sep 17 00:00:00 2001 From: Denis Rouzaud Date: Fri, 1 Mar 2024 19:56:27 +0100 Subject: [PATCH] improve build workflow (#56614) * improve build workflow use upstream cache save/restore actions simplify the build cache saving since we only save build cache on push use native docker action * login on push only * fix tags * fix login * fix tag * do not push local tags * correctly setup ccache * define compiler * fix docker tags * remove personal branch --- .docker/docker-qgis-build.sh | 13 +--- .github/workflows/run-tests.yml | 133 ++++++++++++++------------------ 2 files changed, 60 insertions(+), 86 deletions(-) diff --git a/.docker/docker-qgis-build.sh b/.docker/docker-qgis-build.sh index 61a631ffc21..5596eeb9335 100755 --- a/.docker/docker-qgis-build.sh +++ b/.docker/docker-qgis-build.sh @@ -35,16 +35,11 @@ pushd ${CTEST_BUILD_DIR} > /dev/null echo "${bold}Running cmake...${endbold}" echo "::group::cmake" -if [[ -f "/usr/lib64/ccache/clang" ]]; then - export CC=/usr/lib64/ccache/clang - export CXX=/usr/lib64/ccache/clang++ -else - export CC=/usr/lib/ccache/clang - export CXX=/usr/lib/ccache/clang++ -fi - BUILD_TYPE=Release +export CC=/usr/bin/clang +export CXX=/usr/bin/clang++ + if [[ "${WITH_CLAZY}" = "ON" ]]; then # In release mode, all variables in QgsDebugMsg would be considered unused BUILD_TYPE=Debug @@ -75,7 +70,7 @@ fi cmake \ -GNinja \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - -DUSE_CCACHE=OFF \ + -DUSE_CCACHE=ON \ -DBUILD_WITH_QT6=${BUILD_WITH_QT6} \ -DWITH_DESKTOP=ON \ -DWITH_ANALYSIS=ON \ diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0eec7d03169..10a50e0c5e0 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -33,8 +33,6 @@ permissions: jobs: build: env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} QGIS_WORKSPACE: ${{ github.workspace }} # used in docker compose RUN_FLAKY_TESTS: ${{ contains( github.event.pull_request.labels.*.name, 'run flaky tests') }} @@ -47,7 +45,6 @@ jobs: - distro-version: '22.04' qt-version: 5 run-tests: true - docker-tag-suffix: '' with-qt6: OFF with-qt5: ON with-3d: ON @@ -64,7 +61,6 @@ jobs: - distro-version: '39' qt-version: 6 run-tests: true - docker-tag-suffix: '' with-qt6: ON with-qt5: OFF with-3d: ON @@ -107,7 +103,7 @@ jobs: GITHUB_PR_NUMBER: ${{github.event.number}} run: | # Be aware that these instructions are duplicated in run-tests job - DOCKER_TAG=$(echo $( [[ ${GITHUB_EVENT_NAME} == pull_request ]] && echo ${GITHUB_BASE_REF} || echo ${GITHUB_REF##*/} ) | sed 's/^master$/latest/')${{ matrix.docker-tag-suffix }} + DOCKER_TAG=$(echo $( [[ ${GITHUB_EVENT_NAME} == pull_request ]] && echo ${GITHUB_BASE_REF} || echo ${GITHUB_REF##*/} ) | sed 's/^master$/latest/') CTEST_BUILD_NAME=$( [[ ${GITHUB_EVENT_NAME} == pull_request ]] && echo "PR${GITHUB_PR_NUMBER}" || echo ${GITHUB_REF##*/} )"_${GITHUB_SHA}" echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV echo "CTEST_BUILD_NAME=${CTEST_BUILD_NAME}" >> $GITHUB_ENV @@ -119,57 +115,35 @@ jobs: echo CTEST_BUILD_NAME: ${CTEST_BUILD_NAME} echo QT_VERSION: ${QT_VERSION} + - name: Login to Docker Hub + if: ${{ github.event_name == 'push' }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build Docker Container with Build Environment id: docker-build - uses: whoan/docker-build-with-cache-action@v8 + uses: docker/build-push-action@v5 with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: ghcr.io - image_name: qgis3-build-deps-${{ matrix.distro-version }} - dockerfile: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile - build_extra_args: "--build-arg=DISTRO_VERSION=${{ matrix.distro-version }}" - push_git_tag: true - push_image_and_stages: on:push - pull_image_and_stages: ${{ github.event_name != 'workflow_dispatch' }} + context: . + file: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile + tags: qgis/qgis3-build-deps-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}:${{ github.event.pull_request.head.ref || github.ref_name }} + push: ${{ github.event_name == 'push' }} + pull: true + build-args: + DISTRO_VERSION=${{ matrix.distro-version }} - name: Tag image - run: | - docker tag ${DOCKER_IMAGE} qgis3-build-deps - env: - DOCKER_IMAGE: ${{ steps.docker-build.outputs.FULL_IMAGE_NAME }} + run: docker tag ${{ steps.docker-build.outputs.imageid }} qgis3-build-deps - - name: Push Qt5 deps image to Docker hub - if: ${{ github.repository_owner == 'qgis' && github.event_name != 'pull_request' && matrix.qt-version == '5' }} - run: | - docker tag qgis3-build-deps qgis/qgis3-build-deps:${DOCKER_TAG} - docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}" - docker push "qgis/qgis3-build-deps:${DOCKER_TAG}" - - - name: Prepare build cache for pull request - uses: pat-s/always-upload-cache@v3.0.11 - if: github.event_name == 'pull_request' + - name: Restore build cache + uses: actions/cache/restore@v3 with: path: /home/runner/QGIS/.ccache - key: build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.actor }}-${{ github.head_ref }}-${{ github.sha }} - # The head_ref or source branch of the pull request in a workflow run. - # The base_ref or target branch of the pull request in a workflow run. + key: build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.event.pull_request.head.ref || github.ref_name }} restore-keys: | - build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.actor }}-${{ github.head_ref }}- - build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-refs/heads/${{ github.base_ref }}- - build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-refs/heads/master- - - - name: Prepare build cache for branch/tag - # use a fork of actions/cache@v2 to upload cache even when the build or test failed - uses: pat-s/always-upload-cache@v3.0.11 - if: github.event_name != 'pull_request' - with: - path: /home/runner/QGIS/.ccache - # The branch or tag ref that triggered the workflow run. For branches this in the format refs/heads/, and for tags it is refs/tags/ - key: build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.ref }}-${{ github.sha }} - restore-keys: | - build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.ref }}- - build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-refs/heads/master- + build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-master - name: Compile QGIS id: compile @@ -178,6 +152,7 @@ jobs: -v $(pwd):/root/QGIS \ -v /home/runner/QGIS/.ccache:/root/.ccache \ --env-file .docker/docker-variables.env \ + --env CCACHE_DIR=/root/.ccache \ --env PUSH_TO_CDASH=true \ --env WITH_QT5=${{ matrix.with-qt5 }} \ --env BUILD_WITH_QT6=${{ matrix.with-qt6 }} \ @@ -193,6 +168,13 @@ jobs: qgis3-build-deps \ /root/QGIS/.docker/docker-qgis-build.sh + - name: Save build cache for push only + uses: actions/cache/save@v3 + if: ${{ github.event_name == 'push' }} + with: + path: /home/runner/QGIS/.ccache + key: build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.ref }} + - name: Push artifact id: push_artifact if: ${{ matrix.run-tests }} @@ -301,8 +283,6 @@ jobs: run-tests: name: Run tests env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} QGIS_WORKSPACE: ${{ github.workspace }} # used in docker compose runs-on: ubuntu-latest @@ -366,25 +346,28 @@ jobs: run: | echo CTEST_BUILD_NAME: ${CTEST_BUILD_NAME} + - name: Login to Docker Hub + if: ${{ github.event_name == 'push' }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build Docker Container with Testing Environment id: docker-build - uses: whoan/docker-build-with-cache-action@v8 + uses: docker/build-push-action@v5 with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: ghcr.io - image_name: qgis3-qt${{ matrix.qt-version }}-build-deps-bin-only - dockerfile: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile - build_extra_args: "--target ${{ matrix.docker-target }} --build-arg=DISTRO_VERSION=${{ matrix.distro-version }}" - push_git_tag: true - push_image_and_stages: on:push - pull_image_and_stages: ${{ github.event_name != 'workflow_dispatch' }} + context: . + file: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile + tags: qgis/qgis3-qt${{ matrix.qt-version }}-build-deps-bin-only:${{ github.event.pull_request.head.ref || github.ref_name }} + push: ${{ github.event_name == 'push' }} + pull: true + target: ${{ matrix.docker-target }} + build-args: + DISTRO_VERSION=${{ matrix.distro-version }} - name: Tag image - run: | - docker tag ${DOCKER_IMAGE} qgis3-build-deps-binary-image - env: - DOCKER_IMAGE: ${{ steps.docker-build.outputs.FULL_IMAGE_NAME }} + run: docker tag ${{ steps.docker-build.outputs.imageid }} qgis3-build-deps-binary-image - name: Print disk space run: | @@ -476,23 +459,19 @@ jobs: - name: Build Docker Container with Testing Environment id: docker-build - uses: whoan/docker-build-with-cache-action@v8 + uses: docker/build-push-action@v5 with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: ghcr.io - image_name: qgis3-qt${{ matrix.qt-version }}-build-deps-bin-only - dockerfile: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile - build_extra_args: "--build-arg=DISTRO_VERSION=${{ matrix.distro-version }}" - push_git_tag: true - push_image_and_stages: on:push - pull_image_and_stages: ${{ github.event_name != 'workflow_dispatch' }} + context: . + file: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile + tags: qgis/qgis3-qt${{ matrix.qt-version }}-build-deps-bin-only:${{ github.event.pull_request.head.ref || github.ref_name }} + push: ${{ github.event_name == 'push' }} + pull: true + target: ${{ matrix.docker-target }} + build-args: + DISTRO_VERSION=${{ matrix.distro-version }} - name: Tag image - run: | - docker tag ${DOCKER_IMAGE} qgis3-build-deps - env: - DOCKER_IMAGE: ${{ steps.docker-build.outputs.FULL_IMAGE_NAME }} + run: docker tag ${{ steps.docker-build.outputs.imageid }} qgis3-build-deps-binary-image - name: Download build artifact uses: actions/download-artifact@v4