name: 🧪 QGIS tests

on:
  push:
    branches:
      - master
      - release-**
      - queued_ltr_backports
    paths:
    - 'src/**'
    - 'external/**'
    - 'python/**'
    - 'tests/**'
    - 'CMakeLists.txt'
    - '.github/workflows/run-tests.yml'
    - '.docker/**'
    - '.ci/**'
  pull_request:
    branches:
      - master
      - release-**
    # paths cannot be filtered on this workflow on pull request as it is a required one in the branch protection
    # feature request and hacks: https://github.community/t/feature-request-conditional-required-checks/16761

jobs:
  build:
    env:
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
      GH_WORKSPACE: ${{ github.workspace }} # used in docker compose
      DEFAULT_UBUNTU_BASE: '20.04'
      RUN_FLAKY_TESTS: ${{ contains( github.event.pull_request.labels.*.name, 'run flaky tests') }}

    runs-on: ubuntu-latest

    strategy:
      matrix:
        # tests run on 20.04 (Qt 5.12), compile test on 20.10 (Qt 5.14) and 21.04 (Qt 5.15)
        include:
          - ubuntu-base: '20.04'
            run-tests: true
          - ubuntu-base: '21.04'
            run-tests: false
      fail-fast: false

    outputs:
      compile_outcome: ${{ steps.compile.outcome }}
      tests_failing: ${{ steps.tests.outputs.TESTS_FAILING }}
      cdash_url: ${{ steps.tests.outputs.CDASH_URL }}
      runners_outcome: ${{ steps.runners.outcome }}

    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Set vars
        env:
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          GITHUB_REF: ${{ github.ref }}
          GITHUB_PR_NUMBER: ${{github.event.number}}
          UBUNTU_BASE: ${{ matrix.ubuntu-base }}
        run: |
          # Be aware that these instructions are duplicated in run-tests job
          DOCKER_TAG=$(echo $( [[ ${GITHUB_EVENT_NAME} =~ ^pull_request$ ]] && echo ${GITHUB_BASE_REF} || echo ${GITHUB_REF##*/} ) | sed 's/^master$/latest/')$( [[ ${UBUNTU_BASE} != ${DEFAULT_UBUNTU_BASE} ]] && echo "_${UBUNTU_BASE}" || echo "" )
          CTEST_BUILD_NAME=$( [[ ${GITHUB_EVENT_NAME} =~ ^pull_request$ ]] && echo "PR${GITHUB_PR_NUMBER}" || echo ${GITHUB_REF##*/} )"_${GITHUB_SHA}"
          [[ ${UBUNTU_BASE} == "20.04" ]] && PATCH_QT_3D=true || PATCH_QT_3D=false
          # build w/o 3D for Qt 5.15 (many deprecation warnings)
          [[ ${UBUNTU_BASE} == "21.04" ]] && WITH_3D=FALSE || WITH_3D=TRUE
          echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV
          echo "CTEST_BUILD_NAME=${CTEST_BUILD_NAME}" >> $GITHUB_ENV
          echo "PATCH_QT_3D=${PATCH_QT_3D}" >> $GITHUB_ENV
          echo "WITH_3D=${WITH_3D}" >> $GITHUB_ENV

      - name: Print vars
        run: |
          echo DOCKER_TAG: ${DOCKER_TAG}
          echo CTEST_BUILD_NAME: ${CTEST_BUILD_NAME}
          echo PATCH_QT_3D: ${PATCH_QT_3D}
          echo WITH_3D: ${WITH_3D}

      - name: Build deps
        env:
          UBUNTU_BASE: ${{ matrix.ubuntu-base }}
        run: |
          pushd .docker
          docker pull qgis/qgis3-build-deps:${DOCKER_TAG} || true
          docker build --build-arg UBUNTU_BASE=${UBUNTU_BASE} --cache-from qgis/qgis3-build-deps:${DOCKER_TAG} -t qgis/qgis3-build-deps:${DOCKER_TAG} -f qgis3-build-deps.dockerfile .
          popd

      - name: Push deps image
        if: ${{ github.event_name != 'pull_request' }}
        run: |
          docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}"
          docker push "qgis/qgis3-build-deps:${DOCKER_TAG}"

      - name: Prepare build cache for pull request
        uses: pat-s/always-upload-cache@v2.1.3
        if: github.event_name == 'pull_request'
        with:
          path: /home/runner/QGIS/.ccache
          key: build-ccache-${{ matrix.ubuntu-base }}-${{ github.actor }}-${{ github.head_ref }}-${{ github.sha }}
          # The head_ref or source branch of the pull request in a workflow run.
          # The base_ref or target branch of the pull request in a workflow run.
          restore-keys: |
            build-ccache-${{ matrix.ubuntu-base }}-${{ github.actor }}-${{ github.head_ref }}-
            build-ccache-${{ matrix.ubuntu-base }}-refs/heads/${{ github.base_ref }}-
            build-ccache-${{ matrix.ubuntu-base }}-refs/heads/master-

      - name: Prepare build cache for branch/tag
        # use a fork of actions/cache@v2 to upload cache even when the build or test failed
        uses: pat-s/always-upload-cache@v2.1.3
        if: github.event_name != 'pull_request'
        with:
          path: /home/runner/QGIS/.ccache
          # The branch or tag ref that triggered the workflow run. For branches this in the format refs/heads/<branch_name>, and for tags it is refs/tags/<tag_name>
          key: build-ccache-${{ matrix.ubuntu-base }}-${{ github.ref }}-${{ github.sha }}
          restore-keys: |
            build-ccache-${{ matrix.ubuntu-base }}-${{ github.ref }}-
            build-ccache-${{ matrix.ubuntu-base }}-refs/heads/master-

      - name: Compile QGIS
        id: compile
        run: |
          docker run -t --name qgis_container \
                     -v $(pwd):/root/QGIS \
                     -v /home/runner/QGIS/.ccache:/root/.ccache \
                     --env-file .docker/docker-variables.env \
                     --env PUSH_TO_CDASH=true \
                     --env WITH_3D=${WITH_3D} \
                     qgis/qgis3-build-deps:${DOCKER_TAG} \
                     /root/QGIS/.docker/docker-qgis-build.sh

      - name: Push artifact
        id: push_artifact
        if: ${{ matrix.run-tests }}
        run: tar --exclude='*.o' -cvzf build.tgz build

      - uses: actions/upload-artifact@v2
        if: ${{ matrix.run-tests }}
        with:
          name: build.tgz
          path: ./build.tgz
          retention-days: 1

#      - name: Test QGIS runners
#        id: runners
#        if: ${{ matrix.ubuntu-base == env.DEFAULT_UBUNTU_BASE }}
#        run: |
#          docker run -d --name qgis-testing-environment \
#                     -v $(pwd):/root/QGIS \
#                     -v $(pwd)/tests/src/python:/tests_directory \
#                     -v $(pwd)/.docker/qgis_resources/test_runner:/usr/bin/test_runner \
#                     -v $(pwd)/.docker/qgis_resources/supervisor:/etc/supervisor \
#                     -e QGIS_BUILD_PATH=/root/QGIS/build/output/bin/qgis \
#                     -e TEST_RUNNER_PATH=/usr/bin/test_runner/qgis_testrunner.py \
#                     -e DISPLAY=:99 \
#                     qgis_image \
#                     /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
#          # Wait for xvfb to finish starting
#          printf "Waiting for the docker...🐳..."
#          sleep 10
#          echo " done 🥩"
#
#          declare -A testrunners
#          # Passing cases:
#          testrunners["test_testrunner.run_passing"]=0
#          testrunners["test_testrunner.run_skipped_and_passing"]=0
#          # Failing cases:
#          testrunners["test_testrunner"]=1
#          testrunners["test_testrunner.run_all"]=1
#          testrunners["test_testrunner.run_failing"]=1
#          set +e # do not exit on error
#          # Run tests in the docker
#          for i in "${!testrunners[@]}"
#          do
#            echo "::group::docker_test_runner_${i}"
#            echo "test ${i}..."
#            docker exec -i qgis-testing-environment sh -c "cd /tests_directory && /usr/bin/test_runner/qgis_testrunner.sh ${i}"
#            [[ $? -eq "${testrunners[$i]}" ]] && echo "success" || exit 1
#            echo "::endgroup::"
#          done
#          set -e # switch back
#          docker stop qgis-testing-environment


#  tests-report-comment:
#    name: Write tests report in a comment
#    needs: build
#    runs-on: ubuntu-latest
#    if: always() && ( needs.build.result == 'failure' || needs.build.result == 'success' ) && github.event_name == 'pull_request'
#    steps:
#      - name: Find Comment
#        uses: peter-evans/find-comment@v1
#        id: find-comment
#        with:
#          issue-number: ${{ github.event.pull_request.number }}
#          comment-author: 'github-actions[bot]'
#          body-includes: Tests report
#
#      - name: Create comment
#        if: ${{ steps.find-comment.outputs.comment-id == 0 }}
#        uses: peter-evans/create-or-update-comment@v1
#        id: create-comment
#        with:
#          issue-number: ${{ github.event.pull_request.number }}
#          body: |
#            **Tests report**
#
#      - name: Process
#        id: process-vars
#        env:
#          COMMENT_FOUND: ${{ steps.find-comment.outputs.comment-id }}
#          COMMENT_CREATED: ${{ steps.create-comment.outputs.comment-id }}
#          COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
#          CDASH_URL: ${{ needs.build.outputs.cdash_url }}
#          COMPILE_OUTCOME: ${{ needs.build.outputs.compile_outcome }}
#          TESTS_FAILING: ${{ needs.build.outputs.tests_failing }}
#          RUNNERS_OUTCOME: ${{ needs.build.outputs.runners_outcome }}
#        run: |
#          echo "::set-output name=COMMENT_ID::"$([[ "${COMMENT_FOUND}" -eq "0" ]] && echo ${COMMENT_CREATED} || echo ${COMMENT_FOUND})
#          if [[ ${COMPILE_OUTCOME} != "success" ]]; then
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :scream: compilation failed"
#          elif [[ ${TESTS_FAILING} != "0" ]]; then
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :fire: ${TESTS_FAILING} unit-tests are failing ${CDASH_URL}"
#          elif [[ ${RUNNERS_OUTCOME} != "success" ]]; then
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :broken_heart: QGIS runners test failed"
#          else
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :sunglasses: unit-tests succeeded"
#          fi
#
#      - name: Update comment
#        uses: peter-evans/create-or-update-comment@v1
#        with:
#          comment-id: ${{ steps.process-vars.outputs.COMMENT_ID }}
#          edit-mode: append
#          body: ${{ steps.process-vars.outputs.COMMENT_BODY }}

  run-tests:
    name: Run tests
    env:
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
      GH_WORKSPACE: ${{ github.workspace }} # used in docker compose
      DEFAULT_UBUNTU_BASE: '20.04'

    runs-on: ubuntu-latest

    strategy:
      matrix:
        # tests run on 20.04 (Qt 5.12), compile test on 20.10 (Qt 5.14)
        ubuntu-base: [20.04]
        test-batch: [ALL_BUT_PROVIDERS, POSTGRES, ORACLE, HANA]
      fail-fast: false
      
    services:
      # Run httpbin container for QgsNetwor* tests
      httpbin:
        image: kennethreitz/httpbin:latest
        ports:
          - 8082:80

    needs: build

    steps:

      - name: Print disk space
        run: |
          echo "DF -H"
          sudo df -h

      - name: Checkout
        uses: actions/checkout@v2

      - name: Set vars
        env:
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          GITHUB_REF: ${{ github.ref }}
          GITHUB_PR_NUMBER: ${{github.event.number}}
          UBUNTU_BASE: ${{ matrix.ubuntu-base }}
          TEST_BATCH: ${{ matrix.test-batch }}
        run: |
          # Be aware that these instructions are duplicated in build job
          DOCKER_TAG=$(echo $( [[ ${GITHUB_EVENT_NAME} =~ ^pull_request$ ]] && echo ${GITHUB_BASE_REF} || echo ${GITHUB_REF##*/} ) | sed 's/^master$/latest/')$( [[ ${UBUNTU_BASE} != ${DEFAULT_UBUNTU_BASE} ]] && echo "_${UBUNTU_BASE}" || echo "" )
          CTEST_BUILD_NAME=$( [[ ${GITHUB_EVENT_NAME} =~ ^pull_request$ ]] && echo "PR${GITHUB_PR_NUMBER}" || echo ${GITHUB_REF##*/} )"_${GITHUB_SHA}_${TEST_BATCH}"
          [[ ${UBUNTU_BASE} == "20.04" ]] && PATCH_QT_3D=true || PATCH_QT_3D=false
          echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV
          echo "CTEST_BUILD_NAME=${CTEST_BUILD_NAME}" >> $GITHUB_ENV
          echo "PATCH_QT_3D=${PATCH_QT_3D}" >> $GITHUB_ENV
          echo "TEST_BATCH=${TEST_BATCH}" >> $GITHUB_ENV

      - name: Print vars
        run: |
          echo DOCKER_TAG: ${DOCKER_TAG}
          echo CTEST_BUILD_NAME: ${CTEST_BUILD_NAME}
          echo PATCH_QT_3D: ${PATCH_QT_3D}
          echo TEST_BATCH: ${TEST_BATCH}

      - name: Build deps
        env:
          UBUNTU_BASE: ${{ matrix.ubuntu-base }}
        run: |
          pushd .docker
          TARGET=$( ( [[ ${TEST_BATCH} == "ORACLE"   ]] && echo "binary-for-oracle" ) \
                 || echo "binary-only" )
          docker build --target $TARGET --build-arg UBUNTU_BASE=${UBUNTU_BASE} --cache-from qgis/qgis3-build-deps:${DOCKER_TAG} -t qgis/qgis3-build-deps-bin-only:${DOCKER_TAG} -f qgis3-build-deps.dockerfile .
          popd

      - name: Print disk space
        run: |
          echo "DOCKER IMAGES"
          docker images
          echo "DF -H"
          sudo df -h

      - name: Download build artifact
        uses: actions/download-artifact@v2
        with:
          name: build.tgz
          path: .

      - name: Extract build artifact
        run: |
          tar xvzf build.tgz
          rm -Rf build.tgz

      - name: Print disk space
        run: |
          echo "DF -H"
          sudo df -h

      - name: Run tests
        id: tests
        env:
          TEST_BATCH: ${{matrix.test-batch}}
          QGIS_HTTPBIN_HOST: "localhost:8082"
        run: |
          DOCKERFILE=$( ( [[ ${TEST_BATCH} == "ORACLE"   ]] && echo "docker-compose-testing-oracle.yml" ) \
                     || ( [[ ${TEST_BATCH} == "POSTGRES" ]] && echo "docker-compose-testing-postgres.yml" ) \
                     || echo "docker-compose-testing.yml" )
          echo "TEST_BATCH=$TEST_BATCH"
          echo "DOCKERFILE=$DOCKERFILE"
          docker-compose -f .docker/$DOCKERFILE run qgis-deps /root/QGIS/.docker/docker-qgis-test.sh $TEST_BATCH