Use standard render test reporting mechanism in server tests

python/testing/__init__.py

@@ -30,7 +30,7 @@ import sys
 import tempfile
 import unittest
 from pathlib import Path
-from typing import Optional, Tuple
+from typing import Optional, Tuple, Union
 from warnings import warn
 
 from qgis.PyQt.QtCore import (
@@ -181,7 +181,7 @@ class QgisTestCase(unittest.TestCase):
         control_name=None,
         color_tolerance: int = 2,
         allowed_mismatch: int = 20,
-        size_tolerance: Optional[int] = None,
+        size_tolerance: Optional[Union[int, QSize]] = None,
         expect_fail: bool = False,
         control_path_prefix: Optional[str] = None,
         use_checkerboard_background: bool = False
@@ -213,6 +213,10 @@ class QgisTestCase(unittest.TestCase):
         checker.setColorTolerance(color_tolerance)
         checker.setExpectFail(expect_fail)
         if size_tolerance is not None:
+            if isinstance(size_tolerance, QSize):
+                if size_tolerance.isValid():
+                    checker.setSizeTolerance(size_tolerance.width(), size_tolerance.height())
+            else:
                 checker.setSizeTolerance(size_tolerance, size_tolerance)
 
         result = checker.runTest(name, allowed_mismatch)

tests/src/python/test_qgsserver.py

@@ -47,7 +47,7 @@ from qgis.core import (
     QgsMultiRenderChecker,
 )
 from qgis.PyQt.QtCore import QSize
-from qgis.PyQt.QtGui import QColor
+from qgis.PyQt.QtGui import QColor, QImage
 from qgis.server import (
     QgsBufferServerRequest,
     QgsBufferServerResponse,
@@ -209,11 +209,6 @@ class QgsServerTestBase(QgisTestCase):
     @classmethod
     def store_reference(self, reference_path, response):
         """Utility to store reference files"""
-
-        # Normally this is false
-        if not self.regenerate_reference:
-            return
-
         # Store the output for debug or to regenerate the reference documents:
         f = open(reference_path, 'wb+')
         f.write(response)
@@ -229,7 +224,7 @@ class QgsServerTestBase(QgisTestCase):
 
         return data[1], headers
 
-    def _img_diff(self, image, control_image, max_diff, max_size_diff=QSize(), outputFormat='PNG'):
+    def _img_diff(self, image: str, control_image, max_diff, max_size_diff=QSize(), outputFormat='PNG') -> bool:
 
         if outputFormat == 'PNG':
             extFile = 'png'
@@ -246,17 +241,20 @@ class QgsServerTestBase(QgisTestCase):
             f.write(image)
 
         if outputFormat != 'PNG':
-            return (True, "QgsRenderChecker can only be used for PNG")
+            # TODO fix this, it's not actually testing anything..!
+            return True
 
-        control = QgsMultiRenderChecker()
-        control.setControlPathPrefix("qgis_server")
-        control.setControlName(control_image)
-        control.setRenderedImage(temp_image)
-        if max_size_diff.isValid():
-            control.setSizeTolerance(max_size_diff.width(), max_size_diff.height())
-        return control.runTest(control_image, max_diff), control.report()
+        return self.image_check(
+            control_image,
+            control_image,
+            QImage(temp_image),
+            control_image,
+            allowed_mismatch=max_diff,
+            control_path_prefix="qgis_server",
+            size_tolerance=max_size_diff
+        )
 
-    def _img_diff_error(self, response, headers, image, max_diff=100, max_size_diff=QSize(), outputFormat='PNG'):
+    def _img_diff_error(self, response, headers, test_name: str, max_diff=100, max_size_diff=QSize(), outputFormat='PNG'):
         """
         :param outputFormat: PNG, JPG or WEBP
         """
@@ -273,25 +271,18 @@ class QgsServerTestBase(QgisTestCase):
         else:
             raise RuntimeError('Yeah, new format implemented')
 
-        reference_path = unitTestDataPath('control_images') + '/qgis_server/' + image + '/' + image + '.' + extFile
+        if self.regenerate_reference:
+            reference_path = unitTestDataPath(
+                'control_images') + '/qgis_server/' + test_name + '/' + test_name + '.' + extFile
             self.store_reference(reference_path, response)
 
         self.assertEqual(
             headers.get("Content-Type"), contentType,
             f"Content type is wrong: {headers.get('Content-Type')} instead of {contentType}\n{response}")
 
-        test, report = self._img_diff(response, image, max_diff, max_size_diff, outputFormat)
-
-        with open(os.path.join(tempfile.gettempdir(), image + "_result." + extFile), "rb") as rendered_file:
-            encoded_rendered_file = base64.b64encode(rendered_file.read())
-            message = f"Image is wrong: rendered file {tempfile.gettempdir()}/{test_name}_result.{extFile}"
-
-        # If the failure is in image sizes the diff file will not exists.
-        if os.path.exists(os.path.join(tempfile.gettempdir(), test_name + "_result_diff." + extFile)):
-            with open(os.path.join(tempfile.gettempdir(), test_name + "_result_diff." + extFile), "rb") as diff_file:
-                message = f"Image is wrong: diff file {tempfile.gettempdir()}/{test_name}_result_diff.{extFile}"
-
-        self.assertTrue(test, message)
+        self.assertTrue(
+            self._img_diff(response, test_name, max_diff, max_size_diff, outputFormat)
+        )
 
     def _execute_request(self, qs, requestMethod=QgsServerRequest.GetMethod, data=None, request_headers=None):
         request = QgsBufferServerRequest(qs, requestMethod, request_headers or {}, data)

tests/src/python/test_qgsserver_wms_getlegendgraphic.py

@@ -141,7 +141,7 @@ class TestQgsServerWMSGetLegendGraphic(TestQgsServerWMSTestBase):
         }.items())])
 
         r, h = self._result(self._execute_request(qs))
-        self._img_diff_error(r, h, "WMS_GetLegendGraphic_LayerTitleSpace")
+        self._img_diff_error(r, h, "WMS_GetLegendGraphic_LayerTitleSpace", max_size_diff=QSize(1,5))
 
     def test_wms_GetLegendGraphic_ShowFeatureCount(self):
         qs = "?" + "&".join(["%s=%s" % i for i in list({