Add mechanism to flag that a render check is expected to fail

python/core/auto_generated/qgsmultirenderchecker.sip.in

@@ -70,6 +70,13 @@ Set the path to the rendered image. If this is not set or set to null QString, a
 will be rendered based on the provided mapsettings
 
 :param renderedImagePath: A path to the rendered image with which control images will be compared
+%End
+
+    void setExpectFail( bool expectFail );
+%Docstring
+Sets whether the comparison is expected to fail.
+
+.. versionadded:: 3.28
 %End
 
     void setMapSettings( const QgsMapSettings &mapSettings );

python/core/auto_generated/qgsrenderchecker.sip.in

@@ -131,6 +131,13 @@ Gets an md5 hash that uniquely identifies an image
     void setRenderedImage( const QString &imageFileName );
 %Docstring
 Sets the file name of the rendered image generated by the test.
+%End
+
+    void setExpectFail( bool expectFail );
+%Docstring
+Sets whether the comparison is expected to fail.
+
+.. versionadded:: 3.28
 %End
 
     QString renderedImage() const;

src/core/qgsmultirenderchecker.cpp

@@ -89,7 +89,7 @@ bool QgsMultiRenderChecker::runTest( const QString &testName, unsigned int misma
     mReport += checker.report( false );
   }
 
-  if ( !mResult && mIsCiRun )
+  if ( !mResult && !mExpectFail && mIsCiRun )
   {
     const auto constDartMeasurements = dartMeasurements;
     for ( const QgsDartMeasurement &measurement : constDartMeasurements )
@@ -113,7 +113,7 @@ bool QgsMultiRenderChecker::runTest( const QString &testName, unsigned int misma
 #endif
   }
 
-  if ( !mResult )
+  if ( !mResult && !mExpectFail )
   {
     const QDir reportDir = QgsRenderChecker::testReportDir();
     if ( !reportDir.exists() )

src/core/qgsmultirenderchecker.h

@@ -81,6 +81,13 @@ class CORE_EXPORT QgsMultiRenderChecker
      */
     void setRenderedImage( const QString &renderedImagePath ) { mRenderedImage = renderedImagePath; }
 
+    /**
+     * Sets whether the comparison is expected to fail.
+     *
+     * \since QGIS 3.28
+     */
+    void setExpectFail( bool expectFail ) { mExpectFail = expectFail; }
+
     /**
      * Set the map settings to use to render the image
      *
@@ -148,6 +155,7 @@ class CORE_EXPORT QgsMultiRenderChecker
     int mMaxSizeDifferenceX = 0;
     int mMaxSizeDifferenceY = 0;
     QgsMapSettings mMapSettings;
+    bool mExpectFail = false;
 
     bool mIsCiRun = false;
 };

src/core/qgsrenderchecker.cpp

@@ -199,7 +199,7 @@ void QgsRenderChecker::dumpRenderedImageAsBase64()
 
 void QgsRenderChecker::performPostTestActions( Flags flags )
 {
-  if ( mResult )
+  if ( mResult || mExpectFail )
     return;
 
 #if DUMP_BASE64_IMAGES

src/core/qgsrenderchecker.h

@@ -147,6 +147,13 @@ class CORE_EXPORT QgsRenderChecker
      */
     void setRenderedImage( const QString &imageFileName ) { mRenderedImageFile = imageFileName; }
 
+    /**
+     * Sets whether the comparison is expected to fail.
+     *
+     * \since QGIS 3.28
+     */
+    void setExpectFail( bool expectFail ) { mExpectFail = expectFail; }
+
     /**
      * Returns the path of the rendered image generated by the test.
      *
@@ -285,6 +292,7 @@ class CORE_EXPORT QgsRenderChecker
     void performPostTestActions( Flags flags );
 
     bool mResult = false;
+    bool mExpectFail = false;
 
     QString mBasePath;
 

tests/src/python/test_qgsmapcanvas.py

@@ -110,7 +110,7 @@ class TestQgsMapCanvas(unittest.TestCase):
         canvas.waitWhileRendering()
 
         # now we expect the canvas check to fail (since they'll be a new polygon rendered over it)
-        self.assertFalse(self.canvasImageCheck('empty_canvas', 'empty_canvas', canvas))
+        self.assertFalse(self.canvasImageCheck('empty_canvas', 'empty_canvas', canvas, expect_fail=True))
 
     def testRefreshOnTimer(self):
         """ test that map canvas refreshes with auto refreshing layers """
@@ -165,7 +165,7 @@ class TestQgsMapCanvas(unittest.TestCase):
             self.assertTrue(time.time() < timeout)
 
         # now canvas should look different...
-        self.assertFalse(self.canvasImageCheck('empty_canvas', 'empty_canvas', canvas))
+        self.assertFalse(self.canvasImageCheck('empty_canvas', 'empty_canvas', canvas, expect_fail=True))
 
         # switch off auto refresh
         layer.setAutoRefreshEnabled(False)
@@ -346,7 +346,7 @@ class TestQgsMapCanvas(unittest.TestCase):
         canvas.refresh()
         canvas.waitWhileRendering()
         # should be different - we should now render project layers
-        self.assertFalse(self.canvasImageCheck('theme4', 'theme4', canvas))
+        self.assertFalse(self.canvasImageCheck('theme4', 'theme4', canvas, expect_fail=True))
 
         # set canvas to theme1
         canvas.setTheme('theme1')
@@ -399,7 +399,7 @@ class TestQgsMapCanvas(unittest.TestCase):
         canvas.waitWhileRendering()
 
         # no annotation yet...
-        self.assertFalse(self.canvasImageCheck('main_annotation_layer', 'main_annotation_layer', canvas))
+        self.assertFalse(self.canvasImageCheck('main_annotation_layer', 'main_annotation_layer', canvas, expect_fail=True))
 
         annotation_layer = QgsProject.instance().mainAnnotationLayer()
         annotation_layer.setCrs(QgsCoordinateReferenceSystem('EPSG:4326'))
@@ -417,7 +417,7 @@ class TestQgsMapCanvas(unittest.TestCase):
         self.assertTrue(self.canvasImageCheck('main_annotation_layer', 'main_annotation_layer', canvas))
         annotation_layer.clear()
 
-    def canvasImageCheck(self, name, reference_image, canvas):
+    def canvasImageCheck(self, name, reference_image, canvas, expect_fail=False):
         self.report += "<h2>Render {}</h2>\n".format(name)
         temp_dir = QDir.tempPath() + '/'
         file_name = temp_dir + 'mapcanvas_' + name + ".png"
@@ -428,6 +428,7 @@ class TestQgsMapCanvas(unittest.TestCase):
         checker.setControlName("expected_" + reference_image)
         checker.setRenderedImage(file_name)
         checker.setColorTolerance(2)
+        checker.setExpectFail(expect_fail)
         result = checker.runTest(name, 20)
         self.report += checker.report()
         print((self.report))

tests/src/python/test_qgsrandommarkersymbollayer.py

@@ -128,7 +128,7 @@ class TestQgsRandomMarkerSymbolLayer(unittest.TestCase):
         g = QgsGeometry.fromWkt(
             'Polygon((0 0, 10 0, 10 10, 0 10, 0 0),(1 1, 1 2, 2 2, 2 1, 1 1),(8 8, 9 8, 9 9, 8 9, 8 8))')
         rendered_image = self.renderGeometry(s3, g)
-        self.assertFalse(self.imageCheck('randommarkerfill_seed', 'randommarkerfill_seed', rendered_image))
+        self.assertFalse(self.imageCheck('randommarkerfill_seed', 'randommarkerfill_seed', rendered_image, expect_fail=True))
 
         # density-based count
         s3.symbolLayer(0).setSeed(1)
@@ -324,7 +324,7 @@ class TestQgsRandomMarkerSymbolLayer(unittest.TestCase):
 
         return image
 
-    def imageCheck(self, name, reference_image, image):
+    def imageCheck(self, name, reference_image, image, expect_fail=False):
         self.report += "<h2>Render {}</h2>\n".format(name)
         temp_dir = QDir.tempPath() + '/'
         file_name = temp_dir + 'symbol_' + name + ".png"
@@ -333,6 +333,7 @@ class TestQgsRandomMarkerSymbolLayer(unittest.TestCase):
         checker.setControlPathPrefix("symbol_randommarkerfill")
         checker.setControlName("expected_" + reference_image)
         checker.setRenderedImage(file_name)
+        checker.setExpectFail(expect_fail)
         checker.setColorTolerance(2)
         result = checker.runTest(name, 20)
         TestQgsRandomMarkerSymbolLayer.report += checker.report()