Generate a markdown summary of test failures along with existing

HTML report

python/core/auto_generated/qgsmultirenderchecker.sip.in

@@ -124,9 +124,22 @@ Test using renderer to generate the image to be compared.
 
     QString report() const;
 %Docstring
-Returns a report for this test.
+Returns a HTML report for this test.
 
 The report will be empty if the test was successfully run.
+
+.. seealso:: :py:func:`markdownReport`
+%End
+
+    QString markdownReport() const;
+%Docstring
+Returns a markdown report for this test.
+
+The report will be empty if the test was successfully run.
+
+.. seealso:: :py:func:`report`
+
+.. versionadded:: 3.34
 %End
 
     QString controlImagePath() const;

python/core/auto_generated/qgsrenderchecker.sip.in

@@ -72,6 +72,20 @@ Returns the HTML report describing the results of the test run.
 
 If ``ignoreSuccess`` is ``True`` then the report will always be empty if
 the test was successful.
+
+.. seealso:: :py:func:`markdownReport`
+%End
+
+    QString markdownReport( bool ignoreSuccess = true ) const;
+%Docstring
+Returns the markdown report describing the results of the test run.
+
+If ``ignoreSuccess`` is ``True`` then the report will always be empty if
+the test was successful.
+
+.. seealso:: :py:func:`report`
+
+.. versionadded:: 3.34
 %End
 
     float matchPercent() const;

src/core/qgsmultirenderchecker.cpp

@@ -44,6 +44,7 @@ bool QgsMultiRenderChecker::runTest( const QString &testName, unsigned int misma
   mResult = false;
 
   mReport += "<h2>" + testName + "</h2>\n";
+  mMarkdownReport += QStringLiteral( "### %1\n\n" ).arg( testName );
 
   const QString baseDir = controlImagePath();
   if ( !QFile::exists( baseDir ) )
@@ -97,6 +98,10 @@ bool QgsMultiRenderChecker::runTest( const QString &testName, unsigned int misma
     dartMeasurements << checker.dartMeasurements();
 
     mReport += checker.report( false );
+    if ( subDirs.count() > 1 )
+      mMarkdownReport += QStringLiteral( "* " ) + checker.markdownReport( false );
+    else
+      mMarkdownReport += checker.markdownReport( false );
 
     if ( !mResult && diffImageFile.isEmpty() )
     {
@@ -173,6 +178,11 @@ QString QgsMultiRenderChecker::report() const
   return !mResult ? mReport : QString();
 }
 
+QString QgsMultiRenderChecker::markdownReport() const
+{
+  return !mResult ? mMarkdownReport : QString();
+}
+
 QString QgsMultiRenderChecker::controlImagePath() const
 {
   QString myDataDir( TEST_DATA_DIR ); //defined in CmakeLists.txt

src/core/qgsmultirenderchecker.h

@@ -128,12 +128,24 @@ class CORE_EXPORT QgsMultiRenderChecker
     bool runTest( const QString &testName, unsigned int mismatchCount = 0 );
 
     /**
-     * Returns a report for this test.
+     * Returns a HTML report for this test.
      *
      * The report will be empty if the test was successfully run.
+     *
+     * \see markdownReport()
      */
     QString report() const;
 
+    /**
+     * Returns a markdown report for this test.
+     *
+     * The report will be empty if the test was successfully run.
+     *
+     * \see report()
+     * \since QGIS 3.34
+     */
+    QString markdownReport() const;
+
     /**
      * Returns the path to the control images.
      */
@@ -148,6 +160,7 @@ class CORE_EXPORT QgsMultiRenderChecker
   private:
     bool mResult = false;
     QString mReport;
+    QString mMarkdownReport;
     QString mRenderedImage;
     QString mControlName;
     QString mControlPathPrefix;

src/core/qgsrenderchecker.cpp

@@ -62,6 +62,11 @@ QString QgsRenderChecker::report( bool ignoreSuccess ) const
   return ( ( ignoreSuccess && mResult ) || ( mExpectFail && !mResult ) ) ? QString() : mReport;
 }
 
+QString QgsRenderChecker::markdownReport( bool ignoreSuccess ) const
+{
+  return ( ( ignoreSuccess && mResult ) || ( mExpectFail && !mResult ) ) ? QString() : mMarkdownReport;
+}
+
 void QgsRenderChecker::setControlName( const QString &name )
 {
   mControlName = name;
@@ -252,6 +257,7 @@ bool QgsRenderChecker::runTest( const QString &testName,
               "<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
               "<tr><td>Nothing rendered</td>\n<td>Failed because Expected "
               "Image File not set.</td></tr></table>\n";
+    mMarkdownReport = QStringLiteral( "Failed because expected image file not set\n" );
     performPostTestActions( flags );
     return mResult;
   }
@@ -266,6 +272,7 @@ bool QgsRenderChecker::runTest( const QString &testName,
               "<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
               "<tr><td>Nothing rendered</td>\n<td>Failed because Expected "
               "Image File could not be loaded.</td></tr></table>\n";
+    mMarkdownReport = QStringLiteral( "Failed because expected image file (%1) could not be loaded\n" ).arg( mExpectedImageFile );
     performPostTestActions( flags );
     return mResult;
   }
@@ -304,6 +311,8 @@ bool QgsRenderChecker::runTest( const QString &testName,
               "<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
               "<tr><td>Nothing rendered</td>\n<td>Failed because Rendered "
               "Image File could not be saved.</td></tr></table>\n";
+    mMarkdownReport = QStringLiteral( "Failed because rendered image file could not be saved to %1\n" ).arg( mRenderedImageFile );
+
     performPostTestActions( flags );
     return mResult;
   }
@@ -340,6 +349,8 @@ bool QgsRenderChecker::compareImages( const QString &testName,
               "<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
               "<tr><td>Nothing rendered</td>\n<td>Failed because Expected "
               "Image File not set.</td></tr></table>\n";
+    mMarkdownReport = QStringLiteral( "Failed because expected image file was not set\n" );
+
     performPostTestActions( flags );
     return mResult;
   }
@@ -365,6 +376,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
               "<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
               "<tr><td>Nothing rendered</td>\n<td>Failed because Rendered "
               "Image File not set.</td></tr></table>\n";
+    mMarkdownReport = QStringLiteral( "Failed because rendered image file was not set\n" );
     performPostTestActions( flags );
     return mResult;
   }
@@ -380,6 +392,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
               "<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
               "<tr><td>Nothing rendered</td>\n<td>Failed because control "
               "image file could not be loaded.</td></tr></table>\n";
+    mMarkdownReport = QStringLiteral( "Failed because expected image file (%1) could not be loaded\n" ).arg( referenceImageFile );
     performPostTestActions( flags );
     return mResult;
   }
@@ -400,6 +413,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
                               "<tr><td>Test Result:</td><td>%1:</td></tr>\n"
                               "<tr><td>Nothing rendered</td>\n<td>Failed because Rendered "
                               "Image File could not be loaded.</td></tr></table>\n" ).arg( upperFirst( expectedImageString ) );
+    mMarkdownReport = QStringLiteral( "Failed because rendered image (%1) could not be loaded\n" ).arg( mRenderedImageFile );
     performPostTestActions( flags );
     return mResult;
   }
@@ -507,6 +521,11 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
       mReport += QLatin1String( "<tr><td colspan=3>" );
       mReport += QStringLiteral( "<font color=red>%1 and %2 for " ).arg( upperFirst( expectedImageString ), renderedImageString ) + testName + " are different dimensions - FAILING!</font>";
       mReport += QLatin1String( "</td></tr>" );
+      mMarkdownReport += QStringLiteral( "Failed because rendered image and expected image are different dimensions (%1x%2 v2 %3x%4)\n" )
+                         .arg( myResultImage.width() )
+                         .arg( myResultImage.height() )
+                         .arg( expectedImage.width() )
+                         .arg( expectedImage.height() );
 
       const QString diffSizeImagesString = QString(
                                              "<tr>"
@@ -547,6 +566,8 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
       mReport += "<font color=red>Expected image and rendered image for " + testName + " have different formats (8bit format is expected) - FAILING!</font>";
       mReport += QLatin1String( "</td></tr>" );
       mReport += myImagesString;
+
+      mMarkdownReport += QStringLiteral( "Failed because rendered image and expected image have different formats (8bit format is expected)\n" );
       performPostTestActions( flags );
       return mResult;
     }
@@ -654,6 +675,9 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
       mReport += QLatin1String( "<font color=red>Test failed because render step took too long</font>" );
       mReport += QLatin1String( "</td></tr>" );
       mReport += myImagesString;
+
+      mMarkdownReport += QStringLiteral( "Test failed because render step took too long\n" );
+
       performPostTestActions( flags );
       return mResult;
     }
@@ -679,6 +703,8 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
   mReport += QLatin1String( "</td></tr>" );
   mReport += myImagesString;
 
+  mMarkdownReport += QStringLiteral( "Rendered image did not match %1 (found %2 pixels different)\n" ).arg( referenceImageFile ).arg( mMismatchCount );
+
   performPostTestActions( flags );
   return mResult;
 }

src/core/qgsrenderchecker.h

@@ -86,9 +86,22 @@ class CORE_EXPORT QgsRenderChecker
      *
      * If \a ignoreSuccess is TRUE then the report will always be empty if
      * the test was successful.
+     *
+     * \see markdownReport()
      */
     QString report( bool ignoreSuccess = true ) const;
 
+    /**
+     * Returns the markdown report describing the results of the test run.
+     *
+     * If \a ignoreSuccess is TRUE then the report will always be empty if
+     * the test was successful.
+     *
+     * \see report()
+     * \since QGIS 3.34
+     */
+    QString markdownReport( bool ignoreSuccess = true ) const;
+
     /**
      * Returns the percent of pixels which matched the control image.
      */
@@ -277,7 +290,10 @@ class CORE_EXPORT QgsRenderChecker
     QVector<QgsDartMeasurement> dartMeasurements() const { return mDashMessages; }
 
   protected:
+    //! HTML format report
     QString mReport;
+    //! Markdown report
+    QString mMarkdownReport;
     unsigned int mMatchTarget = 0;
     int mElapsedTime = 0;
     QString mRenderedImageFile;

src/test/qgstest.h

@@ -151,6 +151,8 @@ class TEST_EXPORT QgsTest : public QObject
     {
       if ( !mReport.isEmpty() )
         writeLocalHtmlReport( mReport );
+      if ( !mMarkdownReport.isEmpty() )
+        writeMarkdownReport( mMarkdownReport );
     }
 
     /**
@@ -264,7 +266,7 @@ class TEST_EXPORT QgsTest : public QObject
       const bool result = checker.runTest( name, allowedMismatch );
       if ( !result )
       {
-        appendToReport( name, checker.report() );
+        appendToReport( name, checker.report(), checker.markdownReport() );
 
       }
       return result;
@@ -290,7 +292,7 @@ class TEST_EXPORT QgsTest : public QObject
       const bool result = checker.runTest( name, allowedMismatch );
       if ( !result )
       {
-        appendToReport( name, checker.report() );
+        appendToReport( name, checker.report(), checker.markdownReport() );
       }
       return result;
     }
@@ -308,29 +310,42 @@ class TEST_EXPORT QgsTest : public QObject
       const bool result = checker.testLayout( report, page, allowedMismatch );
       if ( !result )
       {
-        appendToReport( name, report );
+        appendToReport( name, report, checker.markdownReport() );
       }
       return result;
     }
 
     /**
-     * Appends some \a content to the test report.
+     * Appends some \a html and \a markdown to the test report.
      *
      * This should be used only for appending useful information when a test fails.
      */
-    void appendToReport( const QString &testName, const QString &content )
+    void appendToReport( const QString &testName, const QString &html, const QString &markdown = QString() )
     {
       QString testIdentifier;
       if ( QTest::currentDataTag() )
         testIdentifier = QStringLiteral( "%1 (%2: %3)" ).arg( testName, QTest::currentTestFunction(), QTest::currentDataTag() );
       else
         testIdentifier = QStringLiteral( "%1 (%2)" ).arg( testName, QTest::currentTestFunction() );
-      mReport += QStringLiteral( "<h2>%1</h2>\n" ).arg( testIdentifier );
+
-      mReport += content;
+      if ( !html.isEmpty() )
+      {
+        mReport += QStringLiteral( "<h2>%1</h2>\n" ).arg( testIdentifier );
+        mReport += html;
+      }
+
+      const QString markdownContent = markdown.isEmpty() ? html : markdown;
+      if ( !markdownContent.isEmpty() )
+      {
+        mMarkdownReport += QStringLiteral( "## %1\n\n" ).arg( testIdentifier );
+        mMarkdownReport += markdownContent + QStringLiteral( "\n\n" );
+      }
     }
 
   private:
 
+    QString mMarkdownReport;
+
     /**
      * Writes out a HTML report to a temporary file for visual comparison
      * of test results on a local build.
@@ -391,6 +406,33 @@ class TEST_EXPORT QgsTest : public QObject
       }
     }
 
+    /**
+     * Writes out a markdown report to a temporary file for use on CI runs.
+     */
+    void writeMarkdownReport( const QString &report )
+    {
+      const QDir reportDir = QgsRenderChecker::testReportDir();
+      if ( !reportDir.exists() )
+        QDir().mkpath( reportDir.path() );
+
+      const QString reportFile = reportDir.filePath( "summary.md" );
+      QFile file( reportFile );
+
+      QFile::OpenMode mode = QIODevice::WriteOnly;
+      if ( qgetenv( "QGIS_CONTINUOUS_INTEGRATION_RUN" ) == QStringLiteral( "true" )
+           || qgetenv( "QGIS_APPEND_TO_TEST_REPORT" ) == QStringLiteral( "true" ) )
+        mode |= QIODevice::Append;
+      else
+        mode |= QIODevice::Truncate;
+
+      if ( file.open( mode ) )
+      {
+        QTextStream stream( &file );
+        stream << report;
+        file.close();
+      }
+    }
+
 };
 
 /**