diff --git a/python/core/auto_generated/qgsmultirenderchecker.sip.in b/python/core/auto_generated/qgsmultirenderchecker.sip.in
index 5e14d27db78..5d3fa1d859e 100644
--- a/python/core/auto_generated/qgsmultirenderchecker.sip.in
+++ b/python/core/auto_generated/qgsmultirenderchecker.sip.in
@@ -124,9 +124,22 @@ Test using renderer to generate the image to be compared.
QString report() const;
%Docstring
-Returns a report for this test.
+Returns a HTML report for this test.
The report will be empty if the test was successfully run.
+
+.. seealso:: :py:func:`markdownReport`
+%End
+
+ QString markdownReport() const;
+%Docstring
+Returns a markdown report for this test.
+
+The report will be empty if the test was successfully run.
+
+.. seealso:: :py:func:`report`
+
+.. versionadded:: 3.34
%End
QString controlImagePath() const;
diff --git a/python/core/auto_generated/qgsrenderchecker.sip.in b/python/core/auto_generated/qgsrenderchecker.sip.in
index 7b52dcc66b9..42c65b30a98 100644
--- a/python/core/auto_generated/qgsrenderchecker.sip.in
+++ b/python/core/auto_generated/qgsrenderchecker.sip.in
@@ -72,6 +72,20 @@ Returns the HTML report describing the results of the test run.
If ``ignoreSuccess`` is ``True`` then the report will always be empty if
the test was successful.
+
+.. seealso:: :py:func:`markdownReport`
+%End
+
+ QString markdownReport( bool ignoreSuccess = true ) const;
+%Docstring
+Returns the markdown report describing the results of the test run.
+
+If ``ignoreSuccess`` is ``True`` then the report will always be empty if
+the test was successful.
+
+.. seealso:: :py:func:`report`
+
+.. versionadded:: 3.34
%End
float matchPercent() const;
diff --git a/src/core/qgsmultirenderchecker.cpp b/src/core/qgsmultirenderchecker.cpp
index de4f5e05cd5..881513782d1 100644
--- a/src/core/qgsmultirenderchecker.cpp
+++ b/src/core/qgsmultirenderchecker.cpp
@@ -44,6 +44,7 @@ bool QgsMultiRenderChecker::runTest( const QString &testName, unsigned int misma
mResult = false;
mReport += "
" + testName + "
\n";
+ mMarkdownReport += QStringLiteral( "### %1\n\n" ).arg( testName );
const QString baseDir = controlImagePath();
if ( !QFile::exists( baseDir ) )
@@ -97,6 +98,10 @@ bool QgsMultiRenderChecker::runTest( const QString &testName, unsigned int misma
dartMeasurements << checker.dartMeasurements();
mReport += checker.report( false );
+ if ( subDirs.count() > 1 )
+ mMarkdownReport += QStringLiteral( "* " ) + checker.markdownReport( false );
+ else
+ mMarkdownReport += checker.markdownReport( false );
if ( !mResult && diffImageFile.isEmpty() )
{
@@ -173,6 +178,11 @@ QString QgsMultiRenderChecker::report() const
return !mResult ? mReport : QString();
}
+QString QgsMultiRenderChecker::markdownReport() const
+{
+ return !mResult ? mMarkdownReport : QString();
+}
+
QString QgsMultiRenderChecker::controlImagePath() const
{
QString myDataDir( TEST_DATA_DIR ); //defined in CmakeLists.txt
diff --git a/src/core/qgsmultirenderchecker.h b/src/core/qgsmultirenderchecker.h
index 5e34ebdaa37..46eacac9a3a 100644
--- a/src/core/qgsmultirenderchecker.h
+++ b/src/core/qgsmultirenderchecker.h
@@ -128,12 +128,24 @@ class CORE_EXPORT QgsMultiRenderChecker
bool runTest( const QString &testName, unsigned int mismatchCount = 0 );
/**
- * Returns a report for this test.
+ * Returns a HTML report for this test.
*
* The report will be empty if the test was successfully run.
+ *
+ * \see markdownReport()
*/
QString report() const;
+ /**
+ * Returns a markdown report for this test.
+ *
+ * The report will be empty if the test was successfully run.
+ *
+ * \see report()
+ * \since QGIS 3.34
+ */
+ QString markdownReport() const;
+
/**
* Returns the path to the control images.
*/
@@ -148,6 +160,7 @@ class CORE_EXPORT QgsMultiRenderChecker
private:
bool mResult = false;
QString mReport;
+ QString mMarkdownReport;
QString mRenderedImage;
QString mControlName;
QString mControlPathPrefix;
diff --git a/src/core/qgsrenderchecker.cpp b/src/core/qgsrenderchecker.cpp
index 2f31156bf1b..31282546bb5 100644
--- a/src/core/qgsrenderchecker.cpp
+++ b/src/core/qgsrenderchecker.cpp
@@ -62,6 +62,11 @@ QString QgsRenderChecker::report( bool ignoreSuccess ) const
return ( ( ignoreSuccess && mResult ) || ( mExpectFail && !mResult ) ) ? QString() : mReport;
}
+QString QgsRenderChecker::markdownReport( bool ignoreSuccess ) const
+{
+ return ( ( ignoreSuccess && mResult ) || ( mExpectFail && !mResult ) ) ? QString() : mMarkdownReport;
+}
+
void QgsRenderChecker::setControlName( const QString &name )
{
mControlName = name;
@@ -252,6 +257,7 @@ bool QgsRenderChecker::runTest( const QString &testName,
"Test Result: | Expected Result: |
\n"
"Nothing rendered | \nFailed because Expected "
"Image File not set. |
\n";
+ mMarkdownReport = QStringLiteral( "Failed because expected image file not set\n" );
performPostTestActions( flags );
return mResult;
}
@@ -266,6 +272,7 @@ bool QgsRenderChecker::runTest( const QString &testName,
"Test Result: | Expected Result: |
\n"
"Nothing rendered | \nFailed because Expected "
"Image File could not be loaded. |
\n";
+ mMarkdownReport = QStringLiteral( "Failed because expected image file (%1) could not be loaded\n" ).arg( mExpectedImageFile );
performPostTestActions( flags );
return mResult;
}
@@ -304,6 +311,8 @@ bool QgsRenderChecker::runTest( const QString &testName,
"Test Result: | Expected Result: |
\n"
"Nothing rendered | \nFailed because Rendered "
"Image File could not be saved. |
\n";
+ mMarkdownReport = QStringLiteral( "Failed because rendered image file could not be saved to %1\n" ).arg( mRenderedImageFile );
+
performPostTestActions( flags );
return mResult;
}
@@ -340,6 +349,8 @@ bool QgsRenderChecker::compareImages( const QString &testName,
"Test Result: | Expected Result: |
\n"
"Nothing rendered | \nFailed because Expected "
"Image File not set. |
\n";
+ mMarkdownReport = QStringLiteral( "Failed because expected image file was not set\n" );
+
performPostTestActions( flags );
return mResult;
}
@@ -365,6 +376,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
"Test Result: | Expected Result: |
\n"
"Nothing rendered | \nFailed because Rendered "
"Image File not set. |
\n";
+ mMarkdownReport = QStringLiteral( "Failed because rendered image file was not set\n" );
performPostTestActions( flags );
return mResult;
}
@@ -380,6 +392,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
"Test Result: | Expected Result: |
\n"
"Nothing rendered | \nFailed because control "
"image file could not be loaded. |
\n";
+ mMarkdownReport = QStringLiteral( "Failed because expected image file (%1) could not be loaded\n" ).arg( referenceImageFile );
performPostTestActions( flags );
return mResult;
}
@@ -400,6 +413,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
"Test Result: | %1: |
\n"
"Nothing rendered | \nFailed because Rendered "
"Image File could not be loaded. |
\n" ).arg( upperFirst( expectedImageString ) );
+ mMarkdownReport = QStringLiteral( "Failed because rendered image (%1) could not be loaded\n" ).arg( mRenderedImageFile );
performPostTestActions( flags );
return mResult;
}
@@ -507,6 +521,11 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += QLatin1String( "" );
mReport += QStringLiteral( "%1 and %2 for " ).arg( upperFirst( expectedImageString ), renderedImageString ) + testName + " are different dimensions - FAILING!";
mReport += QLatin1String( " |
" );
+ mMarkdownReport += QStringLiteral( "Failed because rendered image and expected image are different dimensions (%1x%2 v2 %3x%4)\n" )
+ .arg( myResultImage.width() )
+ .arg( myResultImage.height() )
+ .arg( expectedImage.width() )
+ .arg( expectedImage.height() );
const QString diffSizeImagesString = QString(
""
@@ -547,6 +566,8 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += "Expected image and rendered image for " + testName + " have different formats (8bit format is expected) - FAILING!";
mReport += QLatin1String( "
" );
mReport += myImagesString;
+
+ mMarkdownReport += QStringLiteral( "Failed because rendered image and expected image have different formats (8bit format is expected)\n" );
performPostTestActions( flags );
return mResult;
}
@@ -654,6 +675,9 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += QLatin1String( "Test failed because render step took too long" );
mReport += QLatin1String( "" );
mReport += myImagesString;
+
+ mMarkdownReport += QStringLiteral( "Test failed because render step took too long\n" );
+
performPostTestActions( flags );
return mResult;
}
@@ -679,6 +703,8 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += QLatin1String( "" );
mReport += myImagesString;
+ mMarkdownReport += QStringLiteral( "Rendered image did not match %1 (found %2 pixels different)\n" ).arg( referenceImageFile ).arg( mMismatchCount );
+
performPostTestActions( flags );
return mResult;
}
diff --git a/src/core/qgsrenderchecker.h b/src/core/qgsrenderchecker.h
index 367ae541ab9..d0763e1115d 100644
--- a/src/core/qgsrenderchecker.h
+++ b/src/core/qgsrenderchecker.h
@@ -86,9 +86,22 @@ class CORE_EXPORT QgsRenderChecker
*
* If \a ignoreSuccess is TRUE then the report will always be empty if
* the test was successful.
+ *
+ * \see markdownReport()
*/
QString report( bool ignoreSuccess = true ) const;
+ /**
+ * Returns the markdown report describing the results of the test run.
+ *
+ * If \a ignoreSuccess is TRUE then the report will always be empty if
+ * the test was successful.
+ *
+ * \see report()
+ * \since QGIS 3.34
+ */
+ QString markdownReport( bool ignoreSuccess = true ) const;
+
/**
* Returns the percent of pixels which matched the control image.
*/
@@ -277,7 +290,10 @@ class CORE_EXPORT QgsRenderChecker
QVector dartMeasurements() const { return mDashMessages; }
protected:
+ //! HTML format report
QString mReport;
+ //! Markdown report
+ QString mMarkdownReport;
unsigned int mMatchTarget = 0;
int mElapsedTime = 0;
QString mRenderedImageFile;
diff --git a/src/test/qgstest.h b/src/test/qgstest.h
index 453b30c9513..6638fd2407b 100644
--- a/src/test/qgstest.h
+++ b/src/test/qgstest.h
@@ -151,6 +151,8 @@ class TEST_EXPORT QgsTest : public QObject
{
if ( !mReport.isEmpty() )
writeLocalHtmlReport( mReport );
+ if ( !mMarkdownReport.isEmpty() )
+ writeMarkdownReport( mMarkdownReport );
}
/**
@@ -264,7 +266,7 @@ class TEST_EXPORT QgsTest : public QObject
const bool result = checker.runTest( name, allowedMismatch );
if ( !result )
{
- appendToReport( name, checker.report() );
+ appendToReport( name, checker.report(), checker.markdownReport() );
}
return result;
@@ -290,7 +292,7 @@ class TEST_EXPORT QgsTest : public QObject
const bool result = checker.runTest( name, allowedMismatch );
if ( !result )
{
- appendToReport( name, checker.report() );
+ appendToReport( name, checker.report(), checker.markdownReport() );
}
return result;
}
@@ -308,29 +310,42 @@ class TEST_EXPORT QgsTest : public QObject
const bool result = checker.testLayout( report, page, allowedMismatch );
if ( !result )
{
- appendToReport( name, report );
+ appendToReport( name, report, checker.markdownReport() );
}
return result;
}
/**
- * Appends some \a content to the test report.
+ * Appends some \a html and \a markdown to the test report.
*
* This should be used only for appending useful information when a test fails.
*/
- void appendToReport( const QString &testName, const QString &content )
+ void appendToReport( const QString &testName, const QString &html, const QString &markdown = QString() )
{
QString testIdentifier;
if ( QTest::currentDataTag() )
testIdentifier = QStringLiteral( "%1 (%2: %3)" ).arg( testName, QTest::currentTestFunction(), QTest::currentDataTag() );
else
testIdentifier = QStringLiteral( "%1 (%2)" ).arg( testName, QTest::currentTestFunction() );
- mReport += QStringLiteral( "%1
\n" ).arg( testIdentifier );
- mReport += content;
+
+ if ( !html.isEmpty() )
+ {
+ mReport += QStringLiteral( "%1
\n" ).arg( testIdentifier );
+ mReport += html;
+ }
+
+ const QString markdownContent = markdown.isEmpty() ? html : markdown;
+ if ( !markdownContent.isEmpty() )
+ {
+ mMarkdownReport += QStringLiteral( "## %1\n\n" ).arg( testIdentifier );
+ mMarkdownReport += markdownContent + QStringLiteral( "\n\n" );
+ }
}
private:
+ QString mMarkdownReport;
+
/**
* Writes out a HTML report to a temporary file for visual comparison
* of test results on a local build.
@@ -391,6 +406,33 @@ class TEST_EXPORT QgsTest : public QObject
}
}
+ /**
+ * Writes out a markdown report to a temporary file for use on CI runs.
+ */
+ void writeMarkdownReport( const QString &report )
+ {
+ const QDir reportDir = QgsRenderChecker::testReportDir();
+ if ( !reportDir.exists() )
+ QDir().mkpath( reportDir.path() );
+
+ const QString reportFile = reportDir.filePath( "summary.md" );
+ QFile file( reportFile );
+
+ QFile::OpenMode mode = QIODevice::WriteOnly;
+ if ( qgetenv( "QGIS_CONTINUOUS_INTEGRATION_RUN" ) == QStringLiteral( "true" )
+ || qgetenv( "QGIS_APPEND_TO_TEST_REPORT" ) == QStringLiteral( "true" ) )
+ mode |= QIODevice::Append;
+ else
+ mode |= QIODevice::Truncate;
+
+ if ( file.open( mode ) )
+ {
+ QTextStream stream( &file );
+ stream << report;
+ file.close();
+ }
+ }
+
};
/**