sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-10-05 00:09:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

Generate a markdown summary of test failures along with existing

Browse Source 
HTML report
This commit is contained in:
Nyall Dawson 2023-10-11 10:45:00 +10:00
parent 68ea9df9ce
commit 82918f9063
7 changed files with 143 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
python/core/auto_generated/qgsmultirenderchecker.sip.in
View File

@ -124,9 +124,22 @@ Test using renderer to generate the image to be compared.
QString report() const;
%Docstring
Returns a report for this test.
Returns a HTML report for this test.
The report will be empty if the test was successfully run.
.. seealso:: :py:func:`markdownReport`
%End
QString markdownReport() const;
%Docstring
Returns a markdown report for this test.
The report will be empty if the test was successfully run.
.. seealso:: :py:func:`report`
.. versionadded:: 3.34
%End
QString controlImagePath() const;

14
python/core/auto_generated/qgsrenderchecker.sip.in
View File

@ -72,6 +72,20 @@ Returns the HTML report describing the results of the test run.
If ``ignoreSuccess`` is ``True`` then the report will always be empty if
the test was successful.
.. seealso:: :py:func:`markdownReport`
%End
QString markdownReport( bool ignoreSuccess = true ) const;
%Docstring
Returns the markdown report describing the results of the test run.
If ``ignoreSuccess`` is ``True`` then the report will always be empty if
the test was successful.
.. seealso:: :py:func:`report`
.. versionadded:: 3.34
%End
float matchPercent() const;

10
src/core/qgsmultirenderchecker.cpp
View File

@ -44,6 +44,7 @@ bool QgsMultiRenderChecker::runTest( const QString &testName, unsigned int misma
mResult = false;
mReport += "<h2>" + testName + "</h2>\n";
mMarkdownReport += QStringLiteral( "### %1\n\n" ).arg( testName );
const QString baseDir = controlImagePath();
if ( !QFile::exists( baseDir ) )
@ -97,6 +98,10 @@ bool QgsMultiRenderChecker::runTest( const QString &testName, unsigned int misma
dartMeasurements << checker.dartMeasurements();
mReport += checker.report( false );
if ( subDirs.count() > 1 )
mMarkdownReport += QStringLiteral( "* " ) + checker.markdownReport( false );
else
mMarkdownReport += checker.markdownReport( false );
if ( !mResult && diffImageFile.isEmpty() )
{
@ -173,6 +178,11 @@ QString QgsMultiRenderChecker::report() const
return !mResult ? mReport : QString();
}
QString QgsMultiRenderChecker::markdownReport() const
{
return !mResult ? mMarkdownReport : QString();
}
QString QgsMultiRenderChecker::controlImagePath() const
{
QString myDataDir( TEST_DATA_DIR ); //defined in CmakeLists.txt

15
src/core/qgsmultirenderchecker.h
View File

@ -128,12 +128,24 @@ class CORE_EXPORT QgsMultiRenderChecker
bool runTest( const QString &testName, unsigned int mismatchCount = 0 );
/**
* Returns a report for this test.
* Returns a HTML report for this test.
*
* The report will be empty if the test was successfully run.
*
* \see markdownReport()
*/
QString report() const;
/**
* Returns a markdown report for this test.
*
* The report will be empty if the test was successfully run.
*
* \see report()
* \since QGIS 3.34
*/
QString markdownReport() const;
/**
* Returns the path to the control images.
*/
@ -148,6 +160,7 @@ class CORE_EXPORT QgsMultiRenderChecker
private:
bool mResult = false;
QString mReport;
QString mMarkdownReport;
QString mRenderedImage;
QString mControlName;
QString mControlPathPrefix;

26
src/core/qgsrenderchecker.cpp
View File

@ -62,6 +62,11 @@ QString QgsRenderChecker::report( bool ignoreSuccess ) const
return ( ( ignoreSuccess && mResult ) || ( mExpectFail && !mResult ) ) ? QString() : mReport;
}
QString QgsRenderChecker::markdownReport( bool ignoreSuccess ) const
{
return ( ( ignoreSuccess && mResult ) || ( mExpectFail && !mResult ) ) ? QString() : mMarkdownReport;
}
void QgsRenderChecker::setControlName( const QString &name )
{
mControlName = name;
@ -252,6 +257,7 @@ bool QgsRenderChecker::runTest( const QString &testName,
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Expected "
"Image File not set.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because expected image file not set\n" );
performPostTestActions( flags );
return mResult;
}
@ -266,6 +272,7 @@ bool QgsRenderChecker::runTest( const QString &testName,
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Expected "
"Image File could not be loaded.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because expected image file (%1) could not be loaded\n" ).arg( mExpectedImageFile );
performPostTestActions( flags );
return mResult;
}
@ -304,6 +311,8 @@ bool QgsRenderChecker::runTest( const QString &testName,
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Rendered "
"Image File could not be saved.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because rendered image file could not be saved to %1\n" ).arg( mRenderedImageFile );
performPostTestActions( flags );
return mResult;
}
@ -340,6 +349,8 @@ bool QgsRenderChecker::compareImages( const QString &testName,
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Expected "
"Image File not set.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because expected image file was not set\n" );
performPostTestActions( flags );
return mResult;
}
@ -365,6 +376,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Rendered "
"Image File not set.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because rendered image file was not set\n" );
performPostTestActions( flags );
return mResult;
}
@ -380,6 +392,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because control "
"image file could not be loaded.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because expected image file (%1) could not be loaded\n" ).arg( referenceImageFile );
performPostTestActions( flags );
return mResult;
}
@ -400,6 +413,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
"<tr><td>Test Result:</td><td>%1:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Rendered "
"Image File could not be loaded.</td></tr></table>\n" ).arg( upperFirst( expectedImageString ) );
mMarkdownReport = QStringLiteral( "Failed because rendered image (%1) could not be loaded\n" ).arg( mRenderedImageFile );
performPostTestActions( flags );
return mResult;
}
@ -507,6 +521,11 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += QLatin1String( "<tr><td colspan=3>" );
mReport += QStringLiteral( "<font color=red>%1 and %2 for " ).arg( upperFirst( expectedImageString ), renderedImageString ) + testName + " are different dimensions - FAILING!</font>";
mReport += QLatin1String( "</td></tr>" );
mMarkdownReport += QStringLiteral( "Failed because rendered image and expected image are different dimensions (%1x%2 v2 %3x%4)\n" )
.arg( myResultImage.width() )
.arg( myResultImage.height() )
.arg( expectedImage.width() )
.arg( expectedImage.height() );
const QString diffSizeImagesString = QString(
"<tr>"
@ -547,6 +566,8 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += "<font color=red>Expected image and rendered image for " + testName + " have different formats (8bit format is expected) - FAILING!</font>";
mReport += QLatin1String( "</td></tr>" );
mReport += myImagesString;
mMarkdownReport += QStringLiteral( "Failed because rendered image and expected image have different formats (8bit format is expected)\n" );
performPostTestActions( flags );
return mResult;
}
@ -654,6 +675,9 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += QLatin1String( "<font color=red>Test failed because render step took too long</font>" );
mReport += QLatin1String( "</td></tr>" );
mReport += myImagesString;
mMarkdownReport += QStringLiteral( "Test failed because render step took too long\n" );
performPostTestActions( flags );
return mResult;
}
@ -679,6 +703,8 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += QLatin1String( "</td></tr>" );
mReport += myImagesString;
mMarkdownReport += QStringLiteral( "Rendered image did not match %1 (found %2 pixels different)\n" ).arg( referenceImageFile ).arg( mMismatchCount );
performPostTestActions( flags );
return mResult;
}

16
src/core/qgsrenderchecker.h
View File

@ -86,9 +86,22 @@ class CORE_EXPORT QgsRenderChecker
*
* If \a ignoreSuccess is TRUE then the report will always be empty if
* the test was successful.
*
* \see markdownReport()
*/
QString report( bool ignoreSuccess = true ) const;
/**
* Returns the markdown report describing the results of the test run.
*
* If \a ignoreSuccess is TRUE then the report will always be empty if
* the test was successful.
*
* \see report()
* \since QGIS 3.34
*/
QString markdownReport( bool ignoreSuccess = true ) const;
/**
* Returns the percent of pixels which matched the control image.
*/
@ -277,7 +290,10 @@ class CORE_EXPORT QgsRenderChecker
QVector<QgsDartMeasurement> dartMeasurements() const { return mDashMessages; }
protected:
//! HTML format report
QString mReport;
//! Markdown report
QString mMarkdownReport;
unsigned int mMatchTarget = 0;
int mElapsedTime = 0;
QString mRenderedImageFile;

56
src/test/qgstest.h
View File

@ -151,6 +151,8 @@ class TEST_EXPORT QgsTest : public QObject
{
if ( !mReport.isEmpty() )
writeLocalHtmlReport( mReport );
if ( !mMarkdownReport.isEmpty() )
writeMarkdownReport( mMarkdownReport );
}
/**
@ -264,7 +266,7 @@ class TEST_EXPORT QgsTest : public QObject
const bool result = checker.runTest( name, allowedMismatch );
if ( !result )
{
appendToReport( name, checker.report() );
appendToReport( name, checker.report(), checker.markdownReport() );
}
return result;
@ -290,7 +292,7 @@ class TEST_EXPORT QgsTest : public QObject
const bool result = checker.runTest( name, allowedMismatch );
if ( !result )
{
appendToReport( name, checker.report() );
appendToReport( name, checker.report(), checker.markdownReport() );
}
return result;
}
@ -308,29 +310,42 @@ class TEST_EXPORT QgsTest : public QObject
const bool result = checker.testLayout( report, page, allowedMismatch );
if ( !result )
{
appendToReport( name, report );
appendToReport( name, report, checker.markdownReport() );
}
return result;
}
/**
* Appends some \a content to the test report.
* Appends some \a html and \a markdown to the test report.
*
* This should be used only for appending useful information when a test fails.
*/
void appendToReport( const QString &testName, const QString &content )
void appendToReport( const QString &testName, const QString &html, const QString &markdown = QString() )
{
QString testIdentifier;
if ( QTest::currentDataTag() )
testIdentifier = QStringLiteral( "%1 (%2: %3)" ).arg( testName, QTest::currentTestFunction(), QTest::currentDataTag() );
else
testIdentifier = QStringLiteral( "%1 (%2)" ).arg( testName, QTest::currentTestFunction() );
mReport += QStringLiteral( "<h2>%1</h2>\n" ).arg( testIdentifier );
mReport += content;
if ( !html.isEmpty() )
{
mReport += QStringLiteral( "<h2>%1</h2>\n" ).arg( testIdentifier );
mReport += html;
}
const QString markdownContent = markdown.isEmpty() ? html : markdown;
if ( !markdownContent.isEmpty() )
{
mMarkdownReport += QStringLiteral( "## %1\n\n" ).arg( testIdentifier );
mMarkdownReport += markdownContent + QStringLiteral( "\n\n" );
}
}
private:
QString mMarkdownReport;
/**
* Writes out a HTML report to a temporary file for visual comparison
* of test results on a local build.
@ -391,6 +406,33 @@ class TEST_EXPORT QgsTest : public QObject
}
}
/**
* Writes out a markdown report to a temporary file for use on CI runs.
*/
void writeMarkdownReport( const QString &report )
{
const QDir reportDir = QgsRenderChecker::testReportDir();
if ( !reportDir.exists() )
QDir().mkpath( reportDir.path() );
const QString reportFile = reportDir.filePath( "summary.md" );
QFile file( reportFile );
QFile::OpenMode mode = QIODevice::WriteOnly;
if ( qgetenv( "QGIS_CONTINUOUS_INTEGRATION_RUN" ) == QStringLiteral( "true" )
|| qgetenv( "QGIS_APPEND_TO_TEST_REPORT" ) == QStringLiteral( "true" ) )
mode |= QIODevice::Append;
else
mode |= QIODevice::Truncate;
if ( file.open( mode ) )
{
QTextStream stream( &file );
stream << report;
file.close();
}
}
};
/**