[unit tests] Add multirenderchecker

The multirenderchecker allows to have several images, each with its own set of
anomalies distributed in several subdirectories.
With the help of multiple reference images, it is possible to apply a color
tolerance to each of these

python/core/core.sip

@@ -66,6 +66,7 @@
 %Include qgsmessagelog.sip
 %Include qgsmessageoutput.sip
 %Include qgsmimedatautils.sip
+%Include qgsmultirenderchecker.sip
 %Include qgsnetworkaccessmanager.sip
 %Include qgsnetworkcontentfetcher.sip
 %Include qgsofflineediting.sip

python/core/qgsmultirenderchecker.sip

@@ -0,0 +1,91 @@
+/***************************************************************************
+    qgsmultirenderchecker.sip
+     --------------------------------------
+    Date                 : 6.11.2014
+    Copyright            : (C) 2014 Matthias Kuhn
+    Email                : matthias dot kuhn at gmx dot ch
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+/**
+ *
+ * @note added in 2.8
+ */
+
+class QgsMultiRenderChecker
+{
+%TypeHeaderCode
+#include <qgsmultirenderchecker.h>
+%End
+  public:
+    QgsMultiRenderChecker();
+
+    /**
+     * Base directory name for the control image (with control image path
+     * suffixed) the path to the image will be constructed like this:
+     * controlImagePath + '/' + mControlName + '/' + mControlName + '.png'
+     */
+    void setControlName( const QString& theName );
+
+    void setControlPathPrefix( const QString& prefix );
+
+    /**
+     * Set the path to the rendered image. If this is not set or set to QString::Null, an image
+     * will be rendered based on the provided mapsettings
+     *
+     * @param renderedImagePath A path to the rendered image with which control images will be compared
+     */
+    void setRenderedImage( const QString& renderedImagePath );
+
+    /**
+     * Set the map settings to use to render the image
+     *
+     * @param mapSettings The map settings
+     */
+    void setMapSettings( const QgsMapSettings& mapSettings );
+
+    /**
+     * Set tolerance for color components used by runTest()
+     * Default value is 0.
+     *
+     * @param theColorTolerance The maximum difference for each color component
+     *                          including alpha to be considered correct.
+     */
+    void setColorTolerance( unsigned int theColorTolerance );
+
+    /**
+     * Test using renderer to generate the image to be compared.
+     *
+     * @param theTestName - to be used as the basis for writing a file to
+     * e.g. /tmp/theTestName.png
+     *
+     * @param theMismatchCount - defaults to 0 - the number of pixels that
+     * are allowed to be different from the control image. In some cases
+     * rendering may be non-deterministic. This parameter allows you to account
+     * for that by providing a tolerance.
+     *
+     * @note make sure to call setExpectedImage and setMapSettings first
+     */
+    bool runTest( const QString& theTestName, unsigned int theMismatchCount = 0 );
+
+    /**
+     * Returns a report for this test
+     *
+     * @return A report
+     */
+    const QString& report() const;
+
+    /**
+     * @brief controlImagePath
+     * @return
+     */
+    const QString controlImagePath() const;
+
+};
+

src/core/CMakeLists.txt

@@ -114,12 +114,13 @@ SET(QGIS_CORE_SRCS
   qgsmapsettings.cpp
   qgsmaptopixel.cpp
   qgsmaptopixelgeometrysimplifier.cpp
+  qgsmessagelog.cpp
   qgsmessageoutput.cpp
   qgsmimedatautils.cpp
-  qgsmessagelog.cpp
+  qgsmultirenderchecker.cpp
   qgsnetworkaccessmanager.cpp
-  qgsnetworkreplyparser.cpp
   qgsnetworkcontentfetcher.cpp
+  qgsnetworkreplyparser.cpp
   qgsobjectcustomproperties.cpp
   qgsofflineediting.cpp
   qgsogcutils.cpp
@@ -131,8 +132,8 @@ SET(QGIS_CORE_SRCS
   qgspoint.cpp
   qgsproject.cpp
   qgsprojectfiletransform.cpp
-  qgsprojectversion.cpp
   qgsprojectproperty.cpp
+  qgsprojectversion.cpp
   qgsprovidercountcalcevent.cpp
   qgsproviderextentcalcevent.cpp
   qgsprovidermetadata.cpp
@@ -140,8 +141,8 @@ SET(QGIS_CORE_SRCS
   qgspythonrunner.cpp
   qgsrelation.cpp
   qgsrelationmanager.cpp
-  qgsrendercontext.cpp
   qgsrenderchecker.cpp
+  qgsrendercontext.cpp
   qgsrectangle.cpp
   qgsrunprocess.cpp
   qgsscalecalculator.cpp
@@ -513,8 +514,9 @@ SET(QGIS_CORE_HDRS
   qgsmapunitscale.h
   qgsmessageoutput.h
   qgsmimedatautils.h
-  qgsnetworkreplyparser.h
+  qgsmultirenderchecker.h
   qgsnetworkcontentfetcher.h
+  qgsnetworkreplyparser.h
   qgsobjectcustomproperties.h
   qgsofflineediting.h
   qgsogcutils.h
@@ -534,10 +536,10 @@ SET(QGIS_CORE_HDRS
   qgsproviderregistry.h
   qgspythonrunner.h
   qgsrectangle.h
-  qgsrendercontext.h
-  qgsrenderchecker.h
   qgsrelation.h
   qgsrelationmanager.h
+  qgsrenderchecker.h
+  qgsrendercontext.h
   qgsrunprocess.h
   qgsscalecalculator.h
   qgsscaleutils.h

src/core/qgsmultirenderchecker.cpp

@@ -0,0 +1,92 @@
+/***************************************************************************
+    qgsmultirenderchecker.cpp
+     --------------------------------------
+    Date                 : 6.11.2014
+    Copyright            : (C) 2014 Matthias Kuhn
+    Email                : matthias dot kuhn at gmx dot ch
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsmultirenderchecker.h"
+
+#include <QDebug>
+
+QgsMultiRenderChecker::QgsMultiRenderChecker()
+{
+}
+
+void QgsMultiRenderChecker::setControlName( const QString& theName )
+{
+  mControlName = theName;
+}
+
+void QgsMultiRenderChecker::setControlPathPrefix( const QString& prefix )
+{
+  mControlPathPrefix = prefix;
+}
+
+void QgsMultiRenderChecker::setMapSettings( const QgsMapSettings& mapSettings )
+{
+  mMapSettings = mapSettings;
+}
+
+bool QgsMultiRenderChecker::runTest( const QString& theTestName, unsigned int theMismatchCount )
+{
+  bool successful = false;
+
+  const QString baseDir = controlImagePath();
+
+  QStringList subDirs = QDir( baseDir ).entryList( QDir::Dirs | QDir::NoDotAndDotDot );
+
+  if ( subDirs.count() == 0 )
+  {
+    subDirs << "";
+  }
+
+  Q_FOREACH( const QString& suffix, subDirs )
+  {
+    qDebug() << "Checking subdir " << suffix;
+    bool result;
+    QgsRenderChecker checker;
+    checker.setColorTolerance( mColorTolerance );
+    checker.setControlPathPrefix( mControlPathPrefix );
+    checker.setControlPathSuffix( suffix );
+    checker.setControlName( mControlName );
+    checker.setMapSettings( mMapSettings );
+
+    if ( !mRenderedImage.isNull() )
+    {
+      checker.setRenderedImage( mRenderedImage );
+      result = checker.compareImages( theTestName, theMismatchCount, mRenderedImage );
+    }
+    else
+    {
+      result = checker.runTest( theTestName, theMismatchCount );
+      mRenderedImage = checker.renderedImage();
+    }
+
+    qDebug() << " * Subdir check " << suffix << ": " << result;
+    successful |= result;
+
+    mReport += checker.report();
+  }
+
+  if ( !successful )
+    qDebug() << "No matching image found. If you think that this result should be considered ok, please copy it into a new subdirectory inside " << baseDir;
+
+  return successful;
+}
+
+const QString QgsMultiRenderChecker::controlImagePath() const
+{
+  QString myDataDir( TEST_DATA_DIR ); //defined in CmakeLists.txt
+  QString myControlImageDir = myDataDir + QDir::separator() + "control_images" +
+                              QDir::separator() + mControlPathPrefix + QDir::separator() + mControlName + QDir::separator();
+  return myControlImageDir;
+}

src/core/qgsmultirenderchecker.h

@@ -0,0 +1,126 @@
+/***************************************************************************
+    qgsmultirenderchecker.h
+     --------------------------------------
+    Date                 : 6.11.2014
+    Copyright            : (C) 2014 Matthias Kuhn
+    Email                : matthias dot kuhn at gmx dot ch
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSMULTIRENDERCHECKER_H
+#define QGSMULTIRENDERCHECKER_H
+
+#include "qgsrenderchecker.h"
+
+/**
+ * This class allows to check rendered images against comparison images.
+ * Its main purpose is for the unit testing framework.
+ *
+ * It will either
+ * <ul>
+ *   <li>take an externally rendered image (setRenderedImage())</li>
+ *   <li>render the image based on provided mapSettings (setMapSettings())</li>
+ * </ul>
+ *
+ * This image will then be compared against one or several images in a folder inside
+ * the control directory (tests/testdata/control_images/{controlName}).
+ *
+ * There are modes for single and for multiple reference images.
+ * <ul>
+ *   <li>If there are no subfolders in the control directory, it will assume an image
+ *       with the name {controlImage}.png in the control directory itself.</li>
+ *
+ *   <li>If there are subfolders inside the control directory, it will search for images
+ *       with the name {controlImage}.png in every subfolder.</li>
+ * </ul>
+ *
+ * For every control image there may be one or several randomly named anomaly images defining
+ * allowed anomalies.
+ * For every control image, the allowed mismatch and color tolerance values will be calculated
+ * individually.
+ *
+ * @note added in 2.8
+ */
+
+class CORE_EXPORT QgsMultiRenderChecker
+{
+  public:
+    QgsMultiRenderChecker();
+
+    /**
+     * Base directory name for the control image (with control image path
+     * suffixed) the path to the image will be constructed like this:
+     * controlImagePath + '/' + mControlName + '/' + mControlName + '.png'
+     */
+    void setControlName( const QString& theName );
+
+    void setControlPathPrefix( const QString& prefix );
+
+    /**
+     * Set the path to the rendered image. If this is not set or set to QString::Null, an image
+     * will be rendered based on the provided mapsettings
+     *
+     * @param renderedImagePath A path to the rendered image with which control images will be compared
+     */
+    void setRenderedImage( const QString& renderedImagePath ) { mRenderedImage = renderedImagePath; }
+
+    /**
+     * Set the map settings to use to render the image
+     *
+     * @param mapSettings The map settings
+     */
+    void setMapSettings( const QgsMapSettings& mapSettings );
+
+    /**
+     * Set tolerance for color components used by runTest()
+     * Default value is 0.
+     *
+     * @param theColorTolerance The maximum difference for each color component
+     *                          including alpha to be considered correct.
+     */
+    void setColorTolerance( unsigned int theColorTolerance ) { mColorTolerance = theColorTolerance; }
+
+    /**
+     * Test using renderer to generate the image to be compared.
+     *
+     * @param theTestName - to be used as the basis for writing a file to
+     * e.g. /tmp/theTestName.png
+     *
+     * @param theMismatchCount - defaults to 0 - the number of pixels that
+     * are allowed to be different from the control image. In some cases
+     * rendering may be non-deterministic. This parameter allows you to account
+     * for that by providing a tolerance.
+     *
+     * @note make sure to call setExpectedImage and setMapSettings first
+     */
+    bool runTest( const QString& theTestName, unsigned int theMismatchCount = 0 );
+
+    /**
+     * Returns a report for this test
+     *
+     * @return A report
+     */
+    const QString& report() const { return mReport; }
+
+    /**
+     * @brief controlImagePath
+     * @return
+     */
+    const QString controlImagePath() const;
+
+  private:
+    QString mReport;
+    QString mRenderedImage;
+    QString mControlName;
+    QString mControlPathPrefix;
+    unsigned int mColorTolerance;
+    QgsMapSettings mMapSettings;
+};
+
+#endif // QGSMULTIRENDERCHECKER_H

src/core/qgsrenderchecker.cpp

@@ -51,7 +51,7 @@ QString QgsRenderChecker::controlImagePath() const
 void QgsRenderChecker::setControlName( const QString theName )
 {
   mControlName = theName;
-  mExpectedImageFile = controlImagePath() + theName + QDir::separator()
+  mExpectedImageFile = controlImagePath() + theName + QDir::separator() + mControlPathSuffix
                        + theName + ".png";
 }
 

src/core/qgsrenderchecker.h

@@ -55,19 +55,32 @@ class CORE_EXPORT QgsRenderChecker
     //only records time for actual render part
     int elapsedTime() { return mElapsedTime; }
     void setElapsedTimeTarget( int theTarget ) { mElapsedTimeTarget = theTarget; };
+
     /** Base directory name for the control image (with control image path
       * suffixed) the path to the image will be constructed like this:
       * controlImagePath + '/' + mControlName + '/' + mControlName + '.png'
       */
     void setControlName( const QString theName );
+
     /** Prefix where the control images are kept.
      * This will be appended to controlImagePath
       */
     void setControlPathPrefix( const QString theName ) { mControlPathPrefix = theName + QDir::separator(); }
+
+    void setControlPathSuffix( const QString& theName ) { mControlPathSuffix = theName + QDir::separator(); }
+
     /** Get an md5 hash that uniquely identifies an image */
     QString imageToHash( QString theImageFile );
 
     void setRenderedImage( QString theImageFileName ) { mRenderedImageFile = theImageFileName; }
+
+    /**
+     * The path of the rendered image can be retrieved through that method.
+     * Will return the path set with setRenderedImage() or generated in runTest()
+     *
+     * @return The path to the rendered image
+     */
+    const QString& renderedImage() { return mRenderedImageFile; }
     //! @deprecated since 2.4 - use setMapSettings()
     Q_DECL_DEPRECATED void setMapRenderer( QgsMapRenderer *  thepMapRenderer );
 
@@ -130,6 +143,7 @@ class CORE_EXPORT QgsRenderChecker
     int mElapsedTimeTarget;
     QgsMapSettings mMapSettings;
     QString mControlPathPrefix;
+    QString mControlPathSuffix;
 
 }; // class QgsRenderChecker
 

tests/src/core/qgscompositionchecker.cpp

@@ -22,9 +22,11 @@
 #include <QPainter>
 
 QgsCompositionChecker::QgsCompositionChecker( const QString& testName, QgsComposition* composition )
-    : QgsRenderChecker(),
+    : QgsMultiRenderChecker(),
     mTestName( testName ),
-    mComposition( composition )
+    mComposition( composition ),
+    mSize( 1122, 794 ),
+    mDotsPerMeter( 96 / 25.4 * 1000 )
 {
 }
 
@@ -36,7 +38,7 @@ QgsCompositionChecker::~QgsCompositionChecker()
 {
 }
 
-bool QgsCompositionChecker::testComposition( QString &report, int page, int pixelDiff )
+bool QgsCompositionChecker::testComposition( QString &theReport, int page, int pixelDiff )
 {
   if ( !mComposition )
   {
@@ -62,17 +64,11 @@ bool QgsCompositionChecker::testComposition( QString &report, int page, int pixe
   return true;
 #endif //0
 
-  //load expected image
+  QImage outputImage( mSize, QImage::Format_ARGB32 );
-  QImage expectedImage( mExpectedImageFile );
-
-  //get width/height, create image and render the composition to it
-  int width = expectedImage.width();
-  int height = expectedImage.height();
-  QImage outputImage( QSize( width, height ), QImage::Format_ARGB32 );
 
   mComposition->setPlotStyle( QgsComposition::Print );
-  outputImage.setDotsPerMeterX( expectedImage.dotsPerMeterX() );
+  outputImage.setDotsPerMeterX( mDotsPerMeter );
-  outputImage.setDotsPerMeterY( expectedImage.dotsPerMeterX() );
+  outputImage.setDotsPerMeterY( mDotsPerMeter );
   outputImage.fill( 0 );
   QPainter p( &outputImage );
   mComposition->renderPage( &p, page );
@@ -81,19 +77,11 @@ bool QgsCompositionChecker::testComposition( QString &report, int page, int pixe
   QString renderedFilePath = QDir::tempPath() + QDir::separator() + QFileInfo( mTestName ).baseName() + "_rendered.png";
   outputImage.save( renderedFilePath, "PNG" );
 
-  QString diffFilePath = QDir::tempPath() + QDir::separator() + QFileInfo( mTestName ).baseName() + "_result_diff.png";
+  setRenderedImage( renderedFilePath );
 
-  bool testResult = compareImages( mTestName, pixelDiff, renderedFilePath );
+  bool testResult = runTest( mTestName, pixelDiff );
 
-  QString myDashMessage = "<DartMeasurementFile name=\"Rendered Image " + mTestName + "\""
+  theReport += report();
-                          " type=\"image/png\">" + renderedFilePath +
-                          "</DartMeasurementFile>"
-                          "<DartMeasurementFile name=\"Expected Image " + mTestName + "\" type=\"image/png\">" +
-                          mExpectedImageFile + "</DartMeasurementFile>"
-                          "<DartMeasurementFile name=\"Difference Image " + mTestName + "\" type=\"image/png\">" +
-                          diffFilePath + "</DartMeasurementFile>";
-  qDebug() << myDashMessage;
 
-  report += mReport;
   return testResult;
 }

tests/src/core/qgscompositionchecker.h

@@ -16,14 +16,14 @@
 #ifndef QGSCOMPOSITIONCHECKER_H
 #define QGSCOMPOSITIONCHECKER_H
 
-#include "qgsrenderchecker.h"
+#include "qgsmultirenderchecker.h"
 #include <QString>
 
 class QgsComposition;
 class QImage;
 
 /**Renders a composition to an image and compares with an expected output*/
-class QgsCompositionChecker : public QgsRenderChecker
+class QgsCompositionChecker : public QgsMultiRenderChecker
 {
   public:
     QgsCompositionChecker( const QString& testName, QgsComposition* composition );
@@ -36,8 +36,8 @@ class QgsCompositionChecker : public QgsRenderChecker
 
     QString mTestName;
     QgsComposition* mComposition;
-
+    QSize mSize;
-
+    int mDotsPerMeter;
 };
 
 #endif // QGSCOMPOSITIONCHECKER_H