[FEATURE][API] Add a content cache for raster images

This new class QgsImageCache is the equivalent of QgsSvgCache
but for raster images.

QgsImageCache stores pre-rendered resampled versions of raster
image files, allowing efficient reuse without incurring the
cost of resampling on every render.

Additionally, it offers the other benefits QgsSvgCache has,
such as thread safety, ability to transparently download remote
images, and support for base64 encoded strings.
This commit is contained in:
Nyall Dawson 2018-12-03 12:22:52 +10:00 committed by Mathieu Pellerin
parent cdba8f5621
commit 84838d1ec6
29 changed files with 788 additions and 5 deletions

View File

@ -33,7 +33,6 @@ Constructor for QgsAbstractContentCacheEntry for an entry relating to the specif
virtual ~QgsAbstractContentCacheEntry();
QString path;
QDateTime fileModified;

View File

@ -649,7 +649,18 @@ providers that may add items to the browser tree.
Returns the application's SVG cache, used for caching SVG images and handling parameter replacement
within SVG files.
.. seealso:: :py:func:`imageCache`
.. versionadded:: 3.0
%End
static QgsImageCache *imageCache();
%Docstring
Returns the application's image cache, used for caching resampled versions of raster images.
.. seealso:: :py:func:`svgCache`
.. versionadded:: 3.6
%End
static QgsNetworkContentFetcherRegistry *networkContentFetcherRegistry();

View File

@ -0,0 +1,71 @@
/************************************************************************
* This file has been generated automatically from *
* *
* src/core/qgsimagecache.h *
* *
* Do not edit manually ! Edit header and run scripts/sipify.pl again *
************************************************************************/
class QgsImageCache : QgsAbstractContentCacheBase
{
%Docstring
A cache for images derived from raster files.
QgsImageCache stores pre-rendered resampled versions of raster image files, allowing efficient
reuse without incurring the cost of resampling on every render.
QgsImageCache is not usually directly created, but rather accessed through
:py:func:`QgsApplication.imageCache()`
.. versionadded:: 3.6
%End
%TypeHeaderCode
#include "qgsimagecache.h"
%End
public:
QgsImageCache( QObject *parent /TransferThis/ = 0 );
%Docstring
Constructor for QgsImageCache, with the specified ``parent`` object.
%End
QImage pathAsImage( const QString &path, QSize size, bool keepAspectRatio, bool &fitsInCache /Out/ );
%Docstring
Returns the specified ``path`` rendered as an image. If possible, a pre-existing cached
version of the image will be used. If not, the image is fetched and resampled to the desired
size, and then the result cached for subsequent lookups.
``path`` may be a local file, remote (HTTP) url, or a base 64 encoded string (with a "base64:" prefix).
The ``size`` parameter dictates the target size of the image. An invalid size indicates the
original raster image size (with no resampling).
If ``keepAspectRatio`` is true, then the original raster aspect ratio will be maintained during
any resampling operations.
If the resultant raster was of a sufficiently small size to store in the cache, then ``fitsInCache``
will be set to true.
%End
signals:
void remoteImageFetched( const QString &url );
%Docstring
Emitted when the cache has finished retrieving an image file from a remote ``url``.
%End
};
/************************************************************************
* This file has been generated automatically from *
* *
* src/core/qgsimagecache.h *
* *
* Do not edit manually ! Edit header and run scripts/sipify.pl again *
************************************************************************/

View File

@ -332,6 +332,7 @@
%Include auto_generated/qgsgeometryvalidator.sip
%Include auto_generated/qgsgml.sip
%Include auto_generated/qgsgmlschema.sip
%Include auto_generated/qgsimagecache.sip
%Include auto_generated/qgsmaplayer.sip
%Include auto_generated/qgsmaplayerlegend.sip
%Include auto_generated/qgsmaplayermodel.sip

View File

@ -6554,7 +6554,7 @@ stength:strength
steriods:steroids
sterotypes:stereotypes
stilus:stylus
stingent:stringent
stingent:stringent:*
stiring:stirring
stirng:string
stirngs:strings

View File

@ -214,6 +214,7 @@ SET(QGIS_CORE_SRCS
qgshistogram.cpp
qgshstoreutils.cpp
qgshtmlutils.cpp
qgsimagecache.cpp
qgsinterval.cpp
qgsjsonutils.cpp
qgslabelfeature.cpp
@ -616,6 +617,7 @@ SET(QGIS_CORE_MOC_HDRS
qgsgeometryvalidator.h
qgsgml.h
qgsgmlschema.h
qgsimagecache.h
qgsmaplayer.h
qgsmaplayerlegend.h
qgsmaplayermodel.h

View File

@ -52,8 +52,6 @@ class CORE_EXPORT QgsAbstractContentCacheEntry
virtual ~QgsAbstractContentCacheEntry() = default;
QgsAbstractContentCacheEntry() = delete;
//! QgsAbstractContentCacheEntry cannot be copied.
QgsAbstractContentCacheEntry( const QgsAbstractContentCacheEntry &rh ) = delete;
//! QgsAbstractContentCacheEntry cannot be copied.
@ -176,7 +174,7 @@ class CORE_EXPORT QgsAbstractContentCacheBase: public QObject
#ifndef SIP_RUN
/**
* \class QgsAbstractContentCacheBase
* \class QgsAbstractContentCache
* \ingroup core
*
* Abstract base class for file content caches, such as SVG or raster image caches.
@ -560,6 +558,7 @@ class CORE_EXPORT QgsAbstractContentCache : public QgsAbstractContentCacheBase
QString mTypeString;
friend class TestQgsSvgCache;
friend class TestQgsImageCache;
};
#endif

View File

@ -30,6 +30,7 @@
#include "qgstaskmanager.h"
#include "qgsfieldformatterregistry.h"
#include "qgssvgcache.h"
#include "qgsimagecache.h"
#include "qgscolorschemeregistry.h"
#include "qgspainteffectregistry.h"
#include "qgsprojectstorageregistry.h"
@ -1772,6 +1773,11 @@ QgsSvgCache *QgsApplication::svgCache()
return members()->mSvgCache;
}
QgsImageCache *QgsApplication::imageCache()
{
return members()->mImageCache;
}
QgsNetworkContentFetcherRegistry *QgsApplication::networkContentFetcherRegistry()
{
return members()->mNetworkContentFetcherRegistry;
@ -1842,6 +1848,7 @@ QgsApplication::ApplicationMembers::ApplicationMembers()
mActionScopeRegistry = new QgsActionScopeRegistry();
mFieldFormatterRegistry = new QgsFieldFormatterRegistry();
mSvgCache = new QgsSvgCache();
mImageCache = new QgsImageCache();
mColorSchemeRegistry = new QgsColorSchemeRegistry();
mPaintEffectRegistry = new QgsPaintEffectRegistry();
mSymbolLayerRegistry = new QgsSymbolLayerRegistry();
@ -1878,6 +1885,7 @@ QgsApplication::ApplicationMembers::~ApplicationMembers()
delete mRasterRendererRegistry;
delete mRendererRegistry;
delete mSvgCache;
delete mImageCache;
delete mSymbolLayerRegistry;
delete mTaskManager;
delete mNetworkContentFetcherRegistry;

View File

@ -34,6 +34,7 @@ class QgsPaintEffectRegistry;
class QgsProjectStorageRegistry;
class QgsRendererRegistry;
class QgsSvgCache;
class QgsImageCache;
class QgsSymbolLayerRegistry;
class QgsRasterRendererRegistry;
class QgsGpsConnectionRegistry;
@ -595,10 +596,20 @@ class CORE_EXPORT QgsApplication : public QApplication
/**
* Returns the application's SVG cache, used for caching SVG images and handling parameter replacement
* within SVG files.
*
* \see imageCache()
* \since QGIS 3.0
*/
static QgsSvgCache *svgCache();
/**
* Returns the application's image cache, used for caching resampled versions of raster images.
*
* \see svgCache()
* \since QGIS 3.6
*/
static QgsImageCache *imageCache();
/**
* Returns the application's network content registry used for fetching temporary files during QGIS session
* \since QGIS 3.2
@ -867,6 +878,7 @@ class CORE_EXPORT QgsApplication : public QApplication
QgsRendererRegistry *mRendererRegistry = nullptr;
QgsRuntimeProfiler *mProfiler = nullptr;
QgsSvgCache *mSvgCache = nullptr;
QgsImageCache *mImageCache = nullptr;
QgsSymbolLayerRegistry *mSymbolLayerRegistry = nullptr;
QgsTaskManager *mTaskManager = nullptr;
QgsLayoutItemRegistry *mLayoutItemRegistry = nullptr;

195
src/core/qgsimagecache.cpp Normal file
View File

@ -0,0 +1,195 @@
/***************************************************************************
qgsimagecache.cpp
-----------------
begin : December 2018
copyright : (C) 2018 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include "qgsimagecache.h"
#include "qgis.h"
#include "qgslogger.h"
#include "qgsnetworkaccessmanager.h"
#include "qgsmessagelog.h"
#include "qgssymbollayerutils.h"
#include "qgsnetworkcontentfetchertask.h"
#include <QApplication>
#include <QCoreApplication>
#include <QCursor>
#include <QDomDocument>
#include <QDomElement>
#include <QFile>
#include <QImage>
#include <QPainter>
#include <QPicture>
#include <QFileInfo>
#include <QNetworkReply>
#include <QNetworkRequest>
#include <QBuffer>
#include <QImageReader>
#include <QSvgRenderer>
///@cond PRIVATE
QgsImageCacheEntry::QgsImageCacheEntry( const QString &path, QSize size, const bool keepAspectRatio )
: QgsAbstractContentCacheEntry( path )
, size( size )
, keepAspectRatio( keepAspectRatio )
{
}
bool QgsImageCacheEntry::isEqual( const QgsAbstractContentCacheEntry *other ) const
{
const QgsImageCacheEntry *otherImage = dynamic_cast< const QgsImageCacheEntry * >( other );
// cheapest checks first!
if ( !otherImage || otherImage->keepAspectRatio != keepAspectRatio || otherImage->size != size || otherImage->path != path )
return false;
return true;
}
int QgsImageCacheEntry::dataSize() const
{
int size = 0;
if ( !image.isNull() )
{
size += ( image.width() * image.height() * 32 );
}
return size;
}
void QgsImageCacheEntry::dump() const
{
QgsDebugMsg( QStringLiteral( "path: %1, size %2x%3" ).arg( path ).arg( size.width() ).arg( size.height() ) );
}
///@endcond
QgsImageCache::QgsImageCache( QObject *parent )
: QgsAbstractContentCache< QgsImageCacheEntry >( parent, QObject::tr( "Image" ) )
{
mMissingSvg = QStringLiteral( "<svg width='10' height='10'><text x='5' y='10' font-size='10' text-anchor='middle'>?</text></svg>" ).toLatin1();
const QString downloadingSvgPath = QgsApplication::defaultThemePath() + QStringLiteral( "downloading_svg.svg" );
if ( QFile::exists( downloadingSvgPath ) )
{
QFile file( downloadingSvgPath );
if ( file.open( QIODevice::ReadOnly ) )
{
mFetchingSvg = file.readAll();
}
}
if ( mFetchingSvg.isEmpty() )
{
mFetchingSvg = QStringLiteral( "<svg width='10' height='10'><text x='5' y='10' font-size='10' text-anchor='middle'>?</text></svg>" ).toLatin1();
}
connect( this, &QgsAbstractContentCacheBase::remoteContentFetched, this, &QgsImageCache::remoteImageFetched );
}
QImage QgsImageCache::pathAsImage( const QString &file, const QSize size, const bool keepAspectRatio, bool &fitsInCache )
{
QMutexLocker locker( &mMutex );
fitsInCache = true;
QgsImageCacheEntry *currentEntry = findExistingEntry( new QgsImageCacheEntry( file, size, keepAspectRatio ) );
QImage result;
//if current entry image is null: create the image
// checks to see if image will fit into cache
//update stats for memory usage
if ( currentEntry->image.isNull() )
{
long cachedDataSize = 0;
cachedDataSize += currentEntry->size.width() * currentEntry->size.height() * 32;
result = renderImage( file, size, keepAspectRatio );
if ( cachedDataSize > mMaxCacheSize / 2 )
{
fitsInCache = false;
currentEntry->image = QImage();
}
else
{
mTotalSize += ( result.width() * result.height() * 32 );
currentEntry->image = result;
}
trimToMaximumSize();
}
else
{
result = currentEntry->image;
}
return result;
}
QImage QgsImageCache::renderImage( const QString &path, QSize size, const bool keepAspectRatio ) const
{
QImage im;
// direct read if path is a file -- maybe more efficient than going the bytearray route? (untested!)
if ( QFile::exists( path ) )
{
im = QImage( path );
}
else
{
QByteArray ba = getContent( path, QByteArray( "broken" ), QByteArray( "fetching" ) );
if ( ba == "broken" )
{
// render "broken" svg
im = QImage( size, QImage::Format_ARGB32_Premultiplied );
im.fill( 0 ); // transparent background
QPainter p( &im );
QSvgRenderer r( mMissingSvg );
QSizeF s( r.viewBox().size() );
s.scale( size.width(), size.height(), Qt::KeepAspectRatio );
QRectF rect( ( size.width() - s.width() ) / 2, ( size.height() - s.height() ) / 2, s.width(), s.height() );
r.render( &p, rect );
}
else if ( ba == "fetching" )
{
// render "fetching" svg
im = QImage( size, QImage::Format_ARGB32_Premultiplied );
im.fill( 0 ); // transparent background
QPainter p( &im );
QSvgRenderer r( mFetchingSvg );
QSizeF s( r.viewBox().size() );
s.scale( size.width(), size.height(), Qt::KeepAspectRatio );
QRectF rect( ( size.width() - s.width() ) / 2, ( size.height() - s.height() ) / 2, s.width(), s.height() );
r.render( &p, rect );
}
else
{
QBuffer buffer( &ba );
buffer.open( QIODevice::ReadOnly );
QImageReader reader( &buffer );
im = reader.read();
}
}
// render image at desired size -- null size means original size
if ( !size.isValid() || size.isNull() || im.size() == size )
return im;
else
return im.scaled( size, keepAspectRatio ? Qt::KeepAspectRatio : Qt::IgnoreAspectRatio, Qt::SmoothTransformation );
}

135
src/core/qgsimagecache.h Normal file
View File

@ -0,0 +1,135 @@
/***************************************************************************
qgsimagecache.h
---------------
begin : December 2018
copyright : (C) 2018 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSIMAGECACHE_H
#define QGSIMAGECACHE_H
#include "qgsabstractcontentcache.h"
#include "qgis.h"
#include "qgis_core.h"
#include <QElapsedTimer>
#include <QSize>
#include <QImage>
#ifndef SIP_RUN
///@cond PRIVATE
/**
* \ingroup core
* \class QgsImageCacheEntry
* An entry for a QgsImageCache, representing a single raster rendered at a specific width and height.
* \since QGIS 3.6
*/
class CORE_EXPORT QgsImageCacheEntry : public QgsAbstractContentCacheEntry
{
public:
/**
* Constructor for QgsImageCacheEntry, corresponding to the specified image \a path and \a size.
*
* If \a keepAspectRatio is true then the original raster aspect ratio will always be preserved
* when resizing.
*/
QgsImageCacheEntry( const QString &path, QSize size, bool keepAspectRatio ) ;
//! Rendered image size
QSize size;
//! True if original raster aspect ratio was kept during resizing
bool keepAspectRatio = true;
//! Rendered, resampled image.
QImage image;
int dataSize() const override;
void dump() const override;
bool isEqual( const QgsAbstractContentCacheEntry *other ) const override;
};
///@endcond
#endif
/**
* \class QgsImageCache
* \ingroup core
* A cache for images derived from raster files.
*
* QgsImageCache stores pre-rendered resampled versions of raster image files, allowing efficient
* reuse without incurring the cost of resampling on every render.
*
* QgsImageCache is not usually directly created, but rather accessed through
* QgsApplication::imageCache().
*
* \since QGIS 3.6
*/
#ifdef SIP_RUN
class CORE_EXPORT QgsImageCache : public QgsAbstractContentCacheBase // for sip we skip to the base class and avoid the template difficulty
{
#else
class CORE_EXPORT QgsImageCache : public QgsAbstractContentCache< QgsImageCacheEntry >
{
#endif
Q_OBJECT
public:
/**
* Constructor for QgsImageCache, with the specified \a parent object.
*/
QgsImageCache( QObject *parent SIP_TRANSFERTHIS = nullptr );
/**
* Returns the specified \a path rendered as an image. If possible, a pre-existing cached
* version of the image will be used. If not, the image is fetched and resampled to the desired
* size, and then the result cached for subsequent lookups.
*
* \a path may be a local file, remote (HTTP) url, or a base 64 encoded string (with a "base64:" prefix).
*
* The \a size parameter dictates the target size of the image. An invalid size indicates the
* original raster image size (with no resampling).
*
* If \a keepAspectRatio is true, then the original raster aspect ratio will be maintained during
* any resampling operations.
*
* If the resultant raster was of a sufficiently small size to store in the cache, then \a fitsInCache
* will be set to true.
*/
QImage pathAsImage( const QString &path, QSize size, bool keepAspectRatio, bool &fitsInCache SIP_OUT );
signals:
/**
* Emitted when the cache has finished retrieving an image file from a remote \a url.
*/
void remoteImageFetched( const QString &url );
private:
QImage renderImage( const QString &path, QSize size, const bool keepAspectRatio ) const;
//! SVG content to be rendered if SVG file was not found.
QByteArray mMissingSvg;
QByteArray mFetchingSvg;
friend class TestQgsImageCache;
};
#endif // QGSIMAGECACHE_H

View File

@ -117,6 +117,7 @@ SET(TESTS
testqgsgraduatedsymbolrenderer.cpp
testqgshistogram.cpp
testqgshstoreutils.cpp
testqgsimagecache.cpp
testqgsimageoperation.cpp
testqgsinternalgeometryengine.cpp
testqgsinvertedpolygonrenderer.cpp

File diff suppressed because one or more lines are too long

View File

@ -75,6 +75,7 @@ ADD_PYTHON_TEST(PyQgsGeometryTest test_qgsgeometry.py)
ADD_PYTHON_TEST(PyQgsGeometryValidator test_qgsgeometryvalidator.py)
ADD_PYTHON_TEST(PyQgsGraduatedSymbolRenderer test_qgsgraduatedsymbolrenderer.py)
ADD_PYTHON_TEST(PyQgsHighlight test_qgshighlight.py)
ADD_PYTHON_TEST(PyQgsImageCache test_qgsimagecache.py)
ADD_PYTHON_TEST(PyQgsInterval test_qgsinterval.py)
ADD_PYTHON_TEST(PyQgsJsonUtils test_qgsjsonutils.py)
ADD_PYTHON_TEST(PyQgsLayerMetadata test_qgslayermetadata.py)

View File

@ -0,0 +1,111 @@
# -*- coding: utf-8 -*-
"""QGIS Unit tests for QgsImageCache.
.. note:: This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
"""
__author__ = '(C) 2018 by Nyall Dawson'
__date__ = '02/10/2018'
__copyright__ = 'Copyright 2018, The QGIS Project'
# This will get replaced with a git SHA1 when you do a git archive
__revision__ = '$Format:%H$'
import qgis # NOQA
import os
import socketserver
import threading
import http.server
from qgis.PyQt.QtCore import QDir, QCoreApplication, QSize
from qgis.PyQt.QtGui import QColor, QImage, QPainter
from qgis.core import (QgsImageCache, QgsRenderChecker, QgsApplication, QgsMultiRenderChecker)
from qgis.testing import start_app, unittest
from utilities import unitTestDataPath
start_app()
TEST_DATA_DIR = unitTestDataPath()
class TestQgsImageCache(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Bring up a simple HTTP server, for remote SVG tests
os.chdir(unitTestDataPath() + '')
handler = http.server.SimpleHTTPRequestHandler
cls.httpd = socketserver.TCPServer(('localhost', 0), handler)
cls.port = cls.httpd.server_address[1]
cls.httpd_thread = threading.Thread(target=cls.httpd.serve_forever)
cls.httpd_thread.setDaemon(True)
cls.httpd_thread.start()
def setUp(self):
self.report = "<h1>Python QgsImageCache Tests</h1>\n"
self.fetched = False
QgsApplication.imageCache().remoteImageFetched.connect(self.imageFetched)
def tearDown(self):
report_file_path = "%s/qgistest.html" % QDir.tempPath()
with open(report_file_path, 'a') as report_file:
report_file.write(self.report)
def imageFetched(self):
self.fetched = True
def waitForFetch(self):
self.fetched = False
while not self.fetched:
QCoreApplication.processEvents()
def testRemoteImage(self):
"""Test fetching remote image."""
url = 'http://localhost:{}/qgis_local_server/sample_image.png'.format(str(TestQgsImageCache.port))
image, in_cache = QgsApplication.imageCache().pathAsImage(url, QSize(100, 100), True)
# first should be waiting image
self.assertTrue(self.imageCheck('Remote Image', 'waiting_image', image))
self.waitForFetch()
# second should be correct image
image, in_cache = QgsApplication.imageCache().pathAsImage(url, QSize(100, 100), True)
self.assertTrue(self.imageCheck('Remote Image', 'remote_image', image))
def testRemoteSvgMissing(self):
"""Test fetching remote image with bad url"""
url = 'http://localhost:{}/qgis_local_server/xxx.png'.format(str(TestQgsImageCache.port)) # oooo naughty
image, in_cache = QgsApplication.imageCache().pathAsImage(url, QSize(100, 100), True)
self.assertTrue(self.imageCheck('Remote SVG missing', 'waiting_image', image))
def imageCheck(self, name, reference_image, image):
self.report += "<h2>Render {}</h2>\n".format(name)
temp_dir = QDir.tempPath() + '/'
file_name = temp_dir + 'image_' + name + ".png"
output_image = QImage(image.size(), QImage.Format_RGB32)
QgsMultiRenderChecker.drawBackground(output_image)
painter = QPainter(output_image)
painter.drawImage(0, 0, image)
painter.end()
output_image.save(file_name, "PNG")
checker = QgsRenderChecker()
checker.setControlPathPrefix("image_cache")
checker.setControlName("expected_" + reference_image)
checker.setRenderedImage(file_name)
checker.setColorTolerance(2)
result = checker.compareImages(name, 20)
self.report += checker.report()
print((self.report))
return result
if __name__ == '__main__':
unittest.main()

Binary file not shown.

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 674 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 694 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 836 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 94 KiB

BIN
tests/testdata/sample_image2.png vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 95 KiB