sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-04-14 00:07:35 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3764 from nyalldawson/cache_fix2

Browse Source 
Cache fixes
This commit is contained in:
Nyall Dawson 2016-11-16 10:38:47 +10:00 committed by GitHub
commit 6727ea78bd
12 changed files with 246 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
python/core/qgscacheindexfeatureid.sip
View File

@ -6,42 +6,8 @@ class QgsCacheIndexFeatureId : QgsAbstractCacheIndex
public:
QgsCacheIndexFeatureId( QgsVectorLayerCache* );
/**
* Is called, whenever a feature is removed from the cache. You should update your indexes, so
* they become invalid in case this feature was required to successfuly answer a request.
*/
virtual void flushFeature( const QgsFeatureId fid );
/**
* Sometimes, the whole cache changes its state and its easier to just withdraw everything.
* In this case, this method is issued. Be sure to clear all cache information in here.
*/
virtual void flush();
/**
* @brief
* Implement this method to update the the indices, in case you need information contained by the request
* to properly index. (E.g. spatial index)
* Does nothing by default
*
* @param featureRequest The feature request that was answered
* @param fids The feature ids that have been returned
*/
virtual void requestCompleted( const QgsFeatureRequest& featureRequest, const QgsFeatureIds& fids );
/**
* Is called, when a feature request is issued on a cached layer.
* If this cache index is able to completely answer the feature request, it will return true
* and write the list of feature ids of cached features to cachedFeatures. If it is not able
* it will return false and the cachedFeatures state is undefined.
*
* @param featureIterator A reference to a {@link QgsFeatureIterator}. A valid featureIterator will
* be assigned in case this index is able to answer the request and the return
* value is true.
* @param featureRequest The feature request, for which this index is queried.
*
* @return True, if this index holds the information to answer the request.
*
*/
virtual bool getCacheIterator( QgsFeatureIterator& featureIterator, const QgsFeatureRequest& featureRequest );
};

18
python/core/qgsvectorlayercache.sip
View File

@ -65,9 +65,18 @@ class QgsVectorLayerCache : QObject
* be used for slow data sources, be aware, that the call to this method might take a long time.
*
* @param fullCache True: enable full caching, False: disable full caching
* @see hasFullCache()
*/
void setFullCache( bool fullCache );
/** Returns true if the cache is complete, ie it contains all features. This may happen as
* a result of a call to setFullCache() or by through a feature request which resulted in
* all available features being cached.
* @see setFullCache()
* @note added in QGIS 3.0
*/
bool hasFullCache() const;
/**
* @brief
* Adds a {@link QgsAbstractCacheIndex} to this cache. Cache indices know about features present
@ -114,8 +123,15 @@ class QgsVectorLayerCache : QObject
* Check if a certain feature id is cached.
* @param fid The feature id to look for
* @return True if this id is in the cache
* @see cachedFeatureIds()
*/
bool isFidCached( const QgsFeatureId fid );
bool isFidCached( const QgsFeatureId fid ) const;
/** Returns the set of feature IDs for features which are cached.
* @note added in QGIS 3.0
* @see isFidCached()
*/
QgsFeatureIds cachedFeatureIds() const;
/**
* Gets the feature at the given feature id. Considers the changed, added, deleted and permanent features

4
src/core/qgscacheindex.h
View File

@ -58,8 +58,8 @@ class CORE_EXPORT QgsAbstractCacheIndex
/**
* Is called, when a feature request is issued on a cached layer.
* If this cache index is able to completely answer the feature request, it will return true
* and write the list of feature ids of cached features to cachedFeatures. If it is not able
* it will return false and the cachedFeatures state is undefined.
* and set the iterator to a valid iterator over the cached features. If it is not able
* it will return false.
*
* @param featureIterator A reference to a {@link QgsFeatureIterator}. A valid featureIterator will
* be assigned in case this index is able to answer the request and the return

32
src/core/qgscacheindexfeatureid.cpp
View File

@ -42,12 +42,36 @@ void QgsCacheIndexFeatureId::requestCompleted( const QgsFeatureRequest& featureR
bool QgsCacheIndexFeatureId::getCacheIterator( QgsFeatureIterator &featureIterator, const QgsFeatureRequest &featureRequest )
{
if ( featureRequest.filterType() == QgsFeatureRequest::FilterFid )
switch ( featureRequest.filterType() )
{
if ( C->isFidCached( featureRequest.filterFid() ) )
case QgsFeatureRequest::FilterFid:
{
featureIterator = QgsFeatureIterator( new QgsCachedFeatureIterator( C, featureRequest ) );
return true;
if ( C->isFidCached( featureRequest.filterFid() ) )
{
featureIterator = QgsFeatureIterator( new QgsCachedFeatureIterator( C, featureRequest ) );
return true;
}
break;
}
case QgsFeatureRequest::FilterFids:
{
if ( C->cachedFeatureIds().contains( featureRequest.filterFids() ) )
{
featureIterator = QgsFeatureIterator( new QgsCachedFeatureIterator( C, featureRequest ) );
return true;
}
break;
}
case QgsFeatureRequest::FilterNone:
case QgsFeatureRequest::FilterRect:
case QgsFeatureRequest::FilterExpression:
{
if ( C->hasFullCache() )
{
featureIterator = QgsFeatureIterator( new QgsCachedFeatureIterator( C, featureRequest ) );
return true;
}
break;
}
}

34
src/core/qgscacheindexfeatureid.h
View File

@ -28,43 +28,9 @@ class CORE_EXPORT QgsCacheIndexFeatureId : public QgsAbstractCacheIndex
public:
QgsCacheIndexFeatureId( QgsVectorLayerCache* );
/**
* Is called, whenever a feature is removed from the cache. You should update your indexes, so
* they become invalid in case this feature was required to successfuly answer a request.
*/
virtual void flushFeature( const QgsFeatureId fid ) override;
/**
* Sometimes, the whole cache changes its state and its easier to just withdraw everything.
* In this case, this method is issued. Be sure to clear all cache information in here.
*/
virtual void flush() override;
/**
* @brief
* Implement this method to update the the indices, in case you need information contained by the request
* to properly index. (E.g. spatial index)
* Does nothing by default
*
* @param featureRequest The feature request that was answered
* @param fids The feature ids that have been returned
*/
virtual void requestCompleted( const QgsFeatureRequest& featureRequest, const QgsFeatureIds& fids ) override;
/**
* Is called, when a feature request is issued on a cached layer.
* If this cache index is able to completely answer the feature request, it will return true
* and write the list of feature ids of cached features to cachedFeatures. If it is not able
* it will return false and the cachedFeatures state is undefined.
*
* @param featureIterator A reference to a {@link QgsFeatureIterator}. A valid featureIterator will
* be assigned in case this index is able to answer the request and the return
* value is true.
* @param featureRequest The feature request, for which this index is queried.
*
* @return True, if this index holds the information to answer the request.
*
*/
virtual bool getCacheIterator( QgsFeatureIterator& featureIterator, const QgsFeatureRequest& featureRequest ) override;
private:

67
src/core/qgsvectorlayercache.cpp
View File

@ -174,12 +174,16 @@ QgsVectorLayer* QgsVectorLayerCache::layer()
void QgsVectorLayerCache::requestCompleted( const QgsFeatureRequest& featureRequest, const QgsFeatureIds& fids )
{
// If a request is too large for the cache don't notify to prevent from indexing incomplete requests
if ( fids.count() < mCache.size() )
if ( fids.count() <= mCache.size() )
{
Q_FOREACH ( QgsAbstractCacheIndex* idx, mCacheIndices )
{
idx->requestCompleted( featureRequest, fids );
}
if ( featureRequest.filterType() == QgsFeatureRequest::FilterNone )
{
mFullCache = true;
}
}
}
@ -266,6 +270,54 @@ void QgsVectorLayerCache::invalidate()
emit invalidated();
}
bool QgsVectorLayerCache::canUseCacheForRequest( const QgsFeatureRequest &featureRequest, QgsFeatureIterator& it )
{
// check first for available indices
Q_FOREACH ( QgsAbstractCacheIndex *idx, mCacheIndices )
{
if ( idx->getCacheIterator( it, featureRequest ) )
{
return true;
}
}
// no indexes available, but maybe we have already cached all required features anyway?
switch ( featureRequest.filterType() )
{
case QgsFeatureRequest::FilterFid:
{
if ( mCache.contains( featureRequest.filterFid() ) )
{
it = QgsFeatureIterator( new QgsCachedFeatureIterator( this, featureRequest ) );
return true;
}
break;
}
case QgsFeatureRequest::FilterFids:
{
if ( mCache.keys().toSet().contains( featureRequest.filterFids() ) )
{
it = QgsFeatureIterator( new QgsCachedFeatureIterator( this, featureRequest ) );
return true;
}
break;
}
case QgsFeatureRequest::FilterNone:
case QgsFeatureRequest::FilterRect:
case QgsFeatureRequest::FilterExpression:
{
if ( mFullCache )
{
it = QgsFeatureIterator( new QgsCachedFeatureIterator( this, featureRequest ) );
return true;
}
break;
}
}
return false;
}
QgsFeatureIterator QgsVectorLayerCache::getFeatures( const QgsFeatureRequest &featureRequest )
{
QgsFeatureIterator it;
@ -281,15 +333,8 @@ QgsFeatureIterator QgsVectorLayerCache::getFeatures( const QgsFeatureRequest &fe
}
else
{
// Check if an index is able to deliver the requested features
Q_FOREACH ( QgsAbstractCacheIndex *idx, mCacheIndices )
{
if ( idx->getCacheIterator( it, featureRequest ) )
{
requiresWriterIt = false;
break;
}
}
// may still be able to satisfy request using cache
requiresWriterIt = !canUseCacheForRequest( featureRequest, it );
}
}
else
@ -319,7 +364,7 @@ QgsFeatureIterator QgsVectorLayerCache::getFeatures( const QgsFeatureRequest &fe
return it;
}
bool QgsVectorLayerCache::isFidCached( const QgsFeatureId fid )
bool QgsVectorLayerCache::isFidCached( const QgsFeatureId fid ) const
{
return mCache.contains( fid );
}

29
src/core/qgsvectorlayercache.h
View File

@ -133,9 +133,18 @@ class CORE_EXPORT QgsVectorLayerCache : public QObject
* be used for slow data sources, be aware, that the call to this method might take a long time.
*
* @param fullCache True: enable full caching, False: disable full caching
* @see hasFullCache()
*/
void setFullCache( bool fullCache );
/** Returns true if the cache is complete, ie it contains all features. This may happen as
* a result of a call to setFullCache() or by through a feature request which resulted in
* all available features being cached.
* @see setFullCache()
* @note added in QGIS 3.0
*/
bool hasFullCache() const { return mFullCache; }
/**
* @brief
* Adds a {@link QgsAbstractCacheIndex} to this cache. Cache indices know about features present
@ -196,8 +205,15 @@ class CORE_EXPORT QgsVectorLayerCache : public QObject
* Check if a certain feature id is cached.
* @param fid The feature id to look for
* @return True if this id is in the cache
* @see cachedFeatureIds()
*/
bool isFidCached( const QgsFeatureId fid );
bool isFidCached( const QgsFeatureId fid ) const;
/** Returns the set of feature IDs for features which are cached.
* @note added in QGIS 3.0
* @see isFidCached()
*/
QgsFeatureIds cachedFeatureIds() const { return mCache.keys().toSet(); }
/**
* Gets the feature at the given feature id. Considers the changed, added, deleted and permanent features
@ -329,5 +345,16 @@ class CORE_EXPORT QgsVectorLayerCache : public QObject
friend class QgsCachedFeatureIterator;
friend class QgsCachedFeatureWriterIterator;
friend class QgsCachedFeature;
/** Returns true if the cache contains all the features required for a specified request.
* @param featureRequest feature request
* @param it will be set to iterator for matching features
* @returns true if cache can satisfy request
* @note this method only checks for available features, not whether the cache
* contains required attributes or geometry. For that, use checkInformationCovered()
*/
bool canUseCacheForRequest( const QgsFeatureRequest& featureRequest, QgsFeatureIterator& it );
friend class TestVectorLayerCache;
};
#endif // QgsVectorLayerCache_H

6
src/gui/attributetable/qgsattributetablefiltermodel.cpp
View File

@ -294,11 +294,6 @@ void QgsAttributeTableFilterModel::setFilterMode( FilterMode filterMode )
disconnect( mCanvas, SIGNAL( extentsChanged() ), this, SLOT( extentsChanged() ) );
}
if ( filterMode == ShowSelected )
{
generateListOfVisibleFeatures();
}
mFilterMode = filterMode;
invalidateFilter();
}
@ -359,7 +354,6 @@ void QgsAttributeTableFilterModel::selectionChanged()
{
if ( ShowSelected == mFilterMode )
{
generateListOfVisibleFeatures();
invalidateFilter();
}
else if ( mSelectedOnTop )

2
src/gui/attributetable/qgsdualview.cpp
View File

@ -706,9 +706,7 @@ void QgsDualView::progress( int i, bool& cancel )
mProgressDlg->show();
}
mProgressDlg->setValue( i );
mProgressDlg->setLabelText( tr( "%1 features loaded." ).arg( i ) );
QCoreApplication::processEvents();
cancel = mProgressDlg && mProgressDlg->wasCanceled();

2
src/ui/qgsoptionsbase.ui
View File

@ -1530,7 +1530,7 @@
<number>0</number>
</property>
<property name="maximum">
<number>100000</number>
<number>10000000</number>
</property>
<property name="singleStep">
<number>1000</number>

112
tests/src/core/testqgsvectorlayercache.cpp
View File

@ -52,6 +52,9 @@ class TestVectorLayerCache : public QObject
void testCacheAttrActions(); // Test attribute add/ attribute delete
void testFeatureActions(); // Test adding/removing features works
void testSubsetRequest();
void testFullCache();
void testFullCacheThroughRequest();
void testCanUseCacheForRequest();
void onCommittedFeaturesAdded( const QString&, const QgsFeatureList& );
@ -219,6 +222,115 @@ void TestVectorLayerCache::testSubsetRequest()
QVERIFY( a == f.attribute( 3 ) );
}
void TestVectorLayerCache::testFullCache()
{
// cache is too small to fit all features
QgsVectorLayerCache cache( mPointsLayer, 2 );
QVERIFY( !cache.hasFullCache() );
QVERIFY( cache.cacheSize() < mPointsLayer->featureCount() );
// but we set it to full cache
cache.setFullCache( true );
// so now it should have sufficient size for all features
QVERIFY( cache.cacheSize() >= mPointsLayer->featureCount() );
QVERIFY( cache.hasFullCache() );
// double check that everything is indeed in the cache
QgsFeatureIterator it = mPointsLayer->getFeatures();
QgsFeature f;
while ( it.nextFeature( f ) )
{
QVERIFY( cache.isFidCached( f.id() ) );
}
}
void TestVectorLayerCache::testFullCacheThroughRequest()
{
// make sure cache is sufficient size for all features
QgsVectorLayerCache cache( mPointsLayer, mPointsLayer->featureCount() * 2 );
QVERIFY( !cache.hasFullCache() );
// now request all features from cache
QgsFeatureIterator it = cache.getFeatures( QgsFeatureRequest() );
QgsFeature f;
while ( it.nextFeature( f ) )
{
// suck in all features
}
// cache should now contain all features
it = mPointsLayer->getFeatures();
while ( it.nextFeature( f ) )
{
QVERIFY( cache.isFidCached( f.id() ) );
}
// so it should be a full cache!
QVERIFY( cache.hasFullCache() );
}
void TestVectorLayerCache::testCanUseCacheForRequest()
{
//first get some feature ids from layer
QgsFeature f;
QgsFeatureIterator it = mPointsLayer->getFeatures();
it.nextFeature( f );
QgsFeatureId id1 = f.id();
it.nextFeature( f );
QgsFeatureId id2 = f.id();
QgsVectorLayerCache cache( mPointsLayer, 10 );
// initially nothing in cache, so can't use it to fulfill the request
QVERIFY( !cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFid( id1 ), it ) );
QVERIFY( !cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFid( id2 ), it ) );
QVERIFY( !cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFids( QgsFeatureIds() << id1 << id2 ), it ) );
QVERIFY( !cache.canUseCacheForRequest( QgsFeatureRequest().setFilterRect( QgsRectangle( 1, 2, 3, 4 ) ), it ) );
QVERIFY( !cache.canUseCacheForRequest( QgsFeatureRequest().setFilterExpression( "$x<5" ), it ) );
// get just the first feature into the cache
it = cache.getFeatures( QgsFeatureRequest().setFilterFid( id1 ) );
while ( it.nextFeature( f ) ) { }
QCOMPARE( cache.cachedFeatureIds(), QgsFeatureIds() << id1 );
QVERIFY( cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFid( id1 ), it ) );
//verify that the returned iterator was correct
QVERIFY( it.nextFeature( f ) );
QCOMPARE( f.id(), id1 );
QVERIFY( !it.nextFeature( f ) );
QVERIFY( !cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFid( id2 ), it ) );
QVERIFY( !cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFids( QgsFeatureIds() << id1 << id2 ), it ) );
QVERIFY( !cache.canUseCacheForRequest( QgsFeatureRequest().setFilterRect( QgsRectangle( 1, 2, 3, 4 ) ), it ) );
QVERIFY( !cache.canUseCacheForRequest( QgsFeatureRequest().setFilterExpression( "$x<5" ), it ) );
// get feature 2 into cache
it = cache.getFeatures( QgsFeatureRequest().setFilterFid( id2 ) );
while ( it.nextFeature( f ) ) { }
QCOMPARE( cache.cachedFeatureIds(), QgsFeatureIds() << id1 << id2 );
QVERIFY( cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFid( id1 ), it ) );
QVERIFY( it.nextFeature( f ) );
QCOMPARE( f.id(), id1 );
QVERIFY( !it.nextFeature( f ) );
QVERIFY( cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFid( id2 ), it ) );
QVERIFY( it.nextFeature( f ) );
QCOMPARE( f.id(), id2 );
QVERIFY( !it.nextFeature( f ) );
QVERIFY( cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFids( QgsFeatureIds() << id1 << id2 ), it ) );
QVERIFY( it.nextFeature( f ) );
QgsFeatureIds result;
result << f.id();
QVERIFY( it.nextFeature( f ) );
result << f.id();
QCOMPARE( result, QgsFeatureIds() << id1 << id2 );
QVERIFY( !cache.canUseCacheForRequest( QgsFeatureRequest().setFilterRect( QgsRectangle( 1, 2, 3, 4 ) ), it ) );
QVERIFY( !cache.canUseCacheForRequest( QgsFeatureRequest().setFilterExpression( "$x<5" ), it ) );
// can only use rect/expression requests if cache has everything
cache.setFullCache( true );
QVERIFY( cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFid( id1 ), it ) );
QVERIFY( cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFid( id2 ), it ) );
QVERIFY( cache.canUseCacheForRequest( QgsFeatureRequest().setFilterFids( QgsFeatureIds() << id1 << id2 ), it ) );
QVERIFY( cache.canUseCacheForRequest( QgsFeatureRequest().setFilterRect( QgsRectangle( 1, 2, 3, 4 ) ), it ) );
QVERIFY( cache.canUseCacheForRequest( QgsFeatureRequest().setFilterExpression( "$x<5" ), it ) );
}
void TestVectorLayerCache::onCommittedFeaturesAdded( const QString& layerId, const QgsFeatureList& features )
{
Q_UNUSED( layerId )

4
tests/src/gui/testqgsdualview.cpp
View File

@ -136,12 +136,12 @@ void TestQgsDualView::testSelectAll()
// Only show parts of the canvas, so only one selected feature is visible
mCanvas->setExtent( QgsRectangle( -139, 23, -100, 48 ) );
mDualView->mTableView->selectAll();
QVERIFY( mPointsLayer->selectedFeatureCount() == 10 );
QCOMPARE( mPointsLayer->selectedFeatureCount(), 10 );
mPointsLayer->selectByIds( QgsFeatureIds() );
mCanvas->setExtent( QgsRectangle( -110, 40, -100, 48 ) );
mDualView->mTableView->selectAll();
QVERIFY( mPointsLayer->selectedFeatureCount() == 1 );
QCOMPARE( mPointsLayer->selectedFeatureCount(), 1 );
}
void TestQgsDualView::testSort()