sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-12-30 00:29:39 -05:00
Code Issues Packages Projects Releases Wiki Activity

[gdal] Correctly implement FastScan for querySublayers

Browse Source
This commit is contained in:
Nyall Dawson 2021-07-22 08:54:44 +10:00
parent 8349457267
commit c7c97101d3
5 changed files with 153 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
src/core/providers/gdal/qgsgdalprovider.cpp
View File

@ -3562,48 +3562,70 @@ QList<QgsProviderSublayerDetails> QgsGdalProviderMetadata::querySublayers( const
}
}
if ( flags & Qgis::SublayerQueryFlag::FastScan )
const QString path = uriParts.value( QStringLiteral( "path" ) ).toString();
const QFileInfo pathInfo( path );
if ( flags & Qgis::SublayerQueryFlag::FastScan && ( pathInfo.isFile() || pathInfo.isDir() ) )
{
// filter based on extension
const QVariantMap uriParts = decodeUri( gdalUri );
const QString path = uriParts.value( QStringLiteral( "path" ) ).toString();
QFileInfo info( path );
if ( info.isFile() )
// fast scan, so we don't actually try to open the dataset and instead just check the extension alone
static QString sFilterString;
static QStringList sExtensions;
static QStringList sWildcards;
// get supported extensions
static std::once_flag initialized;
std::call_once( initialized, [ = ]
{
const QString suffix = info.suffix().toLower();
buildSupportedRasterFileFilterAndExtensions( sFilterString, sExtensions, sWildcards );
QgsDebugMsgLevel( QStringLiteral( "extensions: " ) + sExtensions.join( ' ' ), 2 );
QgsDebugMsgLevel( QStringLiteral( "wildcards: " ) + sWildcards.join( ' ' ), 2 );
} );
static QString sFilterString;
static QStringList sExtensions;
static QStringList sWildcards;
const QString suffix = pathInfo.suffix().toLower();
// get supported extensions
static std::once_flag initialized;
std::call_once( initialized, [ = ]
if ( !sExtensions.contains( suffix ) )
{
bool matches = false;
for ( const QString &wildcard : std::as_const( sWildcards ) )
{
buildSupportedRasterFileFilterAndExtensions( sFilterString, sExtensions, sWildcards );
QgsDebugMsgLevel( QStringLiteral( "extensions: " ) + sExtensions.join( ' ' ), 2 );
QgsDebugMsgLevel( QStringLiteral( "wildcards: " ) + sWildcards.join( ' ' ), 2 );
} );
if ( !sExtensions.contains( suffix ) )
{
bool matches = false;
for ( const QString &wildcard : std::as_const( sWildcards ) )
const thread_local QRegularExpression rx( QRegularExpression::anchoredPattern(
QRegularExpression::wildcardToRegularExpression( wildcard )
), QRegularExpression::CaseInsensitiveOption );
const QRegularExpressionMatch match = rx.match( pathInfo.fileName() );
if ( match.hasMatch() )
{
const thread_local QRegularExpression rx( QRegularExpression::anchoredPattern(
QRegularExpression::wildcardToRegularExpression( wildcard )
), QRegularExpression::CaseInsensitiveOption );
const QRegularExpressionMatch match = rx.match( info.fileName() );
if ( match.hasMatch() )
{
matches = true;
break;
}
matches = true;
break;
}
if ( !matches )
return {};
}
if ( !matches )
return {};
}
// if this is a VRT file make sure it is raster VRT
if ( suffix == QLatin1String( "vrt" ) )
{
CPLPushErrorHandler( CPLQuietErrorHandler );
CPLErrorReset();
GDALDriverH hDriver = GDALIdentifyDriverEx( path.toUtf8().constData(), GDAL_OF_RASTER, nullptr, nullptr );
CPLPopErrorHandler();
if ( !hDriver )
{
// vrt is not a raster vrt, skip it
return {};
}
}
QgsProviderSublayerDetails details;
details.setType( QgsMapLayerType::RasterLayer );
details.setProviderKey( QStringLiteral( "gdal" ) );
details.setUri( uri );
details.setName( QgsProviderUtils::suggestLayerNameFromFilePath( path ) );
if ( QgsGdalUtils::SUPPORTED_DB_LAYERS_EXTENSIONS.contains( suffix ) )
{
// uri may contain sublayers, but query flags prevent us from examining them
details.setSkippedContainerScan( true );
}
return {details};
}
if ( !uriParts.value( QStringLiteral( "vsiPrefix" ) ).toString().isEmpty()

27
src/core/providers/ogr/qgsogrprovidermetadata.cpp
View File

@ -30,6 +30,7 @@ email : nyall dot dawson at gmail dot com
#include "qgsprovidersublayerdetails.h"
#include "qgszipitem.h"
#include "qgsproviderutils.h"
#include "qgsgdalutils.h"
#include <QFileInfo>
#include <QFile>
@ -1134,30 +1135,8 @@ QList<QgsProviderSublayerDetails> QgsOgrProviderMetadata::querySublayers( const
// these extensions are trivial to read, so there's no need to rely on
// the extension only scan here -- avoiding it always gives us the correct data type
// and sublayer visibility
static QStringList sSkipFastTrackExtensions { QStringLiteral( "xlsx" ),
QStringLiteral( "ods" ),
QStringLiteral( "csv" ),
QStringLiteral( "nc" ),
QStringLiteral( "shp.zip" ) };
if ( !sSkipFastTrackExtensions.contains( suffix ) )
if ( !QgsGdalUtils::INEXPENSIVE_TO_SCAN_EXTENSIONS.contains( suffix ) )
{
// Filters out the OGR/GDAL supported formats that can contain multiple layers
// and should be treated as a potential layer container
static QStringList sOgrSupportedDbLayersExtensions { QStringLiteral( "gpkg" ),
QStringLiteral( "sqlite" ),
QStringLiteral( "db" ),
QStringLiteral( "gdb" ),
QStringLiteral( "kml" ),
QStringLiteral( "osm" ),
QStringLiteral( "mdb" ),
QStringLiteral( "accdb" ),
QStringLiteral( "xls" ),
QStringLiteral( "xlsx" ),
QStringLiteral( "gpx" ),
QStringLiteral( "pdf" ),
QStringLiteral( "pbf" ) };
// if this is a VRT file make sure it is vector VRT
if ( suffix == QLatin1String( "vrt" ) )
{
@ -1177,7 +1156,7 @@ QList<QgsProviderSublayerDetails> QgsOgrProviderMetadata::querySublayers( const
details.setProviderKey( QStringLiteral( "ogr" ) );
details.setUri( uri );
details.setName( QgsProviderUtils::suggestLayerNameFromFilePath( path ) );
if ( sOgrSupportedDbLayersExtensions.contains( suffix ) )
if ( QgsGdalUtils::SUPPORTED_DB_LAYERS_EXTENSIONS.contains( suffix ) )
{
// uri may contain sublayers, but query flags prevent us from examining them
details.setSkippedContainerScan( true );

27
src/core/qgsgdalutils.cpp
View File

@ -28,6 +28,33 @@
#include <QImage>
#include <QFileInfo>
// File extensions for formats supported by GDAL which may contain multiple layers
// and should be treated as a potential layer container
const QStringList QgsGdalUtils::SUPPORTED_DB_LAYERS_EXTENSIONS
{
QStringLiteral( "gpkg" ),
QStringLiteral( "sqlite" ),
QStringLiteral( "db" ),
QStringLiteral( "gdb" ),
QStringLiteral( "kml" ),
QStringLiteral( "osm" ),
QStringLiteral( "mdb" ),
QStringLiteral( "accdb" ),
QStringLiteral( "xls" ),
QStringLiteral( "xlsx" ),
QStringLiteral( "gpx" ),
QStringLiteral( "pdf" ),
QStringLiteral( "pbf" ),
QStringLiteral( "nc" ) };
const QStringList QgsGdalUtils::INEXPENSIVE_TO_SCAN_EXTENSIONS
{
QStringLiteral( "xlsx" ),
QStringLiteral( "ods" ),
QStringLiteral( "csv" ),
QStringLiteral( "nc" ),
QStringLiteral( "shp.zip" ) };
bool QgsGdalUtils::supportsRasterCreate( GDALDriverH driver )
{
QString driverShortName = GDALGetDriverShortName( driver );

8
src/core/qgsgdalutils.h
View File

@ -148,6 +148,14 @@ class CORE_EXPORT QgsGdalUtils
*/
static bool pathIsCheapToOpen( const QString &path, int smallFileSizeLimit = 50000 );
/**
* File extensions for formats supported by GDAL which may contain multiple layers
* and should be treated as a potential layer container.
* \since QGIS 3.22
*/
static const QStringList SUPPORTED_DB_LAYERS_EXTENSIONS;
friend class TestQgsGdalUtils;
};

80
tests/src/core/testqgsgdalprovider.cpp
View File

@ -64,6 +64,7 @@ class TestQgsGdalProvider : public QObject
void scale0(); //test when data has scale 0 (#20493)
void transformCoordinates();
void testGdalProviderQuerySublayers();
void testGdalProviderQuerySublayersFastScan();
private:
QString mTestDataDir;
@ -428,8 +429,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers()
// not a raster
res = gdalMetadata->querySublayers( QString( TEST_DATA_DIR ) + "/lines.shp" );
QVERIFY( res.empty() );
res = gdalMetadata->querySublayers( QString( TEST_DATA_DIR ) + "/lines.shp", Qgis::SublayerQueryFlag::FastScan );
QVERIFY( res.empty() );
// single layer raster
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/landsat.tif" );
@ -468,8 +467,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers()
QCOMPARE( res.at( 1 ).driverName(), QStringLiteral( "GPKG" ) );
rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 1 ).toLayer( options ) ) );
QVERIFY( rl->isValid() );
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mixed_layers.gpkg", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 2 );
// netcdf file
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc" );
@ -493,9 +490,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers()
rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 1 ).toLayer( options ) ) );
QVERIFY( rl->isValid() );
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 8 );
// netcdf with open options
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc|option:HONOUR_VALID_RANGE=YES" );
QCOMPARE( res.count(), 8 );
@ -544,19 +538,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers()
rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 0 ).toLayer( options ) ) );
QVERIFY( rl->isValid() );
// aigrid, with fast scan
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/aigrid", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).layerNumber(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "aigrid" ) );
QCOMPARE( res.at( 0 ).description(), QString() );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( "%1/aigrid" ).arg( QStringLiteral( TEST_DATA_DIR ) ) );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QCOMPARE( res.at( 0 ).driverName(), QStringLiteral( "AIG" ) );
rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 0 ).toLayer( options ) ) );
QVERIFY( rl->isValid() );
// zip archive, only 1 file
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/zip/landsat_b1.zip" );
QCOMPARE( res.count(), 1 );
@ -626,5 +607,64 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers()
QVERIFY( rl->isValid() );
}
void TestQgsGdalProvider::testGdalProviderQuerySublayersFastScan()
{
// test querying sub layers for a mesh layer
QgsProviderMetadata *gdalMetadata = QgsProviderRegistry::instance()->providerMetadata( QStringLiteral( "gdal" ) );
// invalid uri
QList< QgsProviderSublayerDetails >res = gdalMetadata->querySublayers( QString(), Qgis::SublayerQueryFlag::FastScan );
QVERIFY( res.empty() );
// not a raster
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/lines.shp", Qgis::SublayerQueryFlag::FastScan );
QVERIFY( res.empty() );
// single layer raster
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/landsat.tif", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "landsat" ) );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/landsat.tif" );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QVERIFY( !res.at( 0 ).skippedContainerScan() );
// geopackage with two raster layers
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mixed_layers.gpkg", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "mixed_layers" ) );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/mixed_layers.gpkg" );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QVERIFY( res.at( 0 ).skippedContainerScan() );
// netcdf file
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "trap_steady_05_3D" ) );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc" );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QVERIFY( res.at( 0 ).skippedContainerScan() );
// netcdf with open options
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc|option:HONOUR_VALID_RANGE=YES", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "trap_steady_05_3D" ) );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc|option:HONOUR_VALID_RANGE=YES" );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QVERIFY( res.at( 0 ).skippedContainerScan() );
// aigrid, pointing to .adf file
res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/aigrid/hdr.adf", Qgis::SublayerQueryFlag::FastScan );
QCOMPARE( res.count(), 1 );
QCOMPARE( res.at( 0 ).name(), QStringLiteral( "aigrid" ) );
QCOMPARE( res.at( 0 ).uri(), QStringLiteral( "%1/aigrid/hdr.adf" ).arg( QStringLiteral( TEST_DATA_DIR ) ) );
QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) );
QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer );
QVERIFY( !res.at( 0 ).skippedContainerScan() );
}
QGSTEST_MAIN( TestQgsGdalProvider )
#include "testqgsgdalprovider.moc"