From c7c97101d3a44ae942bf7b6907147b2e22cd384e Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 22 Jul 2021 08:54:44 +1000 Subject: [PATCH] [gdal] Correctly implement FastScan for querySublayers --- src/core/providers/gdal/qgsgdalprovider.cpp | 88 ++++++++++++------- .../providers/ogr/qgsogrprovidermetadata.cpp | 27 +----- src/core/qgsgdalutils.cpp | 27 ++++++ src/core/qgsgdalutils.h | 8 ++ tests/src/core/testqgsgdalprovider.cpp | 80 ++++++++++++----- 5 files changed, 153 insertions(+), 77 deletions(-) diff --git a/src/core/providers/gdal/qgsgdalprovider.cpp b/src/core/providers/gdal/qgsgdalprovider.cpp index 13888a124b2..80eee2e2c33 100644 --- a/src/core/providers/gdal/qgsgdalprovider.cpp +++ b/src/core/providers/gdal/qgsgdalprovider.cpp @@ -3562,48 +3562,70 @@ QList QgsGdalProviderMetadata::querySublayers( const } } - if ( flags & Qgis::SublayerQueryFlag::FastScan ) + const QString path = uriParts.value( QStringLiteral( "path" ) ).toString(); + const QFileInfo pathInfo( path ); + if ( flags & Qgis::SublayerQueryFlag::FastScan && ( pathInfo.isFile() || pathInfo.isDir() ) ) { - // filter based on extension - const QVariantMap uriParts = decodeUri( gdalUri ); - const QString path = uriParts.value( QStringLiteral( "path" ) ).toString(); - QFileInfo info( path ); - if ( info.isFile() ) + // fast scan, so we don't actually try to open the dataset and instead just check the extension alone + static QString sFilterString; + static QStringList sExtensions; + static QStringList sWildcards; + + // get supported extensions + static std::once_flag initialized; + std::call_once( initialized, [ = ] { - const QString suffix = info.suffix().toLower(); + buildSupportedRasterFileFilterAndExtensions( sFilterString, sExtensions, sWildcards ); + QgsDebugMsgLevel( QStringLiteral( "extensions: " ) + sExtensions.join( ' ' ), 2 ); + QgsDebugMsgLevel( QStringLiteral( "wildcards: " ) + sWildcards.join( ' ' ), 2 ); + } ); - static QString sFilterString; - static QStringList sExtensions; - static QStringList sWildcards; + const QString suffix = pathInfo.suffix().toLower(); - // get supported extensions - static std::once_flag initialized; - std::call_once( initialized, [ = ] + if ( !sExtensions.contains( suffix ) ) + { + bool matches = false; + for ( const QString &wildcard : std::as_const( sWildcards ) ) { - buildSupportedRasterFileFilterAndExtensions( sFilterString, sExtensions, sWildcards ); - QgsDebugMsgLevel( QStringLiteral( "extensions: " ) + sExtensions.join( ' ' ), 2 ); - QgsDebugMsgLevel( QStringLiteral( "wildcards: " ) + sWildcards.join( ' ' ), 2 ); - } ); - - if ( !sExtensions.contains( suffix ) ) - { - bool matches = false; - for ( const QString &wildcard : std::as_const( sWildcards ) ) + const thread_local QRegularExpression rx( QRegularExpression::anchoredPattern( + QRegularExpression::wildcardToRegularExpression( wildcard ) + ), QRegularExpression::CaseInsensitiveOption ); + const QRegularExpressionMatch match = rx.match( pathInfo.fileName() ); + if ( match.hasMatch() ) { - const thread_local QRegularExpression rx( QRegularExpression::anchoredPattern( - QRegularExpression::wildcardToRegularExpression( wildcard ) - ), QRegularExpression::CaseInsensitiveOption ); - const QRegularExpressionMatch match = rx.match( info.fileName() ); - if ( match.hasMatch() ) - { - matches = true; - break; - } + matches = true; + break; } - if ( !matches ) - return {}; + } + if ( !matches ) + return {}; + } + + // if this is a VRT file make sure it is raster VRT + if ( suffix == QLatin1String( "vrt" ) ) + { + CPLPushErrorHandler( CPLQuietErrorHandler ); + CPLErrorReset(); + GDALDriverH hDriver = GDALIdentifyDriverEx( path.toUtf8().constData(), GDAL_OF_RASTER, nullptr, nullptr ); + CPLPopErrorHandler(); + if ( !hDriver ) + { + // vrt is not a raster vrt, skip it + return {}; } } + + QgsProviderSublayerDetails details; + details.setType( QgsMapLayerType::RasterLayer ); + details.setProviderKey( QStringLiteral( "gdal" ) ); + details.setUri( uri ); + details.setName( QgsProviderUtils::suggestLayerNameFromFilePath( path ) ); + if ( QgsGdalUtils::SUPPORTED_DB_LAYERS_EXTENSIONS.contains( suffix ) ) + { + // uri may contain sublayers, but query flags prevent us from examining them + details.setSkippedContainerScan( true ); + } + return {details}; } if ( !uriParts.value( QStringLiteral( "vsiPrefix" ) ).toString().isEmpty() diff --git a/src/core/providers/ogr/qgsogrprovidermetadata.cpp b/src/core/providers/ogr/qgsogrprovidermetadata.cpp index 305e14909ac..9bcf80c50f7 100644 --- a/src/core/providers/ogr/qgsogrprovidermetadata.cpp +++ b/src/core/providers/ogr/qgsogrprovidermetadata.cpp @@ -30,6 +30,7 @@ email : nyall dot dawson at gmail dot com #include "qgsprovidersublayerdetails.h" #include "qgszipitem.h" #include "qgsproviderutils.h" +#include "qgsgdalutils.h" #include #include @@ -1134,30 +1135,8 @@ QList QgsOgrProviderMetadata::querySublayers( const // these extensions are trivial to read, so there's no need to rely on // the extension only scan here -- avoiding it always gives us the correct data type // and sublayer visibility - static QStringList sSkipFastTrackExtensions { QStringLiteral( "xlsx" ), - QStringLiteral( "ods" ), - QStringLiteral( "csv" ), - QStringLiteral( "nc" ), - QStringLiteral( "shp.zip" ) }; - - if ( !sSkipFastTrackExtensions.contains( suffix ) ) + if ( !QgsGdalUtils::INEXPENSIVE_TO_SCAN_EXTENSIONS.contains( suffix ) ) { - // Filters out the OGR/GDAL supported formats that can contain multiple layers - // and should be treated as a potential layer container - static QStringList sOgrSupportedDbLayersExtensions { QStringLiteral( "gpkg" ), - QStringLiteral( "sqlite" ), - QStringLiteral( "db" ), - QStringLiteral( "gdb" ), - QStringLiteral( "kml" ), - QStringLiteral( "osm" ), - QStringLiteral( "mdb" ), - QStringLiteral( "accdb" ), - QStringLiteral( "xls" ), - QStringLiteral( "xlsx" ), - QStringLiteral( "gpx" ), - QStringLiteral( "pdf" ), - QStringLiteral( "pbf" ) }; - // if this is a VRT file make sure it is vector VRT if ( suffix == QLatin1String( "vrt" ) ) { @@ -1177,7 +1156,7 @@ QList QgsOgrProviderMetadata::querySublayers( const details.setProviderKey( QStringLiteral( "ogr" ) ); details.setUri( uri ); details.setName( QgsProviderUtils::suggestLayerNameFromFilePath( path ) ); - if ( sOgrSupportedDbLayersExtensions.contains( suffix ) ) + if ( QgsGdalUtils::SUPPORTED_DB_LAYERS_EXTENSIONS.contains( suffix ) ) { // uri may contain sublayers, but query flags prevent us from examining them details.setSkippedContainerScan( true ); diff --git a/src/core/qgsgdalutils.cpp b/src/core/qgsgdalutils.cpp index abfd34c4366..6e4835fb17b 100644 --- a/src/core/qgsgdalutils.cpp +++ b/src/core/qgsgdalutils.cpp @@ -28,6 +28,33 @@ #include #include +// File extensions for formats supported by GDAL which may contain multiple layers +// and should be treated as a potential layer container +const QStringList QgsGdalUtils::SUPPORTED_DB_LAYERS_EXTENSIONS +{ + QStringLiteral( "gpkg" ), + QStringLiteral( "sqlite" ), + QStringLiteral( "db" ), + QStringLiteral( "gdb" ), + QStringLiteral( "kml" ), + QStringLiteral( "osm" ), + QStringLiteral( "mdb" ), + QStringLiteral( "accdb" ), + QStringLiteral( "xls" ), + QStringLiteral( "xlsx" ), + QStringLiteral( "gpx" ), + QStringLiteral( "pdf" ), + QStringLiteral( "pbf" ), + QStringLiteral( "nc" ) }; + +const QStringList QgsGdalUtils::INEXPENSIVE_TO_SCAN_EXTENSIONS +{ + QStringLiteral( "xlsx" ), + QStringLiteral( "ods" ), + QStringLiteral( "csv" ), + QStringLiteral( "nc" ), + QStringLiteral( "shp.zip" ) }; + bool QgsGdalUtils::supportsRasterCreate( GDALDriverH driver ) { QString driverShortName = GDALGetDriverShortName( driver ); diff --git a/src/core/qgsgdalutils.h b/src/core/qgsgdalutils.h index 1cf843d65dd..274e328fd07 100644 --- a/src/core/qgsgdalutils.h +++ b/src/core/qgsgdalutils.h @@ -148,6 +148,14 @@ class CORE_EXPORT QgsGdalUtils */ static bool pathIsCheapToOpen( const QString &path, int smallFileSizeLimit = 50000 ); + + /** + * File extensions for formats supported by GDAL which may contain multiple layers + * and should be treated as a potential layer container. + * \since QGIS 3.22 + */ + static const QStringList SUPPORTED_DB_LAYERS_EXTENSIONS; + friend class TestQgsGdalUtils; }; diff --git a/tests/src/core/testqgsgdalprovider.cpp b/tests/src/core/testqgsgdalprovider.cpp index 730e190b63f..f63edd521fd 100644 --- a/tests/src/core/testqgsgdalprovider.cpp +++ b/tests/src/core/testqgsgdalprovider.cpp @@ -64,6 +64,7 @@ class TestQgsGdalProvider : public QObject void scale0(); //test when data has scale 0 (#20493) void transformCoordinates(); void testGdalProviderQuerySublayers(); + void testGdalProviderQuerySublayersFastScan(); private: QString mTestDataDir; @@ -428,8 +429,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers() // not a raster res = gdalMetadata->querySublayers( QString( TEST_DATA_DIR ) + "/lines.shp" ); QVERIFY( res.empty() ); - res = gdalMetadata->querySublayers( QString( TEST_DATA_DIR ) + "/lines.shp", Qgis::SublayerQueryFlag::FastScan ); - QVERIFY( res.empty() ); // single layer raster res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/landsat.tif" ); @@ -468,8 +467,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers() QCOMPARE( res.at( 1 ).driverName(), QStringLiteral( "GPKG" ) ); rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 1 ).toLayer( options ) ) ); QVERIFY( rl->isValid() ); - res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mixed_layers.gpkg", Qgis::SublayerQueryFlag::FastScan ); - QCOMPARE( res.count(), 2 ); // netcdf file res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc" ); @@ -493,9 +490,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers() rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 1 ).toLayer( options ) ) ); QVERIFY( rl->isValid() ); - res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc", Qgis::SublayerQueryFlag::FastScan ); - QCOMPARE( res.count(), 8 ); - // netcdf with open options res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc|option:HONOUR_VALID_RANGE=YES" ); QCOMPARE( res.count(), 8 ); @@ -544,19 +538,6 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers() rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 0 ).toLayer( options ) ) ); QVERIFY( rl->isValid() ); - // aigrid, with fast scan - res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/aigrid", Qgis::SublayerQueryFlag::FastScan ); - QCOMPARE( res.count(), 1 ); - QCOMPARE( res.at( 0 ).layerNumber(), 1 ); - QCOMPARE( res.at( 0 ).name(), QStringLiteral( "aigrid" ) ); - QCOMPARE( res.at( 0 ).description(), QString() ); - QCOMPARE( res.at( 0 ).uri(), QStringLiteral( "%1/aigrid" ).arg( QStringLiteral( TEST_DATA_DIR ) ) ); - QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) ); - QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer ); - QCOMPARE( res.at( 0 ).driverName(), QStringLiteral( "AIG" ) ); - rl.reset( qgis::down_cast< QgsRasterLayer * >( res.at( 0 ).toLayer( options ) ) ); - QVERIFY( rl->isValid() ); - // zip archive, only 1 file res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/zip/landsat_b1.zip" ); QCOMPARE( res.count(), 1 ); @@ -626,5 +607,64 @@ void TestQgsGdalProvider::testGdalProviderQuerySublayers() QVERIFY( rl->isValid() ); } +void TestQgsGdalProvider::testGdalProviderQuerySublayersFastScan() +{ + // test querying sub layers for a mesh layer + QgsProviderMetadata *gdalMetadata = QgsProviderRegistry::instance()->providerMetadata( QStringLiteral( "gdal" ) ); + + // invalid uri + QList< QgsProviderSublayerDetails >res = gdalMetadata->querySublayers( QString(), Qgis::SublayerQueryFlag::FastScan ); + QVERIFY( res.empty() ); + + // not a raster + res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/lines.shp", Qgis::SublayerQueryFlag::FastScan ); + QVERIFY( res.empty() ); + + // single layer raster + res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/landsat.tif", Qgis::SublayerQueryFlag::FastScan ); + QCOMPARE( res.count(), 1 ); + QCOMPARE( res.at( 0 ).name(), QStringLiteral( "landsat" ) ); + QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/landsat.tif" ); + QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) ); + QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer ); + QVERIFY( !res.at( 0 ).skippedContainerScan() ); + + // geopackage with two raster layers + res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mixed_layers.gpkg", Qgis::SublayerQueryFlag::FastScan ); + QCOMPARE( res.count(), 1 ); + QCOMPARE( res.at( 0 ).name(), QStringLiteral( "mixed_layers" ) ); + QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/mixed_layers.gpkg" ); + QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) ); + QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer ); + QVERIFY( res.at( 0 ).skippedContainerScan() ); + + // netcdf file + res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc", Qgis::SublayerQueryFlag::FastScan ); + QCOMPARE( res.count(), 1 ); + QCOMPARE( res.at( 0 ).name(), QStringLiteral( "trap_steady_05_3D" ) ); + QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc" ); + QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) ); + QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer ); + QVERIFY( res.at( 0 ).skippedContainerScan() ); + + // netcdf with open options + res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc|option:HONOUR_VALID_RANGE=YES", Qgis::SublayerQueryFlag::FastScan ); + QCOMPARE( res.count(), 1 ); + QCOMPARE( res.at( 0 ).name(), QStringLiteral( "trap_steady_05_3D" ) ); + QCOMPARE( res.at( 0 ).uri(), QStringLiteral( TEST_DATA_DIR ) + "/mesh/trap_steady_05_3D.nc|option:HONOUR_VALID_RANGE=YES" ); + QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) ); + QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer ); + QVERIFY( res.at( 0 ).skippedContainerScan() ); + + // aigrid, pointing to .adf file + res = gdalMetadata->querySublayers( QStringLiteral( TEST_DATA_DIR ) + "/aigrid/hdr.adf", Qgis::SublayerQueryFlag::FastScan ); + QCOMPARE( res.count(), 1 ); + QCOMPARE( res.at( 0 ).name(), QStringLiteral( "aigrid" ) ); + QCOMPARE( res.at( 0 ).uri(), QStringLiteral( "%1/aigrid/hdr.adf" ).arg( QStringLiteral( TEST_DATA_DIR ) ) ); + QCOMPARE( res.at( 0 ).providerKey(), QStringLiteral( "gdal" ) ); + QCOMPARE( res.at( 0 ).type(), QgsMapLayerType::RasterLayer ); + QVERIFY( !res.at( 0 ).skippedContainerScan() ); +} + QGSTEST_MAIN( TestQgsGdalProvider ) #include "testqgsgdalprovider.moc"