From e06e7d0338685677b23ea07ca0efe5fff2340b35 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 21 Jul 2021 08:01:30 +1000 Subject: [PATCH] Create a generic data item provider for all file based datasources This provider uses the QgsProviderRegistry::querySublayers API to automatically create appropriate browser data items for all file based sources, regardless of the underlying provider (be it mdal, gdal, ogr, pdal or ept). This allows us to merge sources which can be handled by multiple different providers into single container items in the browser, so e.g. instead of seeing a .nc file appear twice in the browser (once with a mesh icon and once with a raster icon), we now only see the file ONCE and expanding it out shows BOTH the mesh and raster sublayers. Similarly with all other mixed type formats, such as GeoPDF files (which may contain a mix of raster and vector layers), KML/KMZ,... etc. --- .../browser/qgsinbuiltdataitemproviders.cpp | 2 +- src/core/CMakeLists.txt | 2 + .../browser/qgsdataitemproviderregistry.cpp | 3 + .../browser/qgsfilebaseddataitemprovider.cpp | 353 ++++++++++++++++++ .../browser/qgsfilebaseddataitemprovider.h | 122 ++++++ src/core/providers/gdal/qgsgdalprovider.cpp | 2 +- .../providers/ogr/qgsogrprovidermetadata.cpp | 2 +- src/providers/mdal/qgsmdalprovider.cpp | 2 +- 8 files changed, 484 insertions(+), 4 deletions(-) create mode 100644 src/core/browser/qgsfilebaseddataitemprovider.cpp create mode 100644 src/core/browser/qgsfilebaseddataitemprovider.h diff --git a/src/app/browser/qgsinbuiltdataitemproviders.cpp b/src/app/browser/qgsinbuiltdataitemproviders.cpp index 1e5d2cccbd9..bb2c5cc5664 100644 --- a/src/app/browser/qgsinbuiltdataitemproviders.cpp +++ b/src/app/browser/qgsinbuiltdataitemproviders.cpp @@ -1065,7 +1065,7 @@ void QgsDatabaseItemGuiProvider::populateContextMenu( QgsDataItem *item, QMenu * // SQL dialog if ( std::unique_ptr conn( item->databaseConnection() ); conn && conn->capabilities().testFlag( QgsAbstractDatabaseProviderConnection::Capability::ExecuteSql ) ) { - QAction *sqlAction = new QAction( QObject::tr( "Execute SQL …" ), menu ); + QAction *sqlAction = new QAction( QObject::tr( "Execute SQL…" ), menu ); QObject::connect( sqlAction, &QAction::triggered, item, [ item, context ] { diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 78ddb6a336e..c991afb9084 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -502,6 +502,7 @@ set(QGIS_CORE_SRCS browser/qgsdirectoryitem.cpp browser/qgsfavoritesitem.cpp browser/qgsfieldsitem.cpp + browser/qgsfilebaseddataitemprovider.cpp browser/qgslayeritem.cpp browser/qgsprojectitem.cpp browser/qgszipitem.cpp @@ -1168,6 +1169,7 @@ set(QGIS_CORE_HDRS browser/qgsdirectoryitem.h browser/qgsfavoritesitem.h browser/qgsfieldsitem.h + browser/qgsfilebaseddataitemprovider.h browser/qgslayeritem.h browser/qgsprojectitem.h browser/qgszipitem.h diff --git a/src/core/browser/qgsdataitemproviderregistry.cpp b/src/core/browser/qgsdataitemproviderregistry.cpp index 133a944fea7..fe2da562ca5 100644 --- a/src/core/browser/qgsdataitemproviderregistry.cpp +++ b/src/core/browser/qgsdataitemproviderregistry.cpp @@ -20,9 +20,12 @@ #include "qgsdataprovider.h" #include "qgslogger.h" #include "qgsproviderregistry.h" +#include "qgsfilebaseddataitemprovider.h" QgsDataItemProviderRegistry::QgsDataItemProviderRegistry() { + mProviders << new QgsFileBasedDataItemProvider(); + QStringList providersList = QgsProviderRegistry::instance()->providerList(); const auto constProvidersList = providersList; diff --git a/src/core/browser/qgsfilebaseddataitemprovider.cpp b/src/core/browser/qgsfilebaseddataitemprovider.cpp new file mode 100644 index 00000000000..c5bec6d9f00 --- /dev/null +++ b/src/core/browser/qgsfilebaseddataitemprovider.cpp @@ -0,0 +1,353 @@ +/*************************************************************************** + qgsfilebaseddataitemprovider.cpp + -------------------------------------- + Date : July 2021 + Copyright : (C) 2021 by Nyall Dawson + Email : nyall dot dawson at gmail dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#include "qgsfilebaseddataitemprovider.h" +#include "qgsdataprovider.h" +#include "qgsproviderregistry.h" +#include "qgslogger.h" +#include "qgssettings.h" +#include "qgszipitem.h" +#include "qgsogrproviderutils.h" +#include "qgsstyle.h" +#include "qgsgdalutils.h" +#include "qgsgeopackagedataitems.h" +#include "qgsprovidersublayerdetails.h" +#include "qgsfieldsitem.h" +#include "qgsproviderutils.h" +#include "qgsmbtiles.h" +#include "qgsvectortiledataitems.h" +#include "qgsprovidermetadata.h" +#include + +// +// QgsProviderSublayerItem +// + +QgsProviderSublayerItem::QgsProviderSublayerItem( QgsDataItem *parent, const QString &name, + const QgsProviderSublayerDetails &details, bool isFile ) + : QgsLayerItem( parent, name, details.uri(), details.uri(), layerTypeFromSublayer( details ), details.providerKey() ) + , mDetails( details ) + , mIsFile( isFile ) +{ + mToolTip = details.uri(); + + // no children, except for sqlite, which gets special handling because of the unusual situation with the spatialite provider + setState( details.driverName() == QStringLiteral( "SQLite" ) ? Qgis::BrowserItemState::NotPopulated : Qgis::BrowserItemState::Populated ); +} + +QVector QgsProviderSublayerItem::createChildren() +{ + QVector children; + + if ( mDetails.type() == QgsMapLayerType::VectorLayer ) + { + // sqlite gets special handling because of the spatialite provider which supports the api required for a fields item. + // TODO -- allow read only fields items to be created directly from vector layers, so that all vector layers can show field items. + if ( mDetails.driverName() == QLatin1String( "SQLite" ) ) + { + children.push_back( new QgsFieldsItem( this, + path() + QStringLiteral( "/columns/ " ), + QStringLiteral( R"(dbname="%1")" ).arg( parent()->path().replace( '"', QLatin1String( R"(\")" ) ) ), + QStringLiteral( "spatialite" ), QString(), name() ) ); + } + } + return children; +} + +bool QgsProviderSublayerItem::isFile() const +{ + return mIsFile; +} + +Qgis::BrowserLayerType QgsProviderSublayerItem::layerTypeFromSublayer( const QgsProviderSublayerDetails &sublayer ) +{ + switch ( sublayer.type() ) + { + case QgsMapLayerType::VectorLayer: + { + switch ( QgsWkbTypes::geometryType( sublayer.wkbType() ) ) + { + case QgsWkbTypes::PointGeometry: + return Qgis::BrowserLayerType::Point; + + case QgsWkbTypes::LineGeometry: + return Qgis::BrowserLayerType::Line; + + case QgsWkbTypes::PolygonGeometry: + return Qgis::BrowserLayerType::Polygon; + + case QgsWkbTypes::NullGeometry: + return Qgis::BrowserLayerType::TableLayer; + + case QgsWkbTypes::UnknownGeometry: + return Qgis::BrowserLayerType::Vector; + } + + break; + } + case QgsMapLayerType::RasterLayer: + return Qgis::BrowserLayerType::Raster; + + case QgsMapLayerType::PluginLayer: + return Qgis::BrowserLayerType::Plugin; + + case QgsMapLayerType::MeshLayer: + return Qgis::BrowserLayerType::Mesh; + + case QgsMapLayerType::VectorTileLayer: + return Qgis::BrowserLayerType::VectorTile; + + case QgsMapLayerType::PointCloudLayer: + return Qgis::BrowserLayerType::PointCloud; + + case QgsMapLayerType::AnnotationLayer: + break; + } + return Qgis::BrowserLayerType::NoType; +} + +QString QgsProviderSublayerItem::layerName() const +{ + return mDetails.name(); +} + +// +// QgsFileDataCollectionItem +// + +QgsFileDataCollectionItem::QgsFileDataCollectionItem( QgsDataItem *parent, const QString &name, const QString &path, const QList &sublayers ) + : QgsDataCollectionItem( parent, name, path ) + , mSublayers( sublayers ) +{ + if ( QgsProviderUtils::sublayerDetailsAreIncomplete( mSublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount ) ) + setCapabilities( Qgis::BrowserItemCapability::Fertile ); + else + setCapabilities( Qgis::BrowserItemCapability::Fast | Qgis::BrowserItemCapability::Fertile ); +} + +QVector QgsFileDataCollectionItem::createChildren() +{ + if ( QgsProviderUtils::sublayerDetailsAreIncomplete( mSublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount ) ) + { + mSublayers = QgsProviderRegistry::instance()->querySublayers( path(), Qgis::SublayerQueryFlag::ResolveGeometryType ); + } + + QVector children; + children.reserve( mSublayers.size() ); + for ( const QgsProviderSublayerDetails &sublayer : std::as_const( mSublayers ) ) + { + QgsProviderSublayerItem *item = new QgsProviderSublayerItem( this, sublayer.name(), sublayer, true ); + children.append( item ); + } + + return children; +} + +bool QgsFileDataCollectionItem::hasDragEnabled() const +{ + return true; +} + +QgsMimeDataUtils::UriList QgsFileDataCollectionItem::mimeUris() const +{ + QgsMimeDataUtils::Uri collectionUri; + collectionUri.uri = path(); + collectionUri.layerType = QStringLiteral( "collection" ); + return { collectionUri }; +} + +QgsAbstractDatabaseProviderConnection *QgsFileDataCollectionItem::databaseConnection() const +{ + // sqlite gets special handling because of the spatialite provider which supports the api required database connections + const QFileInfo fi( mPath ); + if ( fi.suffix().toLower() != QLatin1String( "sqlite" ) ) + { + return nullptr; + } + + QgsAbstractDatabaseProviderConnection *conn = nullptr; + + // test that file is valid with OGR + if ( OGRGetDriverCount() == 0 ) + { + OGRRegisterAll(); + } + // do not print errors, but write to debug + CPLPushErrorHandler( CPLQuietErrorHandler ); + CPLErrorReset(); + gdal::dataset_unique_ptr hDS( GDALOpenEx( path().toUtf8().constData(), GDAL_OF_VECTOR, nullptr, nullptr, nullptr ) ); + CPLPopErrorHandler(); + + if ( ! hDS ) + { + QgsDebugMsgLevel( QStringLiteral( "GDALOpen error # %1 : %2 on %3" ).arg( CPLGetLastErrorNo() ).arg( CPLGetLastErrorMsg() ).arg( path() ), 2 ); + return nullptr; + } + + GDALDriverH hDriver = GDALGetDatasetDriver( hDS.get() ); + QString driverName = GDALGetDriverShortName( hDriver ); + + if ( driverName == QLatin1String( "SQLite" ) ) + { + QgsProviderMetadata *md { QgsProviderRegistry::instance()->providerMetadata( QStringLiteral( "spatialite" ) ) }; + if ( md ) + { + QgsDataSourceUri uri; + uri.setDatabase( path( ) ); + conn = static_cast( md->createConnection( uri.uri(), {} ) ); + } + } + return conn; +} + +// +// QgsFileBasedDataItemProvider +// + +QString QgsFileBasedDataItemProvider::name() +{ + return QStringLiteral( "files" ); +} + +int QgsFileBasedDataItemProvider::capabilities() const +{ + return QgsDataProvider::File | QgsDataProvider::Dir; +} + +QgsDataItem *QgsFileBasedDataItemProvider::createDataItem( const QString &pathIn, QgsDataItem *parentItem ) +{ + QString path( pathIn ); + if ( path.isEmpty() ) + return nullptr; + + const QFileInfo info( path ); + QString suffix = info.suffix().toLower(); + const QString name = info.fileName(); + + // special handling for some suffixes + if ( suffix.compare( QLatin1String( "gpkg" ), Qt::CaseInsensitive ) == 0 ) + { + // Geopackage is special -- it gets a dedicated collection item type + return new QgsGeoPackageCollectionItem( parentItem, name, path ); + } + else if ( suffix == QLatin1String( "txt" ) ) + { + // never ever show .txt files as datasets in browser -- they are only used for geospatial data in extremely rare cases + // and are predominantly just noise in the browser + return nullptr; + } + // If a .tab exists, then the corresponding .map/.dat is very likely a + // side-car file of the .tab + else if ( suffix == QLatin1String( "map" ) || suffix == QLatin1String( "dat" ) ) + { + if ( QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".tab" ) ) || QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".TAB" ) ) ) + return nullptr; + } + // .dbf and .shx should only appear if .shp is not present + else if ( suffix == QLatin1String( "dbf" ) || suffix == QLatin1String( "shx" ) ) + { + if ( QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".shp" ) ) || QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".SHP" ) ) ) + return nullptr; + } + // skip QGIS style xml files + else if ( suffix == QLatin1String( "xml" ) && QgsStyle::isXmlStyleFile( path ) ) + { + return nullptr; + } + // GDAL 3.1 Shapefile driver directly handles .shp.zip files + else if ( path.endsWith( QLatin1String( ".shp.zip" ), Qt::CaseInsensitive ) && + GDALIdentifyDriverEx( path.toUtf8().constData(), GDAL_OF_VECTOR, nullptr, nullptr ) ) + { + suffix = QStringLiteral( "shp.zip" ); + } + // special handling for mbtiles files + else if ( suffix == QLatin1String( "mbtiles" ) ) + { + QgsMbTiles reader( path ); + if ( reader.open() ) + { + if ( reader.metadataValue( QStringLiteral( "format" ) ) == QLatin1String( "pbf" ) ) + { + // these are vector tiles + QUrlQuery uq; + uq.addQueryItem( QStringLiteral( "type" ), QStringLiteral( "mbtiles" ) ); + uq.addQueryItem( QStringLiteral( "url" ), path ); + QString encodedUri = uq.toString(); + return new QgsVectorTileLayerItem( parentItem, name, path, encodedUri ); + } + else + { + // handled by WMS provider + QUrlQuery uq; + uq.addQueryItem( QStringLiteral( "type" ), QStringLiteral( "mbtiles" ) ); + uq.addQueryItem( QStringLiteral( "url" ), QUrl::fromLocalFile( path ).toString() ); + QString encodedUri = uq.toString(); + QgsLayerItem *item = new QgsLayerItem( parentItem, name, path, encodedUri, Qgis::BrowserLayerType::Raster, QStringLiteral( "wms" ) ); + item->setState( Qgis::BrowserItemState::Populated ); + return item; + } + } + } + + // hide blocklisted URIs, such as .aux.xml files + if ( QgsProviderRegistry::instance()->uriIsBlocklisted( path ) ) + return nullptr; + + // allow only normal files, supported directories, or VSIFILE items to continue + const QStringList dirExtensions = QgsOgrProviderUtils::directoryExtensions(); + bool isOgrSupportedDirectory = info.isDir() && dirExtensions.contains( suffix ); + if ( !isOgrSupportedDirectory && !info.isFile() ) + return nullptr; + + QgsSettings settings; + + Qgis::SublayerQueryFlags queryFlags = Qgis::SublayerQueryFlags(); + + // should we fast scan only? + if ( ( settings.value( QStringLiteral( "qgis/scanItemsInBrowser2" ), + "extension" ).toString() == QLatin1String( "extension" ) ) || + ( parentItem && settings.value( QStringLiteral( "qgis/scanItemsFastScanUris" ), + QStringList() ).toStringList().contains( parentItem->path() ) ) ) + { + queryFlags |= Qgis::SublayerQueryFlag::FastScan; + } + + const QList sublayers = QgsProviderRegistry::instance()->querySublayers( path, queryFlags ); + + if ( sublayers.size() == 1 + && ( ( ( queryFlags & Qgis::SublayerQueryFlag::FastScan ) && !QgsProviderUtils::sublayerDetailsAreIncomplete( sublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount | QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownGeometryType ) ) + || ( !( queryFlags & Qgis::SublayerQueryFlag::FastScan ) && !QgsProviderUtils::sublayerDetailsAreIncomplete( sublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount ) ) ) + ) + { + return new QgsProviderSublayerItem( parentItem, name, sublayers.at( 0 ), false ); + } + else if ( !sublayers.empty() ) + { + return new QgsFileDataCollectionItem( parentItem, name, path, sublayers ); + } + else + { + return nullptr; + } +} + +bool QgsFileBasedDataItemProvider::handlesDirectoryPath( const QString &path ) +{ + QFileInfo info( path ); + QString suffix = info.suffix().toLower(); + + QStringList dirExtensions = QgsOgrProviderUtils::directoryExtensions(); + return dirExtensions.contains( suffix ); +} diff --git a/src/core/browser/qgsfilebaseddataitemprovider.h b/src/core/browser/qgsfilebaseddataitemprovider.h new file mode 100644 index 00000000000..69978b2694c --- /dev/null +++ b/src/core/browser/qgsfilebaseddataitemprovider.h @@ -0,0 +1,122 @@ +/*************************************************************************** + qgsfilebaseddataitemprovider.h + -------------------------------------- + Date : July 2021 + Copyright : (C) 2021 by Nyall Dawson + Email : nyall dot dawson at gmail dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#ifndef QGSFILEBASEDDATAITEMPROVIDER_H +#define QGSFILEBASEDDATAITEMPROVIDER_H + +#include "qgis_core.h" +#include "qgis_sip.h" +#include "qgsdataitemprovider.h" +#include "qgsdatacollectionitem.h" +#include "qgslayeritem.h" +#include "qgsprovidersublayerdetails.h" +#include +#include + +class QgsProviderSublayerDetails; + +#define SIP_NO_FILE + +class QgsDataItem; + +/** + * \ingroup core + * \brief A generic data item for file based layers. + * + * This is a generic data item for file based layers. It is created by a QgsFileBasedDataItemProvider + * for files which represent a single layer, or as children of a QgsFileDataCollectionItem for + * files which contain multiple layers. + * + * \since QGIS 3.22 + */ +class CORE_EXPORT QgsProviderSublayerItem final: public QgsLayerItem +{ + Q_OBJECT + public: + QgsProviderSublayerItem( QgsDataItem *parent, const QString &name, const QgsProviderSublayerDetails &details, bool isFile ); + QString layerName() const override; + QVector createChildren() override; + + /** + * Returns TRUE if this item directly represents a file, i.e. it is not a sublayer + * of a QgsFileDataCollectionItem. + */ + bool isFile() const; + + private: + + static Qgis::BrowserLayerType layerTypeFromSublayer( const QgsProviderSublayerDetails &sublayer ); + + QgsProviderSublayerDetails mDetails; + bool mIsFile = false; + +}; + +/** + * \ingroup core + * \brief A data collection item for file based data collections (e.g. NetCDF files). + * + * This is a generic data collection item, which is created by a QgsFileBasedDataItemProvider + * for datasets which may potentially contain multiple sublayers. + * + * \since QGIS 3.22 + */ +class CORE_EXPORT QgsFileDataCollectionItem final: public QgsDataCollectionItem +{ + Q_OBJECT + public: + + /** + * Constructor for QgsFileDataCollectionItem. + * \param parent parent item + * \param name data item name (this should usually match the filename of the dataset) + * \param path path to dataset + * \param sublayers list of sublayers to initially populate the item with. If the sublayer details are incomplete + * (see QgsProviderUtils::sublayerDetailsAreIncomplete()) then the item will be populated in a background thread when + * expanded. + */ + QgsFileDataCollectionItem( QgsDataItem *parent, const QString &name, const QString &path, const QList< QgsProviderSublayerDetails> &sublayers ); + + QVector createChildren() override; + bool hasDragEnabled() const override; + QgsMimeDataUtils::UriList mimeUris() const override; + QgsAbstractDatabaseProviderConnection *databaseConnection() const override; + + private: + + QList< QgsProviderSublayerDetails> mSublayers; +}; + + +/** + * \ingroup core + * \brief A data item provider for file based data sources. + * + * This is a generic data item provider, which creates data items for file based data sources from + * registered providers (using the QgsProviderRegistry::querySublayers() API). + * + * \since QGIS 3.22 + */ +class CORE_EXPORT QgsFileBasedDataItemProvider : public QgsDataItemProvider +{ + public: + + QString name() override; + int capabilities() const override; + QgsDataItem *createDataItem( const QString &path, QgsDataItem *parentItem ) override SIP_FACTORY; + bool handlesDirectoryPath( const QString &path ) override; +}; + +#endif // QGSFILEBASEDDATAITEMPROVIDER_H diff --git a/src/core/providers/gdal/qgsgdalprovider.cpp b/src/core/providers/gdal/qgsgdalprovider.cpp index 40d78b19657..2a58ecf56fc 100644 --- a/src/core/providers/gdal/qgsgdalprovider.cpp +++ b/src/core/providers/gdal/qgsgdalprovider.cpp @@ -3706,7 +3706,7 @@ QList QgsGdalProviderMetadata::querySublayers( const QList QgsGdalProviderMetadata::dataItemProviders() const { QList< QgsDataItemProvider * > providers; - providers << new QgsGdalDataItemProvider; + //providers << new QgsGdalDataItemProvider; return providers; } diff --git a/src/core/providers/ogr/qgsogrprovidermetadata.cpp b/src/core/providers/ogr/qgsogrprovidermetadata.cpp index 8b2b0488bea..c287a863fb1 100644 --- a/src/core/providers/ogr/qgsogrprovidermetadata.cpp +++ b/src/core/providers/ogr/qgsogrprovidermetadata.cpp @@ -223,7 +223,7 @@ QString QgsOgrProviderMetadata::encodeUri( const QVariantMap &parts ) const QList QgsOgrProviderMetadata::dataItemProviders() const { QList< QgsDataItemProvider * > providers; - providers << new QgsOgrDataItemProvider; +// providers << new QgsOgrDataItemProvider; providers << new QgsGeoPackageDataItemProvider; return providers; } diff --git a/src/providers/mdal/qgsmdalprovider.cpp b/src/providers/mdal/qgsmdalprovider.cpp index 71ce9a8f3bd..c9c965e18f7 100644 --- a/src/providers/mdal/qgsmdalprovider.cpp +++ b/src/providers/mdal/qgsmdalprovider.cpp @@ -965,7 +965,7 @@ QgsMdalProvider *QgsMdalProviderMetadata::createProvider( const QString &uri, co QList QgsMdalProviderMetadata::dataItemProviders() const { QList providers; - providers << new QgsMdalDataItemProvider; +// providers << new QgsMdalDataItemProvider; return providers; }