Create a generic data item provider for all file based datasources

This provider uses the QgsProviderRegistry::querySublayers API
to automatically create appropriate browser data items for all
file based sources, regardless of the underlying provider (be it
mdal, gdal, ogr, pdal or ept).

This allows us to merge sources which can be handled by multiple
different providers into single container items in the browser,
so e.g. instead of seeing a .nc file appear twice in the browser
(once with a mesh icon and once with a raster icon), we now
only see the file ONCE and expanding it out shows BOTH the mesh
and raster sublayers. Similarly with all other mixed type formats,
such as GeoPDF files (which may contain a mix of raster and vector
layers), KML/KMZ,... etc.
This commit is contained in:
Nyall Dawson 2021-07-21 08:01:30 +10:00
parent 6f942a70ac
commit e06e7d0338
8 changed files with 484 additions and 4 deletions

View File

@ -1065,7 +1065,7 @@ void QgsDatabaseItemGuiProvider::populateContextMenu( QgsDataItem *item, QMenu *
// SQL dialog
if ( std::unique_ptr<QgsAbstractDatabaseProviderConnection> conn( item->databaseConnection() ); conn && conn->capabilities().testFlag( QgsAbstractDatabaseProviderConnection::Capability::ExecuteSql ) )
{
QAction *sqlAction = new QAction( QObject::tr( "Execute SQL " ), menu );
QAction *sqlAction = new QAction( QObject::tr( "Execute SQL" ), menu );
QObject::connect( sqlAction, &QAction::triggered, item, [ item, context ]
{

View File

@ -502,6 +502,7 @@ set(QGIS_CORE_SRCS
browser/qgsdirectoryitem.cpp
browser/qgsfavoritesitem.cpp
browser/qgsfieldsitem.cpp
browser/qgsfilebaseddataitemprovider.cpp
browser/qgslayeritem.cpp
browser/qgsprojectitem.cpp
browser/qgszipitem.cpp
@ -1168,6 +1169,7 @@ set(QGIS_CORE_HDRS
browser/qgsdirectoryitem.h
browser/qgsfavoritesitem.h
browser/qgsfieldsitem.h
browser/qgsfilebaseddataitemprovider.h
browser/qgslayeritem.h
browser/qgsprojectitem.h
browser/qgszipitem.h

View File

@ -20,9 +20,12 @@
#include "qgsdataprovider.h"
#include "qgslogger.h"
#include "qgsproviderregistry.h"
#include "qgsfilebaseddataitemprovider.h"
QgsDataItemProviderRegistry::QgsDataItemProviderRegistry()
{
mProviders << new QgsFileBasedDataItemProvider();
QStringList providersList = QgsProviderRegistry::instance()->providerList();
const auto constProvidersList = providersList;

View File

@ -0,0 +1,353 @@
/***************************************************************************
qgsfilebaseddataitemprovider.cpp
--------------------------------------
Date : July 2021
Copyright : (C) 2021 by Nyall Dawson
Email : nyall dot dawson at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include "qgsfilebaseddataitemprovider.h"
#include "qgsdataprovider.h"
#include "qgsproviderregistry.h"
#include "qgslogger.h"
#include "qgssettings.h"
#include "qgszipitem.h"
#include "qgsogrproviderutils.h"
#include "qgsstyle.h"
#include "qgsgdalutils.h"
#include "qgsgeopackagedataitems.h"
#include "qgsprovidersublayerdetails.h"
#include "qgsfieldsitem.h"
#include "qgsproviderutils.h"
#include "qgsmbtiles.h"
#include "qgsvectortiledataitems.h"
#include "qgsprovidermetadata.h"
#include <QUrlQuery>
//
// QgsProviderSublayerItem
//
QgsProviderSublayerItem::QgsProviderSublayerItem( QgsDataItem *parent, const QString &name,
const QgsProviderSublayerDetails &details, bool isFile )
: QgsLayerItem( parent, name, details.uri(), details.uri(), layerTypeFromSublayer( details ), details.providerKey() )
, mDetails( details )
, mIsFile( isFile )
{
mToolTip = details.uri();
// no children, except for sqlite, which gets special handling because of the unusual situation with the spatialite provider
setState( details.driverName() == QStringLiteral( "SQLite" ) ? Qgis::BrowserItemState::NotPopulated : Qgis::BrowserItemState::Populated );
}
QVector<QgsDataItem *> QgsProviderSublayerItem::createChildren()
{
QVector<QgsDataItem *> children;
if ( mDetails.type() == QgsMapLayerType::VectorLayer )
{
// sqlite gets special handling because of the spatialite provider which supports the api required for a fields item.
// TODO -- allow read only fields items to be created directly from vector layers, so that all vector layers can show field items.
if ( mDetails.driverName() == QLatin1String( "SQLite" ) )
{
children.push_back( new QgsFieldsItem( this,
path() + QStringLiteral( "/columns/ " ),
QStringLiteral( R"(dbname="%1")" ).arg( parent()->path().replace( '"', QLatin1String( R"(\")" ) ) ),
QStringLiteral( "spatialite" ), QString(), name() ) );
}
}
return children;
}
bool QgsProviderSublayerItem::isFile() const
{
return mIsFile;
}
Qgis::BrowserLayerType QgsProviderSublayerItem::layerTypeFromSublayer( const QgsProviderSublayerDetails &sublayer )
{
switch ( sublayer.type() )
{
case QgsMapLayerType::VectorLayer:
{
switch ( QgsWkbTypes::geometryType( sublayer.wkbType() ) )
{
case QgsWkbTypes::PointGeometry:
return Qgis::BrowserLayerType::Point;
case QgsWkbTypes::LineGeometry:
return Qgis::BrowserLayerType::Line;
case QgsWkbTypes::PolygonGeometry:
return Qgis::BrowserLayerType::Polygon;
case QgsWkbTypes::NullGeometry:
return Qgis::BrowserLayerType::TableLayer;
case QgsWkbTypes::UnknownGeometry:
return Qgis::BrowserLayerType::Vector;
}
break;
}
case QgsMapLayerType::RasterLayer:
return Qgis::BrowserLayerType::Raster;
case QgsMapLayerType::PluginLayer:
return Qgis::BrowserLayerType::Plugin;
case QgsMapLayerType::MeshLayer:
return Qgis::BrowserLayerType::Mesh;
case QgsMapLayerType::VectorTileLayer:
return Qgis::BrowserLayerType::VectorTile;
case QgsMapLayerType::PointCloudLayer:
return Qgis::BrowserLayerType::PointCloud;
case QgsMapLayerType::AnnotationLayer:
break;
}
return Qgis::BrowserLayerType::NoType;
}
QString QgsProviderSublayerItem::layerName() const
{
return mDetails.name();
}
//
// QgsFileDataCollectionItem
//
QgsFileDataCollectionItem::QgsFileDataCollectionItem( QgsDataItem *parent, const QString &name, const QString &path, const QList<QgsProviderSublayerDetails> &sublayers )
: QgsDataCollectionItem( parent, name, path )
, mSublayers( sublayers )
{
if ( QgsProviderUtils::sublayerDetailsAreIncomplete( mSublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount ) )
setCapabilities( Qgis::BrowserItemCapability::Fertile );
else
setCapabilities( Qgis::BrowserItemCapability::Fast | Qgis::BrowserItemCapability::Fertile );
}
QVector<QgsDataItem *> QgsFileDataCollectionItem::createChildren()
{
if ( QgsProviderUtils::sublayerDetailsAreIncomplete( mSublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount ) )
{
mSublayers = QgsProviderRegistry::instance()->querySublayers( path(), Qgis::SublayerQueryFlag::ResolveGeometryType );
}
QVector<QgsDataItem *> children;
children.reserve( mSublayers.size() );
for ( const QgsProviderSublayerDetails &sublayer : std::as_const( mSublayers ) )
{
QgsProviderSublayerItem *item = new QgsProviderSublayerItem( this, sublayer.name(), sublayer, true );
children.append( item );
}
return children;
}
bool QgsFileDataCollectionItem::hasDragEnabled() const
{
return true;
}
QgsMimeDataUtils::UriList QgsFileDataCollectionItem::mimeUris() const
{
QgsMimeDataUtils::Uri collectionUri;
collectionUri.uri = path();
collectionUri.layerType = QStringLiteral( "collection" );
return { collectionUri };
}
QgsAbstractDatabaseProviderConnection *QgsFileDataCollectionItem::databaseConnection() const
{
// sqlite gets special handling because of the spatialite provider which supports the api required database connections
const QFileInfo fi( mPath );
if ( fi.suffix().toLower() != QLatin1String( "sqlite" ) )
{
return nullptr;
}
QgsAbstractDatabaseProviderConnection *conn = nullptr;
// test that file is valid with OGR
if ( OGRGetDriverCount() == 0 )
{
OGRRegisterAll();
}
// do not print errors, but write to debug
CPLPushErrorHandler( CPLQuietErrorHandler );
CPLErrorReset();
gdal::dataset_unique_ptr hDS( GDALOpenEx( path().toUtf8().constData(), GDAL_OF_VECTOR, nullptr, nullptr, nullptr ) );
CPLPopErrorHandler();
if ( ! hDS )
{
QgsDebugMsgLevel( QStringLiteral( "GDALOpen error # %1 : %2 on %3" ).arg( CPLGetLastErrorNo() ).arg( CPLGetLastErrorMsg() ).arg( path() ), 2 );
return nullptr;
}
GDALDriverH hDriver = GDALGetDatasetDriver( hDS.get() );
QString driverName = GDALGetDriverShortName( hDriver );
if ( driverName == QLatin1String( "SQLite" ) )
{
QgsProviderMetadata *md { QgsProviderRegistry::instance()->providerMetadata( QStringLiteral( "spatialite" ) ) };
if ( md )
{
QgsDataSourceUri uri;
uri.setDatabase( path( ) );
conn = static_cast<QgsAbstractDatabaseProviderConnection *>( md->createConnection( uri.uri(), {} ) );
}
}
return conn;
}
//
// QgsFileBasedDataItemProvider
//
QString QgsFileBasedDataItemProvider::name()
{
return QStringLiteral( "files" );
}
int QgsFileBasedDataItemProvider::capabilities() const
{
return QgsDataProvider::File | QgsDataProvider::Dir;
}
QgsDataItem *QgsFileBasedDataItemProvider::createDataItem( const QString &pathIn, QgsDataItem *parentItem )
{
QString path( pathIn );
if ( path.isEmpty() )
return nullptr;
const QFileInfo info( path );
QString suffix = info.suffix().toLower();
const QString name = info.fileName();
// special handling for some suffixes
if ( suffix.compare( QLatin1String( "gpkg" ), Qt::CaseInsensitive ) == 0 )
{
// Geopackage is special -- it gets a dedicated collection item type
return new QgsGeoPackageCollectionItem( parentItem, name, path );
}
else if ( suffix == QLatin1String( "txt" ) )
{
// never ever show .txt files as datasets in browser -- they are only used for geospatial data in extremely rare cases
// and are predominantly just noise in the browser
return nullptr;
}
// If a .tab exists, then the corresponding .map/.dat is very likely a
// side-car file of the .tab
else if ( suffix == QLatin1String( "map" ) || suffix == QLatin1String( "dat" ) )
{
if ( QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".tab" ) ) || QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".TAB" ) ) )
return nullptr;
}
// .dbf and .shx should only appear if .shp is not present
else if ( suffix == QLatin1String( "dbf" ) || suffix == QLatin1String( "shx" ) )
{
if ( QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".shp" ) ) || QFile::exists( QDir( info.path() ).filePath( info.baseName() + ".SHP" ) ) )
return nullptr;
}
// skip QGIS style xml files
else if ( suffix == QLatin1String( "xml" ) && QgsStyle::isXmlStyleFile( path ) )
{
return nullptr;
}
// GDAL 3.1 Shapefile driver directly handles .shp.zip files
else if ( path.endsWith( QLatin1String( ".shp.zip" ), Qt::CaseInsensitive ) &&
GDALIdentifyDriverEx( path.toUtf8().constData(), GDAL_OF_VECTOR, nullptr, nullptr ) )
{
suffix = QStringLiteral( "shp.zip" );
}
// special handling for mbtiles files
else if ( suffix == QLatin1String( "mbtiles" ) )
{
QgsMbTiles reader( path );
if ( reader.open() )
{
if ( reader.metadataValue( QStringLiteral( "format" ) ) == QLatin1String( "pbf" ) )
{
// these are vector tiles
QUrlQuery uq;
uq.addQueryItem( QStringLiteral( "type" ), QStringLiteral( "mbtiles" ) );
uq.addQueryItem( QStringLiteral( "url" ), path );
QString encodedUri = uq.toString();
return new QgsVectorTileLayerItem( parentItem, name, path, encodedUri );
}
else
{
// handled by WMS provider
QUrlQuery uq;
uq.addQueryItem( QStringLiteral( "type" ), QStringLiteral( "mbtiles" ) );
uq.addQueryItem( QStringLiteral( "url" ), QUrl::fromLocalFile( path ).toString() );
QString encodedUri = uq.toString();
QgsLayerItem *item = new QgsLayerItem( parentItem, name, path, encodedUri, Qgis::BrowserLayerType::Raster, QStringLiteral( "wms" ) );
item->setState( Qgis::BrowserItemState::Populated );
return item;
}
}
}
// hide blocklisted URIs, such as .aux.xml files
if ( QgsProviderRegistry::instance()->uriIsBlocklisted( path ) )
return nullptr;
// allow only normal files, supported directories, or VSIFILE items to continue
const QStringList dirExtensions = QgsOgrProviderUtils::directoryExtensions();
bool isOgrSupportedDirectory = info.isDir() && dirExtensions.contains( suffix );
if ( !isOgrSupportedDirectory && !info.isFile() )
return nullptr;
QgsSettings settings;
Qgis::SublayerQueryFlags queryFlags = Qgis::SublayerQueryFlags();
// should we fast scan only?
if ( ( settings.value( QStringLiteral( "qgis/scanItemsInBrowser2" ),
"extension" ).toString() == QLatin1String( "extension" ) ) ||
( parentItem && settings.value( QStringLiteral( "qgis/scanItemsFastScanUris" ),
QStringList() ).toStringList().contains( parentItem->path() ) ) )
{
queryFlags |= Qgis::SublayerQueryFlag::FastScan;
}
const QList<QgsProviderSublayerDetails> sublayers = QgsProviderRegistry::instance()->querySublayers( path, queryFlags );
if ( sublayers.size() == 1
&& ( ( ( queryFlags & Qgis::SublayerQueryFlag::FastScan ) && !QgsProviderUtils::sublayerDetailsAreIncomplete( sublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount | QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownGeometryType ) )
|| ( !( queryFlags & Qgis::SublayerQueryFlag::FastScan ) && !QgsProviderUtils::sublayerDetailsAreIncomplete( sublayers, QgsProviderUtils::SublayerCompletenessFlag::IgnoreUnknownFeatureCount ) ) )
)
{
return new QgsProviderSublayerItem( parentItem, name, sublayers.at( 0 ), false );
}
else if ( !sublayers.empty() )
{
return new QgsFileDataCollectionItem( parentItem, name, path, sublayers );
}
else
{
return nullptr;
}
}
bool QgsFileBasedDataItemProvider::handlesDirectoryPath( const QString &path )
{
QFileInfo info( path );
QString suffix = info.suffix().toLower();
QStringList dirExtensions = QgsOgrProviderUtils::directoryExtensions();
return dirExtensions.contains( suffix );
}

View File

@ -0,0 +1,122 @@
/***************************************************************************
qgsfilebaseddataitemprovider.h
--------------------------------------
Date : July 2021
Copyright : (C) 2021 by Nyall Dawson
Email : nyall dot dawson at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSFILEBASEDDATAITEMPROVIDER_H
#define QGSFILEBASEDDATAITEMPROVIDER_H
#include "qgis_core.h"
#include "qgis_sip.h"
#include "qgsdataitemprovider.h"
#include "qgsdatacollectionitem.h"
#include "qgslayeritem.h"
#include "qgsprovidersublayerdetails.h"
#include <QString>
#include <QVector>
class QgsProviderSublayerDetails;
#define SIP_NO_FILE
class QgsDataItem;
/**
* \ingroup core
* \brief A generic data item for file based layers.
*
* This is a generic data item for file based layers. It is created by a QgsFileBasedDataItemProvider
* for files which represent a single layer, or as children of a QgsFileDataCollectionItem for
* files which contain multiple layers.
*
* \since QGIS 3.22
*/
class CORE_EXPORT QgsProviderSublayerItem final: public QgsLayerItem
{
Q_OBJECT
public:
QgsProviderSublayerItem( QgsDataItem *parent, const QString &name, const QgsProviderSublayerDetails &details, bool isFile );
QString layerName() const override;
QVector<QgsDataItem *> createChildren() override;
/**
* Returns TRUE if this item directly represents a file, i.e. it is not a sublayer
* of a QgsFileDataCollectionItem.
*/
bool isFile() const;
private:
static Qgis::BrowserLayerType layerTypeFromSublayer( const QgsProviderSublayerDetails &sublayer );
QgsProviderSublayerDetails mDetails;
bool mIsFile = false;
};
/**
* \ingroup core
* \brief A data collection item for file based data collections (e.g. NetCDF files).
*
* This is a generic data collection item, which is created by a QgsFileBasedDataItemProvider
* for datasets which may potentially contain multiple sublayers.
*
* \since QGIS 3.22
*/
class CORE_EXPORT QgsFileDataCollectionItem final: public QgsDataCollectionItem
{
Q_OBJECT
public:
/**
* Constructor for QgsFileDataCollectionItem.
* \param parent parent item
* \param name data item name (this should usually match the filename of the dataset)
* \param path path to dataset
* \param sublayers list of sublayers to initially populate the item with. If the sublayer details are incomplete
* (see QgsProviderUtils::sublayerDetailsAreIncomplete()) then the item will be populated in a background thread when
* expanded.
*/
QgsFileDataCollectionItem( QgsDataItem *parent, const QString &name, const QString &path, const QList< QgsProviderSublayerDetails> &sublayers );
QVector<QgsDataItem *> createChildren() override;
bool hasDragEnabled() const override;
QgsMimeDataUtils::UriList mimeUris() const override;
QgsAbstractDatabaseProviderConnection *databaseConnection() const override;
private:
QList< QgsProviderSublayerDetails> mSublayers;
};
/**
* \ingroup core
* \brief A data item provider for file based data sources.
*
* This is a generic data item provider, which creates data items for file based data sources from
* registered providers (using the QgsProviderRegistry::querySublayers() API).
*
* \since QGIS 3.22
*/
class CORE_EXPORT QgsFileBasedDataItemProvider : public QgsDataItemProvider
{
public:
QString name() override;
int capabilities() const override;
QgsDataItem *createDataItem( const QString &path, QgsDataItem *parentItem ) override SIP_FACTORY;
bool handlesDirectoryPath( const QString &path ) override;
};
#endif // QGSFILEBASEDDATAITEMPROVIDER_H

View File

@ -3706,7 +3706,7 @@ QList<QgsProviderSublayerDetails> QgsGdalProviderMetadata::querySublayers( const
QList<QgsDataItemProvider *> QgsGdalProviderMetadata::dataItemProviders() const
{
QList< QgsDataItemProvider * > providers;
providers << new QgsGdalDataItemProvider;
//providers << new QgsGdalDataItemProvider;
return providers;
}

View File

@ -223,7 +223,7 @@ QString QgsOgrProviderMetadata::encodeUri( const QVariantMap &parts ) const
QList<QgsDataItemProvider *> QgsOgrProviderMetadata::dataItemProviders() const
{
QList< QgsDataItemProvider * > providers;
providers << new QgsOgrDataItemProvider;
// providers << new QgsOgrDataItemProvider;
providers << new QgsGeoPackageDataItemProvider;
return providers;
}

View File

@ -965,7 +965,7 @@ QgsMdalProvider *QgsMdalProviderMetadata::createProvider( const QString &uri, co
QList<QgsDataItemProvider *> QgsMdalProviderMetadata::dataItemProviders() const
{
QList<QgsDataItemProvider *> providers;
providers << new QgsMdalDataItemProvider;
// providers << new QgsMdalDataItemProvider;
return providers;
}