From b733307109ce7335e3c1ec89c5e35158ebaee387 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Tue, 25 Jun 2024 12:53:07 +1000 Subject: [PATCH] Create QgsGdalCloudProviderConnection class This QgsGdalCloudProviderConnection subclass represents a connection to a cloud storage provider (eg S3) via GDAL's VSI handlers --- src/core/CMakeLists.txt | 2 + .../providers/gdal/qgsgdalcloudconnection.cpp | 182 ++++++++++++++++++ .../providers/gdal/qgsgdalcloudconnection.h | 159 +++++++++++++++ tests/src/core/CMakeLists.txt | 1 + tests/src/core/testqgsgdalcloudconnection.cpp | 131 +++++++++++++ 5 files changed, 475 insertions(+) create mode 100644 src/core/providers/gdal/qgsgdalcloudconnection.cpp create mode 100644 src/core/providers/gdal/qgsgdalcloudconnection.h create mode 100644 tests/src/core/testqgsgdalcloudconnection.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index afc504cb6d1..1c0c1ecefbb 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -307,6 +307,7 @@ set(QGIS_CORE_SRCS providers/arcgis/qgsarcgisrestquery.cpp providers/arcgis/qgsarcgisrestutils.cpp + providers/gdal/qgsgdalcloudconnection.cpp providers/gdal/qgsgdalproviderbase.cpp providers/gdal/qgsgdalprovider.cpp @@ -1813,6 +1814,7 @@ set(QGIS_CORE_HDRS providers/arcgis/qgsarcgisrestquery.h providers/arcgis/qgsarcgisrestutils.h + providers/gdal/qgsgdalcloudconnection.h providers/gdal/qgsgdalprovider.h providers/memory/qgsmemoryfeatureiterator.h diff --git a/src/core/providers/gdal/qgsgdalcloudconnection.cpp b/src/core/providers/gdal/qgsgdalcloudconnection.cpp new file mode 100644 index 00000000000..48153575161 --- /dev/null +++ b/src/core/providers/gdal/qgsgdalcloudconnection.cpp @@ -0,0 +1,182 @@ +/*************************************************************************** + qgsgdalcloudconnection.cpp + --------------------- + begin : June 2024 + copyright : (C) 2024 by Nyall Dawson + email : nyall dot dawson at gmail dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#include "qgsgdalcloudconnection.h" + +#include "qgsdatasourceuri.h" +#include "qgssettingsentryimpl.h" +#include "qgsgdalutils.h" +#include +#include +#include + +///@cond PRIVATE + + +const QgsSettingsEntryString *QgsGdalCloudProviderConnection::settingsVsiHandler = new QgsSettingsEntryString( QStringLiteral( "handler" ), sTreeConnectionCloud ); +const QgsSettingsEntryString *QgsGdalCloudProviderConnection::settingsContainer = new QgsSettingsEntryString( QStringLiteral( "container" ), sTreeConnectionCloud ); +const QgsSettingsEntryString *QgsGdalCloudProviderConnection::settingsPath = new QgsSettingsEntryString( QStringLiteral( "path" ), sTreeConnectionCloud ); +const QgsSettingsEntryVariantMap *QgsGdalCloudProviderConnection::settingsCredentialOptions = new QgsSettingsEntryVariantMap( QStringLiteral( "credential-options" ), sTreeConnectionCloud ); + +///@endcond + +QString QgsGdalCloudProviderConnection::encodedUri( const QgsGdalCloudProviderConnection::Data &data ) +{ + QgsDataSourceUri uri; + + if ( !data.vsiHandler.isEmpty() ) + uri.setParam( QStringLiteral( "handler" ), data.vsiHandler ); + if ( !data.container.isEmpty() ) + uri.setParam( QStringLiteral( "container" ), data.container ); + if ( !data.rootPath.isEmpty() ) + uri.setParam( QStringLiteral( "rootPath" ), data.rootPath ); + + QStringList credentialOptions; + for ( auto it = data.credentialOptions.constBegin(); it != data.credentialOptions.constEnd(); ++it ) + { + if ( !it.value().toString().isEmpty() ) + { + credentialOptions.append( QStringLiteral( "%1=%2" ).arg( it.key(), it.value().toString() ) ); + } + } + if ( !credentialOptions.empty() ) + uri.setParam( "credentialOptions", credentialOptions.join( '|' ) ); + + return uri.encodedUri(); +} + +QgsGdalCloudProviderConnection::Data QgsGdalCloudProviderConnection::decodedUri( const QString &uri ) +{ + QgsDataSourceUri dsUri; + dsUri.setEncodedUri( uri ); + + QgsGdalCloudProviderConnection::Data conn; + conn.vsiHandler = dsUri.param( QStringLiteral( "handler" ) ); + conn.container = dsUri.param( QStringLiteral( "container" ) ); + conn.rootPath = dsUri.param( QStringLiteral( "rootPath" ) ); + + const QStringList credentialOptions = dsUri.param( QStringLiteral( "credentialOptions" ) ).split( '|' ); + for ( const QString &option : credentialOptions ) + { + const thread_local QRegularExpression credentialOptionKeyValueRegex( QStringLiteral( "(.*?)=(.*)" ) ); + const QRegularExpressionMatch keyValueMatch = credentialOptionKeyValueRegex.match( option ); + if ( keyValueMatch.hasMatch() ) + { + conn.credentialOptions.insert( keyValueMatch.captured( 1 ), keyValueMatch.captured( 2 ) ); + } + } + + return conn; +} + +QStringList QgsGdalCloudProviderConnection::connectionList() +{ + return QgsGdalCloudProviderConnection::sTreeConnectionCloud->items(); +} + +QgsGdalCloudProviderConnection::Data QgsGdalCloudProviderConnection::connection( const QString &name ) +{ + if ( !settingsContainer->exists( name ) ) + return QgsGdalCloudProviderConnection::Data(); + + QgsGdalCloudProviderConnection::Data conn; + conn.vsiHandler = settingsVsiHandler->value( name ); + conn.container = settingsContainer->value( name ); + conn.rootPath = settingsPath->value( name ); + conn.credentialOptions = settingsCredentialOptions->value( name ); + + return conn; +} + +void QgsGdalCloudProviderConnection::addConnection( const QString &name, const Data &conn ) +{ + settingsVsiHandler->setValue( conn.vsiHandler, name ); + settingsContainer->setValue( conn.container, name ); + settingsPath->setValue( conn.rootPath, name ); + settingsCredentialOptions->setValue( conn.credentialOptions, name ); +} + +QString QgsGdalCloudProviderConnection::selectedConnection() +{ + return sTreeConnectionCloud->selectedItem(); +} + +void QgsGdalCloudProviderConnection::setSelectedConnection( const QString &name ) +{ + sTreeConnectionCloud->setSelectedItem( name ); +} + +QgsGdalCloudProviderConnection::QgsGdalCloudProviderConnection( const QString &name ) + : QgsAbstractProviderConnection( name ) +{ + const QgsGdalCloudProviderConnection::Data connectionData = connection( name ); + setUri( encodedUri( connectionData ) ); +} + +QgsGdalCloudProviderConnection::QgsGdalCloudProviderConnection( const QString &uri, const QVariantMap &configuration ) + : QgsAbstractProviderConnection( uri, configuration ) +{ +} + +void QgsGdalCloudProviderConnection::store( const QString &name ) const +{ + QgsGdalCloudProviderConnection::Data connectionData = decodedUri( uri() ); + addConnection( name, connectionData ); +} + +void QgsGdalCloudProviderConnection::remove( const QString &name ) const +{ + sTreeConnectionCloud->deleteItem( name ); +} + +QList QgsGdalCloudProviderConnection::contents( const QString &path ) const +{ + const QgsGdalCloudProviderConnection::Data connectionDetails = decodedUri( uri() ); + + if ( !connectionDetails.credentialOptions.isEmpty() ) + { + QgsGdalUtils::applyVsiCredentialOptions( connectionDetails.vsiHandler, + connectionDetails.container, connectionDetails.credentialOptions ); + } + + char **papszOptions = nullptr; + papszOptions = CSLAddString( papszOptions, "NAME_AND_TYPE_ONLY=YES" ); + + const QString vsiPath = QStringLiteral( "/%1/%2/%3" ).arg( connectionDetails.vsiHandler, + connectionDetails.container, + path ); + + VSIDIR *dir = VSIOpenDir( vsiPath.toUtf8().constData(), 0, papszOptions ); + if ( !dir ) + { + CSLDestroy( papszOptions ); + return {}; + } + + QList< QgsGdalCloudProviderConnection::DirectoryObject > objects; + while ( const VSIDIREntry *entry = VSIGetNextDirEntry( dir ) ) + { + QgsGdalCloudProviderConnection::DirectoryObject object; + object.name = QString( entry->pszName ); + object.isFile = VSI_ISREG( entry->nMode ); + object.isDir = VSI_ISDIR( entry->nMode ); + objects << object; + } + + VSICloseDir( dir ); + CSLDestroy( papszOptions ); + + return objects; +} diff --git a/src/core/providers/gdal/qgsgdalcloudconnection.h b/src/core/providers/gdal/qgsgdalcloudconnection.h new file mode 100644 index 00000000000..a8bc50c3c8b --- /dev/null +++ b/src/core/providers/gdal/qgsgdalcloudconnection.h @@ -0,0 +1,159 @@ +/*************************************************************************** + qgsgdalcloudconnection.h + --------------------- + begin : June 2024 + copyright : (C) 2024 by Nyall Dawson + email : nyall dot dawson at gmail dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#ifndef QGSGDALCLOUDCONNECTION_H +#define QGSGDALCLOUDCONNECTION_H + +#include "qgis_core.h" +#include "qgssettingstree.h" +#include "qgssettingstreenode.h" + +#define SIP_NO_FILE + +#include + +#include "qgsabstractproviderconnection.h" +#include "qgshttpheaders.h" + +class QgsSettingsEntryString; +class QgsSettingsEntryInteger; +class QgsSettingsEntryVariantMap; + +/** + * \brief Represents connections to cloud providers via GDAL's VSI handlers. + * + * \ingroup core + * \note Not available in Python bindings. + * + * \since QGIS 3.40 + */ +class CORE_EXPORT QgsGdalCloudProviderConnection : public QgsAbstractProviderConnection +{ + + public: + +#ifndef SIP_RUN + + ///@cond PRIVATE + static inline QgsSettingsTreeNamedListNode *sTreeConnectionCloud = QgsSettingsTree::sTreeConnections->createNamedListNode( QStringLiteral( "cloud" ), Qgis::SettingsTreeNodeOption::NamedListSelectedItemSetting ); + + static const QgsSettingsEntryString *settingsVsiHandler; + static const QgsSettingsEntryString *settingsContainer; + static const QgsSettingsEntryString *settingsPath; + static const QgsSettingsEntryVariantMap *settingsCredentialOptions; + + ///@endcond PRIVATE +#endif + + /** + * Constructor for QgsGdalCloudProviderConnection, using the stored settings with the specified connection \a name. + */ + QgsGdalCloudProviderConnection( const QString &name ); + + /** + * Constructor for QgsGdalCloudProviderConnection, using the a specific connection details. + */ + QgsGdalCloudProviderConnection( const QString &uri, const QVariantMap &configuration ); + + void store( const QString &name ) const override; + void remove( const QString &name ) const override; + + struct DirectoryObject + { + //! Object name + QString name; + //! TRUE if the object represents a file + bool isFile = false; + //! TRUE if the object represents a directory + bool isDir = false; + }; + + /** + * Returns the contents of the bucket at the specified \a path. + */ + QList< DirectoryObject > contents( const QString &path ) const; + + /** + * \brief Represents decoded data of a GDAL cloud provider connection. + * + * \ingroup core + * \note Not available in Python bindings. + * + * \since QGIS 3.40 + */ + struct Data + { + //! VSI handler + QString vsiHandler; + + //! Container or bucket + QString container; + + //! Path + QString rootPath; + + //! Credential options + QVariantMap credentialOptions; + + }; + + /** + * Returns connection \a data encoded as a string. + * + * \see encodedLayerUri() + * \see decodedUri() + */ + static QString encodedUri( const Data &data ); + + /** + * Returns a connection \a uri decoded to a data structure. + * + * \see encodedUri() + * \see encodedLayerUri() + */ + static Data decodedUri( const QString &uri ); + + /** + * Returns a list of the stored connection names. + */ + static QStringList connectionList(); + + /** + * Returns connection details for the stored connection with the specified \a name. + */ + static Data connection( const QString &name ); + + /** + * Stores a new \a connection, under the specified connection \a name. + */ + static void addConnection( const QString &name, const Data &connection ); + + /** + * Returns the name of the last used connection. + * + * \see setSelectedConnection() + */ + static QString selectedConnection(); + + /** + * Stores the \a name of the last used connection. + * + * \see selectedConnection() + */ + static void setSelectedConnection( const QString &name ); + +}; + +#endif // QGSGDALCLOUDCONNECTION_H diff --git a/tests/src/core/CMakeLists.txt b/tests/src/core/CMakeLists.txt index 686f5729305..a44752d9415 100644 --- a/tests/src/core/CMakeLists.txt +++ b/tests/src/core/CMakeLists.txt @@ -62,6 +62,7 @@ set(TESTS testqgsfilledmarker.cpp testqgsfontmarker.cpp testqgsfontutils.cpp + testqgsgdalcloudconnection.cpp testqgsgdalprovider.cpp testqgsgdalutils.cpp testqgsgenericspatialindex.cpp diff --git a/tests/src/core/testqgsgdalcloudconnection.cpp b/tests/src/core/testqgsgdalcloudconnection.cpp new file mode 100644 index 00000000000..3247f433a5e --- /dev/null +++ b/tests/src/core/testqgsgdalcloudconnection.cpp @@ -0,0 +1,131 @@ +/*************************************************************************** + testqgsgdalcloudconnection.cpp + -------------------- + Date : June 2024 + Copyright : (C) 2024 by Nyall Dawson + Email : nyall dot dawson at gmail dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +#include "qgstest.h" +#include +#include +#include +#include "qgsgdalcloudconnection.h" +#include "qgssettings.h" + +class TestQgsGdalCloudConnection : public QObject +{ + Q_OBJECT + + private slots: + void initTestCase();// will be called before the first testfunction is executed. + void cleanupTestCase();// will be called after the last testfunction was executed. + void init() {} // will be called before each testfunction is executed. + void cleanup() {} // will be called after every testfunction. + void encodeDecode(); + void testConnections(); + +}; + + +void TestQgsGdalCloudConnection::initTestCase() +{ + // Set up the QgsSettings environment + QCoreApplication::setOrganizationName( QStringLiteral( "QGIS" ) ); + QCoreApplication::setOrganizationDomain( QStringLiteral( "qgis.org" ) ); + QCoreApplication::setApplicationName( QStringLiteral( "QGIS-TEST" ) ); + + QgsApplication::init(); + QgsApplication::initQgis(); + + QgsSettings().clear(); +} + +void TestQgsGdalCloudConnection::cleanupTestCase() +{ + QgsApplication::exitQgis(); +} + +void TestQgsGdalCloudConnection::encodeDecode() +{ + QgsGdalCloudProviderConnection::Data data; + data.vsiHandler = QStringLiteral( "vsis3" ); + data.container = QStringLiteral( "my_container" ); + data.rootPath = QStringLiteral( "some/path" ); + data.credentialOptions = QVariantMap{ {"pw", QStringLiteral( "xxxx" )}, {"key", QStringLiteral( "yyy" )} }; + + QCOMPARE( QgsGdalCloudProviderConnection::encodedUri( data ), QStringLiteral( "container=my_container&credentialOptions=key%3Dyyy%7Cpw%3Dxxxx&handler=vsis3&rootPath=some/path" ) ); + + const QgsGdalCloudProviderConnection::Data data2 = QgsGdalCloudProviderConnection::decodedUri( QStringLiteral( "container=my_container&credentialOptions=key%3Dyyy%7Cpw%3Dxxxx&handler=vsis3&rootPath=some/path" ) ); + QCOMPARE( data2.vsiHandler, QStringLiteral( "vsis3" ) ); + QCOMPARE( data2.container, QStringLiteral( "my_container" ) ); + QCOMPARE( data2.rootPath, QStringLiteral( "some/path" ) ); + QCOMPARE( data2.credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "xxxx" ) ); + QCOMPARE( data2.credentialOptions.value( QStringLiteral( "key" ) ).toString(), QStringLiteral( "yyy" ) ); +} + +void TestQgsGdalCloudConnection::testConnections() +{ + QVERIFY( QgsGdalCloudProviderConnection::connectionList().isEmpty() ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "does not exist" ) ).container, QString() ); + + QgsGdalCloudProviderConnection conn = QgsGdalCloudProviderConnection( QStringLiteral( "my connection" ) ); + QCOMPARE( conn.uri(), QString() ); + + QgsGdalCloudProviderConnection::Data data; + data.vsiHandler = QStringLiteral( "vsis3" ); + data.container = QStringLiteral( "my_container" ); + data.rootPath = QStringLiteral( "some/path" ); + data.credentialOptions = QVariantMap{ {"pw", QStringLiteral( "xxxx" )}, {"key", QStringLiteral( "yyy" )} }; + + QgsGdalCloudProviderConnection::addConnection( QStringLiteral( "my connection" ), data ); + QCOMPARE( QgsGdalCloudProviderConnection::connectionList(), {QStringLiteral( "my connection" )} ); + + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).vsiHandler, QStringLiteral( "vsis3" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).container, QStringLiteral( "my_container" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).rootPath, QStringLiteral( "some/path" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "xxxx" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "key" ) ).toString(), QStringLiteral( "yyy" ) ); + + // retrieve stored connection + conn = QgsGdalCloudProviderConnection( QStringLiteral( "my connection" ) ); + QCOMPARE( conn.uri(), QStringLiteral( "container=my_container&credentialOptions=key%3Dyyy%7Cpw%3Dxxxx&handler=vsis3&rootPath=some/path" ) ); + + // add a second connection + QgsGdalCloudProviderConnection::Data data2; + data2.vsiHandler = QStringLiteral( "vsiaz" ); + data2.container = QStringLiteral( "some_container" ); + data2.rootPath = QStringLiteral( "path" ); + data2.credentialOptions = QVariantMap{ {"pw", QStringLiteral( "zzz" )} }; + + QgsGdalCloudProviderConnection conn2( QgsGdalCloudProviderConnection::encodedUri( data2 ), {} ); + QCOMPARE( conn2.uri(), QStringLiteral( "container=some_container&credentialOptions=pw%3Dzzz&handler=vsiaz&rootPath=path" ) ); + conn2.store( QStringLiteral( "second connection" ) ); + + // retrieve stored connections + QCOMPARE( qgis::listToSet( QgsGdalCloudProviderConnection::connectionList() ), qgis::listToSet( QStringList() << QStringLiteral( "my connection" ) << QStringLiteral( "second connection" ) ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).vsiHandler, QStringLiteral( "vsis3" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).container, QStringLiteral( "my_container" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).rootPath, QStringLiteral( "some/path" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "xxxx" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "key" ) ).toString(), QStringLiteral( "yyy" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).vsiHandler, QStringLiteral( "vsiaz" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).container, QStringLiteral( "some_container" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).rootPath, QStringLiteral( "path" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "zzz" ) ); + + QgsGdalCloudProviderConnection::setSelectedConnection( QStringLiteral( "second connection" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::selectedConnection(), QStringLiteral( "second connection" ) ); + QgsGdalCloudProviderConnection::setSelectedConnection( QStringLiteral( "my connection" ) ); + QCOMPARE( QgsGdalCloudProviderConnection::selectedConnection(), QStringLiteral( "my connection" ) ); +} + + +QGSTEST_MAIN( TestQgsGdalCloudConnection ) +#include "testqgsgdalcloudconnection.moc"