Create QgsGdalCloudProviderConnection class

This QgsGdalCloudProviderConnection subclass represents a connection
to a cloud storage provider (eg S3) via GDAL's VSI handlers

src/core/CMakeLists.txt

@@ -307,6 +307,7 @@ set(QGIS_CORE_SRCS
   providers/arcgis/qgsarcgisrestquery.cpp
   providers/arcgis/qgsarcgisrestutils.cpp
 
+  providers/gdal/qgsgdalcloudconnection.cpp
   providers/gdal/qgsgdalproviderbase.cpp
   providers/gdal/qgsgdalprovider.cpp
 
@@ -1813,6 +1814,7 @@ set(QGIS_CORE_HDRS
   providers/arcgis/qgsarcgisrestquery.h
   providers/arcgis/qgsarcgisrestutils.h
 
+  providers/gdal/qgsgdalcloudconnection.h
   providers/gdal/qgsgdalprovider.h
 
   providers/memory/qgsmemoryfeatureiterator.h

src/core/providers/gdal/qgsgdalcloudconnection.cpp

@@ -0,0 +1,182 @@
+/***************************************************************************
+    qgsgdalcloudconnection.cpp
+    ---------------------
+    begin                : June 2024
+    copyright            : (C) 2024 by Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsgdalcloudconnection.h"
+
+#include "qgsdatasourceuri.h"
+#include "qgssettingsentryimpl.h"
+#include "qgsgdalutils.h"
+#include <QRegularExpression>
+#include <QRegularExpressionMatch>
+#include <gdal.h>
+
+///@cond PRIVATE
+
+
+const QgsSettingsEntryString *QgsGdalCloudProviderConnection::settingsVsiHandler = new QgsSettingsEntryString( QStringLiteral( "handler" ), sTreeConnectionCloud );
+const QgsSettingsEntryString *QgsGdalCloudProviderConnection::settingsContainer = new QgsSettingsEntryString( QStringLiteral( "container" ), sTreeConnectionCloud );
+const QgsSettingsEntryString *QgsGdalCloudProviderConnection::settingsPath = new QgsSettingsEntryString( QStringLiteral( "path" ), sTreeConnectionCloud );
+const QgsSettingsEntryVariantMap *QgsGdalCloudProviderConnection::settingsCredentialOptions = new QgsSettingsEntryVariantMap( QStringLiteral( "credential-options" ), sTreeConnectionCloud );
+
+///@endcond
+
+QString QgsGdalCloudProviderConnection::encodedUri( const QgsGdalCloudProviderConnection::Data &data )
+{
+  QgsDataSourceUri uri;
+
+  if ( !data.vsiHandler.isEmpty() )
+    uri.setParam( QStringLiteral( "handler" ), data.vsiHandler );
+  if ( !data.container.isEmpty() )
+    uri.setParam( QStringLiteral( "container" ), data.container );
+  if ( !data.rootPath.isEmpty() )
+    uri.setParam( QStringLiteral( "rootPath" ), data.rootPath );
+
+  QStringList credentialOptions;
+  for ( auto it = data.credentialOptions.constBegin(); it != data.credentialOptions.constEnd(); ++it )
+  {
+    if ( !it.value().toString().isEmpty() )
+    {
+      credentialOptions.append( QStringLiteral( "%1=%2" ).arg( it.key(), it.value().toString() ) );
+    }
+  }
+  if ( !credentialOptions.empty() )
+    uri.setParam( "credentialOptions", credentialOptions.join( '|' ) );
+
+  return uri.encodedUri();
+}
+
+QgsGdalCloudProviderConnection::Data QgsGdalCloudProviderConnection::decodedUri( const QString &uri )
+{
+  QgsDataSourceUri dsUri;
+  dsUri.setEncodedUri( uri );
+
+  QgsGdalCloudProviderConnection::Data conn;
+  conn.vsiHandler = dsUri.param( QStringLiteral( "handler" ) );
+  conn.container = dsUri.param( QStringLiteral( "container" ) );
+  conn.rootPath = dsUri.param( QStringLiteral( "rootPath" ) );
+
+  const QStringList credentialOptions = dsUri.param( QStringLiteral( "credentialOptions" ) ).split( '|' );
+  for ( const QString &option : credentialOptions )
+  {
+    const thread_local QRegularExpression credentialOptionKeyValueRegex( QStringLiteral( "(.*?)=(.*)" ) );
+    const QRegularExpressionMatch keyValueMatch = credentialOptionKeyValueRegex.match( option );
+    if ( keyValueMatch.hasMatch() )
+    {
+      conn.credentialOptions.insert( keyValueMatch.captured( 1 ), keyValueMatch.captured( 2 ) );
+    }
+  }
+
+  return conn;
+}
+
+QStringList QgsGdalCloudProviderConnection::connectionList()
+{
+  return QgsGdalCloudProviderConnection::sTreeConnectionCloud->items();
+}
+
+QgsGdalCloudProviderConnection::Data QgsGdalCloudProviderConnection::connection( const QString &name )
+{
+  if ( !settingsContainer->exists( name ) )
+    return QgsGdalCloudProviderConnection::Data();
+
+  QgsGdalCloudProviderConnection::Data conn;
+  conn.vsiHandler = settingsVsiHandler->value( name );
+  conn.container = settingsContainer->value( name );
+  conn.rootPath = settingsPath->value( name );
+  conn.credentialOptions = settingsCredentialOptions->value( name );
+
+  return conn;
+}
+
+void QgsGdalCloudProviderConnection::addConnection( const QString &name, const Data &conn )
+{
+  settingsVsiHandler->setValue( conn.vsiHandler, name );
+  settingsContainer->setValue( conn.container, name );
+  settingsPath->setValue( conn.rootPath, name );
+  settingsCredentialOptions->setValue( conn.credentialOptions, name );
+}
+
+QString QgsGdalCloudProviderConnection::selectedConnection()
+{
+  return sTreeConnectionCloud->selectedItem();
+}
+
+void QgsGdalCloudProviderConnection::setSelectedConnection( const QString &name )
+{
+  sTreeConnectionCloud->setSelectedItem( name );
+}
+
+QgsGdalCloudProviderConnection::QgsGdalCloudProviderConnection( const QString &name )
+  : QgsAbstractProviderConnection( name )
+{
+  const QgsGdalCloudProviderConnection::Data connectionData = connection( name );
+  setUri( encodedUri( connectionData ) );
+}
+
+QgsGdalCloudProviderConnection::QgsGdalCloudProviderConnection( const QString &uri, const QVariantMap &configuration )
+  : QgsAbstractProviderConnection( uri, configuration )
+{
+}
+
+void QgsGdalCloudProviderConnection::store( const QString &name ) const
+{
+  QgsGdalCloudProviderConnection::Data connectionData = decodedUri( uri() );
+  addConnection( name, connectionData );
+}
+
+void QgsGdalCloudProviderConnection::remove( const QString &name ) const
+{
+  sTreeConnectionCloud->deleteItem( name );
+}
+
+QList<QgsGdalCloudProviderConnection::DirectoryObject> QgsGdalCloudProviderConnection::contents( const QString &path ) const
+{
+  const QgsGdalCloudProviderConnection::Data connectionDetails = decodedUri( uri() );
+
+  if ( !connectionDetails.credentialOptions.isEmpty() )
+  {
+    QgsGdalUtils::applyVsiCredentialOptions( connectionDetails.vsiHandler,
+        connectionDetails.container, connectionDetails.credentialOptions );
+  }
+
+  char **papszOptions = nullptr;
+  papszOptions = CSLAddString( papszOptions, "NAME_AND_TYPE_ONLY=YES" );
+
+  const QString vsiPath = QStringLiteral( "/%1/%2/%3" ).arg( connectionDetails.vsiHandler,
+                          connectionDetails.container,
+                          path );
+
+  VSIDIR *dir = VSIOpenDir( vsiPath.toUtf8().constData(), 0, papszOptions );
+  if ( !dir )
+  {
+    CSLDestroy( papszOptions );
+    return {};
+  }
+
+  QList< QgsGdalCloudProviderConnection::DirectoryObject > objects;
+  while ( const VSIDIREntry *entry = VSIGetNextDirEntry( dir ) )
+  {
+    QgsGdalCloudProviderConnection::DirectoryObject object;
+    object.name = QString( entry->pszName );
+    object.isFile = VSI_ISREG( entry->nMode );
+    object.isDir = VSI_ISDIR( entry->nMode );
+    objects << object;
+  }
+
+  VSICloseDir( dir );
+  CSLDestroy( papszOptions );
+
+  return objects;
+}

src/core/providers/gdal/qgsgdalcloudconnection.h

@@ -0,0 +1,159 @@
+/***************************************************************************
+    qgsgdalcloudconnection.h
+    ---------------------
+    begin                : June 2024
+    copyright            : (C) 2024 by Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSGDALCLOUDCONNECTION_H
+#define QGSGDALCLOUDCONNECTION_H
+
+#include "qgis_core.h"
+#include "qgssettingstree.h"
+#include "qgssettingstreenode.h"
+
+#define SIP_NO_FILE
+
+#include <QStringList>
+
+#include "qgsabstractproviderconnection.h"
+#include "qgshttpheaders.h"
+
+class QgsSettingsEntryString;
+class QgsSettingsEntryInteger;
+class QgsSettingsEntryVariantMap;
+
+/**
+ * \brief Represents connections to cloud providers via GDAL's VSI handlers.
+ *
+ * \ingroup core
+ * \note Not available in Python bindings.
+ *
+ * \since QGIS 3.40
+ */
+class CORE_EXPORT QgsGdalCloudProviderConnection : public QgsAbstractProviderConnection
+{
+
+  public:
+
+#ifndef SIP_RUN
+
+    ///@cond PRIVATE
+    static inline QgsSettingsTreeNamedListNode *sTreeConnectionCloud = QgsSettingsTree::sTreeConnections->createNamedListNode( QStringLiteral( "cloud" ), Qgis::SettingsTreeNodeOption::NamedListSelectedItemSetting );
+
+    static const QgsSettingsEntryString *settingsVsiHandler;
+    static const QgsSettingsEntryString *settingsContainer;
+    static const QgsSettingsEntryString *settingsPath;
+    static const QgsSettingsEntryVariantMap *settingsCredentialOptions;
+
+    ///@endcond PRIVATE
+#endif
+
+    /**
+     * Constructor for QgsGdalCloudProviderConnection, using the stored settings with the specified connection \a name.
+     */
+    QgsGdalCloudProviderConnection( const QString &name );
+
+    /**
+     * Constructor for QgsGdalCloudProviderConnection, using the a specific connection details.
+     */
+    QgsGdalCloudProviderConnection( const QString &uri, const QVariantMap &configuration );
+
+    void store( const QString &name ) const override;
+    void remove( const QString &name ) const override;
+
+    struct DirectoryObject
+    {
+      //! Object name
+      QString name;
+      //! TRUE if the object represents a file
+      bool isFile = false;
+      //! TRUE if the object represents a directory
+      bool isDir = false;
+    };
+
+    /**
+     * Returns the contents of the bucket at the specified \a path.
+     */
+    QList< DirectoryObject > contents( const QString &path ) const;
+
+    /**
+    * \brief Represents decoded data of a GDAL cloud provider connection.
+    *
+    * \ingroup core
+    * \note Not available in Python bindings.
+    *
+    * \since QGIS 3.40
+    */
+    struct Data
+    {
+      //! VSI handler
+      QString vsiHandler;
+
+      //! Container or bucket
+      QString container;
+
+      //! Path
+      QString rootPath;
+
+      //! Credential options
+      QVariantMap credentialOptions;
+
+    };
+
+    /**
+     * Returns connection \a data encoded as a string.
+     *
+     * \see encodedLayerUri()
+     * \see decodedUri()
+     */
+    static QString encodedUri( const Data &data );
+
+    /**
+     * Returns a connection \a uri decoded to a data structure.
+     *
+     * \see encodedUri()
+     * \see encodedLayerUri()
+     */
+    static Data decodedUri( const QString &uri );
+
+    /**
+     * Returns a list of the stored connection names.
+     */
+    static QStringList connectionList();
+
+    /**
+     * Returns connection details for the stored connection with the specified \a name.
+     */
+    static Data connection( const QString &name );
+
+    /**
+     * Stores a new \a connection, under the specified connection \a name.
+     */
+    static void addConnection( const QString &name, const Data &connection );
+
+    /**
+     * Returns the name of the last used connection.
+     *
+     * \see setSelectedConnection()
+     */
+    static QString selectedConnection();
+
+    /**
+     * Stores the \a name of the last used connection.
+     *
+     * \see selectedConnection()
+     */
+    static void setSelectedConnection( const QString &name );
+
+};
+
+#endif // QGSGDALCLOUDCONNECTION_H

tests/src/core/CMakeLists.txt

@@ -62,6 +62,7 @@ set(TESTS
  testqgsfilledmarker.cpp
  testqgsfontmarker.cpp
  testqgsfontutils.cpp
+ testqgsgdalcloudconnection.cpp
  testqgsgdalprovider.cpp
  testqgsgdalutils.cpp
  testqgsgenericspatialindex.cpp

tests/src/core/testqgsgdalcloudconnection.cpp

@@ -0,0 +1,131 @@
+/***************************************************************************
+     testqgsgdalcloudconnection.cpp
+     --------------------
+    Date                 : June 2024
+    Copyright            : (C) 2024 by Nyall Dawson
+    Email                : nyall dot dawson at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+#include "qgstest.h"
+#include <QObject>
+#include <QString>
+#include <QtConcurrent>
+#include "qgsgdalcloudconnection.h"
+#include "qgssettings.h"
+
+class TestQgsGdalCloudConnection : public QObject
+{
+    Q_OBJECT
+
+  private slots:
+    void initTestCase();// will be called before the first testfunction is executed.
+    void cleanupTestCase();// will be called after the last testfunction was executed.
+    void init() {} // will be called before each testfunction is executed.
+    void cleanup() {} // will be called after every testfunction.
+    void encodeDecode();
+    void testConnections();
+
+};
+
+
+void TestQgsGdalCloudConnection::initTestCase()
+{
+  // Set up the QgsSettings environment
+  QCoreApplication::setOrganizationName( QStringLiteral( "QGIS" ) );
+  QCoreApplication::setOrganizationDomain( QStringLiteral( "qgis.org" ) );
+  QCoreApplication::setApplicationName( QStringLiteral( "QGIS-TEST" ) );
+
+  QgsApplication::init();
+  QgsApplication::initQgis();
+
+  QgsSettings().clear();
+}
+
+void TestQgsGdalCloudConnection::cleanupTestCase()
+{
+  QgsApplication::exitQgis();
+}
+
+void TestQgsGdalCloudConnection::encodeDecode()
+{
+  QgsGdalCloudProviderConnection::Data data;
+  data.vsiHandler = QStringLiteral( "vsis3" );
+  data.container = QStringLiteral( "my_container" );
+  data.rootPath = QStringLiteral( "some/path" );
+  data.credentialOptions = QVariantMap{ {"pw", QStringLiteral( "xxxx" )}, {"key", QStringLiteral( "yyy" )} };
+
+  QCOMPARE( QgsGdalCloudProviderConnection::encodedUri( data ), QStringLiteral( "container=my_container&credentialOptions=key%3Dyyy%7Cpw%3Dxxxx&handler=vsis3&rootPath=some/path" ) );
+
+  const QgsGdalCloudProviderConnection::Data data2 = QgsGdalCloudProviderConnection::decodedUri( QStringLiteral( "container=my_container&credentialOptions=key%3Dyyy%7Cpw%3Dxxxx&handler=vsis3&rootPath=some/path" ) );
+  QCOMPARE( data2.vsiHandler, QStringLiteral( "vsis3" ) );
+  QCOMPARE( data2.container, QStringLiteral( "my_container" ) );
+  QCOMPARE( data2.rootPath, QStringLiteral( "some/path" ) );
+  QCOMPARE( data2.credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "xxxx" ) );
+  QCOMPARE( data2.credentialOptions.value( QStringLiteral( "key" ) ).toString(), QStringLiteral( "yyy" ) );
+}
+
+void TestQgsGdalCloudConnection::testConnections()
+{
+  QVERIFY( QgsGdalCloudProviderConnection::connectionList().isEmpty() );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "does not exist" ) ).container, QString() );
+
+  QgsGdalCloudProviderConnection conn = QgsGdalCloudProviderConnection( QStringLiteral( "my connection" ) );
+  QCOMPARE( conn.uri(), QString() );
+
+  QgsGdalCloudProviderConnection::Data data;
+  data.vsiHandler = QStringLiteral( "vsis3" );
+  data.container = QStringLiteral( "my_container" );
+  data.rootPath = QStringLiteral( "some/path" );
+  data.credentialOptions = QVariantMap{ {"pw", QStringLiteral( "xxxx" )}, {"key", QStringLiteral( "yyy" )} };
+
+  QgsGdalCloudProviderConnection::addConnection( QStringLiteral( "my connection" ), data );
+  QCOMPARE( QgsGdalCloudProviderConnection::connectionList(), {QStringLiteral( "my connection" )} );
+
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).vsiHandler, QStringLiteral( "vsis3" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).container, QStringLiteral( "my_container" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).rootPath, QStringLiteral( "some/path" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "xxxx" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "key" ) ).toString(), QStringLiteral( "yyy" ) );
+
+  // retrieve stored connection
+  conn = QgsGdalCloudProviderConnection( QStringLiteral( "my connection" ) );
+  QCOMPARE( conn.uri(), QStringLiteral( "container=my_container&credentialOptions=key%3Dyyy%7Cpw%3Dxxxx&handler=vsis3&rootPath=some/path" ) );
+
+  // add a second connection
+  QgsGdalCloudProviderConnection::Data data2;
+  data2.vsiHandler = QStringLiteral( "vsiaz" );
+  data2.container = QStringLiteral( "some_container" );
+  data2.rootPath = QStringLiteral( "path" );
+  data2.credentialOptions = QVariantMap{ {"pw", QStringLiteral( "zzz" )} };
+
+  QgsGdalCloudProviderConnection conn2( QgsGdalCloudProviderConnection::encodedUri( data2 ), {} );
+  QCOMPARE( conn2.uri(), QStringLiteral( "container=some_container&credentialOptions=pw%3Dzzz&handler=vsiaz&rootPath=path" ) );
+  conn2.store( QStringLiteral( "second connection" ) );
+
+  // retrieve stored connections
+  QCOMPARE( qgis::listToSet( QgsGdalCloudProviderConnection::connectionList() ), qgis::listToSet( QStringList() << QStringLiteral( "my connection" ) << QStringLiteral( "second connection" ) ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).vsiHandler, QStringLiteral( "vsis3" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).container, QStringLiteral( "my_container" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).rootPath, QStringLiteral( "some/path" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "xxxx" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "key" ) ).toString(), QStringLiteral( "yyy" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).vsiHandler, QStringLiteral( "vsiaz" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).container, QStringLiteral( "some_container" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).rootPath, QStringLiteral( "path" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "zzz" ) );
+
+  QgsGdalCloudProviderConnection::setSelectedConnection( QStringLiteral( "second connection" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::selectedConnection(), QStringLiteral( "second connection" ) );
+  QgsGdalCloudProviderConnection::setSelectedConnection( QStringLiteral( "my connection" ) );
+  QCOMPARE( QgsGdalCloudProviderConnection::selectedConnection(), QStringLiteral( "my connection" ) );
+}
+
+
+QGSTEST_MAIN( TestQgsGdalCloudConnection )
+#include "testqgsgdalcloudconnection.moc"