sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-10-05 00:09:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

Create QgsGdalCloudProviderConnection class

Browse Source 
This QgsGdalCloudProviderConnection subclass represents a connection
to a cloud storage provider (eg S3) via GDAL's VSI handlers
This commit is contained in:
Nyall Dawson 2024-06-25 12:53:07 +10:00
parent ca1bfbf16a
commit b733307109
5 changed files with 475 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/core/CMakeLists.txt
View File

@ -307,6 +307,7 @@ set(QGIS_CORE_SRCS
providers/arcgis/qgsarcgisrestquery.cpp
providers/arcgis/qgsarcgisrestutils.cpp
providers/gdal/qgsgdalcloudconnection.cpp
providers/gdal/qgsgdalproviderbase.cpp
providers/gdal/qgsgdalprovider.cpp
@ -1813,6 +1814,7 @@ set(QGIS_CORE_HDRS
providers/arcgis/qgsarcgisrestquery.h
providers/arcgis/qgsarcgisrestutils.h
providers/gdal/qgsgdalcloudconnection.h
providers/gdal/qgsgdalprovider.h
providers/memory/qgsmemoryfeatureiterator.h

182
src/core/providers/gdal/qgsgdalcloudconnection.cpp Normal file
View File

@ -0,0 +1,182 @@
/***************************************************************************
qgsgdalcloudconnection.cpp
---------------------
begin : June 2024
copyright : (C) 2024 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include "qgsgdalcloudconnection.h"
#include "qgsdatasourceuri.h"
#include "qgssettingsentryimpl.h"
#include "qgsgdalutils.h"
#include <QRegularExpression>
#include <QRegularExpressionMatch>
#include <gdal.h>
///@cond PRIVATE
const QgsSettingsEntryString *QgsGdalCloudProviderConnection::settingsVsiHandler = new QgsSettingsEntryString( QStringLiteral( "handler" ), sTreeConnectionCloud );
const QgsSettingsEntryString *QgsGdalCloudProviderConnection::settingsContainer = new QgsSettingsEntryString( QStringLiteral( "container" ), sTreeConnectionCloud );
const QgsSettingsEntryString *QgsGdalCloudProviderConnection::settingsPath = new QgsSettingsEntryString( QStringLiteral( "path" ), sTreeConnectionCloud );
const QgsSettingsEntryVariantMap *QgsGdalCloudProviderConnection::settingsCredentialOptions = new QgsSettingsEntryVariantMap( QStringLiteral( "credential-options" ), sTreeConnectionCloud );
///@endcond
QString QgsGdalCloudProviderConnection::encodedUri( const QgsGdalCloudProviderConnection::Data &data )
{
QgsDataSourceUri uri;
if ( !data.vsiHandler.isEmpty() )
uri.setParam( QStringLiteral( "handler" ), data.vsiHandler );
if ( !data.container.isEmpty() )
uri.setParam( QStringLiteral( "container" ), data.container );
if ( !data.rootPath.isEmpty() )
uri.setParam( QStringLiteral( "rootPath" ), data.rootPath );
QStringList credentialOptions;
for ( auto it = data.credentialOptions.constBegin(); it != data.credentialOptions.constEnd(); ++it )
{
if ( !it.value().toString().isEmpty() )
{
credentialOptions.append( QStringLiteral( "%1=%2" ).arg( it.key(), it.value().toString() ) );
}
}
if ( !credentialOptions.empty() )
uri.setParam( "credentialOptions", credentialOptions.join( '|' ) );
return uri.encodedUri();
}
QgsGdalCloudProviderConnection::Data QgsGdalCloudProviderConnection::decodedUri( const QString &uri )
{
QgsDataSourceUri dsUri;
dsUri.setEncodedUri( uri );
QgsGdalCloudProviderConnection::Data conn;
conn.vsiHandler = dsUri.param( QStringLiteral( "handler" ) );
conn.container = dsUri.param( QStringLiteral( "container" ) );
conn.rootPath = dsUri.param( QStringLiteral( "rootPath" ) );
const QStringList credentialOptions = dsUri.param( QStringLiteral( "credentialOptions" ) ).split( '|' );
for ( const QString &option : credentialOptions )
{
const thread_local QRegularExpression credentialOptionKeyValueRegex( QStringLiteral( "(.*?)=(.*)" ) );
const QRegularExpressionMatch keyValueMatch = credentialOptionKeyValueRegex.match( option );
if ( keyValueMatch.hasMatch() )
{
conn.credentialOptions.insert( keyValueMatch.captured( 1 ), keyValueMatch.captured( 2 ) );
}
}
return conn;
}
QStringList QgsGdalCloudProviderConnection::connectionList()
{
return QgsGdalCloudProviderConnection::sTreeConnectionCloud->items();
}
QgsGdalCloudProviderConnection::Data QgsGdalCloudProviderConnection::connection( const QString &name )
{
if ( !settingsContainer->exists( name ) )
return QgsGdalCloudProviderConnection::Data();
QgsGdalCloudProviderConnection::Data conn;
conn.vsiHandler = settingsVsiHandler->value( name );
conn.container = settingsContainer->value( name );
conn.rootPath = settingsPath->value( name );
conn.credentialOptions = settingsCredentialOptions->value( name );
return conn;
}
void QgsGdalCloudProviderConnection::addConnection( const QString &name, const Data &conn )
{
settingsVsiHandler->setValue( conn.vsiHandler, name );
settingsContainer->setValue( conn.container, name );
settingsPath->setValue( conn.rootPath, name );
settingsCredentialOptions->setValue( conn.credentialOptions, name );
}
QString QgsGdalCloudProviderConnection::selectedConnection()
{
return sTreeConnectionCloud->selectedItem();
}
void QgsGdalCloudProviderConnection::setSelectedConnection( const QString &name )
{
sTreeConnectionCloud->setSelectedItem( name );
}
QgsGdalCloudProviderConnection::QgsGdalCloudProviderConnection( const QString &name )
: QgsAbstractProviderConnection( name )
{
const QgsGdalCloudProviderConnection::Data connectionData = connection( name );
setUri( encodedUri( connectionData ) );
}
QgsGdalCloudProviderConnection::QgsGdalCloudProviderConnection( const QString &uri, const QVariantMap &configuration )
: QgsAbstractProviderConnection( uri, configuration )
{
}
void QgsGdalCloudProviderConnection::store( const QString &name ) const
{
QgsGdalCloudProviderConnection::Data connectionData = decodedUri( uri() );
addConnection( name, connectionData );
}
void QgsGdalCloudProviderConnection::remove( const QString &name ) const
{
sTreeConnectionCloud->deleteItem( name );
}
QList<QgsGdalCloudProviderConnection::DirectoryObject> QgsGdalCloudProviderConnection::contents( const QString &path ) const
{
const QgsGdalCloudProviderConnection::Data connectionDetails = decodedUri( uri() );
if ( !connectionDetails.credentialOptions.isEmpty() )
{
QgsGdalUtils::applyVsiCredentialOptions( connectionDetails.vsiHandler,
connectionDetails.container, connectionDetails.credentialOptions );
}
char **papszOptions = nullptr;
papszOptions = CSLAddString( papszOptions, "NAME_AND_TYPE_ONLY=YES" );
const QString vsiPath = QStringLiteral( "/%1/%2/%3" ).arg( connectionDetails.vsiHandler,
connectionDetails.container,
path );
VSIDIR *dir = VSIOpenDir( vsiPath.toUtf8().constData(), 0, papszOptions );
if ( !dir )
{
CSLDestroy( papszOptions );
return {};
}
QList< QgsGdalCloudProviderConnection::DirectoryObject > objects;
while ( const VSIDIREntry *entry = VSIGetNextDirEntry( dir ) )
{
QgsGdalCloudProviderConnection::DirectoryObject object;
object.name = QString( entry->pszName );
object.isFile = VSI_ISREG( entry->nMode );
object.isDir = VSI_ISDIR( entry->nMode );
objects << object;
}
VSICloseDir( dir );
CSLDestroy( papszOptions );
return objects;
}

159
src/core/providers/gdal/qgsgdalcloudconnection.h Normal file
View File

@ -0,0 +1,159 @@
/***************************************************************************
qgsgdalcloudconnection.h
---------------------
begin : June 2024
copyright : (C) 2024 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSGDALCLOUDCONNECTION_H
#define QGSGDALCLOUDCONNECTION_H
#include "qgis_core.h"
#include "qgssettingstree.h"
#include "qgssettingstreenode.h"
#define SIP_NO_FILE
#include <QStringList>
#include "qgsabstractproviderconnection.h"
#include "qgshttpheaders.h"
class QgsSettingsEntryString;
class QgsSettingsEntryInteger;
class QgsSettingsEntryVariantMap;
/**
* \brief Represents connections to cloud providers via GDAL's VSI handlers.
*
* \ingroup core
* \note Not available in Python bindings.
*
* \since QGIS 3.40
*/
class CORE_EXPORT QgsGdalCloudProviderConnection : public QgsAbstractProviderConnection
{
public:
#ifndef SIP_RUN
///@cond PRIVATE
static inline QgsSettingsTreeNamedListNode *sTreeConnectionCloud = QgsSettingsTree::sTreeConnections->createNamedListNode( QStringLiteral( "cloud" ), Qgis::SettingsTreeNodeOption::NamedListSelectedItemSetting );
static const QgsSettingsEntryString *settingsVsiHandler;
static const QgsSettingsEntryString *settingsContainer;
static const QgsSettingsEntryString *settingsPath;
static const QgsSettingsEntryVariantMap *settingsCredentialOptions;
///@endcond PRIVATE
#endif
/**
* Constructor for QgsGdalCloudProviderConnection, using the stored settings with the specified connection \a name.
*/
QgsGdalCloudProviderConnection( const QString &name );
/**
* Constructor for QgsGdalCloudProviderConnection, using the a specific connection details.
*/
QgsGdalCloudProviderConnection( const QString &uri, const QVariantMap &configuration );
void store( const QString &name ) const override;
void remove( const QString &name ) const override;
struct DirectoryObject
{
//! Object name
QString name;
//! TRUE if the object represents a file
bool isFile = false;
//! TRUE if the object represents a directory
bool isDir = false;
};
/**
* Returns the contents of the bucket at the specified \a path.
*/
QList< DirectoryObject > contents( const QString &path ) const;
/**
* \brief Represents decoded data of a GDAL cloud provider connection.
*
* \ingroup core
* \note Not available in Python bindings.
*
* \since QGIS 3.40
*/
struct Data
{
//! VSI handler
QString vsiHandler;
//! Container or bucket
QString container;
//! Path
QString rootPath;
//! Credential options
QVariantMap credentialOptions;
};
/**
* Returns connection \a data encoded as a string.
*
* \see encodedLayerUri()
* \see decodedUri()
*/
static QString encodedUri( const Data &data );
/**
* Returns a connection \a uri decoded to a data structure.
*
* \see encodedUri()
* \see encodedLayerUri()
*/
static Data decodedUri( const QString &uri );
/**
* Returns a list of the stored connection names.
*/
static QStringList connectionList();
/**
* Returns connection details for the stored connection with the specified \a name.
*/
static Data connection( const QString &name );
/**
* Stores a new \a connection, under the specified connection \a name.
*/
static void addConnection( const QString &name, const Data &connection );
/**
* Returns the name of the last used connection.
*
* \see setSelectedConnection()
*/
static QString selectedConnection();
/**
* Stores the \a name of the last used connection.
*
* \see selectedConnection()
*/
static void setSelectedConnection( const QString &name );
};
#endif // QGSGDALCLOUDCONNECTION_H

1
tests/src/core/CMakeLists.txt
View File

@ -62,6 +62,7 @@ set(TESTS
testqgsfilledmarker.cpp
testqgsfontmarker.cpp
testqgsfontutils.cpp
testqgsgdalcloudconnection.cpp
testqgsgdalprovider.cpp
testqgsgdalutils.cpp
testqgsgenericspatialindex.cpp

131
tests/src/core/testqgsgdalcloudconnection.cpp Normal file
View File

@ -0,0 +1,131 @@
/***************************************************************************
testqgsgdalcloudconnection.cpp
--------------------
Date : June 2024
Copyright : (C) 2024 by Nyall Dawson
Email : nyall dot dawson at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include "qgstest.h"
#include <QObject>
#include <QString>
#include <QtConcurrent>
#include "qgsgdalcloudconnection.h"
#include "qgssettings.h"
class TestQgsGdalCloudConnection : public QObject
{
Q_OBJECT
private slots:
void initTestCase();// will be called before the first testfunction is executed.
void cleanupTestCase();// will be called after the last testfunction was executed.
void init() {} // will be called before each testfunction is executed.
void cleanup() {} // will be called after every testfunction.
void encodeDecode();
void testConnections();
};
void TestQgsGdalCloudConnection::initTestCase()
{
// Set up the QgsSettings environment
QCoreApplication::setOrganizationName( QStringLiteral( "QGIS" ) );
QCoreApplication::setOrganizationDomain( QStringLiteral( "qgis.org" ) );
QCoreApplication::setApplicationName( QStringLiteral( "QGIS-TEST" ) );
QgsApplication::init();
QgsApplication::initQgis();
QgsSettings().clear();
}
void TestQgsGdalCloudConnection::cleanupTestCase()
{
QgsApplication::exitQgis();
}
void TestQgsGdalCloudConnection::encodeDecode()
{
QgsGdalCloudProviderConnection::Data data;
data.vsiHandler = QStringLiteral( "vsis3" );
data.container = QStringLiteral( "my_container" );
data.rootPath = QStringLiteral( "some/path" );
data.credentialOptions = QVariantMap{ {"pw", QStringLiteral( "xxxx" )}, {"key", QStringLiteral( "yyy" )} };
QCOMPARE( QgsGdalCloudProviderConnection::encodedUri( data ), QStringLiteral( "container=my_container&credentialOptions=key%3Dyyy%7Cpw%3Dxxxx&handler=vsis3&rootPath=some/path" ) );
const QgsGdalCloudProviderConnection::Data data2 = QgsGdalCloudProviderConnection::decodedUri( QStringLiteral( "container=my_container&credentialOptions=key%3Dyyy%7Cpw%3Dxxxx&handler=vsis3&rootPath=some/path" ) );
QCOMPARE( data2.vsiHandler, QStringLiteral( "vsis3" ) );
QCOMPARE( data2.container, QStringLiteral( "my_container" ) );
QCOMPARE( data2.rootPath, QStringLiteral( "some/path" ) );
QCOMPARE( data2.credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "xxxx" ) );
QCOMPARE( data2.credentialOptions.value( QStringLiteral( "key" ) ).toString(), QStringLiteral( "yyy" ) );
}
void TestQgsGdalCloudConnection::testConnections()
{
QVERIFY( QgsGdalCloudProviderConnection::connectionList().isEmpty() );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "does not exist" ) ).container, QString() );
QgsGdalCloudProviderConnection conn = QgsGdalCloudProviderConnection( QStringLiteral( "my connection" ) );
QCOMPARE( conn.uri(), QString() );
QgsGdalCloudProviderConnection::Data data;
data.vsiHandler = QStringLiteral( "vsis3" );
data.container = QStringLiteral( "my_container" );
data.rootPath = QStringLiteral( "some/path" );
data.credentialOptions = QVariantMap{ {"pw", QStringLiteral( "xxxx" )}, {"key", QStringLiteral( "yyy" )} };
QgsGdalCloudProviderConnection::addConnection( QStringLiteral( "my connection" ), data );
QCOMPARE( QgsGdalCloudProviderConnection::connectionList(), {QStringLiteral( "my connection" )} );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).vsiHandler, QStringLiteral( "vsis3" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).container, QStringLiteral( "my_container" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).rootPath, QStringLiteral( "some/path" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "xxxx" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "key" ) ).toString(), QStringLiteral( "yyy" ) );
// retrieve stored connection
conn = QgsGdalCloudProviderConnection( QStringLiteral( "my connection" ) );
QCOMPARE( conn.uri(), QStringLiteral( "container=my_container&credentialOptions=key%3Dyyy%7Cpw%3Dxxxx&handler=vsis3&rootPath=some/path" ) );
// add a second connection
QgsGdalCloudProviderConnection::Data data2;
data2.vsiHandler = QStringLiteral( "vsiaz" );
data2.container = QStringLiteral( "some_container" );
data2.rootPath = QStringLiteral( "path" );
data2.credentialOptions = QVariantMap{ {"pw", QStringLiteral( "zzz" )} };
QgsGdalCloudProviderConnection conn2( QgsGdalCloudProviderConnection::encodedUri( data2 ), {} );
QCOMPARE( conn2.uri(), QStringLiteral( "container=some_container&credentialOptions=pw%3Dzzz&handler=vsiaz&rootPath=path" ) );
conn2.store( QStringLiteral( "second connection" ) );
// retrieve stored connections
QCOMPARE( qgis::listToSet( QgsGdalCloudProviderConnection::connectionList() ), qgis::listToSet( QStringList() << QStringLiteral( "my connection" ) << QStringLiteral( "second connection" ) ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).vsiHandler, QStringLiteral( "vsis3" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).container, QStringLiteral( "my_container" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).rootPath, QStringLiteral( "some/path" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "xxxx" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "my connection" ) ).credentialOptions.value( QStringLiteral( "key" ) ).toString(), QStringLiteral( "yyy" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).vsiHandler, QStringLiteral( "vsiaz" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).container, QStringLiteral( "some_container" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).rootPath, QStringLiteral( "path" ) );
QCOMPARE( QgsGdalCloudProviderConnection::connection( QStringLiteral( "second connection" ) ).credentialOptions.value( QStringLiteral( "pw" ) ).toString(), QStringLiteral( "zzz" ) );
QgsGdalCloudProviderConnection::setSelectedConnection( QStringLiteral( "second connection" ) );
QCOMPARE( QgsGdalCloudProviderConnection::selectedConnection(), QStringLiteral( "second connection" ) );
QgsGdalCloudProviderConnection::setSelectedConnection( QStringLiteral( "my connection" ) );
QCOMPARE( QgsGdalCloudProviderConnection::selectedConnection(), QStringLiteral( "my connection" ) );
}
QGSTEST_MAIN( TestQgsGdalCloudConnection )
#include "testqgsgdalcloudconnection.moc"