diff --git a/.ci/run_tests.sh b/.ci/run_tests.sh index 45ec8b74965..823d725e8b7 100755 --- a/.ci/run_tests.sh +++ b/.ci/run_tests.sh @@ -123,6 +123,9 @@ else COMMAND=bash fi +# Create an empty minio folder with appropriate permissions so www user can write inside it +mkdir -p /tmp/minio_tests/test_bucket && chmod -R 777 /tmp/minio_tests + # Create an empty webdav folder with appropriate permissions so www user can write inside it mkdir -p /tmp/webdav_tests && chmod 777 /tmp/webdav_tests diff --git a/.docker/docker-compose-testing.yml b/.docker/docker-compose-testing.yml index e15246a61a1..19dd0dc92c9 100755 --- a/.docker/docker-compose-testing.yml +++ b/.docker/docker-compose-testing.yml @@ -18,6 +18,15 @@ services: - ${QGIS_WORKSPACE}/.docker/webdav/passwords.list:/etc/nginx/.passwords.list - /tmp/webdav_tests:/tmp/webdav_tests_root/webdav_tests + minio: + image: minio/minio + volumes: + - /tmp/minio_tests:/data + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=adminio€ + command: server /data + qgis-deps: tty: true image: qgis3-build-deps-binary-image @@ -27,6 +36,7 @@ services: links: # - mssql - webdav + - minio - httpbin env_file: - docker-variables.env diff --git a/.docker/docker-qgis-test.sh b/.docker/docker-qgis-test.sh index a26fef1038f..c851894a140 100755 --- a/.docker/docker-qgis-test.sh +++ b/.docker/docker-qgis-test.sh @@ -208,6 +208,33 @@ EOT fi +####################################### +# Wait for Minio container to be ready +####################################### + +if [ $# -eq 0 ] || [ $1 = "ALL_BUT_PROVIDERS" ] || [ $1 = "ALL" ] ; then + + echo "::group::Setup Minio" + + echo "Wait for minio to be ready..." + COUNT=0 + while ! curl http://$QGIS_MINIO_HOST:$QGIS_MINIO_PORT &> /dev/null; + do + printf "." + sleep 5 + if [[ $(( COUNT++ )) -eq 40 ]]; then + break + fi + done + if [[ ${COUNT} -eq 41 ]]; then + echo "Error: Minio docker timeout!!!" + else + echo "done" + fi + + echo "::endgroup::" +fi + ####################################### # Wait for WebDAV container to be ready ####################################### diff --git a/.docker/docker-variables.env b/.docker/docker-variables.env index 8ab9d66d715..d8caff78226 100644 --- a/.docker/docker-variables.env +++ b/.docker/docker-variables.env @@ -21,5 +21,8 @@ PUSH_TO_CDASH=false XDG_RUNTIME_DIR=/tmp +QGIS_MINIO_HOST=minio +QGIS_MINIO_PORT=9000 + QGIS_WEBDAV_HOST=webdav QGIS_WEBDAV_PORT=80 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 066fcd4e085..0e5e1f1a7dc 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -396,6 +396,7 @@ jobs: echo "TEST_BATCH=$TEST_BATCH" echo "DOCKERFILE=$DOCKERFILE" mkdir -p /tmp/webdav_tests && chmod 777 /tmp/webdav_tests + mkdir -p /tmp/minio_tests/test_bucket && chmod -R 777 /tmp/minio_tests docker-compose -f .docker/$DOCKERFILE run qgis-deps /root/QGIS/.docker/docker-qgis-test.sh $TEST_BATCH - name: Archive test results report diff --git a/src/auth/CMakeLists.txt b/src/auth/CMakeLists.txt index f263cf83904..afb8ade5f77 100644 --- a/src/auth/CMakeLists.txt +++ b/src/auth/CMakeLists.txt @@ -19,6 +19,7 @@ add_subdirectory(pkipaths) add_subdirectory(pkipkcs12) add_subdirectory(apiheader) add_subdirectory(maptiler_hmacsha256) +add_subdirectory(awss3) if (WITH_OAUTH2_PLUGIN) add_subdirectory(oauth2) diff --git a/src/auth/awss3/CMakeLists.txt b/src/auth/awss3/CMakeLists.txt new file mode 100644 index 00000000000..47ba61a9e52 --- /dev/null +++ b/src/auth/awss3/CMakeLists.txt @@ -0,0 +1,77 @@ +set(AUTH_AWSS3_SRCS + core/qgsauthawss3method.cpp +) + +set(AUTH_AWSS3_HDRS + core/qgsauthawss3method.h +) + +set(AUTH_AWSS3_UIS_H "") + +if (WITH_GUI) + set(AUTH_AWSS3_SRCS ${AUTH_AWSS3_SRCS} + gui/qgsauthawss3edit.cpp + ) + set(AUTH_AWSS3_HDRS ${AUTH_AWSS3_HDRS} + gui/qgsauthawss3edit.h + ) + set(AUTH_AWSS3_UIS gui/qgsauthawss3edit.ui) + if (BUILD_WITH_QT6) + QT6_WRAP_UI(AUTH_AWSS3_UIS_H ${AUTH_AWSS3_UIS}) + else() + QT5_WRAP_UI(AUTH_AWSS3_UIS_H ${AUTH_AWSS3_UIS}) + endif() +endif() + + +# static library +add_library(authmethod_awss3_a STATIC ${AUTH_AWSS3_SRCS} ${AUTH_AWSS3_HDRS} ${AUTH_AWSS3_UIS_H}) + +target_include_directories(authmethod_awss3_a PUBLIC ${CMAKE_SOURCE_DIR}/src/auth/awss3/core) + +# require c++17 +target_compile_features(authmethod_awss3_a PRIVATE cxx_std_17) + +target_link_libraries(authmethod_awss3_a qgis_core) + +if (WITH_GUI) + target_include_directories(authmethod_awss3_a PRIVATE + ${CMAKE_SOURCE_DIR}/src/auth/awss3/gui + ${CMAKE_BINARY_DIR}/src/auth/awss3 + ) + + target_link_libraries (authmethod_awss3_a qgis_gui) +endif() + +target_compile_definitions(authmethod_awss3_a PRIVATE "-DQT_NO_FOREACH") + + + +if (FORCE_STATIC_LIBS) + # for (external) mobile apps to be able to pick up provider for linking + install (TARGETS authmethod_awss3_a ARCHIVE DESTINATION ${QGIS_PLUGIN_DIR}) +else() + # dynamically loaded module + add_library(authmethod_awss3 MODULE ${AUTH_AWSS3_SRCS} ${AUTH_AWSS3_HDRS} ${AUTH_AWSS3_UIS_H}) + + # require c++17 + target_compile_features(authmethod_awss3 PRIVATE cxx_std_17) + + target_link_libraries(authmethod_awss3 qgis_core) + + if (WITH_GUI) + target_include_directories(authmethod_awss3 PRIVATE + ${CMAKE_SOURCE_DIR}/src/auth/awss3/gui + ${CMAKE_BINARY_DIR}/src/auth/awss3 + ) + target_link_libraries (authmethod_awss3 qgis_gui) + add_dependencies(authmethod_awss3 ui) + endif() + + target_compile_definitions(authmethod_awss3 PRIVATE "-DQT_NO_FOREACH") + + install (TARGETS authmethod_awss3 + RUNTIME DESTINATION ${QGIS_PLUGIN_DIR} + LIBRARY DESTINATION ${QGIS_PLUGIN_DIR} + ) +endif() diff --git a/src/auth/awss3/core/qgsauthawss3method.cpp b/src/auth/awss3/core/qgsauthawss3method.cpp new file mode 100644 index 00000000000..8fa94aea524 --- /dev/null +++ b/src/auth/awss3/core/qgsauthawss3method.cpp @@ -0,0 +1,208 @@ +/*************************************************************************** + qgsauthawss3method.cpp + -------------------------------------- + Date : December 2022 + Copyright : (C) 2022 by Jacky Volpes + Email : jacky dot volpes at oslandia dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#include "qgsauthawss3method.h" + +#include +#include +#include +#include + +#include "qgsauthmanager.h" +#include "qgslogger.h" +#include "qgsapplication.h" + +#ifdef HAVE_GUI +#include "qgsauthawss3edit.h" +#endif + + +const QString QgsAuthAwsS3Method::AUTH_METHOD_KEY = QStringLiteral( "AWSS3" ); +const QString QgsAuthAwsS3Method::AUTH_METHOD_DESCRIPTION = QStringLiteral( "AWS S3" ); +const QString QgsAuthAwsS3Method::AUTH_METHOD_DISPLAY_DESCRIPTION = tr( "AWS S3" ); + +QMap QgsAuthAwsS3Method::sAuthConfigCache = QMap(); + + +QgsAuthAwsS3Method::QgsAuthAwsS3Method() +{ + setVersion( 4 ); + setExpansions( QgsAuthMethod::NetworkRequest ); + setDataProviders( QStringList() << QStringLiteral( "awss3" ) ); +} + +QString QgsAuthAwsS3Method::key() const +{ + return AUTH_METHOD_KEY; +} + +QString QgsAuthAwsS3Method::description() const +{ + return AUTH_METHOD_DESCRIPTION; +} + +QString QgsAuthAwsS3Method::displayDescription() const +{ + return AUTH_METHOD_DISPLAY_DESCRIPTION; +} + +bool QgsAuthAwsS3Method::updateNetworkRequest( QNetworkRequest &request, const QString &authcfg, + const QString &dataprovider ) +{ + Q_UNUSED( dataprovider ) + const QgsAuthMethodConfig config = getMethodConfig( authcfg ); + if ( !config.isValid() ) + { + QgsDebugMsg( QStringLiteral( "Update request config FAILED for authcfg: %1: config invalid" ).arg( authcfg ) ); + return false; + } + + const QByteArray username = config.config( QStringLiteral( "username" ) ).toLocal8Bit(); + const QByteArray password = config.config( QStringLiteral( "password" ) ).toLocal8Bit(); + const QByteArray region = config.config( QStringLiteral( "region" ) ).toLocal8Bit(); + + const QByteArray headerList = "host;x-amz-content-sha256;x-amz-date"; + const QByteArray encryptionMethod = "AWS4-HMAC-SHA256"; + const QDateTime currentDateTime = QDateTime::currentDateTime().toUTC(); + const QByteArray date = currentDateTime.toString( "yyyyMMdd" ).toLocal8Bit(); + const QByteArray dateTime = currentDateTime.toString( "yyyyMMddThhmmssZ" ).toLocal8Bit(); + + QByteArray canonicalPath = QUrl::toPercentEncoding( request.url().path(), "/" ); // Don't encode slash + if ( canonicalPath.isEmpty() ) + { + canonicalPath = "/"; + } + + QByteArray method; + QByteArray payloadHash; + if ( request.hasRawHeader( "X-Amz-Content-SHA256" ) ) + { + method = "PUT"; + payloadHash = request.rawHeader( "X-Amz-Content-SHA256" ); + } + else + { + method = "GET"; + payloadHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; // Sha256 of empty payload + request.setRawHeader( QByteArray( "X-Amz-Content-SHA256" ), payloadHash ); + } + + const QByteArray canonicalRequest = method + '\n' + + canonicalPath + '\n' + + '\n' + + "host:" + request.url().host().toLocal8Bit() + '\n' + + "x-amz-content-sha256:" + payloadHash + '\n' + + "x-amz-date:" + dateTime + '\n' + + '\n' + + headerList + '\n' + + payloadHash; + + const QByteArray canonicalRequestHash = QCryptographicHash::hash( canonicalRequest, QCryptographicHash::Sha256 ).toHex(); + const QByteArray stringToSign = encryptionMethod + '\n' + + dateTime + '\n' + + date + "/" + region + "/s3/aws4_request" + '\n' + + canonicalRequestHash; + + const QByteArray signingKey = QMessageAuthenticationCode::hash( "aws4_request", + QMessageAuthenticationCode::hash( "s3", + QMessageAuthenticationCode::hash( region, + QMessageAuthenticationCode::hash( date, "AWS4" + password, + QCryptographicHash::Sha256 ), + QCryptographicHash::Sha256 ), + QCryptographicHash::Sha256 ), + QCryptographicHash::Sha256 ); + + const QByteArray signature = QMessageAuthenticationCode::hash( stringToSign, signingKey, QCryptographicHash::Sha256 ).toHex(); + + request.setRawHeader( QByteArray( "Host" ), request.url().host().toLocal8Bit() ); + request.setRawHeader( QByteArray( "X-Amz-Date" ), dateTime ); + request.setRawHeader( QByteArray( "Authorization" ), + encryptionMethod + "Credential=" + username + '/' + date + "/" + region + "/s3/aws4_request, SignedHeaders=" + headerList + ", Signature=" + signature ); + + return true; +} + +void QgsAuthAwsS3Method::clearCachedConfig( const QString &authcfg ) +{ + removeMethodConfig( authcfg ); +} + +void QgsAuthAwsS3Method::updateMethodConfig( QgsAuthMethodConfig &mconfig ) +{ + Q_UNUSED( mconfig ); + // NOTE: add updates as method version() increases due to config storage changes +} + +QgsAuthMethodConfig QgsAuthAwsS3Method::getMethodConfig( const QString &authcfg, bool fullconfig ) +{ + const QMutexLocker locker( &mMutex ); + QgsAuthMethodConfig config; + + // check if it is cached + if ( sAuthConfigCache.contains( authcfg ) ) + { + config = sAuthConfigCache.value( authcfg ); + QgsDebugMsgLevel( QStringLiteral( "Retrieved config for authcfg: %1" ).arg( authcfg ), 2 ); + return config; + } + + // else build bundle + if ( !QgsApplication::authManager()->loadAuthenticationConfig( authcfg, config, fullconfig ) ) + { + QgsDebugMsgLevel( QStringLiteral( "Retrieved config for authcfg: %1" ).arg( authcfg ), 2 ); + return QgsAuthMethodConfig(); + } + + // cache bundle + putMethodConfig( authcfg, config ); + + return config; +} + +void QgsAuthAwsS3Method::putMethodConfig( const QString &authcfg, const QgsAuthMethodConfig &mconfig ) +{ + const QMutexLocker locker( &mMutex ); + QgsDebugMsgLevel( QStringLiteral( "Putting AWS S3 config for authcfg: %1" ).arg( authcfg ), 2 ); + sAuthConfigCache.insert( authcfg, mconfig ); +} + +void QgsAuthAwsS3Method::removeMethodConfig( const QString &authcfg ) +{ + const QMutexLocker locker( &mMutex ); + if ( sAuthConfigCache.contains( authcfg ) ) + { + sAuthConfigCache.remove( authcfg ); + QgsDebugMsgLevel( QStringLiteral( "Removed Aws S3 config for authcfg: %1" ).arg( authcfg ), 2 ); + } +} + +#ifdef HAVE_GUI +QWidget *QgsAuthAwsS3Method::editWidget( QWidget *parent ) const +{ + return new QgsAuthAwsS3Edit( parent ); +} +#endif + +////////////////////////////////////////////// +// Plugin externals +////////////////////////////////////////////// + + +#ifndef HAVE_STATIC_PROVIDERS +QGISEXTERN QgsAuthMethodMetadata *authMethodMetadataFactory() +{ + return new QgsAuthAwsS3MethodMetadata(); +} +#endif diff --git a/src/auth/awss3/core/qgsauthawss3method.h b/src/auth/awss3/core/qgsauthawss3method.h new file mode 100644 index 00000000000..1bca2a3893b --- /dev/null +++ b/src/auth/awss3/core/qgsauthawss3method.h @@ -0,0 +1,77 @@ +/*************************************************************************** + qgsauthawss3method.h + -------------------------------------- + Date : December 2022 + Copyright : (C) 2022 by Jacky Volpes + Email : jacky dot volpes at oslandia dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#ifndef QGSAUTHAWSS3METHOD_H +#define QGSAUTHAWSS3METHOD_H + +#include +#include + +#include "qgsauthconfig.h" +#include "qgsauthmethod.h" +#include "qgsauthmethodmetadata.h" + + +class QgsAuthAwsS3Method : public QgsAuthMethod +{ + Q_OBJECT + + public: + + static const QString AUTH_METHOD_KEY; + static const QString AUTH_METHOD_DESCRIPTION; + static const QString AUTH_METHOD_DISPLAY_DESCRIPTION; + + explicit QgsAuthAwsS3Method(); + + // QgsAuthMethod interface + QString key() const override; + + QString description() const override; + + QString displayDescription() const override; + + bool updateNetworkRequest( QNetworkRequest &request, const QString &authcfg, + const QString &dataprovider = QString() ) override; + + void clearCachedConfig( const QString &authcfg ) override; + void updateMethodConfig( QgsAuthMethodConfig &mconfig ) override; + +#ifdef HAVE_GUI + QWidget *editWidget( QWidget *parent )const override; +#endif + + private: + QgsAuthMethodConfig getMethodConfig( const QString &authcfg, bool fullconfig = true ); + + void putMethodConfig( const QString &authcfg, const QgsAuthMethodConfig &mconfig ); + + void removeMethodConfig( const QString &authcfg ); + + static QMap sAuthConfigCache; + +}; + + +class QgsAuthAwsS3MethodMetadata : public QgsAuthMethodMetadata +{ + public: + QgsAuthAwsS3MethodMetadata() + : QgsAuthMethodMetadata( QgsAuthAwsS3Method::AUTH_METHOD_KEY, QgsAuthAwsS3Method::AUTH_METHOD_DESCRIPTION ) + {} + QgsAuthAwsS3Method *createAuthMethod() const override {return new QgsAuthAwsS3Method;} +}; + +#endif // QGSAUTHAWSS3METHOD_H diff --git a/src/auth/awss3/gui/qgsauthawss3edit.cpp b/src/auth/awss3/gui/qgsauthawss3edit.cpp new file mode 100644 index 00000000000..1af175fa9ef --- /dev/null +++ b/src/auth/awss3/gui/qgsauthawss3edit.cpp @@ -0,0 +1,97 @@ +/*************************************************************************** + qgsauthawss3edit.cpp + -------------------------------------- + Date : December 2022 + Copyright : (C) 2022 by Jacky Volpes + Email : jacky dot volpes at oslandia dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#include "qgsauthawss3edit.h" +#include "ui_qgsauthawss3edit.h" + + +QgsAuthAwsS3Edit::QgsAuthAwsS3Edit( QWidget *parent ) + : QgsAuthMethodEdit( parent ) +{ + setupUi( this ); + connect( leUsername, &QLineEdit::textChanged, this, &QgsAuthAwsS3Edit::leUsername_textChanged ); + connect( lePassword, &QLineEdit::textChanged, this, &QgsAuthAwsS3Edit::lePassword_textChanged ); + connect( leRegion, &QLineEdit::textChanged, this, &QgsAuthAwsS3Edit::leRegion_textChanged ); + connect( chkPasswordShow, &QCheckBox::stateChanged, this, &QgsAuthAwsS3Edit::chkPasswordShow_stateChanged ); +} + +bool QgsAuthAwsS3Edit::validateConfig() +{ + const bool curvalid = !leUsername->text().isEmpty() && !lePassword->text().isEmpty() && !leRegion->text().isEmpty(); + if ( mValid != curvalid ) + { + mValid = curvalid; + emit validityChanged( curvalid ); + } + return curvalid; +} + +QgsStringMap QgsAuthAwsS3Edit::configMap() const +{ + QgsStringMap config; + config.insert( QStringLiteral( "username" ), leUsername->text() ); + config.insert( QStringLiteral( "password" ), lePassword->text() ); + config.insert( QStringLiteral( "region" ), leRegion->text() ); + + return config; +} + +void QgsAuthAwsS3Edit::loadConfig( const QgsStringMap &configmap ) +{ + clearConfig(); + + mConfigMap = configmap; + leUsername->setText( configmap.value( QStringLiteral( "username" ) ) ); + lePassword->setText( configmap.value( QStringLiteral( "password" ) ) ); + leRegion->setText( configmap.value( QStringLiteral( "region" ) ) ); + + validateConfig(); +} + +void QgsAuthAwsS3Edit::resetConfig() +{ + loadConfig( mConfigMap ); +} + +void QgsAuthAwsS3Edit::clearConfig() +{ + leUsername->clear(); + lePassword->clear(); + leRegion->clear(); + chkPasswordShow->setChecked( false ); +} + +void QgsAuthAwsS3Edit::leUsername_textChanged( const QString &txt ) +{ + Q_UNUSED( txt ) + validateConfig(); +} + +void QgsAuthAwsS3Edit::lePassword_textChanged( const QString &txt ) +{ + Q_UNUSED( txt ) + validateConfig(); +} + +void QgsAuthAwsS3Edit::leRegion_textChanged( const QString &txt ) +{ + Q_UNUSED( txt ) + validateConfig(); +} + +void QgsAuthAwsS3Edit::chkPasswordShow_stateChanged( int state ) +{ + lePassword->setEchoMode( ( state > 0 ) ? QLineEdit::Normal : QLineEdit::Password ); +} diff --git a/src/auth/awss3/gui/qgsauthawss3edit.h b/src/auth/awss3/gui/qgsauthawss3edit.h new file mode 100644 index 00000000000..fadd69dba11 --- /dev/null +++ b/src/auth/awss3/gui/qgsauthawss3edit.h @@ -0,0 +1,59 @@ +/*************************************************************************** + qgsauthawss3edit.h + -------------------------------------- + Date : December 2022 + Copyright : (C) 2022 by Jacky Volpes + Email : jacky dot volpes at oslandia dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#ifndef QGSAUTHAWSS3EDIT_H +#define QGSAUTHAWSS3EDIT_H + +#include + +#include "qgsauthmethodedit.h" +#include "ui_qgsauthawss3edit.h" + +#include "qgsauthconfig.h" + + +class QgsAuthAwsS3Edit : public QgsAuthMethodEdit, private Ui::QgsAuthAwsS3Edit +{ + Q_OBJECT + + public: + explicit QgsAuthAwsS3Edit( QWidget *parent = nullptr ); + + bool validateConfig() override; + + QgsStringMap configMap() const override; + + public slots: + void loadConfig( const QgsStringMap &configmap ) override; + + void resetConfig() override; + + void clearConfig() override; + + private slots: + void leUsername_textChanged( const QString &txt ); + + void lePassword_textChanged( const QString &txt ); + + void leRegion_textChanged( const QString &txt ); + + void chkPasswordShow_stateChanged( int state ); + + private: + QgsStringMap mConfigMap; + bool mValid = false; +}; + +#endif // QGSAUTHAWSS3EDIT_H diff --git a/src/auth/awss3/gui/qgsauthawss3edit.ui b/src/auth/awss3/gui/qgsauthawss3edit.ui new file mode 100644 index 00000000000..640e4127c69 --- /dev/null +++ b/src/auth/awss3/gui/qgsauthawss3edit.ui @@ -0,0 +1,129 @@ + + + QgsAuthAwsS3Edit + + + + 0 + 0 + 400 + 300 + + + + + 6 + + + 6 + + + 6 + + + 6 + + + + + Required + + + + + + + AWS AccessKeyId + + + Required + + + + + + + Region + + + + + + + Qt::Vertical + + + + 20 + 173 + + + + + + + + 6 + + + + + AWS SecretAccessKey + + + + + + QLineEdit::Password + + + Required + + + + + + + + 0 + 0 + + + + Show + + + + + + + + + AWS AccessKeyId + + + Username + + + + + + + AWS SecretAccessKey + + + Password + + + + + + + leUsername + lePassword + leRegion + chkPasswordShow + + + + diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 50792f2c674..775a8134f01 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -154,7 +154,7 @@ set(QGIS_CORE_SRCS externalstorage/qgsexternalstorage.cpp externalstorage/qgsexternalstorageregistry.cpp externalstorage/qgssimplecopyexternalstorage.cpp - externalstorage/qgswebdavexternalstorage.cpp + externalstorage/qgshttpexternalstorage.cpp layertree/qgscolorramplegendnode.cpp layertree/qgscolorramplegendnodesettings.cpp @@ -1925,7 +1925,7 @@ set(QGIS_CORE_PRIVATE_HDRS expression/qgsexpression_p.h externalstorage/qgssimplecopyexternalstorage_p.h - externalstorage/qgswebdavexternalstorage_p.h + externalstorage/qgshttpexternalstorage_p.h proj/qgscoordinatereferencesystem_p.h proj/qgscoordinatetransformcontext_p.h diff --git a/src/core/externalstorage/qgsexternalstorageregistry.cpp b/src/core/externalstorage/qgsexternalstorageregistry.cpp index 915bc45c7f4..92d384b417b 100644 --- a/src/core/externalstorage/qgsexternalstorageregistry.cpp +++ b/src/core/externalstorage/qgsexternalstorageregistry.cpp @@ -17,12 +17,13 @@ #include "qgsexternalstorage.h" #include "qgssimplecopyexternalstorage_p.h" -#include "qgswebdavexternalstorage_p.h" +#include "qgshttpexternalstorage_p.h" QgsExternalStorageRegistry::QgsExternalStorageRegistry() { registerExternalStorage( new QgsSimpleCopyExternalStorage() ); - registerExternalStorage( new QgsWebDAVExternalStorage() ); + registerExternalStorage( new QgsWebDavExternalStorage() ); + registerExternalStorage( new QgsAwsS3ExternalStorage() ); } QgsExternalStorageRegistry::~QgsExternalStorageRegistry() diff --git a/src/core/externalstorage/qgswebdavexternalstorage.cpp b/src/core/externalstorage/qgshttpexternalstorage.cpp similarity index 57% rename from src/core/externalstorage/qgswebdavexternalstorage.cpp rename to src/core/externalstorage/qgshttpexternalstorage.cpp index 299061b7343..8c19b489355 100644 --- a/src/core/externalstorage/qgswebdavexternalstorage.cpp +++ b/src/core/externalstorage/qgshttpexternalstorage.cpp @@ -13,7 +13,7 @@ * * ***************************************************************************/ -#include "qgswebdavexternalstorage_p.h" +#include "qgshttpexternalstorage_p.h" #include "qgsnetworkcontentfetcherregistry.h" #include "qgsblockingnetworkrequest.h" @@ -24,10 +24,11 @@ #include #include #include +#include ///@cond PRIVATE -QgsWebDAVExternalStorageStoreTask::QgsWebDAVExternalStorageStoreTask( const QUrl &url, const QString &filePath, const QString &authCfg ) +QgsHttpExternalStorageStoreTask::QgsHttpExternalStorageStoreTask( const QUrl &url, const QString &filePath, const QString &authCfg ) : QgsTask( tr( "Storing %1" ).arg( QFileInfo( filePath ).baseName() ) ) , mUrl( url ) , mFilePath( filePath ) @@ -36,17 +37,20 @@ QgsWebDAVExternalStorageStoreTask::QgsWebDAVExternalStorageStoreTask( const QUrl { } -bool QgsWebDAVExternalStorageStoreTask::run() +bool QgsHttpExternalStorageStoreTask::run() { QgsBlockingNetworkRequest request; request.setAuthCfg( mAuthCfg ); QNetworkRequest req( mUrl ); - QgsSetRequestInitiatorClass( req, QStringLiteral( "QgsWebDAVExternalStorageStoreTask" ) ); + QgsSetRequestInitiatorClass( req, QStringLiteral( "QgsHttpExternalStorageStoreTask" ) ); QFile *f = new QFile( mFilePath ); f->open( QIODevice::ReadOnly ); + if ( mPrepareRequestHandler ) + mPrepareRequestHandler( req, f ); + connect( &request, &QgsBlockingNetworkRequest::uploadProgress, this, [ = ]( qint64 bytesReceived, qint64 bytesTotal ) { if ( !isCanceled() && bytesTotal > 0 ) @@ -66,24 +70,29 @@ bool QgsWebDAVExternalStorageStoreTask::run() return !isCanceled() && err == QgsBlockingNetworkRequest::NoError; } -void QgsWebDAVExternalStorageStoreTask::cancel() +void QgsHttpExternalStorageStoreTask::cancel() { mFeedback->cancel(); QgsTask::cancel(); } -QString QgsWebDAVExternalStorageStoreTask::errorString() const +QString QgsHttpExternalStorageStoreTask::errorString() const { return mErrorString; } -QgsWebDAVExternalStorageStoredContent::QgsWebDAVExternalStorageStoredContent( const QString &filePath, const QString &url, const QString &authcfg ) +void QgsHttpExternalStorageStoreTask::setPrepareRequestHandler( std::function< void( QNetworkRequest &request, QFile *f ) > handler ) +{ + mPrepareRequestHandler = handler; +} + +QgsHttpExternalStorageStoredContent::QgsHttpExternalStorageStoredContent( const QString &filePath, const QString &url, const QString &authcfg ) { QString storageUrl = url; if ( storageUrl.endsWith( "/" ) ) storageUrl.append( QFileInfo( filePath ).fileName() ); - mUploadTask = new QgsWebDAVExternalStorageStoreTask( storageUrl, filePath, authcfg ); + mUploadTask = new QgsHttpExternalStorageStoreTask( storageUrl, filePath, authcfg ); connect( mUploadTask, &QgsTask::taskCompleted, this, [ = ] { @@ -103,14 +112,14 @@ QgsWebDAVExternalStorageStoredContent::QgsWebDAVExternalStorageStoredContent( co } ); } -void QgsWebDAVExternalStorageStoredContent::store() +void QgsHttpExternalStorageStoredContent::store() { mStatus = Qgis::ContentStatus::Running; QgsApplication::taskManager()->addTask( mUploadTask ); } -void QgsWebDAVExternalStorageStoredContent::cancel() +void QgsHttpExternalStorageStoredContent::cancel() { if ( !mUploadTask ) return; @@ -125,16 +134,21 @@ void QgsWebDAVExternalStorageStoredContent::cancel() mUploadTask->cancel(); } -QString QgsWebDAVExternalStorageStoredContent::url() const +QString QgsHttpExternalStorageStoredContent::url() const { return mUrl; } +void QgsHttpExternalStorageStoredContent::setPrepareRequestHandler( std::function< void( QNetworkRequest &request, QFile *f ) > handler ) +{ + mUploadTask->setPrepareRequestHandler( handler ); +} -QgsWebDAVExternalStorageFetchedContent::QgsWebDAVExternalStorageFetchedContent( QgsFetchedContent *fetchedContent ) + +QgsHttpExternalStorageFetchedContent::QgsHttpExternalStorageFetchedContent( QgsFetchedContent *fetchedContent ) : mFetchedContent( fetchedContent ) { - connect( mFetchedContent, &QgsFetchedContent::fetched, this, &QgsWebDAVExternalStorageFetchedContent::onFetched ); + connect( mFetchedContent, &QgsFetchedContent::fetched, this, &QgsHttpExternalStorageFetchedContent::onFetched ); connect( mFetchedContent, &QgsFetchedContent::errorOccurred, this, [ = ]( QNetworkReply::NetworkError code, const QString & errorMsg ) { Q_UNUSED( code ); @@ -142,7 +156,7 @@ QgsWebDAVExternalStorageFetchedContent::QgsWebDAVExternalStorageFetchedContent( } ); } -void QgsWebDAVExternalStorageFetchedContent::fetch() +void QgsHttpExternalStorageFetchedContent::fetch() { if ( !mFetchedContent ) return; @@ -158,12 +172,12 @@ void QgsWebDAVExternalStorageFetchedContent::fetch() } } -QString QgsWebDAVExternalStorageFetchedContent::filePath() const +QString QgsHttpExternalStorageFetchedContent::filePath() const { return mFetchedContent ? mFetchedContent->filePath() : QString(); } -void QgsWebDAVExternalStorageFetchedContent::onFetched() +void QgsHttpExternalStorageFetchedContent::onFetched() { if ( !mFetchedContent ) return; @@ -175,31 +189,68 @@ void QgsWebDAVExternalStorageFetchedContent::onFetched() } } -void QgsWebDAVExternalStorageFetchedContent::cancel() +void QgsHttpExternalStorageFetchedContent::cancel() { mFetchedContent->cancel(); } -QString QgsWebDAVExternalStorage::type() const + +// WEB DAV PROTOCOL + +QString QgsWebDavExternalStorage::type() const { return QStringLiteral( "WebDAV" ); }; -QString QgsWebDAVExternalStorage::displayName() const +QString QgsWebDavExternalStorage::displayName() const { return QObject::tr( "WebDAV Storage" ); }; -QgsExternalStorageStoredContent *QgsWebDAVExternalStorage::doStore( const QString &filePath, const QString &url, const QString &authcfg ) const +QgsExternalStorageStoredContent *QgsWebDavExternalStorage::doStore( const QString &filePath, const QString &url, const QString &authcfg ) const { - return new QgsWebDAVExternalStorageStoredContent( filePath, url, authcfg ); + return new QgsHttpExternalStorageStoredContent( filePath, url, authcfg ); }; -QgsExternalStorageFetchedContent *QgsWebDAVExternalStorage::doFetch( const QString &url, const QString &authConfig ) const +QgsExternalStorageFetchedContent *QgsWebDavExternalStorage::doFetch( const QString &url, const QString &authConfig ) const { QgsFetchedContent *fetchedContent = QgsApplication::networkContentFetcherRegistry()->fetch( url, Qgis::ActionStart::Deferred, authConfig ); - return new QgsWebDAVExternalStorageFetchedContent( fetchedContent ); + return new QgsHttpExternalStorageFetchedContent( fetchedContent ); } + +// AWS S3 PROTOCOL + +QString QgsAwsS3ExternalStorage::type() const +{ + return QStringLiteral( "AWSS3" ); +}; + +QString QgsAwsS3ExternalStorage::displayName() const +{ + return QObject::tr( "AWS S3" ); +}; + +QgsExternalStorageStoredContent *QgsAwsS3ExternalStorage::doStore( const QString &filePath, const QString &url, const QString &authcfg ) const +{ + std::unique_ptr storedContent = std::make_unique( filePath, url, authcfg ); + storedContent->setPrepareRequestHandler( []( QNetworkRequest & request, QFile * f ) + { + QCryptographicHash payloadCrypto( QCryptographicHash::Sha256 ); + payloadCrypto.addData( f ); + QByteArray payloadHash = payloadCrypto.result().toHex(); + f->seek( 0 ); + request.setRawHeader( QByteArray( "X-Amz-Content-SHA256" ), payloadHash ); + } ); + + return storedContent.release(); +}; + +QgsExternalStorageFetchedContent *QgsAwsS3ExternalStorage::doFetch( const QString &url, const QString &authConfig ) const +{ + QgsFetchedContent *fetchedContent = QgsApplication::networkContentFetcherRegistry()->fetch( url, Qgis::ActionStart::Deferred, authConfig ); + + return new QgsHttpExternalStorageFetchedContent( fetchedContent ); +} ///@endcond diff --git a/src/core/externalstorage/qgswebdavexternalstorage_p.h b/src/core/externalstorage/qgshttpexternalstorage_p.h similarity index 59% rename from src/core/externalstorage/qgswebdavexternalstorage_p.h rename to src/core/externalstorage/qgshttpexternalstorage_p.h index c8815dfec5d..d23f86c4e61 100644 --- a/src/core/externalstorage/qgswebdavexternalstorage_p.h +++ b/src/core/externalstorage/qgshttpexternalstorage_p.h @@ -26,7 +26,7 @@ #include class QgsFeedback; -class QgsWebDAVExternalStorageStoreTask; +class QgsHttpExternalStorageStoreTask; class QgsFetchedContent; ///@cond PRIVATE @@ -38,7 +38,7 @@ class QgsFetchedContent; * * \since QGIS 3.22 */ -class CORE_EXPORT QgsWebDAVExternalStorage : public QgsExternalStorage +class CORE_EXPORT QgsWebDavExternalStorage : public QgsExternalStorage { public: @@ -55,17 +55,38 @@ class CORE_EXPORT QgsWebDAVExternalStorage : public QgsExternalStorage /** * \ingroup core - * \brief Class for WebDAV stored content + * \brief External storage implementation using the protocol AWS S3. + * + * \since QGIS 3.28 + */ +class CORE_EXPORT QgsAwsS3ExternalStorage : public QgsExternalStorage +{ + public: + + QString type() const override; + + QString displayName() const override; + + protected: + + QgsExternalStorageStoredContent *doStore( const QString &filePath, const QString &url, const QString &authcfg = QString() ) const override; + + QgsExternalStorageFetchedContent *doFetch( const QString &url, const QString &authConfig = QString() ) const override; +}; + +/** + * \ingroup core + * \brief Class for HTTP stored content * * \since QGIS 3.22 */ -class QgsWebDAVExternalStorageStoredContent : public QgsExternalStorageStoredContent +class QgsHttpExternalStorageStoredContent : public QgsExternalStorageStoredContent { Q_OBJECT public: - QgsWebDAVExternalStorageStoredContent( const QString &filePath, const QString &url, const QString &authcfg = QString() ); + QgsHttpExternalStorageStoredContent( const QString &filePath, const QString &url, const QString &authcfg = QString() ); void cancel() override; @@ -73,25 +94,28 @@ class QgsWebDAVExternalStorageStoredContent : public QgsExternalStorageStoredCo void store() override; + void setPrepareRequestHandler( std::function< void( QNetworkRequest &request, QFile *f ) > ); + private: - QPointer mUploadTask; + std::function< void( QNetworkRequest &request, QFile *f ) > mPrepareRequestHandler = nullptr; + QPointer mUploadTask; QString mUrl; }; /** * \ingroup core - * \brief Class for WebDAV fetched content + * \brief Class for HTTP fetched content * * \since QGIS 3.22 */ -class QgsWebDAVExternalStorageFetchedContent : public QgsExternalStorageFetchedContent +class QgsHttpExternalStorageFetchedContent : public QgsExternalStorageFetchedContent { Q_OBJECT public: - QgsWebDAVExternalStorageFetchedContent( QgsFetchedContent *fetchedContent ); + QgsHttpExternalStorageFetchedContent( QgsFetchedContent *fetchedContent ); QString filePath() const override; @@ -111,17 +135,17 @@ class QgsWebDAVExternalStorageFetchedContent : public QgsExternalStorageFetchedC /** * \ingroup core - * \brief Task to store a file to a given WebDAV url + * \brief Task to store a file to a given url * * \since QGIS 3.22 */ -class QgsWebDAVExternalStorageStoreTask : public QgsTask +class QgsHttpExternalStorageStoreTask : public QgsTask { Q_OBJECT public: - QgsWebDAVExternalStorageStoreTask( const QUrl &url, const QString &filePath, const QString &authCfg ); + QgsHttpExternalStorageStoreTask( const QUrl &url, const QString &filePath, const QString &authCfg ); bool run() override; @@ -129,8 +153,11 @@ class QgsWebDAVExternalStorageStoreTask : public QgsTask QString errorString() const; + void setPrepareRequestHandler( std::function< void( QNetworkRequest &request, QFile *f ) > ); + private: + std::function< void( QNetworkRequest &request, QFile *f ) > mPrepareRequestHandler = nullptr; const QUrl mUrl; const QString mFilePath; const QString mAuthCfg; diff --git a/tests/src/python/CMakeLists.txt b/tests/src/python/CMakeLists.txt index 4bf6cf1bd88..c0a16098134 100644 --- a/tests/src/python/CMakeLists.txt +++ b/tests/src/python/CMakeLists.txt @@ -89,7 +89,8 @@ ADD_PYTHON_TEST(PyQgsExpressionBuilderWidget test_qgsexpressionbuilderwidget.py) ADD_PYTHON_TEST(PyQgsExpressionLineEdit test_qgsexpressionlineedit.py) ADD_PYTHON_TEST(PyQgsExtentGroupBox test_qgsextentgroupbox.py) ADD_PYTHON_TEST(PyQgsExtentWidget test_qgsextentwidget.py) -ADD_PYTHON_TEST(PyQgsExternalStorageWebDAV test_qgsexternalstorage_webdav.py) +ADD_PYTHON_TEST(PyQgsExternalStorageWebDav test_qgsexternalstorage_webdav.py) +ADD_PYTHON_TEST(PyQgsExternalStorageAwsS3 test_qgsexternalstorage_awss3.py) ADD_PYTHON_TEST(PyQgsFeature test_qgsfeature.py) ADD_PYTHON_TEST(PyQgsFeatureSink test_qgsfeaturesink.py) ADD_PYTHON_TEST(PyQgsFeatureSource test_qgsfeaturesource.py) diff --git a/tests/src/python/test_qgsexternalstorage_awss3.py b/tests/src/python/test_qgsexternalstorage_awss3.py new file mode 100644 index 00000000000..cace8c5213b --- /dev/null +++ b/tests/src/python/test_qgsexternalstorage_awss3.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +"""QGIS Unit tests for AWS S3 external storage + +External storage backend must implement a test based on TestPyQgsExternalStorageBase + +.. note:: This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. +""" + +__author__ = "Jacky Volpes" +__date__ = "20/12/2022" +__copyright__ = "Copyright 2022, The QGIS Project" + +from shutil import rmtree +import os +import tempfile +import time + +from utilities import unitTestDataPath, waitServer +from test_qgsexternalstorage_base import TestPyQgsExternalStorageBase + +from qgis.PyQt.QtCore import QCoreApplication, QEventLoop, QUrl + +from qgis.core import ( + QgsApplication, + QgsAuthMethodConfig, + QgsExternalStorageFetchedContent, +) + +from qgis.testing import ( + start_app, + unittest, +) + + +class TestPyQgsExternalStorageAwsS3(TestPyQgsExternalStorageBase, unittest.TestCase): + + storageType = "AWSS3" + badUrl = "http://nothinghere/" + + @classmethod + def setUpClass(cls): + """Run before all tests:""" + + super().setUpClass() + + bucket_name = "test_bucket" + + cls.auth_config = QgsAuthMethodConfig("AWSS3") + cls.auth_config.setConfig("username", "minioadmin") + cls.auth_config.setConfig("password", "adminio€") + cls.auth_config.setConfig("region", "us-east-1") + cls.auth_config.setName("test_awss3_auth_config") + assert cls.authm.storeAuthenticationConfig(cls.auth_config)[0] + assert cls.auth_config.isValid() + + cls.url = "http://{}:{}/{}".format( + os.environ.get("QGIS_MINIO_HOST", "localhost"), + os.environ.get("QGIS_MINIO_PORT", "80"), + bucket_name, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/src/python/test_qgsexternalstorage_base.py b/tests/src/python/test_qgsexternalstorage_base.py index 32faf34dd19..0d2973c9b10 100644 --- a/tests/src/python/test_qgsexternalstorage_base.py +++ b/tests/src/python/test_qgsexternalstorage_base.py @@ -9,9 +9,9 @@ the Free Software Foundation; either version 2 of the License, or (at your option) any later version. """ -__author__ = 'Julien Cabieces' -__date__ = '31/03/2021' -__copyright__ = 'Copyright 2021, The QGIS Project' +__author__ = "Julien Cabieces" +__date__ = "31/03/2021" +__copyright__ = "Copyright 2021, The QGIS Project" from shutil import rmtree import os @@ -27,7 +27,8 @@ from qgis.core import ( Qgis, QgsApplication, QgsAuthMethodConfig, - QgsExternalStorageFetchedContent) + QgsExternalStorageFetchedContent, +) from qgis.testing import ( start_app, @@ -35,7 +36,7 @@ from qgis.testing import ( ) -class TestPyQgsExternalStorageBase(): +class TestPyQgsExternalStorageBase: storageType = None url = None @@ -50,13 +51,13 @@ class TestPyQgsExternalStorageBase(): start_app() cls.authm = QgsApplication.authManager() - assert (cls.authm.setMasterPassword('masterpassword', True)) + assert cls.authm.setMasterPassword("masterpassword", True) assert not cls.authm.isDisabled(), cls.authm.disabledMessage() cls.auth_config = QgsAuthMethodConfig("Basic") - cls.auth_config.setConfig('username', "qgis") - cls.auth_config.setConfig('password', "myPasswd!") - cls.auth_config.setName('test_basic_auth_config') + cls.auth_config.setConfig("username", "qgis") + cls.auth_config.setConfig("password", "myPasswd!") + cls.auth_config.setName("test_basic_auth_config") assert cls.authm.storeAuthenticationConfig(cls.auth_config)[0] assert cls.auth_config.isValid() @@ -80,16 +81,21 @@ class TestPyQgsExternalStorageBase(): """Run after each test.""" pass - def getNewFile(self, content): - """Return a newly created temporary file with content""" - f = tempfile.NamedTemporaryFile(suffix='.txt') + def getNewFile(self, content, with_special_characters=False): + """Return a newly created temporary file with content + if with_special_characters is True then add url reserved characters in the file name""" + + f = tempfile.NamedTemporaryFile( + suffix=".txt", + prefix="é~u!:;=\"',iù[ &²*k (~$£<" if with_special_characters else "", + ) f.write(content) f.flush() return f def checkContent(self, file_path, content): """Check that file content matches given content""" - f = open(file_path, 'r') + f = open(file_path, "r") self.assertTrue(f.read(), b"New content") f.close() @@ -97,8 +103,10 @@ class TestPyQgsExternalStorageBase(): """ Check that storage list in in correct order """ - self.assertEqual([storage.type() for storage in self.registry.externalStorages()], - ["SimpleCopy", "WebDAV"]) + self.assertEqual( + [storage.type() for storage in self.registry.externalStorages()], + ["SimpleCopy", "WebDAV", "AWSS3"], + ) def testStoreFetchFileLater(self): """ @@ -130,7 +138,9 @@ class TestPyQgsExternalStorageBase(): self.assertEqual(spyProgressChanged[-1][0], 100) # fetch - fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id()) + fetchedContent = self.storage.fetch( + self.url + "/" + os.path.basename(f.name), self.auth_config.id() + ) self.assertTrue(fetchedContent) self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.NotStarted) @@ -147,10 +157,12 @@ class TestPyQgsExternalStorageBase(): self.assertFalse(fetchedContent.errorString()) self.assertTrue(fetchedContent.filePath()) self.checkContent(fetchedContent.filePath(), b"New content") - self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt') + self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], ".txt") # fetch again, should be cached - fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id()) + fetchedContent = self.storage.fetch( + self.url + "/" + os.path.basename(f.name), self.auth_config.id() + ) self.assertTrue(fetchedContent) self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.NotStarted) @@ -167,7 +179,7 @@ class TestPyQgsExternalStorageBase(): self.assertTrue(not fetchedContent.errorString()) self.assertTrue(fetchedContent.filePath()) self.checkContent(fetchedContent.filePath(), b"New content") - self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt') + self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], ".txt") # fetch bad url fetchedContent = self.storage.fetch(self.url + "/error", self.auth_config.id()) @@ -187,16 +199,18 @@ class TestPyQgsExternalStorageBase(): self.assertTrue(fetchedContent.errorString()) self.assertFalse(fetchedContent.filePath()) - def testStoreFetchFileImmediately(self): + def testStoreFetchFileImmediatelySpecialCharacters(self): """ - Test file storing and fetching (Immediately mode) + Test file storing and fetching (Immediately mode) with special characters name """ - f = self.getNewFile(b"New content") + f = self.getNewFile(b"New content", True) # store url = self.url + "/" + os.path.basename(f.name) - storedContent = self.storage.store(f.name, url, self.auth_config.id(), Qgis.ActionStart.Immediate) + storedContent = self.storage.store( + f.name, url, self.auth_config.id(), Qgis.ActionStart.Immediate + ) self.assertTrue(storedContent) self.assertEqual(storedContent.status(), Qgis.ContentStatus.Running) @@ -216,14 +230,20 @@ class TestPyQgsExternalStorageBase(): self.assertEqual(spyProgressChanged[-1][0], 100) # fetch - fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id(), Qgis.ActionStart.Immediate) + fetchedContent = self.storage.fetch( + self.url + "/" + os.path.basename(f.name), + self.auth_config.id(), + Qgis.ActionStart.Immediate, + ) self.assertTrue(fetchedContent) # Some external storage (SimpleCopy) doesn't actually need to retrieve the resource - self.assertTrue(fetchedContent.status() == Qgis.ContentStatus.Finished or - fetchedContent.status() == Qgis.ContentStatus.Running) + self.assertTrue( + fetchedContent.status() == Qgis.ContentStatus.Finished + or fetchedContent.status() == Qgis.ContentStatus.Running + ) - if (fetchedContent.status() == Qgis.ContentStatus.Running): + if fetchedContent.status() == Qgis.ContentStatus.Running: spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred) @@ -238,27 +258,136 @@ class TestPyQgsExternalStorageBase(): self.assertFalse(fetchedContent.errorString()) self.assertTrue(fetchedContent.filePath()) self.checkContent(fetchedContent.filePath(), b"New content") - self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt') + self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], ".txt") # fetch again, should be cached - fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id(), Qgis.ActionStart.Immediate) + fetchedContent = self.storage.fetch( + self.url + "/" + os.path.basename(f.name), + self.auth_config.id(), + Qgis.ActionStart.Immediate, + ) self.assertTrue(fetchedContent) self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished) self.assertTrue(not fetchedContent.errorString()) self.assertTrue(fetchedContent.filePath()) self.checkContent(fetchedContent.filePath(), b"New content") - self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt') + self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], ".txt") # fetch bad url - fetchedContent = self.storage.fetch(self.url + "/error", self.auth_config.id(), Qgis.ActionStart.Immediate) + fetchedContent = self.storage.fetch( + self.url + "/error", self.auth_config.id(), Qgis.ActionStart.Immediate + ) self.assertTrue(fetchedContent) # Some external storage (SimpleCopy) doesn't actually need to retrieve the resource - self.assertTrue(fetchedContent.status() == Qgis.ContentStatus.Failed or - fetchedContent.status() == Qgis.ContentStatus.Running) + self.assertTrue( + fetchedContent.status() == Qgis.ContentStatus.Failed + or fetchedContent.status() == Qgis.ContentStatus.Running + ) - if (fetchedContent.status() == Qgis.ContentStatus.Running): + if fetchedContent.status() == Qgis.ContentStatus.Running: + spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred) + + loop = QEventLoop() + fetchedContent.errorOccurred.connect(loop.quit) + fetchedContent.fetched.connect(loop.quit) + loop.exec() + + self.assertEqual(len(spyErrorOccurred), 1) + + self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Failed) + self.assertTrue(fetchedContent.errorString()) + self.assertFalse(fetchedContent.filePath()) + + def testStoreFetchFileImmediately(self): + """ + Test file storing and fetching (Immediately mode) + """ + + f = self.getNewFile(b"New content") + + # store + url = self.url + "/" + os.path.basename(f.name) + storedContent = self.storage.store( + f.name, url, self.auth_config.id(), Qgis.ActionStart.Immediate + ) + self.assertTrue(storedContent) + self.assertEqual(storedContent.status(), Qgis.ContentStatus.Running) + + spyErrorOccurred = QSignalSpy(storedContent.errorOccurred) + spyProgressChanged = QSignalSpy(storedContent.progressChanged) + + loop = QEventLoop() + storedContent.stored.connect(loop.quit) + storedContent.errorOccurred.connect(loop.quit) + loop.exec() + + self.assertEqual(len(spyErrorOccurred), 0) + self.assertEqual(storedContent.url(), url) + self.assertFalse(storedContent.errorString()) + self.assertEqual(storedContent.status(), Qgis.ContentStatus.Finished) + self.assertTrue(len(spyProgressChanged) > 0) + self.assertEqual(spyProgressChanged[-1][0], 100) + + # fetch + fetchedContent = self.storage.fetch( + self.url + "/" + os.path.basename(f.name), + self.auth_config.id(), + Qgis.ActionStart.Immediate, + ) + self.assertTrue(fetchedContent) + + # Some external storage (SimpleCopy) doesn't actually need to retrieve the resource + self.assertTrue( + fetchedContent.status() == Qgis.ContentStatus.Finished + or fetchedContent.status() == Qgis.ContentStatus.Running + ) + + if fetchedContent.status() == Qgis.ContentStatus.Running: + + spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred) + + loop = QEventLoop() + fetchedContent.fetched.connect(loop.quit) + fetchedContent.errorOccurred.connect(loop.quit) + loop.exec() + + self.assertEqual(len(spyErrorOccurred), 0) + + self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished) + self.assertFalse(fetchedContent.errorString()) + self.assertTrue(fetchedContent.filePath()) + self.checkContent(fetchedContent.filePath(), b"New content") + self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], ".txt") + + # fetch again, should be cached + fetchedContent = self.storage.fetch( + self.url + "/" + os.path.basename(f.name), + self.auth_config.id(), + Qgis.ActionStart.Immediate, + ) + self.assertTrue(fetchedContent) + self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.Finished) + + self.assertTrue(not fetchedContent.errorString()) + self.assertTrue(fetchedContent.filePath()) + self.checkContent(fetchedContent.filePath(), b"New content") + self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], ".txt") + + # fetch bad url + fetchedContent = self.storage.fetch( + self.url + "/error", self.auth_config.id(), Qgis.ActionStart.Immediate + ) + self.assertTrue(fetchedContent) + + # Some external storage (SimpleCopy) doesn't actually need to retrieve the resource + self.assertTrue( + fetchedContent.status() == Qgis.ContentStatus.Failed + or fetchedContent.status() == Qgis.ContentStatus.Running + ) + + if fetchedContent.status() == Qgis.ContentStatus.Running: spyErrorOccurred = QSignalSpy(fetchedContent.errorOccurred) loop = QEventLoop() @@ -278,7 +407,9 @@ class TestPyQgsExternalStorageBase(): """ f = self.getNewFile(b"New content") - storedContent = self.storage.store(f.name, self.badUrl + os.path.basename(f.name), self.auth_config.id()) + storedContent = self.storage.store( + f.name, self.badUrl + os.path.basename(f.name), self.auth_config.id() + ) self.assertTrue(storedContent) self.assertEqual(storedContent.status(), Qgis.ContentStatus.NotStarted) @@ -305,7 +436,9 @@ class TestPyQgsExternalStorageBase(): """ f = self.getNewFile(b"New content") - storedContent = self.storage.store(f.name, self.url + "/" + os.path.basename(f.name)) + storedContent = self.storage.store( + f.name, self.url + "/" + os.path.basename(f.name) + ) self.assertTrue(storedContent) self.assertEqual(storedContent.status(), Qgis.ContentStatus.NotStarted) @@ -334,7 +467,9 @@ class TestPyQgsExternalStorageBase(): f = self.getNewFile(b"New content") # store - storedContent = self.storage.store(f.name, self.url + "/", self.auth_config.id()) + storedContent = self.storage.store( + f.name, self.url + "/", self.auth_config.id() + ) self.assertTrue(storedContent) self.assertEqual(storedContent.status(), Qgis.ContentStatus.NotStarted) @@ -355,7 +490,9 @@ class TestPyQgsExternalStorageBase(): self.assertEqual(spyProgressChanged[-1][0], 100) # fetch - fetchedContent = self.storage.fetch(self.url + "/" + os.path.basename(f.name), self.auth_config.id()) + fetchedContent = self.storage.fetch( + self.url + "/" + os.path.basename(f.name), self.auth_config.id() + ) self.assertTrue(fetchedContent) self.assertEqual(fetchedContent.status(), Qgis.ContentStatus.NotStarted) @@ -372,4 +509,4 @@ class TestPyQgsExternalStorageBase(): self.assertFalse(fetchedContent.errorString()) self.assertTrue(fetchedContent.filePath()) self.checkContent(fetchedContent.filePath(), b"New content") - self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], '.txt') + self.assertEqual(os.path.splitext(fetchedContent.filePath())[1], ".txt") diff --git a/tests/src/python/test_qgsexternalstorage_webdav.py b/tests/src/python/test_qgsexternalstorage_webdav.py index 24cdacefe17..70b69d53e61 100644 --- a/tests/src/python/test_qgsexternalstorage_webdav.py +++ b/tests/src/python/test_qgsexternalstorage_webdav.py @@ -34,7 +34,7 @@ from qgis.testing import ( ) -class TestPyQgsExternalStorageWebDAV(TestPyQgsExternalStorageBase, unittest.TestCase): +class TestPyQgsExternalStorageWebDav(TestPyQgsExternalStorageBase, unittest.TestCase): storageType = "WebDAV" badUrl = "http://nothinghere/"