From 96eb75a1189c09fc94d429f61a1fffdc9b62fb38 Mon Sep 17 00:00:00 2001 From: Julien Cabieces Date: Tue, 11 May 2021 10:42:59 +0200 Subject: [PATCH] [ExternalStorage] add WebDAV external storage implementation --- .docker/docker-compose-testing.yml | 10 +- .docker/docker-variables.env | 3 + .docker/webdav/nginx.conf | 21 ++ .docker/webdav/passwords.list | 1 + .github/workflows/run-tests.yml | 1 + .../network/qgsblockingnetworkrequest.sip.in | 29 ++- .../network/qgsnetworkcontentfetcher.sip.in | 8 + .../qgsnetworkcontentfetcherregistry.sip.in | 19 +- .../qgsnetworkcontentfetchertask.sip.in | 8 + src/core/CMakeLists.txt | 3 +- .../qgsexternalstorageregistry.cpp | 2 + .../qgswebdavexternalstorage.cpp | 200 ++++++++++++++++++ .../qgswebdavexternalstorage_p.h | 141 ++++++++++++ .../network/qgsblockingnetworkrequest.cpp | 24 ++- src/core/network/qgsblockingnetworkrequest.h | 26 ++- src/core/network/qgsnetworkcontentfetcher.cpp | 15 +- src/core/network/qgsnetworkcontentfetcher.h | 7 + .../qgsnetworkcontentfetcherregistry.cpp | 17 +- .../qgsnetworkcontentfetcherregistry.h | 21 +- .../network/qgsnetworkcontentfetchertask.cpp | 19 +- .../network/qgsnetworkcontentfetchertask.h | 12 +- tests/src/python/CMakeLists.txt | 1 + .../python/test_qgsexternalstorage_webdav.py | 53 +++++ 23 files changed, 619 insertions(+), 22 deletions(-) create mode 100644 .docker/webdav/nginx.conf create mode 100644 .docker/webdav/passwords.list create mode 100644 src/core/externalstorage/qgswebdavexternalstorage.cpp create mode 100644 src/core/externalstorage/qgswebdavexternalstorage_p.h create mode 100644 tests/src/python/test_qgsexternalstorage_webdav.py diff --git a/.docker/docker-compose-testing.yml b/.docker/docker-compose-testing.yml index 0070064623e..5f8eeab4ee4 100755 --- a/.docker/docker-compose-testing.yml +++ b/.docker/docker-compose-testing.yml @@ -11,12 +11,20 @@ services: httpbin: image: kennethreitz/httpbin:latest + webdav: + image: nginx + volumes: + - ${GH_WORKSPACE}/.docker/webdav/nginx.conf:/etc/nginx/conf.d/default.conf + - ${GH_WORKSPACE}/.docker/webdav/passwords.list:/etc/nginx/.passwords.list + - /tmp/webdav_tests:/tmp/webdav_tests_root/webdav_tests + qgis-deps: tty: true image: qgis3-build-deps-binary-image volumes: - ${GH_WORKSPACE}:/root/QGIS - # links: + links: + - webdav # - mssql links: - httpbin diff --git a/.docker/docker-variables.env b/.docker/docker-variables.env index 384653b26a9..cf1d9ea0a84 100644 --- a/.docker/docker-variables.env +++ b/.docker/docker-variables.env @@ -20,3 +20,6 @@ QGIS_CONTINUOUS_INTEGRATION_RUN=true PUSH_TO_CDASH=false XDG_RUNTIME_DIR=/tmp + +QGIS_WEBDAV_HOST=webdav +QGIS_WEBDAV_PORT=80 diff --git a/.docker/webdav/nginx.conf b/.docker/webdav/nginx.conf new file mode 100644 index 00000000000..3dcca0b3eed --- /dev/null +++ b/.docker/webdav/nginx.conf @@ -0,0 +1,21 @@ +server { + listen 80; + listen [::]:80; + server_name localhost; + + location /webdav_tests { + + auth_basic realm_name; + auth_basic_user_file /etc/nginx/.passwords.list; + + dav_methods PUT DELETE MKCOL COPY MOVE; + #dav_ext_methods PROPFIND OPTIONS; + dav_access user:rw group:rw all:r; + + autoindex on; + + client_max_body_size 0; + create_full_put_path on; + root /tmp/webdav_tests_root; + } +} diff --git a/.docker/webdav/passwords.list b/.docker/webdav/passwords.list new file mode 100644 index 00000000000..8c6448e7b1e --- /dev/null +++ b/.docker/webdav/passwords.list @@ -0,0 +1 @@ +qgis:$apr1$cxID/nB1$3tG4J0FkYvEHyWAB.yqjo. diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index a812be142cc..f13f8262640 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -386,4 +386,5 @@ jobs: [[ ${{ matrix.test-batch }} == "ORACLE" ]] && sudo rm -rf /usr/share/dotnet/sdk echo "TEST_BATCH=$TEST_BATCH" echo "DOCKERFILE=$DOCKERFILE" + mkdir -p /tmp/webdav_tests && chmod 777 /tmp/webdav_tests docker-compose -f .docker/$DOCKERFILE run qgis-deps /root/QGIS/.docker/docker-qgis-test.sh $TEST_BATCH diff --git a/python/core/auto_generated/network/qgsblockingnetworkrequest.sip.in b/python/core/auto_generated/network/qgsblockingnetworkrequest.sip.in index 75f1f2d3b2c..376a3e0bcce 100644 --- a/python/core/auto_generated/network/qgsblockingnetworkrequest.sip.in +++ b/python/core/auto_generated/network/qgsblockingnetworkrequest.sip.in @@ -69,7 +69,7 @@ can be retrieved by calling :py:func:`~QgsBlockingNetworkRequest.errorMessage`. .. seealso:: :py:func:`post` %End - ErrorCode post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh = false, QgsFeedback *feedback = 0 ); + ErrorCode post( QNetworkRequest &request, QIODevice *data, bool forceRefresh = false, QgsFeedback *feedback = 0 ); %Docstring Performs a "post" operation on the specified ``request``, using the given ``data``. @@ -89,6 +89,15 @@ If an error was encountered then a specific ErrorCode will be returned, and a de can be retrieved by calling :py:func:`~QgsBlockingNetworkRequest.errorMessage`. .. seealso:: :py:func:`get` + +.. versionadded:: 3.22 +%End + + ErrorCode post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh = false, QgsFeedback *feedback = 0 ); +%Docstring +This is an overloaded function. + +Performs a "post" operation on the specified ``request``, using the given ``data``. %End ErrorCode head( QNetworkRequest &request, bool forceRefresh = false, QgsFeedback *feedback = 0 ); @@ -113,7 +122,7 @@ can be retrieved by calling :py:func:`~QgsBlockingNetworkRequest.errorMessage`. .. versionadded:: 3.18 %End - ErrorCode put( QNetworkRequest &request, const QByteArray &data, QgsFeedback *feedback = 0 ); + ErrorCode put( QNetworkRequest &request, QIODevice *data, QgsFeedback *feedback = 0 ); %Docstring Performs a "put" operation on the specified ``request``, using the given ``data``. @@ -129,6 +138,15 @@ by calling :py:func:`~QgsBlockingNetworkRequest.reply`. If an error was encountered then a specific ErrorCode will be returned, and a detailed error message can be retrieved by calling :py:func:`~QgsBlockingNetworkRequest.errorMessage`. +.. versionadded:: 3.22 +%End + + ErrorCode put( QNetworkRequest &request, const QByteArray &data, QgsFeedback *feedback = 0 ); +%Docstring +This is an overloaded function. + +Performs a "put" operation on the specified ``request``, using the given ``data``. + .. versionadded:: 3.18 %End @@ -192,6 +210,13 @@ Emitted when when data arrives during a request. void downloadFinished(); %Docstring Emitted once a request has finished downloading. +%End + + void uploadProgress( qint64, qint64 ); +%Docstring +Emitted when when data are sent during a request. + +.. versionadded:: 3.22 %End }; diff --git a/python/core/auto_generated/network/qgsnetworkcontentfetcher.sip.in b/python/core/auto_generated/network/qgsnetworkcontentfetcher.sip.in index 863b547f140..d949e3d054e 100644 --- a/python/core/auto_generated/network/qgsnetworkcontentfetcher.sip.in +++ b/python/core/auto_generated/network/qgsnetworkcontentfetcher.sip.in @@ -95,6 +95,14 @@ Emitted when content has loaded Emitted when data is received. .. versionadded:: 3.2 +%End + + void errorOccurred( QNetworkReply::NetworkError code, const QString &errorMsg ); +%Docstring +Emitted when an error with ``code`` error occured while processing the request +``errorMsg`` is a textual description of the error + +.. versionadded:: 3.22 %End }; diff --git a/python/core/auto_generated/network/qgsnetworkcontentfetcherregistry.sip.in b/python/core/auto_generated/network/qgsnetworkcontentfetcherregistry.sip.in index 0f45eb80280..49bfe739f03 100644 --- a/python/core/auto_generated/network/qgsnetworkcontentfetcherregistry.sip.in +++ b/python/core/auto_generated/network/qgsnetworkcontentfetcherregistry.sip.in @@ -32,7 +32,8 @@ FetchedContent holds useful information about a network content being fetched Failed }; - explicit QgsFetchedContent( const QString &url, QTemporaryFile *file = 0, ContentStatus status = NotStarted ); + explicit QgsFetchedContent( const QString &url, QTemporaryFile *file = 0, ContentStatus status = NotStarted, + const QString &authConfig = QString() ); %Docstring Constructs a FetchedContent with pointer to the downloaded file and status of the download %End @@ -54,6 +55,11 @@ Returns the status of the download QNetworkReply::NetworkError error() const; %Docstring Returns the potential error of the download +%End + + QString authConfig() const; +%Docstring +Returns the authentication configuration id use for this fetched content %End public slots: @@ -74,6 +80,14 @@ Cancel the download operation. void fetched(); %Docstring Emitted when the file is fetched and accessible +%End + + void errorOccurred( QNetworkReply::NetworkError code, const QString &errorMsg ); +%Docstring +Emitted when an error with ``code`` error occured while processing the request +``errorMsg`` is a textual description of the error + +.. versionadded:: 3.22 %End }; @@ -103,12 +117,13 @@ Create the registry for temporary downloaded files ~QgsNetworkContentFetcherRegistry(); - const QgsFetchedContent *fetch( const QString &url, Qgis::ActionStart fetchingMode = Qgis::ActionStart::Deferred ); + QgsFetchedContent *fetch( const QString &url, Qgis::ActionStart fetchingMode = Qgis::ActionStart::Deferred, const QString &authConfig = QString() ); %Docstring Initialize a download for the given URL :param url: the URL to be fetched :param fetchingMode: defines if the download will start immediately or shall be manually triggered +:param authConfig: authentication configuration id to be used while fetching .. note:: diff --git a/python/core/auto_generated/network/qgsnetworkcontentfetchertask.sip.in b/python/core/auto_generated/network/qgsnetworkcontentfetchertask.sip.in index 147d29f328a..daf83740b92 100644 --- a/python/core/auto_generated/network/qgsnetworkcontentfetchertask.sip.in +++ b/python/core/auto_generated/network/qgsnetworkcontentfetchertask.sip.in @@ -91,6 +91,14 @@ of whether the fetch was successful or not. Users of QgsNetworkContentFetcherTask should connect to this signal, and from the associated slot they can then safely access the network :py:func:`~QgsNetworkContentFetcherTask.reply` without danger of the task being first removed by the :py:class:`QgsTaskManager`. +%End + + void errorOccurred( QNetworkReply::NetworkError code, const QString &errorMsg ); +%Docstring +Emitted when an error with ``code`` error occured while processing the request +``errorMsg`` is a textual description of the error + +.. versionadded:: 3.22 %End }; diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index e2ffc88f8ca..5634b5b48de 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -133,6 +133,7 @@ set(QGIS_CORE_SRCS externalstorage/qgsexternalstorage.cpp externalstorage/qgsexternalstorageregistry.cpp externalstorage/qgssimplecopyexternalstorage.cpp + externalstorage/qgswebdavexternalstorage.cpp layertree/qgscolorramplegendnode.cpp layertree/qgscolorramplegendnodesettings.cpp @@ -1750,8 +1751,8 @@ set(QGIS_CORE_PRIVATE_HDRS qgsspatialindexkdbush_p.h editform/qgseditformconfig_p.h - externalstorage/qgssimplecopyexternalstorage_p.h + externalstorage/qgswebdavexternalstorage_p.h proj/qgscoordinatereferencesystem_p.h proj/qgscoordinatetransformcontext_p.h diff --git a/src/core/externalstorage/qgsexternalstorageregistry.cpp b/src/core/externalstorage/qgsexternalstorageregistry.cpp index 355257e06d5..cff065225c1 100644 --- a/src/core/externalstorage/qgsexternalstorageregistry.cpp +++ b/src/core/externalstorage/qgsexternalstorageregistry.cpp @@ -17,10 +17,12 @@ #include "qgsexternalstorage.h" #include "qgssimplecopyexternalstorage_p.h" +#include "qgswebdavexternalstorage_p.h" QgsExternalStorageRegistry::QgsExternalStorageRegistry() { registerExternalStorage( new QgsSimpleCopyExternalStorage() ); + registerExternalStorage( new QgsWebDAVExternalStorage() ); } QgsExternalStorageRegistry::~QgsExternalStorageRegistry() diff --git a/src/core/externalstorage/qgswebdavexternalstorage.cpp b/src/core/externalstorage/qgswebdavexternalstorage.cpp new file mode 100644 index 00000000000..59f9952ab30 --- /dev/null +++ b/src/core/externalstorage/qgswebdavexternalstorage.cpp @@ -0,0 +1,200 @@ +/*************************************************************************** + qgswebdavexternalstorage.cpp + -------------------------------------- + Date : March 2021 + Copyright : (C) 2021 by Julien Cabieces + Email : julien dot cabieces at oslandia dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#include "qgswebdavexternalstorage_p.h" + +#include "qgsnetworkcontentfetcherregistry.h" +#include "qgsblockingnetworkrequest.h" +#include "qgsnetworkaccessmanager.h" +#include "qgsapplication.h" + +#include +#include +#include + +QgsWebDAVExternalStorageStoreTask::QgsWebDAVExternalStorageStoreTask( const QUrl &url, const QString &filePath, const QString &authCfg ) + : QgsTask( tr( "Storing %1" ).arg( QFileInfo( filePath ).baseName() ) ) + , mUrl( url ) + , mFilePath( filePath ) + , mAuthCfg( authCfg ) + , mFeedback( new QgsFeedback( this ) ) +{ +} + +bool QgsWebDAVExternalStorageStoreTask::run() +{ + QgsBlockingNetworkRequest request; + request.setAuthCfg( mAuthCfg ); + + QNetworkRequest req( mUrl ); + QgsSetRequestInitiatorClass( req, QStringLiteral( "QgsWebDAVExternalStorageStoreTask" ) ); + + QFile *f = new QFile( mFilePath ); + f->open( QIODevice::ReadOnly ); + + connect( &request, &QgsBlockingNetworkRequest::uploadProgress, this, [ = ]( qint64 bytesReceived, qint64 bytesTotal ) + { + if ( !isCanceled() && bytesTotal > 0 ) + { + const int progress = ( bytesReceived * 100 ) / bytesTotal; + setProgress( progress ); + } + } ); + + QgsBlockingNetworkRequest::ErrorCode err = request.put( req, f, mFeedback ); + + if ( err != QgsBlockingNetworkRequest::NoError ) + { + mErrorString = request.errorMessage(); + } + + return !isCanceled() && err == QgsBlockingNetworkRequest::NoError; +} + +void QgsWebDAVExternalStorageStoreTask::cancel() +{ + mFeedback->cancel(); + QgsTask::cancel(); +} + +QString QgsWebDAVExternalStorageStoreTask::errorString() const +{ + return mErrorString; +} + +QgsWebDAVExternalStorageStoredContent::QgsWebDAVExternalStorageStoredContent( const QString &filePath, const QString &url, const QString &authcfg ) +{ + QString storageUrl = url; + if ( storageUrl.endsWith( "/" ) ) + storageUrl.append( QFileInfo( filePath ).fileName() ); + + mUploadTask = new QgsWebDAVExternalStorageStoreTask( storageUrl, filePath, authcfg ); + + connect( mUploadTask, &QgsTask::taskCompleted, this, [ = ] + { + mUrl = storageUrl; + mStatus = Qgis::ContentStatus::Finished; + emit stored(); + } ); + + connect( mUploadTask, &QgsTask::taskTerminated, this, [ = ] + { + reportError( mUploadTask->errorString() ); + } ); + + connect( mUploadTask, &QgsTask::progressChanged, this, [ = ]( double progress ) + { + emit progressChanged( progress ); + } ); +} + +void QgsWebDAVExternalStorageStoredContent::store() +{ + mStatus = Qgis::ContentStatus::Running; + QgsApplication::instance()->taskManager()->addTask( mUploadTask ); +} + + +void QgsWebDAVExternalStorageStoredContent::cancel() +{ + if ( !mUploadTask ) + return; + + disconnect( mUploadTask, &QgsTask::taskTerminated, this, nullptr ); + connect( mUploadTask, &QgsTask::taskTerminated, this, [ = ] + { + mStatus = Qgis::ContentStatus::Canceled; + emit canceled(); + } ); + + mUploadTask->cancel(); +} + +QString QgsWebDAVExternalStorageStoredContent::url() const +{ + return mUrl; +} + + +QgsWebDAVExternalStorageFetchedContent::QgsWebDAVExternalStorageFetchedContent( QgsFetchedContent *fetchedContent ) + : mFetchedContent( fetchedContent ) +{ + connect( mFetchedContent, &QgsFetchedContent::fetched, this, &QgsWebDAVExternalStorageFetchedContent::onFetched ); + connect( mFetchedContent, &QgsFetchedContent::errorOccurred, this, [ = ]( QNetworkReply::NetworkError code, const QString & errorMsg ) + { + Q_UNUSED( code ); + reportError( errorMsg ); + } ); +} + +void QgsWebDAVExternalStorageFetchedContent::fetch() +{ + if ( !mFetchedContent ) + return; + + mStatus = Qgis::ContentStatus::Running; + mFetchedContent->download(); + + // could be already fetched/cached + if ( mFetchedContent->status() == QgsFetchedContent::Finished ) + { + mStatus = Qgis::ContentStatus::Finished; + emit fetched(); + } +} + +QString QgsWebDAVExternalStorageFetchedContent::filePath() const +{ + return mFetchedContent ? mFetchedContent->filePath() : QString(); +} + +void QgsWebDAVExternalStorageFetchedContent::onFetched() +{ + if ( !mFetchedContent ) + return; + + if ( mFetchedContent->status() == QgsFetchedContent::Finished ) + { + mStatus = Qgis::ContentStatus::Finished; + emit fetched(); + } +} + +void QgsWebDAVExternalStorageFetchedContent::cancel() +{ + mFetchedContent->cancel(); +} + +QString QgsWebDAVExternalStorage::type() const +{ + return QStringLiteral( "WebDAV" ); +}; + +QString QgsWebDAVExternalStorage::displayName() const +{ + return QObject::tr( "WebDAV Storage" ); +}; + +QgsExternalStorageStoredContent *QgsWebDAVExternalStorage::doStore( const QString &filePath, const QString &url, const QString &authcfg ) const +{ + return new QgsWebDAVExternalStorageStoredContent( filePath, url, authcfg ); +}; + +QgsExternalStorageFetchedContent *QgsWebDAVExternalStorage::doFetch( const QString &url, const QString &authConfig ) const +{ + QgsFetchedContent *fetchedContent = QgsApplication::instance()->networkContentFetcherRegistry()->fetch( url, Qgis::ActionStart::Deferred, authConfig ); + + return new QgsWebDAVExternalStorageFetchedContent( fetchedContent ); +} diff --git a/src/core/externalstorage/qgswebdavexternalstorage_p.h b/src/core/externalstorage/qgswebdavexternalstorage_p.h new file mode 100644 index 00000000000..6dc41b1fbfc --- /dev/null +++ b/src/core/externalstorage/qgswebdavexternalstorage_p.h @@ -0,0 +1,141 @@ +/*************************************************************************** + qgswebdavexternalstorage.h + -------------------------------------- + Date : March 2021 + Copyright : (C) 2021 by Julien Cabieces + Email : julien dot cabieces at oslandia dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#ifndef QGSWEBDAVEXTERNALSTORAGE_H +#define QGSWEBDAVEXTERNALSTORAGE_H + +#include "qgis_core.h" +#include "qgis_sip.h" +#include "qgstaskmanager.h" + +#include "externalstorage/qgsexternalstorage.h" + +#include + +class QgsWebDAVExternalStorageStoreTask; +class QgsFetchedContent; + +///@cond PRIVATE +#define SIP_NO_FILE + +/** + * \ingroup core + * \brief External storage implementation using the protocol WebDAV. + * + * \since QGIS 3.22 + */ +class CORE_EXPORT QgsWebDAVExternalStorage : public QgsExternalStorage +{ + public: + + QString type() const override; + + QString displayName() const override; + + protected: + + QgsExternalStorageStoredContent *doStore( const QString &filePath, const QString &url, const QString &authcfg = QString() ) const override; + + QgsExternalStorageFetchedContent *doFetch( const QString &url, const QString &authConfig = QString() ) const override; +}; + +/** + * \ingroup core + * \brief Class for WebDAV stored content + * + * \since QGIS 3.22 + */ +class QgsWebDAVExternalStorageStoredContent : public QgsExternalStorageStoredContent +{ + Q_OBJECT + + public: + + QgsWebDAVExternalStorageStoredContent( const QString &filePath, const QString &url, const QString &authcfg = QString() ); + + void cancel() override; + + QString url() const override; + + void store() override; + + private: + + QPointer mUploadTask; + QString mUrl; +}; + +/** + * \ingroup core + * \brief Class for WebDAV fetched content + * + * \since QGIS 3.22 + */ +class QgsWebDAVExternalStorageFetchedContent : public QgsExternalStorageFetchedContent +{ + Q_OBJECT + + public: + + QgsWebDAVExternalStorageFetchedContent( QgsFetchedContent *fetchedContent ); + + QString filePath() const override; + + void cancel() override; + + void fetch() override; + + private slots: + + void onFetched(); + + private: + + QPointer mFetchedContent; +}; + + +/** + * \ingroup core + * \brief Task to store a file to a given WebDAV url + * + * \since QGIS 3.22 + */ +class QgsWebDAVExternalStorageStoreTask : public QgsTask +{ + Q_OBJECT + + public: + + QgsWebDAVExternalStorageStoreTask( const QUrl &url, const QString &filePath, const QString &authCfg ); + + bool run() override; + + void cancel() override; + + QString errorString() const; + + private: + + const QUrl mUrl; + const QString mFilePath; + const QString mAuthCfg; + QgsFeedback *mFeedback = nullptr; + QString mErrorString; +}; + + + +#endif // QGSWEBDAVEXTERNALSTORAGE_H diff --git a/src/core/network/qgsblockingnetworkrequest.cpp b/src/core/network/qgsblockingnetworkrequest.cpp index e4bda04e6de..e703e6bc5dd 100644 --- a/src/core/network/qgsblockingnetworkrequest.cpp +++ b/src/core/network/qgsblockingnetworkrequest.cpp @@ -27,6 +27,7 @@ #include #include #include +#include QgsBlockingNetworkRequest::QgsBlockingNetworkRequest() { @@ -60,6 +61,14 @@ QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::get( QNetworkReq } QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh, QgsFeedback *feedback ) +{ + QByteArray ldata( data ); + QBuffer buffer( &ldata ); + buffer.open( QIODevice::ReadOnly ); + return post( request, &buffer, forceRefresh, feedback ); +} + +QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::post( QNetworkRequest &request, QIODevice *data, bool forceRefresh, QgsFeedback *feedback ) { mPayloadData = data; return doRequest( Post, request, forceRefresh, feedback ); @@ -71,6 +80,14 @@ QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::head( QNetworkRe } QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::put( QNetworkRequest &request, const QByteArray &data, QgsFeedback *feedback ) +{ + QByteArray ldata( data ); + QBuffer buffer( &ldata ); + buffer.open( QIODevice::ReadOnly ); + return put( request, &buffer, feedback ); +} + +QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::put( QNetworkRequest &request, QIODevice *data, QgsFeedback *feedback ) { mPayloadData = data; return doRequest( Put, request, true, feedback ); @@ -177,6 +194,7 @@ QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::doRequest( QgsBl // * or the owner thread of mReply is currently not doing anything because it's blocked in future.waitForFinished() (if it is the main thread) connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection ); connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection ); + connect( mReply, &QNetworkReply::uploadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection ); auto resumeMainThread = [&waitConditionMutex, &authRequestBufferNotEmpty ]() { @@ -288,7 +306,10 @@ void QgsBlockingNetworkRequest::replyProgress( qint64 bytesReceived, qint64 byte } } - emit downloadProgress( bytesReceived, bytesTotal ); + if ( mMethod == Put || mMethod == Post ) + emit uploadProgress( bytesReceived, bytesTotal ); + else + emit downloadProgress( bytesReceived, bytesTotal ); } void QgsBlockingNetworkRequest::replyFinished() @@ -351,6 +372,7 @@ void QgsBlockingNetworkRequest::replyFinished() connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection ); connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection ); + connect( mReply, &QNetworkReply::uploadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection ); return; } } diff --git a/src/core/network/qgsblockingnetworkrequest.h b/src/core/network/qgsblockingnetworkrequest.h index 023e8ce90ef..f5cfd73e597 100644 --- a/src/core/network/qgsblockingnetworkrequest.h +++ b/src/core/network/qgsblockingnetworkrequest.h @@ -103,6 +103,14 @@ class CORE_EXPORT QgsBlockingNetworkRequest : public QObject * can be retrieved by calling errorMessage(). * * \see get() + * \since 3.22 + */ + ErrorCode post( QNetworkRequest &request, QIODevice *data, bool forceRefresh = false, QgsFeedback *feedback = nullptr ); + + /** + * This is an overloaded function. + * + * Performs a "post" operation on the specified \a request, using the given \a data. */ ErrorCode post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh = false, QgsFeedback *feedback = nullptr ); @@ -143,6 +151,14 @@ class CORE_EXPORT QgsBlockingNetworkRequest : public QObject * If an error was encountered then a specific ErrorCode will be returned, and a detailed error message * can be retrieved by calling errorMessage(). * + * \since 3.22 + */ + ErrorCode put( QNetworkRequest &request, QIODevice *data, QgsFeedback *feedback = nullptr ); + + /** + * This is an overloaded function. + * + * Performs a "put" operation on the specified \a request, using the given \a data. * \since 3.18 */ ErrorCode put( QNetworkRequest &request, const QByteArray &data, QgsFeedback *feedback = nullptr ); @@ -207,6 +223,12 @@ class CORE_EXPORT QgsBlockingNetworkRequest : public QObject */ void downloadFinished(); + /** + * Emitted when when data are sent during a request. + * \since QGIS 3.22 + */ + void uploadProgress( qint64, qint64 ); + private slots: void replyProgress( qint64, qint64 ); void replyFinished(); @@ -227,7 +249,9 @@ class CORE_EXPORT QgsBlockingNetworkRequest : public QObject QNetworkReply *mReply = nullptr; Method mMethod = Get; - QByteArray mPayloadData; + + //! payload data used in PUT/POST request + QIODevice *mPayloadData; //! Authentication configuration ID QString mAuthCfg; diff --git a/src/core/network/qgsnetworkcontentfetcher.cpp b/src/core/network/qgsnetworkcontentfetcher.cpp index 21f4728c336..98baa01fc0f 100644 --- a/src/core/network/qgsnetworkcontentfetcher.cpp +++ b/src/core/network/qgsnetworkcontentfetcher.cpp @@ -71,6 +71,17 @@ void QgsNetworkContentFetcher::fetchContent( const QNetworkRequest &r, const QSt mReply->setParent( nullptr ); // we don't want thread locale QgsNetworkAccessManagers to delete the reply - we want ownership of it to belong to this object connect( mReply, &QNetworkReply::finished, this, [ = ] { contentLoaded(); } ); connect( mReply, &QNetworkReply::downloadProgress, this, &QgsNetworkContentFetcher::downloadProgress ); + + auto onError = [ = ]( QNetworkReply::NetworkError code ) + { + emit errorOccurred( code, mReply->errorString() ); + }; + +#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0) + connect( mReply, qOverload( &QNetworkReply::error ), this, onError ); +#else + connect( mReply, &QNetworkReply::errorOccurred, this, onError ); +#endif } QNetworkReply *QgsNetworkContentFetcher::reply() @@ -190,7 +201,3 @@ void QgsNetworkContentFetcher::contentLoaded( bool ok ) mReply->deleteLater(); fetchContent( redirect.toUrl(), mAuthCfg ); } - - - - diff --git a/src/core/network/qgsnetworkcontentfetcher.h b/src/core/network/qgsnetworkcontentfetcher.h index efde61019cc..c84246536af 100644 --- a/src/core/network/qgsnetworkcontentfetcher.h +++ b/src/core/network/qgsnetworkcontentfetcher.h @@ -105,6 +105,13 @@ class CORE_EXPORT QgsNetworkContentFetcher : public QObject */ void downloadProgress( qint64 bytesReceived, qint64 bytesTotal ); + /** + * Emitted when an error with \a code error occured while processing the request + * \a errorMsg is a textual description of the error + * \since QGIS 3.22 + */ + void errorOccurred( QNetworkReply::NetworkError code, const QString &errorMsg ); + private: QString mAuthCfg; diff --git a/src/core/network/qgsnetworkcontentfetcherregistry.cpp b/src/core/network/qgsnetworkcontentfetcherregistry.cpp index a3a9e60e3c7..e0d3809bb3e 100644 --- a/src/core/network/qgsnetworkcontentfetcherregistry.cpp +++ b/src/core/network/qgsnetworkcontentfetcherregistry.cpp @@ -20,6 +20,8 @@ #include "qgsapplication.h" #include +#include +#include QgsNetworkContentFetcherRegistry::~QgsNetworkContentFetcherRegistry() { @@ -31,7 +33,7 @@ QgsNetworkContentFetcherRegistry::~QgsNetworkContentFetcherRegistry() mFileRegistry.clear(); } -const QgsFetchedContent *QgsNetworkContentFetcherRegistry::fetch( const QString &url, const Qgis::ActionStart fetchingMode ) +QgsFetchedContent *QgsNetworkContentFetcherRegistry::fetch( const QString &url, const Qgis::ActionStart fetchingMode, const QString &authConfig ) { if ( mFileRegistry.contains( url ) ) @@ -39,7 +41,7 @@ const QgsFetchedContent *QgsNetworkContentFetcherRegistry::fetch( const QString return mFileRegistry.value( url ); } - QgsFetchedContent *content = new QgsFetchedContent( url, nullptr, QgsFetchedContent::NotStarted ); + QgsFetchedContent *content = new QgsFetchedContent( url, nullptr, QgsFetchedContent::NotStarted, authConfig ); mFileRegistry.insert( url, content ); @@ -126,9 +128,11 @@ void QgsFetchedContent::download( bool redownload ) status() == QgsFetchedContent::NotStarted || status() == QgsFetchedContent::Failed ) { - mFetchingTask = new QgsNetworkContentFetcherTask( mUrl ); + mFetchingTask = new QgsNetworkContentFetcherTask( mUrl, mAuthConfig ); // use taskCompleted which is main thread rather than fetched signal in worker thread connect( mFetchingTask, &QgsNetworkContentFetcherTask::taskCompleted, this, &QgsFetchedContent::taskCompleted ); + connect( mFetchingTask, &QgsNetworkContentFetcherTask::taskTerminated, this, &QgsFetchedContent::taskCompleted ); + connect( mFetchingTask, &QgsNetworkContentFetcherTask::errorOccurred, this, &QgsFetchedContent::errorOccurred ); QgsApplication::instance()->taskManager()->addTask( mFetchingTask ); mStatus = QgsFetchedContent::Downloading; } @@ -163,7 +167,12 @@ void QgsFetchedContent::taskCompleted() QNetworkReply *reply = mFetchingTask->reply(); if ( reply->error() == QNetworkReply::NoError ) { - QTemporaryFile *tf = new QTemporaryFile( QStringLiteral( "XXXXXX" ) ); + // keep extension, it can be usefull when guessing file content + // (when loading this file in a Qt WebView for instance) + const QString extension = QFileInfo( reply->request().url().fileName() ).completeSuffix(); + + QTemporaryFile *tf = new QTemporaryFile( extension.isEmpty() ? QString( "XXXXXX" ) : + QString( "%1/XXXXXX.%2" ).arg( QDir::tempPath(), extension ) ); mFile = tf; tf->open(); mFile->write( reply->readAll() ); diff --git a/src/core/network/qgsnetworkcontentfetcherregistry.h b/src/core/network/qgsnetworkcontentfetcherregistry.h index 1d03bfdcdb7..a6537644a7c 100644 --- a/src/core/network/qgsnetworkcontentfetcherregistry.h +++ b/src/core/network/qgsnetworkcontentfetcherregistry.h @@ -51,10 +51,12 @@ class CORE_EXPORT QgsFetchedContent : public QObject }; //! Constructs a FetchedContent with pointer to the downloaded file and status of the download - explicit QgsFetchedContent( const QString &url, QTemporaryFile *file = nullptr, ContentStatus status = NotStarted ) + explicit QgsFetchedContent( const QString &url, QTemporaryFile *file = nullptr, ContentStatus status = NotStarted, + const QString &authConfig = QString() ) : mUrl( url ) , mFile( file ) , mStatus( status ) + , mAuthConfig( authConfig ) {} ~QgsFetchedContent() override @@ -79,6 +81,11 @@ class CORE_EXPORT QgsFetchedContent : public QObject //! Returns the potential error of the download QNetworkReply::NetworkError error() const {return mError;} + /** + * Returns the authentication configuration id use for this fetched content + */ + QString authConfig() const {return mAuthConfig;} + public slots: /** @@ -96,6 +103,13 @@ class CORE_EXPORT QgsFetchedContent : public QObject //! Emitted when the file is fetched and accessible void fetched(); + /** + * Emitted when an error with \a code error occured while processing the request + * \a errorMsg is a textual description of the error + * \since QGIS 3.22 + */ + void errorOccurred( QNetworkReply::NetworkError code, const QString &errorMsg ); + private slots: void taskCompleted(); @@ -106,6 +120,8 @@ class CORE_EXPORT QgsFetchedContent : public QObject QgsNetworkContentFetcherTask *mFetchingTask = nullptr; ContentStatus mStatus = NotStarted; QNetworkReply::NetworkError mError = QNetworkReply::NoError; + QString mAuthConfig; + QString mErrorString; }; /** @@ -134,9 +150,10 @@ class CORE_EXPORT QgsNetworkContentFetcherRegistry : public QObject * \brief Initialize a download for the given URL * \param url the URL to be fetched * \param fetchingMode defines if the download will start immediately or shall be manually triggered + * \param authConfig authentication configuration id to be used while fetching * \note If the download starts immediately, it will not redownload any already fetched or currently fetching file. */ - const QgsFetchedContent *fetch( const QString &url, Qgis::ActionStart fetchingMode = Qgis::ActionStart::Deferred ); + QgsFetchedContent *fetch( const QString &url, Qgis::ActionStart fetchingMode = Qgis::ActionStart::Deferred, const QString &authConfig = QString() ); #ifndef SIP_RUN diff --git a/src/core/network/qgsnetworkcontentfetchertask.cpp b/src/core/network/qgsnetworkcontentfetchertask.cpp index 7e68c542585..f1f37d5aa36 100644 --- a/src/core/network/qgsnetworkcontentfetchertask.cpp +++ b/src/core/network/qgsnetworkcontentfetchertask.cpp @@ -41,8 +41,13 @@ bool QgsNetworkContentFetcherTask::run() { mFetcher = new QgsNetworkContentFetcher(); QEventLoop loop; + + // We need to set the event loop (and not 'this') as receiver for all signal to ensure execution + // in the same thread and in the same order of emission. Indeed 'this' and 'loop' lives in + // different thread because they have been created in different thread. + connect( mFetcher, &QgsNetworkContentFetcher::finished, &loop, &QEventLoop::quit ); - connect( mFetcher, &QgsNetworkContentFetcher::downloadProgress, this, [ = ]( qint64 bytesReceived, qint64 bytesTotal ) + connect( mFetcher, &QgsNetworkContentFetcher::downloadProgress, &loop, [ = ]( qint64 bytesReceived, qint64 bytesTotal ) { if ( !isCanceled() && bytesTotal > 0 ) { @@ -53,12 +58,22 @@ bool QgsNetworkContentFetcherTask::run() setProgress( progress ); } } ); + + + bool hasErrorOccurred = false; + connect( mFetcher, &QgsNetworkContentFetcher::errorOccurred, &loop, [ &hasErrorOccurred, this ]( QNetworkReply::NetworkError code, const QString & errorMsg ) + { + hasErrorOccurred = true; + emit errorOccurred( code, errorMsg ); + } ); + mFetcher->fetchContent( mRequest, mAuthcfg ); loop.exec(); if ( !isCanceled() ) setProgress( 100 ); emit fetched(); - return true; + + return !isCanceled() && !hasErrorOccurred; } void QgsNetworkContentFetcherTask::cancel() diff --git a/src/core/network/qgsnetworkcontentfetchertask.h b/src/core/network/qgsnetworkcontentfetchertask.h index cfdcd481fe1..d2fc6c1062a 100644 --- a/src/core/network/qgsnetworkcontentfetchertask.h +++ b/src/core/network/qgsnetworkcontentfetchertask.h @@ -23,9 +23,9 @@ #include "qgstaskmanager.h" #include "qgis_core.h" #include +#include class QgsNetworkContentFetcher; -class QNetworkReply; /** * \class QgsNetworkContentFetcherTask @@ -103,12 +103,20 @@ class CORE_EXPORT QgsNetworkContentFetcherTask : public QgsTask */ void fetched(); + /** + * Emitted when an error with \a code error occured while processing the request + * \a errorMsg is a textual description of the error + * \since QGIS 3.22 + */ + void errorOccurred( QNetworkReply::NetworkError code, const QString &errorMsg ); + private: QNetworkRequest mRequest; QString mAuthcfg; QgsNetworkContentFetcher *mFetcher = nullptr; - + QString mMode; + QIODevice *mContent = nullptr; }; #endif //QGSNETWORKCONTENTFETCHERTASK_H diff --git a/tests/src/python/CMakeLists.txt b/tests/src/python/CMakeLists.txt index 530c4c926bb..3f3819dc25a 100644 --- a/tests/src/python/CMakeLists.txt +++ b/tests/src/python/CMakeLists.txt @@ -80,6 +80,7 @@ ADD_PYTHON_TEST(PyQgsExpressionBuilderWidget test_qgsexpressionbuilderwidget.py) ADD_PYTHON_TEST(PyQgsExpressionLineEdit test_qgsexpressionlineedit.py) ADD_PYTHON_TEST(PyQgsExtentGroupBox test_qgsextentgroupbox.py) ADD_PYTHON_TEST(PyQgsExtentWidget test_qgsextentwidget.py) +ADD_PYTHON_TEST(PyQgsExternalStorageWebDAV test_qgsexternalstorage_webdav.py) ADD_PYTHON_TEST(PyQgsFeature test_qgsfeature.py) ADD_PYTHON_TEST(PyQgsFeatureSink test_qgsfeaturesink.py) ADD_PYTHON_TEST(PyQgsFeatureSource test_qgsfeaturesource.py) diff --git a/tests/src/python/test_qgsexternalstorage_webdav.py b/tests/src/python/test_qgsexternalstorage_webdav.py new file mode 100644 index 00000000000..24cdacefe17 --- /dev/null +++ b/tests/src/python/test_qgsexternalstorage_webdav.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +"""QGIS Unit tests for WebDAV external storage + +External storage backend must implement a test based on TestPyQgsExternalStorageBase + +.. note:: This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. +""" + +__author__ = 'Julien Cabieces' +__date__ = '31/03/2021' +__copyright__ = 'Copyright 2021, The QGIS Project' + +from shutil import rmtree +import os +import tempfile +import time + +from utilities import unitTestDataPath, waitServer +from test_qgsexternalstorage_base import TestPyQgsExternalStorageBase + +from qgis.PyQt.QtCore import QCoreApplication, QEventLoop, QUrl + +from qgis.core import ( + QgsApplication, + QgsAuthMethodConfig, + QgsExternalStorageFetchedContent) + +from qgis.testing import ( + start_app, + unittest, +) + + +class TestPyQgsExternalStorageWebDAV(TestPyQgsExternalStorageBase, unittest.TestCase): + + storageType = "WebDAV" + badUrl = "http://nothinghere/" + + @classmethod + def setUpClass(cls): + """Run before all tests:""" + + super().setUpClass() + + cls.url = "http://{}:{}/webdav_tests".format( + os.environ.get('QGIS_WEBDAV_HOST', 'localhost'), os.environ.get('QGIS_WEBDAV_PORT', '80')) + + +if __name__ == '__main__': + unittest.main()