diff --git a/python/core/auto_generated/qgsblockingnetworkrequest.sip.in b/python/core/auto_generated/qgsblockingnetworkrequest.sip.in new file mode 100644 index 00000000000..dcc67864e55 --- /dev/null +++ b/python/core/auto_generated/qgsblockingnetworkrequest.sip.in @@ -0,0 +1,142 @@ +/************************************************************************ + * This file has been generated automatically from * + * * + * src/core/qgsblockingnetworkrequest.h * + * * + * Do not edit manually ! Edit header and run scripts/sipify.pl again * + ************************************************************************/ + + + +class QgsBlockingNetworkRequest : QObject +{ +%Docstring +A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy +and authentication settings. + +This class should be used whenever a blocking network request is required. Unlike implementations +which rely on QApplication.processEvents() or creation of a QEventLoop, this class is completely +thread safe and can be used on either the main thread or background threads without issue. + +Redirects are automatically handled by the class. + +After completion of a request, the reply content should be retrieved by calling getReplyContent(). +This method returns a QgsNetworkReplyContent container, which is safe and cheap to copy and pass +between threads without issue. + +.. versionadded:: 3.6 +%End + +%TypeHeaderCode +#include "qgsblockingnetworkrequest.h" +%End + public: + + enum ErrorCode + { + NoError, + NetworkError, + TimeoutError, + ServerExceptionError, + }; + + explicit QgsBlockingNetworkRequest(); +%Docstring +Constructor for QgsBlockingNetworkRequest +%End + + ~QgsBlockingNetworkRequest(); + + ErrorCode get( QNetworkRequest &request, bool forceRefresh = false ); +%Docstring +Performs a "get" operation on the specified ``request``. + +If ``forceRefresh`` is false then previously cached replies may be used for the request. If +it is set to true then a new query is always performed. + +If an authCfg() has been set, then any authentication configuration required will automatically be applied to +``request``. There is no need to manually apply the authentication to the request prior to calling +this method. + +The method will return NoError if the get operation was successful. The contents of the reply can be retrieved +by calling reply(). + +If an error was encountered then a specific ErrorCode will be returned, and a detailed error message +can be retrieved by calling errorMessage(). + +.. seealso:: :py:func:`post` +%End + + ErrorCode post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh = false ); +%Docstring +Performs a "post" operation on the specified ``request``, using the given ``data``. + +If ``forceRefresh`` is false then previously cached replies may be used for the request. If +it is set to true then a new query is always performed. + +If an authCfg() has been set, then any authentication configuration required will automatically be applied to +``request``. There is no need to manually apply the authentication to the request prior to calling +this method. + +The method will return NoError if the get operation was successful. The contents of the reply can be retrieved +by calling reply(). + +If an error was encountered then a specific ErrorCode will be returned, and a detailed error message +can be retrieved by calling errorMessage(). + +.. seealso:: :py:func:`get` +%End + + QString errorMessage() const; +%Docstring +Returns the error message string, after a get() or post() request has been made.\ +%End + + QgsNetworkReplyContent reply() const; +%Docstring +Returns the content of the network reply, after a get() or post() request has been made. +%End + + QString authCfg() const; +%Docstring +Returns the authentication config id which will be used during the request. + +.. seealso:: :py:func:`setAuthCfg` +%End + + void setAuthCfg( const QString &authCfg ); +%Docstring +Sets the authentication config id which should be used during the request. + +.. seealso:: :py:func:`authCfg` +%End + + public slots: + + void abort(); +%Docstring +Aborts the network request immediately. +%End + + signals: + + void downloadProgress( qint64, qint64 ); +%Docstring +Emitted when when data arrives during a request. +%End + + void downloadFinished(); +%Docstring +Emitted once a request has finished downloading. +%End + +}; + + +/************************************************************************ + * This file has been generated automatically from * + * * + * src/core/qgsblockingnetworkrequest.h * + * * + * Do not edit manually ! Edit header and run scripts/sipify.pl again * + ************************************************************************/ diff --git a/python/core/core_auto.sip b/python/core/core_auto.sip index b7f3b8d87da..f2dc3f5b04f 100644 --- a/python/core/core_auto.sip +++ b/python/core/core_auto.sip @@ -313,6 +313,7 @@ %Include auto_generated/qgsactionscoperegistry.sip %Include auto_generated/qgsanimatedicon.sip %Include auto_generated/qgsauxiliarystorage.sip +%Include auto_generated/qgsblockingnetworkrequest.sip %Include auto_generated/qgsbrowsermodel.sip %Include auto_generated/qgsbrowserproxymodel.sip %Include auto_generated/qgscoordinatereferencesystem.sip diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 6d9eb5d2dca..4c0dffe0f8a 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -152,6 +152,7 @@ SET(QGIS_CORE_SRCS qgsattributeeditorelement.cpp qgsauxiliarystorage.cpp qgsbearingutils.cpp + qgsblockingnetworkrequest.cpp qgsbrowsermodel.cpp qgsbrowserproxymodel.cpp qgscachedfeatureiterator.cpp @@ -595,6 +596,7 @@ SET(QGIS_CORE_MOC_HDRS qgsactionscoperegistry.h qgsanimatedicon.h qgsauxiliarystorage.h + qgsblockingnetworkrequest.h qgsbrowsermodel.h qgsbrowserproxymodel.h qgscoordinatereferencesystem.h diff --git a/src/core/qgsblockingnetworkrequest.cpp b/src/core/qgsblockingnetworkrequest.cpp new file mode 100644 index 00000000000..16e342643c2 --- /dev/null +++ b/src/core/qgsblockingnetworkrequest.cpp @@ -0,0 +1,375 @@ +/*************************************************************************** + qgsblockingnetworkrequest.cpp + ----------------------------- + begin : November 2018 + copyright : (C) 2018 by Nyall Dawson + email : nyall dot dawson at gmail dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#include "qgsblockingnetworkrequest.h" +#include "qgslogger.h" +#include "qgsapplication.h" +#include "qgsnetworkaccessmanager.h" +#include "qgsauthmanager.h" +#include "qgsmessagelog.h" +#include +#include +#include +#include +#include +#include +#include + +const qint64 READ_BUFFER_SIZE_HINT = 1024 * 1024; + +QgsBlockingNetworkRequest::QgsBlockingNetworkRequest() +{ + connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::requestTimedOut, this, &QgsBlockingNetworkRequest::requestTimedOut ); +} + +QgsBlockingNetworkRequest::~QgsBlockingNetworkRequest() +{ + abort(); +} + +void QgsBlockingNetworkRequest::requestTimedOut( QNetworkReply *reply ) +{ + if ( reply == mReply ) + mTimedout = true; +} + +QString QgsBlockingNetworkRequest::authCfg() const +{ + return mAuthCfg; +} + +void QgsBlockingNetworkRequest::setAuthCfg( const QString &authCfg ) +{ + mAuthCfg = authCfg; +} + +QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::get( QNetworkRequest &request, bool forceRefresh ) +{ + return doRequest( Get, request, forceRefresh ); +} + +QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh ) +{ + mPostData = data; + return doRequest( Post, request, forceRefresh ); +} + +QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::doRequest( QgsBlockingNetworkRequest::Method method, QNetworkRequest &request, bool forceRefresh ) +{ + mMethod = method; + + abort(); // cancel previous + mIsAborted = false; + mTimedout = false; + mGotNonEmptyResponse = false; + + mErrorMessage.clear(); + mErrorCode = NoError; + mForceRefresh = forceRefresh; + mReplyContent.clear(); + + if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) ) + { + mErrorCode = NetworkError; + mErrorMessage = errorMessageFailedAuth(); + QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); + return NetworkError; + } + + QgsDebugMsgLevel( QStringLiteral( "Calling: %1" ).arg( request.url().toString() ), 2 ); + + request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, forceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache ); + request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true ); + + QWaitCondition waitCondition; + QMutex waitConditionMutex; + bool threadFinished = false; + bool success = false; + + std::function downloaderFunction = [ this, request, &waitConditionMutex, &waitCondition, &threadFinished, &success ]() + { + if ( QThread::currentThread() != QgsApplication::instance()->thread() ) + QgsNetworkAccessManager::instance( Qt::DirectConnection ); + + success = true; + + switch ( mMethod ) + { + case Get: + mReply = QgsNetworkAccessManager::instance()->get( request ); + break; + + case Post: + mReply = QgsNetworkAccessManager::instance()->post( request, mPostData ); + break; + }; + + mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT ); + + if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) ) + { + mErrorCode = NetworkError; + mErrorMessage = errorMessageFailedAuth(); + QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); + waitCondition.wakeAll(); + success = false; + } + else + { + // We are able to use direct connection here, because we + // * either run on the thread mReply lives in, so DirectConnection is standard and safe anyway + // * or the owner thread of mReply is currently not doing anything because it's blocked in future.waitForFinished() (if it is the main thread) + connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection ); + connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection ); + + auto resumeMainThread = [&waitConditionMutex, &waitCondition]() + { + waitConditionMutex.lock(); + waitCondition.wakeAll(); + waitConditionMutex.unlock(); + + waitConditionMutex.lock(); + waitCondition.wait( &waitConditionMutex ); + waitConditionMutex.unlock(); + }; + + connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::authenticationRequired, this, resumeMainThread, Qt::DirectConnection ); + connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::proxyAuthenticationRequired, this, resumeMainThread, Qt::DirectConnection ); + +#ifndef QT_NO_SSL + connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::sslErrors, this, resumeMainThread, Qt::DirectConnection ); +#endif + QEventLoop loop; + connect( this, &QgsBlockingNetworkRequest::downloadFinished, &loop, &QEventLoop::quit, Qt::DirectConnection ); + loop.exec(); + } + waitConditionMutex.lock(); + threadFinished = true; + waitCondition.wakeAll(); + waitConditionMutex.unlock(); + }; + + if ( QThread::currentThread() == QApplication::instance()->thread() ) + { + std::unique_ptr downloaderThread = qgis::make_unique( downloaderFunction ); + downloaderThread->start(); + + while ( true ) + { + waitConditionMutex.lock(); + if ( threadFinished ) + { + waitConditionMutex.unlock(); + break; + } + waitCondition.wait( &waitConditionMutex ); + + // If the downloader thread wakes us (the main thread) up and is not yet finished + // he needs the authentication to run. + // The processEvents() call gives the auth manager the chance to show a dialog and + // once done with that, we can wake the downloaderThread again and continue the download. + if ( !threadFinished ) + { + waitConditionMutex.unlock(); + + QgsApplication::instance()->processEvents(); + waitConditionMutex.lock(); + waitCondition.wakeAll(); + waitConditionMutex.unlock(); + } + else + { + waitConditionMutex.unlock(); + } + } + // wait for thread to gracefully exit + downloaderThread->wait(); + } + else + { + downloaderFunction(); + } + return mErrorCode; +} + +void QgsBlockingNetworkRequest::abort() +{ + mIsAborted = true; + if ( mReply ) + { + mReply->deleteLater(); + mReply = nullptr; + } +} + +void QgsBlockingNetworkRequest::replyProgress( qint64 bytesReceived, qint64 bytesTotal ) +{ + QgsDebugMsgLevel( QStringLiteral( "%1 of %2 bytes downloaded." ).arg( bytesReceived ).arg( bytesTotal < 0 ? QStringLiteral( "unknown number of" ) : QString::number( bytesTotal ) ), 2 ); + + if ( bytesReceived != 0 ) + mGotNonEmptyResponse = true; + + if ( !mIsAborted && mReply ) + { + if ( mReply->error() == QNetworkReply::NoError ) + { + QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute ); + if ( !redirect.isNull() ) + { + // We don't want to emit downloadProgress() for a redirect + return; + } + } + } + + emit downloadProgress( bytesReceived, bytesTotal ); +} + +void QgsBlockingNetworkRequest::replyFinished() +{ + if ( !mIsAborted && mReply ) + { + if ( mReply->error() == QNetworkReply::NoError ) + { + QgsDebugMsgLevel( QStringLiteral( "reply OK" ), 2 ); + QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute ); + if ( !redirect.isNull() ) + { + QgsDebugMsgLevel( QStringLiteral( "Request redirected." ), 2 ); + + const QUrl &toUrl = redirect.toUrl(); + mReply->request(); + if ( toUrl == mReply->url() ) + { + mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() ); + QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); + mReplyContent.clear(); + } + else + { + QNetworkRequest request( toUrl ); + + if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) ) + { + mReplyContent.clear(); + mErrorMessage = errorMessageFailedAuth(); + mErrorCode = NetworkError; + QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); + emit downloadFinished(); + return; + } + + request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache ); + request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true ); + + mReply->deleteLater(); + mReply = nullptr; + + QgsDebugMsgLevel( QStringLiteral( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ), 2 ); + switch ( mMethod ) + { + case Get: + mReply = QgsNetworkAccessManager::instance()->get( request ); + break; + + case Post: + mReply = QgsNetworkAccessManager::instance()->post( request, mPostData ); + break; + }; + + mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT ); + + if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) ) + { + mReplyContent.clear(); + mErrorMessage = errorMessageFailedAuth(); + mErrorCode = NetworkError; + QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); + emit downloadFinished(); + return; + } + + connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection ); + connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection ); + return; + } + } + else + { + const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance(); + + if ( nam->cache() ) + { + QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() ); + + QNetworkCacheMetaData::RawHeaderList hl; + Q_FOREACH ( const QNetworkCacheMetaData::RawHeader &h, cmd.rawHeaders() ) + { + if ( h.first != "Cache-Control" ) + hl.append( h ); + } + cmd.setRawHeaders( hl ); + + QgsDebugMsgLevel( QStringLiteral( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ), 2 ); + if ( cmd.expirationDate().isNull() ) + { + cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( mExpirationSec ) ); + } + + nam->cache()->updateMetaData( cmd ); + } + else + { + QgsDebugMsgLevel( QStringLiteral( "No cache!" ), 2 ); + } + +#ifdef QGISDEBUG + bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool(); + QgsDebugMsgLevel( QStringLiteral( "Reply was cached: %1" ).arg( fromCache ), 2 ); +#endif + + mReplyContent = QgsNetworkReplyContent( mReply ); + if ( mReplyContent.content().isEmpty() && !mGotNonEmptyResponse ) + { + mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() ); + mErrorCode = ServerExceptionError; + QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); + } + } + } + else + { + mErrorMessage = mReply->errorString(); + mErrorCode = ServerExceptionError; + QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); + mReplyContent.clear(); + } + } + if ( mTimedout ) + mErrorCode = TimeoutError; + + if ( mReply ) + { + mReply->deleteLater(); + mReply = nullptr; + } + + emit downloadFinished(); +} + +QString QgsBlockingNetworkRequest::errorMessageFailedAuth() +{ + return tr( "network request update failed for authentication config" ); +} diff --git a/src/core/qgsblockingnetworkrequest.h b/src/core/qgsblockingnetworkrequest.h new file mode 100644 index 00000000000..a721b92465f --- /dev/null +++ b/src/core/qgsblockingnetworkrequest.h @@ -0,0 +1,220 @@ +/*************************************************************************** + qgsblockingnetworkrequest.h + --------------------------- + begin : November 2018 + copyright : (C) 2018 by Nyall Dawson + email : nyall dot dawson at gmail dot com + *************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +#ifndef QGSBLOCKINGNETWORKREQUEST_H +#define QGSBLOCKINGNETWORKREQUEST_H + +#include "qgis_core.h" +#include "qgsnetworkreply.h" +#include +#include +#include + +class QNetworkRequest; +class QNetworkReply; + +/** + * A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy + * and authentication settings. + * + * This class should be used whenever a blocking network request is required. Unlike implementations + * which rely on QApplication::processEvents() or creation of a QEventLoop, this class is completely + * thread safe and can be used on either the main thread or background threads without issue. + * + * Redirects are automatically handled by the class. + * + * After completion of a request, the reply content should be retrieved by calling getReplyContent(). + * This method returns a QgsNetworkReplyContent container, which is safe and cheap to copy and pass + * between threads without issue. + * + * \ingroup core + * \since QGIS 3.6 + */ +class CORE_EXPORT QgsBlockingNetworkRequest : public QObject +{ + Q_OBJECT + public: + + //! Error codes + enum ErrorCode + { + NoError, //!< No error was encountered + NetworkError, //!< A network error occurred + TimeoutError, //!< Timeout was reached before a reply was received + ServerExceptionError, //!< An exception was raised by the server + }; + + //! Constructor for QgsBlockingNetworkRequest + explicit QgsBlockingNetworkRequest(); + + ~QgsBlockingNetworkRequest() override; + + /** + * Performs a "get" operation on the specified \a request. + * + * If \a forceRefresh is false then previously cached replies may be used for the request. If + * it is set to true then a new query is always performed. + * + * If an authCfg() has been set, then any authentication configuration required will automatically be applied to + * \a request. There is no need to manually apply the authentication to the request prior to calling + * this method. + * + * The method will return NoError if the get operation was successful. The contents of the reply can be retrieved + * by calling reply(). + * + * If an error was encountered then a specific ErrorCode will be returned, and a detailed error message + * can be retrieved by calling errorMessage(). + * + * \see post() + */ + ErrorCode get( QNetworkRequest &request, bool forceRefresh = false ); + + /** + * Performs a "post" operation on the specified \a request, using the given \a data. + * + * If \a forceRefresh is false then previously cached replies may be used for the request. If + * it is set to true then a new query is always performed. + * + * If an authCfg() has been set, then any authentication configuration required will automatically be applied to + * \a request. There is no need to manually apply the authentication to the request prior to calling + * this method. + * + * The method will return NoError if the get operation was successful. The contents of the reply can be retrieved + * by calling reply(). + * + * If an error was encountered then a specific ErrorCode will be returned, and a detailed error message + * can be retrieved by calling errorMessage(). + * + * \see get() + */ + ErrorCode post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh = false ); + + /** + * Returns the error message string, after a get() or post() request has been made.\ + */ + QString errorMessage() const { return mErrorMessage; } + + /** + * Returns the content of the network reply, after a get() or post() request has been made. + */ + QgsNetworkReplyContent reply() const { return mReplyContent; } + + /** + * Returns the authentication config id which will be used during the request. + * \see setAuthCfg() + */ + QString authCfg() const; + + /** + * Sets the authentication config id which should be used during the request. + * \see authCfg() + */ + void setAuthCfg( const QString &authCfg ); + + public slots: + + /** + * Aborts the network request immediately. + */ + void abort(); + + signals: + + /** + * Emitted when when data arrives during a request. + */ + void downloadProgress( qint64, qint64 ); + + /** + * Emitted once a request has finished downloading. + */ + void downloadFinished(); + + private slots: + void replyProgress( qint64, qint64 ); + void replyFinished(); + void requestTimedOut( QNetworkReply *reply ); + + private : + + enum Method + { + Get, + Post + }; + + //! The reply to the request + QNetworkReply *mReply = nullptr; + + Method mMethod = Get; + QByteArray mPostData; + + //! Authentication configuration ID + QString mAuthCfg; + + //! The error message associated with the last error. + QString mErrorMessage; + + //! Error code + ErrorCode mErrorCode = NoError; + + QgsNetworkReplyContent mReplyContent; + + //! Whether the request is aborted. + bool mIsAborted = false; + + //! Whether to force refresh (i.e. issue a network request and not use cache) + bool mForceRefresh = false; + + //! Whether the request has timed-out + bool mTimedout = false; + + //! Whether we already received bytes + bool mGotNonEmptyResponse = false; + + int mExpirationSec = 30; + + ErrorCode doRequest( Method method, QNetworkRequest &request, bool forceRefresh ); + + QString errorMessageFailedAuth(); + +}; + +///@cond PRIVATE +#ifndef SIP_RUN + +class DownloaderThread : public QThread +{ + Q_OBJECT + + public: + DownloaderThread( const std::function &function, QObject *parent = nullptr ) + : QThread( parent ) + , mFunction( function ) + { + } + + void run() override + { + mFunction(); + } + + private: + std::function mFunction; +}; + +#endif +///@endcond + +#endif // QGSBLOCKINGNETWORKREQUEST_H