diff --git a/python/core/auto_generated/qgsblockingnetworkrequest.sip.in b/python/core/auto_generated/qgsblockingnetworkrequest.sip.in deleted file mode 100644 index f4c231aa90a..00000000000 --- a/python/core/auto_generated/qgsblockingnetworkrequest.sip.in +++ /dev/null @@ -1,146 +0,0 @@ -/************************************************************************ - * This file has been generated automatically from * - * * - * src/core/qgsblockingnetworkrequest.h * - * * - * Do not edit manually ! Edit header and run scripts/sipify.pl again * - ************************************************************************/ - - - -class QgsBlockingNetworkRequest : QObject -{ -%Docstring -A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy -and authentication settings. - -This class should be used whenever a blocking network request is required. Unlike implementations -which rely on QApplication.processEvents() or creation of a QEventLoop, this class is completely -thread safe and can be used on either the main thread or background threads without issue. - -Redirects are automatically handled by the class. - -After completion of a request, the reply content should be retrieved by calling getReplyContent(). -This method returns a QgsNetworkReplyContent container, which is safe and cheap to copy and pass -between threads without issue. - -.. versionadded:: 3.6 -%End - -%TypeHeaderCode -#include "qgsblockingnetworkrequest.h" -%End - public: - - enum ErrorCode - { - NoError, - NetworkError, - TimeoutError, - ServerExceptionError, - }; - - explicit QgsBlockingNetworkRequest(); -%Docstring -Constructor for QgsBlockingNetworkRequest -%End - - ~QgsBlockingNetworkRequest(); - - ErrorCode get( QNetworkRequest &request, bool forceRefresh = false, QgsFeedback *feedback = 0 ); -%Docstring -Performs a "get" operation on the specified ``request``. - -If ``forceRefresh`` is false then previously cached replies may be used for the request. If -it is set to true then a new query is always performed. - -If an authCfg() has been set, then any authentication configuration required will automatically be applied to -``request``. There is no need to manually apply the authentication to the request prior to calling -this method. - -The optional ``feedback`` argument can be used to abort ongoing requests. - -The method will return NoError if the get operation was successful. The contents of the reply can be retrieved -by calling reply(). - -If an error was encountered then a specific ErrorCode will be returned, and a detailed error message -can be retrieved by calling errorMessage(). - -.. seealso:: :py:func:`post` -%End - - ErrorCode post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh = false, QgsFeedback *feedback = 0 ); -%Docstring -Performs a "post" operation on the specified ``request``, using the given ``data``. - -If ``forceRefresh`` is false then previously cached replies may be used for the request. If -it is set to true then a new query is always performed. - -If an authCfg() has been set, then any authentication configuration required will automatically be applied to -``request``. There is no need to manually apply the authentication to the request prior to calling -this method. - -The optional ``feedback`` argument can be used to abort ongoing requests. - -The method will return NoError if the get operation was successful. The contents of the reply can be retrieved -by calling reply(). - -If an error was encountered then a specific ErrorCode will be returned, and a detailed error message -can be retrieved by calling errorMessage(). - -.. seealso:: :py:func:`get` -%End - - QString errorMessage() const; -%Docstring -Returns the error message string, after a get() or post() request has been made.\ -%End - - QgsNetworkReplyContent reply() const; -%Docstring -Returns the content of the network reply, after a get() or post() request has been made. -%End - - QString authCfg() const; -%Docstring -Returns the authentication config id which will be used during the request. - -.. seealso:: :py:func:`setAuthCfg` -%End - - void setAuthCfg( const QString &authCfg ); -%Docstring -Sets the authentication config id which should be used during the request. - -.. seealso:: :py:func:`authCfg` -%End - - public slots: - - void abort(); -%Docstring -Aborts the network request immediately. -%End - - signals: - - void downloadProgress( qint64, qint64 ); -%Docstring -Emitted when when data arrives during a request. -%End - - void downloadFinished(); -%Docstring -Emitted once a request has finished downloading. -%End - -}; - - -/************************************************************************ - * This file has been generated automatically from * - * * - * src/core/qgsblockingnetworkrequest.h * - * * - * Do not edit manually ! Edit header and run scripts/sipify.pl again * - ************************************************************************/ diff --git a/python/core/auto_generated/qgsnetworkreply.sip.in b/python/core/auto_generated/qgsnetworkreply.sip.in deleted file mode 100644 index 5edcb1b62f2..00000000000 --- a/python/core/auto_generated/qgsnetworkreply.sip.in +++ /dev/null @@ -1,115 +0,0 @@ -/************************************************************************ - * This file has been generated automatically from * - * * - * src/core/qgsnetworkreply.h * - * * - * Do not edit manually ! Edit header and run scripts/sipify.pl again * - ************************************************************************/ - - - -class QgsNetworkReplyContent -{ -%Docstring -Encapsulates a network reply within a container which is inexpensive to copy and safe to pass between threads. - -.. versionadded:: 3.6 -%End - -%TypeHeaderCode -#include "qgsnetworkreply.h" -%End - public: - - QgsNetworkReplyContent(); -%Docstring -Default constructor for an empty reply. -%End - - explicit QgsNetworkReplyContent( QNetworkReply *reply ); -%Docstring -Constructor for QgsNetworkReplyContent, populated from the specified ``reply``. -%End - - void clear(); -%Docstring -Clears the reply, resetting it back to a default, empty reply. -%End - - QVariant attribute( QNetworkRequest::Attribute code ) const; -%Docstring -Returns the attribute associated with the ``code``. If the attribute has not been set, it returns an -invalid QVariant. - -You can expect the default values listed in QNetworkRequest.Attribute to be -applied to the values returned by this function. - -.. seealso:: :py:func:`attributes` -%End - - - QByteArray content() const; -%Docstring -Returns the raw reply content. -%End - - QNetworkReply::NetworkError error() const; -%Docstring -Returns the reply's error message, or QNetworkReply.NoError if no -error was encountered. - -.. seealso:: :py:func:`errorString` -%End - - QString errorString() const; -%Docstring -Returns the error text for the reply, or an empty string if no -error was encountered. - -.. seealso:: :py:func:`error` -%End - - - bool hasRawHeader( const QByteArray &headerName ) const; -%Docstring -Returns true if the reply contains a header with the specified ``headerName``. - -.. seealso:: :py:func:`rawHeaderPairs` - -.. seealso:: :py:func:`rawHeaderList` - -.. seealso:: :py:func:`rawHeader` -%End - - QList rawHeaderList() const; -%Docstring -Returns a list of raw header names contained within the reply. - -.. seealso:: :py:func:`rawHeaderPairs` - -.. seealso:: :py:func:`hasRawHeader` - -.. seealso:: :py:func:`rawHeader` -%End - - QByteArray rawHeader( const QByteArray &headerName ) const; -%Docstring -Returns the content of the header with the specified ``headerName``, or an -empty QByteArray if the specified header was not found in the reply. - -.. seealso:: :py:func:`rawHeaderPairs` - -.. seealso:: :py:func:`hasRawHeader` - -.. seealso:: :py:func:`rawHeaderList` -%End - -}; - -/************************************************************************ - * This file has been generated automatically from * - * * - * src/core/qgsnetworkreply.h * - * * - * Do not edit manually ! Edit header and run scripts/sipify.pl again * - ************************************************************************/ diff --git a/python/core/core_auto.sip b/python/core/core_auto.sip index f2dc3f5b04f..a1abcd32cb1 100644 --- a/python/core/core_auto.sip +++ b/python/core/core_auto.sip @@ -80,7 +80,6 @@ %Include auto_generated/qgsmargins.sip %Include auto_generated/qgsmimedatautils.sip %Include auto_generated/qgsmultirenderchecker.sip -%Include auto_generated/qgsnetworkreply.sip %Include auto_generated/qgsobjectcustomproperties.sip %Include auto_generated/qgsogcutils.sip %Include auto_generated/qgsoptional.sip @@ -313,7 +312,6 @@ %Include auto_generated/qgsactionscoperegistry.sip %Include auto_generated/qgsanimatedicon.sip %Include auto_generated/qgsauxiliarystorage.sip -%Include auto_generated/qgsblockingnetworkrequest.sip %Include auto_generated/qgsbrowsermodel.sip %Include auto_generated/qgsbrowserproxymodel.sip %Include auto_generated/qgscoordinatereferencesystem.sip diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 4c0dffe0f8a..d1b53c57906 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -152,7 +152,6 @@ SET(QGIS_CORE_SRCS qgsattributeeditorelement.cpp qgsauxiliarystorage.cpp qgsbearingutils.cpp - qgsblockingnetworkrequest.cpp qgsbrowsermodel.cpp qgsbrowserproxymodel.cpp qgscachedfeatureiterator.cpp @@ -258,7 +257,6 @@ SET(QGIS_CORE_SRCS qgsnetworkcontentfetcher.cpp qgsnetworkcontentfetcherregistry.cpp qgsnetworkcontentfetchertask.cpp - qgsnetworkreply.cpp qgsnetworkreplyparser.cpp qgsobjectcustomproperties.cpp qgsofflineediting.cpp @@ -596,7 +594,6 @@ SET(QGIS_CORE_MOC_HDRS qgsactionscoperegistry.h qgsanimatedicon.h qgsauxiliarystorage.h - qgsblockingnetworkrequest.h qgsbrowsermodel.h qgsbrowserproxymodel.h qgscoordinatereferencesystem.h @@ -911,7 +908,6 @@ SET(QGIS_CORE_HDRS qgsmargins.h qgsmimedatautils.h qgsmultirenderchecker.h - qgsnetworkreply.h qgsobjectcustomproperties.h qgsogcutils.h qgsoptional.h diff --git a/src/core/qgsblockingnetworkrequest.cpp b/src/core/qgsblockingnetworkrequest.cpp deleted file mode 100644 index 2b0cd6f4cfe..00000000000 --- a/src/core/qgsblockingnetworkrequest.cpp +++ /dev/null @@ -1,391 +0,0 @@ -/*************************************************************************** - qgsblockingnetworkrequest.cpp - ----------------------------- - begin : November 2018 - copyright : (C) 2018 by Nyall Dawson - email : nyall dot dawson at gmail dot com - *************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ - -#include "qgsblockingnetworkrequest.h" -#include "qgslogger.h" -#include "qgsapplication.h" -#include "qgsnetworkaccessmanager.h" -#include "qgsauthmanager.h" -#include "qgsmessagelog.h" -#include "qgsfeedback.h" -#include -#include -#include -#include -#include -#include -#include - -const qint64 READ_BUFFER_SIZE_HINT = 1024 * 1024; - -QgsBlockingNetworkRequest::QgsBlockingNetworkRequest() -{ - connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::requestTimedOut, this, &QgsBlockingNetworkRequest::requestTimedOut ); -} - -QgsBlockingNetworkRequest::~QgsBlockingNetworkRequest() -{ - abort(); -} - -void QgsBlockingNetworkRequest::requestTimedOut( QNetworkReply *reply ) -{ - if ( reply == mReply ) - mTimedout = true; -} - -QString QgsBlockingNetworkRequest::authCfg() const -{ - return mAuthCfg; -} - -void QgsBlockingNetworkRequest::setAuthCfg( const QString &authCfg ) -{ - mAuthCfg = authCfg; -} - -QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::get( QNetworkRequest &request, bool forceRefresh, QgsFeedback *feedback ) -{ - return doRequest( Get, request, forceRefresh, feedback ); -} - -QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh, QgsFeedback *feedback ) -{ - mPostData = data; - return doRequest( Post, request, forceRefresh, feedback ); -} - -QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::doRequest( QgsBlockingNetworkRequest::Method method, QNetworkRequest &request, bool forceRefresh, QgsFeedback *feedback ) -{ - mMethod = method; - mFeedback = feedback; - - abort(); // cancel previous - mIsAborted = false; - mTimedout = false; - mGotNonEmptyResponse = false; - - mErrorMessage.clear(); - mErrorCode = NoError; - mForceRefresh = forceRefresh; - mReplyContent.clear(); - - if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) ) - { - mErrorCode = NetworkError; - mErrorMessage = errorMessageFailedAuth(); - QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); - return NetworkError; - } - - QgsDebugMsgLevel( QStringLiteral( "Calling: %1" ).arg( request.url().toString() ), 2 ); - - request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, forceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache ); - request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true ); - - // QWaitCondition based producer/consumer problem - // taken from http://doc.qt.io/qt-5/qtcore-threads-waitconditions-example.html - // in this case the buffer has size 1 and potentially contains authentication requests - // from the producer (downloader thread), which the consumer (main thread) needs to - // handle - QWaitCondition authRequestBufferNotEmpty; - QWaitCondition authRequestBufferNotFull; - QMutex waitConditionMutex; - - bool threadFinished = false; - bool success = false; - - if ( mFeedback ) - connect( mFeedback, &QgsFeedback::canceled, this, &QgsBlockingNetworkRequest::abort ); - - std::function downloaderFunction = [ this, request, &waitConditionMutex, &authRequestBufferNotEmpty, &authRequestBufferNotFull, &threadFinished, &success ]() - { - if ( QThread::currentThread() != QgsApplication::instance()->thread() ) - QgsNetworkAccessManager::instance( Qt::DirectConnection ); - - success = true; - - switch ( mMethod ) - { - case Get: - mReply = QgsNetworkAccessManager::instance()->get( request ); - break; - - case Post: - mReply = QgsNetworkAccessManager::instance()->post( request, mPostData ); - break; - }; - - mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT ); - if ( mFeedback ) - connect( mFeedback, &QgsFeedback::canceled, mReply, &QNetworkReply::abort ); - - if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) ) - { - mErrorCode = NetworkError; - mErrorMessage = errorMessageFailedAuth(); - QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); - authRequestBufferNotEmpty.wakeAll(); - success = false; - } - else - { - // We are able to use direct connection here, because we - // * either run on the thread mReply lives in, so DirectConnection is standard and safe anyway - // * or the owner thread of mReply is currently not doing anything because it's blocked in future.waitForFinished() (if it is the main thread) - connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection ); - connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection ); - - auto resumeMainThread = [&waitConditionMutex, &authRequestBufferNotEmpty, &authRequestBufferNotFull ]() - { - // when this method is called we have "produced" a single authentication request -- so the buffer is now full - // and it's time for the "consumer" (main thread) to do its part - waitConditionMutex.lock(); - authRequestBufferNotFull.wait( &waitConditionMutex ); - waitConditionMutex.unlock(); - - waitConditionMutex.lock(); - authRequestBufferNotEmpty.wakeAll(); - waitConditionMutex.unlock(); - }; - - connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::authenticationRequired, this, resumeMainThread, Qt::DirectConnection ); - connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::proxyAuthenticationRequired, this, resumeMainThread, Qt::DirectConnection ); - -#ifndef QT_NO_SSL - connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::sslErrors, this, resumeMainThread, Qt::DirectConnection ); -#endif - QEventLoop loop; - connect( this, &QgsBlockingNetworkRequest::downloadFinished, &loop, &QEventLoop::quit, Qt::DirectConnection ); - loop.exec(); - } - waitConditionMutex.lock(); - threadFinished = true; - authRequestBufferNotEmpty.wakeAll(); - waitConditionMutex.unlock(); - }; - - if ( QThread::currentThread() == QApplication::instance()->thread() ) - { - std::unique_ptr downloaderThread = qgis::make_unique( downloaderFunction ); - downloaderThread->start(); - - while ( true ) - { - waitConditionMutex.lock(); - if ( threadFinished ) - { - waitConditionMutex.unlock(); - break; - } - authRequestBufferNotEmpty.wait( &waitConditionMutex ); - - // If the downloader thread wakes us (the main thread) up and is not yet finished - // then it has "produced" an authentication request which we need to now "consume". - // The processEvents() call gives the auth manager the chance to show a dialog and - // once done with that, we can wake the downloaderThread again and continue the download. - if ( !threadFinished ) - { - waitConditionMutex.unlock(); - - QgsApplication::instance()->processEvents(); - waitConditionMutex.lock(); - authRequestBufferNotFull.wakeAll(); - waitConditionMutex.unlock(); - } - else - { - waitConditionMutex.unlock(); - } - } - // wait for thread to gracefully exit - downloaderThread->wait(); - } - else - { - downloaderFunction(); - } - return mErrorCode; -} - -void QgsBlockingNetworkRequest::abort() -{ - mIsAborted = true; - if ( mReply ) - { - mReply->deleteLater(); - mReply = nullptr; - } -} - -void QgsBlockingNetworkRequest::replyProgress( qint64 bytesReceived, qint64 bytesTotal ) -{ - QgsDebugMsgLevel( QStringLiteral( "%1 of %2 bytes downloaded." ).arg( bytesReceived ).arg( bytesTotal < 0 ? QStringLiteral( "unknown number of" ) : QString::number( bytesTotal ) ), 2 ); - - if ( bytesReceived != 0 ) - mGotNonEmptyResponse = true; - - if ( !mIsAborted && mReply && ( !mFeedback || !mFeedback->isCanceled() ) ) - { - if ( mReply->error() == QNetworkReply::NoError ) - { - QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute ); - if ( !redirect.isNull() ) - { - // We don't want to emit downloadProgress() for a redirect - return; - } - } - } - - emit downloadProgress( bytesReceived, bytesTotal ); -} - -void QgsBlockingNetworkRequest::replyFinished() -{ - if ( !mIsAborted && mReply ) - { - if ( mReply->error() == QNetworkReply::NoError && ( !mFeedback || !mFeedback->isCanceled() ) ) - { - QgsDebugMsgLevel( QStringLiteral( "reply OK" ), 2 ); - QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute ); - if ( !redirect.isNull() ) - { - QgsDebugMsgLevel( QStringLiteral( "Request redirected." ), 2 ); - - const QUrl &toUrl = redirect.toUrl(); - mReply->request(); - if ( toUrl == mReply->url() ) - { - mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() ); - QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); - mReplyContent.clear(); - } - else - { - QNetworkRequest request( toUrl ); - - if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) ) - { - mReplyContent.clear(); - mErrorMessage = errorMessageFailedAuth(); - mErrorCode = NetworkError; - QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); - emit downloadFinished(); - return; - } - - request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache ); - request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true ); - - mReply->deleteLater(); - mReply = nullptr; - - QgsDebugMsgLevel( QStringLiteral( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ), 2 ); - switch ( mMethod ) - { - case Get: - mReply = QgsNetworkAccessManager::instance()->get( request ); - break; - - case Post: - mReply = QgsNetworkAccessManager::instance()->post( request, mPostData ); - break; - }; - - mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT ); - - if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) ) - { - mReplyContent.clear(); - mErrorMessage = errorMessageFailedAuth(); - mErrorCode = NetworkError; - QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); - emit downloadFinished(); - return; - } - - connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection ); - connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection ); - return; - } - } - else - { - const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance(); - - if ( nam->cache() ) - { - QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() ); - - QNetworkCacheMetaData::RawHeaderList hl; - Q_FOREACH ( const QNetworkCacheMetaData::RawHeader &h, cmd.rawHeaders() ) - { - if ( h.first != "Cache-Control" ) - hl.append( h ); - } - cmd.setRawHeaders( hl ); - - QgsDebugMsgLevel( QStringLiteral( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ), 2 ); - if ( cmd.expirationDate().isNull() ) - { - cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( mExpirationSec ) ); - } - - nam->cache()->updateMetaData( cmd ); - } - else - { - QgsDebugMsgLevel( QStringLiteral( "No cache!" ), 2 ); - } - -#ifdef QGISDEBUG - bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool(); - QgsDebugMsgLevel( QStringLiteral( "Reply was cached: %1" ).arg( fromCache ), 2 ); -#endif - - mReplyContent = QgsNetworkReplyContent( mReply ); - if ( mReplyContent.content().isEmpty() && !mGotNonEmptyResponse ) - { - mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() ); - mErrorCode = ServerExceptionError; - QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); - } - } - } - else - { - mErrorMessage = mReply->errorString(); - mErrorCode = ServerExceptionError; - QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) ); - mReplyContent.clear(); - } - } - if ( mTimedout ) - mErrorCode = TimeoutError; - - if ( mReply ) - { - mReply->deleteLater(); - mReply = nullptr; - } - - emit downloadFinished(); -} - -QString QgsBlockingNetworkRequest::errorMessageFailedAuth() -{ - return tr( "network request update failed for authentication config" ); -} diff --git a/src/core/qgsblockingnetworkrequest.h b/src/core/qgsblockingnetworkrequest.h deleted file mode 100644 index 72185110d73..00000000000 --- a/src/core/qgsblockingnetworkrequest.h +++ /dev/null @@ -1,228 +0,0 @@ -/*************************************************************************** - qgsblockingnetworkrequest.h - --------------------------- - begin : November 2018 - copyright : (C) 2018 by Nyall Dawson - email : nyall dot dawson at gmail dot com - *************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ -#ifndef QGSBLOCKINGNETWORKREQUEST_H -#define QGSBLOCKINGNETWORKREQUEST_H - -#include "qgis_core.h" -#include "qgsnetworkreply.h" -#include "qgsfeedback.h" -#include -#include -#include -#include - -class QNetworkRequest; -class QNetworkReply; - -/** - * A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy - * and authentication settings. - * - * This class should be used whenever a blocking network request is required. Unlike implementations - * which rely on QApplication::processEvents() or creation of a QEventLoop, this class is completely - * thread safe and can be used on either the main thread or background threads without issue. - * - * Redirects are automatically handled by the class. - * - * After completion of a request, the reply content should be retrieved by calling getReplyContent(). - * This method returns a QgsNetworkReplyContent container, which is safe and cheap to copy and pass - * between threads without issue. - * - * \ingroup core - * \since QGIS 3.6 - */ -class CORE_EXPORT QgsBlockingNetworkRequest : public QObject -{ - Q_OBJECT - public: - - //! Error codes - enum ErrorCode - { - NoError, //!< No error was encountered - NetworkError, //!< A network error occurred - TimeoutError, //!< Timeout was reached before a reply was received - ServerExceptionError, //!< An exception was raised by the server - }; - - //! Constructor for QgsBlockingNetworkRequest - explicit QgsBlockingNetworkRequest(); - - ~QgsBlockingNetworkRequest() override; - - /** - * Performs a "get" operation on the specified \a request. - * - * If \a forceRefresh is false then previously cached replies may be used for the request. If - * it is set to true then a new query is always performed. - * - * If an authCfg() has been set, then any authentication configuration required will automatically be applied to - * \a request. There is no need to manually apply the authentication to the request prior to calling - * this method. - * - * The optional \a feedback argument can be used to abort ongoing requests. - * - * The method will return NoError if the get operation was successful. The contents of the reply can be retrieved - * by calling reply(). - * - * If an error was encountered then a specific ErrorCode will be returned, and a detailed error message - * can be retrieved by calling errorMessage(). - * - * \see post() - */ - ErrorCode get( QNetworkRequest &request, bool forceRefresh = false, QgsFeedback *feedback = nullptr ); - - /** - * Performs a "post" operation on the specified \a request, using the given \a data. - * - * If \a forceRefresh is false then previously cached replies may be used for the request. If - * it is set to true then a new query is always performed. - * - * If an authCfg() has been set, then any authentication configuration required will automatically be applied to - * \a request. There is no need to manually apply the authentication to the request prior to calling - * this method. - * - * The optional \a feedback argument can be used to abort ongoing requests. - * - * The method will return NoError if the get operation was successful. The contents of the reply can be retrieved - * by calling reply(). - * - * If an error was encountered then a specific ErrorCode will be returned, and a detailed error message - * can be retrieved by calling errorMessage(). - * - * \see get() - */ - ErrorCode post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh = false, QgsFeedback *feedback = nullptr ); - - /** - * Returns the error message string, after a get() or post() request has been made.\ - */ - QString errorMessage() const { return mErrorMessage; } - - /** - * Returns the content of the network reply, after a get() or post() request has been made. - */ - QgsNetworkReplyContent reply() const { return mReplyContent; } - - /** - * Returns the authentication config id which will be used during the request. - * \see setAuthCfg() - */ - QString authCfg() const; - - /** - * Sets the authentication config id which should be used during the request. - * \see authCfg() - */ - void setAuthCfg( const QString &authCfg ); - - public slots: - - /** - * Aborts the network request immediately. - */ - void abort(); - - signals: - - /** - * Emitted when when data arrives during a request. - */ - void downloadProgress( qint64, qint64 ); - - /** - * Emitted once a request has finished downloading. - */ - void downloadFinished(); - - private slots: - void replyProgress( qint64, qint64 ); - void replyFinished(); - void requestTimedOut( QNetworkReply *reply ); - - private : - - enum Method - { - Get, - Post - }; - - //! The reply to the request - QNetworkReply *mReply = nullptr; - - Method mMethod = Get; - QByteArray mPostData; - - //! Authentication configuration ID - QString mAuthCfg; - - //! The error message associated with the last error. - QString mErrorMessage; - - //! Error code - ErrorCode mErrorCode = NoError; - - QgsNetworkReplyContent mReplyContent; - - //! Whether the request is aborted. - bool mIsAborted = false; - - //! Whether to force refresh (i.e. issue a network request and not use cache) - bool mForceRefresh = false; - - //! Whether the request has timed-out - bool mTimedout = false; - - //! Whether we already received bytes - bool mGotNonEmptyResponse = false; - - int mExpirationSec = 30; - - QPointer< QgsFeedback > mFeedback; - - ErrorCode doRequest( Method method, QNetworkRequest &request, bool forceRefresh, QgsFeedback *feedback = nullptr ); - - QString errorMessageFailedAuth(); - -}; - -///@cond PRIVATE -#ifndef SIP_RUN - -class DownloaderThread : public QThread -{ - Q_OBJECT - - public: - DownloaderThread( const std::function &function, QObject *parent = nullptr ) - : QThread( parent ) - , mFunction( function ) - { - } - - void run() override - { - mFunction(); - } - - private: - std::function mFunction; -}; - -#endif -///@endcond - -#endif // QGSBLOCKINGNETWORKREQUEST_H diff --git a/src/core/qgsnetworkreply.cpp b/src/core/qgsnetworkreply.cpp deleted file mode 100644 index 3aef4427439..00000000000 --- a/src/core/qgsnetworkreply.cpp +++ /dev/null @@ -1,75 +0,0 @@ -/*************************************************************************** - qgsnetworkreply.cpp - ------------------- - begin : November 2018 - copyright : (C) 2018 by Nyall Dawson - email : nyall dot dawson at gmail dot com - *************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ - -#include "qgsnetworkreply.h" -#include - -QgsNetworkReplyContent::QgsNetworkReplyContent( QNetworkReply *reply ) - : mContent( reply->readAll() ) - , mError( reply->error() ) - , mErrorString( reply->errorString() ) - , mRawHeaderPairs( reply->rawHeaderPairs() ) -{ - int maxAttribute = static_cast< int >( QNetworkRequest::RedirectPolicyAttribute ); -#if QT_VERSION >= QT_VERSION_CHECK( 5, 11, 0 ) - maxAttribute = static_cast< int >( QNetworkRequest::Http2DirectAttribute ); -#endif - for ( int i = 0; i <= maxAttribute; ++i ) - { - if ( reply->attribute( static_cast< QNetworkRequest::Attribute>( i ) ).isValid() ) - mAttributes[ static_cast< QNetworkRequest::Attribute>( i ) ] = reply->attribute( static_cast< QNetworkRequest::Attribute>( i ) ); - } -} - -void QgsNetworkReplyContent::clear() -{ - *this = QgsNetworkReplyContent(); -} - -QVariant QgsNetworkReplyContent::attribute( QNetworkRequest::Attribute code ) const -{ - return mAttributes.value( code ); -} - -bool QgsNetworkReplyContent::hasRawHeader( const QByteArray &headerName ) const -{ - for ( auto &header : mRawHeaderPairs ) - { - if ( header.first == headerName ) - return true; - } - return false; -} - -QList QgsNetworkReplyContent::rawHeaderList() const -{ - QList< QByteArray > res; - res.reserve( mRawHeaderPairs.length() ); - for ( auto &header : mRawHeaderPairs ) - { - res << header.first; - } - return res; -} - -QByteArray QgsNetworkReplyContent::rawHeader( const QByteArray &headerName ) const -{ - for ( auto &header : mRawHeaderPairs ) - { - if ( header.first == headerName ) - return header.second; - } - return QByteArray(); -} diff --git a/src/core/qgsnetworkreply.h b/src/core/qgsnetworkreply.h deleted file mode 100644 index e233025125f..00000000000 --- a/src/core/qgsnetworkreply.h +++ /dev/null @@ -1,148 +0,0 @@ -/*************************************************************************** - qgsnetworkreply.h - ----------------- - begin : November 2018 - copyright : (C) 2018 by Nyall Dawson - email : nyall dot dawson at gmail dot com - *************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ -#ifndef QGSNETWORKREPLY_H -#define QGSNETWORKREPLY_H - -#include "qgis_core.h" - -#include - -/** - * Encapsulates a network reply within a container which is inexpensive to copy and safe to pass between threads. - * \ingroup core - * \since QGIS 3.6 - */ -class CORE_EXPORT QgsNetworkReplyContent -{ - public: - - /** - * Default constructor for an empty reply. - */ - QgsNetworkReplyContent() = default; - - /** - * Constructor for QgsNetworkReplyContent, populated from the specified \a reply. - */ - explicit QgsNetworkReplyContent( QNetworkReply *reply ); - - /** - * Clears the reply, resetting it back to a default, empty reply. - */ - void clear(); - - /** - * Returns the attribute associated with the \a code. If the attribute has not been set, it returns an - * invalid QVariant. - * - * You can expect the default values listed in QNetworkRequest::Attribute to be - * applied to the values returned by this function. - * - * \see attributes() - */ - QVariant attribute( QNetworkRequest::Attribute code ) const; - -#ifndef SIP_RUN - - /** - * Returns a list of valid attributes received in the reply. - * - * \see attribute() - * \note Not available in Python bindings - */ - QMap< QNetworkRequest::Attribute, QVariant > attributes() const { return mAttributes; } -#endif - - /** - * Returns the raw reply content. - */ - QByteArray content() const - { - return mContent; - } - - /** - * Returns the reply's error message, or QNetworkReply::NoError if no - * error was encountered. - * - * \see errorString() - */ - QNetworkReply::NetworkError error() const - { - return mError; - } - - /** - * Returns the error text for the reply, or an empty string if no - * error was encountered. - * - * \see error() - */ - QString errorString() const - { - return mErrorString; - } - -#ifndef SIP_RUN - typedef QPair RawHeaderPair; - - /** - * Returns the list of raw header pairs in the reply. - * \see hasRawHeader() - * \see rawHeaderList() - * \see rawHeader() - * \note Not available in Python bindings - */ - const QList &rawHeaderPairs() const - { - return mRawHeaderPairs; - } -#endif - - /** - * Returns true if the reply contains a header with the specified \a headerName. - * \see rawHeaderPairs() - * \see rawHeaderList() - * \see rawHeader() - */ - bool hasRawHeader( const QByteArray &headerName ) const; - - /** - * Returns a list of raw header names contained within the reply. - * \see rawHeaderPairs() - * \see hasRawHeader() - * \see rawHeader() - */ - QList rawHeaderList() const; - - /** - * Returns the content of the header with the specified \a headerName, or an - * empty QByteArray if the specified header was not found in the reply. - * \see rawHeaderPairs() - * \see hasRawHeader() - * \see rawHeaderList() - */ - QByteArray rawHeader( const QByteArray &headerName ) const; - - private: - - QByteArray mContent; - QNetworkReply::NetworkError mError = QNetworkReply::NoError; - QString mErrorString; - QList mRawHeaderPairs; - QMap< QNetworkRequest::Attribute, QVariant > mAttributes; -}; - -#endif // QGSNETWORKREPLY_H diff --git a/src/providers/arcgisrest/qgsafsprovider.cpp b/src/providers/arcgisrest/qgsafsprovider.cpp index 661917d277a..38ca8e154ae 100644 --- a/src/providers/arcgisrest/qgsafsprovider.cpp +++ b/src/providers/arcgisrest/qgsafsprovider.cpp @@ -21,6 +21,8 @@ #include "qgsdatasourceuri.h" #include "qgsafsdataitems.h" #include "qgslogger.h" +#include "geometry/qgsgeometry.h" +#include "qgsnetworkaccessmanager.h" #include "qgsdataitemprovider.h" #ifdef HAVE_GUI @@ -28,6 +30,12 @@ #include "qgssourceselectprovider.h" #endif +#include +#include +#include +#include + + static const QString TEXT_PROVIDER_KEY = QStringLiteral( "arcgisfeatureserver" ); static const QString TEXT_PROVIDER_DESCRIPTION = QStringLiteral( "ArcGIS Feature Server data provider" ); diff --git a/src/providers/arcgisrest/qgsarcgisrestutils.cpp b/src/providers/arcgisrest/qgsarcgisrestutils.cpp index c8ca14deea0..f8fe27034ae 100644 --- a/src/providers/arcgisrest/qgsarcgisrestutils.cpp +++ b/src/providers/arcgisrest/qgsarcgisrestutils.cpp @@ -42,8 +42,10 @@ #include "qgscategorizedsymbolrenderer.h" #include "qgsvectorlayerlabeling.h" +#include #include -#include "qgsblockingnetworkrequest.h" +#include +#include #include #include @@ -469,27 +471,57 @@ QList QgsArcGisRestUtils::getObjectIdsByExtent( const QString &layerurl QByteArray QgsArcGisRestUtils::queryService( const QUrl &u, const QString &authcfg, QString &errorTitle, QString &errorText, QgsFeedback *feedback ) { + QEventLoop loop; QUrl url = parseUrl( u ); QNetworkRequest request( url ); - QgsBlockingNetworkRequest networkRequest; - networkRequest.setAuthCfg( authcfg ); - const QgsBlockingNetworkRequest::ErrorCode error = networkRequest.get( request, false, feedback ); - if ( feedback && feedback->isCanceled() ) - return QByteArray(); - - // Handle network errors - if ( error != QgsBlockingNetworkRequest::NoError ) + if ( !authcfg.isEmpty() ) { - QgsDebugMsg( QStringLiteral( "Network error: %1" ).arg( networkRequest.errorMessage() ) ); - errorTitle = QStringLiteral( "Network error" ); - errorText = networkRequest.errorMessage(); - return QByteArray(); + QgsApplication::authManager()->updateNetworkRequest( request, authcfg ); } - const QgsNetworkReplyContent content = networkRequest.reply(); - return content.content(); + QNetworkReply *reply = nullptr; + QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance(); + + // Request data, handling redirects + while ( true ) + { + reply = nam->get( request ); + QObject::connect( reply, &QNetworkReply::finished, &loop, &QEventLoop::quit ); + if ( feedback ) + { + QObject::connect( feedback, &QgsFeedback::canceled, reply, &QNetworkReply::abort ); + } + + loop.exec( QEventLoop::ExcludeUserInputEvents ); + + reply->deleteLater(); + + if ( feedback && feedback->isCanceled() ) + return QByteArray(); + + // Handle network errors + if ( reply->error() != QNetworkReply::NoError ) + { + QgsDebugMsg( QStringLiteral( "Network error: %1" ).arg( reply->errorString() ) ); + errorTitle = QStringLiteral( "Network error" ); + errorText = reply->errorString(); + return QByteArray(); + } + + // Handle HTTP redirects + QVariant redirect = reply->attribute( QNetworkRequest::RedirectionTargetAttribute ); + if ( redirect.isNull() ) + { + break; + } + + QgsDebugMsg( "redirecting to " + redirect.toUrl().toString() ); + request.setUrl( redirect.toUrl() ); + } + QByteArray result = reply->readAll(); + return result; } QVariantMap QgsArcGisRestUtils::queryServiceJSON( const QUrl &url, const QString &authcfg, QString &errorTitle, QString &errorText, QgsFeedback *feedback ) diff --git a/tests/src/python/CMakeLists.txt b/tests/src/python/CMakeLists.txt index b9556907344..b0bd61f9ae9 100644 --- a/tests/src/python/CMakeLists.txt +++ b/tests/src/python/CMakeLists.txt @@ -23,7 +23,6 @@ ADD_PYTHON_TEST(PyQgsAuthenticationSystem test_qgsauthsystem.py) ADD_PYTHON_TEST(PyQgsBearingUtils test_qgsbearingutils.py) ADD_PYTHON_TEST(PyQgsBinaryWidget test_qgsbinarywidget.py) ADD_PYTHON_TEST(PyQgsBlendModes test_qgsblendmodes.py) -ADD_PYTHON_TEST(PyQgsBlockingNetworkRequest test_qgsblockingnetworkrequest.py) ADD_PYTHON_TEST(PyQgsBox3d test_qgsbox3d.py) ADD_PYTHON_TEST(PyQgsCategorizedSymbolRenderer test_qgscategorizedsymbolrenderer.py) ADD_PYTHON_TEST(PyQgsCheckableComboBox test_qgscheckablecombobox.py) diff --git a/tests/src/python/test_qgsblockingnetworkrequest.py b/tests/src/python/test_qgsblockingnetworkrequest.py deleted file mode 100644 index 47983d3c852..00000000000 --- a/tests/src/python/test_qgsblockingnetworkrequest.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- coding: utf-8 -*- -"""QGIS Unit tests for QgsBlockingNetworkRequest - -.. note:: This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. -""" - -from builtins import chr -from builtins import str -__author__ = 'Nyall Dawson' -__date__ = '12/11/2018' -__copyright__ = 'Copyright 2018, The QGIS Project' -# This will get replaced with a git SHA1 when you do a git archive -__revision__ = '$Format:%H$' - -import qgis # NOQA - -import os -from qgis.testing import unittest, start_app -from qgis.core import QgsBlockingNetworkRequest -from utilities import unitTestDataPath -from qgis.PyQt.QtCore import QUrl -from qgis.PyQt.QtTest import QSignalSpy -from qgis.PyQt.QtNetwork import QNetworkReply, QNetworkRequest -import socketserver -import threading -import http.server - -app = start_app() - - -class TestQgsBlockingNetworkRequest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - # Bring up a simple HTTP server - os.chdir(unitTestDataPath() + '') - handler = http.server.SimpleHTTPRequestHandler - - cls.httpd = socketserver.TCPServer(('localhost', 0), handler) - cls.port = cls.httpd.server_address[1] - - cls.httpd_thread = threading.Thread(target=cls.httpd.serve_forever) - cls.httpd_thread.setDaemon(True) - cls.httpd_thread.start() - - def testFetchEmptyUrl(self): - request = QgsBlockingNetworkRequest() - spy = QSignalSpy(request.downloadFinished) - err = request.get(QNetworkRequest(QUrl())) - self.assertEqual(len(spy), 1) - self.assertEqual(err, QgsBlockingNetworkRequest.ServerExceptionError) - self.assertEqual(request.errorMessage(), 'Protocol "" is unknown') - reply = request.reply() - self.assertFalse(reply.content()) - - def testFetchBadUrl(self): - request = QgsBlockingNetworkRequest() - spy = QSignalSpy(request.downloadFinished) - err = request.get(QNetworkRequest(QUrl('http://x'))) - self.assertEqual(len(spy), 1) - self.assertEqual(err, QgsBlockingNetworkRequest.ServerExceptionError) - self.assertEqual(request.errorMessage(), 'Host x not found') - reply = request.reply() - self.assertFalse(reply.content()) - - def testFetchBadUrl2(self): - request = QgsBlockingNetworkRequest() - spy = QSignalSpy(request.downloadFinished) - err = request.get(QNetworkRequest(QUrl('http://localhost:' + str(TestQgsBlockingNetworkRequest.port) + '/ffff'))) - self.assertEqual(len(spy), 1) - self.assertEqual(err, QgsBlockingNetworkRequest.ServerExceptionError) - self.assertIn('File not found', request.errorMessage()) - reply = request.reply() - self.assertFalse(reply.content()) - self.assertEqual(reply.rawHeaderList(), []) - - def testGet(self): - request = QgsBlockingNetworkRequest() - spy = QSignalSpy(request.downloadFinished) - err = request.get(QNetworkRequest(QUrl('http://localhost:' + str(TestQgsBlockingNetworkRequest.port) + '/qgis_local_server/index.html'))) - self.assertEqual(len(spy), 1) - self.assertEqual(err, QgsBlockingNetworkRequest.NoError) - self.assertEqual(request.errorMessage(), '') - reply = request.reply() - self.assertEqual(reply.error(), QNetworkReply.NoError) - self.assertEqual(reply.content(), '\n\n\n\t\n\tLocal QGIS Server Default Index\n\n\n

Web Server Working

\n\n\n') - self.assertEqual(reply.rawHeaderList(), [b'Server', - b'Date', - b'Content-type', - b'Content-Length', - b'Last-Modified']) - self.assertEqual(reply.rawHeader(b'Content-type'), 'text/html') - self.assertEqual(reply.rawHeader(b'xxxxxxxxx'), '') - self.assertEqual(reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), 200) - self.assertEqual(reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute), 'OK') - self.assertEqual(reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), 200) - self.assertEqual(reply.attribute(QNetworkRequest.RedirectionTargetAttribute), None) - - -if __name__ == "__main__": - unittest.main()