sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-10-15 00:02:52 -04:00
Code Issues Packages Projects Releases Wiki Activity

[FEATURE][API] New class for blocking (non-async) network requests

Browse Source 
This new class, QgsBlockingNetworkRequest, is designed for
performing SAFE blocking requests. It is thread safe and
has full support for QGIS proxy and authentication settings.

This class should be used whenever a blocking network
request is required. Unlike implementations
which rely on QApplication::processEvents() or creation of a
QEventLoop, this class is completely
thread safe and can be used on either the main thread or
background threads without issue.

Redirects are automatically handled by the class.

After completion of a request, the reply content should be
retrieved by calling getReplyContent().
This method returns a QgsNetworkReplyContent container,
which is safe and cheap to copy and pass
between threads without issue.

The guts of this class have been copied from QgsWfsRequest (which
has been using the same approach since 3.2)
This commit is contained in:
Nyall Dawson 2018-11-08 17:14:29 +10:00
parent e4959a6b9a
commit 1774e68f39
5 changed files with 740 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
python/core/auto_generated/qgsblockingnetworkrequest.sip.in Normal file
View File

@ -0,0 +1,142 @@
/************************************************************************
* This file has been generated automatically from *
* *
* src/core/qgsblockingnetworkrequest.h *
* *
* Do not edit manually ! Edit header and run scripts/sipify.pl again *
************************************************************************/
class QgsBlockingNetworkRequest : QObject
{
%Docstring
A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy
and authentication settings.
This class should be used whenever a blocking network request is required. Unlike implementations
which rely on QApplication.processEvents() or creation of a QEventLoop, this class is completely
thread safe and can be used on either the main thread or background threads without issue.
Redirects are automatically handled by the class.
After completion of a request, the reply content should be retrieved by calling getReplyContent().
This method returns a QgsNetworkReplyContent container, which is safe and cheap to copy and pass
between threads without issue.
.. versionadded:: 3.6
%End
%TypeHeaderCode
#include "qgsblockingnetworkrequest.h"
%End
public:
enum ErrorCode
{
NoError,
NetworkError,
TimeoutError,
ServerExceptionError,
};
explicit QgsBlockingNetworkRequest();
%Docstring
Constructor for QgsBlockingNetworkRequest
%End
~QgsBlockingNetworkRequest();
ErrorCode get( QNetworkRequest &request, bool forceRefresh = false );
%Docstring
Performs a "get" operation on the specified ``request``.
If ``forceRefresh`` is false then previously cached replies may be used for the request. If
it is set to true then a new query is always performed.
If an authCfg() has been set, then any authentication configuration required will automatically be applied to
``request``. There is no need to manually apply the authentication to the request prior to calling
this method.
The method will return NoError if the get operation was successful. The contents of the reply can be retrieved
by calling reply().
If an error was encountered then a specific ErrorCode will be returned, and a detailed error message
can be retrieved by calling errorMessage().
.. seealso:: :py:func:`post`
%End
ErrorCode post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh = false );
%Docstring
Performs a "post" operation on the specified ``request``, using the given ``data``.
If ``forceRefresh`` is false then previously cached replies may be used for the request. If
it is set to true then a new query is always performed.
If an authCfg() has been set, then any authentication configuration required will automatically be applied to
``request``. There is no need to manually apply the authentication to the request prior to calling
this method.
The method will return NoError if the get operation was successful. The contents of the reply can be retrieved
by calling reply().
If an error was encountered then a specific ErrorCode will be returned, and a detailed error message
can be retrieved by calling errorMessage().
.. seealso:: :py:func:`get`
%End
QString errorMessage() const;
%Docstring
Returns the error message string, after a get() or post() request has been made.\
%End
QgsNetworkReplyContent reply() const;
%Docstring
Returns the content of the network reply, after a get() or post() request has been made.
%End
QString authCfg() const;
%Docstring
Returns the authentication config id which will be used during the request.
.. seealso:: :py:func:`setAuthCfg`
%End
void setAuthCfg( const QString &authCfg );
%Docstring
Sets the authentication config id which should be used during the request.
.. seealso:: :py:func:`authCfg`
%End
public slots:
void abort();
%Docstring
Aborts the network request immediately.
%End
signals:
void downloadProgress( qint64, qint64 );
%Docstring
Emitted when when data arrives during a request.
%End
void downloadFinished();
%Docstring
Emitted once a request has finished downloading.
%End
};
/************************************************************************
* This file has been generated automatically from *
* *
* src/core/qgsblockingnetworkrequest.h *
* *
* Do not edit manually ! Edit header and run scripts/sipify.pl again *
************************************************************************/

1
python/core/core_auto.sip
View File

@ -313,6 +313,7 @@
%Include auto_generated/qgsactionscoperegistry.sip
%Include auto_generated/qgsanimatedicon.sip
%Include auto_generated/qgsauxiliarystorage.sip
%Include auto_generated/qgsblockingnetworkrequest.sip
%Include auto_generated/qgsbrowsermodel.sip
%Include auto_generated/qgsbrowserproxymodel.sip
%Include auto_generated/qgscoordinatereferencesystem.sip

2
src/core/CMakeLists.txt
View File

@ -152,6 +152,7 @@ SET(QGIS_CORE_SRCS
qgsattributeeditorelement.cpp
qgsauxiliarystorage.cpp
qgsbearingutils.cpp
qgsblockingnetworkrequest.cpp
qgsbrowsermodel.cpp
qgsbrowserproxymodel.cpp
qgscachedfeatureiterator.cpp
@ -595,6 +596,7 @@ SET(QGIS_CORE_MOC_HDRS
qgsactionscoperegistry.h
qgsanimatedicon.h
qgsauxiliarystorage.h
qgsblockingnetworkrequest.h
qgsbrowsermodel.h
qgsbrowserproxymodel.h
qgscoordinatereferencesystem.h

375
src/core/qgsblockingnetworkrequest.cpp Normal file
View File

@ -0,0 +1,375 @@
/***************************************************************************
qgsblockingnetworkrequest.cpp
-----------------------------
begin : November 2018
copyright : (C) 2018 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include "qgsblockingnetworkrequest.h"
#include "qgslogger.h"
#include "qgsapplication.h"
#include "qgsnetworkaccessmanager.h"
#include "qgsauthmanager.h"
#include "qgsmessagelog.h"
#include <QUrl>
#include <QNetworkRequest>
#include <QNetworkReply>
#include <QMutex>
#include <QWaitCondition>
#include <QNetworkCacheMetaData>
#include <QAuthenticator>
const qint64 READ_BUFFER_SIZE_HINT = 1024 * 1024;
QgsBlockingNetworkRequest::QgsBlockingNetworkRequest()
{
connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::requestTimedOut, this, &QgsBlockingNetworkRequest::requestTimedOut );
}
QgsBlockingNetworkRequest::~QgsBlockingNetworkRequest()
{
abort();
}
void QgsBlockingNetworkRequest::requestTimedOut( QNetworkReply *reply )
{
if ( reply == mReply )
mTimedout = true;
}
QString QgsBlockingNetworkRequest::authCfg() const
{
return mAuthCfg;
}
void QgsBlockingNetworkRequest::setAuthCfg( const QString &authCfg )
{
mAuthCfg = authCfg;
}
QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::get( QNetworkRequest &request, bool forceRefresh )
{
return doRequest( Get, request, forceRefresh );
}
QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh )
{
mPostData = data;
return doRequest( Post, request, forceRefresh );
}
QgsBlockingNetworkRequest::ErrorCode QgsBlockingNetworkRequest::doRequest( QgsBlockingNetworkRequest::Method method, QNetworkRequest &request, bool forceRefresh )
{
mMethod = method;
abort(); // cancel previous
mIsAborted = false;
mTimedout = false;
mGotNonEmptyResponse = false;
mErrorMessage.clear();
mErrorCode = NoError;
mForceRefresh = forceRefresh;
mReplyContent.clear();
if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) )
{
mErrorCode = NetworkError;
mErrorMessage = errorMessageFailedAuth();
QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
return NetworkError;
}
QgsDebugMsgLevel( QStringLiteral( "Calling: %1" ).arg( request.url().toString() ), 2 );
request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, forceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
QWaitCondition waitCondition;
QMutex waitConditionMutex;
bool threadFinished = false;
bool success = false;
std::function<void()> downloaderFunction = [ this, request, &waitConditionMutex, &waitCondition, &threadFinished, &success ]()
{
if ( QThread::currentThread() != QgsApplication::instance()->thread() )
QgsNetworkAccessManager::instance( Qt::DirectConnection );
success = true;
switch ( mMethod )
{
case Get:
mReply = QgsNetworkAccessManager::instance()->get( request );
break;
case Post:
mReply = QgsNetworkAccessManager::instance()->post( request, mPostData );
break;
};
mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) )
{
mErrorCode = NetworkError;
mErrorMessage = errorMessageFailedAuth();
QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
waitCondition.wakeAll();
success = false;
}
else
{
// We are able to use direct connection here, because we
// * either run on the thread mReply lives in, so DirectConnection is standard and safe anyway
// * or the owner thread of mReply is currently not doing anything because it's blocked in future.waitForFinished() (if it is the main thread)
connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection );
connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
auto resumeMainThread = [&waitConditionMutex, &waitCondition]()
{
waitConditionMutex.lock();
waitCondition.wakeAll();
waitConditionMutex.unlock();
waitConditionMutex.lock();
waitCondition.wait( &waitConditionMutex );
waitConditionMutex.unlock();
};
connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::authenticationRequired, this, resumeMainThread, Qt::DirectConnection );
connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::proxyAuthenticationRequired, this, resumeMainThread, Qt::DirectConnection );
#ifndef QT_NO_SSL
connect( QgsNetworkAccessManager::instance(), &QgsNetworkAccessManager::sslErrors, this, resumeMainThread, Qt::DirectConnection );
#endif
QEventLoop loop;
connect( this, &QgsBlockingNetworkRequest::downloadFinished, &loop, &QEventLoop::quit, Qt::DirectConnection );
loop.exec();
}
waitConditionMutex.lock();
threadFinished = true;
waitCondition.wakeAll();
waitConditionMutex.unlock();
};
if ( QThread::currentThread() == QApplication::instance()->thread() )
{
std::unique_ptr<DownloaderThread> downloaderThread = qgis::make_unique<DownloaderThread>( downloaderFunction );
downloaderThread->start();
while ( true )
{
waitConditionMutex.lock();
if ( threadFinished )
{
waitConditionMutex.unlock();
break;
}
waitCondition.wait( &waitConditionMutex );
// If the downloader thread wakes us (the main thread) up and is not yet finished
// he needs the authentication to run.
// The processEvents() call gives the auth manager the chance to show a dialog and
// once done with that, we can wake the downloaderThread again and continue the download.
if ( !threadFinished )
{
waitConditionMutex.unlock();
QgsApplication::instance()->processEvents();
waitConditionMutex.lock();
waitCondition.wakeAll();
waitConditionMutex.unlock();
}
else
{
waitConditionMutex.unlock();
}
}
// wait for thread to gracefully exit
downloaderThread->wait();
}
else
{
downloaderFunction();
}
return mErrorCode;
}
void QgsBlockingNetworkRequest::abort()
{
mIsAborted = true;
if ( mReply )
{
mReply->deleteLater();
mReply = nullptr;
}
}
void QgsBlockingNetworkRequest::replyProgress( qint64 bytesReceived, qint64 bytesTotal )
{
QgsDebugMsgLevel( QStringLiteral( "%1 of %2 bytes downloaded." ).arg( bytesReceived ).arg( bytesTotal < 0 ? QStringLiteral( "unknown number of" ) : QString::number( bytesTotal ) ), 2 );
if ( bytesReceived != 0 )
mGotNonEmptyResponse = true;
if ( !mIsAborted && mReply )
{
if ( mReply->error() == QNetworkReply::NoError )
{
QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
if ( !redirect.isNull() )
{
// We don't want to emit downloadProgress() for a redirect
return;
}
}
}
emit downloadProgress( bytesReceived, bytesTotal );
}
void QgsBlockingNetworkRequest::replyFinished()
{
if ( !mIsAborted && mReply )
{
if ( mReply->error() == QNetworkReply::NoError )
{
QgsDebugMsgLevel( QStringLiteral( "reply OK" ), 2 );
QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
if ( !redirect.isNull() )
{
QgsDebugMsgLevel( QStringLiteral( "Request redirected." ), 2 );
const QUrl &toUrl = redirect.toUrl();
mReply->request();
if ( toUrl == mReply->url() )
{
mErrorMessage = tr( "Redirect loop detected: %1" ).arg( toUrl.toString() );
QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
mReplyContent.clear();
}
else
{
QNetworkRequest request( toUrl );
if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkRequest( request, mAuthCfg ) )
{
mReplyContent.clear();
mErrorMessage = errorMessageFailedAuth();
mErrorCode = NetworkError;
QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
emit downloadFinished();
return;
}
request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, mForceRefresh ? QNetworkRequest::AlwaysNetwork : QNetworkRequest::PreferCache );
request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
mReply->deleteLater();
mReply = nullptr;
QgsDebugMsgLevel( QStringLiteral( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ), 2 );
switch ( mMethod )
{
case Get:
mReply = QgsNetworkAccessManager::instance()->get( request );
break;
case Post:
mReply = QgsNetworkAccessManager::instance()->post( request, mPostData );
break;
};
mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
if ( !mAuthCfg.isEmpty() && !QgsApplication::authManager()->updateNetworkReply( mReply, mAuthCfg ) )
{
mReplyContent.clear();
mErrorMessage = errorMessageFailedAuth();
mErrorCode = NetworkError;
QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
emit downloadFinished();
return;
}
connect( mReply, &QNetworkReply::finished, this, &QgsBlockingNetworkRequest::replyFinished, Qt::DirectConnection );
connect( mReply, &QNetworkReply::downloadProgress, this, &QgsBlockingNetworkRequest::replyProgress, Qt::DirectConnection );
return;
}
}
else
{
const QgsNetworkAccessManager *nam = QgsNetworkAccessManager::instance();
if ( nam->cache() )
{
QNetworkCacheMetaData cmd = nam->cache()->metaData( mReply->request().url() );
QNetworkCacheMetaData::RawHeaderList hl;
Q_FOREACH ( const QNetworkCacheMetaData::RawHeader &h, cmd.rawHeaders() )
{
if ( h.first != "Cache-Control" )
hl.append( h );
}
cmd.setRawHeaders( hl );
QgsDebugMsgLevel( QStringLiteral( "expirationDate:%1" ).arg( cmd.expirationDate().toString() ), 2 );
if ( cmd.expirationDate().isNull() )
{
cmd.setExpirationDate( QDateTime::currentDateTime().addSecs( mExpirationSec ) );
}
nam->cache()->updateMetaData( cmd );
}
else
{
QgsDebugMsgLevel( QStringLiteral( "No cache!" ), 2 );
}
#ifdef QGISDEBUG
bool fromCache = mReply->attribute( QNetworkRequest::SourceIsFromCacheAttribute ).toBool();
QgsDebugMsgLevel( QStringLiteral( "Reply was cached: %1" ).arg( fromCache ), 2 );
#endif
mReplyContent = QgsNetworkReplyContent( mReply );
if ( mReplyContent.content().isEmpty() && !mGotNonEmptyResponse )
{
mErrorMessage = tr( "empty response: %1" ).arg( mReply->errorString() );
mErrorCode = ServerExceptionError;
QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
}
}
}
else
{
mErrorMessage = mReply->errorString();
mErrorCode = ServerExceptionError;
QgsMessageLog::logMessage( mErrorMessage, tr( "Network" ) );
mReplyContent.clear();
}
}
if ( mTimedout )
mErrorCode = TimeoutError;
if ( mReply )
{
mReply->deleteLater();
mReply = nullptr;
}
emit downloadFinished();
}
QString QgsBlockingNetworkRequest::errorMessageFailedAuth()
{
return tr( "network request update failed for authentication config" );
}

220
src/core/qgsblockingnetworkrequest.h Normal file
View File

@ -0,0 +1,220 @@
/***************************************************************************
qgsblockingnetworkrequest.h
---------------------------
begin : November 2018
copyright : (C) 2018 by Nyall Dawson
email : nyall dot dawson at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSBLOCKINGNETWORKREQUEST_H
#define QGSBLOCKINGNETWORKREQUEST_H
#include "qgis_core.h"
#include "qgsnetworkreply.h"
#include <QThread>
#include <QObject>
#include <functional>
class QNetworkRequest;
class QNetworkReply;
/**
* A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy
* and authentication settings.
*
* This class should be used whenever a blocking network request is required. Unlike implementations
* which rely on QApplication::processEvents() or creation of a QEventLoop, this class is completely
* thread safe and can be used on either the main thread or background threads without issue.
*
* Redirects are automatically handled by the class.
*
* After completion of a request, the reply content should be retrieved by calling getReplyContent().
* This method returns a QgsNetworkReplyContent container, which is safe and cheap to copy and pass
* between threads without issue.
*
* \ingroup core
* \since QGIS 3.6
*/
class CORE_EXPORT QgsBlockingNetworkRequest : public QObject
{
Q_OBJECT
public:
//! Error codes
enum ErrorCode
{
NoError, //!< No error was encountered
NetworkError, //!< A network error occurred
TimeoutError, //!< Timeout was reached before a reply was received
ServerExceptionError, //!< An exception was raised by the server
};
//! Constructor for QgsBlockingNetworkRequest
explicit QgsBlockingNetworkRequest();
~QgsBlockingNetworkRequest() override;
/**
* Performs a "get" operation on the specified \a request.
*
* If \a forceRefresh is false then previously cached replies may be used for the request. If
* it is set to true then a new query is always performed.
*
* If an authCfg() has been set, then any authentication configuration required will automatically be applied to
* \a request. There is no need to manually apply the authentication to the request prior to calling
* this method.
*
* The method will return NoError if the get operation was successful. The contents of the reply can be retrieved
* by calling reply().
*
* If an error was encountered then a specific ErrorCode will be returned, and a detailed error message
* can be retrieved by calling errorMessage().
*
* \see post()
*/
ErrorCode get( QNetworkRequest &request, bool forceRefresh = false );
/**
* Performs a "post" operation on the specified \a request, using the given \a data.
*
* If \a forceRefresh is false then previously cached replies may be used for the request. If
* it is set to true then a new query is always performed.
*
* If an authCfg() has been set, then any authentication configuration required will automatically be applied to
* \a request. There is no need to manually apply the authentication to the request prior to calling
* this method.
*
* The method will return NoError if the get operation was successful. The contents of the reply can be retrieved
* by calling reply().
*
* If an error was encountered then a specific ErrorCode will be returned, and a detailed error message
* can be retrieved by calling errorMessage().
*
* \see get()
*/
ErrorCode post( QNetworkRequest &request, const QByteArray &data, bool forceRefresh = false );
/**
* Returns the error message string, after a get() or post() request has been made.\
*/
QString errorMessage() const { return mErrorMessage; }
/**
* Returns the content of the network reply, after a get() or post() request has been made.
*/
QgsNetworkReplyContent reply() const { return mReplyContent; }
/**
* Returns the authentication config id which will be used during the request.
* \see setAuthCfg()
*/
QString authCfg() const;
/**
* Sets the authentication config id which should be used during the request.
* \see authCfg()
*/
void setAuthCfg( const QString &authCfg );
public slots:
/**
* Aborts the network request immediately.
*/
void abort();
signals:
/**
* Emitted when when data arrives during a request.
*/
void downloadProgress( qint64, qint64 );
/**
* Emitted once a request has finished downloading.
*/
void downloadFinished();
private slots:
void replyProgress( qint64, qint64 );
void replyFinished();
void requestTimedOut( QNetworkReply *reply );
private :
enum Method
{
Get,
Post
};
//! The reply to the request
QNetworkReply *mReply = nullptr;
Method mMethod = Get;
QByteArray mPostData;
//! Authentication configuration ID
QString mAuthCfg;
//! The error message associated with the last error.
QString mErrorMessage;
//! Error code
ErrorCode mErrorCode = NoError;
QgsNetworkReplyContent mReplyContent;
//! Whether the request is aborted.
bool mIsAborted = false;
//! Whether to force refresh (i.e. issue a network request and not use cache)
bool mForceRefresh = false;
//! Whether the request has timed-out
bool mTimedout = false;
//! Whether we already received bytes
bool mGotNonEmptyResponse = false;
int mExpirationSec = 30;
ErrorCode doRequest( Method method, QNetworkRequest &request, bool forceRefresh );
QString errorMessageFailedAuth();
};
///@cond PRIVATE
#ifndef SIP_RUN
class DownloaderThread : public QThread
{
Q_OBJECT
public:
DownloaderThread( const std::function<void()> &function, QObject *parent = nullptr )
: QThread( parent )
, mFunction( function )
{
}
void run() override
{
mFunction();
}
private:
std::function<void()> mFunction;
};
#endif
///@endcond
#endif // QGSBLOCKINGNETWORKREQUEST_H