Merge pull request #7170 from rouault/wfs_pagesize

[WFS provider] [FEATURE] Allow user to enable/disable paging and specify page size (fixes #18935)
This commit is contained in:
Even Rouault 2018-06-22 14:57:37 +02:00 committed by GitHub
commit 14a913e25b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 645 additions and 60 deletions

View File

@ -92,6 +92,11 @@ Returns the "test connection" button.
.. versionadded:: 3.0
%End
virtual QString wfsSettingsKey( const QString &base, const QString &connectionName ) const;
%Docstring
Returns the QSettings key for WFS related settings for the connection.

View File

@ -62,10 +62,16 @@ QgsNewHttpConnection::QgsNewHttpConnection( QWidget *parent, ConnectionTypes typ
cmbDpiMode->addItem( tr( "GeoServer" ) );
cmbVersion->clear();
cmbVersion->addItem( tr( "Auto-detect" ) );
cmbVersion->addItem( tr( "Maximum" ) );
cmbVersion->addItem( tr( "1.0" ) );
cmbVersion->addItem( tr( "1.1" ) );
cmbVersion->addItem( tr( "2.0" ) );
connect( cmbVersion,
static_cast<void ( QComboBox::* )( int )>( &QComboBox::currentIndexChanged ),
this, &QgsNewHttpConnection::wfsVersionCurrentIndexChanged );
connect( cbxWfsFeaturePaging, &QCheckBox::stateChanged,
this, &QgsNewHttpConnection::wfsFeaturePagingStateChanged );
if ( !connectionName.isEmpty() )
{
@ -86,6 +92,7 @@ QgsNewHttpConnection::QgsNewHttpConnection( QWidget *parent, ConnectionTypes typ
mAuthSettings->setPassword( settings.value( credentialsKey + "/password" ).toString() );
mAuthSettings->setConfigId( settings.value( credentialsKey + "/authcfg" ).toString() );
}
mWfsVersionDetectButton->setDisabled( txtUrl->text().isEmpty() );
if ( !( mTypes & ConnectionWms ) && !( mTypes & ConnectionWcs ) )
{
@ -148,6 +155,20 @@ QgsNewHttpConnection::QgsNewHttpConnection( QWidget *parent, ConnectionTypes typ
nameChanged( connectionName );
}
void QgsNewHttpConnection::wfsVersionCurrentIndexChanged( int index )
{
cbxWfsFeaturePaging->setEnabled( index == 0 || index == 3 );
lblPageSize->setEnabled( index == 0 || index == 3 );
txtPageSize->setEnabled( index == 0 || index == 3 );
cbxWfsIgnoreAxisOrientation->setEnabled( index != 1 );
}
void QgsNewHttpConnection::wfsFeaturePagingStateChanged( int state )
{
lblPageSize->setEnabled( state == Qt::Checked );
txtPageSize->setEnabled( state == Qt::Checked );
}
QString QgsNewHttpConnection::name() const
{
return txtName->text();
@ -168,6 +189,7 @@ void QgsNewHttpConnection::urlChanged( const QString &text )
{
Q_UNUSED( text );
buttonBox->button( QDialogButtonBox::Ok )->setDisabled( txtName->text().isEmpty() || txtUrl->text().isEmpty() );
mWfsVersionDetectButton->setDisabled( txtUrl->text().isEmpty() );
}
void QgsNewHttpConnection::updateOkButtonState()
@ -209,6 +231,26 @@ QPushButton *QgsNewHttpConnection::testConnectButton()
return mTestConnectionButton;
}
QPushButton *QgsNewHttpConnection::wfsVersionDetectButton()
{
return mWfsVersionDetectButton;
}
QComboBox *QgsNewHttpConnection::wfsVersionComboBox()
{
return cmbVersion;
}
QCheckBox *QgsNewHttpConnection::wfsPagingEnabledCheckBox()
{
return cbxWfsFeaturePaging;
}
QLineEdit *QgsNewHttpConnection::wfsPageSizeLineEdit()
{
return txtPageSize;
}
QString QgsNewHttpConnection::wfsSettingsKey( const QString &base, const QString &connectionName ) const
{
return base + connectionName;
@ -266,24 +308,18 @@ void QgsNewHttpConnection::updateServiceSpecificSettings()
txtReferer->setText( settings.value( wmsKey + "/referer" ).toString() );
txtMaxNumFeatures->setText( settings.value( wfsKey + "/maxnumfeatures" ).toString() );
bool pagingEnabled = settings.value( wfsKey + "/pagingenabled", true ).toBool();
txtPageSize->setText( settings.value( wfsKey + "/pagesize" ).toString() );
cbxWfsFeaturePaging->setChecked( pagingEnabled );
txtPageSize->setEnabled( pagingEnabled );
lblPageSize->setEnabled( pagingEnabled );
cbxWfsFeaturePaging->setEnabled( pagingEnabled );
}
void QgsNewHttpConnection::accept()
QUrl QgsNewHttpConnection::urlTrimmed() const
{
QgsSettings settings;
QString key = mBaseKey + txtName->text();
QString credentialsKey = "qgis/" + mCredentialsBaseKey + '/' + txtName->text();
if ( !validate() )
return;
// on rename delete original entry first
if ( !mOriginalConnName.isNull() && mOriginalConnName != key )
{
settings.remove( mBaseKey + mOriginalConnName );
settings.remove( "qgis/" + mCredentialsBaseKey + '/' + mOriginalConnName );
settings.sync();
}
QUrl url( txtUrl->text().trimmed() );
const QList< QPair<QByteArray, QByteArray> > &items = url.encodedQueryItems();
@ -306,7 +342,27 @@ void QgsNewHttpConnection::accept()
{
url.setEncodedPath( "/" );
}
return url;
}
void QgsNewHttpConnection::accept()
{
QgsSettings settings;
QString key = mBaseKey + txtName->text();
QString credentialsKey = "qgis/" + mCredentialsBaseKey + '/' + txtName->text();
if ( !validate() )
return;
// on rename delete original entry first
if ( !mOriginalConnName.isNull() && mOriginalConnName != key )
{
settings.remove( mBaseKey + mOriginalConnName );
settings.remove( "qgis/" + mCredentialsBaseKey + '/' + mOriginalConnName );
settings.sync();
}
QUrl url( urlTrimmed() );
settings.setValue( key + "/url", url.toString() );
QString wfsKey = wfsSettingsKey( mBaseKey, txtName->text() );
@ -374,6 +430,9 @@ void QgsNewHttpConnection::accept()
settings.setValue( wfsKey + "/version", version );
settings.setValue( wfsKey + "/maxnumfeatures", txtMaxNumFeatures->text() );
settings.setValue( wfsKey + "/pagesize", txtPageSize->text() );
settings.setValue( wfsKey + "/pagingenabled", cbxWfsFeaturePaging->isChecked() );
}
settings.setValue( credentialsKey + "/username", mAuthSettings->username() );

View File

@ -97,6 +97,8 @@ class GUI_EXPORT QgsNewHttpConnection : public QDialog, private Ui::QgsNewHttpCo
void nameChanged( const QString & );
void urlChanged( const QString & );
void updateOkButtonState();
void wfsVersionCurrentIndexChanged( int index );
void wfsFeaturePagingStateChanged( int state );
protected:
@ -113,6 +115,36 @@ class GUI_EXPORT QgsNewHttpConnection : public QDialog, private Ui::QgsNewHttpCo
*/
QPushButton *testConnectButton();
/**
* Returns the "WFS version detect" button.
* \since QGIS 3.2
*/
QPushButton *wfsVersionDetectButton() SIP_SKIP;
/**
* Returns the "WFS version" combobox.
* \since QGIS 3.2
*/
QComboBox *wfsVersionComboBox() SIP_SKIP;
/**
* Returns the "WFS paging enabled" checkbox
* \since QGIS 3.2
*/
QCheckBox *wfsPagingEnabledCheckBox() SIP_SKIP;
/**
* Returns the "WFS page size" edit
* \since QGIS 3.2
*/
QLineEdit *wfsPageSizeLineEdit() SIP_SKIP;
/**
* Returns the url.
* \since QGIS 3.2
*/
QUrl urlTrimmed() const SIP_SKIP;
/**
* Returns the QSettings key for WFS related settings for the connection.
* \see wmsSettingsKey()

View File

@ -33,9 +33,11 @@ SET (WFS_MOC_HDRS
IF (WITH_GUI)
SET(WFS_SRCS ${WFS_SRCS}
qgswfssourceselect.cpp
qgswfsnewconnection.cpp
)
SET(WFS_MOC_HDRS ${WFS_MOC_HDRS}
qgswfssourceselect.h
qgswfsnewconnection.h
)
ENDIF ()

View File

@ -39,6 +39,20 @@ QgsWfsConnection::QgsWfsConnection( const QString &connName )
mUri.setParam( QgsWFSConstants::URI_PARAM_MAXNUMFEATURES, maxnumfeatures );
}
const QString &pagesize = settings.value( key + "/" + QgsWFSConstants::SETTINGS_PAGE_SIZE ).toString();
if ( !pagesize.isEmpty() )
{
mUri.removeParam( QgsWFSConstants::URI_PARAM_PAGE_SIZE ); // setParam allow for duplicates!
mUri.setParam( QgsWFSConstants::URI_PARAM_PAGE_SIZE, pagesize );
}
if ( settings.contains( key + "/" + QgsWFSConstants::SETTINGS_PAGING_ENABLED ) )
{
mUri.removeParam( QgsWFSConstants::URI_PARAM_PAGING_ENABLED ); // setParam allow for duplicates!
mUri.setParam( QgsWFSConstants::URI_PARAM_PAGING_ENABLED,
settings.value( key + "/" + QgsWFSConstants::SETTINGS_PAGING_ENABLED, true ).toBool() ? "true" : "false" );
}
QgsDebugMsg( QString( "WFS full uri: '%1'." ).arg( QString( mUri.uri() ) ) );
}

View File

@ -38,12 +38,16 @@ const QString QgsWFSConstants::URI_PARAM_IGNOREAXISORIENTATION( QStringLiteral(
const QString QgsWFSConstants::URI_PARAM_INVERTAXISORIENTATION( QStringLiteral( "InvertAxisOrientation" ) );
const QString QgsWFSConstants::URI_PARAM_VALIDATESQLFUNCTIONS( QStringLiteral( "validateSQLFunctions" ) );
const QString QgsWFSConstants::URI_PARAM_HIDEDOWNLOADPROGRESSDIALOG( QStringLiteral( "hideDownloadProgressDialog" ) );
const QString QgsWFSConstants::URI_PARAM_PAGING_ENABLED( "pagingEnabled" );
const QString QgsWFSConstants::URI_PARAM_PAGE_SIZE( "pageSize" );
const QString QgsWFSConstants::VERSION_AUTO( QStringLiteral( "auto" ) );
const QString QgsWFSConstants::CONNECTIONS_WFS( QStringLiteral( "qgis/connections-wfs/" ) );
const QString QgsWFSConstants::SETTINGS_VERSION( QStringLiteral( "version" ) );
const QString QgsWFSConstants::SETTINGS_MAXNUMFEATURES( QStringLiteral( "maxnumfeatures" ) );
const QString QgsWFSConstants::SETTINGS_PAGING_ENABLED( QStringLiteral( "pagingenabled" ) );
const QString QgsWFSConstants::SETTINGS_PAGE_SIZE( QStringLiteral( "pagesize" ) );
const QString QgsWFSConstants::FIELD_GEN_COUNTER( QStringLiteral( "__qgis_gen_counter" ) );
const QString QgsWFSConstants::FIELD_GMLID( QStringLiteral( "__qgis_gmlid" ) );

View File

@ -46,6 +46,8 @@ struct QgsWFSConstants
static const QString URI_PARAM_INVERTAXISORIENTATION;
static const QString URI_PARAM_VALIDATESQLFUNCTIONS;
static const QString URI_PARAM_HIDEDOWNLOADPROGRESSDIALOG;
static const QString URI_PARAM_PAGING_ENABLED;
static const QString URI_PARAM_PAGE_SIZE;
//
static const QString VERSION_AUTO;
@ -54,6 +56,8 @@ struct QgsWFSConstants
static const QString CONNECTIONS_WFS;
static const QString SETTINGS_VERSION;
static const QString SETTINGS_MAXNUMFEATURES;
static const QString SETTINGS_PAGING_ENABLED;
static const QString SETTINGS_PAGE_SIZE;
// Special fields of the cache
static const QString FIELD_GEN_COUNTER;

View File

@ -207,6 +207,20 @@ void QgsWFSDataSourceURI::setMaxNumFeatures( int maxNumFeatures )
mURI.setParam( QgsWFSConstants::URI_PARAM_MAXNUMFEATURES, QString( maxNumFeatures ) );
}
int QgsWFSDataSourceURI::pageSize() const
{
if ( !mURI.hasParam( QgsWFSConstants::URI_PARAM_PAGE_SIZE ) )
return 0;
return mURI.param( QgsWFSConstants::URI_PARAM_PAGE_SIZE ).toInt();
}
bool QgsWFSDataSourceURI::pagingEnabled() const
{
if ( !mURI.hasParam( QgsWFSConstants::URI_PARAM_PAGING_ENABLED ) )
return true;
return mURI.param( QgsWFSConstants::URI_PARAM_PAGING_ENABLED ) == QStringLiteral( "true" );
}
void QgsWFSDataSourceURI::setTypeName( const QString &typeName )
{
mURI.removeParam( QgsWFSConstants::URI_PARAM_TYPENAME );

View File

@ -102,6 +102,12 @@ class QgsWFSDataSourceURI
//! Sets user defined limit of features to download
void setMaxNumFeatures( int maxNumFeatures );
//! Returns user defined limit page size. 0=server udefault
int pageSize() const;
//! Returns whether paging is enabled.
bool pagingEnabled() const;
//! Gets typename (with prefix)
QString typeName() const;

View File

@ -29,6 +29,7 @@
#include "qgsexception.h"
#include "qgsfeedback.h"
#include <algorithm>
#include <QDir>
#include <QProgressDialog>
#include <QTimer>
@ -81,7 +82,7 @@ QgsWFSFeatureDownloader::QgsWFSFeatureDownloader( QgsWFSSharedData *shared )
, mShared( shared )
, mStop( false )
, mProgressDialogShowImmediately( false )
, mSupportsPaging( shared->mCaps.supportsPaging )
, mPageSize( shared->mPageSize )
, mRemoveNSPrefix( false )
, mNumberMatched( -1 )
, mFeatureHitsAsyncRequest( shared->mURI )
@ -185,7 +186,7 @@ QString QgsWFSFeatureDownloader::sanitizeFilter( QString filter )
return filter;
}
QUrl QgsWFSFeatureDownloader::buildURL( int startIndex, int maxFeatures, bool forHits )
QUrl QgsWFSFeatureDownloader::buildURL( qint64 startIndex, int maxFeatures, bool forHits )
{
QUrl getFeatureUrl( mShared->mURI.requestUrl( QStringLiteral( "GetFeature" ) ) );
getFeatureUrl.addQueryItem( QStringLiteral( "VERSION" ), mShared->mWFSVersion );
@ -231,7 +232,7 @@ QUrl QgsWFSFeatureDownloader::buildURL( int startIndex, int maxFeatures, bool fo
}
else if ( maxFeatures > 0 )
{
if ( mSupportsPaging )
if ( mPageSize > 0 )
{
// Note: always include the STARTINDEX, even for zero, has some (likely buggy)
// implementations do not return the same results if STARTINDEX=0 is specified
@ -383,6 +384,10 @@ QUrl QgsWFSFeatureDownloader::buildURL( int startIndex, int maxFeatures, bool fo
void QgsWFSFeatureDownloader::gotHitsResponse()
{
mNumberMatched = mFeatureHitsAsyncRequest.numberMatched();
if ( mShared->mMaxFeatures > 0 )
{
mNumberMatched = std::min( mNumberMatched, mShared->mMaxFeatures );
}
if ( mNumberMatched >= 0 )
{
if ( mTotalDownloadedFeatureCount == 0 )
@ -456,14 +461,41 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
int pagingIter = 1;
QString gmlIdFirstFeatureFirstIter;
bool disablePaging = false;
qint64 maxTotalFeatures = 0;
if ( maxFeatures > 0 && mShared->mMaxFeatures > 0 )
{
maxTotalFeatures = std::min( maxFeatures, mShared->mMaxFeatures );
}
else if ( maxFeatures > 0 )
{
maxTotalFeatures = maxFeatures;
}
else
{
maxTotalFeatures = mShared->mMaxFeatures;
}
// Top level loop to do feature paging in WFS 2.0
while ( true )
{
success = true;
QgsGmlStreamingParser *parser = mShared->createParser();
int maxFeaturesThisRequest = static_cast<int>(
std::min( maxTotalFeatures - mTotalDownloadedFeatureCount,
static_cast<qint64>( std::numeric_limits<int>::max() ) ) );
if ( mShared->mPageSize > 0 )
{
if ( maxFeaturesThisRequest > 0 )
{
maxFeaturesThisRequest = std::min( maxFeaturesThisRequest, mShared->mPageSize );
}
else
{
maxFeaturesThisRequest = mShared->mPageSize;
}
}
QUrl url( buildURL( mTotalDownloadedFeatureCount,
maxFeatures ? maxFeatures : mShared->mMaxFeatures, false ) );
maxFeaturesThisRequest, false ) );
// Small hack for testing purposes
if ( retryIter > 0 && url.toString().contains( QLatin1String( "fake_qgis_http_endpoint" ) ) )
@ -527,11 +559,11 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
// Some GeoServer instances in WFS 2.0 with paging throw an exception
// e.g. http://ows.region-bretagne.fr/geoserver/wfs?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=rb:etudes&STARTINDEX=0&COUNT=1
// Disabling paging helps in those cases
if ( mSupportsPaging && mTotalDownloadedFeatureCount == 0 &&
if ( mPageSize > 0 && mTotalDownloadedFeatureCount == 0 &&
parser->exceptionText().contains( QLatin1String( "Cannot do natural order without a primary key" ) ) )
{
QgsDebugMsg( QString( "Got exception %1. Re-trying with paging disabled" ).arg( parser->exceptionText() ) );
mSupportsPaging = false;
mPageSize = 0;
}
// GeoServer doesn't like typenames prefixed by namespace prefix, despite
// the examples in the WFS 2.0 spec showing that
@ -560,11 +592,15 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
if ( parser->numberMatched() > 0 && mTotalDownloadedFeatureCount == 0 )
mNumberMatched = parser->numberMatched();
// The number returned can only be used if we aren't in paging mode
else if ( parser->numberReturned() > 0 && !mSupportsPaging )
else if ( parser->numberReturned() > 0 && mPageSize == 0 )
mNumberMatched = parser->numberMatched();
// We can only use the layer feature count if we don't apply a BBOX
else if ( mShared->isFeatureCountExact() && mShared->mRect.isNull() )
mNumberMatched = mShared->getFeatureCount( false );
if ( mNumberMatched > 0 && mShared->mMaxFeatures > 0 )
{
mNumberMatched = std::min( mNumberMatched, mShared->mMaxFeatures );
}
// If we didn't get a valid mNumberMatched, we will possibly issue
// a explicit RESULTTYPE=hits request 4 second after the beginning of
@ -686,7 +722,7 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
if ( finished )
{
if ( parser->isTruncatedResponse() && !mSupportsPaging )
if ( parser->isTruncatedResponse() && mPageSize == 0 )
{
// e.g: http://services.cuzk.cz/wfs/inspire-cp-wfs.asp?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=cp:CadastralParcel
truncatedResponse = true;
@ -715,19 +751,20 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
retryIter = 0;
lastValidTotalDownloadedFeatureCount = mTotalDownloadedFeatureCount;
if ( !mSupportsPaging )
if ( mPageSize == 0 )
break;
if ( maxFeatures == 1 )
break;
// Detect if we are at the last page
if ( ( mShared->mMaxFeatures > 0 && featureCountForThisResponse < mShared->mMaxFeatures ) || featureCountForThisResponse == 0 )
if ( ( mShared->mPageSize > 0 && featureCountForThisResponse < mShared->mPageSize ) || featureCountForThisResponse == 0 )
break;
++ pagingIter;
if ( disablePaging )
{
mSupportsPaging = mShared->mCaps.supportsPaging = false;
mShared->mPageSize = mPageSize = 0;
mTotalDownloadedFeatureCount = 0;
if ( mShared->mMaxFeaturesWasSetFromDefaultForPaging )
mShared->mPageSize = 0;
if ( mShared->mMaxFeatures == mShared->mURI.maxNumFeatures() )
{
mShared->mMaxFeatures = 0;
}

View File

@ -135,7 +135,7 @@ class QgsWFSFeatureDownloader: public QgsWfsRequest
void hideProgressDialog();
private:
QUrl buildURL( int startIndex, int maxFeatures, bool forHits );
QUrl buildURL( qint64 startIndex, int maxFeatures, bool forHits );
void pushError( const QString &errorMsg );
QString sanitizeFilter( QString filter );
@ -150,13 +150,13 @@ class QgsWFSFeatureDownloader: public QgsWfsRequest
* If the progress dialog should be shown immediately, or if it should be
let to QProgressDialog logic to decide when to show it */
bool mProgressDialogShowImmediately;
bool mSupportsPaging;
int mPageSize;
bool mRemoveNSPrefix;
int mNumberMatched;
QWidget *mMainWindow = nullptr;
QTimer *mTimer = nullptr;
QgsWFSFeatureHitsAsyncRequest mFeatureHitsAsyncRequest;
int mTotalDownloadedFeatureCount;
qint64 mTotalDownloadedFeatureCount;
};
//! Downloader thread

View File

@ -0,0 +1,116 @@
/***************************************************************************
qgswfsnewconnection.cpp
---------------------
begin : June 2018
copyright : (C) 2018 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include "qgswfsnewconnection.h"
#include <QMessageBox>
QgsWFSNewConnection::QgsWFSNewConnection( QWidget *parent, const QString &connName ):
QgsNewHttpConnection( parent, QgsNewHttpConnection::ConnectionWfs, QgsWFSConstants::CONNECTIONS_WFS, connName )
{
connect( wfsVersionDetectButton(), &QPushButton::clicked, this, &QgsWFSNewConnection::versionDetectButton );
}
QgsWFSNewConnection::~QgsWFSNewConnection()
{
if ( mCapabilities )
{
QApplication::restoreOverrideCursor();
delete mCapabilities;
}
}
void QgsWFSNewConnection::versionDetectButton()
{
delete mCapabilities;
mCapabilities = new QgsWfsCapabilities( urlTrimmed().toString() );
connect( mCapabilities, &QgsWfsCapabilities::gotCapabilities, this, &QgsWFSNewConnection::capabilitiesReplyFinished );
const bool synchronous = false;
const bool forceRefresh = true;
if ( mCapabilities->requestCapabilities( synchronous, forceRefresh ) )
{
QApplication::setOverrideCursor( Qt::WaitCursor );
}
else
{
QMessageBox *box = new QMessageBox( QMessageBox::Critical, tr( "Error" ), tr( "Could not get capabilities" ), QMessageBox::Ok, this );
box->setAttribute( Qt::WA_DeleteOnClose );
box->setModal( true );
box->open();
delete mCapabilities;
mCapabilities = nullptr;
}
}
void QgsWFSNewConnection::capabilitiesReplyFinished()
{
if ( !mCapabilities )
return;
QApplication::restoreOverrideCursor();
QgsWfsCapabilities::ErrorCode err = mCapabilities->errorCode();
if ( err != QgsWfsCapabilities::NoError )
{
QString title;
switch ( err )
{
case QgsWfsCapabilities::NetworkError:
title = tr( "Network Error" );
break;
case QgsWfsCapabilities::XmlError:
title = tr( "Capabilities document is not valid" );
break;
case QgsWfsCapabilities::ServerExceptionError:
title = tr( "Server Exception" );
break;
default:
title = tr( "Error" );
break;
}
// handle errors
QMessageBox *box = new QMessageBox( QMessageBox::Critical, title, mCapabilities->errorMessage(), QMessageBox::Ok, this );
box->setAttribute( Qt::WA_DeleteOnClose );
box->setModal( true );
box->open();
delete mCapabilities;
mCapabilities = nullptr;
return;
}
const auto &caps = mCapabilities->capabilities();
int versionIdx = 0;
wfsPageSizeLineEdit()->clear();
if ( caps.version.startsWith( QLatin1String( "1.0" ) ) )
{
versionIdx = 1;
}
else if ( caps.version.startsWith( QLatin1String( "1.1" ) ) )
{
versionIdx = 2;
}
else if ( caps.version.startsWith( QLatin1String( "2.0" ) ) )
{
versionIdx = 3;
wfsPageSizeLineEdit()->setText( QString::number( caps.maxFeatures ) );
}
wfsVersionComboBox()->setCurrentIndex( versionIdx );
wfsPagingEnabledCheckBox()->setChecked( caps.supportsPaging );
delete mCapabilities;
mCapabilities = nullptr;
}

View File

@ -0,0 +1,41 @@
/***************************************************************************
qgswfsnewconnection.h
---------------------
begin : June 2018
copyright : (C) 2018 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSWFSNEWCONNECTION_H
#define QGSWFSNEWCONNECTION_H
#include "qgsnewhttpconnection.h"
#include "qgswfsconstants.h"
#include "qgswfscapabilities.h"
class QgsWFSNewConnection : public QgsNewHttpConnection
{
Q_OBJECT
public:
//! Constructor
QgsWFSNewConnection( QWidget *parent = nullptr, const QString &connName = QString::null );
~QgsWFSNewConnection();
private slots:
void versionDetectButton();
void capabilitiesReplyFinished();
private:
QgsWfsCapabilities *mCapabilities = nullptr;
};
#endif //QGSWFSNEWCONNECTION_H

View File

@ -1684,17 +1684,46 @@ bool QgsWFSProvider::getCapabilities()
}
mShared->mWFSVersion = mShared->mCaps.version;
if ( mShared->mURI.maxNumFeatures() > 0 )
mShared->mMaxFeatures = mShared->mURI.maxNumFeatures();
else
mShared->mMaxFeatures = mShared->mCaps.maxFeatures;
if ( mShared->mMaxFeatures <= 0 && mShared->mCaps.supportsPaging )
if ( mShared->mURI.maxNumFeatures() > 0 && mShared->mCaps.maxFeatures > 0 )
{
QgsSettings settings;
mShared->mMaxFeatures = settings.value( QStringLiteral( "wfs/max_feature_count_if_not_provided" ), "1000" ).toInt();
mShared->mMaxFeaturesWasSetFromDefaultForPaging = true;
QgsDebugMsg( QString( "Server declares paging but does not advertize max feature count and user did not specify it. Using %1" ).arg( mShared->mMaxFeatures ) );
mShared->mMaxFeatures = std::min( mShared->mURI.maxNumFeatures(), mShared->mCaps.maxFeatures );
}
else if ( mShared->mURI.maxNumFeatures() > 0 )
{
mShared->mMaxFeatures = mShared->mURI.maxNumFeatures();
}
else if ( mShared->mCaps.maxFeatures > 0 )
{
mShared->mMaxFeatures = mShared->mCaps.maxFeatures;
}
else
{
mShared->mMaxFeatures = 0;
}
if ( mShared->mCaps.supportsPaging && mShared->mURI.pagingEnabled() )
{
if ( mShared->mURI.pageSize() > 0 )
{
if ( mShared->mMaxFeatures > 0 )
{
mShared->mPageSize = std::min( mShared->mURI.pageSize(), mShared->mMaxFeatures );
}
else
{
mShared->mPageSize = mShared->mURI.pageSize();
}
}
else
{
QgsSettings settings;
mShared->mPageSize = settings.value( QStringLiteral( "wfs/max_feature_count_if_not_provided" ), "1000" ).toInt();
QgsDebugMsg( QString( "Server declares paging but does not advertize max feature count and user did not specify it. Using %1" ).arg( mShared->mMaxFeatures ) );
}
}
else
{
mShared->mPageSize = 0;
}
//find the <FeatureType> for this layer

View File

@ -39,7 +39,7 @@ QgsWFSSharedData::QgsWFSSharedData( const QString &uri )
: mURI( uri )
, mSourceCRS( 0 )
, mMaxFeatures( 0 )
, mMaxFeaturesWasSetFromDefaultForPaging( false )
, mPageSize( 0 )
, mRequestLimit( 0 )
, mHideProgressDialog( mURI.hideDownloadProgressDialog() )
, mDistinctSelect( false )

View File

@ -155,11 +155,11 @@ class QgsWFSSharedData : public QObject
//! Current BBOX used by the downloader
QgsRectangle mRect;
//! Server-side or user-side limit of downloaded features (in a single GetFeature()). Valid if > 0
//! Server-side or user-side limit of downloaded features (including with paging). Valid if > 0
int mMaxFeatures;
//! Whether mMaxFeatures was set to a non 0 value for the purpose of paging
bool mMaxFeaturesWasSetFromDefaultForPaging;
//! Page size for WFS 2.0. 0 = disabled
int mPageSize;
//! Limit of retrieved number of features for the current request
int mRequestLimit;

View File

@ -22,7 +22,7 @@
#include "qgswfsprovider.h"
#include "qgswfsdatasourceuri.h"
#include "qgswfsutils.h"
#include "qgsnewhttpconnection.h"
#include "qgswfsnewconnection.h"
#include "qgsprojectionselectiondialog.h"
#include "qgsproject.h"
#include "qgscoordinatereferencesystem.h"
@ -108,6 +108,8 @@ QgsWFSSourceSelect::QgsWFSSourceSelect( QWidget *parent, Qt::WindowFlags fl, Qgs
QgsWFSSourceSelect::~QgsWFSSourceSelect()
{
QApplication::restoreOverrideCursor();
QgsSettings settings;
QgsDebugMsg( "saving settings" );
settings.setValue( QStringLiteral( "Windows/WFSSourceSelect/geometry" ), saveGeometry() );
@ -204,6 +206,7 @@ void QgsWFSSourceSelect::refresh()
void QgsWFSSourceSelect::capabilitiesReplyFinished()
{
QApplication::restoreOverrideCursor();
btnConnect->setEnabled( true );
if ( !mCapabilities )
@ -290,7 +293,7 @@ void QgsWFSSourceSelect::capabilitiesReplyFinished()
void QgsWFSSourceSelect::addEntryToServerList()
{
QgsNewHttpConnection *nc = new QgsNewHttpConnection( this, QgsNewHttpConnection::ConnectionWfs, QgsWFSConstants::CONNECTIONS_WFS );
auto nc = new QgsWFSNewConnection( this );
nc->setAttribute( Qt::WA_DeleteOnClose );
nc->setWindowTitle( tr( "Create a New WFS Connection" ) );
@ -303,7 +306,7 @@ void QgsWFSSourceSelect::addEntryToServerList()
void QgsWFSSourceSelect::modifyEntryOfServerList()
{
QgsNewHttpConnection *nc = new QgsNewHttpConnection( this, QgsNewHttpConnection::ConnectionWfs, QgsWFSConstants::CONNECTIONS_WFS, cmbConnections->currentText() );
auto nc = new QgsWFSNewConnection( this, cmbConnections->currentText() );
nc->setAttribute( Qt::WA_DeleteOnClose );
nc->setWindowTitle( tr( "Modify WFS Connection" ) );
@ -356,6 +359,7 @@ void QgsWFSSourceSelect::connectToServer()
const bool synchronous = false;
const bool forceRefresh = true;
mCapabilities->requestCapabilities( synchronous, forceRefresh );
QApplication::setOverrideCursor( Qt::WaitCursor );
}
}

View File

@ -32,14 +32,14 @@
<string>WFS Options</string>
</property>
<layout class="QGridLayout" name="gridLayout1">
<item row="2" column="0" colspan="2">
<item row="4" column="0" colspan="2">
<widget class="QCheckBox" name="cbxWfsIgnoreAxisOrientation">
<property name="text">
<string>Ignore axis orientation (WFS 1.1/WFS 2.0)</string>
</property>
</widget>
</item>
<item row="3" column="0" colspan="2">
<item row="5" column="0" colspan="2">
<widget class="QCheckBox" name="cbxWfsInvertAxisOrientation">
<property name="text">
<string>Invert axis orientation</string>
@ -60,17 +60,48 @@
</property>
</widget>
</item>
<item row="0" column="1">
<widget class="QComboBox" name="cmbVersion">
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Select protocol version&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
</widget>
</item>
<item row="1" column="1">
<widget class="QLineEdit" name="txtMaxNumFeatures">
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Enter a number to limit the maximum number of features retrieved in a single GetFeature request. If let to empty, server default will apply.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Enter a number to limit the maximum number of features retrieved per feature request. If let to empty, no limit is set.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
</widget>
</item>
<item row="0" column="1">
<layout class="QHBoxLayout" name="layoutWfsVersion">
<item>
<widget class="QComboBox" name="cmbVersion"/>
</item>
<item>
<widget class="QPushButton" name="mWfsVersionDetectButton">
<property name="text">
<string>Detect</string>
</property>
</widget>
</item>
</layout>
</item>
<item row="2" column="0">
<widget class="QCheckBox" name="cbxWfsFeaturePaging">
<property name="text">
<string>Enable feature paging</string>
</property>
<property name="checked">
<bool>true</bool>
</property>
</widget>
</item>
<item row="3" column="0">
<widget class="QLabel" name="lblPageSize">
<property name="text">
<string>Page size</string>
</property>
</widget>
</item>
<item row="3" column="1">
<widget class="QLineEdit" name="txtPageSize">
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Enter a number to limit the maximum number of features retrieved in a single GetFeature request when paging is enabled. If let to empty, server default will apply.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
</widget>
</item>
@ -158,7 +189,7 @@
<item row="1" column="0">
<widget class="QLabel" name="lblDpiMode">
<property name="text">
<string>&amp;DPI-Mode</string>
<string>DPI-&amp;Mode</string>
</property>
<property name="buddy">
<cstring>cmbDpiMode</cstring>
@ -168,7 +199,7 @@
<item row="0" column="0">
<widget class="QLabel" name="lblReferer">
<property name="text">
<string>Referer</string>
<string>&amp;Referer</string>
</property>
<property name="buddy">
<cstring>txtReferer</cstring>
@ -289,6 +320,25 @@
<container>1</container>
</customwidget>
</customwidgets>
<tabstops>
<tabstop>txtName</tabstop>
<tabstop>txtUrl</tabstop>
<tabstop>cmbVersion</tabstop>
<tabstop>mWfsVersionDetectButton</tabstop>
<tabstop>txtMaxNumFeatures</tabstop>
<tabstop>cbxWfsFeaturePaging</tabstop>
<tabstop>txtPageSize</tabstop>
<tabstop>cbxWfsIgnoreAxisOrientation</tabstop>
<tabstop>cbxWfsInvertAxisOrientation</tabstop>
<tabstop>txtReferer</tabstop>
<tabstop>cmbDpiMode</tabstop>
<tabstop>cbxIgnoreGetMapURI</tabstop>
<tabstop>cbxIgnoreGetFeatureInfoURI</tabstop>
<tabstop>cbxWmsIgnoreAxisOrientation</tabstop>
<tabstop>cbxWmsInvertAxisOrientation</tabstop>
<tabstop>cbxSmoothPixmapTransform</tabstop>
<tabstop>mTestConnectionButton</tabstop>
</tabstops>
<resources/>
<connections>
<connection>

View File

@ -1088,6 +1088,174 @@ class TestPyQgsWFSProvider(unittest.TestCase, ProviderTestCase):
</wfs:FeatureCollection>""".encode('UTF-8'))
self.assertEqual(vl.featureCount(), 2)
def testWFS20PagingPageSizeOverride(self):
"""Test WFS 2.0 paging"""
endpoint = self.__class__.basetestpath + '/fake_qgis_http_endpoint_WFS_2.0_paging_override'
with open(sanitize(endpoint, '?SERVICE=WFS?REQUEST=GetCapabilities?ACCEPTVERSIONS=2.0.0,1.1.0,1.0.0'), 'wb') as f:
f.write("""
<wfs:WFS_Capabilities version="2.0.0" xmlns="http://www.opengis.net/wfs/2.0" xmlns:wfs="http://www.opengis.net/wfs/2.0" xmlns:ows="http://www.opengis.net/ows/1.1" xmlns:gml="http://schemas.opengis.net/gml/3.2" xmlns:fes="http://www.opengis.net/fes/2.0">
<OperationsMetadata>
<Operation name="GetFeature">
<Constraint name="CountDefault">
<NoValues/>
<DefaultValue>10</DefaultValue>
</Constraint>
</Operation>
<Constraint name="ImplementsResultPaging">
<NoValues/>
<DefaultValue>TRUE</DefaultValue>
</Constraint>
</OperationsMetadata>
<FeatureTypeList>
<FeatureType>
<Name>my:typename</Name>
<Title>Title</Title>
<Abstract>Abstract</Abstract>
<DefaultCRS>urn:ogc:def:crs:EPSG::4326</DefaultCRS>
<WGS84BoundingBox>
<LowerCorner>-71.123 66.33</LowerCorner>
<UpperCorner>-65.32 78.3</UpperCorner>
</WGS84BoundingBox>
</FeatureType>
</FeatureTypeList>
</wfs:WFS_Capabilities>""".encode('UTF-8'))
with open(sanitize(endpoint, '?SERVICE=WFS&REQUEST=DescribeFeatureType&VERSION=2.0.0&TYPENAME=my:typename'), 'wb') as f:
f.write("""
<xsd:schema xmlns:my="http://my" xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:xsd="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" targetNamespace="http://my">
<xsd:import namespace="http://www.opengis.net/gml/3.2"/>
<xsd:complexType name="typenameType">
<xsd:complexContent>
<xsd:extension base="gml:AbstractFeatureType">
<xsd:sequence>
<xsd:element maxOccurs="1" minOccurs="0" name="id" nillable="true" type="xsd:int"/>
<xsd:element maxOccurs="1" minOccurs="0" name="geometryProperty" nillable="true" type="gml:PointPropertyType"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:element name="typename" substitutionGroup="gml:_Feature" type="my:typenameType"/>
</xsd:schema>
""".encode('UTF-8'))
# user pageSize < user maxNumFeatures < server pagesize
vl = QgsVectorLayer("url='http://" + endpoint + "' typename='my:typename' maxNumFeatures='3' pageSize='2'", 'test', 'WFS')
self.assertTrue(vl.isValid())
self.assertEqual(vl.wkbType(), QgsWkbTypes.Point)
with open(sanitize(endpoint, '?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=my:typename&STARTINDEX=0&COUNT=2&SRSNAME=urn:ogc:def:crs:EPSG::4326'), 'wb') as f:
f.write("""
<wfs:FeatureCollection xmlns:wfs="http://www.opengis.net/wfs/2.0"
xmlns:gml="http://www.opengis.net/gml/3.2"
xmlns:my="http://my"
numberMatched="2" numberReturned="2" timeStamp="2016-03-25T14:51:48.998Z">
<wfs:member>
<my:typename gml:id="typename.100">
<my:geometryProperty><gml:Point srsName="urn:ogc:def:crs:EPSG::4326" gml:id="typename.geom.0"><gml:pos>66.33 -70.332</gml:pos></gml:Point></my:geometryProperty>
<my:id>1</my:id>
</my:typename>
<my:typename gml:id="typename.101">
<my:geometryProperty><gml:Point srsName="urn:ogc:def:crs:EPSG::4326" gml:id="typename.geom.1"><gml:pos>66.33 -70.332</gml:pos></gml:Point></my:geometryProperty>
<my:id>2</my:id>
</my:typename>
</wfs:member>
</wfs:FeatureCollection>""".encode('UTF-8'))
with open(sanitize(endpoint, '?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=my:typename&STARTINDEX=2&COUNT=1&SRSNAME=urn:ogc:def:crs:EPSG::4326'), 'wb') as f:
f.write("""
<wfs:FeatureCollection xmlns:wfs="http://www.opengis.net/wfs/2.0"
xmlns:gml="http://www.opengis.net/gml/3.2"
xmlns:my="http://my"
numberMatched="1" numberReturned="1" timeStamp="2016-03-25T14:51:48.998Z">
<wfs:member>
<my:typename gml:id="typename.200">
<my:geometryProperty><gml:Point srsName="urn:ogc:def:crs:EPSG::4326" gml:id="typename.geom.0"><gml:pos>66.33 -70.332</gml:pos></gml:Point></my:geometryProperty>
<my:id>3</my:id>
</my:typename>
</wfs:member>
</wfs:FeatureCollection>""".encode('UTF-8'))
values = [f['id'] for f in vl.getFeatures()]
self.assertEqual(values, [1, 2, 3])
os.unlink(sanitize(endpoint, '?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=my:typename&STARTINDEX=0&COUNT=2&SRSNAME=urn:ogc:def:crs:EPSG::4326'))
os.unlink(sanitize(endpoint, '?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=my:typename&STARTINDEX=2&COUNT=1&SRSNAME=urn:ogc:def:crs:EPSG::4326'))
# user maxNumFeatures < user pageSize < server pagesize
vl = QgsVectorLayer("url='http://" + endpoint + "' typename='my:typename' maxNumFeatures='1' pageSize='2'", 'test', 'WFS')
self.assertTrue(vl.isValid())
self.assertEqual(vl.wkbType(), QgsWkbTypes.Point)
with open(sanitize(endpoint, '?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=my:typename&STARTINDEX=0&COUNT=1&SRSNAME=urn:ogc:def:crs:EPSG::4326'), 'wb') as f:
f.write("""
<wfs:FeatureCollection xmlns:wfs="http://www.opengis.net/wfs/2.0"
xmlns:gml="http://www.opengis.net/gml/3.2"
xmlns:my="http://my"
numberMatched="1" numberReturned="1" timeStamp="2016-03-25T14:51:48.998Z">
<wfs:member>
<my:typename gml:id="typename.100">
<my:geometryProperty><gml:Point srsName="urn:ogc:def:crs:EPSG::4326" gml:id="typename.geom.0"><gml:pos>66.33 -70.332</gml:pos></gml:Point></my:geometryProperty>
<my:id>1</my:id>
</my:typename>
</wfs:member>
</wfs:FeatureCollection>""".encode('UTF-8'))
values = [f['id'] for f in vl.getFeatures()]
self.assertEqual(values, [1])
os.unlink(sanitize(endpoint, '?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=my:typename&STARTINDEX=0&COUNT=1&SRSNAME=urn:ogc:def:crs:EPSG::4326'))
# user user pageSize > server pagesize
vl = QgsVectorLayer("url='http://" + endpoint + "' typename='my:typename' pageSize='100'", 'test', 'WFS')
self.assertTrue(vl.isValid())
self.assertEqual(vl.wkbType(), QgsWkbTypes.Point)
with open(sanitize(endpoint, '?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=my:typename&STARTINDEX=0&COUNT=10&SRSNAME=urn:ogc:def:crs:EPSG::4326'), 'wb') as f:
f.write("""
<wfs:FeatureCollection xmlns:wfs="http://www.opengis.net/wfs/2.0"
xmlns:gml="http://www.opengis.net/gml/3.2"
xmlns:my="http://my"
numberMatched="1" numberReturned="1" timeStamp="2016-03-25T14:51:48.998Z">
<wfs:member>
<my:typename gml:id="typename.100">
<my:geometryProperty><gml:Point srsName="urn:ogc:def:crs:EPSG::4326" gml:id="typename.geom.0"><gml:pos>66.33 -70.332</gml:pos></gml:Point></my:geometryProperty>
<my:id>1</my:id>
</my:typename>
</wfs:member>
</wfs:FeatureCollection>""".encode('UTF-8'))
values = [f['id'] for f in vl.getFeatures()]
self.assertEqual(values, [1])
os.unlink(sanitize(endpoint, '?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=my:typename&STARTINDEX=0&COUNT=10&SRSNAME=urn:ogc:def:crs:EPSG::4326'))
vl = QgsVectorLayer("url='http://" + endpoint + "' typename='my:typename' pagingEnabled='false' maxNumFeatures='3'", 'test', 'WFS')
self.assertTrue(vl.isValid())
self.assertEqual(vl.wkbType(), QgsWkbTypes.Point)
with open(sanitize(endpoint, '?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=my:typename&COUNT=3&SRSNAME=urn:ogc:def:crs:EPSG::4326'), 'wb') as f:
f.write("""
<wfs:FeatureCollection xmlns:wfs="http://www.opengis.net/wfs/2.0"
xmlns:gml="http://www.opengis.net/gml/3.2"
xmlns:my="http://my"
numberMatched="2" numberReturned="2" timeStamp="2016-03-25T14:51:48.998Z">
<wfs:member>
<my:typename gml:id="typename.100">
<my:geometryProperty><gml:Point srsName="urn:ogc:def:crs:EPSG::4326" gml:id="typename.geom.0"><gml:pos>66.33 -70.332</gml:pos></gml:Point></my:geometryProperty>
<my:id>1000</my:id>
</my:typename>
<my:typename gml:id="typename.101">
<my:geometryProperty><gml:Point srsName="urn:ogc:def:crs:EPSG::4326" gml:id="typename.geom.1"><gml:pos>66.33 -70.332</gml:pos></gml:Point></my:geometryProperty>
<my:id>2000</my:id>
</my:typename>
</wfs:member>
</wfs:FeatureCollection>""".encode('UTF-8'))
values = [f['id'] for f in vl.getFeatures()]
self.assertEqual(values, [1000, 2000])
def testWFSGetOnlyFeaturesInViewExtent(self):
"""Test 'get only features in view extent' """