sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-03-04 00:30:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

[Feature] Add 'OGC API - Features' provider, shortnamed as OAPIF. Only non-GUI elements in this commit

Browse Source 
Funded by Planet
This commit is contained in:
Even Rouault 2019-10-13 12:51:46 +02:00 committed by Nyall Dawson
parent 05eee425ad
commit 4a6b49fe8f
16 changed files with 2257 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/providers/wfs/CMakeLists.txt
View File

@ -3,6 +3,7 @@
# Files
SET(WFS_SRCS
${CMAKE_SOURCE_DIR}/external/nlohmann/json.hpp
qgswfsprovider.cpp
qgswfscapabilities.cpp
qgswfsdataitems.cpp
@ -19,6 +20,12 @@ SET(WFS_SRCS
qgsbackgroundcachedshareddata.cpp
qgscachedirectorymanager.cpp
qgsbasenetworkrequest.cpp
qgsoapiflandingpagerequest.cpp
qgsoapifapirequest.cpp
qgsoapifcollection.cpp
qgsoapifitemsrequest.cpp
qgsoapifprovider.cpp
qgsoapifutils.cpp
)
SET (WFS_MOC_HDRS
@ -35,6 +42,11 @@ SET (WFS_MOC_HDRS
qgswfsutils.h
qgsbackgroundcachedfeatureiterator.h
qgscachedirectorymanager.h
qgsoapiflandingpagerequest.h
qgsoapifapirequest.h
qgsoapifcollection.h
qgsoapifitemsrequest.h
qgsoapifprovider.h
)
IF (WITH_GUI)

132
src/providers/wfs/qgsoapifapirequest.cpp Normal file
View File

@ -0,0 +1,132 @@
/***************************************************************************
qgsoapifapirequest.cpp
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include <nlohmann/json.hpp>
using namespace nlohmann;
#include "qgslogger.h"
#include "qgsoapifapirequest.h"
#include <QTextCodec>
QgsOapifApiRequest::QgsOapifApiRequest( const QgsDataSourceUri &baseUri, const QString &url ):
QgsBaseNetworkRequest( QgsAuthorizationSettings( baseUri.username(), baseUri.password(), baseUri.authConfigId() ), tr( "OAPIF" ) ),
mUrl( url )
{
// Using Qt::DirectConnection since the download might be running on a different thread.
// In this case, the request was sent from the main thread and is executed with the main
// thread being blocked in future.waitForFinished() so we can run code on this object which
// lives in the main thread without risking havoc.
connect( this, &QgsBaseNetworkRequest::downloadFinished, this, &QgsOapifApiRequest::processReply, Qt::DirectConnection );
}
bool QgsOapifApiRequest::request( bool synchronous, bool forceRefresh )
{
if ( !sendGET( QUrl( mUrl ), QStringLiteral( "application/vnd.oai.openapi+json;version=3.0, application/openapi+json;version=3.0, application/json" ), synchronous, forceRefresh ) )
{
emit gotResponse();
return false;
}
return true;
}
QString QgsOapifApiRequest::errorMessageWithReason( const QString &reason )
{
return tr( "Download of API page failed: %1" ).arg( reason );
}
void QgsOapifApiRequest::processReply()
{
if ( mErrorCode != QgsBaseNetworkRequest::NoError )
{
emit gotResponse();
return;
}
const QByteArray &buffer = mResponse;
if ( buffer.isEmpty() )
{
mErrorMessage = tr( "empty response" );
mErrorCode = QgsBaseNetworkRequest::ServerExceptionError;
emit gotResponse();
return;
}
QgsDebugMsgLevel( QStringLiteral( "parsing API response: " ) + buffer, 4 );
QTextCodec::ConverterState state;
QTextCodec *codec = QTextCodec::codecForName( "UTF-8" );
Q_ASSERT( codec );
const QString utf8Text = codec->toUnicode( buffer.constData(), buffer.size(), &state );
if ( state.invalidChars != 0 )
{
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::JsonError;
mErrorMessage = errorMessageWithReason( tr( "Invalid UTF-8 content" ) );
emit gotResponse();
return;
}
try
{
const json j = json::parse( utf8Text.toStdString() );
if ( j.is_object() && j.contains( "components" ) )
{
const auto components = j["components"];
if ( components.is_object() && components.contains( "parameters" ) )
{
const auto parameters = components["parameters"];
if ( parameters.is_object() && parameters.contains( "limit" ) )
{
const auto limit = parameters["limit"];
if ( limit.is_object() && limit.contains( "schema" ) )
{
const auto schema = limit["schema"];
if ( schema.is_object() )
{
if ( schema.contains( "maximum" ) )
{
const auto maximum = schema["maximum"];
if ( maximum.is_number_integer() )
{
mMaxLimit = maximum.get<int>();
}
}
if ( schema.contains( "default" ) )
{
const auto defaultL = schema["default"];
if ( defaultL.is_number_integer() )
{
mDefaultLimit = defaultL.get<int>();
}
}
}
}
}
}
}
}
catch ( const json::parse_error &ex )
{
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::JsonError;
mErrorMessage = errorMessageWithReason( tr( "Cannot decode JSon document: %1" ).arg( QString::fromStdString( ex.what() ) ) );
emit gotResponse();
return;
}
emit gotResponse();
}

72
src/providers/wfs/qgsoapifapirequest.h Normal file
View File

@ -0,0 +1,72 @@
/***************************************************************************
qgsoapifapirequest.h
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSOAPIFAPIREQUEST_H
#define QGSOAPIFAPIREQUEST_H
#include <QObject>
#include "qgsdatasourceuri.h"
#include "qgsbasenetworkrequest.h"
//! Manages the /api request
class QgsOapifApiRequest : public QgsBaseNetworkRequest
{
Q_OBJECT
public:
explicit QgsOapifApiRequest( const QgsDataSourceUri &baseUri, const QString &url );
//! Issue the request
bool request( bool synchronous, bool forceRefresh );
//! Application level error
enum class ApplicationLevelError
{
NoError,
JsonError,
IncompleteInformation
};
//! Returns application level error
ApplicationLevelError applicationLevelError() const { return mAppLevelError; }
//! Return the maximum number of features that can be requested at once (-1 if unknown)
int maxLimit() const { return mMaxLimit; }
//! Return the default number of features that are requested at once (-1 if unknown)
int defaultLimit() const { return mDefaultLimit; }
signals:
//! emitted when the capabilities have been fully parsed, or an error occurred */
void gotResponse();
private slots:
void processReply();
protected:
QString errorMessageWithReason( const QString &reason ) override;
private:
QString mUrl;
int mMaxLimit = -1;
int mDefaultLimit = -1;
ApplicationLevelError mAppLevelError = ApplicationLevelError::NoError;
};
#endif // QGSOAPIFAPIREQUEST_H

298
src/providers/wfs/qgsoapifcollection.cpp Normal file
View File

@ -0,0 +1,298 @@
/***************************************************************************
qgsoapifcollection.cpp
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include <nlohmann/json.hpp>
using namespace nlohmann;
#include "qgslogger.h"
#include "qgsoapifcollection.h"
#include "qgsoapifutils.h"
#include <QTextCodec>
bool QgsOapifCollection::deserialize( const json &j )
{
if ( !j.is_object() )
return false;
const char *idPropertyName = "id";
if ( !j.contains( "id" ) )
{
#ifndef REMOVE_SUPPORT_DRAFT_VERSIONS
if ( j.contains( "name" ) )
{
idPropertyName = "name";
}
else
#endif
{
QgsDebugMsg( QStringLiteral( "missing id in collection" ) );
return false;
}
}
const auto id = j[idPropertyName];
if ( !id.is_string() )
return false;
mId = QString::fromStdString( id.get<std::string>() );
if ( j.contains( "title" ) )
{
const auto title = j["title"];
if ( title.is_string() )
mTitle = QString::fromStdString( title.get<std::string>() );
}
if ( j.contains( "description" ) )
{
const auto description = j["description"];
if ( description.is_string() )
mDescription = QString::fromStdString( description.get<std::string>() );
}
if ( j.contains( "extent" ) )
{
const auto extent = j["extent"];
if ( extent.is_object() && extent.contains( "spatial" ) )
{
const auto spatial = extent["spatial"];
if ( spatial.is_object() && spatial.contains( "bbox" ) )
{
const auto bbox = spatial["bbox"];
if ( bbox.is_array() && !bbox.empty() )
{
const auto firstBbox = bbox[0];
if ( firstBbox.is_array() && ( firstBbox.size() == 4 || firstBbox.size() == 6 ) )
{
std::vector<double> values;
for ( size_t i = 0; i < firstBbox.size(); i++ )
{
if ( !firstBbox[i].is_number() )
{
values.clear();
break;
}
values.push_back( firstBbox[i].get<double>() );
}
if ( values.size() == 4 )
{
mBbox.set( values[0], values[1], values[2], values[3] );
}
else if ( values.size() == 6 ) // with zmin at [2] and zmax at [5]
{
mBbox.set( values[0], values[1], values[3], values[4] );
}
}
}
}
}
#ifndef REMOVE_SUPPORT_DRAFT_VERSIONS
else if ( extent.is_object() && extent.contains( "bbox" ) )
{
const auto bbox = extent["bbox"];
if ( bbox.is_array() && bbox.size() == 4 )
{
std::vector<double> values;
for ( size_t i = 0; i < bbox.size(); i++ )
{
if ( !bbox[i].is_number() )
{
values.clear();
break;
}
values.push_back( bbox[i].get<double>() );
}
if ( values.size() == 4 )
{
mBbox.set( values[0], values[1], values[2], values[3] );
}
}
}
#endif
}
return true;
}
// -----------------------------------------
QgsOapifCollectionsRequest::QgsOapifCollectionsRequest( const QgsDataSourceUri &baseUri, const QString &url ):
QgsBaseNetworkRequest( QgsAuthorizationSettings( baseUri.username(), baseUri.password(), baseUri.authConfigId() ), tr( "OAPIF" ) ),
mUrl( url )
{
// Using Qt::DirectConnection since the download might be running on a different thread.
// In this case, the request was sent from the main thread and is executed with the main
// thread being blocked in future.waitForFinished() so we can run code on this object which
// lives in the main thread without risking havoc.
connect( this, &QgsBaseNetworkRequest::downloadFinished, this, &QgsOapifCollectionsRequest::processReply, Qt::DirectConnection );
}
bool QgsOapifCollectionsRequest::request( bool synchronous, bool forceRefresh )
{
if ( !sendGET( QUrl( mUrl ), QStringLiteral( "application/json" ), synchronous, forceRefresh ) )
{
emit gotResponse();
return false;
}
return true;
}
QString QgsOapifCollectionsRequest::errorMessageWithReason( const QString &reason )
{
return tr( "Download of collections description failed: %1" ).arg( reason );
}
void QgsOapifCollectionsRequest::processReply()
{
if ( mErrorCode != QgsBaseNetworkRequest::NoError )
{
emit gotResponse();
return;
}
const QByteArray &buffer = mResponse;
if ( buffer.isEmpty() )
{
mErrorMessage = tr( "empty response" );
mErrorCode = QgsBaseNetworkRequest::ServerExceptionError;
emit gotResponse();
return;
}
QgsDebugMsgLevel( QStringLiteral( "parsing collections response: " ) + buffer, 4 );
QTextCodec::ConverterState state;
QTextCodec *codec = QTextCodec::codecForName( "UTF-8" );
Q_ASSERT( codec );
const QString utf8Text = codec->toUnicode( buffer.constData(), buffer.size(), &state );
if ( state.invalidChars != 0 )
{
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::JsonError;
mErrorMessage = errorMessageWithReason( tr( "Invalid UTF-8 content" ) );
emit gotResponse();
return;
}
try
{
const json j = json::parse( utf8Text.toStdString() );
if ( j.is_object() && j.contains( "collections" ) )
{
const auto collections = j["collections"];
if ( collections.is_array() )
{
for ( const auto jCollection : collections )
{
QgsOapifCollection collection;
if ( collection.deserialize( jCollection ) )
{
mCollections.emplace_back( collection );
}
}
}
}
// Paging informal extension used by api.planet.com/
const auto links = QgsOAPIFJson::parseLinks( j );
mNextUrl = QgsOAPIFJson::findLink( links,
QStringLiteral( "next" ),
QStringList() << QStringLiteral( "application/json" ) );
}
catch ( const json::parse_error &ex )
{
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::JsonError;
mErrorMessage = errorMessageWithReason( tr( "Cannot decode JSon document: %1" ).arg( QString::fromStdString( ex.what() ) ) );
emit gotResponse();
return;
}
emit gotResponse();
}
// -----------------------------------------
QgsOapifCollectionRequest::QgsOapifCollectionRequest( const QgsDataSourceUri &baseUri, const QString &url ):
QgsBaseNetworkRequest( QgsAuthorizationSettings( baseUri.username(), baseUri.password(), baseUri.authConfigId() ), tr( "OAPIF" ) ),
mUrl( url )
{
// Using Qt::DirectConnection since the download might be running on a different thread.
// In this case, the request was sent from the main thread and is executed with the main
// thread being blocked in future.waitForFinished() so we can run code on this object which
// lives in the main thread without risking havoc.
connect( this, &QgsBaseNetworkRequest::downloadFinished, this, &QgsOapifCollectionRequest::processReply, Qt::DirectConnection );
}
bool QgsOapifCollectionRequest::request( bool synchronous, bool forceRefresh )
{
if ( !sendGET( QUrl( mUrl ), QStringLiteral( "application/json" ), synchronous, forceRefresh ) )
{
emit gotResponse();
return false;
}
return true;
}
QString QgsOapifCollectionRequest::errorMessageWithReason( const QString &reason )
{
return tr( "Download of collection description failed: %1" ).arg( reason );
}
void QgsOapifCollectionRequest::processReply()
{
if ( mErrorCode != QgsBaseNetworkRequest::NoError )
{
emit gotResponse();
return;
}
const QByteArray &buffer = mResponse;
if ( buffer.isEmpty() )
{
mErrorMessage = tr( "empty response" );
mErrorCode = QgsBaseNetworkRequest::ServerExceptionError;
emit gotResponse();
return;
}
QgsDebugMsgLevel( QStringLiteral( "parsing collection response: " ) + buffer, 4 );
QTextCodec::ConverterState state;
QTextCodec *codec = QTextCodec::codecForName( "UTF-8" );
Q_ASSERT( codec );
const QString utf8Text = codec->toUnicode( buffer.constData(), buffer.size(), &state );
if ( state.invalidChars != 0 )
{
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::JsonError;
mErrorMessage = errorMessageWithReason( tr( "Invalid UTF-8 content" ) );
emit gotResponse();
return;
}
try
{
const json j = json::parse( utf8Text.toStdString() );
mCollection.deserialize( j );
}
catch ( const json::parse_error &ex )
{
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::JsonError;
mErrorMessage = errorMessageWithReason( tr( "Cannot decode JSon document: %1" ).arg( QString::fromStdString( ex.what() ) ) );
emit gotResponse();
return;
}
emit gotResponse();
}

139
src/providers/wfs/qgsoapifcollection.h Normal file
View File

@ -0,0 +1,139 @@
/***************************************************************************
qgsoapifcollection.h
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSOAPIFCOLLECTION_H
#define QGSOAPIFCOLLECTION_H
#include <QObject>
#include "qgsdatasourceuri.h"
#include "qgsbasenetworkrequest.h"
#include "qgsrectangle.h"
#include <nlohmann/json.hpp>
using namespace nlohmann;
#include <vector>
//! Describes a collection
struct QgsOapifCollection
{
//! Identifier
QString mId;
//! Title
QString mTitle;
//! Description
QString mDescription;
//! Bounding box (in CRS84)
QgsRectangle mBbox;
//! Fills a collection from its JSon serialization
bool deserialize( const json &j );
};
//! Manages the /collections request
class QgsOapifCollectionsRequest : public QgsBaseNetworkRequest
{
Q_OBJECT
public:
explicit QgsOapifCollectionsRequest( const QgsDataSourceUri &baseUri, const QString &url );
//! Issue the request
bool request( bool synchronous, bool forceRefresh );
//! Application level error
enum class ApplicationLevelError
{
NoError,
JsonError,
IncompleteInformation
};
//! Returns application level error
ApplicationLevelError applicationLevelError() const { return mAppLevelError; }
//! Returns collections description.
const std::vector<QgsOapifCollection> &collections() const { return mCollections; }
//! Return the url of the next page (extension to the spec)
const QString &nextUrl() const { return mNextUrl; }
signals:
//! emitted when the capabilities have been fully parsed, or an error occurred
void gotResponse();
private slots:
void processReply();
protected:
QString errorMessageWithReason( const QString &reason ) override;
private:
QString mUrl;
std::vector<QgsOapifCollection> mCollections;
QString mNextUrl;
ApplicationLevelError mAppLevelError = ApplicationLevelError::NoError;
};
//! Manages the /collection/{collectionId} request
class QgsOapifCollectionRequest : public QgsBaseNetworkRequest
{
Q_OBJECT
public:
explicit QgsOapifCollectionRequest( const QgsDataSourceUri &baseUri, const QString &url );
//! Issue the request
bool request( bool synchronous, bool forceRefresh );
//! Application level error
enum class ApplicationLevelError
{
NoError,
JsonError,
IncompleteInformation
};
//! Returns application level error
ApplicationLevelError applicationLevelError() const { return mAppLevelError; }
//! Returns collection description.
const QgsOapifCollection &collection() const { return mCollection; }
signals:
//! emitted when the capabilities have been fully parsed, or an error occurred */
void gotResponse();
private slots:
void processReply();
protected:
QString errorMessageWithReason( const QString &reason ) override;
private:
QString mUrl;
QgsOapifCollection mCollection;
ApplicationLevelError mAppLevelError = ApplicationLevelError::NoError;
};
#endif // QGSOAPIFCOLLECTION_H

175
src/providers/wfs/qgsoapifitemsrequest.cpp Normal file
View File

@ -0,0 +1,175 @@
/***************************************************************************
qgsoapifitemsrequest.cpp
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include <nlohmann/json.hpp>
using namespace nlohmann;
#include "qgslogger.h"
#include "qgsoapifitemsrequest.h"
#include "qgsoapifutils.h"
#include "qgswfsconstants.h"
#include "qgsproviderregistry.h"
#include "cpl_vsi.h"
#include <QTextCodec>
QgsOapifItemsRequest::QgsOapifItemsRequest( const QgsDataSourceUri &baseUri, const QString &url ):
QgsBaseNetworkRequest( QgsAuthorizationSettings( baseUri.username(), baseUri.password(), baseUri.authConfigId() ), tr( "OAPIF" ) ),
mUrl( url )
{
// Using Qt::DirectConnection since the download might be running on a different thread.
// In this case, the request was sent from the main thread and is executed with the main
// thread being blocked in future.waitForFinished() so we can run code on this object which
// lives in the main thread without risking havoc.
connect( this, &QgsBaseNetworkRequest::downloadFinished, this, &QgsOapifItemsRequest::processReply, Qt::DirectConnection );
}
bool QgsOapifItemsRequest::request( bool synchronous, bool forceRefresh )
{
if ( !sendGET( QUrl( mUrl ), QString( "application/geo+json, application/json" ), synchronous, forceRefresh ) )
{
emit gotResponse();
return false;
}
return true;
}
QString QgsOapifItemsRequest::errorMessageWithReason( const QString &reason )
{
return tr( "Download of items failed: %1" ).arg( reason );
}
void QgsOapifItemsRequest::processReply()
{
if ( mErrorCode != QgsBaseNetworkRequest::NoError )
{
emit gotResponse();
return;
}
const QByteArray &buffer = mResponse;
if ( buffer.isEmpty() )
{
mErrorMessage = tr( "empty response" );
mErrorCode = QgsBaseNetworkRequest::ServerExceptionError;
emit gotResponse();
return;
}
QgsDebugMsgLevel( QStringLiteral( "parsing items response: " ) + buffer, 4 );
QTextCodec::ConverterState state;
QTextCodec *codec = QTextCodec::codecForName( "UTF-8" );
Q_ASSERT( codec );
const QString utf8Text = codec->toUnicode( buffer.constData(), buffer.size(), &state );
if ( state.invalidChars != 0 )
{
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::JsonError;
mErrorMessage = errorMessageWithReason( tr( "Invalid UTF-8 content" ) );
emit gotResponse();
return;
}
QString vsimemFilename;
vsimemFilename.sprintf( "/vsimem/oaipf_%p.json", &buffer );
VSIFCloseL( VSIFileFromMemBuffer( vsimemFilename.toUtf8().constData(),
const_cast<GByte *>( reinterpret_cast<const GByte *>( buffer.constData() ) ),
buffer.size(),
false ) );
QgsProviderRegistry *pReg = QgsProviderRegistry::instance();
QgsDataProvider::ProviderOptions providerOptions;
auto vectorProvider = std::unique_ptr<QgsVectorDataProvider>(
qobject_cast< QgsVectorDataProvider * >( pReg->createProvider( "ogr", vsimemFilename, providerOptions ) ) );
if ( !vectorProvider || !vectorProvider->isValid() )
{
VSIUnlink( vsimemFilename.toUtf8().constData() );
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::JsonError;
mErrorMessage = errorMessageWithReason( tr( "Loading of items failed" ) );
emit gotResponse();
return;
}
mFields = vectorProvider->fields();
mWKBType = vectorProvider->wkbType();
if ( mComputeBbox )
{
mBbox = vectorProvider->extent();
}
auto iter = vectorProvider->getFeatures();
while ( true )
{
QgsFeature f;
if ( !iter.nextFeature( f ) )
break;
mFeatures.push_back( QgsFeatureUniqueIdPair( f, QString() ) );
}
vectorProvider.reset();
VSIUnlink( vsimemFilename.toUtf8().constData() );
try
{
const json j = json::parse( utf8Text.toStdString() );
if ( j.is_object() && j.contains( "features" ) )
{
const auto features = j["features"];
if ( features.is_array() && features.size() == mFeatures.size() )
{
for ( size_t i = 0; i < features.size(); i++ )
{
const auto &jFeature = features[i];
if ( jFeature.is_object() && jFeature.contains( "id" ) )
{
const auto id = jFeature["id"];
if ( id.is_string() )
{
mFeatures[i].second = QString::fromStdString( id.get<std::string>() );
}
else if ( id.is_number_integer() )
{
mFeatures[i].second = QStringLiteral( "%1" ).arg( id.get<qint64>() );
}
}
}
}
}
const auto links = QgsOAPIFJson::parseLinks( j );
mNextUrl = QgsOAPIFJson::findLink( links,
QStringLiteral( "next" ),
QStringList() << QStringLiteral( "application/geo+json" ) );
if ( j.is_object() && j.contains( "numberMatched" ) )
{
const auto numberMatched = j["numberMatched"];
if ( numberMatched.is_number_integer() )
{
mNumberMatched = numberMatched.get<int>();
}
}
}
catch ( const json::parse_error &ex )
{
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::JsonError;
mErrorMessage = errorMessageWithReason( tr( "Cannot decode JSon document: %1" ).arg( QString::fromStdString( ex.what() ) ) );
emit gotResponse();
return;
}
emit gotResponse();
}

102
src/providers/wfs/qgsoapifitemsrequest.h Normal file
View File

@ -0,0 +1,102 @@
/***************************************************************************
qgsoapifitemsrequest.h
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSOAPIFITEMSREQUEST_H
#define QGSOAPIFITEMSREQUEST_H
#include <QObject>
#include "qgsdatasourceuri.h"
#include "qgsbasenetworkrequest.h"
#include "qgsfeature.h"
#include "qgsbackgroundcachedfeatureiterator.h"
#include "qgsrectangle.h"
#include <vector>
//! Manages the /items request
class QgsOapifItemsRequest : public QgsBaseNetworkRequest
{
Q_OBJECT
public:
explicit QgsOapifItemsRequest( const QgsDataSourceUri &uri, const QString &url );
//! Ask to compute the bbox of the returned items.
void setComputeBbox() { mComputeBbox = true; }
//! Issue the request
bool request( bool synchronous, bool forceRefresh );
//! Application level error
enum class ApplicationLevelError
{
NoError,
JsonError,
IncompleteInformation
};
//! Returns application level error
ApplicationLevelError applicationLevelError() const { return mAppLevelError; }
//! Return fields.
const QgsFields &fields() const { return mFields; }
//! Return geometry type.
QgsWkbTypes::Type wkbType() const { return mWKBType; }
//! Return features.
const std::vector<QgsFeatureUniqueIdPair> &features() const { return mFeatures; }
//! Return features bounding box
const QgsRectangle &bbox() const { return mBbox; }
//! Return number of matched features, or -1 if unknown.
int numberMatched() const { return mNumberMatched; }
//! Return the url of the next page
const QString &nextUrl() const { return mNextUrl; }
signals:
//! emitted when the capabilities have been fully parsed, or an error occurred
void gotResponse();
private slots:
void processReply();
protected:
QString errorMessageWithReason( const QString &reason ) override;
private:
QString mUrl;
bool mComputeBbox = false;
QgsFields mFields;
QgsWkbTypes::Type mWKBType = QgsWkbTypes::Unknown;
std::vector<QgsFeatureUniqueIdPair> mFeatures;
QgsRectangle mBbox;
int mNumberMatched = -1;
QString mNextUrl;
ApplicationLevelError mAppLevelError = ApplicationLevelError::NoError;
};
#endif // QGSOAPIFITEMSREQUEST_H

146
src/providers/wfs/qgsoapiflandingpagerequest.cpp Normal file
View File

@ -0,0 +1,146 @@
/***************************************************************************
qgsoapiflandingpagerequest.cpp
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include <nlohmann/json.hpp>
using namespace nlohmann;
#include "qgslogger.h"
#include "qgsoapiflandingpagerequest.h"
#include "qgsoapifutils.h"
#include "qgswfsconstants.h"
#include <QTextCodec>
QgsOapifLandingPageRequest::QgsOapifLandingPageRequest( const QgsDataSourceUri &uri ):
QgsBaseNetworkRequest( QgsAuthorizationSettings( uri.username(), uri.password(), uri.authConfigId() ), "OAPIF" ),
mUri( uri )
{
// Using Qt::DirectConnection since the download might be running on a different thread.
// In this case, the request was sent from the main thread and is executed with the main
// thread being blocked in future.waitForFinished() so we can run code on this object which
// lives in the main thread without risking havoc.
connect( this, &QgsBaseNetworkRequest::downloadFinished, this, &QgsOapifLandingPageRequest::processReply, Qt::DirectConnection );
}
bool QgsOapifLandingPageRequest::request( bool synchronous, bool forceRefresh )
{
if ( !sendGET( QUrl( mUri.param( QgsWFSConstants::URI_PARAM_URL ) ), QString( "application/json" ), synchronous, forceRefresh ) )
{
emit gotResponse();
return false;
}
return true;
}
QString QgsOapifLandingPageRequest::errorMessageWithReason( const QString &reason )
{
return tr( "Download of landing page failed: %1" ).arg( reason );
}
void QgsOapifLandingPageRequest::processReply()
{
if ( mErrorCode != QgsBaseNetworkRequest::NoError )
{
emit gotResponse();
return;
}
const QByteArray &buffer = mResponse;
if ( buffer.isEmpty() )
{
mErrorMessage = tr( "empty response" );
mErrorCode = QgsBaseNetworkRequest::ServerExceptionError;
emit gotResponse();
return;
}
QgsDebugMsgLevel( QStringLiteral( "parsing GetLandingPage response: " ) + buffer, 4 );
QTextCodec::ConverterState state;
QTextCodec *codec = QTextCodec::codecForName( "UTF-8" );
Q_ASSERT( codec );
const QString utf8Text = codec->toUnicode( buffer.constData(), buffer.size(), &state );
if ( state.invalidChars != 0 )
{
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::JsonError;
mErrorMessage = errorMessageWithReason( tr( "Invalid UTF-8 content" ) );
emit gotResponse();
return;
}
try
{
const json j = json::parse( utf8Text.toStdString() );
const auto links = QgsOAPIFJson::parseLinks( j );
QStringList apiTypes;
apiTypes << QStringLiteral( "application/vnd.oai.openapi+json;version=3.0" );
#ifndef REMOVE_SUPPORT_DRAFT_VERSIONS
apiTypes << QStringLiteral( "application/openapi+json;version=3.0" );
#endif
mApiUrl = QgsOAPIFJson::findLink( links,
QStringLiteral( "service-desc" ),
apiTypes );
#ifndef REMOVE_SUPPORT_DRAFT_VERSIONS
if ( mApiUrl.isEmpty() )
{
mApiUrl = QgsOAPIFJson::findLink( links,
QStringLiteral( "service" ),
apiTypes );
}
#endif
QStringList collectionsTypes;
collectionsTypes << QStringLiteral( "application/json" );
mCollectionsUrl = QgsOAPIFJson::findLink( links,
QStringLiteral( "data" ),
collectionsTypes );
#ifndef REMOVE_SUPPORT_DRAFT_VERSIONS
if ( mCollectionsUrl.isEmpty() )
{
mCollectionsUrl = QgsOAPIFJson::findLink( links,
QStringLiteral( "collections" ),
apiTypes );
}
#endif
}
catch ( const json::parse_error &ex )
{
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::JsonError;
mErrorMessage = errorMessageWithReason( tr( "Cannot decode JSon document: %1" ).arg( QString::fromStdString( ex.what() ) ) );
emit gotResponse();
return;
}
// Strip off suffixex like /collections?f=json
auto posQuotationMark = mCollectionsUrl.indexOf( '?' );
if ( posQuotationMark > 0 )
{
mCollectionsUrl = mCollectionsUrl.mid( 0, posQuotationMark );
}
if ( mApiUrl.isEmpty() || mCollectionsUrl.isEmpty() )
{
mErrorCode = QgsBaseNetworkRequest::ApplicationLevelError;
mAppLevelError = ApplicationLevelError::IncompleteInformation;
mErrorMessage = errorMessageWithReason( tr( "Missing information in response" ) );
emit gotResponse();
return;
}
emit gotResponse();
}

74
src/providers/wfs/qgsoapiflandingpagerequest.h Normal file
View File

@ -0,0 +1,74 @@
/***************************************************************************
qgsoapiflandingpagerequest.h
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSOAPIFLANDINGPAGEREQUEST_H
#define QGSOAPIFLANDINGPAGEREQUEST_H
#include <QObject>
#include "qgsdatasourceuri.h"
#include "qgsbasenetworkrequest.h"
//! Manages the GetLandingPage request
class QgsOapifLandingPageRequest : public QgsBaseNetworkRequest
{
Q_OBJECT
public:
explicit QgsOapifLandingPageRequest( const QgsDataSourceUri &uri );
//! Issue the request
bool request( bool synchronous, bool forceRefresh );
//! Application level error
enum class ApplicationLevelError
{
NoError,
JsonError,
IncompleteInformation
};
//! Returns application level error
ApplicationLevelError applicationLevelError() const { return mAppLevelError; }
//! Return URL of the api endpoint
const QString &apiUrl() const { return mApiUrl; }
//! Return URL of the api endpoint
const QString &collectionsUrl() const { return mCollectionsUrl; }
signals:
//! emitted when the capabilities have been fully parsed, or an error occurred
void gotResponse();
private slots:
void processReply();
protected:
QString errorMessageWithReason( const QString &reason ) override;
private:
QgsDataSourceUri mUri;
//! URL of the api endpoint.
QString mApiUrl;
//! URL of the collections endpoint.
QString mCollectionsUrl;
ApplicationLevelError mAppLevelError = ApplicationLevelError::NoError;
};
#endif // QGSOAPIFLANDINGPAGEREQUEST_H

492
src/providers/wfs/qgsoapifprovider.cpp Normal file
View File

@ -0,0 +1,492 @@
/***************************************************************************
qgsoapifprovider.cpp
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include "qgslogger.h"
#include "qgsmessagelog.h"
#include "qgsoapifprovider.h"
#include "qgsoapiflandingpagerequest.h"
#include "qgsoapifapirequest.h"
#include "qgsoapifcollection.h"
#include "qgsoapifitemsrequest.h"
#include <algorithm>
const QString QgsOapifProvider::OAPIF_PROVIDER_KEY = QStringLiteral( "OAPIF" );
const QString QgsOapifProvider::OAPIF_PROVIDER_DESCRIPTION = QStringLiteral( "OGC API - Features data provider" );
QgsOapifProvider::QgsOapifProvider( const QString &uri, const ProviderOptions &options )
: QgsVectorDataProvider( uri, options ),
mShared( new QgsOapifSharedData( uri ) )
{
connect( mShared.get(), &QgsOapifSharedData::raiseError, this, &QgsOapifProvider::pushErrorSlot );
connect( mShared.get(), &QgsOapifSharedData::extentUpdated, this, &QgsOapifProvider::fullExtentCalculated );
if ( uri.isEmpty() )
{
mValid = false;
return;
}
if ( !init() )
{
mValid = false;
return;
}
mShared->mSourceCrs = QgsCoordinateReferenceSystem::fromOgcWmsCrs( QStringLiteral( "EPSG:4326" ) );
qRegisterMetaType<QgsRectangle>( "QgsRectangle" );
}
QgsOapifProvider::~QgsOapifProvider()
{
}
bool QgsOapifProvider::init()
{
const bool synchronous = true;
const bool forceRefresh = false;
QgsOapifLandingPageRequest landingPageRequest( mShared->mURI.uri() );
if ( !landingPageRequest.request( synchronous, forceRefresh ) )
return false;
if ( landingPageRequest.errorCode() != QgsBaseNetworkRequest::NoError )
return false;
QgsOapifApiRequest apiRequest( mShared->mURI.uri(), landingPageRequest.apiUrl() );
if ( !apiRequest.request( synchronous, forceRefresh ) )
return false;
if ( apiRequest.errorCode() != QgsBaseNetworkRequest::NoError )
return false;
mShared->mServerMaxFeatures = apiRequest.maxLimit();
if ( mShared->mURI.maxNumFeatures() > 0 && mShared->mServerMaxFeatures > 0 && !mShared->mURI.pagingEnabled() )
{
mShared->mMaxFeatures = std::min( mShared->mURI.maxNumFeatures(), mShared->mServerMaxFeatures );
}
else if ( mShared->mURI.maxNumFeatures() > 0 )
{
mShared->mMaxFeatures = mShared->mURI.maxNumFeatures();
}
else if ( mShared->mServerMaxFeatures > 0 && !mShared->mURI.pagingEnabled() )
{
mShared->mMaxFeatures = mShared->mServerMaxFeatures;
}
if ( mShared->mURI.pagingEnabled() && mShared->mURI.pageSize() > 0 )
{
if ( mShared->mServerMaxFeatures > 0 )
{
mShared->mPageSize = std::min( mShared->mURI.pageSize(), mShared->mServerMaxFeatures );
}
else
{
mShared->mPageSize = mShared->mURI.pageSize();
}
}
else if ( mShared->mURI.pagingEnabled() )
{
if ( apiRequest.defaultLimit() > 0 && apiRequest.maxLimit() > 0 )
{
// Use the default, but if it is below 1000, aim for 1000
// but clamp to the maximum limit
mShared->mPageSize = std::min( std::max( 1000, apiRequest.defaultLimit() ), apiRequest.maxLimit() );
}
else if ( apiRequest.defaultLimit() > 0 )
mShared->mPageSize = apiRequest.defaultLimit();
else if ( apiRequest.maxLimit() > 0 )
mShared->mPageSize = apiRequest.maxLimit();
else
mShared->mPageSize = 100; // fallback to arbitrary page size
}
mShared->mCollectionUrl =
landingPageRequest.collectionsUrl() + QStringLiteral( "/" ) + mShared->mURI.typeName();
QgsOapifCollectionRequest collectionRequest( mShared->mURI.uri(), mShared->mCollectionUrl );
if ( !collectionRequest.request( synchronous, forceRefresh ) )
return false;
if ( collectionRequest.errorCode() != QgsBaseNetworkRequest::NoError )
return false;
mShared->mCapabilityExtent = collectionRequest.collection().mBbox;
mShared->mItemsUrl = mShared->mCollectionUrl + QStringLiteral( "/items" );
QgsOapifItemsRequest itemsRequest( mShared->mURI.uri(), mShared->mItemsUrl + QStringLiteral( "?limit=10" ) );
if ( mShared->mCapabilityExtent.isNull() )
{
itemsRequest.setComputeBbox();
}
if ( !itemsRequest.request( synchronous, forceRefresh ) )
return false;
if ( itemsRequest.errorCode() != QgsBaseNetworkRequest::NoError )
return false;
if ( itemsRequest.numberMatched() >= 0 )
{
mShared->setFeatureCount( itemsRequest.numberMatched() );
}
if ( mShared->mCapabilityExtent.isNull() )
{
mShared->mCapabilityExtent = itemsRequest.bbox();
}
mShared->mFields = itemsRequest.fields();
mShared->mWKBType = itemsRequest.wkbType();
return true;
}
void QgsOapifProvider::pushErrorSlot( const QString &errorMsg )
{
pushError( errorMsg );
}
QgsAbstractFeatureSource *QgsOapifProvider::featureSource() const
{
return new QgsBackgroundCachedFeatureSource( mShared );
}
QgsFeatureIterator QgsOapifProvider::getFeatures( const QgsFeatureRequest &request ) const
{
return QgsFeatureIterator( new QgsBackgroundCachedFeatureIterator( new QgsBackgroundCachedFeatureSource( mShared ), true, mShared, request ) );
}
QgsWkbTypes::Type QgsOapifProvider::wkbType() const
{
return mShared->mWKBType;
}
long QgsOapifProvider::featureCount() const
{
return mShared->getFeatureCount();
}
QgsFields QgsOapifProvider::fields() const
{
return mShared->mFields;
}
QgsCoordinateReferenceSystem QgsOapifProvider::crs() const
{
return mShared->mSourceCrs;
}
QgsRectangle QgsOapifProvider::extent() const
{
return mShared->consolidatedExtent();
}
void QgsOapifProvider::reloadData()
{
mShared->invalidateCache();
QgsVectorDataProvider::reloadData();
}
bool QgsOapifProvider::isValid() const
{
return mValid;
}
QgsVectorDataProvider::Capabilities QgsOapifProvider::capabilities() const
{
return QgsVectorDataProvider::SelectAtId;
}
bool QgsOapifProvider::empty() const
{
if ( subsetString().isEmpty() && mShared->isFeatureCountExact() )
{
return mShared->getFeatureCount( false ) == 0;
}
QgsFeature f;
QgsFeatureRequest request;
request.setNoAttributes();
request.setFlags( QgsFeatureRequest::NoGeometry );
// Whoops, the provider returns an empty iterator when we are using
// a setLimit call in combination with a subsetString.
// Remove this method (and default to the QgsVectorDataProvider one)
// once this is fixed
#if 0
request.setLimit( 1 );
#endif
return !getFeatures( request ).nextFeature( f );
};
QString QgsOapifProvider::name() const
{
return OAPIF_PROVIDER_KEY;
}
QString QgsOapifProvider::description() const
{
return OAPIF_PROVIDER_DESCRIPTION;
}
// ---------------------------------
QgsOapifSharedData::QgsOapifSharedData( const QString &uri )
: QgsBackgroundCachedSharedData( "oapif", tr( "OAPIF" ) )
, mURI( uri )
, mHideProgressDialog( mURI.hideDownloadProgressDialog() )
{
}
QgsOapifSharedData::~QgsOapifSharedData()
{
QgsDebugMsgLevel( QStringLiteral( "~QgsOapifSharedData()" ), 4 );
cleanup();
}
bool QgsOapifSharedData::isRestrictedToRequestBBOX() const
{
return mURI.isRestrictedToRequestBBOX();
}
std::unique_ptr<QgsFeatureDownloaderImpl> QgsOapifSharedData::newFeatureDownloaderImpl( QgsFeatureDownloader *downloader )
{
return std::unique_ptr<QgsFeatureDownloaderImpl>( new QgsOapifFeatureDownloaderImpl( this, downloader ) );
}
void QgsOapifSharedData::invalidateCacheBaseUnderLock()
{
}
void QgsOapifSharedData::pushError( const QString &errorMsg )
{
QgsMessageLog::logMessage( errorMsg, tr( "OAPIF" ) );
emit raiseError( errorMsg );
}
// ---------------------------------
QgsOapifFeatureDownloaderImpl::QgsOapifFeatureDownloaderImpl( QgsOapifSharedData *shared, QgsFeatureDownloader *downloader ):
QgsFeatureDownloaderImpl( downloader ),
mShared( shared )
{
}
QgsOapifFeatureDownloaderImpl::~QgsOapifFeatureDownloaderImpl()
{
stop();
}
void QgsOapifFeatureDownloaderImpl::stop()
{
QgsDebugMsgLevel( QStringLiteral( "QgsOapifFeatureDownloaderImpl::stop()" ), 4 );
mStop = true;
emit doStop();
}
void QgsOapifFeatureDownloaderImpl::run( bool serializeFeatures, int maxFeatures )
{
QEventLoop loop;
connect( this, &QgsOapifFeatureDownloaderImpl::doStop, &loop, &QEventLoop::quit );
qint64 maxTotalFeatures = 0;
if ( maxFeatures > 0 && mShared->mMaxFeatures > 0 )
{
maxTotalFeatures = std::min( maxFeatures, mShared->mMaxFeatures );
}
else if ( maxFeatures > 0 )
{
maxTotalFeatures = maxFeatures;
}
else
{
maxTotalFeatures = mShared->mMaxFeatures;
}
qint64 totalDownloadedFeatureCount = 0;
bool interrupted = false;
bool success = true;
QString errorMessage;
QString url;
int maxFeaturesThisRequest = maxTotalFeatures;
if ( mShared->mPageSize > 0 )
{
if ( maxFeaturesThisRequest > 0 )
{
maxFeaturesThisRequest = std::min( maxFeaturesThisRequest, mShared->mPageSize );
}
else
{
maxFeaturesThisRequest = mShared->mPageSize;
}
}
url = mShared->mItemsUrl;
if ( maxFeaturesThisRequest > 0 )
{
url += QStringLiteral( "?limit=%1" ).arg( maxFeaturesThisRequest );
}
const auto &rect = mShared->currentRect();
if ( !rect.isNull() )
{
// Clamp to avoid server errors.
double minx = std::max( -180.0, rect.xMinimum() );
double miny = std::max( -90.0, rect.yMinimum() );
double maxx = std::min( 180.0, rect.xMaximum() );
double maxy = std::min( 90.0, rect.yMaximum() );
if ( minx > 180.0 || miny > 90.0 || maxx < -180.0 || maxy < -90.0 )
{
// completely out of range. Servers could error out
url.clear();
}
else if ( minx > -180.0 || miny > -90.0 || maxx < 180.0 || maxy < 90.0 )
{
if ( maxFeaturesThisRequest > 0 )
{
url += QStringLiteral( "&" );
}
else
{
url += QStringLiteral( "?" );
}
url += QStringLiteral( "bbox=%1,%2,%3,%4" )
.arg( qgsDoubleToString( minx ),
qgsDoubleToString( miny ),
qgsDoubleToString( maxx ),
qgsDoubleToString( maxy ) );
}
}
while ( !url.isEmpty() )
{
if ( maxTotalFeatures > 0 && totalDownloadedFeatureCount >= maxTotalFeatures )
{
break;
}
QgsOapifItemsRequest itemsRequest( mShared->mURI.uri(), url );
connect( &itemsRequest, &QgsOapifItemsRequest::gotResponse, &loop, &QEventLoop::quit );
itemsRequest.request( false /* synchronous*/, true /* forceRefresh */ );
loop.exec( QEventLoop::ExcludeUserInputEvents );
if ( mStop )
{
interrupted = true;
success = false;
break;
}
if ( itemsRequest.errorCode() != QgsBaseNetworkRequest::NoError )
{
errorMessage = itemsRequest.errorMessage();
success = false;
break;
}
if ( itemsRequest.features().empty() )
{
break;
}
url = itemsRequest.nextUrl();
totalDownloadedFeatureCount += itemsRequest.features().size();
emit updateProgress( totalDownloadedFeatureCount );
QVector<QgsFeatureUniqueIdPair> featureList;
size_t i = 0;
const auto &srcFields = itemsRequest.fields();
const auto &dstFields = mShared->fields();
for ( const auto &pair : itemsRequest.features() )
{
// In the case the features of the current page have not the same schema
// as the layer, convert them
const QgsFeature &f = pair.first;
QgsFeature dstFeat( dstFields, f.id() );
dstFeat.setGeometry( f.geometry() );
const auto srcAttrs = f.attributes();
for ( int j = 0; j < dstFields.size(); j++ )
{
int srcIdx = srcFields.indexOf( dstFields[j].name() );
if ( srcIdx >= 0 )
{
const QVariant &v = srcAttrs.value( srcIdx );
const auto dstFieldType = dstFields.at( j ).type();
if ( v.isNull() )
dstFeat.setAttribute( j, QVariant( dstFieldType ) );
else if ( v.type() == dstFieldType )
dstFeat.setAttribute( j, v );
else
dstFeat.setAttribute( j, QgsVectorDataProvider::convertValue( dstFieldType, v.toString() ) );
}
}
QString uniqueId( pair.second );
if ( uniqueId.isEmpty() )
{
uniqueId = QgsBackgroundCachedSharedData::getMD5( f );
}
featureList.push_back( QgsFeatureUniqueIdPair( dstFeat, uniqueId ) );
if ( ( i > 0 && ( i % 1000 ) == 0 ) || i + 1 == itemsRequest.features().size() )
{
// We call it directly to avoid asynchronous signal notification, and
// as serializeFeatures() can modify the featureList to remove features
// that have already been cached, so as to avoid to notify them several
// times to subscribers
if ( serializeFeatures )
mShared->serializeFeatures( featureList );
if ( !featureList.isEmpty() )
{
emitFeatureReceived( featureList );
emitFeatureReceived( featureList.size() );
}
featureList.clear();
}
i++;
}
if ( mShared->mPageSize <= 0 )
{
break;
}
}
if ( serializeFeatures )
mShared->endOfDownload( success, totalDownloadedFeatureCount, false /* truncatedResponse */, interrupted, errorMessage );
// We must emit the signal *AFTER* the previous call to mShared->endOfDownload()
// to avoid issues with iterators that would start just now, wouldn't detect
// that the downloader has finished, would register to itself, but would never
// receive the endOfDownload signal. This is not just a theoretical problem.
// If you switch both calls, it happens quite easily in Release mode with the
// test suite.
emitEndOfDownload( success );
}
// ---------------------------------
QgsOapifProvider *QgsOapifProviderMetadata::createProvider( const QString &uri, const QgsDataProvider::ProviderOptions &options )
{
return new QgsOapifProvider( uri, options );
}
QgsOapifProviderMetadata::QgsOapifProviderMetadata():
QgsProviderMetadata( QgsOapifProvider::OAPIF_PROVIDER_KEY, QgsOapifProvider::OAPIF_PROVIDER_DESCRIPTION ) {}

192
src/providers/wfs/qgsoapifprovider.h Normal file
View File

@ -0,0 +1,192 @@
/***************************************************************************
qgsoapifprovider.h
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSOAPIFPROVIDER_H
#define QGSOAPIFPROVIDER_H
#include "qgis.h"
#include "qgsrectangle.h"
#include "qgscoordinatereferencesystem.h"
#include "qgsvectordataprovider.h"
#include "qgsbackgroundcachedshareddata.h"
#include "qgswfsdatasourceuri.h"
#include "qgsprovidermetadata.h"
class QgsOapifSharedData;
class QgsOapifProvider : public QgsVectorDataProvider
{
Q_OBJECT
public:
static const QString OAPIF_PROVIDER_KEY;
static const QString OAPIF_PROVIDER_DESCRIPTION;
explicit QgsOapifProvider( const QString &uri, const QgsDataProvider::ProviderOptions &providerOptions );
~QgsOapifProvider() override;
/* Inherited from QgsVectorDataProvider */
QgsAbstractFeatureSource *featureSource() const override;
QgsFeatureIterator getFeatures( const QgsFeatureRequest &request = QgsFeatureRequest() ) const override;
QgsWkbTypes::Type wkbType() const override;
long featureCount() const override;
QgsFields fields() const override;
QgsCoordinateReferenceSystem crs() const override;
//QString subsetString() const override;
//bool setSubsetString( const QString &theSQL, bool updateFeatureCount = true ) override;
//bool supportsSubsetString() const override { return true; }
bool supportsSubsetString() const override { return false; }
/* Inherited from QgsDataProvider */
QgsRectangle extent() const override;
bool isValid() const override;
QString name() const override;
QString description() const override;
QgsVectorDataProvider::Capabilities capabilities() const override;
bool empty() const override;
public slots:
void reloadData() override;
private slots:
void pushErrorSlot( const QString &errorMsg );
private:
std::shared_ptr<QgsOapifSharedData> mShared;
//! Flag if provider is valid
bool mValid = true;
//! Initial requests
bool init();
};
class QgsOapifProviderMetadata: public QgsProviderMetadata
{
public:
QgsOapifProviderMetadata();
QgsOapifProvider *createProvider( const QString &uri, const QgsDataProvider::ProviderOptions &options ) override;
};
// !Class shared between provider and feature source
class QgsOapifSharedData : public QObject, public QgsBackgroundCachedSharedData
{
Q_OBJECT
public:
explicit QgsOapifSharedData( const QString &uri );
~QgsOapifSharedData() override;
bool hasGeometry() const override { return mWKBType != QgsWkbTypes::Unknown; }
std::unique_ptr<QgsFeatureDownloaderImpl> newFeatureDownloaderImpl( QgsFeatureDownloader * ) override;
bool isRestrictedToRequestBBOX() const override;
signals:
//! Raise error
void raiseError( const QString &errorMsg );
//! Extent has been updated
void extentUpdated();
protected:
friend class QgsOapifProvider;
friend class QgsOapifFeatureDownloaderImpl;
//! Datasource URI
QgsWFSDataSourceURI mURI;
//! Geometry type of the features in this layer
QgsWkbTypes::Type mWKBType = QgsWkbTypes::Unknown;
//! Page size. 0 = disabled
int mPageSize = 0;
//! Whether progress dialog should be hidden
bool mHideProgressDialog = false;
//! Url to /collections/{collectionId}
QString mCollectionUrl;
//! Url to /collections/{collectionId}/items
QString mItemsUrl;
private:
//! Log error to QgsMessageLog and raise it to the provider
void pushError( const QString &errorMsg ) override;
void emitExtentUpdated() override { emit extentUpdated(); }
void invalidateCacheBaseUnderLock() override;
bool supportsLimitedFeatureCountDownloads() const override { return true; }
QString layerName() const override { return mURI.typeName(); }
bool hasServerSideFilter() const override { return false; }
bool supportsFastFeatureCount() const override { return false; }
QgsRectangle getExtentFromSingleFeatureRequest() const override { return QgsRectangle(); }
int getFeatureCountFromServer() const override { return -1; }
};
class QgsOapifFeatureDownloaderImpl: public QObject, public QgsFeatureDownloaderImpl
{
Q_OBJECT
public:
QgsOapifFeatureDownloaderImpl( QgsOapifSharedData *shared, QgsFeatureDownloader *downloader );
~QgsOapifFeatureDownloaderImpl() override;
void run( bool serializeFeatures, int maxFeatures ) override;
void stop() override;
signals:
//! Used internally by the stop() method
void doStop();
//! Emitted with the total accumulated number of features downloaded.
void updateProgress( int totalFeatureCount );
private:
//! Mutable data shared between provider, feature sources and downloader.
QgsOapifSharedData *mShared = nullptr;
//! Whether the download should stop
bool mStop = false;
};
#endif /* QGSOAPIFPROVIDER_H */

85
src/providers/wfs/qgsoapifutils.cpp Normal file
View File

@ -0,0 +1,85 @@
/***************************************************************************
qgsoapifutils.cpp
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include "qgsoapifutils.h"
#include <limits>
std::vector<QgsOAPIFJson::Link> QgsOAPIFJson::parseLinks( const json &jParent )
{
std::vector<Link> links;
if ( jParent.is_object() && jParent.contains( "links" ) )
{
const auto jLinks = jParent["links"];
if ( jLinks.is_array() )
{
for ( const auto &jLink : jLinks )
{
if ( jLink.is_object() &&
jLink.contains( "href" ) &&
jLink.contains( "rel" ) )
{
const auto href = jLink["href"];
const auto rel = jLink["rel"];
if ( href.is_string() && rel.is_string() )
{
Link link;
link.href = QString::fromStdString( href.get<std::string>() );
link.rel = QString::fromStdString( rel.get<std::string>() );
if ( jLink.contains( "type" ) )
{
const auto type = jLink["type"];
if ( type.is_string() )
{
link.type = QString::fromStdString( type.get<std::string>() );
}
}
links.push_back( link );
}
}
}
}
}
return links;
}
QString QgsOAPIFJson::findLink( const std::vector<QgsOAPIFJson::Link> &links,
const QString &rel,
const QStringList &preferableTypes )
{
QString resultHref;
int resultPriority = std::numeric_limits<int>::max();
for ( const auto &link : links )
{
if ( link.rel == rel )
{
int priority = -1;
if ( !link.type.isEmpty() && !preferableTypes.isEmpty() )
{
priority = preferableTypes.indexOf( link.type );
}
if ( priority < 0 )
{
priority = static_cast<int>( preferableTypes.size() );
}
if ( priority < resultPriority )
{
resultHref = link.href;
resultPriority = priority;
}
}
}
return resultHref;
}

45
src/providers/wfs/qgsoapifutils.h Normal file
View File

@ -0,0 +1,45 @@
/***************************************************************************
qgsoapifutils.h
---------------------
begin : October 2019
copyright : (C) 2019 by Even Rouault
email : even.rouault at spatialys.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSOAPIFUTILS_H
#define QGSOAPIFUTILS_H
#include <nlohmann/json.hpp>
using namespace nlohmann;
#include <QString>
#include <QStringList>
/**
* Utility class */
class QgsOAPIFJson
{
public:
//! A OAPIF Link
struct Link
{
QString href;
QString rel;
QString type;
};
//! Parses the "link" property of jParet
static std::vector<Link> parseLinks( const json &jParent );
//! Find among links the one that matches rel, by using an optional list of preferable types.
static QString findLink( const std::vector<Link> &links, const QString &rel, const QStringList &preferableTypes = QStringList() );
};
#endif // QGSOAPIFUTILS_H

3
src/providers/wfs/qgswfsprovider.cpp
View File

@ -24,6 +24,7 @@
#include "qgslogger.h"
#include "qgsmessagelog.h"
#include "qgsogcutils.h"
#include "qgsoapifprovider.h"
#include "qgswfsdataitems.h"
#include "qgswfsconstants.h"
#include "qgswfsfeatureiterator.h"
@ -1871,5 +1872,5 @@ QgsWfsProviderMetadata::QgsWfsProviderMetadata():
QGISEXTERN std::vector<QgsProviderMetadata *> multipleProviderMetadataFactory()
{
return std::vector<QgsProviderMetadata *> { new QgsWfsProviderMetadata() };
return std::vector<QgsProviderMetadata *> { new QgsWfsProviderMetadata(), new QgsOapifProviderMetadata() };
}

1
tests/src/python/CMakeLists.txt
View File

@ -243,6 +243,7 @@ ADD_PYTHON_TEST(PyQgsVirtualLayerDefinition test_qgsvirtuallayerdefinition.py)
ADD_PYTHON_TEST(PyQgsLayerDefinition test_qgslayerdefinition.py)
ADD_PYTHON_TEST(PyQgsWFSProvider test_provider_wfs.py)
ADD_PYTHON_TEST(PyQgsWFSProviderGUI test_provider_wfs_gui.py)
ADD_PYTHON_TEST(PyQgsOapifProvider test_provider_oapif.py)
ADD_PYTHON_TEST(PyQgsConsole test_console.py)
ADD_PYTHON_TEST(PyQgsLayerDependencies test_layer_dependencies.py)
ADD_PYTHON_TEST(PyQgsVersionCompare test_versioncompare.py)

290
tests/src/python/test_provider_oapif.py Normal file
View File

@ -0,0 +1,290 @@
# -*- coding: utf-8 -*-
"""QGIS Unit tests for the OAPIF provider.
.. note:: This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
"""
__author__ = 'Even Rouault'
__date__ = '2019-10-12'
__copyright__ = 'Copyright 2019, Even Rouault'
import json
import hashlib
import os
import re
import shutil
import tempfile
from qgis.PyQt.QtCore import QCoreApplication, Qt, QObject, QDateTime
from qgis.core import (
QgsWkbTypes,
QgsVectorLayer,
QgsFeature,
QgsGeometry,
QgsRectangle,
QgsPointXY,
QgsVectorDataProvider,
QgsFeatureRequest,
QgsApplication,
QgsSettings,
QgsExpression,
QgsExpressionContextUtils,
QgsExpressionContext,
)
from qgis.testing import (start_app,
unittest
)
from providertestbase import ProviderTestCase
def sanitize(endpoint, x):
if len(endpoint + x) > 256:
ret = endpoint + hashlib.md5(x.replace('/', '_').encode()).hexdigest()
#print('Before: ' + endpoint + x)
#print('After: ' + ret)
return ret
ret = endpoint + x.replace('?', '_').replace('&', '_').replace('<', '_').replace('>', '_').replace('"', '_').replace("'", '_').replace(' ', '_').replace(':', '_').replace('/', '_').replace('\n', '_')
return ret
ACCEPT_LANDING = 'Accept=application/json'
ACCEPT_API = 'Accept=application/vnd.oai.openapi+json;version=3.0, application/openapi+json;version=3.0, application/json'
ACCEPT_COLLECTION = 'Accept=application/json'
ACCEPT_ITEMS = 'Accept=application/geo+json, application/json'
def create_landing_page_api_collection(endpoint):
# Landing page
with open(sanitize(endpoint, '?' + ACCEPT_LANDING), 'wb') as f:
f.write(json.dumps({
"links": [
{"href": "http://" + endpoint + "/api", "rel": "service-desc"},
{"href": "http://" + endpoint + "/collections", "rel": "data"}
]}).encode('UTF-8'))
# API
with open(sanitize(endpoint, '/api?' + ACCEPT_API), 'wb') as f:
f.write(json.dumps({
"components": {
"parameters": {
"limit": {
"schema": {
"maximum": 1000,
"default": 100
}
}
}
}
}).encode('UTF-8'))
# collection
with open(sanitize(endpoint, '/collections/mycollection?' + ACCEPT_COLLECTION), 'wb') as f:
f.write(json.dumps({
"id": "mycollection",
"title": "my title",
"description": "my description",
"extent": {
"spatial": {
"bbox": [
[-71.123, 66.33, -65.32, 78.3]
]
}
}
}).encode('UTF-8'))
class TestPyQgsOapifProvider(unittest.TestCase, ProviderTestCase):
@classmethod
def setUpClass(cls):
"""Run before all tests"""
QCoreApplication.setOrganizationName("QGIS_Test")
QCoreApplication.setOrganizationDomain("TestPyQgsOapifProvider.com")
QCoreApplication.setApplicationName("TestPyQgsOapifProvider")
QgsSettings().clear()
start_app()
# On Windows we must make sure that any backslash in the path is
# replaced by a forward slash so that QUrl can process it
cls.basetestpath = tempfile.mkdtemp().replace('\\', '/')
endpoint = cls.basetestpath + '/fake_qgis_http_endpoint'
create_landing_page_api_collection(endpoint)
items = {
"type": "FeatureCollection",
"features": [
{"type": "Feature", "id": "feat.1", "properties": {"pk": 1, "cnt": 100, "name": "Orange", "name2": "oranGe", "num_char": "1"}, "geometry": {"type": "Point", "coordinates": [-70.332, 66.33]}},
{"type": "Feature", "id": "feat.2", "properties": {"pk": 2, "cnt": 200, "name": "Apple", "name2": "Apple", "num_char": "2"}, "geometry": {"type": "Point", "coordinates": [-68.2, 70.8]}},
{"type": "Feature", "id": "feat.3", "properties": {"pk": 4, "cnt": 400, "name": "Honey", "name2": "Honey", "num_char": "4"}, "geometry": {"type": "Point", "coordinates": [-65.32, 78.3]}},
{"type": "Feature", "id": "feat.4", "properties": {"pk": 3, "cnt": 300, "name": "Pear", "name2": "PEaR", "num_char": "3"}, "geometry": None},
{"type": "Feature", "id": "feat.5", "properties": {"pk": 5, "cnt": -200, "name": None, "name2": "NuLl", "num_char": "5"}, "geometry": {"type": "Point", "coordinates": [-71.123, 78.23]}}
]
}
# first items
with open(sanitize(endpoint, '/collections/mycollection/items?limit=10&' + ACCEPT_ITEMS), 'wb') as f:
f.write(json.dumps(items).encode('UTF-8'))
# real page
with open(sanitize(endpoint, '/collections/mycollection/items?limit=1000&' + ACCEPT_ITEMS), 'wb') as f:
f.write(json.dumps(items).encode('UTF-8'))
# Create test layer
cls.vl = QgsVectorLayer("url='http://" + endpoint + "' typename='mycollection'", 'test', 'OAPIF')
assert cls.vl.isValid()
cls.source = cls.vl.dataProvider()
@classmethod
def tearDownClass(cls):
"""Run after all tests"""
QgsSettings().clear()
shutil.rmtree(cls.basetestpath, True)
cls.vl = None # so as to properly close the provider and remove any temporary file
def testFeaturePaging(self):
endpoint = self.__class__.basetestpath + '/fake_qgis_http_endpoint_testFeaturePaging'
create_landing_page_api_collection(endpoint)
# first items
first_items = {
"type": "FeatureCollection",
"features": [
{"type": "Feature", "id": "feat.1", "properties": {"pk": 1, "cnt": 100}, "geometry": {"type": "Point", "coordinates": [-70.332, 66.33]}}
]
}
with open(sanitize(endpoint, '/collections/mycollection/items?limit=10&' + ACCEPT_ITEMS), 'wb') as f:
f.write(json.dumps(first_items).encode('UTF-8'))
vl = QgsVectorLayer("url='http://" + endpoint + "' typename='mycollection'", 'test', 'OAPIF')
self.assertTrue(vl.isValid())
# first real page
first_page = {
"type": "FeatureCollection",
"features": [
{"type": "Feature", "id": "feat.1", "properties": {"pk": 1, "cnt": 100}, "geometry": {"type": "Point", "coordinates": [-70.332, 66.33]}},
{"type": "Feature", "id": "feat.2", "properties": {"pk": 2, "cnt": 200}, "geometry": {"type": "Point", "coordinates": [-68.2, 70.8]}}
],
"links": [
# Test multiple media types for next
{"href": "http://" + endpoint + "/second_page.html", "rel": "next", "type": "text/html"},
{"href": "http://" + endpoint + "/second_page", "rel": "next", "type": "application/geo+json"},
{"href": "http://" + endpoint + "/second_page.xml", "rel": "next", "type": "text/xml"}
]
}
with open(sanitize(endpoint, '/collections/mycollection/items?limit=1000&' + ACCEPT_ITEMS), 'wb') as f:
f.write(json.dumps(first_page).encode('UTF-8'))
# second page
second_page = {
"type": "FeatureCollection",
"features": [
# Also add a non expected property
{"type": "Feature", "id": "feat.3", "properties": {"a_non_expected": "foo", "pk": 4, "cnt": 400}, "geometry": {"type": "Point", "coordinates": [-65.32, 78.3]}}
],
"links": [
{"href": "http://" + endpoint + "/third_page", "rel": "next"}
]
}
with open(sanitize(endpoint, '/second_page?' + ACCEPT_ITEMS), 'wb') as f:
f.write(json.dumps(second_page).encode('UTF-8'))
# third page
third_page = {
"type": "FeatureCollection",
"features": [],
"links": [
{"href": "http://" + endpoint + "/third_page", "rel": "next"} # dummy link to ourselves
]
}
with open(sanitize(endpoint, '/third_page?' + ACCEPT_ITEMS), 'wb') as f:
f.write(json.dumps(third_page).encode('UTF-8'))
values = [f['pk'] for f in vl.getFeatures()]
self.assertEqual(values, [1, 2, 4])
values = [f['pk'] for f in vl.getFeatures()]
self.assertEqual(values, [1, 2, 4])
def testBbox(self):
endpoint = self.__class__.basetestpath + '/fake_qgis_http_endpoint_testBbox'
create_landing_page_api_collection(endpoint)
# first items
first_items = {
"type": "FeatureCollection",
"features": [
{"type": "Feature", "id": "feat.1", "properties": {"pk": 1, "cnt": 100}, "geometry": {"type": "Point", "coordinates": [-70.332, 66.33]}}
]
}
with open(sanitize(endpoint, '/collections/mycollection/items?limit=10&' + ACCEPT_ITEMS), 'wb') as f:
f.write(json.dumps(first_items).encode('UTF-8'))
vl = QgsVectorLayer("url='http://" + endpoint + "' typename='mycollection' restrictToRequestBBOX=1", 'test', 'OAPIF')
self.assertTrue(vl.isValid())
items = {
"type": "FeatureCollection",
"features": [
{"type": "Feature", "id": "feat.1", "properties": {"pk": 1, "cnt": 100}, "geometry": {"type": "Point", "coordinates": [-70.332, 66.33]}},
{"type": "Feature", "id": "feat.2", "properties": {"pk": 2, "cnt": 200}, "geometry": {"type": "Point", "coordinates": [-68.2, 70.8]}}
]
}
with open(sanitize(endpoint, '/collections/mycollection/items?limit=1000&bbox=-71,65.5,-65,78&' + ACCEPT_ITEMS), 'wb') as f:
f.write(json.dumps(items).encode('UTF-8'))
extent = QgsRectangle(-71, 65.5, -65, 78)
request = QgsFeatureRequest().setFilterRect(extent)
values = [f['pk'] for f in vl.getFeatures(request)]
self.assertEqual(values, [1, 2])
# Test request inside above one
EPS = 0.1
extent = QgsRectangle(-71 + EPS, 65.5 + EPS, -65 - EPS, 78 - EPS)
request = QgsFeatureRequest().setFilterRect(extent)
values = [f['pk'] for f in vl.getFeatures(request)]
self.assertEqual(values, [1, 2])
# Test clamping of bbox
with open(sanitize(endpoint, '/collections/mycollection/items?limit=1000&bbox=-180,64.5,-65,78&' + ACCEPT_ITEMS), 'wb') as f:
f.write(json.dumps(items).encode('UTF-8'))
extent = QgsRectangle(-190, 64.5, -65, 78)
request = QgsFeatureRequest().setFilterRect(extent)
values = [f['pk'] for f in vl.getFeatures(request)]
self.assertEqual(values, [1, 2])
# Test request completely outside of -180,-90,180,90
extent = QgsRectangle(-1000, -1000, -900, -900)
request = QgsFeatureRequest().setFilterRect(extent)
values = [f['pk'] for f in vl.getFeatures(request)]
self.assertEqual(values, [])
# Test request containing -180,-90,180,90
items = {
"type": "FeatureCollection",
"features": [
{"type": "Feature", "id": "feat.1", "properties": {"pk": 1, "cnt": 100}, "geometry": {"type": "Point", "coordinates": [-70.332, 66.33]}},
{"type": "Feature", "id": "feat.2", "properties": {"pk": 2, "cnt": 200}, "geometry": {"type": "Point", "coordinates": [-68.2, 70.8]}},
{"type": "Feature", "id": "feat.3", "properties": {"pk": 4, "cnt": 400}, "geometry": {"type": "Point", "coordinates": [-65.32, 78.3]}}
]
}
with open(sanitize(endpoint, '/collections/mycollection/items?limit=1000&' + ACCEPT_ITEMS), 'wb') as f:
f.write(json.dumps(items).encode('UTF-8'))
extent = QgsRectangle(-181, -91, 181, 91)
request = QgsFeatureRequest().setFilterRect(extent)
values = [f['pk'] for f in vl.getFeatures(request)]
self.assertEqual(values, [1, 2, 4])
if __name__ == '__main__':
unittest.main()