sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-04-14 00:07:35 -04:00
Code Issues Packages Projects Releases Wiki Activity

[server] Proper convertion of literals in Filters

Browse Source 
Convert OGC Filter's literals accordings to field type.
This can have a huge impact on performance in some cases.
For example for a filter like "num_char" = '+2' converted to "num_char" = 2,
this result with PostgreSQL provider in a fallback to client side evaluation for the whole filter,
including the bbox if present.
This commit is contained in:
arnaud.morvan@camptocamp.com 2018-05-25 16:52:48 +02:00
parent facf7a22f0
commit b07c334723
14 changed files with 126 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
python/core/auto_generated/qgsogcutils.sip.in
View File

@ -128,7 +128,7 @@ Exports the rectangle to GML3 Envelope
Parse XML with OGC fill into QColor
%End
static QgsExpression *expressionFromOgcFilter( const QDomElement &element ) /Factory/;
static QgsExpression *expressionFromOgcFilter( const QDomElement &element, QgsVectorLayer *layer = 0 ) /Factory/;
%Docstring
Parse XML with OGC filter into QGIS expression
%End

56
src/core/qgsogcutils.cpp
View File

@ -1598,7 +1598,7 @@ QColor QgsOgcUtils::colorFromOgcFill( const QDomElement &fillElement )
}
QgsExpression *QgsOgcUtils::expressionFromOgcFilter( const QDomElement &element )
QgsExpression *QgsOgcUtils::expressionFromOgcFilter( const QDomElement &element, QgsVectorLayer *layer )
{
if ( element.isNull() || !element.hasChildNodes() )
return nullptr;
@ -1619,7 +1619,7 @@ QgsExpression *QgsOgcUtils::expressionFromOgcFilter( const QDomElement &element
while ( !childElem.isNull() )
{
QString errorMsg;
QgsExpressionNode *node = nodeFromOgcFilter( childElem, errorMsg );
QgsExpressionNode *node = nodeFromOgcFilter( childElem, errorMsg, layer );
if ( !node )
{
// invalid expression, parser error
@ -1702,7 +1702,7 @@ static bool isSpatialOperator( const QString &tagName )
QgsExpressionNode *QgsOgcUtils::nodeFromOgcFilter( QDomElement &element, QString &errorMessage )
QgsExpressionNode *QgsOgcUtils::nodeFromOgcFilter( QDomElement &element, QString &errorMessage, QgsVectorLayer *layer )
{
if ( element.isNull() )
return nullptr;
@ -1710,7 +1710,7 @@ QgsExpressionNode *QgsOgcUtils::nodeFromOgcFilter( QDomElement &element, QString
// check for binary operators
if ( isBinaryOperator( element.tagName() ) )
{
return nodeBinaryOperatorFromOgcFilter( element, errorMessage );
return nodeBinaryOperatorFromOgcFilter( element, errorMessage, layer );
}
// check for spatial operators
@ -1731,7 +1731,7 @@ QgsExpressionNode *QgsOgcUtils::nodeFromOgcFilter( QDomElement &element, QString
}
else if ( element.tagName() == QLatin1String( "Literal" ) )
{
return nodeLiteralFromOgcFilter( element, errorMessage );
return nodeLiteralFromOgcFilter( element, errorMessage, layer );
}
else if ( element.tagName() == QLatin1String( "Function" ) )
{
@ -1752,7 +1752,7 @@ QgsExpressionNode *QgsOgcUtils::nodeFromOgcFilter( QDomElement &element, QString
QgsExpressionNodeBinaryOperator *QgsOgcUtils::nodeBinaryOperatorFromOgcFilter( QDomElement &element, QString &errorMessage )
QgsExpressionNodeBinaryOperator *QgsOgcUtils::nodeBinaryOperatorFromOgcFilter( QDomElement &element, QString &errorMessage, QgsVectorLayer *layer )
{
if ( element.isNull() )
return nullptr;
@ -1771,7 +1771,7 @@ QgsExpressionNodeBinaryOperator *QgsOgcUtils::nodeBinaryOperatorFromOgcFilter( Q
}
QDomElement operandElem = element.firstChildElement();
QgsExpressionNode *expr = nodeFromOgcFilter( operandElem, errorMessage ), *leftOp = expr;
QgsExpressionNode *expr = nodeFromOgcFilter( operandElem, errorMessage, layer ), *leftOp = expr;
if ( !expr )
{
if ( errorMessage.isEmpty() )
@ -1781,7 +1781,7 @@ QgsExpressionNodeBinaryOperator *QgsOgcUtils::nodeBinaryOperatorFromOgcFilter( Q
for ( operandElem = operandElem.nextSiblingElement(); !operandElem.isNull(); operandElem = operandElem.nextSiblingElement() )
{
QgsExpressionNode *opRight = nodeFromOgcFilter( operandElem, errorMessage );
QgsExpressionNode *opRight = nodeFromOgcFilter( operandElem, errorMessage, layer );
if ( !opRight )
{
if ( errorMessage.isEmpty() )
@ -1960,7 +1960,7 @@ QgsExpressionNodeFunction *QgsOgcUtils::nodeFunctionFromOgcFilter( QDomElement &
QgsExpressionNode *QgsOgcUtils::nodeLiteralFromOgcFilter( QDomElement &element, QString &errorMessage )
QgsExpressionNode *QgsOgcUtils::nodeLiteralFromOgcFilter( QDomElement &element, QString &errorMessage, QgsVectorLayer *layer )
{
if ( element.isNull() || element.tagName() != QLatin1String( "Literal" ) )
{
@ -1980,7 +1980,7 @@ QgsExpressionNode *QgsOgcUtils::nodeLiteralFromOgcFilter( QDomElement &element,
{
// found a element node (e.g. PropertyName), convert it
QDomElement operandElem = childNode.toElement();
operand = nodeFromOgcFilter( operandElem, errorMessage );
operand = nodeFromOgcFilter( operandElem, errorMessage, layer );
if ( !operand )
{
delete root;
@ -1994,12 +1994,36 @@ QgsExpressionNode *QgsOgcUtils::nodeLiteralFromOgcFilter( QDomElement &element,
// probably a text/CDATA node
QVariant value = childNode.nodeValue();
// try to convert the node content to number if possible,
// otherwise let's use it as string
bool ok;
double d = value.toDouble( &ok );
if ( ok )
value = d;
bool converted = false;
// try to convert the node content to corresponding field type if possible
if ( layer != nullptr )
{
QDomElement propertyNameElement = element.previousSiblingElement( QLatin1String( "PropertyName" ) );
if ( propertyNameElement.isNull() || propertyNameElement.tagName() != QLatin1String( "PropertyName" ) )
{
propertyNameElement = element.nextSiblingElement( QLatin1String( "PropertyName" ) );
}
if ( !propertyNameElement.isNull() || propertyNameElement.tagName() == QLatin1String( "PropertyName" ) )
{
int fieldIndex = layer->fields().indexOf( propertyNameElement.firstChild().nodeValue() );
if ( fieldIndex != -1 )
{
QgsField field = layer->fields().field( propertyNameElement.firstChild().nodeValue() );
field.convertCompatible( value );
converted = true;
}
}
}
if ( !converted )
{
// try to convert the node content to number if possible,
// otherwise let's use it as string
bool ok;
double d = value.toDouble( &ok );
if ( ok )
value = d;
}
operand = new QgsExpressionNodeLiteral( value );
if ( !operand )

9
src/core/qgsogcutils.h
View File

@ -37,6 +37,7 @@ class QgsRectangle;
#include "qgsexpressionnode.h"
#include "qgsexpressionnodeimpl.h"
#include "qgssqlstatement.h"
#include "qgsvectorlayer.h"
/**
* \ingroup core
@ -140,7 +141,7 @@ class CORE_EXPORT QgsOgcUtils
static QColor colorFromOgcFill( const QDomElement &fillElement );
//! Parse XML with OGC filter into QGIS expression
static QgsExpression *expressionFromOgcFilter( const QDomElement &element ) SIP_FACTORY;
static QgsExpression *expressionFromOgcFilter( const QDomElement &element, QgsVectorLayer *layer = nullptr ) SIP_FACTORY;
/**
* Creates OGC filter XML element. Supports minimum standard filter
@ -298,9 +299,9 @@ class CORE_EXPORT QgsOgcUtils
static QDomElement createGMLPositions( const QgsPolylineXY &points, QDomDocument &doc );
//! handle a generic sub-expression
static QgsExpressionNode *nodeFromOgcFilter( QDomElement &element, QString &errorMessage );
static QgsExpressionNode *nodeFromOgcFilter( QDomElement &element, QString &errorMessage, QgsVectorLayer *layer = nullptr );
//! handle a generic binary operator
static QgsExpressionNodeBinaryOperator *nodeBinaryOperatorFromOgcFilter( QDomElement &element, QString &errorMessage );
static QgsExpressionNodeBinaryOperator *nodeBinaryOperatorFromOgcFilter( QDomElement &element, QString &errorMessage, QgsVectorLayer *layer = nullptr );
//! handles various spatial operation tags (\verbatim <Intersects> \endverbatim, \verbatim <Touches> \endverbatim etc.)
static QgsExpressionNodeFunction *nodeSpatialOperatorFromOgcFilter( QDomElement &element, QString &errorMessage );
//! handle \verbatim <Not> \endverbatim tag
@ -308,7 +309,7 @@ class CORE_EXPORT QgsOgcUtils
//! handles \verbatim <Function> \endverbatim tag
static QgsExpressionNodeFunction *nodeFunctionFromOgcFilter( QDomElement &element, QString &errorMessage );
//! handles \verbatim <Literal> \endverbatim tag
static QgsExpressionNode *nodeLiteralFromOgcFilter( QDomElement &element, QString &errorMessage );
static QgsExpressionNode *nodeLiteralFromOgcFilter( QDomElement &element, QString &errorMessage, QgsVectorLayer *layer = nullptr );
//! handles \verbatim <PropertyName> \endverbatim tag
static QgsExpressionNodeColumnRef *nodeColumnRefFromOgcFilter( QDomElement &element, QString &errorMessage );
//! handles \verbatim <PropertyIsBetween> \endverbatim tag

1
src/server/qgsserverprojectutils.h
View File

@ -20,6 +20,7 @@
#include "qgis_server.h"
#include "qgsproject.h"
#include "qgsvectorlayer.h"
#ifdef SIP_RUN
% ModuleHeaderCode

10
src/server/services/wfs/qgswfsdescribefeaturetype.cpp
View File

@ -139,10 +139,7 @@ namespace QgsWfs
continue;
}
QString name = layer->name();
if ( !layer->shortName().isEmpty() )
name = layer->shortName();
name = name.replace( ' ', '_' );
QString name = layerTypeName( layer );
if ( !typeNameList.isEmpty() && !typeNameList.contains( name ) )
{
@ -180,10 +177,7 @@ namespace QgsWfs
return;
}
QString typeName = layer->name();
if ( !layer->shortName().isEmpty() )
typeName = layer->shortName();
typeName = typeName.replace( ' ', '_' );
QString typeName = layerTypeName( layer );
//xsd:element
QDomElement elementElem = doc.createElement( QStringLiteral( "element" )/*xsd:element*/ );

6
src/server/services/wfs/qgswfsgetcapabilities.cpp
View File

@ -438,11 +438,7 @@ namespace QgsWfs
//create Name
QDomElement nameElem = doc.createElement( QStringLiteral( "Name" ) );
QString typeName = layer->name();
if ( !layer->shortName().isEmpty() )
typeName = layer->shortName();
typeName = typeName.replace( QLatin1String( " " ), QLatin1String( "_" ) );
QDomText nameText = doc.createTextNode( typeName );
QDomText nameText = doc.createTextNode( layerTypeName( layer ) );
nameElem.appendChild( nameText );
layerElem.appendChild( nameElem );

21
src/server/services/wfs/qgswfsgetfeature.cpp
View File

@ -101,11 +101,11 @@ namespace QgsWfs
if ( doc.setContent( mRequestParameters.value( QStringLiteral( "REQUEST_BODY" ) ), true, &errorMsg ) )
{
QDomElement docElem = doc.documentElement();
aRequest = parseGetFeatureRequestBody( docElem );
aRequest = parseGetFeatureRequestBody( docElem, project );
}
else
{
aRequest = parseGetFeatureParameters();
aRequest = parseGetFeatureParameters( project );
}
// store typeName
@ -141,10 +141,7 @@ namespace QgsWfs
continue;
}
QString name = layer->name();
if ( !layer->shortName().isEmpty() )
name = layer->shortName();
name = name.replace( ' ', '_' );
QString name = layerTypeName( layer );
if ( typeNameList.contains( name ) )
{
@ -433,7 +430,7 @@ namespace QgsWfs
}
getFeatureRequest parseGetFeatureParameters()
getFeatureRequest parseGetFeatureParameters( const QgsProject *project )
{
getFeatureRequest request;
request.maxFeatures = mWfsParameters.maxFeaturesAsInt();;
@ -767,7 +764,7 @@ namespace QgsWfs
}
QDomElement filterElem = filter.firstChildElement();
query.featureRequest = parseFilterElement( query.typeName, filterElem );
query.featureRequest = parseFilterElement( query.typeName, filterElem, project );
if ( filterIt != filterList.constEnd() )
{
@ -821,7 +818,7 @@ namespace QgsWfs
return request;
}
getFeatureRequest parseGetFeatureRequestBody( QDomElement &docElem )
getFeatureRequest parseGetFeatureRequestBody( QDomElement &docElem, const QgsProject *project )
{
getFeatureRequest request;
request.maxFeatures = mWfsParameters.maxFeaturesAsInt();;
@ -833,7 +830,7 @@ namespace QgsWfs
for ( int i = 0; i < queryNodes.size(); i++ )
{
queryElem = queryNodes.at( i ).toElement();
getFeatureQuery query = parseQueryElement( queryElem );
getFeatureQuery query = parseQueryElement( queryElem, project );
request.queries.append( query );
}
return request;
@ -887,7 +884,7 @@ namespace QgsWfs
}
}
getFeatureQuery parseQueryElement( QDomElement &queryElem )
getFeatureQuery parseQueryElement( QDomElement &queryElem, const QgsProject *project )
{
QString typeName = queryElem.attribute( QStringLiteral( "typeName" ), QLatin1String( "" ) );
if ( typeName.contains( ':' ) )
@ -923,7 +920,7 @@ namespace QgsWfs
}
else if ( queryChildElem.tagName() == QLatin1String( "Filter" ) )
{
featureRequest = parseFilterElement( typeName, queryChildElem );
featureRequest = parseFilterElement( typeName, queryChildElem, project );
}
else if ( queryChildElem.tagName() == QLatin1String( "SortBy" ) )
{

6
src/server/services/wfs/qgswfsgetfeature.h
View File

@ -58,17 +58,17 @@ namespace QgsWfs
/**
* Transform Query element to getFeatureQuery
*/
getFeatureQuery parseQueryElement( QDomElement &queryElem );
getFeatureQuery parseQueryElement( QDomElement &queryElem, const QgsProject *project = nullptr );
/**
* Transform RequestBody root element to getFeatureRequest
*/
getFeatureRequest parseGetFeatureRequestBody( QDomElement &docElem );
getFeatureRequest parseGetFeatureRequestBody( QDomElement &docElem, const QgsProject *project = nullptr );
/**
* Transform parameters to getFeatureRequest
*/
getFeatureRequest parseGetFeatureParameters();
getFeatureRequest parseGetFeatureParameters( const QgsProject *project = nullptr );
/**
* Output WFS GetFeature response

5
src/server/services/wfs/qgswfstransaction.cpp
View File

@ -259,10 +259,7 @@ namespace QgsWfs
continue;
}
QString name = layer->name();
if ( !layer->shortName().isEmpty() )
name = layer->shortName();
name = name.replace( ' ', '_' );
QString name = layerTypeName( layer );
if ( !typeNameList.contains( name ) )
{

5
src/server/services/wfs/qgswfstransaction_1_0_0.cpp
View File

@ -242,10 +242,7 @@ namespace QgsWfs
continue;
}
QString name = layer->name();
if ( !layer->shortName().isEmpty() )
name = layer->shortName();
name = name.replace( ' ', '_' );
QString name = layerTypeName( layer );
if ( !typeNameList.contains( name ) )
{

41
src/server/services/wfs/qgswfsutils.cpp
View File

@ -58,7 +58,39 @@ namespace QgsWfs
return href;
}
QgsFeatureRequest parseFilterElement( const QString &typeName, QDomElement &filterElem )
QString layerTypeName( const QgsMapLayer *layer )
{
QString name = layer->name();
if ( !layer->shortName().isEmpty() )
name = layer->shortName();
name = name.replace( ' ', '_' );
return name;
}
QgsVectorLayer *layerByTypeName( const QgsProject *project, const QString &typeName )
{
QStringList layerIds = QgsServerProjectUtils::wfsLayerIds( *project );
for ( const QString &layerId : layerIds )
{
QgsMapLayer *layer = project->mapLayer( layerId );
if ( !layer )
{
continue;
}
if ( layer->type() != QgsMapLayer::LayerType::VectorLayer )
{
continue;
}
if ( layerTypeName( layer ) == typeName )
{
return qobject_cast<QgsVectorLayer *>( layer );
}
}
return nullptr;
}
QgsFeatureRequest parseFilterElement( const QString &typeName, QDomElement &filterElem, const QgsProject *project )
{
QgsFeatureRequest request;
@ -196,7 +228,12 @@ namespace QgsWfs
}
else
{
std::shared_ptr<QgsExpression> filter( QgsOgcUtils::expressionFromOgcFilter( filterElem ) );
QgsVectorLayer *layer = nullptr;
if ( project != nullptr )
{
layer = layerByTypeName( project, typeName );
}
std::shared_ptr<QgsExpression> filter( QgsOgcUtils::expressionFromOgcFilter( filterElem, layer ) );
if ( filter )
{
if ( filter->hasParserError() )

12
src/server/services/wfs/qgswfsutils.h
View File

@ -47,10 +47,20 @@ namespace QgsWfs
*/
QString serviceUrl( const QgsServerRequest &request, const QgsProject *project );
/**
* Returns typename from vector layer
*/
QString layerTypeName( const QgsMapLayer *layer );
/**
* Retrieve a layer by typename
*/
QgsVectorLayer *layerByTypeName( const QgsProject *project, const QString &typeName );
/**
* Transform a Filter element to a feature request
*/
QgsFeatureRequest parseFilterElement( const QString &typeName, QDomElement &filterElem );
QgsFeatureRequest parseFilterElement( const QString &typeName, QDomElement &filterElem, const QgsProject *project = nullptr );
// Define namespaces used in WFS documents
const QString WFS_NAMESPACE = QStringLiteral( "http://www.opengis.net/wfs" );

2
src/server/services/wms/qgswmsrenderer.cpp
View File

@ -2798,7 +2798,7 @@ namespace QgsWms
QStringLiteral( "error message: %1. The XML string was: %2" ).arg( errorMsg, filter ) );
}
QDomElement filterElem = filterXml.firstChildElement();
QScopedPointer<QgsExpression> expression( QgsOgcUtils::expressionFromOgcFilter( filterElem ) );
QScopedPointer<QgsExpression> expression( QgsOgcUtils::expressionFromOgcFilter( filterElem, filteredLayer ) );
mFeatureFilter.setFilter( filteredLayer, *expression );
}
else

12
tests/src/core/testqgsogcutils.cpp
View File

@ -21,6 +21,7 @@
#include <qgsgeometry.h>
#include <qgsogcutils.h>
#include "qgsapplication.h"
#include "qgsvectorlayer.h"
/**
* \ingroup UnitTests
@ -368,6 +369,13 @@ void TestQgsOgcUtils::testExpressionFromOgcFilter_data()
"</Intersects>"
"</Filter>" )
<< QStringLiteral( "intersects($geometry, geom_from_gml('<Point><coordinates>123,456</coordinates></Point>'))" );
QTest::newRow( "Literal conversion" ) << QString(
"<Filter><PropertyIsEqualTo>"
"<PropertyName>LITERAL</PropertyName>"
"<Literal>+2</Literal>"
"</PropertyIsEqualTo></Filter>" )
<< QStringLiteral( "LITERAL = '+2'" );
}
void TestQgsOgcUtils::testExpressionFromOgcFilter()
@ -379,7 +387,9 @@ void TestQgsOgcUtils::testExpressionFromOgcFilter()
QVERIFY( doc.setContent( xmlText, true ) );
QDomElement rootElem = doc.documentElement();
std::shared_ptr<QgsExpression> expr( QgsOgcUtils::expressionFromOgcFilter( rootElem ) );
QgsVectorLayer layer( "Point?crs=epsg:4326&field=LITERAL:string(20)", "temp", "memory" );
std::shared_ptr<QgsExpression> expr( QgsOgcUtils::expressionFromOgcFilter( rootElem, &layer ) );
QVERIFY( expr.get() );
qDebug( "OGC XML : %s", xmlText.toAscii().data() );