From aa96e78682cee9f7950346784f4da3e2f5bd3ad5 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 21 Jun 2017 22:12:19 +1000 Subject: [PATCH] Native extract by expression and attribute algs --- .../tests/testdata/qgis_algorithm_tests.yaml | 166 +++++----- src/core/processing/qgsnativealgorithms.cpp | 298 ++++++++++++++++++ src/core/processing/qgsnativealgorithms.h | 61 ++++ 3 files changed, 442 insertions(+), 83 deletions(-) diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 08ef4a93ab0..b20ca97431f 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -1258,7 +1258,7 @@ tests: # name: expected/remove_null_polys.gml # type: vector # - - algorithm: qgis:extractbyexpression + - algorithm: native:extractbyexpression name: Extract by Expression params: EXPRESSION: left( "Name",1)='A' @@ -1484,88 +1484,88 @@ tests: # geometry: # precision: 7 # -# - algorithm: qgis:extractbyattribute -# name: Extract by attribute (is null) -# params: -# FIELD: intval -# INPUT: -# name: polys.gml -# type: vector -# OPERATOR: '8' -# results: -# OUTPUT: -# name: expected/extract_by_attribute_null.gml -# type: vector -# -# - algorithm: qgis:extractbyattribute -# name: Extract by attribute (is not null) -# params: -# FIELD: intval -# INPUT: -# name: polys.gml -# type: vector -# OPERATOR: '9' -# results: -# OUTPUT: -# name: expected/extract_by_attribute_not_null.gml -# type: vector -# -# - algorithm: qgis:extractbyattribute -# name: Extract by attribute (starts with) -# params: -# FIELD: name -# INPUT: -# name: polys.gml -# type: vector -# OPERATOR: '6' -# VALUE: A -# results: -# OUTPUT: -# name: expected/extract_by_attribute_startswith.gml -# type: vector -# -# - algorithm: qgis:extractbyattribute -# name: Extract by attribute (contains) -# params: -# FIELD: name -# INPUT: -# name: polys.gml -# type: vector -# OPERATOR: '7' -# VALUE: aaa -# results: -# OUTPUT: -# name: expected/extract_by_attribute_contains.gml -# type: vector -# -# - algorithm: qgis:extractbyattribute -# name: Extract by attribute (does not contain) -# params: -# FIELD: name -# INPUT: -# name: polys.gml -# type: vector -# OPERATOR: '10' -# VALUE: a -# results: -# OUTPUT: -# name: expected/extract_by_attribute_does_not_contain.gml -# type: vector -# -# - algorithm: qgis:extractbyattribute -# name: Extract by attribute (greater) -# params: -# FIELD: floatval -# INPUT: -# name: polys.gml -# type: vector -# OPERATOR: '2' -# VALUE: '1' -# results: -# OUTPUT: -# name: expected/extract_by_attribute_greater.gml -# type: vector -# + - algorithm: native:extractbyattribute + name: Extract by attribute (is null) + params: + FIELD: intval + INPUT: + name: polys.gml + type: vector + OPERATOR: '8' + results: + OUTPUT: + name: expected/extract_by_attribute_null.gml + type: vector + + - algorithm: native:extractbyattribute + name: Extract by attribute (is not null) + params: + FIELD: intval + INPUT: + name: polys.gml + type: vector + OPERATOR: '9' + results: + OUTPUT: + name: expected/extract_by_attribute_not_null.gml + type: vector + + - algorithm: native:extractbyattribute + name: Extract by attribute (starts with) + params: + FIELD: name + INPUT: + name: polys.gml + type: vector + OPERATOR: '6' + VALUE: A + results: + OUTPUT: + name: expected/extract_by_attribute_startswith.gml + type: vector + + - algorithm: native:extractbyattribute + name: Extract by attribute (contains) + params: + FIELD: name + INPUT: + name: polys.gml + type: vector + OPERATOR: '7' + VALUE: aaa + results: + OUTPUT: + name: expected/extract_by_attribute_contains.gml + type: vector + + - algorithm: native:extractbyattribute + name: Extract by attribute (does not contain) + params: + FIELD: name + INPUT: + name: polys.gml + type: vector + OPERATOR: '10' + VALUE: a + results: + OUTPUT: + name: expected/extract_by_attribute_does_not_contain.gml + type: vector + + - algorithm: native:extractbyattribute + name: Extract by attribute (greater) + params: + FIELD: floatval + INPUT: + name: polys.gml + type: vector + OPERATOR: '2' + VALUE: '1' + results: + OUTPUT: + name: expected/extract_by_attribute_greater.gml + type: vector + # - algorithm: qgis:createattributeindex # name: Create Attribute Index (only tests for python errors, does not check result) # params: diff --git a/src/core/processing/qgsnativealgorithms.cpp b/src/core/processing/qgsnativealgorithms.cpp index 9dd2f74cd62..64cc90b74b4 100644 --- a/src/core/processing/qgsnativealgorithms.cpp +++ b/src/core/processing/qgsnativealgorithms.cpp @@ -62,6 +62,8 @@ void QgsNativeAlgorithms::loadAlgorithms() addAlgorithm( new QgsCentroidAlgorithm() ); addAlgorithm( new QgsClipAlgorithm() ); addAlgorithm( new QgsDissolveAlgorithm() ); + addAlgorithm( new QgsExtractByAttributeAlgorithm() ); + addAlgorithm( new QgsExtractByExpressionAlgorithm() ); addAlgorithm( new QgsMultipartToSinglepartAlgorithm() ); addAlgorithm( new QgsSubdivideAlgorithm() ); addAlgorithm( new QgsTransformAlgorithm() ); @@ -745,4 +747,300 @@ QVariantMap QgsMultipartToSinglepartAlgorithm::processAlgorithm( const QVariantM } +QgsExtractByExpressionAlgorithm::QgsExtractByExpressionAlgorithm() +{ + addParameter( new QgsProcessingParameterFeatureSource( QStringLiteral( "INPUT" ), QObject::tr( "Input layer" ) ) ); + addParameter( new QgsProcessingParameterExpression( QStringLiteral( "EXPRESSION" ), QObject::tr( "Expression" ), QVariant(), QStringLiteral( "INPUT" ) ) ); + + addParameter( new QgsProcessingParameterFeatureSink( QStringLiteral( "OUTPUT" ), QObject::tr( "Matching features" ) ) ); + addOutput( new QgsProcessingOutputVectorLayer( QStringLiteral( "OUTPUT" ), QObject::tr( "Matching (expression)" ) ) ); + addParameter( new QgsProcessingParameterFeatureSink( QStringLiteral( "FAIL_OUTPUT" ), QObject::tr( "Non-matching" ), + QgsProcessingParameterDefinition::TypeVectorAny, QVariant(), true ) ); + addOutput( new QgsProcessingOutputVectorLayer( QStringLiteral( "FAIL_OUTPUT" ), QObject::tr( "Non-matching (expression)" ) ) ); +} + +QString QgsExtractByExpressionAlgorithm::shortHelpString() const +{ + return QObject::tr( "This algorithm creates a new vector layer that only contains matching features from an input layer. " + "The criteria for adding features to the resulting layer is based on a QGIS expression.\n\n" + "For more information about expressions see the user manual" ); +} + +QVariantMap QgsExtractByExpressionAlgorithm::processAlgorithm( const QVariantMap ¶meters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) const +{ + std::unique_ptr< QgsFeatureSource > source( parameterAsSource( parameters, QStringLiteral( "INPUT" ), context ) ); + if ( !source ) + return QVariantMap(); + + QString expressionString = parameterAsExpression( parameters, QStringLiteral( "EXPRESSION" ), context ); + + QString matchingSinkId; + std::unique_ptr< QgsFeatureSink > matchingSink( parameterAsSink( parameters, QStringLiteral( "OUTPUT" ), context, source->fields(), + source->wkbType(), source->sourceCrs(), matchingSinkId ) ); + if ( !matchingSink ) + return QVariantMap(); + + QString nonMatchingSinkId; + std::unique_ptr< QgsFeatureSink > nonMatchingSink( parameterAsSink( parameters, QStringLiteral( "FAIL_OUTPUT" ), context, source->fields(), + source->wkbType(), source->sourceCrs(), nonMatchingSinkId ) ); + + QgsExpression expression( expressionString ); + if ( expression.hasParserError() ) + { + // raise GeoAlgorithmExecutionException(expression.parserErrorString()) + return QVariantMap(); + } + + QgsExpressionContext expressionContext = createExpressionContext( parameters, context ); + + long count = source->featureCount(); + if ( count <= 0 ) + return QVariantMap(); + + double step = 100.0 / count; + int current = 0; + + if ( !nonMatchingSink ) + { + // not saving failing features - so only fetch good features + QgsFeatureRequest req; + req.setFilterExpression( expressionString ); + req.setExpressionContext( expressionContext ); + + QgsFeatureIterator it = source->getFeatures( req ); + QgsFeature f; + while ( it.nextFeature( f ) ) + { + if ( feedback->isCanceled() ) + { + break; + } + + matchingSink->addFeature( f ); + + feedback->setProgress( current * step ); + current++; + } + } + else + { + // saving non-matching features, so we need EVERYTHING + expressionContext.setFields( source->fields() ); + expression.prepare( &expressionContext ); + + QgsFeatureIterator it = source->getFeatures(); + QgsFeature f; + while ( it.nextFeature( f ) ) + { + if ( feedback->isCanceled() ) + { + break; + } + + expressionContext.setFeature( f ); + if ( expression.evaluate( &expressionContext ).toBool() ) + { + matchingSink->addFeature( f ); + } + else + { + nonMatchingSink->addFeature( f ); + } + + feedback->setProgress( current * step ); + current++; + } + } + + + QVariantMap outputs; + outputs.insert( QStringLiteral( "OUTPUT" ), matchingSinkId ); + if ( nonMatchingSink ) + outputs.insert( QStringLiteral( "FAIL_OUTPUT" ), nonMatchingSinkId ); + return outputs; +} + + +QgsExtractByAttributeAlgorithm::QgsExtractByAttributeAlgorithm() +{ + addParameter( new QgsProcessingParameterFeatureSource( QStringLiteral( "INPUT" ), QObject::tr( "Input layer" ) ) ); + addParameter( new QgsProcessingParameterTableField( QStringLiteral( "FIELD" ), QObject::tr( "Selection attribute" ), QVariant(), QStringLiteral( "INPUT" ) ) ); + addParameter( new QgsProcessingParameterEnum( QStringLiteral( "OPERATOR" ), QObject::tr( "Operator" ), QStringList() + << QObject::tr( "=" ) + << QObject::trUtf8( "≠" ) + << QObject::tr( ">" ) + << QObject::tr( ">=" ) + << QObject::tr( "<" ) + << QObject::tr( "<=" ) + << QObject::tr( "begins with" ) + << QObject::tr( "contains" ) + << QObject::tr( "is null" ) + << QObject::tr( "is not null" ) + << QObject::tr( "does not contain" ) ) ); + addParameter( new QgsProcessingParameterString( QStringLiteral( "VALUE" ), QObject::tr( "Value" ), QVariant(), false, true ) ); + + addParameter( new QgsProcessingParameterFeatureSink( QStringLiteral( "OUTPUT" ), QObject::tr( "Extracted (attribute)" ) ) ); + addOutput( new QgsProcessingOutputVectorLayer( QStringLiteral( "OUTPUT" ), QObject::tr( "Matching (attribute)" ) ) ); + addParameter( new QgsProcessingParameterFeatureSink( QStringLiteral( "FAIL_OUTPUT" ), QObject::tr( "Extracted (non-matching)" ), + QgsProcessingParameterDefinition::TypeVectorAny, QVariant(), true ) ); + addOutput( new QgsProcessingOutputVectorLayer( QStringLiteral( "FAIL_OUTPUT" ), QObject::tr( "Non-matching (attribute)" ) ) ); +} + +QString QgsExtractByAttributeAlgorithm::shortHelpString() const +{ + return QObject::tr( " This algorithm creates a new vector layer that only contains matching features from an input layer. " + "The criteria for adding features to the resulting layer is defined based on the values " + "of an attribute from the input layer." ); +} + +QVariantMap QgsExtractByAttributeAlgorithm::processAlgorithm( const QVariantMap ¶meters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) const +{ + std::unique_ptr< QgsFeatureSource > source( parameterAsSource( parameters, QStringLiteral( "INPUT" ), context ) ); + if ( !source ) + return QVariantMap(); + + QString fieldName = parameterAsString( parameters, QStringLiteral( "FIELD" ), context ); + Operation op = static_cast< Operation >( parameterAsEnum( parameters, QStringLiteral( "OPERATOR" ), context ) ); + QString value = parameterAsString( parameters, QStringLiteral( "VALUE" ), context ); + + QString matchingSinkId; + std::unique_ptr< QgsFeatureSink > matchingSink( parameterAsSink( parameters, QStringLiteral( "OUTPUT" ), context, source->fields(), + source->wkbType(), source->sourceCrs(), matchingSinkId ) ); + if ( !matchingSink ) + return QVariantMap(); + + QString nonMatchingSinkId; + std::unique_ptr< QgsFeatureSink > nonMatchingSink( parameterAsSink( parameters, QStringLiteral( "FAIL_OUTPUT" ), context, source->fields(), + source->wkbType(), source->sourceCrs(), nonMatchingSinkId ) ); + + + int idx = source->fields().lookupField( fieldName ); + QVariant::Type fieldType = source->fields().at( idx ).type(); + + if ( fieldType != QVariant::String && ( op == BeginsWith || op == Contains || op == DoesNotContain ) ) + { +#if 0 + op = ''.join( ['"%s", ' % o for o in self.STRING_OPERATORS] ) + raise GeoAlgorithmExecutionException( + self.tr( 'Operators {0} can be used only with string fields.' ).format( op ) ) +#endif + return QVariantMap(); + } + + QString fieldRef = QgsExpression::quotedColumnRef( fieldName ); + QString quotedVal = QgsExpression::quotedValue( value ); + QString expr; + switch ( op ) + { + case Equals: + expr = QStringLiteral( "%1 = %3" ).arg( fieldRef, quotedVal ); + break; + case NotEquals: + expr = QStringLiteral( "%1 != %3" ).arg( fieldRef, quotedVal ); + break; + case GreaterThan: + expr = QStringLiteral( "%1 > %3" ).arg( fieldRef, quotedVal ); + break; + case GreaterThanEqualTo: + expr = QStringLiteral( "%1 >= %3" ).arg( fieldRef, quotedVal ); + break; + case LessThan: + expr = QStringLiteral( "%1 < %3" ).arg( fieldRef, quotedVal ); + break; + case LessThanEqualTo: + expr = QStringLiteral( "%1 <= %3" ).arg( fieldRef, quotedVal ); + break; + case BeginsWith: + expr = QStringLiteral( "%1 LIKE '%2%'" ).arg( fieldRef, value ); + break; + case Contains: + expr = QStringLiteral( "%1 LIKE '%%2%'" ).arg( fieldRef, value ); + break; + case IsNull: + expr = QStringLiteral( "%1 IS NULL" ).arg( fieldRef ); + break; + case IsNotNull: + expr = QStringLiteral( "%1 IS NOT NULL" ).arg( fieldRef ); + break; + case DoesNotContain: + expr = QStringLiteral( "%1 NOT LIKE '%%2%'" ).arg( fieldRef, value ); + break; + } + + QgsExpression expression( expr ); + if ( expression.hasParserError() ) + { + // raise GeoAlgorithmExecutionException(expression.parserErrorString()) + return QVariantMap(); + } + + QgsExpressionContext expressionContext = createExpressionContext( parameters, context ); + + long count = source->featureCount(); + if ( count <= 0 ) + return QVariantMap(); + + double step = 100.0 / count; + int current = 0; + + if ( !nonMatchingSink ) + { + // not saving failing features - so only fetch good features + QgsFeatureRequest req; + req.setFilterExpression( expr ); + req.setExpressionContext( expressionContext ); + + QgsFeatureIterator it = source->getFeatures( req ); + QgsFeature f; + while ( it.nextFeature( f ) ) + { + if ( feedback->isCanceled() ) + { + break; + } + + matchingSink->addFeature( f ); + + feedback->setProgress( current * step ); + current++; + } + } + else + { + // saving non-matching features, so we need EVERYTHING + expressionContext.setFields( source->fields() ); + expression.prepare( &expressionContext ); + + QgsFeatureIterator it = source->getFeatures(); + QgsFeature f; + while ( it.nextFeature( f ) ) + { + if ( feedback->isCanceled() ) + { + break; + } + + expressionContext.setFeature( f ); + if ( expression.evaluate( &expressionContext ).toBool() ) + { + matchingSink->addFeature( f ); + } + else + { + nonMatchingSink->addFeature( f ); + } + + feedback->setProgress( current * step ); + current++; + } + } + + + QVariantMap outputs; + outputs.insert( QStringLiteral( "OUTPUT" ), matchingSinkId ); + if ( nonMatchingSink ) + outputs.insert( QStringLiteral( "FAIL_OUTPUT" ), nonMatchingSinkId ); + return outputs; +} + ///@endcond diff --git a/src/core/processing/qgsnativealgorithms.h b/src/core/processing/qgsnativealgorithms.h index 85eaee5e0ef..8669df80955 100644 --- a/src/core/processing/qgsnativealgorithms.h +++ b/src/core/processing/qgsnativealgorithms.h @@ -135,6 +135,67 @@ class QgsDissolveAlgorithm : public QgsProcessingAlgorithm }; +/** + * Native extract by attribute algorithm. + */ +class QgsExtractByAttributeAlgorithm : public QgsProcessingAlgorithm +{ + + public: + + enum Operation + { + Equals, + NotEquals, + GreaterThan, + GreaterThanEqualTo, + LessThan, + LessThanEqualTo, + BeginsWith, + Contains, + IsNull, + IsNotNull, + DoesNotContain, + }; + + QgsExtractByAttributeAlgorithm(); + + QString name() const override { return QStringLiteral( "extractbyattribute" ); } + QString displayName() const override { return QObject::tr( "Extract by attribute" ); } + virtual QStringList tags() const override { return QObject::tr( "extract,filter,attribute,value,contains,null,field" ).split( ',' ); } + QString group() const override { return QObject::tr( "Vector selection tools" ); } + QString shortHelpString() const override; + + protected: + + virtual QVariantMap processAlgorithm( const QVariantMap ¶meters, + QgsProcessingContext &context, QgsProcessingFeedback *feedback ) const override; + +}; + +/** + * Native extract by expression algorithm. + */ +class QgsExtractByExpressionAlgorithm : public QgsProcessingAlgorithm +{ + + public: + + QgsExtractByExpressionAlgorithm(); + + QString name() const override { return QStringLiteral( "extractbyexpression" ); } + QString displayName() const override { return QObject::tr( "Extract by expression" ); } + virtual QStringList tags() const override { return QObject::tr( "extract,filter,expression,field" ).split( ',' ); } + QString group() const override { return QObject::tr( "Vector selection tools" ); } + QString shortHelpString() const override; + + protected: + + virtual QVariantMap processAlgorithm( const QVariantMap ¶meters, + QgsProcessingContext &context, QgsProcessingFeedback *feedback ) const override; + +}; + /** * Native clip algorithm. */