diff --git a/python/plugins/processing/tests/testdata/custom/hstore.gml b/python/plugins/processing/tests/testdata/custom/hstore.gml new file mode 100644 index 00000000000..17d819316e0 --- /dev/null +++ b/python/plugins/processing/tests/testdata/custom/hstore.gml @@ -0,0 +1,49 @@ + + + + + -1-3 + 106 + + + + + + -1,-1 -1,3 3,3 3,2 2,2 2,-1 -1,-1 + "amenity"=>"restaurant","barrier"=>"wall","cuisine"=>"chinese","internet_access"=>"yes" + + + + + 5,5 6,4 4,4 5,5 + "amenity"=>"fuel","building"=>"roof","name"=>"foo" + + + + + 2,5 2,6 3,6 3,5 2,5 + "building"=>"yes" + + + + + 6,1 10,1 10,-3 6,-3 6,17,0 7,-2 9,-2 9,0 7,0 + "amenity"=>"restaurant","cuisine"=>"burger","name"=>"bar","operator"=>"foo" + + + + + "amenity"=>"bank","atm"=>"yes" + + + + + 3,2 6,1 6,-3 2,-1 2,2 3,2 + "stars"=>"5","tourism"=>"hotel" + + + diff --git a/python/plugins/processing/tests/testdata/custom/hstore.xsd b/python/plugins/processing/tests/testdata/custom/hstore.xsd new file mode 100644 index 00000000000..95b38299631 --- /dev/null +++ b/python/plugins/processing/tests/testdata/custom/hstore.xsd @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/python/plugins/processing/tests/testdata/expected/hstore_all_keys.gml b/python/plugins/processing/tests/testdata/expected/hstore_all_keys.gml new file mode 100644 index 00000000000..6649c29311f --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hstore_all_keys.gml @@ -0,0 +1,109 @@ + + + + + -1-3 + 106 + + + + + + -1,-1 -1,3 3,3 3,2 2,2 2,-1 -1,-1 + "amenity"=>"restaurant","barrier"=>"wall","cuisine"=>"chinese","internet_access"=>"yes" + + + + + + + yes + chinese + wall + restaurant + + + + + 5,5 6,4 4,4 5,5 + "amenity"=>"fuel","building"=>"roof","name"=>"foo" + + + + + foo + roof + + + + fuel + + + + + 2,5 2,6 3,6 3,5 2,5 + "building"=>"yes" + + + + + + yes + + + + + + + + + 6,1 10,1 10,-3 6,-3 6,17,0 7,-2 9,-2 9,0 7,0 + "amenity"=>"restaurant","cuisine"=>"burger","name"=>"bar","operator"=>"foo" + + + + foo + bar + + + burger + + restaurant + + + + + "amenity"=>"bank","atm"=>"yes" + + + yes + + + + + + + bank + + + + + 3,2 6,1 6,-3 2,-1 2,2 3,2 + "stars"=>"5","tourism"=>"hotel" + hotel + 5 + + + + + + + + + + + diff --git a/python/plugins/processing/tests/testdata/expected/hstore_all_keys.xsd b/python/plugins/processing/tests/testdata/expected/hstore_all_keys.xsd new file mode 100644 index 00000000000..db08fbff847 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hstore_all_keys.xsd @@ -0,0 +1,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/python/plugins/processing/tests/testdata/expected/hstore_two_fields.gml b/python/plugins/processing/tests/testdata/expected/hstore_two_fields.gml new file mode 100644 index 00000000000..403d88a95f8 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hstore_two_fields.gml @@ -0,0 +1,67 @@ + + + + + -1-3 + 106 + + + + + + -1,-1 -1,3 3,3 3,2 2,2 2,-1 -1,-1 + "amenity"=>"restaurant","barrier"=>"wall" + yes + chinese + + + + + + 5,5 6,4 4,4 5,5 + "amenity"=>"fuel","building"=>"roof","name"=>"foo" + + + + + + + + 2,5 2,6 3,6 3,5 2,5 + "building"=>"yes" + + + + + + + + 6,1 10,1 10,-3 6,-3 6,17,0 7,-2 9,-2 9,0 7,0 + "amenity"=>"restaurant","name"=>"bar","operator"=>"foo" + + burger + + + + + + "amenity"=>"bank","atm"=>"yes" + + + + + + + + 3,2 6,1 6,-3 2,-1 2,2 3,2 + "stars"=>"5","tourism"=>"hotel" + + + + + + diff --git a/python/plugins/processing/tests/testdata/expected/hstore_two_fields.xsd b/python/plugins/processing/tests/testdata/expected/hstore_two_fields.xsd new file mode 100644 index 00000000000..d36548566af --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hstore_two_fields.xsd @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 475d390ec61..9bae1383da6 100755 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -914,6 +914,32 @@ tests: name: expected/zm_dropped.shp type: vector + - algorithm: native:explodehstorefield + name: Test explode HStore field with all keys + params: + EXPECTED_FIELDS: '' + FIELD: hstore + INPUT: + name: custom/hstore.gml + type: vector + results: + OUTPUT: + name: expected/hstore_all_keys.gml + type: vector + + - algorithm: native:explodehstorefield + name: Test explode HStore field with 2 fields + params: + EXPECTED_FIELDS: internet_access,cuisine,doesntexist + FIELD: hstore + INPUT: + name: custom/hstore.gml + type: vector + results: + OUTPUT: + name: expected/hstore_two_fields.gml + type: vector + - algorithm: native:pointonsurface name: Point on polygon surface params: diff --git a/src/analysis/CMakeLists.txt b/src/analysis/CMakeLists.txt index d3d385f6862..8d2a9342030 100644 --- a/src/analysis/CMakeLists.txt +++ b/src/analysis/CMakeLists.txt @@ -38,6 +38,7 @@ SET(QGIS_ANALYSIS_SRCS processing/qgsalgorithmdropgeometry.cpp processing/qgsalgorithmdropmzvalues.cpp processing/qgsalgorithmexplode.cpp + processing/qgsalgorithmexplodehstore.cpp processing/qgsalgorithmextendlines.cpp processing/qgsalgorithmextenttolayer.cpp processing/qgsalgorithmextractbyattribute.cpp diff --git a/src/analysis/processing/qgsalgorithmexplodehstore.cpp b/src/analysis/processing/qgsalgorithmexplodehstore.cpp new file mode 100644 index 00000000000..5f3d7fdddb2 --- /dev/null +++ b/src/analysis/processing/qgsalgorithmexplodehstore.cpp @@ -0,0 +1,184 @@ +/*************************************************************************** + qgsalgorithmexplodehstore.h + --------------------- + begin : September 2018 + copyright : (C) 2018 by Etienne Trimaille + email : etienne dot trimaille at gmail dot com + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#include "qgis.h" +#include "qgsalgorithmexplodehstore.h" +#include "qgshstoreutils.h" +#include "qgsprocessingutils.h" + +///@cond PRIVATE + +QString QgsExplodeHstoreAlgorithm::name() const +{ + return QStringLiteral( "explodehstorefield" ); +} + +QString QgsExplodeHstoreAlgorithm::displayName() const +{ + return QObject::tr( "Explode HStore Field" ); +} + +QStringList QgsExplodeHstoreAlgorithm::tags() const +{ + return QObject::tr( "field,explode,hstore,osm,openstreetmap" ).split( ',' ); +} + +QString QgsExplodeHstoreAlgorithm::group() const +{ + return QObject::tr( "Vector table" ); +} + +QString QgsExplodeHstoreAlgorithm::groupId() const +{ + return QStringLiteral( "vectortable" ); +} + +QString QgsExplodeHstoreAlgorithm::shortHelpString() const +{ + return QObject::tr( "This algorithm creates a copy of the input layer and adds a new field for every unique key in the HStore field.\n" + "The expected field list is an optional comma separated list. By default, all unique keys are added. If this list is specified, only these fields are added and the HStore field is updated." ); +} + +QgsProcessingAlgorithm *QgsExplodeHstoreAlgorithm::createInstance() const +{ + return new QgsExplodeHstoreAlgorithm(); +} + +void QgsExplodeHstoreAlgorithm::initAlgorithm( const QVariantMap & ) +{ + addParameter( new QgsProcessingParameterFeatureSource( QStringLiteral( "INPUT" ), + QObject::tr( "Input layer" ) ) ); + addParameter( new QgsProcessingParameterField( QStringLiteral( "FIELD" ), + QObject::tr( "HStore field" ), QVariant(), QStringLiteral( "INPUT" ) ) ); + addParameter( new QgsProcessingParameterString( QStringLiteral( "EXPECTED_FIELDS" ), QObject::tr( "Expected list of fields separated by a comma" ), QVariant(), false, true ) ); + addParameter( new QgsProcessingParameterFeatureSink( QStringLiteral( "OUTPUT" ), QObject::tr( "Exploded" ) ) ); +} + +QVariantMap QgsExplodeHstoreAlgorithm::processAlgorithm( const QVariantMap ¶meters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) +{ + std::unique_ptr< QgsProcessingFeatureSource > source( parameterAsSource( parameters, QStringLiteral( "INPUT" ), context ) ); + if ( !source ) + throw QgsProcessingException( invalidSourceError( parameters, QStringLiteral( "INPUT" ) ) ); + int attrSourceCount = source->fields().count(); + + QString fieldName = parameterAsString( parameters, QStringLiteral( "FIELD" ), context ); + int fieldIndex = source->fields().lookupField( fieldName ); + if ( fieldIndex < 0 ) + throw QgsProcessingException( QObject::tr( "Invalid HStore field" ) ); + + QStringList expectedFields; + QString fieldList = parameterAsString( parameters, QStringLiteral( "EXPECTED_FIELDS" ), context ); + if ( ! fieldList.trimmed().isEmpty() ) + { + expectedFields = fieldList.split( ',' ); + } + + QList fieldsToAdd; + QHash hstoreFeatures; + QList features; + + double step = source->featureCount() > 0 ? 50.0 / source->featureCount() : 1; + int i = 0; + QgsFeatureIterator featIterator = source->getFeatures( ); + QgsFeature feat; + while ( featIterator.nextFeature( feat ) ) + { + i++; + if ( feedback->isCanceled() ) + break; + + double progress = i * step; + if ( progress >= 50 ) + feedback->setProgress( 50.0 ); + else + feedback->setProgress( progress ); + + QVariantMap currentHStore = QgsHstoreUtils::parse( feat.attribute( fieldName ).toString() ); + for ( const QString &key : currentHStore.keys() ) + { + if ( expectedFields.isEmpty() && ! fieldsToAdd.contains( key ) ) + fieldsToAdd.insert( 0, key ); + } + hstoreFeatures.insert( feat.id(), currentHStore ); + features.append( feat ); + } + + if ( ! expectedFields.isEmpty() ) + { + fieldsToAdd = expectedFields; + } + + QgsFields hstoreFields; + for ( const QString &fieldName : fieldsToAdd ) + { + hstoreFields.append( QgsField( fieldName, QVariant::String ) ); + } + + QgsFields outFields = QgsProcessingUtils::combineFields( source->fields(), hstoreFields ); + + QString sinkId; + std::unique_ptr< QgsFeatureSink > sink( parameterAsSink( parameters, QStringLiteral( "OUTPUT" ), context, sinkId, outFields, source->wkbType(), source->sourceCrs() ) ); + if ( !sink ) + throw QgsProcessingException( invalidSinkError( parameters, QStringLiteral( "OUTPUT" ) ) ); + + QList fieldIndicesInput = QgsProcessingUtils::fieldNamesToIndices( QStringList(), source->fields() ); + int attrCount = attrSourceCount + fieldsToAdd.count(); + QgsFeature outFeature; + step = !features.empty() ? 50.0 / features.count() : 1; + i = 0; + for ( const QgsFeature &feat : qgis::as_const( features ) ) + { + i++; + if ( feedback->isCanceled() ) + break; + + feedback->setProgress( i * step + 50.0 ); + + QgsAttributes outAttributes( attrCount ); + + const QgsAttributes attrs( feat.attributes() ); + for ( int i = 0; i < fieldIndicesInput.count(); ++i ) + outAttributes[i] = attrs[fieldIndicesInput[i]]; + + QVariantMap currentHStore = hstoreFeatures.take( feat.id() ); + + QString current; + for ( int i = 0; i < fieldsToAdd.count(); ++i ) + { + current = fieldsToAdd.at( i ); + if ( currentHStore.contains( current ) ) + { + outAttributes[attrSourceCount + i] = currentHStore.take( current ); + } + } + + if ( ! expectedFields.isEmpty() ) + { + outAttributes[fieldIndex] = QgsHstoreUtils::build( currentHStore ); + } + + outFeature.setGeometry( QgsGeometry( feat.geometry() ) ); + outFeature.setAttributes( outAttributes ); + sink->addFeature( outFeature, QgsFeatureSink::FastInsert ); + } + + QVariantMap outputs; + outputs.insert( QStringLiteral( "OUTPUT" ), sinkId ); + return outputs; +} + +///@endcond diff --git a/src/analysis/processing/qgsalgorithmexplodehstore.h b/src/analysis/processing/qgsalgorithmexplodehstore.h new file mode 100644 index 00000000000..5040cbe6405 --- /dev/null +++ b/src/analysis/processing/qgsalgorithmexplodehstore.h @@ -0,0 +1,54 @@ +/*************************************************************************** + qgsalgorithmexplodehstore.h + --------------------- + begin : September 2018 + copyright : (C) 2018 by Etienne Trimaille + email : etienne dot trimaille at gmail dot com + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#ifndef QGSALGORITHMEXPLODEHSTORE_H +#define QGSALGORITHMEXPLODEHSTORE_H + +#define SIP_NO_FILE + +#include "qgis.h" +#include "qgsprocessingalgorithm.h" + +///@cond PRIVATE + +/** + * Native explode hstore algorithm. + */ +class QgsExplodeHstoreAlgorithm : public QgsProcessingAlgorithm +{ + + public: + QgsExplodeHstoreAlgorithm() = default; + QString name() const override; + QString displayName() const override; + QString group() const override; + QString groupId() const override; + QStringList tags() const override; + QString shortHelpString() const override; + + protected: + QgsProcessingAlgorithm *createInstance() const override; + void initAlgorithm( const QVariantMap &configuration = QVariantMap() ) override; + QVariantMap processAlgorithm( const QVariantMap ¶meters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) override; + +}; + +///@endcond PRIVATE + +#endif // QGSALGORITHMEXPLODEHSTORE_H + + diff --git a/src/analysis/processing/qgsnativealgorithms.cpp b/src/analysis/processing/qgsnativealgorithms.cpp index 51cb3ec9333..db86d96ff41 100644 --- a/src/analysis/processing/qgsnativealgorithms.cpp +++ b/src/analysis/processing/qgsnativealgorithms.cpp @@ -33,6 +33,7 @@ #include "qgsalgorithmdropgeometry.h" #include "qgsalgorithmdropmzvalues.h" #include "qgsalgorithmexplode.h" +#include "qgsalgorithmexplodehstore.h" #include "qgsalgorithmextendlines.h" #include "qgsalgorithmextenttolayer.h" #include "qgsalgorithmextractbyattribute.h" @@ -158,6 +159,7 @@ void QgsNativeAlgorithms::loadAlgorithms() addAlgorithm( new QgsDropGeometryAlgorithm() ); addAlgorithm( new QgsDropMZValuesAlgorithm() ); addAlgorithm( new QgsExplodeAlgorithm() ); + addAlgorithm( new QgsExplodeHstoreAlgorithm() ); addAlgorithm( new QgsExtendLinesAlgorithm() ); addAlgorithm( new QgsExtentToLayerAlgorithm() ); addAlgorithm( new QgsExtractByAttributeAlgorithm() );