From a8a3cc82ed209fed9f65bb3879339f822e9a1962 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Tue, 2 May 2017 21:15:54 +1000 Subject: [PATCH] [processing] Port vector.createVectorWriter to c++ This implements an improved version of vector.createVectorWriter in QgsProcessingUtils. The improved version relies on the core class QgsVectorLayerImport to create empty layers, which: - reduces duplicate code and reuses the mature QgsVectorLayerImport routines - avoids manual conversion of field types to destination provider field types - potentially allows any writable provider to be used as a feature sink for algorithms (e.g. output direct to MSSQL/Oracle/db2). This should work now - it just needs exposing via UI. --- .../core/processing/qgsprocessingcontext.sip | 14 ++ python/core/processing/qgsprocessingutils.sip | 32 +++++ .../plugins/processing/tools/dataobjects.py | 3 + python/plugins/processing/tools/vector.py | 1 + src/core/processing/qgsprocessingcontext.h | 14 ++ src/core/processing/qgsprocessingutils.cpp | 115 +++++++++++++++ src/core/processing/qgsprocessingutils.h | 52 +++++++ tests/src/core/testqgsprocessing.cpp | 136 ++++++++++++++++++ 8 files changed, 367 insertions(+) diff --git a/python/core/processing/qgsprocessingcontext.sip b/python/core/processing/qgsprocessingcontext.sip index a554970843c..fa3b8805813 100644 --- a/python/core/processing/qgsprocessingcontext.sip +++ b/python/core/processing/qgsprocessingcontext.sip @@ -118,9 +118,23 @@ class QgsProcessingContext %End + QString defaultEncoding() const; +%Docstring + Returns the default encoding to use for newly created files. +.. seealso:: setDefaultEncoding() + :rtype: str +%End + + void setDefaultEncoding( const QString &encoding ); +%Docstring + Sets the default ``encoding`` to use for newly created files. +.. seealso:: defaultEncoding() +%End + private: QgsProcessingContext( const QgsProcessingContext &other ); }; + QFlags operator|(QgsProcessingContext::Flag f1, QFlags f2); diff --git a/python/core/processing/qgsprocessingutils.sip b/python/core/processing/qgsprocessingutils.sip index 1f932fae07e..f20db541449 100644 --- a/python/core/processing/qgsprocessingutils.sip +++ b/python/core/processing/qgsprocessingutils.sip @@ -122,6 +122,38 @@ class QgsProcessingUtils :rtype: list of QVariant %End + + static void createFeatureSinkPython( + QgsFeatureSink **sink /Out,TransferBack/, + QString &destination /In,Out/, + const QString &encoding, + const QgsFields &fields, + QgsWkbTypes::Type geometryType, + const QgsCoordinateReferenceSystem &crs, + QgsProcessingContext &context, + QgsVectorLayer **outputLayer /Out/ ) /PyName=createFeatureSink/; +%Docstring + Creates a feature sink ready for adding features. The ``destination`` specifies a destination + URI for the resultant layer. It may be updated in place to reflect the actual destination + for the layer. + + Sink parameters such as desired ``encoding``, ``fields``, ``geometryType`` and ``crs`` must be specified. + + If the ``encoding`` is not specified, the default encoding from the ``context`` will be used. + + If a layer is created for the feature sink, the layer will automatically be added to the ``context``'s + temporary layer store, and the ``outputLayer`` argument updated to point at this newly created layer. + +.. note:: + + this version of the createFeatureSink() function has an API designed around use from the + SIP bindings. c++ code should call the other createFeatureSink() version. +.. note:: + + available in Python bindings as createFeatureSink() +%End + + }; diff --git a/python/plugins/processing/tools/dataobjects.py b/python/plugins/processing/tools/dataobjects.py index 45aa4835af5..ac029fbb25b 100644 --- a/python/plugins/processing/tools/dataobjects.py +++ b/python/plugins/processing/tools/dataobjects.py @@ -85,6 +85,9 @@ def createContext(): context.setInvalidGeometryCallback(raise_error) + settings = QgsSettings() + context.setDefaultEncoding(settings.value("/Processing/encoding", "System")) + return context diff --git a/python/plugins/processing/tools/vector.py b/python/plugins/processing/tools/vector.py index 787c5f5623d..279409aeccd 100644 --- a/python/plugins/processing/tools/vector.py +++ b/python/plugins/processing/tools/vector.py @@ -460,6 +460,7 @@ NOGEOMETRY_EXTENSIONS = [ def createVectorWriter(destination, encoding, fields, geometryType, crs, context): + return QgsProcessingUtils.createFeatureSink(destination, encoding, fields, geometryType, crs, context) layer = None sink = None diff --git a/src/core/processing/qgsprocessingcontext.h b/src/core/processing/qgsprocessingcontext.h index d7f692d87c0..348ee6defb3 100644 --- a/src/core/processing/qgsprocessingcontext.h +++ b/src/core/processing/qgsprocessingcontext.h @@ -142,6 +142,18 @@ class CORE_EXPORT QgsProcessingContext */ SIP_SKIP std::function< void( const QgsFeature & ) > invalidGeometryCallback() const { return mInvalidGeometryCallback; } + /** + * Returns the default encoding to use for newly created files. + * \see setDefaultEncoding() + */ + QString defaultEncoding() const { return mDefaultEncoding; } + + /** + * Sets the default \a encoding to use for newly created files. + * \see defaultEncoding() + */ + void setDefaultEncoding( const QString &encoding ) { mDefaultEncoding = encoding; } + private: QgsProcessingContext::Flags mFlags = 0; @@ -151,11 +163,13 @@ class CORE_EXPORT QgsProcessingContext QgsExpressionContext mExpressionContext; QgsFeatureRequest::InvalidGeometryCheck mInvalidGeometryCheck = QgsFeatureRequest::GeometryNoCheck; std::function< void( const QgsFeature & ) > mInvalidGeometryCallback; + QString mDefaultEncoding; #ifdef SIP_RUN QgsProcessingContext( const QgsProcessingContext &other ); #endif }; + Q_DECLARE_OPERATORS_FOR_FLAGS( QgsProcessingContext::Flags ) #endif // QGSPROCESSINGPARAMETERS_H diff --git a/src/core/processing/qgsprocessingutils.cpp b/src/core/processing/qgsprocessingutils.cpp index 1f012af1381..3e33e60b5a0 100644 --- a/src/core/processing/qgsprocessingutils.cpp +++ b/src/core/processing/qgsprocessingutils.cpp @@ -19,6 +19,9 @@ #include "qgsproject.h" #include "qgssettings.h" #include "qgsprocessingcontext.h" +#include "qgsvectorlayerimport.h" +#include "qgsvectorfilewriter.h" +#include "qgsmemoryproviderutils.h" QList QgsProcessingUtils::compatibleRasterLayers( QgsProject *project, bool sort ) { @@ -289,4 +292,116 @@ QList QgsProcessingUtils::uniqueValues( QgsVectorLayer *layer, int fie } } +void parseDestinationString( QString &destination, QString &providerKey, QString &uri, QString &format, QMap &options ) +{ + QRegularExpression splitRx( "^(.*?):(.*)$" ); + QRegularExpressionMatch match = splitRx.match( destination ); + if ( match.hasMatch() ) + { + providerKey = match.captured( 1 ); + if ( providerKey == QStringLiteral( "postgis" ) ) // older processing used "postgis" instead of "postgres" + { + providerKey = QStringLiteral( "postgres" ); + } + uri = match.captured( 2 ); + } + else + { + providerKey = QStringLiteral( "ogr" ); + QRegularExpression splitRx( "^(.*)\\.(.*?)$" ); + QRegularExpressionMatch match = splitRx.match( destination ); + QString extension; + if ( match.hasMatch() ) + { + extension = match.captured( 2 ); + format = QgsVectorFileWriter::driverForExtension( extension ); + } + + if ( format.isEmpty() ) + { + format = QStringLiteral( "ESRI Shapefile" ); + destination = destination + QStringLiteral( ".shp" ); + } + + options.insert( QStringLiteral( "driverName" ), format ); + uri = destination; + } +} + +QgsFeatureSink *QgsProcessingUtils::createFeatureSink( QString &destination, const QString &encoding, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, QgsProcessingContext &context, QgsVectorLayer *&outputLayer ) +{ + outputLayer = nullptr; + QgsVectorLayer *layer = nullptr; + + QString destEncoding = encoding; + if ( destEncoding.isEmpty() ) + { + // no destination encoding specified, use default + destEncoding = context.defaultEncoding().isEmpty() ? QStringLiteral( "system" ) : context.defaultEncoding(); + } + + if ( destination.isEmpty() || destination.startsWith( QStringLiteral( "memory:" ) ) ) + { + // memory provider cannot be used with QgsVectorLayerImport - so create layer manually + layer = QgsMemoryProviderUtils::createMemoryLayer( destination, fields, geometryType, crs ); + if ( layer && layer->isValid() ) + destination = layer->id(); + } + else + { + QMap options; + options.insert( QStringLiteral( "fileEncoding" ), destEncoding ); + + QString providerKey; + QString uri; + QString format; + parseDestinationString( destination, providerKey, uri, format, options ); + + if ( providerKey == "ogr" ) + { + // use QgsVectorFileWriter for OGR destinations instead of QgsVectorLayerImport, as that allows + // us to use any OGR format which supports feature addition + QString finalFileName; + QgsVectorFileWriter *writer = new QgsVectorFileWriter( destination, destEncoding, fields, geometryType, crs, format, QgsVectorFileWriter::defaultDatasetOptions( format ), + QgsVectorFileWriter::defaultLayerOptions( format ), &finalFileName ); + destination = finalFileName; + return writer; + } + else + { + //create empty layer + { + QgsVectorLayerImport import( uri, providerKey, fields, geometryType, crs, false, &options ); + if ( import.hasError() ) + return nullptr; + } + + layer = new QgsVectorLayer( uri, destination, providerKey ); + } + } + + if ( !layer ) + return nullptr; + + if ( !layer->isValid() ) + { + delete layer; + return nullptr; + } + + context.temporaryLayerStore()->addMapLayer( layer ); + + outputLayer = layer; + // this is a factory, so we need to return a proxy + return new QgsProxyFeatureSink( layer->dataProvider() ); +} + +void QgsProcessingUtils::createFeatureSinkPython( QgsFeatureSink **sink, QString &destination, const QString &encoding, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, QgsProcessingContext &context, QgsVectorLayer **outputLayer ) +{ + QgsVectorLayer *layer = nullptr; + *sink = createFeatureSink( destination, encoding, fields, geometryType, crs, context, layer ); + if ( outputLayer ) + *outputLayer = layer; +} + diff --git a/src/core/processing/qgsprocessingutils.h b/src/core/processing/qgsprocessingutils.h index a26dfc5d941..f400cccbb81 100644 --- a/src/core/processing/qgsprocessingutils.h +++ b/src/core/processing/qgsprocessingutils.h @@ -131,6 +131,58 @@ class CORE_EXPORT QgsProcessingUtils */ static QList< QVariant > uniqueValues( QgsVectorLayer *layer, int fieldIndex, const QgsProcessingContext &context ); + /** + * Creates a feature sink ready for adding features. The \a destination specifies a destination + * URI for the resultant layer. It may be updated in place to reflect the actual destination + * for the layer. + * + * Sink parameters such as desired \a encoding, \a fields, \a geometryType and \a crs must be specified. + * + * If the \a encoding is not specified, the default encoding from the \a context will be used. + * + * If a layer is created for the feature sink, the layer will automatically be added to the \a context's + * temporary layer store, and the \a outputLayer argument updated to point at this newly created layer. + * + * The caller takes responsibility for deleting the returned sink. + */ +#ifndef SIP_RUN + static QgsFeatureSink *createFeatureSink( + QString &destination, + const QString &encoding, + const QgsFields &fields, + QgsWkbTypes::Type geometryType, + const QgsCoordinateReferenceSystem &crs, + QgsProcessingContext &context, + QgsVectorLayer *&outputLayer ) SIP_FACTORY; +#endif + + /** + * Creates a feature sink ready for adding features. The \a destination specifies a destination + * URI for the resultant layer. It may be updated in place to reflect the actual destination + * for the layer. + * + * Sink parameters such as desired \a encoding, \a fields, \a geometryType and \a crs must be specified. + * + * If the \a encoding is not specified, the default encoding from the \a context will be used. + * + * If a layer is created for the feature sink, the layer will automatically be added to the \a context's + * temporary layer store, and the \a outputLayer argument updated to point at this newly created layer. + * + * \note this version of the createFeatureSink() function has an API designed around use from the + * SIP bindings. c++ code should call the other createFeatureSink() version. + * \note available in Python bindings as createFeatureSink() + */ + static void createFeatureSinkPython( + QgsFeatureSink **sink SIP_OUT SIP_TRANSFERBACK, + QString &destination SIP_INOUT, + const QString &encoding, + const QgsFields &fields, + QgsWkbTypes::Type geometryType, + const QgsCoordinateReferenceSystem &crs, + QgsProcessingContext &context, + QgsVectorLayer **outputLayer SIP_OUT ) SIP_PYNAME( createFeatureSink ); + + private: static bool canUseLayer( const QgsRasterLayer *layer ); diff --git a/tests/src/core/testqgsprocessing.cpp b/tests/src/core/testqgsprocessing.cpp index 76d5c122827..7379d5517b8 100644 --- a/tests/src/core/testqgsprocessing.cpp +++ b/tests/src/core/testqgsprocessing.cpp @@ -28,6 +28,7 @@ #include "qgsproject.h" #include "qgspointv2.h" #include "qgsgeometry.h" +#include "qgsvectorfilewriter.h" class DummyAlgorithm : public QgsProcessingAlgorithm { @@ -104,6 +105,7 @@ class TestQgsProcessing: public QObject void removeProvider(); void compatibleLayers(); void normalizeLayerSource(); + void context(); void mapLayers(); void mapLayerFromStore(); void mapLayerFromString(); @@ -111,6 +113,7 @@ class TestQgsProcessing: public QObject void features(); void uniqueValues(); void createIndex(); + void createFeatureSink(); private: @@ -124,6 +127,9 @@ void TestQgsProcessing::initTestCase() void TestQgsProcessing::cleanupTestCase() { + QFile::remove( QDir::tempPath() + "/create_feature_sink.tab" ); + QgsVectorFileWriter::deleteShapeFile( QDir::tempPath() + "/create_feature_sink2.shp" ); + QgsApplication::exitQgis(); } @@ -334,6 +340,27 @@ void TestQgsProcessing::normalizeLayerSource() QCOMPARE( QgsProcessingUtils::normalizeLayerSource( "data\\layers \"new\"\\test.shp" ), QString( "data/layers 'new'/test.shp" ) ); } +void TestQgsProcessing::context() +{ + QgsProcessingContext context; + + // simple tests for getters/setters + context.setDefaultEncoding( "my_enc" ); + QCOMPARE( context.defaultEncoding(), QStringLiteral( "my_enc" ) ); + + context.setFlags( QgsProcessingContext::UseSelectionIfPresent ); + QCOMPARE( context.flags(), QgsProcessingContext::UseSelectionIfPresent ); + context.setFlags( QgsProcessingContext::Flags( 0 ) ); + QCOMPARE( context.flags(), QgsProcessingContext::Flags( 0 ) ); + + QgsProject p; + context.setProject( &p ); + QCOMPARE( context.project(), &p ); + + context.setInvalidGeometryCheck( QgsFeatureRequest::GeometrySkipInvalid ); + QCOMPARE( context.invalidGeometryCheck(), QgsFeatureRequest::GeometrySkipInvalid ); +} + void TestQgsProcessing::mapLayers() { QString testDataDir = QStringLiteral( TEST_DATA_DIR ) + '/'; //defined in CmakeLists.txt @@ -711,5 +738,114 @@ void TestQgsProcessing::createIndex() } +void TestQgsProcessing::createFeatureSink() +{ + QgsProcessingContext context; + + // empty destination + QString destination; + destination = QString(); + QgsVectorLayer *layer = nullptr; + + // should create a memory layer + QgsFeatureSink *sink = QgsProcessingUtils::createFeatureSink( destination, QString(), QgsFields(), QgsWkbTypes::Point, QgsCoordinateReferenceSystem(), context, layer ); + QVERIFY( sink ); + QVERIFY( layer ); + QCOMPARE( static_cast< QgsProxyFeatureSink *>( sink )->destinationSink(), layer->dataProvider() ); + QCOMPARE( layer->dataProvider()->name(), QStringLiteral( "memory" ) ); + QCOMPARE( destination, layer->id() ); + QCOMPARE( context.temporaryLayerStore()->mapLayer( layer->id() ), layer ); // layer should be in store + QgsFeature f; + QCOMPARE( layer->featureCount(), 0L ); + QVERIFY( sink->addFeature( f ) ); + QCOMPARE( layer->featureCount(), 1L ); + context.temporaryLayerStore()->removeAllMapLayers(); + layer = nullptr; + delete sink; + + // specific memory layer output + destination = QStringLiteral( "memory:mylayer" ); + sink = QgsProcessingUtils::createFeatureSink( destination, QString(), QgsFields(), QgsWkbTypes::Point, QgsCoordinateReferenceSystem(), context, layer ); + QVERIFY( sink ); + QVERIFY( layer ); + QCOMPARE( static_cast< QgsProxyFeatureSink *>( sink )->destinationSink(), layer->dataProvider() ); + QCOMPARE( layer->dataProvider()->name(), QStringLiteral( "memory" ) ); + QCOMPARE( layer->name(), QStringLiteral( "memory:mylayer" ) ); + QCOMPARE( destination, layer->id() ); + QCOMPARE( context.temporaryLayerStore()->mapLayer( layer->id() ), layer ); // layer should be in store + QCOMPARE( layer->featureCount(), 0L ); + QVERIFY( sink->addFeature( f ) ); + QCOMPARE( layer->featureCount(), 1L ); + context.temporaryLayerStore()->removeAllMapLayers(); + layer = nullptr; + delete sink; + + // memory layer parameters + destination = QStringLiteral( "memory:mylayer" ); + QgsFields fields; + fields.append( QgsField( QStringLiteral( "my_field" ), QVariant::String, QString(), 100 ) ); + sink = QgsProcessingUtils::createFeatureSink( destination, QString(), fields, QgsWkbTypes::PointZM, QgsCoordinateReferenceSystem::fromEpsgId( 3111 ), context, layer ); + QVERIFY( sink ); + QVERIFY( layer ); + QCOMPARE( static_cast< QgsProxyFeatureSink *>( sink )->destinationSink(), layer->dataProvider() ); + QCOMPARE( layer->dataProvider()->name(), QStringLiteral( "memory" ) ); + QCOMPARE( layer->name(), QStringLiteral( "memory:mylayer" ) ); + QCOMPARE( layer->wkbType(), QgsWkbTypes::PointZM ); + QCOMPARE( layer->crs().authid(), QStringLiteral( "EPSG:3111" ) ); + QCOMPARE( layer->fields().size(), 1 ); + QCOMPARE( layer->fields().at( 0 ).name(), QStringLiteral( "my_field" ) ); + QCOMPARE( layer->fields().at( 0 ).type(), QVariant::String ); + QCOMPARE( destination, layer->id() ); + QCOMPARE( context.temporaryLayerStore()->mapLayer( layer->id() ), layer ); // layer should be in store + QCOMPARE( layer->featureCount(), 0L ); + QVERIFY( sink->addFeature( f ) ); + QCOMPARE( layer->featureCount(), 1L ); + context.temporaryLayerStore()->removeAllMapLayers(); + layer = nullptr; + delete sink; + + // non memory layer output + destination = QDir::tempPath() + "/create_feature_sink.tab"; + QString prevDest = destination; + sink = QgsProcessingUtils::createFeatureSink( destination, QString(), fields, QgsWkbTypes::Polygon, QgsCoordinateReferenceSystem::fromEpsgId( 3111 ), context, layer ); + QVERIFY( sink ); + f = QgsFeature( fields ); + f.setGeometry( QgsGeometry::fromWkt( QStringLiteral( "Polygon((0 0, 0 1, 1 1, 1 0, 0 0 ))" ) ) ); + f.setAttributes( QgsAttributes() << "val" ); + QVERIFY( sink->addFeature( f ) ); + QVERIFY( !layer ); + QCOMPARE( destination, prevDest ); + delete sink; + layer = new QgsVectorLayer( destination, "test_layer", "ogr" ); + QVERIFY( layer->isValid() ); + QCOMPARE( layer->crs().authid(), QStringLiteral( "EPSG:3111" ) ); + QCOMPARE( layer->fields().size(), 1 ); + QCOMPARE( layer->fields().at( 0 ).name(), QStringLiteral( "my_field" ) ); + QCOMPARE( layer->fields().at( 0 ).type(), QVariant::String ); + QCOMPARE( layer->featureCount(), 1L ); + delete layer; + layer = nullptr; + + // no extension, should default to shp + destination = QDir::tempPath() + "/create_feature_sink2"; + prevDest = QDir::tempPath() + "/create_feature_sink2.shp"; + sink = QgsProcessingUtils::createFeatureSink( destination, QString(), fields, QgsWkbTypes::Point25D, QgsCoordinateReferenceSystem::fromEpsgId( 3111 ), context, layer ); + QVERIFY( sink ); + f.setGeometry( QgsGeometry::fromWkt( QStringLiteral( "PointZ(1 2 3)" ) ) ); + QVERIFY( sink->addFeature( f ) ); + QVERIFY( !layer ); + QCOMPARE( destination, prevDest ); + delete sink; + layer = new QgsVectorLayer( destination, "test_layer", "ogr" ); + QCOMPARE( layer->wkbType(), QgsWkbTypes::Point25D ); + QCOMPARE( layer->crs().authid(), QStringLiteral( "EPSG:3111" ) ); + QCOMPARE( layer->fields().size(), 1 ); + QCOMPARE( layer->fields().at( 0 ).name(), QStringLiteral( "my_field" ) ); + QCOMPARE( layer->fields().at( 0 ).type(), QVariant::String ); + QCOMPARE( layer->featureCount(), 1L ); + delete layer; + layer = nullptr; +} + QGSTEST_MAIN( TestQgsProcessing ) #include "testqgsprocessing.moc"