[processing] Port vector.createVectorWriter to c++

This implements an improved version of vector.createVectorWriter
in QgsProcessingUtils. The improved version relies on the
core class QgsVectorLayerImport to create empty layers,
which:
- reduces duplicate code and reuses the mature QgsVectorLayerImport
routines
- avoids manual conversion of field types to destination provider
field types
- potentially allows any writable provider to be used as a feature
sink for algorithms (e.g. output direct to MSSQL/Oracle/db2). This
should work now - it just needs exposing via UI.

python/core/processing/qgsprocessingcontext.sip

@@ -118,9 +118,23 @@ class QgsProcessingContext
 %End
 
 
+    QString defaultEncoding() const;
+%Docstring
+ Returns the default encoding to use for newly created files.
+.. seealso:: setDefaultEncoding()
+ :rtype: str
+%End
+
+    void setDefaultEncoding( const QString &encoding );
+%Docstring
+ Sets the default ``encoding`` to use for newly created files.
+.. seealso:: defaultEncoding()
+%End
+
   private:
     QgsProcessingContext( const QgsProcessingContext &other );
 };
+
 QFlags<QgsProcessingContext::Flag> operator|(QgsProcessingContext::Flag f1, QFlags<QgsProcessingContext::Flag> f2);
 
 

python/core/processing/qgsprocessingutils.sip

@@ -122,6 +122,38 @@ class QgsProcessingUtils
  :rtype: list of QVariant
 %End
 
+
+    static void createFeatureSinkPython(
+      QgsFeatureSink **sink /Out,TransferBack/,
+      QString &destination /In,Out/,
+      const QString &encoding,
+      const QgsFields &fields,
+      QgsWkbTypes::Type geometryType,
+      const QgsCoordinateReferenceSystem &crs,
+      QgsProcessingContext &context,
+      QgsVectorLayer **outputLayer /Out/ ) /PyName=createFeatureSink/;
+%Docstring
+ Creates a feature sink ready for adding features. The ``destination`` specifies a destination
+ URI for the resultant layer. It may be updated in place to reflect the actual destination
+ for the layer.
+
+ Sink parameters such as desired ``encoding``, ``fields``, ``geometryType`` and ``crs`` must be specified.
+
+ If the ``encoding`` is not specified, the default encoding from the ``context`` will be used.
+
+ If a layer is created for the feature sink, the layer will automatically be added to the ``context``'s
+ temporary layer store, and the ``outputLayer`` argument updated to point at this newly created layer.
+
+.. note::
+
+   this version of the createFeatureSink() function has an API designed around use from the
+ SIP bindings. c++ code should call the other createFeatureSink() version.
+.. note::
+
+   available in Python bindings as createFeatureSink()
+%End
+
+
 };
 
 

python/plugins/processing/tools/dataobjects.py

@@ -85,6 +85,9 @@ def createContext():
 
     context.setInvalidGeometryCallback(raise_error)
 
+    settings = QgsSettings()
+    context.setDefaultEncoding(settings.value("/Processing/encoding", "System"))
+
     return context
 
 

python/plugins/processing/tools/vector.py

@@ -460,6 +460,7 @@ NOGEOMETRY_EXTENSIONS = [
 
 
 def createVectorWriter(destination, encoding, fields, geometryType, crs, context):
+    return QgsProcessingUtils.createFeatureSink(destination, encoding, fields, geometryType, crs, context)
     layer = None
     sink = None
 

src/core/processing/qgsprocessingcontext.h

@@ -142,6 +142,18 @@ class CORE_EXPORT QgsProcessingContext
      */
     SIP_SKIP std::function< void( const QgsFeature & ) > invalidGeometryCallback() const { return mInvalidGeometryCallback; }
 
+    /**
+     * Returns the default encoding to use for newly created files.
+     * \see setDefaultEncoding()
+     */
+    QString defaultEncoding() const { return mDefaultEncoding; }
+
+    /**
+     * Sets the default \a encoding to use for newly created files.
+     * \see defaultEncoding()
+     */
+    void setDefaultEncoding( const QString &encoding ) { mDefaultEncoding = encoding; }
+
   private:
 
     QgsProcessingContext::Flags mFlags = 0;
@@ -151,11 +163,13 @@ class CORE_EXPORT QgsProcessingContext
     QgsExpressionContext mExpressionContext;
     QgsFeatureRequest::InvalidGeometryCheck mInvalidGeometryCheck = QgsFeatureRequest::GeometryNoCheck;
     std::function< void( const QgsFeature & ) > mInvalidGeometryCallback;
+    QString mDefaultEncoding;
 
 #ifdef SIP_RUN
     QgsProcessingContext( const QgsProcessingContext &other );
 #endif
 };
+
 Q_DECLARE_OPERATORS_FOR_FLAGS( QgsProcessingContext::Flags )
 
 #endif // QGSPROCESSINGPARAMETERS_H

src/core/processing/qgsprocessingutils.cpp

@@ -19,6 +19,9 @@
 #include "qgsproject.h"
 #include "qgssettings.h"
 #include "qgsprocessingcontext.h"
+#include "qgsvectorlayerimport.h"
+#include "qgsvectorfilewriter.h"
+#include "qgsmemoryproviderutils.h"
 
 QList<QgsRasterLayer *> QgsProcessingUtils::compatibleRasterLayers( QgsProject *project, bool sort )
 {
@@ -289,4 +292,116 @@ QList<QVariant> QgsProcessingUtils::uniqueValues( QgsVectorLayer *layer, int fie
   }
 }
 
+void parseDestinationString( QString &destination, QString &providerKey, QString &uri, QString &format, QMap<QString, QVariant> &options )
+{
+  QRegularExpression splitRx( "^(.*?):(.*)$" );
+  QRegularExpressionMatch match = splitRx.match( destination );
+  if ( match.hasMatch() )
+  {
+    providerKey = match.captured( 1 );
+    if ( providerKey == QStringLiteral( "postgis" ) ) // older processing used "postgis" instead of "postgres"
+    {
+      providerKey = QStringLiteral( "postgres" );
+    }
+    uri = match.captured( 2 );
+  }
+  else
+  {
+    providerKey = QStringLiteral( "ogr" );
+    QRegularExpression splitRx( "^(.*)\\.(.*?)$" );
+    QRegularExpressionMatch match = splitRx.match( destination );
+    QString extension;
+    if ( match.hasMatch() )
+    {
+      extension = match.captured( 2 );
+      format = QgsVectorFileWriter::driverForExtension( extension );
+    }
+
+    if ( format.isEmpty() )
+    {
+      format = QStringLiteral( "ESRI Shapefile" );
+      destination = destination + QStringLiteral( ".shp" );
+    }
+
+    options.insert( QStringLiteral( "driverName" ), format );
+    uri = destination;
+  }
+}
+
+QgsFeatureSink *QgsProcessingUtils::createFeatureSink( QString &destination, const QString &encoding, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, QgsProcessingContext &context, QgsVectorLayer *&outputLayer )
+{
+  outputLayer = nullptr;
+  QgsVectorLayer *layer = nullptr;
+
+  QString destEncoding = encoding;
+  if ( destEncoding.isEmpty() )
+  {
+    // no destination encoding specified, use default
+    destEncoding = context.defaultEncoding().isEmpty() ? QStringLiteral( "system" ) : context.defaultEncoding();
+  }
+
+  if ( destination.isEmpty() || destination.startsWith( QStringLiteral( "memory:" ) ) )
+  {
+    // memory provider cannot be used with QgsVectorLayerImport - so create layer manually
+    layer = QgsMemoryProviderUtils::createMemoryLayer( destination, fields, geometryType, crs );
+    if ( layer && layer->isValid() )
+      destination = layer->id();
+  }
+  else
+  {
+    QMap<QString, QVariant> options;
+    options.insert( QStringLiteral( "fileEncoding" ), destEncoding );
+
+    QString providerKey;
+    QString uri;
+    QString format;
+    parseDestinationString( destination, providerKey, uri, format, options );
+
+    if ( providerKey == "ogr" )
+    {
+      // use QgsVectorFileWriter for OGR destinations instead of QgsVectorLayerImport, as that allows
+      // us to use any OGR format which supports feature addition
+      QString finalFileName;
+      QgsVectorFileWriter *writer = new QgsVectorFileWriter( destination, destEncoding, fields, geometryType, crs, format, QgsVectorFileWriter::defaultDatasetOptions( format ),
+          QgsVectorFileWriter::defaultLayerOptions( format ), &finalFileName );
+      destination = finalFileName;
+      return writer;
+    }
+    else
+    {
+      //create empty layer
+      {
+        QgsVectorLayerImport import( uri, providerKey, fields, geometryType, crs, false, &options );
+        if ( import.hasError() )
+          return nullptr;
+      }
+
+      layer = new QgsVectorLayer( uri, destination, providerKey );
+    }
+  }
+
+  if ( !layer )
+    return nullptr;
+
+  if ( !layer->isValid() )
+  {
+    delete layer;
+    return nullptr;
+  }
+
+  context.temporaryLayerStore()->addMapLayer( layer );
+
+  outputLayer = layer;
+  // this is a factory, so we need to return a proxy
+  return new QgsProxyFeatureSink( layer->dataProvider() );
+}
+
+void QgsProcessingUtils::createFeatureSinkPython( QgsFeatureSink **sink, QString &destination, const QString &encoding, const QgsFields &fields, QgsWkbTypes::Type geometryType, const QgsCoordinateReferenceSystem &crs, QgsProcessingContext &context, QgsVectorLayer **outputLayer )
+{
+  QgsVectorLayer *layer = nullptr;
+  *sink = createFeatureSink( destination, encoding, fields, geometryType, crs, context, layer );
+  if ( outputLayer )
+    *outputLayer = layer;
+}
+
 

src/core/processing/qgsprocessingutils.h

@@ -131,6 +131,58 @@ class CORE_EXPORT QgsProcessingUtils
      */
     static QList< QVariant > uniqueValues( QgsVectorLayer *layer, int fieldIndex, const QgsProcessingContext &context );
 
+    /**
+     * Creates a feature sink ready for adding features. The \a destination specifies a destination
+     * URI for the resultant layer. It may be updated in place to reflect the actual destination
+     * for the layer.
+     *
+     * Sink parameters such as desired \a encoding, \a fields, \a geometryType and \a crs must be specified.
+     *
+     * If the \a encoding is not specified, the default encoding from the \a context will be used.
+     *
+     * If a layer is created for the feature sink, the layer will automatically be added to the \a context's
+     * temporary layer store, and the \a outputLayer argument updated to point at this newly created layer.
+     *
+     * The caller takes responsibility for deleting the returned sink.
+     */
+#ifndef SIP_RUN
+    static QgsFeatureSink *createFeatureSink(
+      QString &destination,
+      const QString &encoding,
+      const QgsFields &fields,
+      QgsWkbTypes::Type geometryType,
+      const QgsCoordinateReferenceSystem &crs,
+      QgsProcessingContext &context,
+      QgsVectorLayer *&outputLayer ) SIP_FACTORY;
+#endif
+
+    /**
+     * Creates a feature sink ready for adding features. The \a destination specifies a destination
+     * URI for the resultant layer. It may be updated in place to reflect the actual destination
+     * for the layer.
+     *
+     * Sink parameters such as desired \a encoding, \a fields, \a geometryType and \a crs must be specified.
+     *
+     * If the \a encoding is not specified, the default encoding from the \a context will be used.
+     *
+     * If a layer is created for the feature sink, the layer will automatically be added to the \a context's
+     * temporary layer store, and the \a outputLayer argument updated to point at this newly created layer.
+     *
+     * \note this version of the createFeatureSink() function has an API designed around use from the
+     * SIP bindings. c++ code should call the other createFeatureSink() version.
+     * \note available in Python bindings as createFeatureSink()
+     */
+    static void createFeatureSinkPython(
+      QgsFeatureSink **sink SIP_OUT SIP_TRANSFERBACK,
+      QString &destination SIP_INOUT,
+      const QString &encoding,
+      const QgsFields &fields,
+      QgsWkbTypes::Type geometryType,
+      const QgsCoordinateReferenceSystem &crs,
+      QgsProcessingContext &context,
+      QgsVectorLayer **outputLayer SIP_OUT ) SIP_PYNAME( createFeatureSink );
+
+
   private:
 
     static bool canUseLayer( const QgsRasterLayer *layer );

tests/src/core/testqgsprocessing.cpp

@@ -28,6 +28,7 @@
 #include "qgsproject.h"
 #include "qgspointv2.h"
 #include "qgsgeometry.h"
+#include "qgsvectorfilewriter.h"
 
 class DummyAlgorithm : public QgsProcessingAlgorithm
 {
@@ -104,6 +105,7 @@ class TestQgsProcessing: public QObject
     void removeProvider();
     void compatibleLayers();
     void normalizeLayerSource();
+    void context();
     void mapLayers();
     void mapLayerFromStore();
     void mapLayerFromString();
@@ -111,6 +113,7 @@ class TestQgsProcessing: public QObject
     void features();
     void uniqueValues();
     void createIndex();
+    void createFeatureSink();
 
   private:
 
@@ -124,6 +127,9 @@ void TestQgsProcessing::initTestCase()
 
 void TestQgsProcessing::cleanupTestCase()
 {
+  QFile::remove( QDir::tempPath() + "/create_feature_sink.tab" );
+  QgsVectorFileWriter::deleteShapeFile( QDir::tempPath() + "/create_feature_sink2.shp" );
+
   QgsApplication::exitQgis();
 }
 
@@ -334,6 +340,27 @@ void TestQgsProcessing::normalizeLayerSource()
   QCOMPARE( QgsProcessingUtils::normalizeLayerSource( "data\\layers \"new\"\\test.shp" ), QString( "data/layers 'new'/test.shp" ) );
 }
 
+void TestQgsProcessing::context()
+{
+  QgsProcessingContext context;
+
+  // simple tests for getters/setters
+  context.setDefaultEncoding( "my_enc" );
+  QCOMPARE( context.defaultEncoding(), QStringLiteral( "my_enc" ) );
+
+  context.setFlags( QgsProcessingContext::UseSelectionIfPresent );
+  QCOMPARE( context.flags(), QgsProcessingContext::UseSelectionIfPresent );
+  context.setFlags( QgsProcessingContext::Flags( 0 ) );
+  QCOMPARE( context.flags(), QgsProcessingContext::Flags( 0 ) );
+
+  QgsProject p;
+  context.setProject( &p );
+  QCOMPARE( context.project(), &p );
+
+  context.setInvalidGeometryCheck( QgsFeatureRequest::GeometrySkipInvalid );
+  QCOMPARE( context.invalidGeometryCheck(), QgsFeatureRequest::GeometrySkipInvalid );
+}
+
 void TestQgsProcessing::mapLayers()
 {
   QString testDataDir = QStringLiteral( TEST_DATA_DIR ) + '/'; //defined in CmakeLists.txt
@@ -711,5 +738,114 @@ void TestQgsProcessing::createIndex()
 
 }
 
+void TestQgsProcessing::createFeatureSink()
+{
+  QgsProcessingContext context;
+
+  // empty destination
+  QString destination;
+  destination = QString();
+  QgsVectorLayer *layer = nullptr;
+
+  // should create a memory layer
+  QgsFeatureSink *sink = QgsProcessingUtils::createFeatureSink( destination, QString(), QgsFields(), QgsWkbTypes::Point, QgsCoordinateReferenceSystem(), context, layer );
+  QVERIFY( sink );
+  QVERIFY( layer );
+  QCOMPARE( static_cast< QgsProxyFeatureSink *>( sink )->destinationSink(), layer->dataProvider() );
+  QCOMPARE( layer->dataProvider()->name(), QStringLiteral( "memory" ) );
+  QCOMPARE( destination, layer->id() );
+  QCOMPARE( context.temporaryLayerStore()->mapLayer( layer->id() ), layer ); // layer should be in store
+  QgsFeature f;
+  QCOMPARE( layer->featureCount(), 0L );
+  QVERIFY( sink->addFeature( f ) );
+  QCOMPARE( layer->featureCount(), 1L );
+  context.temporaryLayerStore()->removeAllMapLayers();
+  layer = nullptr;
+  delete sink;
+
+  // specific memory layer output
+  destination = QStringLiteral( "memory:mylayer" );
+  sink = QgsProcessingUtils::createFeatureSink( destination, QString(), QgsFields(), QgsWkbTypes::Point, QgsCoordinateReferenceSystem(), context, layer );
+  QVERIFY( sink );
+  QVERIFY( layer );
+  QCOMPARE( static_cast< QgsProxyFeatureSink *>( sink )->destinationSink(), layer->dataProvider() );
+  QCOMPARE( layer->dataProvider()->name(), QStringLiteral( "memory" ) );
+  QCOMPARE( layer->name(), QStringLiteral( "memory:mylayer" ) );
+  QCOMPARE( destination, layer->id() );
+  QCOMPARE( context.temporaryLayerStore()->mapLayer( layer->id() ), layer ); // layer should be in store
+  QCOMPARE( layer->featureCount(), 0L );
+  QVERIFY( sink->addFeature( f ) );
+  QCOMPARE( layer->featureCount(), 1L );
+  context.temporaryLayerStore()->removeAllMapLayers();
+  layer = nullptr;
+  delete sink;
+
+  // memory layer parameters
+  destination = QStringLiteral( "memory:mylayer" );
+  QgsFields fields;
+  fields.append( QgsField( QStringLiteral( "my_field" ), QVariant::String, QString(), 100 ) );
+  sink = QgsProcessingUtils::createFeatureSink( destination, QString(), fields, QgsWkbTypes::PointZM, QgsCoordinateReferenceSystem::fromEpsgId( 3111 ), context, layer );
+  QVERIFY( sink );
+  QVERIFY( layer );
+  QCOMPARE( static_cast< QgsProxyFeatureSink *>( sink )->destinationSink(), layer->dataProvider() );
+  QCOMPARE( layer->dataProvider()->name(), QStringLiteral( "memory" ) );
+  QCOMPARE( layer->name(), QStringLiteral( "memory:mylayer" ) );
+  QCOMPARE( layer->wkbType(), QgsWkbTypes::PointZM );
+  QCOMPARE( layer->crs().authid(), QStringLiteral( "EPSG:3111" ) );
+  QCOMPARE( layer->fields().size(), 1 );
+  QCOMPARE( layer->fields().at( 0 ).name(), QStringLiteral( "my_field" ) );
+  QCOMPARE( layer->fields().at( 0 ).type(), QVariant::String );
+  QCOMPARE( destination, layer->id() );
+  QCOMPARE( context.temporaryLayerStore()->mapLayer( layer->id() ), layer ); // layer should be in store
+  QCOMPARE( layer->featureCount(), 0L );
+  QVERIFY( sink->addFeature( f ) );
+  QCOMPARE( layer->featureCount(), 1L );
+  context.temporaryLayerStore()->removeAllMapLayers();
+  layer = nullptr;
+  delete sink;
+
+  // non memory layer output
+  destination = QDir::tempPath() + "/create_feature_sink.tab";
+  QString prevDest = destination;
+  sink = QgsProcessingUtils::createFeatureSink( destination, QString(), fields, QgsWkbTypes::Polygon, QgsCoordinateReferenceSystem::fromEpsgId( 3111 ), context, layer );
+  QVERIFY( sink );
+  f = QgsFeature( fields );
+  f.setGeometry( QgsGeometry::fromWkt( QStringLiteral( "Polygon((0 0, 0 1, 1 1, 1 0, 0 0 ))" ) ) );
+  f.setAttributes( QgsAttributes() << "val" );
+  QVERIFY( sink->addFeature( f ) );
+  QVERIFY( !layer );
+  QCOMPARE( destination, prevDest );
+  delete sink;
+  layer = new QgsVectorLayer( destination, "test_layer", "ogr" );
+  QVERIFY( layer->isValid() );
+  QCOMPARE( layer->crs().authid(), QStringLiteral( "EPSG:3111" ) );
+  QCOMPARE( layer->fields().size(), 1 );
+  QCOMPARE( layer->fields().at( 0 ).name(), QStringLiteral( "my_field" ) );
+  QCOMPARE( layer->fields().at( 0 ).type(), QVariant::String );
+  QCOMPARE( layer->featureCount(), 1L );
+  delete layer;
+  layer = nullptr;
+
+  // no extension, should default to shp
+  destination = QDir::tempPath() + "/create_feature_sink2";
+  prevDest = QDir::tempPath() + "/create_feature_sink2.shp";
+  sink = QgsProcessingUtils::createFeatureSink( destination, QString(), fields, QgsWkbTypes::Point25D, QgsCoordinateReferenceSystem::fromEpsgId( 3111 ), context, layer );
+  QVERIFY( sink );
+  f.setGeometry( QgsGeometry::fromWkt( QStringLiteral( "PointZ(1 2 3)" ) ) );
+  QVERIFY( sink->addFeature( f ) );
+  QVERIFY( !layer );
+  QCOMPARE( destination, prevDest );
+  delete sink;
+  layer = new QgsVectorLayer( destination, "test_layer", "ogr" );
+  QCOMPARE( layer->wkbType(), QgsWkbTypes::Point25D );
+  QCOMPARE( layer->crs().authid(), QStringLiteral( "EPSG:3111" ) );
+  QCOMPARE( layer->fields().size(), 1 );
+  QCOMPARE( layer->fields().at( 0 ).name(), QStringLiteral( "my_field" ) );
+  QCOMPARE( layer->fields().at( 0 ).type(), QVariant::String );
+  QCOMPARE( layer->featureCount(), 1L );
+  delete layer;
+  layer = nullptr;
+}
+
 QGSTEST_MAIN( TestQgsProcessing )
 #include "testqgsprocessing.moc"