Cache unique values when creating features

Fixes #21305 - pasting features is very slow

Aggressively optimize createFeature for speed
and introduces createFeatures for bulk creation.

python/core/auto_generated/qgsvectorlayerutils.sip.in

@@ -57,6 +57,40 @@ Returns the duplicated features in the given layer
 
     };
 
+    class QgsFeaturesData
+{
+%Docstring
+Encapsulate geometry and attributes for new features, to be passed to createFeatures
+
+.. seealso:: :py:func:`createFeatures`
+
+.. versionadded:: 3.6
+%End
+
+%TypeHeaderCode
+#include "qgsvectorlayerutils.h"
+%End
+      public:
+
+        QgsFeaturesData( const QgsGeometry &geometry = QgsGeometry(), const QgsAttributeMap &attributes = QgsAttributeMap() );
+%Docstring
+Constructs a new QgsFeaturesData with given ``geometry`` and ``attributes``
+%End
+
+        QgsGeometry geometry() const;
+%Docstring
+Returns geometry
+%End
+
+        QgsAttributeMap attributes() const;
+%Docstring
+Returns attributes
+%End
+
+    };
+
+    typedef QList<QgsVectorLayerUtils::QgsFeaturesData> QgsFeaturesDataList;
+
     static QgsFeatureIterator getValuesIterator( const QgsVectorLayer *layer, const QString &fieldOrExpression, bool &ok, bool selectedOnly );
 %Docstring
 Create a feature iterator for a specified field name or expression.
@@ -143,6 +177,17 @@ Creates a new feature ready for insertion into a layer. Default values and const
 passed for the new feature to copy as many attribute values as possible from the map,
 assuming that they respect the layer's constraints. Note that the created feature is not
 automatically inserted into the layer.
+%End
+
+    static QgsFeatureList createFeatures( const QgsVectorLayer *layer,
+                                          const QgsFeaturesDataList &featuresData,
+                                          QgsExpressionContext *context = 0 );
+%Docstring
+Creates a set of new features ready for insertion into a layer. Default values and constraints
+(e.g., unique constraints) will automatically be handled. An optional attribute map can be
+passed for the new feature to copy as many attribute values as possible from the map,
+assuming that they respect the layer's constraints. Note that the created features are not
+automatically inserted into the layer.
 %End
 
     static QgsFeature duplicateFeature( QgsVectorLayer *layer, const QgsFeature &feature, QgsProject *project, int depth, QgsDuplicateFeatureContext &duplicateFeatureContext /Out/ );

src/app/qgisapp.cpp

@@ -8996,11 +8996,12 @@ void QgisApp::pasteFromClipboard( QgsMapLayer *destinationLayer )
   QgsExpressionContext context = pasteVectorLayer->createExpressionContext();
 
   QgsFeatureList compatibleFeatures( QgsVectorLayerUtils::makeFeaturesCompatible( features, pasteVectorLayer ) );
-  QgsFeatureList newFeatures;
+  QgsVectorLayerUtils::QgsFeaturesDataList newFeaturesDataList;
+  newFeaturesDataList.reserve( compatibleFeatures.size() );
+
   // Count collapsed geometries
   int invalidGeometriesCount = 0;
 
-  newFeatures.reserve( compatibleFeatures.size() );
   for ( const auto &feature : qgis::as_const( compatibleFeatures ) )
   {
 
@@ -9022,8 +9023,10 @@ void QgisApp::pasteFromClipboard( QgsMapLayer *destinationLayer )
     }
     // now create new feature using pasted feature as a template. This automatically handles default
     // values and field constraints
-    newFeatures << QgsVectorLayerUtils::createFeature( pasteVectorLayer, geom, attrMap, &context );
+    newFeaturesDataList << QgsVectorLayerUtils::QgsFeaturesData( geom, attrMap );
   }
+
+  QgsFeatureList newFeatures {QgsVectorLayerUtils::createFeatures( pasteVectorLayer, newFeaturesDataList, &context )};
   pasteVectorLayer->addFeatures( newFeatures );
   QgsFeatureIds newIds;
   newIds.reserve( newFeatures.size() );
@@ -9032,7 +9035,6 @@ void QgisApp::pasteFromClipboard( QgsMapLayer *destinationLayer )
     newIds << f.id();
   }
 
-
   pasteVectorLayer->selectByIds( newIds );
   pasteVectorLayer->endEditCommand();
   pasteVectorLayer->updateExtents();

src/core/qgsvectorlayerutils.cpp

@@ -358,11 +358,17 @@ bool QgsVectorLayerUtils::validateAttribute( const QgsVectorLayer *layer, const
 
 QgsFeature QgsVectorLayerUtils::createFeature( const QgsVectorLayer *layer, const QgsGeometry &geometry,
     const QgsAttributeMap &attributes, QgsExpressionContext *context )
+{
+  return createFeatures( layer, QgsFeaturesDataList() << QgsFeaturesData( geometry, attributes ), context ).first();
+}
+
+QgsFeatureList QgsVectorLayerUtils::createFeatures( const QgsVectorLayer *layer, const QgsFeaturesDataList &featuresData, QgsExpressionContext *context )
 {
   if ( !layer )
-  {
-    return QgsFeature();
-  }
+    return QgsFeatureList();
+
+  QgsFeatureList result;
+  result.reserve( featuresData.length() );
 
   QgsExpressionContext *evalContext = context;
   std::unique_ptr< QgsExpressionContext > tempContext;
@@ -375,94 +381,104 @@ QgsFeature QgsVectorLayerUtils::createFeature( const QgsVectorLayer *layer, cons
 
   QgsFields fields = layer->fields();
 
-  QgsFeature newFeature( fields );
-  newFeature.setValid( true );
-  newFeature.setGeometry( geometry );
+  // Cache unique values
+  QMap<int, QSet<QVariant>> uniqueValueCaches;
 
-  // initialize attributes
-  newFeature.initAttributes( fields.count() );
-  for ( int idx = 0; idx < fields.count(); ++idx )
+  for ( const auto &fd : qgis::as_const( featuresData ) )
   {
-    QVariant v;
-    bool checkUnique = true;
 
-    // in order of priority:
-    // 1. passed attribute value and if field does not have a unique constraint like primary key
-    if ( attributes.contains( idx ) )
+    QgsFeature newFeature( fields );
+    newFeature.setValid( true );
+    newFeature.setGeometry( fd.geometry() );
+
+    // initialize attributes
+    newFeature.initAttributes( fields.count() );
+    for ( int idx = 0; idx < fields.count(); ++idx )
     {
-      v = attributes.value( idx );
-    }
+      QVariant v;
+      bool checkUnique = true;
+      const bool hasUniqueConstraint { static_cast<bool>( fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique ) };
 
-    // Cache unique values
-    QSet<QVariant> uniqueValues { layer->uniqueValues( idx ) };
-
-    // 2. client side default expression
-    // note - deliberately not using else if!
-    if ( ( !v.isValid() || ( fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique
-                             && uniqueValues.contains( v ) ) )
-         && layer->defaultValueDefinition( idx ).isValid() )
-    {
-      // client side default expression set - takes precedence over all. Why? Well, this is the only default
-      // which QGIS users have control over, so we assume that they're deliberately overriding any
-      // provider defaults for some good reason and we should respect that
-      v = layer->defaultValue( idx, newFeature, evalContext );
-    }
-
-    // 3. provider side default value clause
-    // note - not an else if deliberately. Users may return null from a default value expression to fallback to provider defaults
-    if ( ( !v.isValid() || ( fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique
-                             && uniqueValues.contains( v ) ) )
-         && fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
-    {
-      int providerIndex = fields.fieldOriginIndex( idx );
-      QString providerDefault = layer->dataProvider()->defaultValueClause( providerIndex );
-      if ( !providerDefault.isEmpty() )
+      // in order of priority:
+      // 1. passed attribute value and if field does not have a unique constraint like primary key
+      if ( fd.attributes().contains( idx ) )
       {
-        v = providerDefault;
-        checkUnique = false;
+        v = fd.attributes().value( idx );
       }
-    }
 
-    // 4. provider side default literal
-    // note - deliberately not using else if!
-    if ( ( !v.isValid() || ( checkUnique && fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique
-                             && uniqueValues.contains( v ) ) )
-         && fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
-    {
-      int providerIndex = fields.fieldOriginIndex( idx );
-      v = layer->dataProvider()->defaultValue( providerIndex );
-      if ( v.isValid() )
+      // Cache unique values
+      if ( hasUniqueConstraint && ! uniqueValueCaches.contains( idx ) )
+        uniqueValueCaches[ idx ] = layer->uniqueValues( idx );
+
+      // 2. client side default expression
+      // note - deliberately not using else if!
+      if ( ( !v.isValid() || ( hasUniqueConstraint
+                               && uniqueValueCaches[ idx ].contains( v ) ) )
+           && layer->defaultValueDefinition( idx ).isValid() )
       {
-        //trust that the provider default has been sensibly set not to violate any constraints
-        checkUnique = false;
+        // client side default expression set - takes precedence over all. Why? Well, this is the only default
+        // which QGIS users have control over, so we assume that they're deliberately overriding any
+        // provider defaults for some good reason and we should respect that
+        v = layer->defaultValue( idx, newFeature, evalContext );
       }
-    }
 
-    // 5. passed attribute value
-    // note - deliberately not using else if!
-    if ( !v.isValid() && attributes.contains( idx ) )
-    {
-      v = attributes.value( idx );
-    }
-
-    // last of all... check that unique constraints are respected
-    // we can't handle not null or expression constraints here, since there's no way to pick a sensible
-    // value if the constraint is violated
-    if ( checkUnique && fields.at( idx ).constraints().constraints() & QgsFieldConstraints::ConstraintUnique )
-    {
-      if ( uniqueValues.contains( v ) )
+      // 3. provider side default value clause
+      // note - not an else if deliberately. Users may return null from a default value expression to fallback to provider defaults
+      if ( ( !v.isValid() || ( hasUniqueConstraint
+                               && uniqueValueCaches[ idx ].contains( v ) ) )
+           && fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
       {
-        // unique constraint violated
-        QVariant uniqueValue = QgsVectorLayerUtils::createUniqueValue( layer, idx, v );
-        if ( uniqueValue.isValid() )
-          v = uniqueValue;
+        int providerIndex = fields.fieldOriginIndex( idx );
+        QString providerDefault = layer->dataProvider()->defaultValueClause( providerIndex );
+        if ( !providerDefault.isEmpty() )
+        {
+          v = providerDefault;
+          checkUnique = false;
+        }
       }
-    }
 
-    newFeature.setAttribute( idx, v );
+      // 4. provider side default literal
+      // note - deliberately not using else if!
+      if ( ( !v.isValid() || ( checkUnique && hasUniqueConstraint
+                               && uniqueValueCaches[ idx ].contains( v ) ) )
+           && fields.fieldOrigin( idx ) == QgsFields::OriginProvider )
+      {
+        int providerIndex = fields.fieldOriginIndex( idx );
+        v = layer->dataProvider()->defaultValue( providerIndex );
+        if ( v.isValid() )
+        {
+          //trust that the provider default has been sensibly set not to violate any constraints
+          checkUnique = false;
+        }
+      }
+
+      // 5. passed attribute value
+      // note - deliberately not using else if!
+      if ( !v.isValid() && fd.attributes().contains( idx ) )
+      {
+        v = fd.attributes().value( idx );
+      }
+
+      // last of all... check that unique constraints are respected
+      // we can't handle not null or expression constraints here, since there's no way to pick a sensible
+      // value if the constraint is violated
+      if ( checkUnique && hasUniqueConstraint )
+      {
+        if ( uniqueValueCaches[ idx ].contains( v ) )
+        {
+          // unique constraint violated
+          QVariant uniqueValue = QgsVectorLayerUtils::createUniqueValue( layer, idx, v );
+          if ( uniqueValue.isValid() )
+            v = uniqueValue;
+        }
+      }
+      if ( hasUniqueConstraint )
+        uniqueValueCaches[ idx ].insert( v );
+      newFeature.setAttribute( idx, v );
+    }
+    result.append( newFeature );
   }
-
-  return newFeature;
+  return result;
 }
 
 QgsFeature QgsVectorLayerUtils::duplicateFeature( QgsVectorLayer *layer, const QgsFeature &feature, QgsProject *project, int depth, QgsDuplicateFeatureContext &duplicateFeatureContext )
@@ -772,3 +788,18 @@ QMap<QgsVectorLayer *, QgsFeatureIds>  QgsVectorLayerUtils::QgsDuplicateFeatureC
   return mDuplicatedFeatures;
 }
 */
+
+QgsVectorLayerUtils::QgsFeaturesData::QgsFeaturesData( const QgsGeometry &geometry, const QgsAttributeMap &attributes ):
+  mGeometry( geometry ),
+  mAttributes( attributes )
+{}
+
+QgsGeometry QgsVectorLayerUtils::QgsFeaturesData::geometry() const
+{
+  return mGeometry;
+}
+
+QgsAttributeMap QgsVectorLayerUtils::QgsFeaturesData::attributes() const
+{
+  return mAttributes;
+}

src/core/qgsvectorlayerutils.h

@@ -70,6 +70,37 @@ class CORE_EXPORT QgsVectorLayerUtils
         void setDuplicatedFeatures( QgsVectorLayer *layer, const QgsFeatureIds &ids );
     };
 
+    /**
+     * \ingroup core
+     * \class QgsFeatureSetData
+     * \brief Encapsulate geometry and attributes for new features, to be passed to createFeatures
+     * \see createFeatures()
+     * \since QGIS 3.6
+     */
+    class CORE_EXPORT QgsFeaturesData
+    {
+      public:
+
+        /**
+         * Constructs a new QgsFeaturesData with given \a geometry and \a attributes
+         */
+        QgsFeaturesData( const QgsGeometry &geometry = QgsGeometry(), const QgsAttributeMap &attributes = QgsAttributeMap() );
+
+        //! Returns geometry
+        QgsGeometry geometry() const;
+
+        //! Returns attributes
+        QgsAttributeMap attributes() const;
+
+      private:
+        QgsGeometry mGeometry;
+        QgsAttributeMap mAttributes;
+    };
+
+    // SIP does not lile "using", use legacy typedef
+    //! Alias for list of QgsFeaturesData
+    typedef QList<QgsVectorLayerUtils::QgsFeaturesData> QgsFeaturesDataList;
+
     /**
      * Create a feature iterator for a specified field name or expression.
      * \param layer vector layer to retrieve values from
@@ -145,6 +176,17 @@ class CORE_EXPORT QgsVectorLayerUtils
                                      const QgsAttributeMap &attributes = QgsAttributeMap(),
                                      QgsExpressionContext *context = nullptr );
 
+    /**
+     * Creates a set of new features ready for insertion into a layer. Default values and constraints
+     * (e.g., unique constraints) will automatically be handled. An optional attribute map can be
+     * passed for the new feature to copy as many attribute values as possible from the map,
+     * assuming that they respect the layer's constraints. Note that the created features are not
+     * automatically inserted into the layer.
+     */
+    static QgsFeatureList createFeatures( const QgsVectorLayer *layer,
+                                          const QgsFeaturesDataList &featuresData,
+                                          QgsExpressionContext *context = nullptr );
+
     /**
      * Duplicates a feature and it's children (one level deep). It calls CreateFeature, so
      * default values and constraints (e.g., unique constraints) will automatically be handled.

tests/src/python/test_provider_postgres.py

@@ -1183,6 +1183,40 @@ class TestPyQgsPostgresProvider(unittest.TestCase, ProviderTestCase):
         self.assertEqual(g.childCount(), 1)
         self.assertTrue(g.childGeometry(0).vertexCount() > 3)
 
+    def testMassivePaste(self):
+        """Speed test to compare createFeature and createFeatures, for regression #21303"""
+
+        import time
+
+        self.execSQLCommand('CREATE TABLE IF NOT EXISTS massive_paste(pk SERIAL NOT NULL PRIMARY KEY, geom public.geometry(Polygon, 4326))')
+        self.execSQLCommand('TRUNCATE massive_paste')
+
+        start_time = time.time()
+        vl = QgsVectorLayer(self.dbconn + ' sslmode=disable key=\'pk\' srid=4326 type=POLYGON table="massive_paste" (geom) sql=', 'test_massive_paste', 'postgres')
+        self.assertTrue(vl.startEditing())
+        features = []
+        context = vl.createExpressionContext()
+        for i in range(4000):
+            features.append(QgsVectorLayerUtils.createFeature(vl, QgsGeometry.fromWkt('Polygon ((7 44, 8 45, 8 46, 7 46, 7 44))'), {0: i}, context))
+        self.assertTrue(vl.addFeatures(features))
+        self.assertTrue(vl.commitChanges())
+        self.assertEqual(vl.featureCount(), 4000)
+        print("--- %s seconds ---" % (time.time() - start_time))
+
+        self.execSQLCommand('TRUNCATE massive_paste')
+        start_time = time.time()
+        vl = QgsVectorLayer(self.dbconn + ' sslmode=disable key=\'pk\' srid=4326 type=POLYGON table="massive_paste" (geom) sql=', 'test_massive_paste', 'postgres')
+        self.assertTrue(vl.startEditing())
+        features_data = []
+        context = vl.createExpressionContext()
+        for i in range(4000):
+            features_data.append(QgsVectorLayerUtils.QgsFeaturesData(QgsGeometry.fromWkt('Polygon ((7 44, 8 45, 8 46, 7 46, 7 44))'), {0: i}))
+        features = QgsVectorLayerUtils.createFeatures(vl, features_data, context)
+        self.assertTrue(vl.addFeatures(features))
+        self.assertTrue(vl.commitChanges())
+        self.assertEqual(vl.featureCount(), 4000)
+        print("--- %s seconds ---" % (time.time() - start_time))
+
 
 class TestPyQgsPostgresProviderCompoundKey(unittest.TestCase, ProviderTestCase):