Merge pull request #8722 from m-kuhn/expression_function_sqlite_fetch_and_increment

Expression function sqlite_fetch_and_increment

resources/function_help/json/sqlite_fetch_and_increment

@@ -0,0 +1,16 @@
+{
+  "name": "sqlite_fetch_and_increment",
+  "type": "function",
+  "description": "Manage autoincrementing values in sqlite databases.<p>SQlite default values can only be applied on insert and not prefetched.</p><p>This makes it impossible to acquire an incremented primary key via AUTO_INCREMENT before creating the row in the database. Sidenote: with postgres, this works via the option <i>evaluate default values</i>.</p><p>When adding new features with relations, it is really nice to be able to already add children for a parent, while the parents form is still open and hence the parent feature uncommitted.</p><p>To get around this limitation, this function can be used to manage sequence values in a separate table on sqlite based formats like gpkg.</p><p>The sequence table will be filtered for a sequence id (filter_attribute and filter_value) and the current value of the id_field will be incremented by 1 ond the incremented value returned.</p><p>If additional columns require values to be specified, the default_value map can be used for this purpose.</p><p><b>Note</b><br/>This function modifies the target sqlite table. It is intended for usage with default value configurations for attributes.</p>",
+  "arguments": [
+	  {"arg":"database", "description":"Path to the sqlite file"},
+    {"arg":"table", "description":"Name of the table that manages the sequences"},
+    {"arg":"id_field", "description":"Name of the field that contains the current value"},
+    {"arg":"filter_attribute", "description":"Name the field that contains a unique identifier for this sequence. Must have a UNIQUE index."},
+    {"arg":"filter_value", "description":"Name of the sequence to use."},
+    {"arg":"default_values", "description":"Map with default values for additional columns on the table. The values need to be fully quoted. Functions are allowed.", "optional": true}
+  ],
+  "examples": [
+    { "expression":"sqlite_fetch_and_increment(layer_property(@layer, 'path'), 'sequence_table', 'last_unique_id', 'sequence_id', 'global', map('last_change','date(''now'')','user','''' || @user_account_name || ''''))", "returns":"0"}
+  ]
+}

src/core/expression/qgsexpressionfunction.cpp

@@ -47,6 +47,7 @@
 #include "qgsfieldformatter.h"
 #include "qgsvectorlayerfeatureiterator.h"
 #include "qgsproviderregistry.h"
+#include "sqlite3.h"
 
 const QString QgsExpressionFunction::helpText() const
 {
@@ -1360,6 +1361,82 @@ static QVariant fcnNumSelected( const QVariantList &values, const QgsExpressionC
   return layer->selectedFeatureCount();
 }
 
+static QVariant fcnSqliteFetchAndIncrement( const QVariantList &values, const QgsExpressionContext *, QgsExpression *parent, const QgsExpressionNodeFunction * )
+{
+  const QString database = values.at( 0 ).toString();
+  const QString table = values.at( 1 ).toString();
+  const QString idColumn = values.at( 2 ).toString();
+  const QString filterAttribute = values.at( 3 ).toString();
+  const QVariant filterValue = values.at( 4 ).toString();
+  const QVariantMap defaultValues = values.at( 5 ).toMap();
+
+
+  // read from database
+  sqlite3_database_unique_ptr sqliteDb;
+  sqlite3_statement_unique_ptr sqliteStatement;
+
+  if ( sqliteDb.open_v2( database, SQLITE_OPEN_READWRITE, nullptr ) != SQLITE_OK )
+  {
+    parent->setEvalErrorString( QObject::tr( "Could not open sqlite database %1. Error %2. " ).arg( database, sqliteDb.errorMessage() ) );
+    return QVariant();
+  }
+
+  QString currentValSql;
+  currentValSql = QStringLiteral( "SELECT %1 FROM %2" ).arg( QgsSqliteUtils::quotedIdentifier( idColumn ), QgsSqliteUtils::quotedIdentifier( table ) );
+  if ( !filterAttribute.isNull() )
+  {
+    currentValSql += QStringLiteral( " WHERE %1 = %2" ).arg( QgsSqliteUtils::quotedIdentifier( filterAttribute ), QgsSqliteUtils::quotedValue( filterValue ) );
+  }
+
+  int result;
+  sqliteStatement = sqliteDb.prepare( currentValSql, result );
+  if ( result == SQLITE_OK )
+  {
+    qlonglong nextId = 0;
+    if ( sqliteStatement.step() == SQLITE_ROW )
+    {
+      nextId = sqliteStatement.columnAsInt64( 0 ) + 1;
+    }
+
+    QString upsertSql;
+    upsertSql = QStringLiteral( "INSERT OR REPLACE INTO %1" ).arg( QgsSqliteUtils::quotedIdentifier( table ) );
+    QStringList cols;
+    QStringList vals;
+    cols << QgsSqliteUtils::quotedIdentifier( idColumn );
+    vals << QgsSqliteUtils::quotedValue( nextId );
+
+    if ( !filterAttribute.isNull() )
+    {
+      cols << QgsSqliteUtils::quotedIdentifier( filterAttribute );
+      vals << QgsSqliteUtils::quotedValue( filterValue );
+    }
+
+    for ( QVariantMap::const_iterator iter = defaultValues.constBegin(); iter != defaultValues.constEnd(); ++iter )
+    {
+      cols << QgsSqliteUtils::quotedIdentifier( iter.key() );
+      vals << iter.value().toString();
+    }
+
+    upsertSql += QLatin1String( " (" ) + cols.join( ',' ) + ')';
+    upsertSql += QLatin1String( " VALUES " );
+    upsertSql += '(' + vals.join( ',' ) + ')';
+
+    QString errorMessage;
+    result = sqliteDb.exec( upsertSql, errorMessage );
+    if ( result == SQLITE_OK )
+    {
+      return nextId;
+    }
+    else
+    {
+      parent->setEvalErrorString( QStringLiteral( "Could not increment value: SQLite error: \"%1\" (%2)." ).arg( errorMessage, QString::number( result ) ) );
+      return QVariant();
+    }
+  }
+
+  return QVariant(); // really?
+}
+
 static QVariant fcnConcat( const QVariantList &values, const QgsExpressionContext *, QgsExpression *parent, const QgsExpressionNodeFunction * )
 {
   QString concat;
@@ -4916,6 +4993,20 @@ const QList<QgsExpressionFunction *> &QgsExpression::Functions()
           QSet<QString>()
         );
 
+    sFunctions
+        << new QgsStaticExpressionFunction(
+          QStringLiteral( "sqlite_fetch_and_increment" ),
+          QgsExpressionFunction::ParameterList()
+          << QgsExpressionFunction::Parameter( QStringLiteral( "database" ) )
+          << QgsExpressionFunction::Parameter( QStringLiteral( "table" ) )
+          << QgsExpressionFunction::Parameter( QStringLiteral( "id_field" ) )
+          << QgsExpressionFunction::Parameter( QStringLiteral( "filter_attribute" ) )
+          << QgsExpressionFunction::Parameter( QStringLiteral( "filter_value" ) )
+          << QgsExpressionFunction::Parameter( QStringLiteral( "default_values" ), true ),
+          fcnSqliteFetchAndIncrement,
+          QStringLiteral( "Record and Attributes" )
+        );
+
     // **Fields and Values** functions
     QgsStaticExpressionFunction *representValueFunc = new QgsStaticExpressionFunction( QStringLiteral( "represent_value" ), QgsExpressionFunction::ParameterList() << QgsExpressionFunction::Parameter( QStringLiteral( "attribute" ) ) << QgsExpressionFunction::Parameter( QStringLiteral( "field_name" ), true ), fcnRepresentValue, QStringLiteral( "Record and Attributes" ) );
 

src/core/qgssqliteutils.cpp

@@ -92,6 +92,21 @@ sqlite3_statement_unique_ptr sqlite3_database_unique_ptr::prepare( const QString
   return s;
 }
 
+int sqlite3_database_unique_ptr::exec( const QString &sql, QString &errorMessage ) const
+{
+  char *errMsg;
+
+  int ret = sqlite3_exec( get(), sql.toUtf8(), nullptr, nullptr, &errMsg );
+
+  if ( errMsg )
+  {
+    errorMessage = QString::fromUtf8( errMsg );
+    sqlite3_free( errMsg );
+  }
+
+  return ret;
+}
+
 QString QgsSqliteUtils::quotedString( const QString &value )
 {
   if ( value.isNull() )

src/core/qgssqliteutils.h

@@ -21,6 +21,8 @@
 #define SIP_NO_FILE
 
 #include "qgis_core.h"
+#include "qgis_sip.h"
+
 #include <memory>
 #include <QString>
 
@@ -135,8 +137,17 @@ class CORE_EXPORT sqlite3_database_unique_ptr : public std::unique_ptr< sqlite3,
      * Prepares a \a sql statement, returning the result. The \a resultCode
      * argument will be filled with the sqlite3 result code.
      */
-    sqlite3_statement_unique_ptr prepare( const QString &sql, int &resultCode ) const;
+    sqlite3_statement_unique_ptr prepare( const QString &sql, int &resultCode SIP_OUT ) const;
 
+    /**
+     * Executes the \a sql command in the database. Multiple sql queries can be run within
+     * one single command.
+     * Errors are reported to \a errorMessage.
+     * Returns SQLITE_OK in case of success or an sqlite error code.
+     *
+     * \since QGIS 3.6
+     */
+    int exec( const QString &sql, QString &errorMessage SIP_OUT ) const;
 };
 
 /**

tests/src/core/testqgsexpression.cpp

@@ -1556,6 +1556,47 @@ class TestQgsExpression: public QObject
       }
     }
 
+    void test_sqliteFetchAndIncrement()
+    {
+      QTemporaryDir dir;
+      QString testGpkgName = QStringLiteral( "humanbeings.gpkg" );
+      QFile::copy( QStringLiteral( TEST_DATA_DIR ) + '/' + testGpkgName, dir.filePath( testGpkgName ) );
+
+      QgsExpressionContext context;
+      QgsExpressionContextScope *scope = new QgsExpressionContextScope();
+      scope->setVariable( QStringLiteral( "test_database" ), dir.filePath( testGpkgName ) );
+      scope->setVariable( QStringLiteral( "username" ), "some_username" );
+      context << scope;
+
+      // Test database file does not exist
+      QgsExpression exp1( QStringLiteral( "sqlite_fetch_and_increment('/path/does/not/exist', 'T_KEY_OBJECT', 'T_LastUniqueId', 'T_Key', 'T_Id')" ) );
+
+      exp1.evaluate( &context );
+      QCOMPARE( exp1.hasEvalError(), true );
+      const QString evalErrorString1 = exp1.evalErrorString();
+      QVERIFY2( evalErrorString1.contains( "/path/does/not/exist" ), QStringLiteral( "Path not found in %1" ).arg( evalErrorString1 ).toUtf8().constData() );
+      QVERIFY2( evalErrorString1.contains( "Error" ), QStringLiteral( "\"Error\" not found in %1" ).arg( evalErrorString1 ).toUtf8().constData() );
+
+      // Test default values are not properly quoted
+      QgsExpression exp2( QStringLiteral( "sqlite_fetch_and_increment(@test_database, 'T_KEY_OBJECT', 'T_LastUniqueId', 'T_Key', 'T_Id', map('T_LastChange','date(''now'')','T_CreateDate','date(''now'')','T_User', @username))" ) );
+      exp2.evaluate( &context );
+      QCOMPARE( exp2.hasEvalError(), true );
+      const QString evalErrorString2 = exp2.evalErrorString();
+      QVERIFY2( evalErrorString2.contains( "some_username" ), QStringLiteral( "'some_username' not found in '%1'" ).arg( evalErrorString2 ).toUtf8().constData() );
+
+      // Test incrementation logic
+      QgsExpression exp( QStringLiteral( "sqlite_fetch_and_increment(@test_database, 'T_KEY_OBJECT', 'T_LastUniqueId', 'T_Key', 'T_Id', map('T_LastChange','date(''now'')','T_CreateDate','date(''now'')','T_User','''me'''))" ) );
+      QVariant res = exp.evaluate( &context );
+      QCOMPARE( res.toInt(), 0 );
+
+      res = exp.evaluate( &context );
+      if ( exp.hasEvalError() )
+        qDebug() << exp.evalErrorString();
+      QCOMPARE( exp.hasEvalError(), false );
+
+      QCOMPARE( res.toInt(), 1 );
+    }
+
     void aggregate_data()
     {
       QTest::addColumn<QString>( "string" );