Merge pull request #40726 from elpaso/connections-api-results-iterator

Connections API: execSql iterator

python/core/auto_generated/qgsabstractdatabaseproviderconnection.sip.in

@@ -49,7 +49,7 @@ is not supported or cannot be performed without errors.
     {
         SIP_PYOBJECT __repr__();
 %MethodCode
-        QString str = QStringLiteral( "<QgsAbstractDatabaseProviderConnection.QueryResult: %1 rows>" ).arg( sipCpp->rows().size() );
+        QString str = QStringLiteral( "<QgsAbstractDatabaseProviderConnection.QueryResult: %1 rows>" ).arg( sipCpp->rowCount() );
         sipRes = PyUnicode_FromString( str.toUtf8().constData() );
 %End
 
@@ -63,16 +63,54 @@ Returns the column names
 Returns the results rows
 %End
 
-        void appendColumn( const QString &columnName );
+        qlonglong rowCount() const;
 %Docstring
-Appends ``columnName`` to the list of column names.
+Returns the row count
+
+.. note::
+
+   the value may not be exact or it may be -1 if not known
 %End
 
-        void appendRow( const QList<QVariant> &row );
+        bool hasNextRow() const;
 %Docstring
-Appends ``row`` to the results.
+Returns ``True`` if there are more rows to fetch
 %End
 
+        QList<QVariant> nextRow();
+%Docstring
+Returns the next result row or an empty row if there are no rows left
+%End
+
+        QueryResult *__iter__();
+%MethodCode
+        sipRes = sipCpp;
+%End
+
+        SIP_PYOBJECT __next__();
+%MethodCode
+        QList<QVariant> result;
+        Py_BEGIN_ALLOW_THREADS
+        result = sipCpp->nextRow( );
+        Py_END_ALLOW_THREADS
+        if ( ! result.isEmpty() )
+        {
+          const sipTypeDef *qvariantlist_type = sipFindType( "QList<QVariant>" );
+          sipRes = sipConvertFromNewType( new QList<QVariant>( result ), qvariantlist_type, Py_None );
+        }
+        else
+        {
+          PyErr_SetString( PyExc_StopIteration, "" );
+        }
+%End
+
+
+
+
+
+
+
+
     };
 
 

src/core/providers/ogr/qgsgeopackageproviderconnection.cpp

@@ -70,7 +70,7 @@ QString QgsGeoPackageProviderConnection::tableUri( const QString &schema, const
   const auto tableInfo { table( schema, name ) };
   if ( tableInfo.flags().testFlag( QgsAbstractDatabaseProviderConnection::TableFlag::Raster ) )
   {
-    return QStringLiteral( "GPKG:%1:%2" ).arg( uri() ).arg( name );
+    return QStringLiteral( "GPKG:%1:%2" ).arg( uri(), name );
   }
   else
   {
@@ -123,7 +123,7 @@ void QgsGeoPackageProviderConnection::dropVectorTable( const QString &schema, co
   const QString layerUri { QStringLiteral( "%1|layername=%2" ).arg( uri(), name ) };
   if ( ! QgsOgrProviderUtils::deleteLayer( layerUri, errCause ) )
   {
-    throw QgsProviderConnectionException( QObject::tr( "Error deleting vector/aspatial table %1: %2" ).arg( name ).arg( errCause ) );
+    throw QgsProviderConnectionException( QObject::tr( "Error deleting vector/aspatial table %1: %2" ).arg( name, errCause ) );
   }
 }
 
@@ -306,7 +306,7 @@ QList<QgsGeoPackageProviderConnection::TableProperty> QgsGeoPackageProviderConne
 
   if ( ! errCause.isEmpty() )
   {
-    throw QgsProviderConnectionException( QObject::tr( "Error listing tables from %1: %2" ).arg( uri() ).arg( errCause ) );
+    throw QgsProviderConnectionException( QObject::tr( "Error listing tables from %1: %2" ).arg( uri(), errCause ) );
   }
   // Filters
   if ( flags )
@@ -356,11 +356,10 @@ void QgsGeoPackageProviderConnection::setDefaultCapabilities()
 
 QgsAbstractDatabaseProviderConnection::QueryResult QgsGeoPackageProviderConnection::executeGdalSqlPrivate( const QString &sql, QgsFeedback *feedback ) const
 {
-  QgsAbstractDatabaseProviderConnection::QueryResult results;
 
   if ( feedback && feedback->isCanceled() )
   {
-    return results;
+    return QgsAbstractDatabaseProviderConnection::QueryResult();
   }
 
   QString errCause;
@@ -370,54 +369,46 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsGeoPackageProviderConnecti
 
     if ( feedback && feedback->isCanceled() )
     {
-      return results;
+      return QgsAbstractDatabaseProviderConnection::QueryResult();
     }
 
     OGRLayerH ogrLayer( GDALDatasetExecuteSQL( hDS.get(), sql.toUtf8().constData(), nullptr, nullptr ) );
+
+    // Read fields
     if ( ogrLayer )
     {
+
+      auto iterator = std::make_shared<QgsGeoPackageProviderResultIterator>( std::move( hDS ), ogrLayer );
+      QgsAbstractDatabaseProviderConnection::QueryResult results( iterator );
+      // Note: Returns the number of features in the layer. For dynamic databases the count may not be exact.
+      //       If bForce is FALSE, and it would be expensive to establish the feature count a value of -1 may
+      //       be returned indicating that the count isn’t know.
+      results.setRowCount( OGR_L_GetFeatureCount( ogrLayer, 0 /* bForce=false: do not scan the whole layer */ ) );
+
       gdal::ogr_feature_unique_ptr fet;
-      QgsFields fields;
-      while ( fet.reset( OGR_L_GetNextFeature( ogrLayer ) ), fet )
+      if ( fet.reset( OGR_L_GetNextFeature( ogrLayer ) ), fet )
       {
 
-        if ( feedback && feedback->isCanceled() )
-        {
-          break;
-        }
+        QgsFields fields { QgsOgrUtils::readOgrFields( fet.get(), QTextCodec::codecForName( "UTF-8" ) ) };
+        iterator->setFields( fields );
 
-        QVariantList row;
-
-        // Try to get the right type for the returned values
-        if ( fields.isEmpty() )
+        for ( const auto &f : qgis::as_const( fields ) )
         {
-          fields = QgsOgrUtils::readOgrFields( fet.get(), QTextCodec::codecForName( "UTF-8" ) );
-          for ( const auto &f : qgis::as_const( fields ) )
-          {
-            results.appendColumn( f.name() );
-          }
+          results.appendColumn( f.name() );
         }
-
-        if ( ! fields.isEmpty() )
-        {
-          QgsFeature f { QgsOgrUtils::readOgrFeature( fet.get(), fields, QTextCodec::codecForName( "UTF-8" ) ) };
-          const QgsAttributes &constAttrs { f.attributes() };
-          for ( int i = 0; i < constAttrs.length(); i++ )
-          {
-            row.push_back( constAttrs.at( i ) );
-          }
-        }
-        else // Fallback to strings
-        {
-          for ( int i = 0; i < OGR_F_GetFieldCount( fet.get() ); i++ )
-          {
-            row.push_back( QVariant( QString::fromUtf8( OGR_F_GetFieldAsString( fet.get(), i ) ) ) );
-          }
-        }
-
-        results.appendRow( row );
       }
-      GDALDatasetReleaseResultSet( hDS.get(), ogrLayer );
+
+      // Check for errors
+      errCause = CPLGetLastErrorMsg( );
+
+      if ( ! errCause.isEmpty() )
+      {
+        throw QgsProviderConnectionException( QObject::tr( "Error executing SQL %1: %2" ).arg( sql, errCause ) );
+      }
+
+      OGR_L_ResetReading( ogrLayer );
+      iterator->nextRow();
+      return results;
     }
     errCause = CPLGetLastErrorMsg( );
   }
@@ -425,17 +416,63 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsGeoPackageProviderConnecti
   {
     errCause = QObject::tr( "There was an error opening GPKG %1!" ).arg( uri() );
   }
-  if ( ! errCause.isEmpty() )
+
+  if ( !errCause.isEmpty() )
   {
-    throw QgsProviderConnectionException( QObject::tr( "Error executing SQL %1: %2" ).arg( sql ).arg( errCause ) );
+    throw QgsProviderConnectionException( QObject::tr( "Error executing SQL %1: %2" ).arg( sql, errCause ) );
   }
-  return results;
+
+  return QgsAbstractDatabaseProviderConnection::QueryResult();
 }
 
+QVariantList QgsGeoPackageProviderResultIterator::nextRow()
+{
+  const QVariantList currentRow { mNextRow };
+  mNextRow = nextRowPrivate();
+  return currentRow;
+}
+
+QVariantList QgsGeoPackageProviderResultIterator::nextRowPrivate()
+{
+  QVariantList row;
+  if ( mHDS && mOgrLayer )
+  {
+    gdal::ogr_feature_unique_ptr fet;
+    if ( fet.reset( OGR_L_GetNextFeature( mOgrLayer ) ), fet )
+    {
+      if ( ! mFields.isEmpty() )
+      {
+        QgsFeature f { QgsOgrUtils::readOgrFeature( fet.get(), mFields, QTextCodec::codecForName( "UTF-8" ) ) };
+        const QgsAttributes &constAttrs { f.attributes() };
+        for ( int i = 0; i < constAttrs.length(); i++ )
+        {
+          row.push_back( constAttrs.at( i ) );
+        }
+      }
+      else // Fallback to strings
+      {
+        for ( int i = 0; i < OGR_F_GetFieldCount( fet.get() ); i++ )
+        {
+          row.push_back( QVariant( QString::fromUtf8( OGR_F_GetFieldAsString( fet.get(), i ) ) ) );
+        }
+      }
+    }
+  }
+  return row;
+}
+
+bool QgsGeoPackageProviderResultIterator::hasNextRow() const
+{
+  return ! mNextRow.isEmpty();
+}
+
+void QgsGeoPackageProviderResultIterator::setFields( const QgsFields &fields )
+{
+  mFields = fields;
+}
 
 QList<QgsVectorDataProvider::NativeType> QgsGeoPackageProviderConnection::nativeTypes() const
 {
-  QList<QgsVectorDataProvider::NativeType> types;
   QgsVectorLayer::LayerOptions options { false, true };
   options.skipCrsValidation = true;
   const QgsVectorLayer vl { uri(), QStringLiteral( "temp_layer" ), QStringLiteral( "ogr" ), options };
@@ -444,7 +481,12 @@ QList<QgsVectorDataProvider::NativeType> QgsGeoPackageProviderConnection::native
     const QString errorCause { vl.dataProvider() &&vl.dataProvider()->hasErrors() ?
                                vl.dataProvider()->errors().join( '\n' ) :
                                QObject::tr( "unknown error" ) };
-    throw QgsProviderConnectionException( QObject::tr( "Error retrieving native types for %1: %2" ).arg( uri() ).arg( errorCause ) );
+    throw QgsProviderConnectionException( QObject::tr( "Error retrieving native types for %1: %2" ).arg( uri(), errorCause ) );
   }
   return vl.dataProvider()->nativeTypes();
 }
+
+QgsGeoPackageProviderResultIterator::~QgsGeoPackageProviderResultIterator()
+{
+  GDALDatasetReleaseResultSet( mHDS.get(), mOgrLayer );
+}

src/core/providers/ogr/qgsgeopackageproviderconnection.h

@@ -13,14 +13,43 @@
  *   (at your option) any later version.                                   *
  *                                                                         *
  ***************************************************************************/
+
 #ifndef QGSGEOPACKAGEPROVIDERCONNECTION_H
 #define QGSGEOPACKAGEPROVIDERCONNECTION_H
 
 #include "qgsabstractdatabaseproviderconnection.h"
+#include "qgsogrutils.h"
 
 ///@cond PRIVATE
 #define SIP_NO_FILE
 
+
+
+struct QgsGeoPackageProviderResultIterator: public QgsAbstractDatabaseProviderConnection::QueryResult::QueryResultIterator
+{
+    QgsGeoPackageProviderResultIterator( gdal::ogr_datasource_unique_ptr hDS, OGRLayerH ogrLayer )
+      : mHDS( std::move( hDS ) )
+      , mOgrLayer( ogrLayer )
+    {}
+
+    ~QgsGeoPackageProviderResultIterator();
+
+    QVariantList nextRow() override;
+    bool hasNextRow() const override;
+
+    void setFields( const QgsFields &fields );
+
+  private:
+
+    gdal::ogr_datasource_unique_ptr mHDS;
+    OGRLayerH mOgrLayer;
+    QgsFields mFields;
+    QVariantList mNextRow;
+
+    QVariantList nextRowPrivate();
+
+};
+
 class QgsGeoPackageProviderConnection : public QgsAbstractDatabaseProviderConnection
 {
   public:

src/core/qgsabstractdatabaseproviderconnection.cpp

@@ -442,15 +442,65 @@ QStringList QgsAbstractDatabaseProviderConnection::QueryResult::columns() const
 
 QList<QList<QVariant> > QgsAbstractDatabaseProviderConnection::QueryResult::rows() const
 {
+  // mRowCount might be -1 (unknown)
+  while ( mResultIterator && ( mRowCount < 0 || mRows.count() < mRowCount ) )
+  {
+    const QVariantList row { mResultIterator->nextRow() };
+    if ( row.isEmpty() )
+    {
+      break;
+    }
+    mRows.push_back( row );
+  }
   return mRows;
 }
 
+QList<QVariant> QgsAbstractDatabaseProviderConnection::QueryResult::nextRow()
+{
+  if ( ! mResultIterator && ! mResultIterator->hasNextRow() )
+  {
+    return QList<QVariant>();
+  }
+
+  const QList<QVariant> row { mResultIterator->nextRow() };
+
+  if ( ! row.isEmpty() )
+  {
+    mRows.push_back( row );
+  }
+
+  return row;
+}
+
+
+qlonglong QgsAbstractDatabaseProviderConnection::QueryResult::rowCount() const
+{
+  return mRowCount;
+}
+
+bool QgsAbstractDatabaseProviderConnection::QueryResult::hasNextRow() const
+{
+  if ( ! mResultIterator )
+  {
+    return false;
+  }
+  return mResultIterator->hasNextRow();
+}
+
+///@cond PRIVATE
+
 void QgsAbstractDatabaseProviderConnection::QueryResult::appendColumn( const QString &columnName )
 {
   mColumns.push_back( columnName );
 }
 
-void QgsAbstractDatabaseProviderConnection::QueryResult::appendRow( const QList<QVariant> &row )
+void QgsAbstractDatabaseProviderConnection::QueryResult::setRowCount( const qlonglong &rowCount )
 {
-  mRows.push_back( row );
+  mRowCount = rowCount;
 }
+
+QgsAbstractDatabaseProviderConnection::QueryResult::QueryResult( std::shared_ptr<QgsAbstractDatabaseProviderConnection::QueryResult::QueryResultIterator> iterator )
+  : mResultIterator( iterator )
+{}
+
+///@endcond private

src/core/qgsabstractdatabaseproviderconnection.h

@@ -72,6 +72,8 @@ class CORE_EXPORT QgsAbstractDatabaseProviderConnection : public QgsAbstractProv
      *
      * It encapsulates the result rows and a list of the column names.
      * The query result may be empty in case the query returns nothing.
+     *
+     *
      * \since QGIS 3.18
      */
     struct CORE_EXPORT QueryResult
@@ -79,7 +81,7 @@ class CORE_EXPORT QgsAbstractDatabaseProviderConnection : public QgsAbstractProv
 #ifdef SIP_RUN
         SIP_PYOBJECT __repr__();
         % MethodCode
-        QString str = QStringLiteral( "<QgsAbstractDatabaseProviderConnection.QueryResult: %1 rows>" ).arg( sipCpp->rows().size() );
+        QString str = QStringLiteral( "<QgsAbstractDatabaseProviderConnection.QueryResult: %1 rows>" ).arg( sipCpp->rowCount() );
         sipRes = PyUnicode_FromString( str.toUtf8().constData() );
         % End
 #endif
@@ -90,24 +92,95 @@ class CORE_EXPORT QgsAbstractDatabaseProviderConnection : public QgsAbstractProv
         QStringList columns() const;
 
         /**
-         *Returns the results rows
+         * Returns the results rows
          */
         QList<QList<QVariant> > rows() const;
 
         /**
-         * Appends \a columnName to the list of column names.
+          * Returns the row count
+          * \note the value may not be exact or it may be -1 if not known
          */
-        void appendColumn( const QString &columnName );
+        qlonglong rowCount() const;
 
         /**
-         * Appends \a row to the results.
+         * Returns TRUE if there are more rows to fetch
          */
-        void appendRow( const QList<QVariant> &row );
+        bool hasNextRow() const;
+
+        /**
+         * Returns the next result row or an empty row if there are no rows left
+         */
+        QList<QVariant> nextRow();
+
+#ifdef SIP_RUN
+        QueryResult *__iter__();
+        % MethodCode
+        sipRes = sipCpp;
+        % End
+
+        SIP_PYOBJECT __next__();
+        % MethodCode
+        QList<QVariant> result;
+        Py_BEGIN_ALLOW_THREADS
+        result = sipCpp->nextRow( );
+        Py_END_ALLOW_THREADS
+        if ( ! result.isEmpty() )
+        {
+          const sipTypeDef *qvariantlist_type = sipFindType( "QList<QVariant>" );
+          sipRes = sipConvertFromNewType( new QList<QVariant>( result ), qvariantlist_type, Py_None );
+        }
+        else
+        {
+          PyErr_SetString( PyExc_StopIteration, "" );
+        }
+        % End
+#endif
+
+///@cond private
+
+        /**
+         * The QueryResultIterator struct is an abstract interface for provider query results iterators.
+         * Providers must implement their own concrete iterator over query results.
+         */
+        struct QueryResultIterator SIP_SKIP
+        {
+          virtual QVariantList nextRow() = 0;
+          virtual bool hasNextRow() const = 0;
+          virtual ~QueryResultIterator() = default;
+        };
+
+        /**
+          * Sets \a rowCount
+          * \note Not available in Python bindings
+          */
+        void setRowCount( const qlonglong &rowCount ) SIP_SKIP;
+
+        /**
+         * Appends \a columnName to the list of column names.
+         * \note Not available in Python bindings
+         */
+        void appendColumn( const QString &columnName ) SIP_SKIP;
+
+        /**
+         * Constructs a QueryResult object from an \a iterator
+         * \note Not available in Python bindings
+         */
+        QueryResult( std::shared_ptr<QueryResultIterator> iterator ) SIP_SKIP;
+
+        /**
+         * Default constructor, used to return empty results
+         * \note Not available in Python bindings
+         */
+        QueryResult( ) = default SIP_SKIP;
+
+///@endcond private
 
       private:
 
+        std::shared_ptr<QueryResultIterator> mResultIterator;
         QStringList mColumns;
-        QList<QList<QVariant>> mRows;
+        mutable QList<QList<QVariant>> mRows;
+        qlonglong mRowCount = 0;
 
     };
 

src/providers/mssql/qgsmssqlproviderconnection.cpp

@@ -222,11 +222,10 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsMssqlProviderConnection::e
 
 QgsAbstractDatabaseProviderConnection::QueryResult QgsMssqlProviderConnection::executeSqlPrivate( const QString &sql, bool resolveTypes, QgsFeedback *feedback ) const
 {
-  QgsAbstractDatabaseProviderConnection::QueryResult results;
 
   if ( feedback && feedback->isCanceled() )
   {
-    return results;
+    return QgsAbstractDatabaseProviderConnection::QueryResult();
   }
 
   const QgsDataSourceUri dsUri { uri() };
@@ -237,15 +236,14 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsMssqlProviderConnection::e
   if ( !QgsMssqlConnection::openDatabase( db ) )
   {
     throw QgsProviderConnectionException( QObject::tr( "Connection to %1 failed: %2" )
-                                          .arg( uri() )
-                                          .arg( db.lastError().text() ) );
+                                          .arg( uri(), db.lastError().text() ) );
   }
   else
   {
 
     if ( feedback && feedback->isCanceled() )
     {
-      return results;
+      return QgsAbstractDatabaseProviderConnection::QueryResult();
     }
 
     //qDebug() << "MSSQL QUERY:" << sql;
@@ -256,8 +254,7 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsMssqlProviderConnection::e
     {
       const QString errorMessage { q.lastError().text() };
       throw QgsProviderConnectionException( QObject::tr( "SQL error: %1 \n %2" )
-                                            .arg( sql )
-                                            .arg( errorMessage ) );
+                                            .arg( sql, errorMessage ) );
 
     }
 
@@ -265,32 +262,55 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsMssqlProviderConnection::e
     {
       const QSqlRecord rec { q.record() };
       const int numCols { rec.count() };
+      auto iterator = std::make_shared<QgssMssqlProviderResultIterator>( resolveTypes, numCols, q );
+      QgsAbstractDatabaseProviderConnection::QueryResult results( iterator );
+      results.setRowCount( q.size() );
       for ( int idx = 0; idx < numCols; ++idx )
       {
         results.appendColumn( rec.field( idx ).name() );
       }
-      while ( q.next() && ( ! feedback || ! feedback->isCanceled() ) )
-      {
-        QVariantList row;
-        for ( int col = 0; col < numCols; ++col )
-        {
-          if ( resolveTypes )
-          {
-            row.push_back( q.value( col ) );
-          }
-          else
-          {
-            row.push_back( q.value( col ).toString() );
-          }
-        }
-        results.appendRow( row );
-      }
+      iterator->nextRow();
+      return results;
     }
 
   }
-  return results;
+  return QgsAbstractDatabaseProviderConnection::QueryResult();
 }
 
+
+QVariantList QgssMssqlProviderResultIterator::nextRow()
+{
+  const QVariantList currentRow { mNextRow };
+  mNextRow = nextRowPrivate();
+  return currentRow;
+}
+
+bool QgssMssqlProviderResultIterator::hasNextRow() const
+{
+  return ! mNextRow.isEmpty();
+}
+
+QVariantList QgssMssqlProviderResultIterator::nextRowPrivate()
+{
+  QVariantList row;
+  if ( mQuery.next() )
+  {
+    for ( int col = 0; col < mColumnCount; ++col )
+    {
+      if ( mResolveTypes )
+      {
+        row.push_back( mQuery.value( col ) );
+      }
+      else
+      {
+        row.push_back( mQuery.value( col ).toString() );
+      }
+    }
+  }
+  return row;
+}
+
+
 QList<QgsMssqlProviderConnection::TableProperty> QgsMssqlProviderConnection::tables( const QString &schema, const TableFlags &flags ) const
 {
   checkCapability( Capability::Tables );

src/providers/mssql/qgsmssqlproviderconnection.h

@@ -13,10 +13,37 @@
  *   (at your option) any later version.                                   *
  *                                                                         *
  ***************************************************************************/
+
 #ifndef QGSMSSQLPROVIDERCONNECTION_H
 #define QGSMSSQLPROVIDERCONNECTION_H
+
 #include "qgsabstractdatabaseproviderconnection.h"
 
+#include <QSqlQuery>
+
+
+struct QgssMssqlProviderResultIterator: public QgsAbstractDatabaseProviderConnection::QueryResult::QueryResultIterator
+{
+
+    QgssMssqlProviderResultIterator( bool resolveTypes, int columnCount, const QSqlQuery &query )
+      : mResolveTypes( resolveTypes )
+      , mColumnCount( columnCount )
+      , mQuery( query )
+    {}
+
+    QVariantList nextRow() override;
+    bool hasNextRow() const override;
+
+  private:
+
+    bool mResolveTypes = true;
+    int mColumnCount = 0;
+    QSqlQuery mQuery;
+    QVariantList mNextRow;
+
+    QVariantList nextRowPrivate();
+
+};
 
 class QgsMssqlProviderConnection : public QgsAbstractDatabaseProviderConnection
 

src/providers/postgres/qgspostgresproviderconnection.cpp

@@ -206,7 +206,13 @@ QList<QVariantList> QgsPostgresProviderConnection::executeSqlPrivate( const QStr
 
 QgsAbstractDatabaseProviderConnection::QueryResult QgsPostgresProviderConnection::execSqlPrivate( const QString &sql, bool resolveTypes, QgsFeedback *feedback, std::shared_ptr<QgsPoolPostgresConn> pgconn ) const
 {
-  QueryResult results;
+  if ( ! pgconn )
+  {
+    pgconn = std::make_shared<QgsPoolPostgresConn>( QgsDataSourceUri( uri() ).connectionInfo( false ) );
+  }
+
+  std::shared_ptr<QgsAbstractDatabaseProviderConnection::QueryResult::QueryResultIterator> iterator = std::make_shared<QgsPostgresProviderResultIterator>( resolveTypes, pgconn );
+  QueryResult results( iterator );
 
   // Check feedback first!
   if ( feedback && feedback->isCanceled() )
@@ -214,11 +220,6 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsPostgresProviderConnection
     return results;
   }
 
-  if ( ! pgconn )
-  {
-    pgconn = std::make_shared<QgsPoolPostgresConn>( QgsDataSourceUri( uri() ).connectionInfo( false ) );
-  }
-
   QgsPostgresConn *conn = pgconn->get();
 
   if ( ! conn )
@@ -237,20 +238,22 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsPostgresProviderConnection
     QMetaObject::Connection qtConnection;
     if ( feedback )
     {
-      qtConnection = QObject::connect( feedback, &QgsFeedback::canceled, [ &conn ]
+      qtConnection = QObject::connect( feedback, &QgsFeedback::canceled, [ &pgconn ]
       {
-        conn->PQCancel();
+        if ( pgconn )
+          pgconn->get()->PQCancel();
       } );
     }
 
-    QgsPostgresResult res( conn->PQexec( sql ) );
+    std::unique_ptr<QgsPostgresResult> res = qgis::make_unique<QgsPostgresResult>( conn->PQexec( sql ) );
+
     if ( feedback )
     {
       QObject::disconnect( qtConnection );
     }
 
     QString errCause;
-    if ( conn->PQstatus() != CONNECTION_OK || ! res.result() )
+    if ( conn->PQstatus() != CONNECTION_OK || ! res->result() )
     {
       errCause = QObject::tr( "Connection error: %1 returned %2 [%3]" )
                  .arg( sql ).arg( conn->PQstatus() )
@@ -268,18 +271,20 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsPostgresProviderConnection
       }
     }
 
-    if ( res.PQntuples() > 0 )
+    const qlonglong numRows { res->PQntuples() };
+    results.setRowCount( numRows );
+
+    if ( numRows > 0 )
     {
 
       // Get column names
-      for ( int rowIdx = 0; rowIdx < res.PQnfields(); rowIdx++ )
+      for ( int rowIdx = 0; rowIdx < res->PQnfields(); rowIdx++ )
       {
-        results.appendColumn( res.PQfname( rowIdx ) );
+        results.appendColumn( res->PQfname( rowIdx ) );
       }
 
       // Try to convert value types at least for basic simple types that can be directly mapped to Python
-      QMap<int, QVariant::Type> typeMap;
-      const int numFields { res.PQnfields() };
+      const int numFields { res->PQnfields() };
       if ( resolveTypes )
       {
         // Collect oids
@@ -291,7 +296,7 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsPostgresProviderConnection
           {
             break;
           }
-          const QString oidStr { QString::number( res.PQftype( rowIdx ) ) };
+          const QString oidStr { QString::number( res->PQftype( rowIdx ) ) };
           oids.push_back( oidStr );
         }
 
@@ -353,61 +358,72 @@ QgsAbstractDatabaseProviderConnection::QueryResult QgsPostgresProviderConnection
             // Just a warning, usually ok
             QgsDebugMsgLevel( QStringLiteral( "Unhandled PostgreSQL type %1, assuming string" ).arg( typName ), 2 );
           }
-          typeMap[ rowIdx ] = vType;
+          static_cast<QgsPostgresProviderResultIterator *>( iterator.get() )->typeMap[ rowIdx ] = vType;
         }
       }
-
-      // Get results
-      for ( int rowIdx = 0; rowIdx < res.PQntuples(); rowIdx++ )
-      {
-        if ( feedback && feedback->isCanceled() )
-        {
-          break;
-        }
-        QVariantList row;
-        for ( int colIdx = 0; colIdx < res.PQnfields(); colIdx++ )
-        {
-          if ( resolveTypes )
-          {
-            const QVariant::Type vType { typeMap.value( colIdx, QVariant::Type::String ) };
-            QVariant val { res.PQgetvalue( rowIdx, colIdx ) };
-            // Special case for bools: 'f' and 't'
-            if ( vType == QVariant::Bool )
-            {
-              const QString boolStrVal { val.toString() };
-              if ( ! boolStrVal.isEmpty() )
-              {
-                val = boolStrVal == 't';
-              }
-            }
-            else if ( val.canConvert( static_cast<int>( vType ) ) )
-            {
-              val.convert( static_cast<int>( vType ) );
-            }
-            row.push_back( val );
-          }
-          else
-          {
-            row.push_back( res.PQgetvalue( rowIdx, colIdx ) );
-          }
-        }
-        results.appendRow( row );
-      }
     }
     if ( ! errCause.isEmpty() )
     {
       throw QgsProviderConnectionException( errCause );
     }
+    static_cast<QgsPostgresProviderResultIterator *>( iterator.get() )->result = std::move( res );
   }
   return results;
 }
 
+
+QVariantList QgsPostgresProviderResultIterator::nextRow()
+{
+  // Get results
+  QVariantList row;
+
+  if ( mRowIndex >= result->PQntuples() )
+  {
+    return row;
+  }
+
+  for ( int colIdx = 0; colIdx < result->PQnfields(); colIdx++ )
+  {
+    if ( mResolveTypes )
+    {
+      const QVariant::Type vType { typeMap.value( colIdx, QVariant::Type::String ) };
+      QVariant val { result->PQgetvalue( mRowIndex, colIdx ) };
+      // Special case for bools: 'f' and 't'
+      if ( vType == QVariant::Bool )
+      {
+        const QString boolStrVal { val.toString() };
+        if ( ! boolStrVal.isEmpty() )
+        {
+          val = boolStrVal == 't';
+        }
+      }
+      else if ( val.canConvert( static_cast<int>( vType ) ) )
+      {
+        val.convert( static_cast<int>( vType ) );
+      }
+      row.push_back( val );
+    }
+    else
+    {
+      row.push_back( result->PQgetvalue( mRowIndex, colIdx ) );
+    }
+  }
+  ++mRowIndex;
+  return row;
+}
+
+bool QgsPostgresProviderResultIterator::hasNextRow() const
+{
+  return mRowIndex < result->PQntuples();
+}
+
+
 void QgsPostgresProviderConnection::vacuum( const QString &schema, const QString &name ) const
 {
   checkCapability( Capability::Vacuum );
   executeSql( QStringLiteral( "VACUUM FULL ANALYZE %1.%2" )
-              .arg( QgsPostgresConn::quotedIdentifier( schema ) )
-              .arg( QgsPostgresConn::quotedIdentifier( name ) ) );
+              .arg( QgsPostgresConn::quotedIdentifier( schema ),
+                    QgsPostgresConn::quotedIdentifier( name ) ) );
 }
 
 void QgsPostgresProviderConnection::createSpatialIndex( const QString &schema, const QString &name, const QgsAbstractDatabaseProviderConnection::SpatialIndexOptions &options ) const

src/providers/postgres/qgspostgresproviderconnection.h

@@ -16,10 +16,30 @@
 #ifndef QGSPOSTGRESPROVIDERCONNECTION_H
 #define QGSPOSTGRESPROVIDERCONNECTION_H
 #include "qgsabstractdatabaseproviderconnection.h"
+#include "qgspostgresconnpool.h"
 
+struct QgsPostgresProviderResultIterator: public QgsAbstractDatabaseProviderConnection::QueryResult::QueryResultIterator
+{
+    QgsPostgresProviderResultIterator( bool resolveTypes, std::shared_ptr<QgsPoolPostgresConn> pgConn )
+      : mResolveTypes( resolveTypes )
+      , mConn( pgConn )
+    {}
+
+    QVariantList nextRow() override;
+    bool hasNextRow() const override;
+
+    QMap<int, QVariant::Type> typeMap;
+    std::unique_ptr<QgsPostgresResult> result;
+
+  private:
+
+    bool mResolveTypes = true;
+    std::shared_ptr<QgsPoolPostgresConn> mConn;
+    qlonglong mRowIndex = 0;
+
+};
 
 class QgsPostgresProviderConnection : public QgsAbstractDatabaseProviderConnection
-
 {
   public:
 

src/providers/spatialite/qgsspatialiteproviderconnection.cpp

@@ -397,79 +397,125 @@ void QgsSpatiaLiteProviderConnection::setDefaultCapabilities()
 
 QgsAbstractDatabaseProviderConnection::QueryResult QgsSpatiaLiteProviderConnection::executeSqlPrivate( const QString &sql, QgsFeedback *feedback ) const
 {
-  QgsAbstractDatabaseProviderConnection::QueryResult results;
 
   if ( feedback && feedback->isCanceled() )
   {
-    return results;
+    return QgsAbstractDatabaseProviderConnection::QueryResult();
   }
 
   QString errCause;
-
   gdal::ogr_datasource_unique_ptr hDS( GDALOpenEx( pathFromUri().toUtf8().constData(), GDAL_OF_VECTOR | GDAL_OF_UPDATE, nullptr, nullptr, nullptr ) );
   if ( hDS )
   {
 
     if ( feedback && feedback->isCanceled() )
     {
-      return results;
+      return QgsAbstractDatabaseProviderConnection::QueryResult();
     }
 
     OGRLayerH ogrLayer( GDALDatasetExecuteSQL( hDS.get(), sql.toUtf8().constData(), nullptr, nullptr ) );
+
+    // Read fields
     if ( ogrLayer )
     {
+
+      auto iterator = std::make_shared<QgsSpatialiteProviderResultIterator>( std::move( hDS ), ogrLayer );
+      QgsAbstractDatabaseProviderConnection::QueryResult results( iterator );
+      // Note: Returns the number of features in the layer. For dynamic databases the count may not be exact.
+      //       If bForce is FALSE, and it would be expensive to establish the feature count a value of -1 may
+      //       be returned indicating that the count isn’t know.
+      results.setRowCount( OGR_L_GetFeatureCount( ogrLayer, 0 /* force=false: do not scan the whole layer */ ) );
+
       gdal::ogr_feature_unique_ptr fet;
-      QgsFields fields;
-      while ( fet.reset( OGR_L_GetNextFeature( ogrLayer ) ), fet )
+      if ( fet.reset( OGR_L_GetNextFeature( ogrLayer ) ), fet )
       {
 
-        if ( feedback && feedback->isCanceled() )
-        {
-          break;
-        }
+        QgsFields fields { QgsOgrUtils::readOgrFields( fet.get(), QTextCodec::codecForName( "UTF-8" ) ) };
+        iterator->setFields( fields );
 
-        QVariantList row;
-        // Try to get the right type for the returned values
-        if ( fields.isEmpty() )
+        for ( const auto &f : qgis::as_const( fields ) )
         {
-          fields = QgsOgrUtils::readOgrFields( fet.get(), QTextCodec::codecForName( "UTF-8" ) );
-          for ( const auto &f : qgis::as_const( fields ) )
-          {
-            results.appendColumn( f.name() );
-          }
+          results.appendColumn( f.name() );
         }
-        if ( ! fields.isEmpty() )
-        {
-          QgsFeature f { QgsOgrUtils::readOgrFeature( fet.get(), fields, QTextCodec::codecForName( "UTF-8" ) ) };
-          const QgsAttributes &constAttrs { f.attributes() };
-          for ( int i = 0; i < constAttrs.length(); i++ )
-          {
-            row.push_back( constAttrs.at( i ) );
-          }
-        }
-        else // Fallback to strings
-        {
-          for ( int i = 0; i < OGR_F_GetFieldCount( fet.get() ); i++ )
-          {
-            row.push_back( QVariant( QString::fromUtf8( OGR_F_GetFieldAsString( fet.get(), i ) ) ) );
-          }
-        }
-
-        results.appendRow( row );
       }
-      GDALDatasetReleaseResultSet( hDS.get(), ogrLayer );
+
+      // Check for errors
+      errCause = CPLGetLastErrorMsg( );
+
+      if ( ! errCause.isEmpty() )
+      {
+        throw QgsProviderConnectionException( QObject::tr( "Error executing SQL %1: %2" ).arg( sql, errCause ) );
+      }
+
+      OGR_L_ResetReading( ogrLayer );
+      iterator->nextRow();
+      return results;
     }
     errCause = CPLGetLastErrorMsg( );
   }
   else
   {
-    errCause = QObject::tr( "There was an error opening Spatialite %1!" ).arg( pathFromUri() );
+    errCause = QObject::tr( "There was an error opening GPKG %1!" ).arg( uri() );
   }
-  if ( ! errCause.isEmpty() )
+
+  if ( !errCause.isEmpty() )
   {
-    throw QgsProviderConnectionException( QObject::tr( "Error executing SQL %1: %2" ).arg( sql ).arg( errCause ) );
+    throw QgsProviderConnectionException( QObject::tr( "Error executing SQL %1: %2" ).arg( sql, errCause ) );
   }
-  return results;
+
+  return QgsAbstractDatabaseProviderConnection::QueryResult();
+}
+
+
+void QgsSpatialiteProviderResultIterator::setFields( const QgsFields &fields )
+{
+  mFields = fields;
+}
+
+QgsSpatialiteProviderResultIterator::~QgsSpatialiteProviderResultIterator()
+{
+  GDALDatasetReleaseResultSet( mHDS.get(), mOgrLayer );
+}
+
+QVariantList QgsSpatialiteProviderResultIterator::nextRow()
+{
+  const QVariantList currentRow { mNextRow };
+  mNextRow = nextRowPrivate();
+  return currentRow;
+}
+
+QVariantList QgsSpatialiteProviderResultIterator::nextRowPrivate()
+{
+  QVariantList row;
+  if ( mHDS && mOgrLayer )
+  {
+    gdal::ogr_feature_unique_ptr fet;
+    if ( fet.reset( OGR_L_GetNextFeature( mOgrLayer ) ), fet )
+    {
+      if ( ! mFields.isEmpty() )
+      {
+        QgsFeature f { QgsOgrUtils::readOgrFeature( fet.get(), mFields, QTextCodec::codecForName( "UTF-8" ) ) };
+        const QgsAttributes &constAttrs { f.attributes() };
+        for ( int i = 0; i < constAttrs.length(); i++ )
+        {
+          row.push_back( constAttrs.at( i ) );
+        }
+      }
+      else // Fallback to strings
+      {
+        for ( int i = 0; i < OGR_F_GetFieldCount( fet.get() ); i++ )
+        {
+          row.push_back( QVariant( QString::fromUtf8( OGR_F_GetFieldAsString( fet.get(), i ) ) ) );
+        }
+      }
+    }
+  }
+  return row;
+}
+
+bool QgsSpatialiteProviderResultIterator::hasNextRow() const
+{
+  return ! mNextRow.isEmpty();
 }
 
 bool QgsSpatiaLiteProviderConnection::executeSqlDirect( const QString &sql ) const

src/providers/spatialite/qgsspatialiteproviderconnection.h

@@ -17,10 +17,37 @@
 #define QGSSPATIALITEPROVIDERCONNECTION_H
 
 #include "qgsabstractdatabaseproviderconnection.h"
+#include "qgsogrutils.h"
 
 ///@cond PRIVATE
 #define SIP_NO_FILE
 
+
+struct QgsSpatialiteProviderResultIterator: public QgsAbstractDatabaseProviderConnection::QueryResult::QueryResultIterator
+{
+    QgsSpatialiteProviderResultIterator( gdal::ogr_datasource_unique_ptr hDS, OGRLayerH ogrLayer )
+      : mHDS( std::move( hDS ) )
+      , mOgrLayer( ogrLayer )
+    {}
+
+    ~QgsSpatialiteProviderResultIterator();
+
+    QVariantList nextRow() override;
+    bool hasNextRow() const override;
+
+    void setFields( const QgsFields &fields );
+
+  private:
+
+    gdal::ogr_datasource_unique_ptr mHDS;
+    OGRLayerH mOgrLayer;
+    QgsFields mFields;
+    QVariantList mNextRow;
+
+    QVariantList nextRowPrivate();
+};
+
+
 class QgsSpatiaLiteProviderConnection : public QgsAbstractDatabaseProviderConnection
 {
   public:

tests/src/python/test_qgsproviderconnection_base.py

@@ -236,6 +236,30 @@ class TestPyQgsProviderConnectionBase():
                 self.assertEqual(res.rows(), [['QGIS Rocks - \U0001f604', 666, 1.234, 1234, QtCore.QDate(2019, 7, 8) if not self.treat_date_as_string() else '2019-07-08', QtCore.QDateTime(2019, 7, 8, 12, 0, 12)]])
                 self.assertEqual(res.columns(), ['string_t', 'long_t', 'double_t', 'integer_t', 'date_t', 'datetime_t'])
 
+                # Test iterator
+                old_rows = res.rows()
+                res = conn.execSql(sql)
+                rows = []
+                self.assertTrue(res.hasNextRow())
+
+                for row in res:
+                    rows.append(row)
+
+                self.assertEqual(rows, old_rows)
+                self.assertEqual(rows, res.rows())
+
+                # Java style
+                res = conn.execSql(sql)
+                rows = []
+                self.assertTrue(res.hasNextRow())
+                while res.hasNextRow():
+                    rows.append(res.nextRow())
+
+                self.assertFalse(res.hasNextRow())
+
+                # But we still have access to rows:
+                self.assertEqual(rows, res.rows())
+
                 sql = "SELECT time_t FROM %s" % table
                 res = conn.executeSql(sql)
 
@@ -294,13 +318,13 @@ class TestPyQgsProviderConnectionBase():
                 if capabilities & QgsAbstractDatabaseProviderConnection.SpatialIndexExists:
                     self.assertFalse(conn.spatialIndexExists('myNewSchema', 'myNewTable', 'geom'))
 
-            if capabilities & QgsAbstractDatabaseProviderConnection.CreateSpatialIndex:
+            if capabilities & (QgsAbstractDatabaseProviderConnection.CreateSpatialIndex | QgsAbstractDatabaseProviderConnection.SpatialIndexExists):
                 options = QgsAbstractDatabaseProviderConnection.SpatialIndexOptions()
                 options.geometryColumnName = 'geom'
-                conn.createSpatialIndex('myNewSchema', 'myNewTable', options)
+                if not conn.spatialIndexExists('myNewSchema', 'myNewTable', options.geometryColumnName):
+                    conn.createSpatialIndex('myNewSchema', 'myNewTable', options)
 
-                if capabilities & QgsAbstractDatabaseProviderConnection.SpatialIndexExists:
-                    self.assertTrue(conn.spatialIndexExists('myNewSchema', 'myNewTable', 'geom'))
+                self.assertTrue(conn.spatialIndexExists('myNewSchema', 'myNewTable', 'geom'))
 
                 # now we know for certain a spatial index exists, let's retry dropping it
                 if capabilities & QgsAbstractDatabaseProviderConnection.DeleteSpatialIndex:

tests/src/python/test_qgsproviderconnection_mssql.py

@@ -94,7 +94,7 @@ class TestPyQgsProviderConnectionMssql(unittest.TestCase, TestPyQgsProviderConne
         vl = QgsVectorLayer(conn.tableUri('qgis_test', 'someData'), 'my', 'mssql')
         self.assertTrue(vl.isValid())
 
-    def test_gpkg_fields(self):
+    def test_mssql_fields(self):
         """Test fields"""
 
         md = QgsProviderRegistry.instance().providerMetadata('mssql')
@@ -102,9 +102,6 @@ class TestPyQgsProviderConnectionMssql(unittest.TestCase, TestPyQgsProviderConne
         fields = conn.fields('qgis_test', 'someData')
         self.assertEqual(fields.names(), ['pk', 'cnt', 'name', 'name2', 'num_char', 'dt', 'date', 'time'])
 
-    def treat_date_as_string(self):
-        return True
-
 
 if __name__ == '__main__':
     unittest.main()