QGIS/src/providers/postgres/qgspostgresfeatureiterator.cpp

712 lines
22 KiB
C++

/***************************************************************************
qgspostgresfeatureiterator.cpp
---------------------
begin : Juli 2012
copyright : (C) 2012 by Martin Dobias
email : wonder dot sk at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include "qgsgeometry.h"
#include "qgspostgresconnpool.h"
#include "qgspostgresexpressioncompiler.h"
#include "qgspostgresfeatureiterator.h"
#include "qgspostgresprovider.h"
#include "qgspostgrestransaction.h"
#include "qgslogger.h"
#include "qgsmessagelog.h"
#include <QObject>
#include <QSettings>
const int QgsPostgresFeatureIterator::sFeatureQueueSize = 2000;
QgsPostgresFeatureIterator::QgsPostgresFeatureIterator( QgsPostgresFeatureSource* source, bool ownSource, const QgsFeatureRequest& request )
: QgsAbstractFeatureIteratorFromSource<QgsPostgresFeatureSource>( source, ownSource, request )
, mFeatureQueueSize( sFeatureQueueSize )
, mFetched( 0 )
, mFetchGeometry( false )
, mExpressionCompiled( false )
, mLastFetch( false )
{
if ( !source->mTransactionConnection )
{
mConn = QgsPostgresConnPool::instance()->acquireConnection( mSource->mConnInfo );
mIsTransactionConnection = false;
}
else
{
mConn = source->mTransactionConnection;
mConn->lock();
mIsTransactionConnection = true;
}
if ( !mConn )
{
mClosed = true;
iteratorClosed();
return;
}
mCursorName = mConn->uniqueCursorName();
QString whereClause;
bool limitAtProvider = ( mRequest.limit() >= 0 );
bool useFallbackWhereClause = false;
QString fallbackWhereClause;
if ( !request.filterRect().isNull() && !mSource->mGeometryColumn.isNull() )
{
whereClause = whereClauseRect();
}
if ( !mSource->mSqlWhereClause.isEmpty() )
{
whereClause = QgsPostgresUtils::andWhereClauses( whereClause, '(' + mSource->mSqlWhereClause + ')' );
}
if ( request.filterType() == QgsFeatureRequest::FilterFid )
{
QString fidWhereClause = QgsPostgresUtils::whereClause( mRequest.filterFid(), mSource->mFields, mConn, mSource->mPrimaryKeyType, mSource->mPrimaryKeyAttrs, mSource->mShared );
whereClause = QgsPostgresUtils::andWhereClauses( whereClause, fidWhereClause );
}
else if ( request.filterType() == QgsFeatureRequest::FilterFids )
{
QString fidsWhereClause = QgsPostgresUtils::whereClause( mRequest.filterFids(), mSource->mFields, mConn, mSource->mPrimaryKeyType, mSource->mPrimaryKeyAttrs, mSource->mShared );
whereClause = QgsPostgresUtils::andWhereClauses( whereClause, fidsWhereClause );
}
else if ( request.filterType() == QgsFeatureRequest::FilterExpression )
{
if ( QSettings().value( "/qgis/compileExpressions", true ).toBool() )
{
//IMPORTANT - this MUST be the last clause added!
QgsPostgresExpressionCompiler compiler = QgsPostgresExpressionCompiler( source );
if ( compiler.compile( request.filterExpression() ) == QgsSqlExpressionCompiler::Complete )
{
useFallbackWhereClause = true;
fallbackWhereClause = whereClause;
whereClause = QgsPostgresUtils::andWhereClauses( whereClause, compiler.result() );
mExpressionCompiled = true;
}
else
{
limitAtProvider = false;
}
}
else
{
limitAtProvider = false;
}
}
bool success = declareCursor( whereClause, limitAtProvider ? mRequest.limit() : -1, false );
if ( !success && useFallbackWhereClause )
{
//try with the fallback where clause, eg for cases when using compiled expression failed to prepare
mExpressionCompiled = false;
success = declareCursor( fallbackWhereClause, -1, false );
}
if ( !success )
{
close();
mClosed = true;
iteratorClosed();
}
mFetched = 0;
}
QgsPostgresFeatureIterator::~QgsPostgresFeatureIterator()
{
close();
}
bool QgsPostgresFeatureIterator::fetchFeature( QgsFeature& feature )
{
feature.setValid( false );
if ( mClosed )
return false;
if ( mFeatureQueue.empty() && !mLastFetch )
{
QString fetch = QString( "FETCH FORWARD %1 FROM %2" ).arg( mFeatureQueueSize ).arg( mCursorName );
QgsDebugMsgLevel( QString( "fetching %1 features." ).arg( mFeatureQueueSize ), 4 );
if ( mConn->PQsendQuery( fetch ) == 0 ) // fetch features asynchronously
{
QgsMessageLog::logMessage( QObject::tr( "Fetching from cursor %1 failed\nDatabase error: %2" ).arg( mCursorName, mConn->PQerrorMessage() ), QObject::tr( "PostGIS" ) );
}
QgsPostgresResult queryResult;
for ( ;; )
{
queryResult = mConn->PQgetResult();
if ( !queryResult.result() )
break;
if ( queryResult.PQresultStatus() != PGRES_TUPLES_OK )
{
QgsMessageLog::logMessage( QObject::tr( "Fetching from cursor %1 failed\nDatabase error: %2" ).arg( mCursorName, mConn->PQerrorMessage() ), QObject::tr( "PostGIS" ) );
break;
}
int rows = queryResult.PQntuples();
if ( rows == 0 )
continue;
mLastFetch = rows < mFeatureQueueSize;
for ( int row = 0; row < rows; row++ )
{
mFeatureQueue.enqueue( QgsFeature() );
getFeature( queryResult, row, mFeatureQueue.back() );
} // for each row in queue
}
}
if ( mFeatureQueue.empty() )
{
QgsDebugMsg( QString( "Finished after %1 features" ).arg( mFetched ) );
close();
mSource->mShared->ensureFeaturesCountedAtLeast( mFetched );
return false;
}
feature = mFeatureQueue.dequeue();
mFetched++;
feature.setValid( true );
feature.setFields( mSource->mFields ); // allow name-based attribute lookups
return true;
}
bool QgsPostgresFeatureIterator::nextFeatureFilterExpression( QgsFeature& f )
{
if ( !mExpressionCompiled )
return QgsAbstractFeatureIterator::nextFeatureFilterExpression( f );
else
return fetchFeature( f );
}
bool QgsPostgresFeatureIterator::prepareSimplification( const QgsSimplifyMethod& simplifyMethod )
{
// setup simplification of geometries to fetch
if ( !( mRequest.flags() & QgsFeatureRequest::NoGeometry ) &&
simplifyMethod.methodType() != QgsSimplifyMethod::NoSimplification &&
!simplifyMethod.forceLocalOptimization() )
{
QgsSimplifyMethod::MethodType methodType = simplifyMethod.methodType();
if ( methodType == QgsSimplifyMethod::OptimizeForRendering || methodType == QgsSimplifyMethod::PreserveTopology )
{
return true;
}
else
{
QgsDebugMsg( QString( "Simplification method type (%1) is not recognised by PostgresFeatureIterator" ).arg( methodType ) );
}
}
return QgsAbstractFeatureIterator::prepareSimplification( simplifyMethod );
}
bool QgsPostgresFeatureIterator::providerCanSimplify( QgsSimplifyMethod::MethodType methodType ) const
{
return methodType == QgsSimplifyMethod::OptimizeForRendering || methodType == QgsSimplifyMethod::PreserveTopology;
}
bool QgsPostgresFeatureIterator::rewind()
{
if ( mClosed )
return false;
// move cursor to first record
mConn->PQexecNR( QString( "move absolute 0 in %1" ).arg( mCursorName ) );
mFeatureQueue.clear();
mFetched = 0;
mLastFetch = false;
return true;
}
bool QgsPostgresFeatureIterator::close()
{
if ( mClosed )
return false;
mConn->closeCursor( mCursorName );
if ( !mIsTransactionConnection )
{
QgsPostgresConnPool::instance()->releaseConnection( mConn );
}
else
{
mConn->unlock();
}
mConn = 0;
while ( !mFeatureQueue.empty() )
{
mFeatureQueue.dequeue();
}
iteratorClosed();
mClosed = true;
return true;
}
///////////////
QString QgsPostgresFeatureIterator::whereClauseRect()
{
QgsRectangle rect = mRequest.filterRect();
if ( mSource->mSpatialColType == sctGeography )
{
rect = QgsRectangle( -180.0, -90.0, 180.0, 90.0 ).intersect( &rect );
}
if ( !rect.isFinite() )
{
QgsMessageLog::logMessage( QObject::tr( "Infinite filter rectangle specified" ), QObject::tr( "PostGIS" ) );
return "false";
}
QString qBox;
if ( mConn->majorVersion() < 2 )
{
qBox = QString( "setsrid('BOX3D(%1)'::box3d,%2)" )
.arg( rect.asWktCoordinates(),
mSource->mRequestedSrid.isEmpty() ? mSource->mDetectedSrid : mSource->mRequestedSrid );
}
else
{
qBox = QString( "st_makeenvelope(%1,%2,%3,%4,%5)" )
.arg( qgsDoubleToString( rect.xMinimum() ),
qgsDoubleToString( rect.yMinimum() ),
qgsDoubleToString( rect.xMaximum() ),
qgsDoubleToString( rect.yMaximum() ),
mSource->mRequestedSrid.isEmpty() ? mSource->mDetectedSrid : mSource->mRequestedSrid );
}
bool castToGeometry = mSource->mSpatialColType == sctGeography ||
mSource->mSpatialColType == sctPcPatch;
QString whereClause = QString( "%1%2 && %3" )
.arg( QgsPostgresConn::quotedIdentifier( mSource->mGeometryColumn ),
castToGeometry ? "::geometry" : "",
qBox );
if ( mRequest.flags() & QgsFeatureRequest::ExactIntersect )
{
QString curveToLineFn; // in postgis < 1.5 the st_curvetoline function does not exist
if ( mConn->majorVersion() >= 2 || ( mConn->majorVersion() == 1 && mConn->minorVersion() >= 5 ) )
curveToLineFn = "st_curvetoline"; // st_ prefix is always used
whereClause += QString( " AND %1(%2(%3%4),%5)" )
.arg( mConn->majorVersion() < 2 ? "intersects" : "st_intersects",
curveToLineFn,
QgsPostgresConn::quotedIdentifier( mSource->mGeometryColumn ),
castToGeometry ? "::geometry" : "",
qBox );
}
if ( !mSource->mRequestedSrid.isEmpty() && ( mSource->mRequestedSrid != mSource->mDetectedSrid || mSource->mRequestedSrid.toInt() == 0 ) )
{
whereClause += QString( " AND %1(%2%3)=%4" )
.arg( mConn->majorVersion() < 2 ? "srid" : "st_srid",
QgsPostgresConn::quotedIdentifier( mSource->mGeometryColumn ),
castToGeometry ? "::geometry" : "",
mSource->mRequestedSrid );
}
if ( mSource->mRequestedGeomType != QGis::WKBUnknown && mSource->mRequestedGeomType != mSource->mDetectedGeomType )
{
whereClause += QString( " AND %1" ).arg( QgsPostgresConn::postgisTypeFilter( mSource->mGeometryColumn, ( QgsWKBTypes::Type )mSource->mRequestedGeomType, castToGeometry ) );
}
return whereClause;
}
bool QgsPostgresFeatureIterator::declareCursor( const QString& whereClause, long limit, bool closeOnFail )
{
mFetchGeometry = !( mRequest.flags() & QgsFeatureRequest::NoGeometry ) && !mSource->mGeometryColumn.isNull();
#if 0
// TODO: check that all field indexes exist
if ( !hasAllFields )
{
rewind();
return false;
}
#endif
QString query( "SELECT " ), delim( "" );
if ( mFetchGeometry )
{
QString geom = QgsPostgresConn::quotedIdentifier( mSource->mGeometryColumn );
if ( mSource->mSpatialColType == sctGeography ||
mSource->mSpatialColType == sctPcPatch )
geom += "::geometry";
if ( mSource->mForce2d )
{
geom = QString( "%1(%2)" )
// Force_2D before 2.0
.arg( mConn->majorVersion() < 2 ? "force_2d"
// ST_Force2D since 2.1.0
: mConn->majorVersion() > 2 || mConn->minorVersion() > 0 ? "st_force2d"
// ST_Force_2D in 2.0.x
: "st_force_2d",
geom );
}
if ( !mRequest.simplifyMethod().forceLocalOptimization() &&
mRequest.simplifyMethod().methodType() != QgsSimplifyMethod::NoSimplification &&
QGis::flatType( QGis::singleType( mSource->mRequestedGeomType != QGis::WKBUnknown
? mSource->mRequestedGeomType
: mSource->mDetectedGeomType ) ) != QGis::WKBPoint )
{
// PostGIS simplification method to use
QString simplifyPostgisMethod;
// Simplify again with st_simplify after first simplification ?
bool postSimplification;
postSimplification = false; // default to false. Set to true only for postgis >= 2.2 when using st_removerepeatedpoints
if ( mRequest.simplifyMethod().methodType() == QgsSimplifyMethod::OptimizeForRendering )
{
// Optimize simplification for rendering
if ( mConn->majorVersion() < 2 )
{
simplifyPostgisMethod = "snaptogrid";
}
else
{
// Default to st_snaptogrid
simplifyPostgisMethod = "st_snaptogrid";
if (( mConn->majorVersion() == 2 && mConn->minorVersion() >= 2 ) ||
mConn->majorVersion() > 2 )
{
// For postgis >= 2.2 Use ST_RemoveRepeatedPoints instead
// Do it only if threshold is <= 1 pixel to avoid holes in adjacent polygons
// We should perhaps use it always for Linestrings, even if threshold > 1 ?
if ( mRequest.simplifyMethod().threshold() <= 1.0 )
{
simplifyPostgisMethod = "st_removerepeatedpoints";
postSimplification = true; // Ask to apply a post-filtering simplification
}
}
}
}
else
{
// preserve topology
if ( mConn->majorVersion() < 2 )
{
simplifyPostgisMethod = "simplifypreservetopology";
}
else
{
simplifyPostgisMethod = "st_simplifypreservetopology";
}
}
QgsDebugMsg(
QString( "PostGIS Server side simplification : threshold %1 pixels - method %2" )
.arg( mRequest.simplifyMethod().threshold() )
.arg( simplifyPostgisMethod )
);
geom = QString( "%1(%2,%3)" )
.arg( simplifyPostgisMethod, geom )
.arg( mRequest.simplifyMethod().tolerance() * 0.8 ); //-> Default factor for the maximum displacement distance for simplification, similar as GeoServer does
// Post-simplification
if ( postSimplification )
{
geom = QString( "st_simplify( %1, %2, true )" )
.arg( geom )
.arg( mRequest.simplifyMethod().tolerance() * 0.7 ); //-> We use a smaller tolerance than pre-filtering to be on the safe side
}
}
geom = QString( "%1(%2,'%3')" )
.arg( mConn->majorVersion() < 2 ? "asbinary" : "st_asbinary",
geom,
QgsPostgresProvider::endianString() );
query += delim + geom;
delim = ',';
}
switch ( mSource->mPrimaryKeyType )
{
case pktOid:
query += delim + "oid";
delim = ',';
break;
case pktTid:
query += delim + "ctid";
delim = ',';
break;
case pktInt:
query += delim + QgsPostgresConn::quotedIdentifier( mSource->mFields.at( mSource->mPrimaryKeyAttrs.at( 0 ) ).name() );
delim = ',';
break;
case pktFidMap:
Q_FOREACH ( int idx, mSource->mPrimaryKeyAttrs )
{
query += delim + mConn->fieldExpression( mSource->mFields.at( idx ) );
delim = ',';
}
break;
case pktUnknown:
QgsDebugMsg( "Cannot declare cursor without primary key." );
return false;
break;
}
bool subsetOfAttributes = mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes;
Q_FOREACH ( int idx, subsetOfAttributes ? mRequest.subsetOfAttributes() : mSource->mFields.allAttributesList() )
{
if ( mSource->mPrimaryKeyAttrs.contains( idx ) )
continue;
query += delim + mConn->fieldExpression( mSource->mFields.at( idx ) );
}
query += " FROM " + mSource->mQuery;
if ( !whereClause.isEmpty() )
query += QString( " WHERE %1" ).arg( whereClause );
if ( limit >= 0 )
query += QString( " LIMIT %1" ).arg( limit );
if ( !mConn->openCursor( mCursorName, query ) )
{
// reloading the fields might help next time around
// TODO how to cleanly force reload of fields? P->loadFields();
if ( closeOnFail )
close();
return false;
}
mLastFetch = false;
return true;
}
bool QgsPostgresFeatureIterator::getFeature( QgsPostgresResult &queryResult, int row, QgsFeature &feature )
{
feature.initAttributes( mSource->mFields.count() );
int col = 0;
if ( mFetchGeometry )
{
int returnedLength = ::PQgetlength( queryResult.result(), row, col );
if ( returnedLength > 0 )
{
unsigned char *featureGeom = new unsigned char[returnedLength + 1];
memcpy( featureGeom, PQgetvalue( queryResult.result(), row, col ), returnedLength );
memset( featureGeom + returnedLength, 0, 1 );
unsigned int wkbType;
memcpy( &wkbType, featureGeom + 1, sizeof( wkbType ) );
QgsWKBTypes::Type newType = QgsPostgresConn::wkbTypeFromOgcWkbType( wkbType );
if (( unsigned int )newType != wkbType )
{
// overwrite type
unsigned int n = newType;
memcpy( featureGeom + 1, &n, sizeof( n ) );
}
// PostGIS stores TIN as a collection of Triangles.
// Since Triangles are not supported, they have to be converted to Polygons
const int nDims = 2 + ( QgsWKBTypes::hasZ( newType ) ? 1 : 0 ) + ( QgsWKBTypes::hasM( newType ) ? 1 : 0 );
if ( wkbType % 1000 == 16 )
{
unsigned int numGeoms;
memcpy( &numGeoms, featureGeom + 5, sizeof( unsigned int ) );
unsigned char *wkb = featureGeom + 9;
for ( unsigned int i = 0; i < numGeoms; ++i )
{
const unsigned int localType = QgsWKBTypes::singleType( newType ); // polygon(Z|M)
memcpy( wkb + 1, &localType, sizeof( localType ) );
// skip endian and type info
wkb += sizeof( unsigned int ) + 1;
// skip coordinates
unsigned int nRings;
memcpy( &nRings, wkb, sizeof( int ) );
wkb += sizeof( int );
for ( unsigned int j = 0; j < nRings; ++j )
{
unsigned int nPoints;
memcpy( &nPoints, wkb, sizeof( int ) );
wkb += sizeof( nPoints ) + sizeof( double ) * nDims * nPoints;
}
}
}
QgsGeometry *g = new QgsGeometry();
g->fromWkb( featureGeom, returnedLength + 1 );
feature.setGeometry( g );
}
else
{
feature.setGeometry( 0 );
}
col++;
}
QgsFeatureId fid = 0;
bool subsetOfAttributes = mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes;
const QgsAttributeList& fetchAttributes = mRequest.subsetOfAttributes();
switch ( mSource->mPrimaryKeyType )
{
case pktOid:
case pktTid:
case pktInt:
fid = mConn->getBinaryInt( queryResult, row, col++ );
if ( mSource->mPrimaryKeyType == pktInt &&
( !subsetOfAttributes || fetchAttributes.contains( mSource->mPrimaryKeyAttrs[0] ) ) )
feature.setAttribute( mSource->mPrimaryKeyAttrs[0], fid );
break;
case pktFidMap:
{
QList<QVariant> primaryKeyVals;
Q_FOREACH ( int idx, mSource->mPrimaryKeyAttrs )
{
const QgsField &fld = mSource->mFields.at( idx );
QVariant v = QgsPostgresProvider::convertValue( fld.type(), queryResult.PQgetvalue( row, col ) );
primaryKeyVals << v;
if ( !subsetOfAttributes || fetchAttributes.contains( idx ) )
feature.setAttribute( idx, v );
col++;
}
fid = mSource->mShared->lookupFid( QVariant( primaryKeyVals ) );
}
break;
case pktUnknown:
Q_ASSERT( !"FAILURE: cannot get feature with unknown primary key" );
return false;
}
feature.setFeatureId( fid );
QgsDebugMsgLevel( QString( "fid=%1" ).arg( fid ), 4 );
// iterate attributes
if ( subsetOfAttributes )
{
Q_FOREACH ( int idx, fetchAttributes )
getFeatureAttribute( idx, queryResult, row, col, feature );
}
else
{
for ( int idx = 0; idx < mSource->mFields.count(); ++idx )
getFeatureAttribute( idx, queryResult, row, col, feature );
}
return true;
}
void QgsPostgresFeatureIterator::getFeatureAttribute( int idx, QgsPostgresResult& queryResult, int row, int& col, QgsFeature& feature )
{
if ( mSource->mPrimaryKeyAttrs.contains( idx ) )
return;
QVariant v = QgsPostgresProvider::convertValue( mSource->mFields.at( idx ).type(), queryResult.PQgetvalue( row, col ) );
feature.setAttribute( idx, v );
col++;
}
// ------------------
QgsPostgresFeatureSource::QgsPostgresFeatureSource( const QgsPostgresProvider* p )
: mConnInfo( p->mUri.connectionInfo() )
, mGeometryColumn( p->mGeometryColumn )
, mFields( p->mAttributeFields )
, mSpatialColType( p->mSpatialColType )
, mRequestedSrid( p->mRequestedSrid )
, mDetectedSrid( p->mDetectedSrid )
, mForce2d( p->mForce2d )
, mRequestedGeomType( p->mRequestedGeomType )
, mDetectedGeomType( p->mDetectedGeomType )
, mPrimaryKeyType( p->mPrimaryKeyType )
, mPrimaryKeyAttrs( p->mPrimaryKeyAttrs )
, mQuery( p->mQuery )
, mShared( p->mShared )
{
mSqlWhereClause = p->filterWhereClause();
if ( mSqlWhereClause.startsWith( " WHERE " ) )
mSqlWhereClause = mSqlWhereClause.mid( 7 );
if ( p->mTransaction )
{
mTransactionConnection = p->mTransaction->connection();
mTransactionConnection->ref();
}
else
{
mTransactionConnection = 0;
}
}
QgsPostgresFeatureSource::~QgsPostgresFeatureSource()
{
if ( mTransactionConnection )
{
mTransactionConnection->unref();
}
}
QgsFeatureIterator QgsPostgresFeatureSource::getFeatures( const QgsFeatureRequest& request )
{
return QgsFeatureIterator( new QgsPostgresFeatureIterator( this, false, request ) );
}