From d96a2748b67a29dfde723ed69acf9fb1238b2d0c Mon Sep 17 00:00:00 2001 From: Matthias Kuhn Date: Wed, 16 Dec 2015 23:17:39 +0100 Subject: [PATCH] Add Order By Clause --- python/core/qgsfeaturerequest.sip | 90 +++++++++++++++ src/core/qgsfeatureiterator.cpp | 186 +++++++++++++++++++++++++++--- src/core/qgsfeatureiterator.h | 30 +++++ src/core/qgsfeaturerequest.cpp | 64 ++++++++++ src/core/qgsfeaturerequest.h | 112 +++++++++++++++++- 5 files changed, 468 insertions(+), 14 deletions(-) diff --git a/python/core/qgsfeaturerequest.sip b/python/core/qgsfeaturerequest.sip index d1bf69df586..c44fe1c5894 100644 --- a/python/core/qgsfeaturerequest.sip +++ b/python/core/qgsfeaturerequest.sip @@ -24,6 +24,63 @@ class QgsFeatureRequest FilterFids //!< Filter using feature IDs }; + /** + * @brief The OrderByClause class represents an order by clause for a QgsFeatureRequest + */ + class OrderByClause + { + public: + /** + * Creates a new OrderByClause for a QgsFeatureRequest + * + * @param expression The expression to use for ordering + * @param ascending If the order should be ascending (1,2,3) or descending (3,2,1) + * If thr order is ascending, by default nulls are last + * If thr order is descending, by default nulls are first + */ + OrderByClause( const QString &expression, bool ascending = true ); + /** + * Creates a new OrderByClause for a QgsFeatureRequest + * + * @param expression The expression to use for ordering + * @param ascending If the order should be ascending (1,2,3) or descending (3,2,1) + * @param nullsfirst If true, NULLS are at the beginning, if false, NULLS are at the end + */ + OrderByClause( const QString &expression, bool ascending, bool nullsfirst ); + + /** + * The expression + * @return the expression + */ + QgsExpression expression() const; + + /** + * Order ascending + * @return If ascending order is requested + */ + bool ascending() const; + + /** + * Set if ascending order is requested + */ + void setAscending( bool ascending ); + + /** + * Set if NULLS should be returned first + * @return if NULLS should be returned first + */ + bool nullsFirst() const; + + /** + * Set if NULLS should be returned first + */ + void setNullsFirst( bool nullsFirst ); + + }; + + /** + * A special attribute that if set matches all attributes + */ static const QString AllAttributes; //! construct a default request: for all features get attributes and geometries @@ -92,6 +149,39 @@ class QgsFeatureRequest */ QgsFeatureRequest& disableFilter(); + /** + * Adds a new OrderByClause + * + * @param expression The expression to use for ordering + * @param ascending If the order should be ascending (1,2,3) or descending (3,2,1) + * If the order is ascending, by default nulls are last + * If the order is descending, by default nulls are first + * + * @added in QGIS 2.14 + */ + + QgsFeatureRequest& addOrderBy( const QString &expression, bool ascending = true ); + /** + * Adds a new OrderByClause, appending it as the least important one. + * + * @param expression The expression to use for ordering + * @param ascending If the order should be ascending (1,2,3) or descending (3,2,1) + * @param nullsfirst If true, NULLS are at the beginning, if false, NULLS are at the end + * + * @added in QGIS 2.14 + */ + QgsFeatureRequest& addOrderBy( const QString &expression, bool ascending, bool nullsfirst ); + + /** + * Return a list of order by clauses specified for this feature request. + */ + QList orderBys() const; + + /** + * Set a list of order by clauses. + */ + void setOrderBys(const QList& orderBys ); + /** Set the maximum number of features to request. * @param limit maximum number of features, or -1 to request all features. * @see limit() diff --git a/src/core/qgsfeatureiterator.cpp b/src/core/qgsfeatureiterator.cpp index a865ac41557..4938884fbf9 100644 --- a/src/core/qgsfeatureiterator.cpp +++ b/src/core/qgsfeatureiterator.cpp @@ -18,6 +18,102 @@ #include "qgsgeometrysimplifier.h" #include "qgssimplifymethod.h" +#include + +class QgsExpressionSorter +{ + public: + QgsExpressionSorter( const QList& preparedOrderBys ) + : mPreparedOrderBys( preparedOrderBys ) + {} + + bool operator()( const QgsIndexedFeature& f1, const QgsIndexedFeature& f2 ) const + { + int i = 0; + Q_FOREACH ( const QgsFeatureRequest::OrderByClause& orderBy, mPreparedOrderBys ) + { + const QVariant& v1 = f1.mIndexes.at( i ); + const QVariant& v2 = f2.mIndexes.at( i ); + ++i; + + // Both NULL: don't care + if ( v1.isNull() && v2.isNull() ) + continue; + + // Check for NULLs first + if ( v1.isNull() != v2.isNull() ) + { + if ( orderBy.nullsFirst() ) + return v1.isNull(); + else + return !v1.isNull(); + } + + // Both values are not NULL + switch ( v1.type() ) + { + case QVariant::Int: + case QVariant::UInt: + case QVariant::LongLong: + case QVariant::ULongLong: + if ( v1.toLongLong() == v2.toLongLong() ) + continue; + if ( orderBy.ascending() ) + return v1.toLongLong() < v2.toLongLong(); + else + return v1.toLongLong() > v2.toLongLong(); + + case QVariant::Double: + if ( v1.toDouble() == v2.toDouble() ) + continue; + if ( orderBy.ascending() ) + return v1.toDouble() < v2.toDouble(); + else + return v1.toDouble() > v2.toDouble(); + + case QVariant::Date: + if ( v1.toDate() == v2.toDate() ) + continue; + if ( orderBy.ascending() ) + return v1.toDate() < v2.toDate(); + else + return v1.toDate() > v2.toDate(); + + case QVariant::DateTime: + if ( v1.toDateTime() == v2.toDateTime() ) + continue; + if ( orderBy.ascending() ) + return v1.toDateTime() < v2.toDateTime(); + else + return v1.toDateTime() > v2.toDateTime(); + + case QVariant::Bool: + if ( v1.toBool() == v2.toBool() ) + continue; + if ( orderBy.ascending() ) + return !v1.toBool(); + else + return v1.toBool(); + + default: + if ( 0 == v1.toString().localeAwareCompare( v2.toString() ) ) + continue; + if ( orderBy.ascending() ) + return v1.toString().localeAwareCompare( v2.toString() ) < 0; + else + return v1.toString().localeAwareCompare( v2.toString() ) > 0; + } + } + + // Equal + return true; + } + + private: + QList mPreparedOrderBys; +}; + + QgsAbstractFeatureIterator::QgsAbstractFeatureIterator( const QgsFeatureRequest& request ) : mRequest( request ) , mClosed( false ) @@ -25,13 +121,13 @@ QgsAbstractFeatureIterator::QgsAbstractFeatureIterator( const QgsFeatureRequest& , mFetchedCount( 0 ) , mGeometrySimplifier( nullptr ) , mLocalSimplification( false ) + , mUseCachedFeatures( false ) { } QgsAbstractFeatureIterator::~QgsAbstractFeatureIterator() { delete mGeometrySimplifier; - mGeometrySimplifier = nullptr; } bool QgsAbstractFeatureIterator::nextFeature( QgsFeature& f ) @@ -42,26 +138,37 @@ bool QgsAbstractFeatureIterator::nextFeature( QgsFeature& f ) return false; } - switch ( mRequest.filterType() ) + if ( mUseCachedFeatures ) { - case QgsFeatureRequest::FilterExpression: - dataOk = nextFeatureFilterExpression( f ); - break; + if ( mFeatureIterator != mCachedFeatures.constEnd() ) + { + f = mFeatureIterator->mFeature; + ++mFeatureIterator; + dataOk = true; + } + } + else + { + switch ( mRequest.filterType() ) + { + case QgsFeatureRequest::FilterExpression: + dataOk = nextFeatureFilterExpression( f ); + break; - case QgsFeatureRequest::FilterFids: - dataOk = nextFeatureFilterFids( f ); - break; + case QgsFeatureRequest::FilterFids: + dataOk = nextFeatureFilterFids( f ); + break; - default: - dataOk = fetchFeature( f ); - break; + default: + dataOk = fetchFeature( f ); + break; + } } // simplify the geometry using the simplifier configured if ( dataOk && mLocalSimplification ) { - const QgsGeometry* geometry = f.constGeometry(); - if ( geometry ) + if ( f.constGeometry() ) simplify( f ); } if ( dataOk ) @@ -101,6 +208,9 @@ void QgsAbstractFeatureIterator::ref() if ( refs == 0 ) { prepareSimplification( mRequest.simplifyMethod() ); + + // Should be called as last preparation step since it possibly will already fetch all features + setupOrderBy( mRequest.orderBys() ); } refs++; } @@ -129,6 +239,50 @@ bool QgsAbstractFeatureIterator::prepareSimplification( const QgsSimplifyMethod& return false; } +void QgsAbstractFeatureIterator::setupOrderBy( const QList& orderBys ) +{ + // Let the provider try using an efficient order by strategy first + if ( !orderBys.isEmpty() && !prepareOrderBy( orderBys ) ) + { + // No success from the provider + + // Prepare the expressions + QList preparedOrderBys( orderBys ); + QList::iterator orderByIt( preparedOrderBys.begin() ); + + QgsExpressionContext* expressionContext( mRequest.expressionContext() ); + do + { + orderByIt->expression().prepare( expressionContext ); + } + while ( ++orderByIt != preparedOrderBys.end() ); + + // Fetch all features + QgsIndexedFeature indexedFeature; + indexedFeature.mIndexes.resize( preparedOrderBys.size() ); + + while ( nextFeature( indexedFeature.mFeature ) ) + { + expressionContext->setFeature( indexedFeature.mFeature ); + int i = 0; + Q_FOREACH ( const QgsFeatureRequest::OrderByClause& orderBy, preparedOrderBys ) + { + indexedFeature.mIndexes.replace( i++, orderBy.expression().evaluate( expressionContext ) ); + } + + // We need all features, to ignore the limit for this pre-fetch + // keep the fetched count at 0. + mFetchedCount = 0; + mCachedFeatures.append( indexedFeature ); + } + + std::sort( mCachedFeatures.begin(), mCachedFeatures.end(), QgsExpressionSorter( preparedOrderBys ) ); + + mFeatureIterator = mCachedFeatures.constBegin(); + mUseCachedFeatures = true; + } +} + bool QgsAbstractFeatureIterator::providerCanSimplify( QgsSimplifyMethod::MethodType methodType ) const { Q_UNUSED( methodType ) @@ -149,6 +303,12 @@ bool QgsAbstractFeatureIterator::simplify( QgsFeature& feature ) return false; } +bool QgsAbstractFeatureIterator::prepareOrderBy( const QList& orderBys ) +{ + Q_UNUSED( orderBys ) + return false; +} + /////// QgsFeatureIterator& QgsFeatureIterator::operator=( const QgsFeatureIterator & other ) diff --git a/src/core/qgsfeatureiterator.h b/src/core/qgsfeatureiterator.h index a74489a9b3a..63af4ecbd35 100644 --- a/src/core/qgsfeatureiterator.h +++ b/src/core/qgsfeatureiterator.h @@ -20,6 +20,14 @@ class QgsAbstractGeometrySimplifier; +// Temporarily used structure to cache order by information +class QgsIndexedFeature +{ + public: + QVector mIndexes; + QgsFeature mFeature; +}; + /** \ingroup core * Internal feature iterator to be implemented within data providers */ @@ -99,11 +107,33 @@ class CORE_EXPORT QgsAbstractFeatureIterator //! this iterator runs local simplification bool mLocalSimplification; + bool mUseCachedFeatures; + QList mCachedFeatures; + QList::ConstIterator mFeatureIterator; + //! returns whether the iterator supports simplify geometries on provider side virtual bool providerCanSimplify( QgsSimplifyMethod::MethodType methodType ) const; //! simplify the specified geometry if it was configured virtual bool simplify( QgsFeature& feature ); + + /** + * Should be overwritten by providers which implement an own order by strategy + * If the own order by strategy is successful, return true, if not, return false + * and a local order by will be triggered instead. + * By default returns false + * + * @note added in QGIS 2.14 + */ + virtual bool prepareOrderBy( const QList& orderBys ); + + /** + * Setup the orderby. Internally calls prepareOrderBy and if false is returned will + * cache all features and order them with local expression evaluation. + * + * @note added in QGIS 2.14 + */ + void setupOrderBy( const QList& orderBys ); }; diff --git a/src/core/qgsfeaturerequest.cpp b/src/core/qgsfeaturerequest.cpp index 34690c307c0..61b47dbab0d 100644 --- a/src/core/qgsfeaturerequest.cpp +++ b/src/core/qgsfeaturerequest.cpp @@ -84,6 +84,7 @@ QgsFeatureRequest& QgsFeatureRequest::operator=( const QgsFeatureRequest & rh ) mAttrs = rh.mAttrs; mSimplifyMethod = rh.mSimplifyMethod; mLimit = rh.mLimit; + mOrderBys = rh.mOrderBys; return *this; } @@ -141,6 +142,28 @@ QgsFeatureRequest &QgsFeatureRequest::setExpressionContext( const QgsExpressionC return *this; } +QgsFeatureRequest& QgsFeatureRequest::addOrderBy( const QString& expression, bool ascending ) +{ + mOrderBys.append( OrderByClause( expression, ascending ) ); + return *this; +} + +QgsFeatureRequest& QgsFeatureRequest::addOrderBy( const QString& expression, bool ascending, bool nullsfirst ) +{ + mOrderBys.append( OrderByClause( expression, ascending, nullsfirst ) ); + return *this; +} + +QList QgsFeatureRequest::orderBys() const +{ + return mOrderBys; +} + +void QgsFeatureRequest::setOrderBys( const QList& orderBys ) +{ + mOrderBys = orderBys; +} + QgsFeatureRequest& QgsFeatureRequest::setLimit( long limit ) { mLimit = limit; @@ -228,6 +251,7 @@ bool QgsFeatureRequest::acceptFeature( const QgsFeature& feature ) return true; } + #include "qgsfeatureiterator.h" #include "qgslogger.h" @@ -252,3 +276,43 @@ void QgsAbstractFeatureSource::iteratorClosed( QgsAbstractFeatureIterator* it ) } + +QgsFeatureRequest::OrderByClause::OrderByClause( const QString& expression, bool ascending ) + : mExpression( expression ) + , mAscending( ascending ) +{ + // postgres behavior: default for ASC: NULLS LAST, default for DESC: NULLS FIRST + mNullsFirst = !ascending; +} + +QgsFeatureRequest::OrderByClause::OrderByClause( const QString& expression, bool ascending, bool nullsfirst ) + : mExpression( expression ) + , mAscending( ascending ) + , mNullsFirst( nullsfirst ) +{ +} + +bool QgsFeatureRequest::OrderByClause::ascending() const +{ + return mAscending; +} + +void QgsFeatureRequest::OrderByClause::setAscending( bool ascending ) +{ + mAscending = ascending; +} + +bool QgsFeatureRequest::OrderByClause::nullsFirst() const +{ + return mNullsFirst; +} + +void QgsFeatureRequest::OrderByClause::setNullsFirst( bool nullsFirst ) +{ + mNullsFirst = nullsFirst; +} + +QgsExpression QgsFeatureRequest::OrderByClause::expression() const +{ + return mExpression; +} diff --git a/src/core/qgsfeaturerequest.h b/src/core/qgsfeaturerequest.h index 37f220e3bfd..b08acff48bf 100644 --- a/src/core/qgsfeaturerequest.h +++ b/src/core/qgsfeaturerequest.h @@ -84,6 +84,83 @@ class CORE_EXPORT QgsFeatureRequest FilterFids //!< Filter using feature IDs }; + /** + * The OrderByClause class represents an order by clause for a QgsFeatureRequest. + * + * It can be a simple field or an expression. Multiple order by clauses can be added to + * a QgsFeatureRequest to fine tune the behavior if a single field or expression is not + * enough to completely specify the required behavior. + * + * If expression compilation is activated in the settings and the expression can be + * translated for the provider in question, it will be evaluated on provider side. + * If one of these two premises does not apply, the ordering will take place locally + * which results in increased memory and CPU usage. + * + * If the ordering is done on strings, the order depends on the system's locale if the + * local fallback implementation is used. The order depends on the server system's locale + * and implementation if ordering is done on the server. + * + * In case the fallback code needs to be used, a limit set on the request will be respected + * for the features returned by the iterator but internally all features will be requested + * from the provider. + * + * @note added in QGIS 2.14 + */ + class OrderByClause + { + public: + /** + * Creates a new OrderByClause for a QgsFeatureRequest + * + * @param expression The expression to use for ordering + * @param ascending If the order should be ascending (1,2,3) or descending (3,2,1) + * If the order is ascending, by default nulls are last + * If the order is descending, by default nulls are first + */ + OrderByClause( const QString &expression, bool ascending = true ); + /** + * Creates a new OrderByClause for a QgsFeatureRequest + * + * @param expression The expression to use for ordering + * @param ascending If the order should be ascending (1,2,3) or descending (3,2,1) + * @param nullsfirst If true, NULLS are at the beginning, if false, NULLS are at the end + */ + OrderByClause( const QString &expression, bool ascending, bool nullsfirst ); + + /** + * The expression + * @return the expression + */ + QgsExpression expression() const; + + /** + * Order ascending + * @return If ascending order is requested + */ + bool ascending() const; + + /** + * Set if ascending order is requested + */ + void setAscending( bool ascending ); + + /** + * Set if NULLS should be returned first + * @return if NULLS should be returned first + */ + bool nullsFirst() const; + + /** + * Set if NULLS should be returned first + */ + void setNullsFirst( bool nullsFirst ); + + private: + QgsExpression mExpression; + bool mAscending; + bool mNullsFirst; + }; + /** * A special attribute that if set matches all attributes */ @@ -175,6 +252,39 @@ class CORE_EXPORT QgsFeatureRequest */ QgsFeatureRequest& disableFilter() { mFilter = FilterNone; return *this; } + /** + * Adds a new OrderByClause, appending it as the least important one. + * + * @param expression The expression to use for ordering + * @param ascending If the order should be ascending (1,2,3) or descending (3,2,1) + * If the order is ascending, by default nulls are last + * If the order is descending, by default nulls are first + * + * @note added in QGIS 2.14 + */ + + QgsFeatureRequest& addOrderBy( const QString &expression, bool ascending = true ); + /** + * Adds a new OrderByClause, appending it as the least important one. + * + * @param expression The expression to use for ordering + * @param ascending If the order should be ascending (1,2,3) or descending (3,2,1) + * @param nullsfirst If true, NULLS are at the beginning, if false, NULLS are at the end + * + * @note added in QGIS 2.14 + */ + QgsFeatureRequest& addOrderBy( const QString &expression, bool ascending, bool nullsfirst ); + + /** + * Return a list of order by clauses specified for this feature request. + */ + QList orderBys() const; + + /** + * Set a list of order by clauses. + */ + void setOrderBys( const QList& orderBys ); + /** Set the maximum number of features to request. * @param limit maximum number of features, or -1 to request all features. * @see limit() @@ -224,7 +334,6 @@ class CORE_EXPORT QgsFeatureRequest // TODO: in future // void setFilterNativeExpression(con QString& expr); // using provider's SQL (if supported) - // void setLimit(int limit); protected: FilterType mFilter; @@ -237,6 +346,7 @@ class CORE_EXPORT QgsFeatureRequest QgsAttributeList mAttrs; QgsSimplifyMethod mSimplifyMethod; long mLimit; + QList mOrderBys; }; Q_DECLARE_OPERATORS_FOR_FLAGS( QgsFeatureRequest::Flags )