sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-04-17 00:04:02 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add a provider for virtual layers

Browse Source
This commit is contained in:
Hugo Mercier 2015-12-15 19:12:35 +02:00
parent 13f4081d07
commit e60712e7cf
24 changed files with 4132 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
python/core/core.sip
View File

@ -141,6 +141,7 @@
%Include qgsvectorlayerfeatureiterator.sip
%Include qgsvisibilitypresetcollection.sip
%Include qgslayerdefinition.sip
%Include qgsvirtuallayerdefinition.sip
%Include auth/qgsauthcertutils.sip
%Include auth/qgsauthconfig.sip

125
python/core/qgsvirtuallayerdefinition.sip Normal file
View File

@ -0,0 +1,125 @@
/**
* Class to manipulate the definition of a virtual layer
*
* It is used to extract parameters from an initial virtual layer definition as well as
* to store the complete, expanded definition once types have been detected.
*/
class QgsVirtualLayerDefinition
{
%TypeHeaderCode
#include <qgsvirtuallayerdefinition.h>
%End
public:
/**
* A SourceLayer is either a reference to a live layer in the registry
* or all the parameters needed to load it (provider key, source, etc.)
*/
class SourceLayer
{
public:
//! Constructor variant to build a live layer reference
SourceLayer( const QString& name, const QString& ref );
//! Constructor variant to build a layer with a provider and a source
SourceLayer( const QString& name, const QString& source, const QString& provider, const QString& encoding );
//! Is it a live layer or not ?
bool isReferenced() const;
//! The reference (id) of the live layer
QString reference() const;
//! Name of the layer
QString name() const;
//! Provider key
QString provider() const;
//! The source url used by the provider to build the layer
QString source() const;
//! Optional encoding for the provider
QString encoding() const;
};
//! Constructor with an optional file path
QgsVirtualLayerDefinition( const QString& filePath = "" );
//! Constructor to build a definition from a QUrl
//! The path part of the URL is extracted as well as the following optional keys:
//! layer_ref=layer_id[:name] represents a live layer referenced by its ID. An optional name can be given
//! layer=provider:source[:name[:encoding]] represents a layer given by its provider key, its source url (URL-encoded).
//! An optional name and encoding can be given
//! geometry=column_name[:type:srid] gives the definition of the geometry column.
//! Type can be either a WKB type code or a string (point, linestring, etc.)
//! srid is an integer
//! uid=column_name is the name of a column with unique integer values.
//! nogeometry is a flag to force the layer to be a non-geometry layer
//! query=sql represents the SQL query. Must be URL-encoded
//! field=column_name:[int|real|text] represents a field with its name and its type
static QgsVirtualLayerDefinition fromUrl( const QUrl& url );
//! Convert the definition into a QUrl
QUrl toUrl() const;
//! Convert into a QString that can be read by the virtual layer provider
QString toString() const;
//! Add a live layer source layer
void addSource( const QString& name, const QString ref );
//! Add a layer with a source, a provider and an encoding
void addSource( const QString& name, const QString source, const QString& provider, const QString& encoding = "" );
//! List of source layers
typedef QList<QgsVirtualLayerDefinition::SourceLayer> SourceLayers;
//! Get access to the source layers
const SourceLayers& sourceLayers() const;
//! Get the SQL query
QString query() const;
//! Set the SQL query
void setQuery( const QString& query );
//! Get the file path. May be empty
QString filePath() const;
//! Set the file path
void setFilePath( const QString& filePath );
//! Get the name of the field with unique identifiers
QString uid() const;
//! Set the name of the field with unique identifiers
void setUid( const QString& uid );
//! Get the name of the geometry field. Empty if no geometry field
QString geometryField() const;
//! Set the name of the geometry field
void setGeometryField( const QString& geometryField );
//! Get the type of the geometry
//! QgsWKBTypes::NoGeometry to hide any geometry
//! QgsWKBTypes::Unknown for unknown types
QgsWKBTypes::Type geometryWkbType() const;
//! Set the type of the geometry
void setGeometryWkbType( QgsWKBTypes::Type t );
//! Get the SRID of the geometry
long geometrySrid() const;
//! Set the SRID of the geometry
void setGeometrySrid( long srid );
//! Get field definitions
const QgsFields& fields() const;
//! Set field definitions
void setFields( const QgsFields& fields );
//! Convenience method to test if a given source layer is part of the definition
bool hasSourceLayer( QString name ) const;
//! Convenience method to test whether the definition has referenced (live) layers
bool hasReferencedLayers() const;
//! Convenient method to test if the geometry is defined (not NoGeometry and not Unknown)
bool hasDefinedGeometry() const;
};

1
src/core/CMakeLists.txt
View File

@ -202,6 +202,7 @@ SET(QGIS_CORE_SRCS
qgsvectorlayerundocommand.cpp
qgsvectorsimplifymethod.cpp
qgsvisibilitypresetcollection.cpp
qgsvirtuallayerdefinition.cpp
qgsxmlutils.cpp
qgsslconnect.cpp
qgslocalec.cpp

250
src/core/qgsvirtuallayerdefinition.cpp Normal file
View File

@ -0,0 +1,250 @@
/***************************************************************************
qgsvirtuallayerdefinition.cpp
begin : December 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include <QUrl>
#include <QRegExp>
#include <QStringList>
#include "qgsvirtuallayerdefinition.h"
QgsVirtualLayerDefinition::QgsVirtualLayerDefinition( const QString& filePath ) :
mFilePath( filePath ),
mGeometryWkbType( QgsWKBTypes::Unknown ),
mGeometrySrid( 0 )
{
}
QgsVirtualLayerDefinition QgsVirtualLayerDefinition::fromUrl( const QUrl& url )
{
QgsVirtualLayerDefinition def;
def.setFilePath( url.path() );
// regexp for column name
const QString columnNameRx( "[a-zA-Z_\x80-\xFF][a-zA-Z0-9_\x80-\xFF]*" );
QgsFields fields;
int layerIdx = 0;
QList<QPair<QString, QString> > items = url.queryItems();
for ( int i = 0; i < items.size(); i++ )
{
QString key = items.at( i ).first;
QString value = items.at( i ).second;
if ( key == "layer_ref" )
{
layerIdx++;
// layer id, with optional layer_name
int pos = value.indexOf( ':' );
QString layerId, vlayerName;
if ( pos == -1 )
{
layerId = value;
vlayerName = QString( "vtab%1" ).arg( layerIdx );
}
else
{
layerId = value.left( pos );
vlayerName = QUrl::fromPercentEncoding( value.mid( pos + 1 ).toUtf8() );
}
// add the layer to the list
def.addSource( vlayerName, layerId );
}
else if ( key == "layer" )
{
layerIdx++;
// syntax: layer=provider:url_encoded_source_URI(:name(:encoding)?)?
int pos = value.indexOf( ':' );
if ( pos != -1 )
{
QString providerKey, source, vlayerName, encoding = "UTF-8";
providerKey = value.left( pos );
int pos2 = value.indexOf( ':', pos + 1 );
if ( pos2 != -1 )
{
source = QUrl::fromPercentEncoding( value.mid( pos + 1, pos2 - pos - 1 ).toUtf8() );
int pos3 = value.indexOf( ':', pos2 + 1 );
if ( pos3 != -1 )
{
vlayerName = QUrl::fromPercentEncoding( value.mid( pos2 + 1, pos3 - pos2 - 1 ).toUtf8() );
encoding = value.mid( pos3 + 1 );
}
else
{
vlayerName = QUrl::fromPercentEncoding( value.mid( pos2 + 1 ).toUtf8() );
}
}
else
{
source = QUrl::fromPercentEncoding( value.mid( pos + 1 ).toUtf8() );
vlayerName = QString( "vtab%1" ).arg( layerIdx );
}
def.addSource( vlayerName, source, providerKey, encoding );
}
}
else if ( key == "geometry" )
{
// geometry field definition, optional
// geometry_column(:wkb_type:srid)?
QRegExp reGeom( "(" + columnNameRx + ")(?::([a-zA-Z0-9]+):(\\d+))?" );
int pos = reGeom.indexIn( value );
if ( pos >= 0 )
{
def.setGeometryField( reGeom.cap( 1 ) );
if ( reGeom.captureCount() > 1 )
{
// not used by the spatialite provider for now ...
QgsWKBTypes::Type wkbType = QgsWKBTypes::parseType( reGeom.cap( 2 ) );
if ( wkbType == QgsWKBTypes::Unknown )
{
wkbType = static_cast<QgsWKBTypes::Type>( reGeom.cap( 2 ).toLong() );
}
def.setGeometryWkbType( wkbType );
def.setGeometrySrid( reGeom.cap( 3 ).toLong() );
}
}
}
else if ( key == "nogeometry" )
{
def.setGeometryWkbType( QgsWKBTypes::NoGeometry );
}
else if ( key == "uid" )
{
def.setUid( value );
}
else if ( key == "query" )
{
// url encoded query
def.setQuery( value );
}
else if ( key == "field" )
{
// field_name:type (int, real, text)
QRegExp reField( "(" + columnNameRx + "):(int|real|text)" );
int pos = reField.indexIn( value );
if ( pos >= 0 )
{
QString fieldName( reField.cap( 1 ) );
QString fieldType( reField.cap( 2 ) );
if ( fieldType == "int" )
{
fields.append( QgsField( fieldName, QVariant::Int, fieldType ) );
}
else if ( fieldType == "real" )
{
fields.append( QgsField( fieldName, QVariant::Double, fieldType ) );
}
if ( fieldType == "text" )
{
fields.append( QgsField( fieldName, QVariant::String, fieldType ) );
}
}
}
}
def.setFields( fields );
return def;
}
QUrl QgsVirtualLayerDefinition::toUrl() const
{
QUrl url;
url.setPath( filePath() );
foreach ( const QgsVirtualLayerDefinition::SourceLayer& l, sourceLayers() )
{
if ( l.isReferenced() )
url.addQueryItem( "layer_ref", QString( "%1:%2" ).arg( l.reference() ).arg( l.name() ) );
else
url.addQueryItem( "layer", QString( "%1:%4:%2:%3" ) // the order is important, since the 4th argument may contain '%2' as well
.arg( l.provider() )
.arg( QString( QUrl::toPercentEncoding( l.name() ) ) )
.arg( l.encoding() )
.arg( QString( QUrl::toPercentEncoding( l.source() ) ) ) );
}
if ( !query().isEmpty() )
{
url.addQueryItem( "query", query() );
}
if ( !uid().isEmpty() )
url.addQueryItem( "uid", uid() );
if ( geometryWkbType() == QgsWKBTypes::NoGeometry )
url.addQueryItem( "nogeometry", "" );
else if ( !geometryField().isEmpty() )
{
if ( hasDefinedGeometry() )
url.addQueryItem( "geometry", QString( "%1:%2:%3" ).arg( geometryField() ). arg( geometryWkbType() ).arg( geometrySrid() ).toUtf8() );
else
url.addQueryItem( "geometry", geometryField() );
}
for ( int i = 0; i < fields().count(); i++ )
{
const QgsField& f = fields()[i];
if ( f.type() == QVariant::Int )
url.addQueryItem( "field", f.name() + ":int" );
else if ( f.type() == QVariant::Double )
url.addQueryItem( "field", f.name() + ":real" );
else if ( f.type() == QVariant::String )
url.addQueryItem( "field", f.name() + ":text" );
}
return url;
}
QString QgsVirtualLayerDefinition::toString() const
{
return QString( toUrl().toEncoded() );
}
void QgsVirtualLayerDefinition::addSource( const QString& name, const QString ref )
{
mSourceLayers.append( SourceLayer( name, ref ) );
}
void QgsVirtualLayerDefinition::addSource( const QString& name, const QString source, const QString& provider, const QString& encoding )
{
mSourceLayers.append( SourceLayer( name, source, provider, encoding ) );
}
bool QgsVirtualLayerDefinition::hasSourceLayer( QString name ) const
{
foreach ( const QgsVirtualLayerDefinition::SourceLayer& l, sourceLayers() )
{
if ( l.name() == name )
{
return true;
}
}
return false;
}
bool QgsVirtualLayerDefinition::hasReferencedLayers() const
{
foreach ( const QgsVirtualLayerDefinition::SourceLayer& l, sourceLayers() )
{
if ( l.isReferenced() )
{
return true;
}
}
return false;
}

166
src/core/qgsvirtuallayerdefinition.h Normal file
View File

@ -0,0 +1,166 @@
/***************************************************************************
qgsvirtuallayerdefinition.h
begin : Feb, 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSVIRTUALLAYERDEFINITION_H
#define QGSVIRTUALLAYERDEFINITION_H
#include <qgsfield.h>
#include <qgis.h>
/**
* Class to manipulate the definition of a virtual layer
*
* It is used to extract parameters from an initial virtual layer definition as well as
* to store the complete, expanded definition once types have been detected.
*/
class CORE_EXPORT QgsVirtualLayerDefinition
{
public:
/**
* A SourceLayer is either a reference to a live layer in the registry
* or all the parameters needed to load it (provider key, source, etc.)
*/
class CORE_EXPORT SourceLayer
{
public:
//! Constructor variant to build a live layer reference
SourceLayer( const QString& name, const QString& ref ) : mName( name ), mRef( ref ) {}
//! Constructor variant to build a layer with a provider and a source
SourceLayer( const QString& name, const QString& source, const QString& provider, const QString& encoding )
: mName( name ), mSource( source ), mProvider( provider ), mEncoding( encoding ) {}
//! Is it a live layer or not ?
bool isReferenced() const { return !mRef.isEmpty(); }
//! The reference (id) of the live layer
QString reference() const { return mRef; }
//! Name of the layer
QString name() const { return mName; }
//! Provider key
QString provider() const { return mProvider; }
//! The source url used by the provider to build the layer
QString source() const { return mSource; }
//! Optional encoding for the provider
QString encoding() const { return mEncoding; }
private:
QString mName;
QString mSource;
QString mProvider;
QString mRef;
QString mEncoding;
};
//! Constructor with an optional file path
QgsVirtualLayerDefinition( const QString& filePath = "" );
//! Constructor to build a definition from a QUrl
//! The path part of the URL is extracted as well as the following optional keys:
//! layer_ref=layer_id[:name] represents a live layer referenced by its ID. An optional name can be given
//! layer=provider:source[:name[:encoding]] represents a layer given by its provider key, its source url (URL-encoded).
//! An optional name and encoding can be given
//! geometry=column_name[:type:srid] gives the definition of the geometry column.
//! Type can be either a WKB type code or a string (point, linestring, etc.)
//! srid is an integer
//! uid=column_name is the name of a column with unique integer values.
//! nogeometry is a flag to force the layer to be a non-geometry layer
//! query=sql represents the SQL query. Must be URL-encoded
//! field=column_name:[int|real|text] represents a field with its name and its type
static QgsVirtualLayerDefinition fromUrl( const QUrl& url );
//! Convert the definition into a QUrl
QUrl toUrl() const;
//! Convert into a QString that can be read by the virtual layer provider
QString toString() const;
//! Add a live layer source layer
void addSource( const QString& name, const QString ref );
//! Add a layer with a source, a provider and an encoding
void addSource( const QString& name, const QString source, const QString& provider, const QString& encoding = "" );
//! List of source layers
typedef QList<SourceLayer> SourceLayers;
//! Get access to the source layers
const SourceLayers& sourceLayers() const { return mSourceLayers; }
//! Get the SQL query
QString query() const { return mQuery; }
//! Set the SQL query
void setQuery( const QString& query ) { mQuery = query; }
//! Get the file path. May be empty
QString filePath() const { return mFilePath; }
//! Set the file path
void setFilePath( const QString& filePath ) { mFilePath = filePath; }
//! Get the name of the field with unique identifiers
QString uid() const { return mUid; }
//! Set the name of the field with unique identifiers
void setUid( const QString& uid ) { mUid = uid; }
//! Get the name of the geometry field. Empty if no geometry field
QString geometryField() const { return mGeometryField; }
//! Set the name of the geometry field
void setGeometryField( const QString& geometryField ) { mGeometryField = geometryField; }
//! Get the type of the geometry
//! QgsWKBTypes::NoGeometry to hide any geometry
//! QgsWKBTypes::Unknown for unknown types
QgsWKBTypes::Type geometryWkbType() const { return mGeometryWkbType; }
//! Set the type of the geometry
void setGeometryWkbType( QgsWKBTypes::Type t ) { mGeometryWkbType = t; }
//! Get the SRID of the geometry
long geometrySrid() const { return mGeometrySrid; }
//! Set the SRID of the geometry
void setGeometrySrid( long srid ) { mGeometrySrid = srid; }
//! Get field definitions
const QgsFields& fields() const { return mFields; }
//! Set field definitions
void setFields( const QgsFields& fields ) { mFields = fields; }
//! Convenience method to test if a given source layer is part of the definition
bool hasSourceLayer( QString name ) const;
//! Convenience method to test whether the definition has referenced (live) layers
bool hasReferencedLayers() const;
//! Convenient method to test if the geometry is defined (not NoGeometry and not Unknown)
bool hasDefinedGeometry() const
{
return geometryWkbType() != QgsWKBTypes::NoGeometry && geometryWkbType() != QgsWKBTypes::Unknown;
}
private:
SourceLayers mSourceLayers;
QString mQuery;
QString mUid;
QString mGeometryField;
QString mFilePath;
QgsFields mFields;
QgsWKBTypes::Type mGeometryWkbType;
long mGeometrySrid;
};
#endif

1
src/providers/CMakeLists.txt
View File

@ -13,6 +13,7 @@ ADD_SUBDIRECTORY(wcs)
ADD_SUBDIRECTORY(gpx)
ADD_SUBDIRECTORY(wfs)
ADD_SUBDIRECTORY(spatialite)
ADD_SUBDIRECTORY(virtual)
IF (WITH_ORACLE)
ADD_SUBDIRECTORY(oracle)

56
src/providers/virtual/CMakeLists.txt Normal file
View File

@ -0,0 +1,56 @@
########################################################
# Files
QT4_WRAP_CPP(vlayer_provider_MOC_SRCS qgsvirtuallayerprovider.h qgsslottofunction.h
)
QT4_WRAP_UI(vlayer_provider_UI_H qgsvirtuallayersourceselectbase.ui qgsembeddedlayerselect.ui)
SET(QGIS_VLAYER_PROVIDER_SRCS
${vlayer_provider_MOC_SRCS}
qgsvirtuallayerprovider.cpp
qgsvirtuallayerfeatureiterator.cpp
qgsvirtuallayerblob.cpp
qgsvirtuallayersqlitemodule.cpp
qgsvirtuallayersqlitehelper.cpp
qgsvirtuallayerqueryparser.cpp
)
ADD_LIBRARY(virtuallayerprovider MODULE
${QGIS_VLAYER_PROVIDER_SRCS}
)
TARGET_LINK_LIBRARIES( virtuallayerprovider
qgis_core
qgis_gui
${QT_QTCORE_LIBRARY}
${QT_QTGUI_LIBRARY}
${SQLITE3_LIBRARY}
${SPATIALITE_LIBRARY}
)
INCLUDE_DIRECTORIES(
.
../../core
../../core/auth
../../core/geometry
)
INCLUDE_DIRECTORIES(SYSTEM
${POSTGRES_INCLUDE_DIR}
${GEOS_INCLUDE_DIR}
${QSCINTILLA_INCLUDE_DIR}
${QCA_INCLUDE_DIR}
)
INCLUDE_DIRECTORIES(
../../core
../../gui
../../gui/auth
../../ui
${CMAKE_CURRENT_BINARY_DIR}/../../ui
)
INSTALL(TARGETS virtuallayerprovider
RUNTIME DESTINATION ${QGIS_PLUGIN_DIR}
LIBRARY DESTINATION ${QGIS_PLUGIN_DIR}
)

25
src/providers/virtual/qgsslottofunction.h Normal file
View File

@ -0,0 +1,25 @@
#ifndef QGSSLOT_TO_FUNCTION_H
#define QGSSLOT_TO_FUNCTION_H
#include <QObject>
/**
* This is an helper Qt object used by the SQLite virtual layer module
* in order to connect to the deletion signal of a vector layer,
* since internal classes of the SQLite module cannot derive from QObject
*/
class QgsSlotToFunction : public QObject
{
Q_OBJECT
public:
QgsSlotToFunction() : mCallback( nullptr ), mArg( nullptr ) {}
QgsSlotToFunction( void ( *callback )( void* ), void* arg ) : mCallback( callback ), mArg( arg ) {}
private slots:
void onSignal() { if ( mCallback ) mCallback( mArg ); }
private:
void ( *mCallback )( void* );
void* mArg;
};
#endif

220
src/providers/virtual/qgsvirtuallayerblob.cpp Normal file
View File

@ -0,0 +1,220 @@
/***************************************************************************
qgsvirtuallayerblob.cpp : Functions to manipulate Spatialite geometry blobs
begin : Nov 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include "qgsvirtuallayerblob.h"
#include <string.h>
SpatialiteBlobHeader::SpatialiteBlobHeader() :
start( 0x00 ), endianness( 0x01 ), end( 0x7C )
{}
void SpatialiteBlobHeader::readFrom( const char* p )
{
// we cannot use directly memcpy( this, p, sizeof(this) ),
// since there may be padding between struct members
memcpy( &start, p, 1 ); p++;
memcpy( &endianness, p, 1 ); p++;
memcpy( &srid, p, 4 ); p += 4;
memcpy( &mbrMinX, p, 8 ); p += 8;
memcpy( &mbrMinY, p, 8 ); p += 8;
memcpy( &mbrMaxX, p, 8 ); p += 8;
memcpy( &mbrMaxY, p, 8 ); p += 8;
memcpy( &end, p, 1 );
}
void SpatialiteBlobHeader::writeTo( char* p ) const
{
// we cannot use directly memcpy( this, p, sizeof(this) ),
// since there may be padding between struct members
memcpy( p, &start, 1 ); p++;
memcpy( p, &endianness, 1 ); p++;
memcpy( p, &srid, 4 ); p += 4;
memcpy( p, &mbrMinX, 8 ); p += 8;
memcpy( p, &mbrMinY, 8 ); p += 8;
memcpy( p, &mbrMaxX, 8 ); p += 8;
memcpy( p, &mbrMaxY, 8 ); p += 8;
memcpy( p, &end, 1 );
}
//
// Convert a QgsGeometry into a Spatialite geometry BLOB
void qgsGeometryToSpatialiteBlob( const QgsGeometry& geom, int32_t srid, char *&blob, size_t& size )
{
const size_t header_len = SpatialiteBlobHeader::length;
const size_t wkb_size = geom.wkbSize();
size = header_len + wkb_size;
blob = new char[size];
char* p = blob;
// write the header
SpatialiteBlobHeader pHeader;
QgsRectangle bbox = const_cast<QgsGeometry&>( geom ).boundingBox(); // boundingBox should be const
pHeader.srid = srid;
pHeader.mbrMinX = bbox.xMinimum();
pHeader.mbrMinY = bbox.yMinimum();
pHeader.mbrMaxX = bbox.xMaximum();
pHeader.mbrMaxY = bbox.yMaximum();
pHeader.writeTo( blob );
p += header_len;
// wkb of the geometry is
// name size value
// endianness 1 01
// type 4 int
// blob geometry = header + wkb[1:] + 'end'
// copy wkb
const unsigned char* wkb = geom.asWkb();
memcpy( p, wkb + 1, wkb_size - 1 );
p += wkb_size - 1;
// end marker
*p = 0xFE;
}
//
// Return the bouding box of a spatialite geometry blob
QgsRectangle spatialiteBlobBbox( const char* blob, size_t size )
{
Q_UNUSED( size );
SpatialiteBlobHeader h;
h.readFrom( blob );
return QgsRectangle( h.mbrMinX, h.mbrMinY, h.mbrMaxX, h.mbrMaxY );
}
void copySpatialiteSingleWkbToQgsGeometry( QgsWKBTypes::Type type, const char* iwkb, char* owkb, uint32_t& osize )
{
int n_dims = QgsWKBTypes::coordDimensions( type );
switch ( QgsWKBTypes::flatType( type ) )
{
case QgsWKBTypes::Point:
memcpy( owkb, iwkb, n_dims*8 );
iwkb += n_dims * 8;
iwkb += n_dims * 8;
osize = n_dims * 8;
break;
case QgsWKBTypes::LineString:
{
uint32_t n_points = *( uint32_t* )iwkb;
memcpy( owkb, iwkb, 4 );
iwkb += 4; owkb += 4;
for ( uint32_t i = 0; i < n_points; i++ )
{
memcpy( owkb, iwkb, n_dims*8 );
iwkb += n_dims * 8;
owkb += n_dims * 8;
}
osize += n_dims * 8 * n_points + 4;
break;
}
case QgsWKBTypes::Polygon:
{
uint32_t n_rings = *( uint32_t* )iwkb;
memcpy( owkb, iwkb, 4 );
iwkb += 4; owkb += 4;
osize = 4;
for ( uint32_t i = 0; i < n_rings; i++ )
{
uint32_t n_points = *( uint32_t* )iwkb;
memcpy( owkb, iwkb, 4 );
iwkb += 4; owkb += 4;
osize += 4;
for ( uint32_t j = 0; j < n_points; j++ )
{
memcpy( owkb, iwkb, n_dims*8 );
iwkb += n_dims * 8;
owkb += n_dims * 8;
osize += n_dims * 8;
}
}
break;
}
default:
break;
}
}
//
// copy the spatialite blob to wkb for qgsgeometry
// the only difference is
// each spatialite sub geometry begins with the byte 0x69 (ENTITY)
// which should be converted to an endianness code
void copySpatialiteCollectionWkbToQgsGeometry( const char* iwkb, char* owkb, uint32_t& osize, int endianness )
{
// copy first byte + type
memcpy( owkb, iwkb, 5 );
// replace 0x69 by the endianness
owkb[0] = endianness;
QgsWKBTypes::Type type = static_cast<QgsWKBTypes::Type>( *( uint32_t* )( iwkb + 1 ) );
if ( QgsWKBTypes::isMultiType( type ) )
{
// multi type
uint32_t n_elements = *( uint32_t* )( iwkb + 5 );
memcpy( owkb + 5, iwkb + 5, 4 );
uint32_t p = 0;
for ( uint32_t i = 0; i < n_elements; i++ )
{
uint32_t rsize = 0;
copySpatialiteCollectionWkbToQgsGeometry( iwkb + 9 + p, owkb + 9 + p, rsize, endianness );
p += rsize;
}
osize = p + 9;
}
else
{
osize = 0;
copySpatialiteSingleWkbToQgsGeometry( type, iwkb + 5, owkb + 5, osize );
osize += 5;
}
}
QgsGeometry spatialiteBlobToQgsGeometry( const char* blob, size_t size )
{
const size_t header_size = SpatialiteBlobHeader::length;
const size_t wkb_size = size - header_size;
char* wkb = new char[wkb_size];
uint32_t osize = 0;
copySpatialiteCollectionWkbToQgsGeometry( blob + header_size - 1, wkb, osize, /*endianness*/blob[1] );
QgsGeometry geom;
geom.fromWkb(( unsigned char* )wkb, wkb_size );
return geom;
}
QPair<QgsWKBTypes::Type, long> spatialiteBlobGeometryType( const char* blob, size_t size )
{
if ( size < SpatialiteBlobHeader::length + 4 ) // the header + the type on 4 bytes
{
return qMakePair( QgsWKBTypes::NoGeometry, long( 0 ) );
}
uint32_t srid = *( uint32_t* )( blob + 2 );
uint32_t type = *( uint32_t* )( blob + SpatialiteBlobHeader::length );
return qMakePair( static_cast<QgsWKBTypes::Type>( type ), long( srid ) );
}

71
src/providers/virtual/qgsvirtuallayerblob.h Normal file
View File

@ -0,0 +1,71 @@
/***************************************************************************
qgsvirtuallayerblob.h : Functions to manipulate Spatialite geometry blobs
begin : Nov 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSVIRTUALLAYER_BLOB_H
#define QGSVIRTUALLAYER_BLOB_H
#include <stdint.h>
#include <qgsgeometry.h>
// BLOB header
// name size value
// start 1 00
// endian 1 01
// srid 4 int
// mbr_min_x 8 double
// mbr_min_y 8 double
// mbr_max_x 8 double
// mbr_max_y 8 double
// mbr_end 1 7C
struct SpatialiteBlobHeader
{
unsigned char start;
unsigned char endianness;
uint32_t srid;
double mbrMinX;
double mbrMinY;
double mbrMaxX;
double mbrMaxY;
unsigned char end;
SpatialiteBlobHeader();
static const size_t length = 39;
void readFrom( const char* p );
void writeTo( char* p ) const;
};
//!
//! Convert a QgsGeometry into a Spatialite geometry BLOB
//! The blob will be allocated and must be handled by the caller
void qgsGeometryToSpatialiteBlob( const QgsGeometry& geom, int32_t srid, char *&blob, size_t& size );
//!
//! Return the bouding box of a spatialite geometry blob
QgsRectangle spatialiteBlobBbox( const char* blob, size_t size );
//!
//! Convert a Spatialite geometry BLOB to a QgsGeometry
QgsGeometry spatialiteBlobToQgsGeometry( const char* blob, size_t size );
//!
//! Get geometry type and srid from a spatialite geometry blob
QPair<QgsWKBTypes::Type, long> spatialiteBlobGeometryType( const char* blob, size_t size );
#endif

236
src/providers/virtual/qgsvirtuallayerfeatureiterator.cpp Normal file
View File

@ -0,0 +1,236 @@
/***************************************************************************
qgsvirtuallayerfeatureiterator.cpp
Feature iterator for the virtual layer provider
begin : Nov 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include <qgsvirtuallayerfeatureiterator.h>
#include <qgsmessagelog.h>
#include <qgsgeometry.h>
#include <stdexcept>
#include "qgsvirtuallayerblob.h"
static QString quotedColumn( QString name )
{
return "\"" + name.replace( "\"", "\"\"" ) + "\"";
}
QgsVirtualLayerFeatureIterator::QgsVirtualLayerFeatureIterator( QgsVirtualLayerFeatureSource* source, bool ownSource, const QgsFeatureRequest& request )
: QgsAbstractFeatureIteratorFromSource<QgsVirtualLayerFeatureSource>( source, ownSource, request )
{
try
{
mSqlite = mSource->provider()->mSqlite.get();
mDefinition = mSource->provider()->mDefinition;
QString tableName = mSource->provider()->mTableName;
QStringList wheres;
QString subset = mSource->provider()->mSubset;
if ( !subset.isNull() )
{
wheres << subset;
}
if ( mDefinition.hasDefinedGeometry() && !request.filterRect().isNull() )
{
bool do_exact = request.flags() & QgsFeatureRequest::ExactIntersect;
QgsRectangle rect( request.filterRect() );
QString mbr = QString( "%1,%2,%3,%4" ).arg( rect.xMinimum() ).arg( rect.yMinimum() ).arg( rect.xMaximum() ).arg( rect.yMaximum() );
wheres << quotedColumn( mDefinition.geometryField() ) + " is not null";
wheres << QString( "%1Intersects(%2,BuildMbr(%3))" )
.arg( do_exact ? "" : "Mbr" )
.arg( quotedColumn( mDefinition.geometryField() ) )
.arg( mbr );
}
else if ( !mDefinition.uid().isNull() && request.filterType() == QgsFeatureRequest::FilterFid )
{
wheres << QString( "%1=%2" )
.arg( quotedColumn( mDefinition.uid() ) )
.arg( request.filterFid() );
}
else if ( !mDefinition.uid().isNull() && request.filterType() == QgsFeatureRequest::FilterFids )
{
QString values = quotedColumn( mDefinition.uid() ) + " IN (";
bool first = true;
foreach ( auto& v, request.filterFids() )
{
if ( !first )
{
values += ",";
}
first = false;
values += QString::number( v );
}
values += ")";
wheres << values;
}
mFields = mSource->provider()->fields();
if ( request.flags() & QgsFeatureRequest::SubsetOfAttributes )
{
// copy only selected fields
foreach ( int idx, request.subsetOfAttributes() )
{
mAttributes << idx;
}
}
else
{
mAttributes = mFields.allAttributesList();
}
QString columns;
{
// the first column is always the uid (or 0)
if ( !mDefinition.uid().isNull() )
{
columns = quotedColumn( mDefinition.uid() );
}
else
{
columns = "0";
}
foreach ( int i, mAttributes )
{
columns += ",";
QString cname = mFields.at( i ).name().toLower();
columns += quotedColumn( cname );
}
}
// the last column is the geometry, if any
if ( !( request.flags() & QgsFeatureRequest::NoGeometry ) && !mDefinition.geometryField().isNull() && mDefinition.geometryField() != "*no*" )
{
columns += "," + quotedColumn( mDefinition.geometryField() );
}
mSqlQuery = "SELECT " + columns + " FROM " + tableName;
if ( !wheres.isEmpty() )
{
mSqlQuery += " WHERE " + wheres.join( " AND " );
}
mQuery.reset( new Sqlite::Query( mSqlite, mSqlQuery ) );
mFid = 0;
}
catch ( std::runtime_error& e )
{
QgsMessageLog::logMessage( e.what(), QObject::tr( "VLayer" ) );
close();
}
}
QgsVirtualLayerFeatureIterator::~QgsVirtualLayerFeatureIterator()
{
close();
}
bool QgsVirtualLayerFeatureIterator::rewind()
{
if ( mClosed )
{
return false;
}
mQuery->reset();
return true;
}
bool QgsVirtualLayerFeatureIterator::close()
{
if ( mClosed )
{
return false;
}
// this call is absolutely needed
iteratorClosed();
mClosed = true;
return true;
}
bool QgsVirtualLayerFeatureIterator::fetchFeature( QgsFeature& feature )
{
if ( mClosed )
{
return false;
}
if ( mQuery->step() != SQLITE_ROW )
{
return false;
}
feature.setFields( mFields, /* init */ true );
if ( mDefinition.uid().isNull() )
{
// no id column => autoincrement
feature.setFeatureId( mFid++ );
}
else
{
// first column: uid
feature.setFeatureId( mQuery->columnInt64( 0 ) );
}
int n = mQuery->columnCount();
int i = 0;
foreach ( int idx, mAttributes )
{
int type = mQuery->columnType( i + 1 );
switch ( type )
{
case SQLITE_INTEGER:
feature.setAttribute( idx, mQuery->columnInt64( i + 1 ) );
break;
case SQLITE_FLOAT:
feature.setAttribute( idx, mQuery->columnDouble( i + 1 ) );
break;
case SQLITE_TEXT:
default:
feature.setAttribute( idx, mQuery->columnText( i + 1 ) );
break;
};
i++;
}
if ( n > mAttributes.size() + 1 )
{
// geometry field
QByteArray blob( mQuery->columnBlob( n - 1 ) );
if ( blob.size() > 0 )
{
feature.setGeometry( spatialiteBlobToQgsGeometry( blob.constData(), blob.size() ) );
}
}
return true;
}
QgsVirtualLayerFeatureSource::QgsVirtualLayerFeatureSource( const QgsVirtualLayerProvider* p ) :
mProvider( p )
{
}
QgsVirtualLayerFeatureSource::~QgsVirtualLayerFeatureSource()
{
}
QgsFeatureIterator QgsVirtualLayerFeatureSource::getFeatures( const QgsFeatureRequest& request )
{
return QgsFeatureIterator( new QgsVirtualLayerFeatureIterator( this, false, request ) );
}

71
src/providers/virtual/qgsvirtuallayerfeatureiterator.h Normal file
View File

@ -0,0 +1,71 @@
/***************************************************************************
qgsvirtuallayerfeatureiterator.h
Feature iterator for the virtual layer provider
begin : Feb 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSVIRTUALLAYER_FEATURE_ITERATOR_H
#define QGSVIRTUALLAYER_FEATURE_ITERATOR_H
#include <qgsvirtuallayerprovider.h>
#include <qgsfeatureiterator.h>
class QgsVirtualLayerFeatureSource : public QgsAbstractFeatureSource
{
public:
QgsVirtualLayerFeatureSource( const QgsVirtualLayerProvider* p );
~QgsVirtualLayerFeatureSource();
virtual QgsFeatureIterator getFeatures( const QgsFeatureRequest& request ) override;
const QgsVirtualLayerProvider* provider() const { return mProvider; }
private:
const QgsVirtualLayerProvider* mProvider;
};
class QgsVirtualLayerFeatureIterator : public QgsAbstractFeatureIteratorFromSource<QgsVirtualLayerFeatureSource>
{
public:
QgsVirtualLayerFeatureIterator( QgsVirtualLayerFeatureSource* source, bool ownSource, const QgsFeatureRequest& request );
~QgsVirtualLayerFeatureIterator();
//! reset the iterator to the starting position
virtual bool rewind() override;
//! end of iterating: free the resources / lock
virtual bool close() override;
protected:
//! fetch next feature, return true on success
virtual bool fetchFeature( QgsFeature& feature ) override;
QScopedPointer<Sqlite::Query> mQuery;
QgsFeatureId mFid;
QString mPath;
sqlite3* mSqlite;
QgsVirtualLayerDefinition mDefinition;
QgsFields mFields;
QString mSqlQuery;
// Index of the id column, -1 if none
int mUidColumn;
QgsAttributeList mAttributes;
};
#endif

636
src/providers/virtual/qgsvirtuallayerprovider.cpp Normal file
View File

@ -0,0 +1,636 @@
/***************************************************************************
qgsvirtuallayerprovider.cpp Virtual layer data provider
begin : Jan, 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
extern "C"
{
#include <sqlite3.h>
#include <spatialite.h>
}
#include <QUrl>
#include <stdexcept>
#include <qgsvirtuallayerprovider.h>
#include <qgsvirtuallayerdefinition.h>
#include <qgsvirtuallayerfeatureiterator.h>
#include <qgsvectorlayer.h>
#include <qgsmaplayerregistry.h>
#include <qgsdatasourceuri.h>
#include "qgsvirtuallayerprovider.h"
#include "qgsvirtuallayersqlitemodule.h"
#include "qgsvirtuallayerqueryparser.h"
const QString VIRTUAL_LAYER_KEY = "virtual";
const QString VIRTUAL_LAYER_DESCRIPTION = "Virtual layer data provider";
const QString VIRTUAL_LAYER_QUERY_VIEW = "_query";
static QString quotedColumn( QString name )
{
return "\"" + name.replace( "\"", "\"\"" ) + "\"";
}
#define PROVIDER_ERROR( msg ) do { mError = QgsError( msg, VIRTUAL_LAYER_KEY ); QgsDebugMsg( msg ); } while(0)
QgsVirtualLayerProvider::QgsVirtualLayerProvider( QString const &uri )
: QgsVectorDataProvider( uri ),
mValid( true ),
mCachedStatistics( false )
{
mError.clear();
QUrl url = QUrl::fromEncoded( uri.toUtf8() );
if ( !url.isValid() )
{
mValid = false;
PROVIDER_ERROR( "Malformed URL" );
return;
}
// xxxxx = open a virtual layer
// xxxxx?key=value&key=value = create a virtual layer
// ?key=value = create a temporary virtual layer
// read url
try
{
mDefinition = QgsVirtualLayerDefinition::fromUrl( url );
if ( mDefinition.sourceLayers().empty() && !mDefinition.filePath().isEmpty() && mDefinition.query().isEmpty() )
{
// open the file
mValid = openIt();
}
else
{
// create the file
mValid = createIt();
}
}
catch ( std::runtime_error& e )
{
mValid = false;
PROVIDER_ERROR( e.what() );
return;
}
if ( mDefinition.geometrySrid() != -1 )
{
mCrs = QgsCoordinateReferenceSystem( mDefinition.geometrySrid() );
}
}
bool QgsVirtualLayerProvider::loadSourceLayers()
{
foreach ( const QgsVirtualLayerDefinition::SourceLayer& layer, mDefinition.sourceLayers() )
{
if ( layer.isReferenced() )
{
QgsMapLayer *l = QgsMapLayerRegistry::instance()->mapLayer( layer.reference() );
if ( l == 0 )
{
PROVIDER_ERROR( QString( "Cannot find layer %1" ).arg( layer.reference() ) );
return false;
}
if ( l->type() != QgsMapLayer::VectorLayer )
{
PROVIDER_ERROR( QString( "Layer %1 is not a vector layer" ).arg( layer.reference() ) );
return false;
}
// add the layer to the list
QgsVectorLayer* vl = static_cast<QgsVectorLayer*>( l );
mLayers << SourceLayer( vl, layer.name() );
// connect to modification signals to invalidate statistics
connect( vl, SIGNAL( featureAdded( QgsFeatureId ) ), this, SLOT( invalidateStatistics() ) );
connect( vl, SIGNAL( featureDeleted( QgsFeatureId ) ), this, SLOT( invalidateStatistics() ) );
connect( vl, SIGNAL( geometryChanged( QgsFeatureId, QgsGeometry& ) ), this, SLOT( invalidateStatistics() ) );
}
else
{
mLayers << SourceLayer( layer.provider(), layer.source(), layer.name(), layer.encoding() );
}
}
return true;
}
bool QgsVirtualLayerProvider::openIt()
{
spatialite_init( 0 );
mPath = mDefinition.filePath();
try
{
QgsScopedSqlite p( mPath );
mSqlite = p;
}
catch ( std::runtime_error& e )
{
PROVIDER_ERROR( e.what() );
return false;
}
{
Sqlite::Query q( mSqlite.get(), "SELECT name FROM sqlite_master WHERE name='_meta'" );
if ( q.step() != SQLITE_ROW )
{
PROVIDER_ERROR( "No metadata tables !" );
return false;
}
}
// look for the correct version and the stored url
{
Sqlite::Query q( mSqlite.get(), "SELECT version, url FROM _meta" );
int version = 0;
if ( q.step() == SQLITE_ROW )
{
version = q.columnInt( 0 );
if ( version != VIRTUAL_LAYER_VERSION )
{
PROVIDER_ERROR( "Wrong virtual layer version !" );
return false;
}
mDefinition = QgsVirtualLayerDefinition::fromUrl( QUrl( q.columnText( 1 ) ) );
}
}
// overwrite the uri part of the definition
mDefinition.setFilePath( mPath );
// load source layers
if ( !loadSourceLayers() )
{
return false;
}
/* only one table */
if ( mDefinition.query().isEmpty() )
{
mTableName = mLayers[0].name;
}
else
{
mTableName = VIRTUAL_LAYER_QUERY_VIEW;
}
return true;
}
bool QgsVirtualLayerProvider::createIt()
{
using namespace QgsVirtualLayerQueryParser;
// consistency check
if ( mDefinition.sourceLayers().size() > 1 && mDefinition.query().isEmpty() )
{
PROVIDER_ERROR( QString( "Don't know how to join layers, please specify a query" ) );
return false;
}
if ( mDefinition.sourceLayers().empty() && mDefinition.filePath().isEmpty() && mDefinition.query().isEmpty() )
{
PROVIDER_ERROR( QString( "Please specify at least one source layer or a query" ) );
return false;
}
if ( !mDefinition.filePath().isEmpty() && mDefinition.hasReferencedLayers() )
{
PROVIDER_ERROR( QString( "Cannot store referenced layers" ) );
return false;
}
QList<ColumnDef> fields, gFields;
QMap<QString, TableDef> refTables;
if ( !mDefinition.query().isEmpty() )
{
QStringList tables = referencedTables( mDefinition.query() );
foreach ( const QString& tname, tables )
{
// is it in source layers ?
if ( mDefinition.hasSourceLayer( tname ) )
{
continue;
}
// is it in loaded layers ?
bool found = false;
foreach ( const QgsMapLayer* l, QgsMapLayerRegistry::instance()->mapLayers() )
{
if ( l->type() != QgsMapLayer::VectorLayer )
continue;
const QgsVectorLayer* vl = static_cast<const QgsVectorLayer*>( l );
if (( vl->name() == tname ) || ( vl->id() == tname ) )
{
mDefinition.addSource( tname, vl->id() );
found = true;
break;
}
}
if ( !found )
{
PROVIDER_ERROR( QString( "Referenced table %1 in query not found!" ).arg( tname ) );
return false;
}
}
}
QString path;
mPath = mDefinition.filePath();
// use a temporary file if needed
if ( mPath.isEmpty() )
path = ":memory:";
else
path = mPath;
spatialite_init( 0 );
try
{
QgsScopedSqlite sqlite( path );
mSqlite = sqlite;
}
catch ( std::runtime_error& e )
{
PROVIDER_ERROR( e.what() );
return false;
}
resetSqlite();
initVirtualLayerMetadata( mSqlite.get() );
bool noGeometry = mDefinition.geometryWkbType() == QgsWKBTypes::NoGeometry;
// load source layers (and populate mLayers)
if ( !loadSourceLayers() )
{
return false;
}
// now create virtual tables based on layers
for ( int i = 0; i < mLayers.size(); i++ )
{
QgsVectorLayer* vlayer = mLayers.at( i ).layer;
QString vname = mLayers.at( i ).name;
if ( vlayer )
{
QString createStr = QString( "DROP TABLE IF EXISTS \"%1\"; CREATE VIRTUAL TABLE \"%1\" USING QgsVLayer(%2);" ).arg( vname ).arg( vlayer->id() );
Sqlite::Query::exec( mSqlite.get(), createStr );
}
else
{
QString provider = mLayers.at( i ).provider;
// double each single quote
provider.replace( "'", "''" );
QString source = mLayers.at( i ).source;
source.replace( "'", "''" );
QString encoding = mLayers.at( i ).encoding;
QString createStr = QString( "DROP TABLE IF EXISTS \"%1\"; CREATE VIRTUAL TABLE \"%1\" USING QgsVLayer('%2','%4',%3)" )
.arg( vname )
.arg( provider )
.arg( encoding )
.arg( source ); // source must be the last argument here, since it can contains '%x' strings that would be replaced
Sqlite::Query::exec( mSqlite.get(), createStr );
}
}
QgsFields tfields;
QList<QString> geometryFields;
if ( !mDefinition.query().isEmpty() )
{
// look for column types of the query
TableDef columns = columnDefinitionsFromQuery( mSqlite.get(), mDefinition.query() );
for ( int i = 0; i < columns.size(); i++ )
{
ColumnDef& c = columns[i];
if ( c.name().isEmpty() )
{
PROVIDER_ERROR( QString( "Result column #%1 has no name !" ).arg( i + 1 ) );
return false;
}
// then override types by the ones defined in the url
if ( mDefinition.fields().indexFromName( c.name() ) != -1 )
{
c.setScalarType( mDefinition.fields().field( c.name() ).type() );
}
if ( c.isGeometry() )
{
gFields << c;
}
// if the geometry field is not detected as a geometry, move it to the geometry fields
// with the provided type and srid
else if ( mDefinition.hasDefinedGeometry() && c.name() == mDefinition.geometryField() )
{
ColumnDef g;
g.setName( mDefinition.geometryField() );
g.setGeometry( mDefinition.geometryWkbType() );
g.setSrid( mDefinition.geometrySrid() );
gFields << g;
}
// default type: string
else if ( c.scalarType() == QVariant::Invalid )
{
c.setScalarType( QVariant::String );
}
else
{
tfields.append( QgsField( c.name(), c.scalarType() ) );
}
}
// process geometry field
if ( !noGeometry )
{
// no geometry field defined yet, take the first detected
if ( mDefinition.geometryField().isEmpty() )
{
if ( gFields.count() > 0 )
{
mDefinition.setGeometryField( gFields[0].name() );
mDefinition.setGeometryWkbType( gFields[0].wkbType() );
mDefinition.setGeometrySrid( gFields[0].srid() );
}
}
// a geometry field is named, but has no type yet
// look for a detected type
else if ( !mDefinition.hasDefinedGeometry() )
{
bool found = false;
for ( int i = 0; i < gFields.size(); i++ )
{
if ( gFields[i].name() == mDefinition.geometryField() )
{
// override the geometry type
mDefinition.setGeometryWkbType( gFields[i].wkbType() );
mDefinition.setGeometrySrid( gFields[i].srid() );
found = true;
break;
}
}
if ( !found )
{
PROVIDER_ERROR( "Cannot find the specified geometry field !" );
return false;
}
}
if ( !mDefinition.geometryField().isEmpty() && !mDefinition.hasDefinedGeometry() )
{
PROVIDER_ERROR( "Can't deduce the geometry type of the geometry field !" );
return false;
}
}
// save field definitions
mDefinition.setFields( tfields );
mTableName = VIRTUAL_LAYER_QUERY_VIEW;
// create a view
QString viewStr = QString( "DROP VIEW IF EXISTS %1; CREATE VIEW %1 AS %2" )
.arg( VIRTUAL_LAYER_QUERY_VIEW )
.arg( mDefinition.query() );
Sqlite::Query::exec( mSqlite.get(), viewStr );
}
else
{
// no query => implies we must only have one virtual table
mTableName = mLayers[0].name;
TableDef td = tableDefinitionFromVirtualTable( mSqlite.get(), mTableName );
foreach ( const ColumnDef& c, td )
{
if ( !c.isGeometry() )
{
tfields.append( QgsField( c.name(), c.scalarType() ) );
}
else if ( !noGeometry )
{
mDefinition.setGeometryField( "geometry" );
mDefinition.setGeometryWkbType( c.wkbType() );
mDefinition.setGeometrySrid( c.srid() );
}
}
mDefinition.setFields( tfields );
}
// Save the definition back to the sqlite file
{
Sqlite::Query q( mSqlite.get(), "UPDATE _meta SET url=?" );
q.bind( mDefinition.toUrl().toString() );
q.step();
}
return true;
}
QgsVirtualLayerProvider::~QgsVirtualLayerProvider()
{
}
void QgsVirtualLayerProvider::resetSqlite()
{
bool hasSpatialrefsys = false;
{
Sqlite::Query q( mSqlite.get(), "SELECT name FROM sqlite_master WHERE name='spatial_ref_sys'" );
hasSpatialrefsys = q.step() == SQLITE_ROW;
}
QString sql = "DROP TABLE IF EXISTS _meta;";
if ( !hasSpatialrefsys )
{
sql += "SELECT InitSpatialMetadata(1);";
}
Sqlite::Query::exec( mSqlite.get(), sql );
}
QgsAbstractFeatureSource* QgsVirtualLayerProvider::featureSource() const
{
return new QgsVirtualLayerFeatureSource( this );
}
QString QgsVirtualLayerProvider::storageType() const
{
return "No storage per se, view data from other data sources";
}
QgsCoordinateReferenceSystem QgsVirtualLayerProvider::crs()
{
return mCrs;
}
QgsFeatureIterator QgsVirtualLayerProvider::getFeatures( const QgsFeatureRequest& request )
{
return QgsFeatureIterator( new QgsVirtualLayerFeatureIterator( new QgsVirtualLayerFeatureSource( this ), false, request ) );
}
QString QgsVirtualLayerProvider::subsetString()
{
return mSubset;
}
bool QgsVirtualLayerProvider::setSubsetString( const QString& subset, bool updateFeatureCount )
{
mSubset = subset;
if ( updateFeatureCount )
updateStatistics();
return true;
}
QGis::WkbType QgsVirtualLayerProvider::geometryType() const
{
return static_cast<QGis::WkbType>( mDefinition.geometryWkbType() );
}
long QgsVirtualLayerProvider::featureCount() const
{
if ( !mCachedStatistics )
{
updateStatistics();
}
return mFeatureCount;
}
QgsRectangle QgsVirtualLayerProvider::extent()
{
if ( !mCachedStatistics )
{
updateStatistics();
}
return mExtent;
}
void QgsVirtualLayerProvider::updateStatistics() const
{
bool hasGeometry = mDefinition.geometryWkbType() != QgsWKBTypes::NoGeometry;
QString subset = mSubset.isEmpty() ? "" : " WHERE " + mSubset;
QString sql = QString( "SELECT Count(*)%1 FROM %2%3" )
.arg( hasGeometry ? QString( ",Min(MbrMinX(%1)),Min(MbrMinY(%1)),Max(MbrMaxX(%1)),Max(MbrMaxY(%1))" ).arg( quotedColumn( mDefinition.geometryField() ) ) : "" )
.arg( mTableName )
.arg( subset );
Sqlite::Query q( mSqlite.get(), sql );
if ( q.step() == SQLITE_ROW )
{
mFeatureCount = q.columnInt64( 0 );
if ( hasGeometry )
{
double x1, y1, x2, y2;
x1 = q.columnDouble( 1 );
y1 = q.columnDouble( 2 );
x2 = q.columnDouble( 3 );
y2 = q.columnDouble( 4 );
mExtent = QgsRectangle( x1, y1, x2, y2 );
}
mCachedStatistics = true;
}
}
void QgsVirtualLayerProvider::invalidateStatistics()
{
mCachedStatistics = false;
}
const QgsFields & QgsVirtualLayerProvider::fields() const
{
return mDefinition.fields();
}
bool QgsVirtualLayerProvider::isValid()
{
return mValid;
}
int QgsVirtualLayerProvider::capabilities() const
{
if ( !mDefinition.uid().isNull() )
{
return SelectAtId | SelectGeometryAtId;
}
return 0;
}
QString QgsVirtualLayerProvider::name() const
{
return VIRTUAL_LAYER_KEY;
}
QString QgsVirtualLayerProvider::description() const
{
return VIRTUAL_LAYER_DESCRIPTION;
}
QgsAttributeList QgsVirtualLayerProvider::pkAttributeIndexes()
{
if ( !mDefinition.uid().isNull() )
{
const QgsFields& fields = mDefinition.fields();
for ( int i = 0; i < fields.size(); i++ )
{
if ( fields.at( i ).name().toLower() == mDefinition.uid().toLower() )
{
QgsAttributeList l;
l << i;
return l;
}
}
}
return QgsAttributeList();
}
/**
* Class factory to return a pointer to a newly created
* QgsSpatiaLiteProvider object
*/
QGISEXTERN QgsVirtualLayerProvider *classFactory( const QString * uri )
{
return new QgsVirtualLayerProvider( *uri );
}
/** Required key function (used to map the plugin to a data store type)
*/
QGISEXTERN QString providerKey()
{
return VIRTUAL_LAYER_KEY;
}
/**
* Required description function
*/
QGISEXTERN QString description()
{
return VIRTUAL_LAYER_DESCRIPTION;
}
/**
* Required isProvider function. Used to determine if this shared library
* is a data provider plugin
*/
QGISEXTERN bool isProvider()
{
return true;
}
QGISEXTERN void cleanupProvider()
{
}

146
src/providers/virtual/qgsvirtuallayerprovider.h Normal file
View File

@ -0,0 +1,146 @@
/***************************************************************************
qgsvirtuallayerprovider.cpp Virtual layer data provider
begin : Jan, 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSVIRTUAL_LAYER_PROVIDER_H
#define QGSVIRTUAL_LAYER_PROVIDER_H
#include <qgsvectordataprovider.h>
#include "qgscoordinatereferencesystem.h"
#include "qgsvirtuallayerdefinition.h"
#include "qgsvirtuallayersqlitehelper.h"
class QgsVirtualLayerFeatureIterator;
class QgsVirtualLayerProvider: public QgsVectorDataProvider
{
Q_OBJECT
public:
/**
* Constructor of the vector provider
* @param uri uniform resource locator (URI) for a dataset
*/
QgsVirtualLayerProvider( QString const &uri = "" );
/** Destructor */
virtual ~QgsVirtualLayerProvider();
virtual QgsAbstractFeatureSource* featureSource() const override;
/** Returns the permanent storage type for this layer as a friendly name */
virtual QString storageType() const override;
/** Get the QgsCoordinateReferenceSystem for this layer */
virtual QgsCoordinateReferenceSystem crs() override;
/** Access features through an iterator */
virtual QgsFeatureIterator getFeatures( const QgsFeatureRequest& request ) override;
/** Get the feature geometry type */
QGis::WkbType geometryType() const override;
/** Get the number of features in the layer */
long featureCount() const override;
/** Return the extent for this data layer */
virtual QgsRectangle extent() override;
/** Accessor for sql where clause used to limit dataset */
virtual QString subsetString() override;
/** Set the subset string used to create a subset of features in the layer (WHERE clause) */
virtual bool setSubsetString( const QString& subset, bool updateFeatureCount = true ) override;
/** Provider supports setting of subset strings */
virtual bool supportsSubsetString() override { return true; }
/**
* Get the field information for the layer
* @return vector of QgsField objects
*/
const QgsFields & fields() const override;
/** Returns true if layer is valid */
bool isValid() override;
/** Returns a bitmask containing the supported capabilities*/
int capabilities() const override;
/** Return the provider name */
QString name() const override;
/** Return description */
QString description() const override;
/** Return list of indexes of fields that make up the primary key */
QgsAttributeList pkAttributeIndexes() override;
private:
// file on disk
QString mPath;
QgsScopedSqlite mSqlite;
// underlying vector layers
struct SourceLayer
{
SourceLayer(): layer( 0 ) {}
SourceLayer( QgsVectorLayer *l, const QString& n = "" ) : layer( l ), name( n ) {}
SourceLayer( const QString& p, const QString& s, const QString& n, const QString& e = "UTF-8" ) :
layer( 0 ), name( n ), source( s ), provider( p ), encoding( e ) {}
// non-null if it refers to a live layer
QgsVectorLayer* layer;
QString name;
// non-empty if it is an embedded layer
QString source;
QString provider;
QString encoding;
};
typedef QVector<SourceLayer> SourceLayers;
SourceLayers mLayers;
bool mValid;
QString mTableName;
QgsCoordinateReferenceSystem mCrs;
QgsVirtualLayerDefinition mDefinition;
QString mSubset;
void resetSqlite();
mutable bool mCachedStatistics;
mutable qint64 mFeatureCount;
mutable QgsRectangle mExtent;
void updateStatistics() const;
bool openIt();
bool createIt();
bool loadSourceLayers();
friend class QgsVirtualLayerFeatureIterator;
private slots:
void invalidateStatistics();
};
#endif

274
src/providers/virtual/qgsvirtuallayerqueryparser.cpp Normal file
View File

@ -0,0 +1,274 @@
#include "qgsvirtuallayerqueryparser.h"
#include "qgsvirtuallayersqlitehelper.h"
#include "qgsvirtuallayerblob.h"
#include <QRegExp>
namespace QgsVirtualLayerQueryParser
{
QStringList referencedTables( const QString& query )
{
QStringList tables;
//
// open an empty in-memory sqlite database and execute the query
// sqlite will return an error for each missing table
// this way we know the list of tables referenced by the query
QgsScopedSqlite db( ":memory:", /*withExtension=*/ false );
const QString noSuchError = "no such table: ";
while ( true )
{
char *errMsg = 0;
int r = sqlite3_exec( db.get(), query.toLocal8Bit().constData(), NULL, NULL, &errMsg );
QString err = errMsg;
if ( r && err.startsWith( noSuchError ) )
{
QString tableName = err.mid( noSuchError.size() );
tables << tableName;
// create a dummy table to skip this error
QString createStr = QString( "CREATE TABLE \"%1\" (id int)" ).arg( tableName.replace( "\"", "\"\"" ) );
sqlite3_exec( db.get(), createStr.toLocal8Bit().constData(), NULL, NULL, NULL );
}
else
{
// no error, or another error
break;
}
}
return tables;
}
QMap<QString, ColumnDef> columnCommentDefinitions( const QString& query )
{
QMap<QString, ColumnDef> defs;
// look for special comments in SQL
// a column name followed by /*:type*/
QRegExp rx( "([a-zA-Z_\x80-\xFF][a-zA-Z0-9_\x80-\xFF]*)\\s*/\\*:(int|real|text|((?:multi)?(?:point|linestring|polygon)):(\\d+))\\s*\\*/", Qt::CaseInsensitive );
int pos = 0;
while (( pos = rx.indexIn( query, pos ) ) != -1 )
{
QString column = rx.cap( 1 );
QString type = rx.cap( 2 );
ColumnDef def;
def.setName( column );
if ( type == "int" )
def.setScalarType( QVariant::Int );
else if ( type == "real" )
def.setScalarType( QVariant::Double );
else if ( type == "text" )
def.setScalarType( QVariant::String );
else
{
// there should be 2 more captures
def.setGeometry( QgsWKBTypes::parseType( rx.cap( 3 ) ) );
def.setSrid( static_cast<QgsWKBTypes::Type>( rx.cap( 4 ).toLong() ) );
}
defs[column] = def;
pos += rx.matchedLength();
}
return defs;
}
bool isValidColumnName( const QString& columnName )
{
// identifier name with possible accents
static QRegExp columnNameRx( "[a-zA-Z_\x80-\xFF][a-zA-Z0-9_\x80-\xFF]*" );
return columnNameRx.exactMatch( columnName );
}
// set the type of the column type, given its text representation
void setColumnDefType( const QString& columnType, ColumnDef& d )
{
// geometry type
QRegExp geometryTypeRx( "\\(([0-9]+),([0-9]+)\\)" );
// see qgsvirtuallayersqlitemodule for possible declared types
// the type returned by PRAGMA table_info will be either
// the type declared by one of the virtual tables
// or null
if ( columnType == "int" )
d.setScalarType( QVariant::Int );
else if ( columnType == "real" )
d.setScalarType( QVariant::Double );
else if ( columnType == "text" )
d.setScalarType( QVariant::String );
else if ( columnType.startsWith( "geometry" ) )
{
// parse the geometry type and srid
// geometry(type,srid)
int pos = geometryTypeRx.indexIn( columnType, 0 );
if ( pos != -1 )
{
QgsWKBTypes::Type type = static_cast<QgsWKBTypes::Type>( geometryTypeRx.cap( 1 ).toInt() );
long srid = geometryTypeRx.cap( 2 ).toLong();
d.setGeometry( type );
d.setSrid( srid );
}
}
}
ColumnDef geometryDefinitionFromVirtualTable( sqlite3* db, const QString& tableName )
{
ColumnDef d;
Sqlite::Query q( db, QString( "PRAGMA table_info(%1)" ).arg( tableName ) );
while ( q.step() == SQLITE_ROW )
{
QString columnName = q.columnText( 1 );
QString columnType = q.columnText( 2 );
if ( ! columnType.startsWith( "geometry" ) )
continue;
d.setName( columnName );
setColumnDefType( columnType, d );
break;
}
return d;
}
TableDef columnDefinitionsFromQuery( sqlite3* db, const QString& query )
{
// get column types defined by comments
QMap<QString, ColumnDef> definedColumns = columnCommentDefinitions( query );
// create a view to detect column names and types, using PRAGMA table_info
QString viewStr = "CREATE TEMP VIEW _tview AS " + query;
Sqlite::Query::exec( db, viewStr );
QStringList columns;
bool hasInvalidName = false;
QVector<int> undefinedColumns;
TableDef tableDef;
{
Sqlite::Query q( db, "PRAGMA table_info(_tview)" );
int columnNumber = 0;
while ( q.step() == SQLITE_ROW )
{
QString columnName = q.columnText( 1 );
if ( !isValidColumnName( columnName ) )
{
std::cout << "Invalid name: " << columnName.toLocal8Bit().constData() << std::endl;
hasInvalidName = true;
// add an unnamed column
ColumnDef d;
tableDef << d;
break;
}
columns << columnName;
QString columnType = q.columnText( 2 );
// column type defined by comments
if ( definedColumns.contains( columnName ) )
{
tableDef << definedColumns[columnName];
}
else
{
ColumnDef d;
d.setName( columnName );
setColumnDefType( columnType, d );
if ( d.scalarType() == QVariant::Invalid )
{
// else no type is defined
undefinedColumns << columnNumber;
}
tableDef << d;
}
columnNumber++;
}
}
if ( hasInvalidName || undefinedColumns.size() == 0 )
return tableDef;
// get the first row to introspect types
{
QString qs = "SELECT ";
for ( int i = 0; i < undefinedColumns.size(); i++ )
{
qs += columns[undefinedColumns[i]];
if ( i != undefinedColumns.size() - 1 )
qs += ", ";
}
qs += " FROM _tview LIMIT 1";
std::cout << qs.toLocal8Bit().constData() << std::endl;
Sqlite::Query q( db, qs );
if ( q.step() == SQLITE_ROW )
{
for ( int i = 0; i < undefinedColumns.size(); i++ )
{
int colIdx = undefinedColumns[i];
int type = q.columnType( i );
switch ( type )
{
case SQLITE_INTEGER:
tableDef[colIdx].setScalarType( QVariant::Int );
break;
case SQLITE_FLOAT:
tableDef[colIdx].setScalarType( QVariant::Double );
break;
case SQLITE_BLOB:
{
// might be a geometry, parse the type
QByteArray ba( q.columnBlob( i ) );
QPair<QgsWKBTypes::Type, long> p( spatialiteBlobGeometryType( ba.constData(), ba.size() ) );
if ( p.first != QgsWKBTypes::NoGeometry )
{
tableDef[colIdx].setGeometry( p.first );
tableDef[colIdx].setSrid( p.second );
}
else
{
// interpret it as a string
tableDef[colIdx].setScalarType( QVariant::String );
}
}
break;
case SQLITE_TEXT:
default:
tableDef[colIdx].setScalarType( QVariant::String );
break;
};
}
}
}
return tableDef;
}
TableDef tableDefinitionFromVirtualTable( sqlite3* db, const QString& tableName )
{
TableDef td;
Sqlite::Query q( db, QString( "PRAGMA table_info(%1)" ).arg( tableName ) );
while ( q.step() == SQLITE_ROW )
{
ColumnDef d;
QString columnName = q.columnText( 1 );
QString columnType = q.columnText( 2 );
d.setName( columnName );
setColumnDefType( columnType, d );
td << d;
}
return td;
}
} // namespace

70
src/providers/virtual/qgsvirtuallayerqueryparser.h Normal file
View File

@ -0,0 +1,70 @@
#ifndef QGSVIRTUALLAYER_QUERY_PARSER_H
#define QGSVIRTUALLAYER_QUERY_PARSER_H
#include <qgis.h>
#include <qgswkbtypes.h>
#include <qgsvectorlayer.h>
namespace QgsVirtualLayerQueryParser
{
//!
//! Return the list of tables referenced in the SQL query
QStringList referencedTables( const QString& q );
/**
* Type used to define a column
*
* It can hold a name and a type.
* The type can be a 'scalar' type (int, double, string) or a geometry type (WKB) and an SRID
*/
class ColumnDef
{
public:
ColumnDef()
: mType( QVariant::Invalid ), mWkbType( QgsWKBTypes::Unknown ), mSrid( -1 )
{}
ColumnDef( const QString& name, QgsWKBTypes::Type aWkbType, long aSrid )
: mName( name ), mType( QVariant::UserType ), mWkbType( aWkbType ), mSrid( aSrid )
{}
ColumnDef( const QString& name, QVariant::Type aType )
: mName( name ), mType( aType ), mWkbType( QgsWKBTypes::NoGeometry ), mSrid( -1 )
{}
QString name() const { return mName; }
void setName( QString name ) { mName = name; }
bool isGeometry() const { return mType == QVariant::UserType; }
void setGeometry( QgsWKBTypes::Type wkbType ) { mType = QVariant::UserType; mWkbType = wkbType; }
long srid() const { return mSrid; }
void setSrid( long srid ) { mSrid = srid; }
void setScalarType( QVariant::Type t ) { mType = t; mWkbType = QgsWKBTypes::NoGeometry; }
QVariant::Type scalarType() const { return mType; }
QgsWKBTypes::Type wkbType() const { return mWkbType; }
private:
QString mName;
QVariant::Type mType;
QgsWKBTypes::Type mWkbType;
long mSrid;
};
//!
//! Type used by the parser to type a query. It is slightly different from a QgsVirtualLayerDefinition since more than one geometry column can be represented
typedef QList<ColumnDef> TableDef;
//! Get the column names and types that can be deduced from the query, using SQLite introspection
//! Special comments can also be used in the query to type columns
//! Comments should be set after the name of the column and are introduced by "/*:"
//! For instance 'SELECT t+1 /*:int*/ FROM table' will type the column 't' as integer
//! A geometry column can also be set by specifying a type and an SRID
//! For instance 'SELECT t, GeomFromText('POINT(0 0)',4326) as geom /*:point:4326*/
TableDef columnDefinitionsFromQuery( sqlite3* db, const QString& query );
//! Get the column types of a virtual table
TableDef tableDefinitionFromVirtualTable( sqlite3* db, const QString& tableName );
}
#endif

189
src/providers/virtual/qgsvirtuallayersqlitehelper.cpp Normal file
View File

@ -0,0 +1,189 @@
/***************************************************************************
qgsvirtuallayersqlitehelper.cpp
begin : December 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include <QString>
#include <stdexcept>
#include "qgsvirtuallayersqlitehelper.h"
QgsScopedSqlite::QgsScopedSqlite( const QString& path, bool withExtension )
{
if ( withExtension )
{
// register a statically-linked function as extension
// for all future database connection
sqlite3_auto_extension(( void( * )() )qgsvlayer_module_init );
}
int r;
r = sqlite3_open( path.toLocal8Bit().constData(), &db_ );
if ( withExtension )
{
// reset the automatic extensions
sqlite3_reset_auto_extension();
}
if ( r )
{
throw std::runtime_error( sqlite3_errmsg( db_ ) );
}
// enable extended result codes
sqlite3_extended_result_codes( db_, 1 );
}
QgsScopedSqlite::QgsScopedSqlite( QgsScopedSqlite& other )
{
db_ = other.db_;
other.db_ = 0;
}
QgsScopedSqlite& QgsScopedSqlite::operator=( QgsScopedSqlite & other )
{
reset( other.release() );
return *this;
}
QgsScopedSqlite::~QgsScopedSqlite()
{
close_();
}
sqlite3* QgsScopedSqlite::get() const { return db_; }
sqlite3* QgsScopedSqlite::release()
{
sqlite3* pp = db_;
db_ = 0;
return pp;
}
void QgsScopedSqlite::reset( sqlite3* db )
{
close_();
db_ = db;
}
void QgsScopedSqlite::close_()
{
if ( db_ )
sqlite3_close( db_ );
}
namespace Sqlite
{
Query::Query( sqlite3* db, const QString& q ) : db_( db ), nBind_( 1 )
{
QByteArray ba( q.toLocal8Bit() );
int r = sqlite3_prepare_v2( db, ba.constData(), ba.size(), &stmt_, NULL );
if ( r )
{
QString err = QString( "Query preparation error on %1" ).arg( q );
throw std::runtime_error( err.toLocal8Bit().constData() );
}
}
Query::~Query()
{
sqlite3_finalize( stmt_ );
}
int Query::step() { return sqlite3_step( stmt_ ); }
Query& Query::bind( const QString& str, int idx )
{
QByteArray ba( str.toLocal8Bit() );
int r = sqlite3_bind_text( stmt_, idx, ba.constData(), ba.size(), SQLITE_TRANSIENT );
if ( r )
{
throw std::runtime_error( sqlite3_errmsg( db_ ) );
}
return *this;
}
Query& Query::bind( const QString& str )
{
return bind( str, nBind_++ );
}
void Query::exec( sqlite3* db, const QString& sql )
{
char *errMsg = 0;
int r = sqlite3_exec( db, sql.toLocal8Bit().constData(), NULL, NULL, &errMsg );
if ( r )
{
QString err = QString( "Query execution error on %1: %2 - %3" ).arg( sql ).arg( r ).arg( errMsg );
throw std::runtime_error( err.toLocal8Bit().constData() );
}
}
void Query::reset()
{
int r = sqlite3_reset( stmt_ );
if ( r )
{
throw std::runtime_error( sqlite3_errmsg( db_ ) );
}
nBind_ = 1;
}
int Query::columnCount() const
{
return sqlite3_column_count( stmt_ );
}
QString Query::columnName( int i ) const
{
return QString( sqlite3_column_name( stmt_, i ) );
}
int Query::columnType( int i ) const
{
return sqlite3_column_type( stmt_, i );
}
int Query::columnInt( int i ) const
{
return sqlite3_column_int( stmt_, i );
}
qint64 Query::columnInt64( int i ) const
{
return sqlite3_column_int64( stmt_, i );
}
double Query::columnDouble( int i ) const
{
return sqlite3_column_double( stmt_, i );
}
QString Query::columnText( int i ) const
{
int size = sqlite3_column_bytes( stmt_, i );
const char* str = ( const char* )sqlite3_column_text( stmt_, i );
return QString::fromUtf8( str, size );
}
QByteArray Query::columnBlob( int i ) const
{
int size = sqlite3_column_bytes( stmt_, i );
const char* data = ( const char* )sqlite3_column_blob( stmt_, i );
// data is not copied. QByteArray is just here a augmented pointer
return QByteArray::fromRawData( data, size );
}
sqlite3_stmt* Query::stmt() { return stmt_; }
}

94
src/providers/virtual/qgsvirtuallayersqlitehelper.h Normal file
View File

@ -0,0 +1,94 @@
/***************************************************************************
qgsvirtuallayersqlitehelper.h
begin : December 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSVIRTUALLAYER_SQLITE_UTILS_H
#define QGSVIRTUALLAYER_SQLITE_UTILS_H
extern "C"
{
#include <sqlite3.h>
int qgsvlayer_module_init( sqlite3 *db,
char **pzErrMsg,
void * unused /*const sqlite3_api_routines *pApi*/ );
}
// RAII class for sqlite3*
// Similar to std::unique_ptr
class QgsScopedSqlite
{
public:
QgsScopedSqlite() : db_( 0 ) {}
explicit QgsScopedSqlite( const QString& path, bool withExtension = true );
QgsScopedSqlite( QgsScopedSqlite& other );
QgsScopedSqlite& operator=( QgsScopedSqlite& other );
~QgsScopedSqlite();
sqlite3* get() const;
sqlite3* release();
void reset( sqlite3* db );
private:
sqlite3* db_;
void close_();
};
namespace Sqlite
{
struct Query
{
Query( sqlite3* db, const QString& q );
~Query();
int step();
Query& bind( const QString& str, int idx );
Query& bind( const QString& str );
static void exec( sqlite3* db, const QString& sql );
void reset();
int columnCount() const;
QString columnName( int i ) const;
int columnType( int i ) const;
int columnInt( int i ) const;
qint64 columnInt64( int i ) const;
double columnDouble( int i ) const;
QString columnText( int i ) const;
QByteArray columnBlob( int i ) const;
sqlite3_stmt* stmt();
private:
sqlite3* db_;
sqlite3_stmt* stmt_;
int nBind_;
};
}
#endif

682
src/providers/virtual/qgsvirtuallayersqlitemodule.cpp Normal file
View File

@ -0,0 +1,682 @@
/***************************************************************************
qgsvirtuallayersqlitemodule.cpp : SQLite module for QGIS virtual layers
begin : Nov 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include <string.h>
#include <iostream>
#include <stdint.h>
#include <stdexcept>
#include <QCoreApplication>
#include <qgsapplication.h>
#include <qgsvectorlayer.h>
#include <qgsvectordataprovider.h>
#include <qgsgeometry.h>
#include <qgsmaplayerregistry.h>
#include <qgsproviderregistry.h>
#include <sqlite3.h>
#include <spatialite.h>
#include <stdio.h>
#include "qgsvirtuallayersqlitemodule.h"
#include "qgsvirtuallayerblob.h"
#include "qgsslottofunction.h"
/**
* Structure created in SQLITE module creation and passed to xCreate/xConnect
*/
struct ModuleContext
{
// private pointer needed for spatialite_init_ex;
// allows to know whether the database has been initialied (null or not)
bool init;
ModuleContext() : init( false ) {}
};
/**
* Create metadata tables if needed
*/
void initVirtualLayerMetadata( sqlite3* db )
{
bool create_meta = false;
sqlite3_stmt *stmt;
int r;
r = sqlite3_prepare_v2( db, "SELECT name FROM sqlite_master WHERE name='_meta'", -1, &stmt, NULL );
if ( r )
{
throw std::runtime_error( sqlite3_errmsg( db ) );
}
create_meta = sqlite3_step( stmt ) != SQLITE_ROW;
sqlite3_finalize( stmt );
char *errMsg;
if ( create_meta )
{
r = sqlite3_exec( db, QString( "CREATE TABLE _meta (version INT, url TEXT); INSERT INTO _meta (version) VALUES(%1);" ).arg( VIRTUAL_LAYER_VERSION ).toLocal8Bit().constData(), NULL, NULL, &errMsg );
if ( r )
{
throw std::runtime_error( errMsg );
}
}
}
void deleteGeometryBlob( void * p )
{
delete[]( unsigned char* )p;
}
//-----------------------------------------------------------------------
//
// Functions and structures used by the SQLite virtual table module
//
//-----------------------------------------------------------------------
// function called when a lived layer is deleted
void invalidateTable( void* b );
struct VTable
{
// minimal set of members (see sqlite3.h)
const sqlite3_module *pModule; /* The module for this virtual table */
int nRef; /* NO LONGER USED */
char *zErrMsg; /* Error message from sqlite3_mprintf() */
VTable( sqlite3* db, QgsVectorLayer* layer )
: zErrMsg( 0 ), mSql( db ), mProvider( 0 ), mLayer( layer ), mSlotToFunction( invalidateTable, this ), mName( layer->name() ), mPkColumn( -1 ), mValid( true )
{
if ( mLayer )
{
QObject::connect( layer, SIGNAL( layerDeleted() ), &mSlotToFunction, SLOT( onSignal() ) );
}
init_();
}
VTable( sqlite3* db, const QString& provider, const QString& source, const QString& name, const QString& encoding )
: zErrMsg( 0 ), mSql( db ), mLayer( 0 ), mName( name ), mEncoding( encoding ), mPkColumn( -1 ), mValid( true )
{
mProvider = static_cast<QgsVectorDataProvider*>( QgsProviderRegistry::instance()->provider( provider, source ) );
if ( mProvider == 0 || !mProvider->isValid() )
{
throw std::runtime_error( "Invalid provider" );
}
if ( mProvider->capabilities() & QgsVectorDataProvider::SelectEncoding )
{
mProvider->setEncoding( mEncoding );
}
init_();
}
~VTable()
{
if ( mProvider )
{
delete mProvider;
}
}
QgsVectorDataProvider* provider() { return mProvider; }
QgsVectorLayer* layer() { return mLayer; }
QString name() const { return mName; }
QString creationString() const { return mCreationStr; }
long crs() const { return mCrs; }
sqlite3* sql() { return mSql; }
int pkColumn() const { return mPkColumn; }
void invalidate() { mValid = false; }
bool valid() const { return mValid; }
private:
// connection
sqlite3* mSql;
// pointer to the underlying vector provider
QgsVectorDataProvider* mProvider;
// pointer to the vector layer, for referenced layer
QgsVectorLayer* mLayer;
// the QObjet responsible of receiving the deletion signal
QgsSlotToFunction mSlotToFunction;
QString mName;
QString mEncoding;
// primary key column (default = -1: none)
int mPkColumn;
// CREATE TABLE string
QString mCreationStr;
long mCrs;
bool mValid;
void init_()
{
const QgsFields& fields = mLayer ? mLayer->fields() : mProvider->fields();
QStringList sql_fields;
// add a hidden field for rtree filtering
sql_fields << "_search_frame_ HIDDEN BLOB";
for ( int i = 0; i < fields.count(); i++ )
{
QString typeName = "text";
switch ( fields.at( i ).type() )
{
case QVariant::Int:
case QVariant::UInt:
case QVariant::Bool:
typeName = "int";
break;
case QVariant::Double:
typeName = "real";
break;
case QVariant::String:
default:
typeName = "text";
break;
}
sql_fields << fields.at( i ).name() + " " + typeName;
}
QgsVectorDataProvider* provider = mLayer ? mLayer->dataProvider() : mProvider;
if ( provider->geometryType() != QGis::WKBNoGeometry )
{
// we have here a convenient hack
// the type of a column can be declared with two numeric arguments, usually for setting numeric precision
// we are using them to set the geometry type and srid
// these will be reused by the provider when it will introspect the query to detect types
sql_fields << QString( "geometry geometry(%1,%2)" ).arg( provider->geometryType() ).arg( provider->crs().postgisSrid() );
}
if ( provider->pkAttributeIndexes().size() == 1 )
{
mPkColumn = provider->pkAttributeIndexes()[0] + 1;
}
mCreationStr = "CREATE TABLE vtable (" + sql_fields.join( "," ) + ")";
mCrs = provider->crs().postgisSrid();
}
};
// function called when a lived layer is deleted
void invalidateTable( void* p )
{
reinterpret_cast<VTable *>( p )->invalidate();
}
struct VTableCursor
{
// minimal set of members (see sqlite3.h)
VTable *mVtab;
// specific members
QgsFeature mCurrentFeature;
QgsFeatureIterator mIterator;
bool mEof;
VTableCursor( VTable *vtab ) : mVtab( vtab ), mEof( true ) {}
void filter( QgsFeatureRequest request )
{
if ( !mVtab->valid() )
{
mEof = true;
return;
}
mIterator = mVtab->layer() ? mVtab->layer()->getFeatures( request ) : mVtab->provider()->getFeatures( request );
// get on the first record
mEof = false;
next();
}
void next()
{
if ( !mEof )
{
mEof = !mIterator.nextFeature( mCurrentFeature );
}
}
bool eof() const { return mEof; }
int nColumns() const
{
if ( !mVtab->valid() )
return 0;
return mVtab->layer() ? mVtab->layer()->fields().count() : mVtab->provider()->fields().count();
}
sqlite3_int64 currentId() const { return mCurrentFeature.id(); }
QVariant currentAttribute( int column ) const { return mCurrentFeature.attribute( column ); }
QPair<char*, size_t> currentGeometry() const
{
size_t blob_len = 0;
char* blob = 0;
const QgsGeometry* g = mCurrentFeature.constGeometry();
if ( g && ! g->isEmpty() )
{
qgsGeometryToSpatialiteBlob( *g, mVtab->crs(), blob, blob_len );
}
return qMakePair( blob, blob_len );
}
};
void getGeometryType( const QgsVectorDataProvider* provider, QString& geometryTypeStr, int& geometryDim, int& geometryWkbType, long& srid )
{
srid = const_cast<QgsVectorDataProvider*>( provider )->crs().postgisSrid();
QgsWKBTypes::Type t = static_cast<QgsWKBTypes::Type>( provider->geometryType() );
geometryTypeStr = QgsWKBTypes::displayString( t );
geometryDim = QgsWKBTypes::coordDimensions( t );
if (( t != QgsWKBTypes::NoGeometry ) && ( t != QgsWKBTypes::Unknown ) )
geometryWkbType = static_cast<int>( t );
else
geometryWkbType = 0;
}
int vtable_create_connect( sqlite3* sql, void* aux, int argc, const char* const* argv, sqlite3_vtab **out_vtab, char** out_err, bool is_created )
{
Q_UNUSED( aux );
Q_UNUSED( is_created );
#define RETURN_CSTR_ERROR(err) if (out_err) {size_t s = strlen(err); *out_err=(char*)sqlite3_malloc(s+1); strncpy(*out_err, err, s);}
#define RETURN_CPPSTR_ERROR(err) if (out_err) {*out_err=(char*)sqlite3_malloc(err.size()+1); strncpy(*out_err, err.c_str(), err.size());}
if ( argc < 4 )
{
std::string err( "Missing arguments: layer_id | provider, source" );
RETURN_CPPSTR_ERROR( err );
return SQLITE_ERROR;
}
QScopedPointer<VTable> new_vtab;
QString vname( argv[2] );
int r;
if ( argc == 4 )
{
// CREATE VIRTUAL TABLE vtab USING QgsVLayer(layer_id)
// vtab = argv[2]
// layer_id = argv[3]
QString layerid( argv[3] );
if ( layerid.size() >= 1 && layerid[0] == '\'' )
{
layerid = layerid.mid( 1, layerid.size() - 2 );
}
QgsMapLayer *l = QgsMapLayerRegistry::instance()->mapLayer( layerid );
if ( l == 0 || l->type() != QgsMapLayer::VectorLayer )
{
if ( out_err )
{
std::string err( "Cannot find layer " );
err += argv[3];
RETURN_CPPSTR_ERROR( err );
}
return SQLITE_ERROR;
}
new_vtab.reset( new VTable( sql, static_cast<QgsVectorLayer*>( l ) ) );
}
else if ( argc == 5 || argc == 6 )
{
// CREATE VIRTUAL TABLE vtab USING QgsVLayer(provider,source[,encoding])
// vtab = argv[2]
// provider = argv[3]
// source = argv[4]
// encoding = argv[5]
QString provider = argv[3];
QString source = argv[4];
QString encoding = "UTF-8";
if ( argc == 6 )
{
encoding = argv[5];
}
if ( provider.size() >= 1 && provider[0] == '\'' )
{
// trim and undouble single quotes
provider = provider.mid( 1, provider.size() - 2 ).replace( "''", "'" );
}
if ( source.size() >= 1 && source[0] == '\'' )
{
// trim and undouble single quotes
source = source.mid( 1, source.size() - 2 ).replace( "''", "'" );
}
try
{
new_vtab.reset( new VTable( sql, provider, source, argv[2], encoding ) );
}
catch ( std::runtime_error& e )
{
std::string err( e.what() );
RETURN_CPPSTR_ERROR( err );
return SQLITE_ERROR;
}
}
r = sqlite3_declare_vtab( sql, new_vtab->creationString().toLocal8Bit().constData() );
if ( r )
{
RETURN_CSTR_ERROR( sqlite3_errmsg( sql ) );
return r;
}
*out_vtab = ( sqlite3_vtab* )new_vtab.take();
return SQLITE_OK;
#undef RETURN_CSTR_ERROR
#undef RETURN_CPPSTR_ERROR
}
void db_init( sqlite3* db, ModuleContext* context )
{
if ( context->init )
{
// db already initialized
return;
}
// create metadata tables
initVirtualLayerMetadata( db );
}
int vtable_create( sqlite3* sql, void* aux, int argc, const char* const* argv, sqlite3_vtab **out_vtab, char** out_err )
{
try
{
db_init( sql, reinterpret_cast<ModuleContext*>( aux ) );
}
catch ( std::runtime_error& e )
{
if ( out_err )
{
*out_err = ( char* )sqlite3_malloc( strlen( e.what() ) + 1 );
strcpy( *out_err, e.what() );
}
return SQLITE_ERROR;
}
return vtable_create_connect( sql, aux, argc, argv, out_vtab, out_err, /* is_created */ true );
}
int vtable_connect( sqlite3* sql, void* aux, int argc, const char* const* argv, sqlite3_vtab **out_vtab, char** out_err )
{
return vtable_create_connect( sql, aux, argc, argv, out_vtab, out_err, /* is_created */ false );
}
int vtable_destroy( sqlite3_vtab *vtab )
{
if ( vtab )
{
delete reinterpret_cast<VTable*>( vtab );
}
return SQLITE_OK;
}
int vtable_disconnect( sqlite3_vtab *vtab )
{
if ( vtab )
{
delete reinterpret_cast<VTable*>( vtab );
}
return SQLITE_OK;
}
int vtable_rename( sqlite3_vtab *vtab, const char *new_name )
{
Q_UNUSED( vtab );
Q_UNUSED( new_name );
return SQLITE_OK;
}
int vtable_bestindex( sqlite3_vtab *pvtab, sqlite3_index_info* index_info )
{
VTable *vtab = ( VTable* )pvtab;
for ( int i = 0; i < index_info->nConstraint; i++ )
{
if (( index_info->aConstraint[i].usable ) &&
( vtab->pkColumn() == index_info->aConstraint[i].iColumn ) &&
( index_info->aConstraint[i].op == SQLITE_INDEX_CONSTRAINT_EQ ) )
{
// request for primary key filter
index_info->aConstraintUsage[i].argvIndex = 1;
index_info->aConstraintUsage[i].omit = 1;
index_info->idxNum = 1; // PK filter
index_info->estimatedCost = 1.0; // ??
//index_info->estimatedRows = 1;
index_info->idxStr = NULL;
index_info->needToFreeIdxStr = 0;
return SQLITE_OK;
}
if (( index_info->aConstraint[i].usable ) &&
( 0 == index_info->aConstraint[i].iColumn ) &&
( index_info->aConstraint[i].op == SQLITE_INDEX_CONSTRAINT_EQ ) )
{
// request for rtree filtering
index_info->aConstraintUsage[i].argvIndex = 1;
// do not test for equality, since it is used for filtering, not to return an actual value
index_info->aConstraintUsage[i].omit = 1;
index_info->idxNum = 2; // RTree filter
index_info->estimatedCost = 1.0; // ??
//index_info->estimatedRows = 1;
index_info->idxStr = NULL;
index_info->needToFreeIdxStr = 0;
return SQLITE_OK;
}
}
index_info->idxNum = 0;
index_info->estimatedCost = 10.0;
//index_info->estimatedRows = 10;
index_info->idxStr = NULL;
index_info->needToFreeIdxStr = 0;
return SQLITE_OK;
}
int vtable_open( sqlite3_vtab *vtab, sqlite3_vtab_cursor **out_cursor )
{
VTableCursor *ncursor = new VTableCursor(( VTable* )vtab );
*out_cursor = ( sqlite3_vtab_cursor* )ncursor;
return SQLITE_OK;
}
int vtable_close( sqlite3_vtab_cursor * cursor )
{
if ( cursor )
{
delete reinterpret_cast<VTableCursor*>( cursor );
}
return SQLITE_OK;
}
int vtable_filter( sqlite3_vtab_cursor * cursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv )
{
Q_UNUSED( argc );
Q_UNUSED( idxStr );
QgsFeatureRequest request;
if ( idxNum == 1 )
{
// id filter
request.setFilterFid( sqlite3_value_int( argv[0] ) );
}
else if ( idxNum == 2 )
{
// rtree filter
const char* blob = ( const char* )sqlite3_value_blob( argv[0] );
int bytes = sqlite3_value_bytes( argv[0] );
QgsRectangle r( spatialiteBlobBbox( blob, bytes ) );
request.setFilterRect( r );
}
VTableCursor *c = reinterpret_cast<VTableCursor*>( cursor );
c->filter( request );
return SQLITE_OK;
}
int vtable_next( sqlite3_vtab_cursor *cursor )
{
VTableCursor* c = reinterpret_cast<VTableCursor*>( cursor );
c->next();
return SQLITE_OK;
}
int vtable_eof( sqlite3_vtab_cursor *cursor )
{
VTableCursor* c = reinterpret_cast<VTableCursor*>( cursor );
return c->eof();
}
int vtable_rowid( sqlite3_vtab_cursor *cursor, sqlite3_int64 *out_rowid )
{
VTableCursor* c = reinterpret_cast<VTableCursor*>( cursor );
*out_rowid = c->currentId();
return SQLITE_OK;
}
int vtable_column( sqlite3_vtab_cursor *cursor, sqlite3_context* ctxt, int idx )
{
VTableCursor* c = reinterpret_cast<VTableCursor*>( cursor );
if ( idx == 0 )
{
// _search_frame_, return null
sqlite3_result_null( ctxt );
return SQLITE_OK;
}
if ( idx == c->nColumns() + 1 )
{
QPair<char*, size_t> g = c->currentGeometry();
if ( !g.first )
sqlite3_result_null( ctxt );
else
sqlite3_result_blob( ctxt, g.first, g.second, deleteGeometryBlob );
return SQLITE_OK;
}
QVariant v = c->currentAttribute( idx - 1 );
if ( v.isNull() )
{
sqlite3_result_null( ctxt );
}
else
{
switch ( v.type() )
{
case QVariant::Int:
case QVariant::UInt:
sqlite3_result_int( ctxt, v.toInt() );
break;
case QVariant::Double:
sqlite3_result_double( ctxt, v.toDouble() );
break;
default:
{
sqlite3_result_text( ctxt, v.toString().toUtf8(), -1, SQLITE_TRANSIENT );
}
break;
}
}
return SQLITE_OK;
}
int vtable_findfunction( sqlite3_vtab *pVtab,
int nArg,
const char *zName,
void ( **pxFunc )( sqlite3_context*, int, sqlite3_value** ),
void **ppArg )
{
Q_UNUSED( pVtab );
Q_UNUSED( nArg );
Q_UNUSED( zName );
Q_UNUSED( pxFunc );
Q_UNUSED( ppArg );
return SQLITE_OK;
}
sqlite3_module module;
static QCoreApplication* core_app = 0;
static int module_argc = 1;
static char module_name[] = "qgsvlayer_module";
static char* module_argv[] = { module_name };
void module_destroy( void * d )
{
delete reinterpret_cast<ModuleContext*>( d );
if ( core_app )
{
delete core_app;
}
}
int qgsvlayer_module_init( sqlite3 *db, char **pzErrMsg, void * unused /*const sqlite3_api_routines *pApi*/ )
{
Q_UNUSED( pzErrMsg );
Q_UNUSED( unused );
int rc = SQLITE_OK;
// check if qgis providers are loaded
if ( QCoreApplication::instance() == 0 )
{
// if run standalone
core_app = new QCoreApplication( module_argc, module_argv );
QgsApplication::init();
QgsApplication::initQgis();
}
module.xCreate = vtable_create;
module.xConnect = vtable_connect;
module.xBestIndex = vtable_bestindex;
module.xDisconnect = vtable_disconnect;
module.xDestroy = vtable_destroy;
module.xOpen = vtable_open;
module.xClose = vtable_close;
module.xFilter = vtable_filter;
module.xNext = vtable_next;
module.xEof = vtable_eof;
module.xColumn = vtable_column;
module.xRowid = vtable_rowid;
module.xRename = vtable_rename;
module.xUpdate = NULL;
module.xBegin = NULL;
module.xSync = NULL;
module.xCommit = NULL;
module.xRollback = NULL;
module.xFindFunction = NULL;
module.xSavepoint = NULL;
module.xRelease = NULL;
module.xRollbackTo = NULL;
ModuleContext* context = new ModuleContext;
sqlite3_create_module_v2( db, "QgsVLayer", &module, context, module_destroy );
return rc;
}

74
src/providers/virtual/qgsvirtuallayersqlitemodule.h Normal file
View File

@ -0,0 +1,74 @@
/***************************************************************************
sqlite_vlayer_module.h : SQLite module for QGIS virtual layers
begin : Nov 2015
copyright : (C) 2015 Hugo Mercier, Oslandia
email : hugo dot mercier at oslandia dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSVIRTUAL_SQLITE_LAYER_MODULE_H
#define QGSVIRTUAL_SQLITE_LAYER_MODULE_H
#ifdef __cplusplus
extern "C"
{
#endif
int vtable_create( sqlite3* sql, void* aux, int argc, const char* const* argv, sqlite3_vtab **out_vtab, char** out_err );
int vtable_connect( sqlite3* sql, void* aux, int argc, const char* const* argv, sqlite3_vtab **out_vtab, char** out_err );
int vtable_rename( sqlite3_vtab *vtab, const char *new_name );
int vtable_bestindex( sqlite3_vtab *vtab, sqlite3_index_info* );
int vtable_disconnect( sqlite3_vtab *vtab );
int vtable_destroy( sqlite3_vtab *vtab );
int vtable_open( sqlite3_vtab *vtab, sqlite3_vtab_cursor **out_cursor );
int vtable_close( sqlite3_vtab_cursor * );
int vtable_filter( sqlite3_vtab_cursor * cursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv );
int vtable_next( sqlite3_vtab_cursor *cursor );
int vtable_eof( sqlite3_vtab_cursor *cursor );
int vtable_column( sqlite3_vtab_cursor *cursor, sqlite3_context*, int );
int vtable_rowid( sqlite3_vtab_cursor *cursor, sqlite3_int64 *out_rowid );
int vtable_findfunction( sqlite3_vtab *pVtab,
int nArg,
const char *zName,
void ( **pxFunc )( sqlite3_context*, int, sqlite3_value** ),
void **ppArg );
int qgsvlayer_module_init( sqlite3 *db,
char **pzErrMsg,
void * unused /*const sqlite3_api_routines *pApi*/ );
#ifdef __cplusplus
}
#include <qgsgeometry.h>
/**
* Convert a spatialite geometry blob to a QgsGeometry
*
* \param blob Pointer to the raw blob memory
* \param size Size in bytes of the blob
* \returns a QgsGeometry (that are implicitly shared)
*/
QgsGeometry spatialite_blob_to_qgsgeometry( const unsigned char* blob, size_t size );
/**
* Init the SQLite file with proper metadata tables
*/
void initVirtualLayerMetadata( sqlite3* db );
#endif
#define VIRTUAL_LAYER_VERSION 1
#endif

3
tests/src/python/CMakeLists.txt
View File

@ -64,7 +64,8 @@ ADD_PYTHON_TEST(PyQgsVectorFileWriter test_qgsvectorfilewriter.py)
ADD_PYTHON_TEST(PyQgsVectorLayer test_qgsvectorlayer.py)
ADD_PYTHON_TEST(PyQgsZonalStatistics test_qgszonalstatistics.py)
ADD_PYTHON_TEST(PyQgsMapLayerRegistry test_qgsmaplayerregistry.py)
ADD_PYTHON_TEST(PyQgsVirtualLayerProvider test_provider_virtual.py)
ADD_PYTHON_TEST(PyQgsVirtualLayerDefinition test_qgsvirtuallayerdefinition.py)
IF (NOT WIN32)
ADD_PYTHON_TEST(PyQgsLogger test_qgslogger.py)

664
tests/src/python/test_provider_virtual.py Normal file
View File

@ -0,0 +1,664 @@
# -*- coding: utf-8 -*-
"""QGIS Unit tests for QgsVirtualLayerProvider
.. note:: This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
"""
__author__ = 'Hugo Mercier'
__date__ = '26/11/2015'
__copyright__ = 'Copyright 2015, The QGIS Project'
# This will get replaced with a git SHA1 when you do a git archive
__revision__ = '$Format:%H$'
import qgis
import os
import tempfile
import sys
from qgis.core import (QGis,
QgsVectorLayer,
QgsFeature,
QgsFeatureRequest,
QgsField,
QgsGeometry,
QgsPoint,
QgsMapLayerRegistry,
QgsRectangle,
QgsErrorMessage,
QgsProviderRegistry,
QgsVirtualLayerDefinition
)
from utilities import (unitTestDataPath,
getQgisTestApp,
TestCase,
unittest
)
from providertestbase import ProviderTestCase
from PyQt4.QtCore import *
try:
from pyspatialite import dbapi2 as sqlite3
except ImportError:
print "You should install pyspatialite to run the tests"
raise ImportError
import tempfile
# Convenience instances in case you may need them
QGISAPP, CANVAS, IFACE, PARENT = getQgisTestApp()
TEST_DATA_DIR = unitTestDataPath()
class TestQgsVirtualLayerProvider(TestCase, ProviderTestCase):
@classmethod
def setUpClass(cls):
"""Run before all tests"""
# Create the layer for the common provider tests
shp = os.path.join(TEST_DATA_DIR, 'provider/shapefile.shp')
d = QgsVirtualLayerDefinition()
d.addSource("vtab1", shp, "ogr")
d.setUid("pk")
cls.vl = QgsVectorLayer(d.toString(), u'test', u'virtual')
assert (cls.vl.isValid())
cls.provider = cls.vl.dataProvider()
@classmethod
def tearDownClass(cls):
"""Run after all tests"""
pass
def setUp(self):
"""Run before each test."""
self.testDataDir = unitTestDataPath()
print "****************************************************"
print "In method", self._testMethodName
print "****************************************************"
pass
def tearDown(self):
"""Run after each test."""
pass
def test_CsvNoGeometry(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "delimitedtext/test.csv") + "?type=csv&geomType=none&subsetIndex=no&watchFile=no", "test", "delimitedtext", False)
self.assertEqual(l1.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l1)
l2 = QgsVectorLayer("?layer_ref=" + l1.id(), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
QgsMapLayerRegistry.instance().removeMapLayer(l1.id())
def test_source_escaping(self):
# the source contains ':'
source = "file:///" + os.path.join(self.testDataDir, "delimitedtext/test.csv") + "?type=csv&geomType=none&subsetIndex=no&watchFile=no"
d = QgsVirtualLayerDefinition()
d.addSource("t", source, "delimitedtext")
l = QgsVectorLayer(d.toString(), "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
def test_source_escaping2(self):
def create_test_db(dbfile):
if os.path.exists(dbfile):
os.remove(dbfile)
con = sqlite3.connect(dbfile)
cur = con.cursor()
cur.execute("SELECT InitSpatialMetadata(1)")
cur.execute("CREATE TABLE test (id INTEGER, name TEXT)")
cur.execute("SELECT AddGeometryColumn('test', 'geometry', 4326, 'POINT', 'XY')")
sql = "INSERT INTO test (id, name, geometry) "
sql += "VALUES (1, 'toto',GeomFromText('POINT(0 0)',4326))"
cur.execute(sql)
con.close()
# the source contains ',' and single quotes
fn = os.path.join(tempfile.gettempdir(), "test,.db")
create_test_db(fn)
source = "dbname='%s' table=\"test\" (geometry) sql=" % fn
d = QgsVirtualLayerDefinition()
d.addSource("t", source, "spatialite")
l = QgsVectorLayer(d.toString(), "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
# the source contains ':' and single quotes
fn = os.path.join(tempfile.gettempdir(), "test:.db")
create_test_db(fn)
source = "dbname='%s' table=\"test\" (geometry) sql=" % fn
d = QgsVirtualLayerDefinition()
d.addSource("t", source, "spatialite")
l = QgsVectorLayer(d.toString(), "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
def test_DynamicGeometry(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "delimitedtext/testextpt.txt") + "?type=csv&delimiter=%7C&geomType=none&subsetIndex=no&watchFile=no", "test", "delimitedtext", False)
self.assertEqual(l1.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l1)
query = QUrl.toPercentEncoding("select *,makepoint(x,y) as geom from vtab1")
l2 = QgsVectorLayer("?layer_ref=%s&query=%s&geometry=geom:point:0&uid=id" % (l1.id(), query), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
QgsMapLayerRegistry.instance().removeMapLayer(l1)
def test_ShapefileWithGeometry(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "france_parts.shp"), "france_parts", "ogr", False)
self.assertEqual(l1.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l1)
# use a temporary file
l2 = QgsVectorLayer("?layer_ref=" + l1.id(), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
l2 = QgsVectorLayer("?layer_ref=%s:nn" % l1.id(), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
QgsMapLayerRegistry.instance().removeMapLayer(l1.id())
def test_Query(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "france_parts.shp"), "france_parts", "ogr", False)
self.assertEqual(l1.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l1)
ref_sum = sum(f.attributes()[0] for f in l1.getFeatures())
query = QUrl.toPercentEncoding("SELECT * FROM vtab1")
l2 = QgsVectorLayer("?layer_ref=%s&geometry=geometry:3:4326&query=%s&uid=OBJECTID" % (l1.id(), query), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().geometryType(), 3)
ref_sum2 = sum(f.attributes()[0] for f in l2.getFeatures())
ref_sum3 = sum(f.id() for f in l2.getFeatures())
# check we have the same rows
self.assertEqual(ref_sum, ref_sum2)
# check the id is ok
self.assertEqual(ref_sum, ref_sum3)
# the same, without specifying the geometry column name
l2 = QgsVectorLayer("?layer_ref=%s&query=%s&uid=OBJECTID" % (l1.id(), query), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().geometryType(), 3)
ref_sum2 = sum(f.attributes()[0] for f in l2.getFeatures())
ref_sum3 = sum(f.id() for f in l2.getFeatures())
# check we have the same rows
self.assertEqual(ref_sum, ref_sum2)
# check the id is ok
self.assertEqual(ref_sum, ref_sum3)
# with two geometry columns
query = QUrl.toPercentEncoding("SELECT *,geometry as geom FROM vtab1")
l2 = QgsVectorLayer("?layer_ref=%s&query=%s&uid=OBJECTID&geometry=geom:3:4326" % (l1.id(), query), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().geometryType(), 3)
ref_sum2 = sum(f.attributes()[0] for f in l2.getFeatures())
ref_sum3 = sum(f.id() for f in l2.getFeatures())
# check we have the same rows
self.assertEqual(ref_sum, ref_sum2)
# check the id is ok
self.assertEqual(ref_sum, ref_sum3)
# with two geometry columns, but no geometry column specified (will take the first)
l2 = QgsVectorLayer("?layer_ref=%s&query=%s&uid=OBJECTID" % (l1.id(), query), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().geometryType(), 3)
ref_sum2 = sum(f.attributes()[0] for f in l2.getFeatures())
ref_sum3 = sum(f.id() for f in l2.getFeatures())
# check we have the same rows
self.assertEqual(ref_sum, ref_sum2)
# check the id is ok
self.assertEqual(ref_sum, ref_sum3)
# the same, without geometry
query = QUrl.toPercentEncoding("SELECT * FROM ww")
l2 = QgsVectorLayer("?layer_ref=%s:ww&query=%s&uid=ObJeCtId&nogeometry" % (l1.id(), query), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().geometryType(), 100) # NoGeometry
ref_sum2 = sum(f.attributes()[0] for f in l2.getFeatures())
ref_sum3 = sum(f.id() for f in l2.getFeatures())
self.assertEqual(ref_sum, ref_sum2)
self.assertEqual(ref_sum, ref_sum3)
# check that it fails when a query has a wrong geometry column
l2 = QgsVectorLayer("?layer_ref=%s&query=%s&geometry=geo" % (l1.id(), query), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), False)
QgsMapLayerRegistry.instance().removeMapLayer(l1.id())
def test_QueryUrlEncoding(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "france_parts.shp"), "france_parts", "ogr", False)
self.assertEqual(l1.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l1)
query = str(QUrl.toPercentEncoding("SELECT * FROM vtab1"))
l2 = QgsVectorLayer("?layer_ref=%s&query=%s&uid=ObjectId&nogeometry" % (l1.id(), query), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
QgsMapLayerRegistry.instance().removeMapLayer(l1.id())
def test_QueryTableName(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "france_parts.shp"), "france_parts", "ogr", False)
self.assertEqual(l1.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l1)
query = str(QUrl.toPercentEncoding("SELECT * FROM vt"))
l2 = QgsVectorLayer("?layer_ref=%s:vt&query=%s&uid=ObJeCtId&nogeometry" % (l1.id(), query), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().geometryType(), 100) # NoGeometry
QgsMapLayerRegistry.instance().removeMapLayer(l1.id())
def test_Join(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "points.shp"), "points", "ogr", False)
self.assertEqual(l1.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l1)
l2 = QgsVectorLayer(os.path.join(self.testDataDir, "points_relations.shp"), "points_relations", "ogr", False)
self.assertEqual(l2.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l2)
ref_sum = sum(f.attributes()[1] for f in l2.getFeatures())
# use a temporary file
query = QUrl.toPercentEncoding("select id,Pilots,vtab1.geometry from vtab1,vtab2 where intersects(vtab1.geometry,vtab2.geometry)")
l3 = QgsVectorLayer("?layer_ref=%s&layer_ref=%s&uid=id&query=%s&geometry=geometry:1:4326" % (l1.id(), l2.id(), query), "vtab", "virtual", False)
self.assertEqual(l3.isValid(), True)
self.assertEqual(l3.dataProvider().geometryType(), 1)
self.assertEqual(l3.dataProvider().fields().count(), 2)
ref_sum2 = sum(f.id() for f in l3.getFeatures())
self.assertEqual(ref_sum, ref_sum2)
QgsMapLayerRegistry.instance().removeMapLayer(l1)
QgsMapLayerRegistry.instance().removeMapLayer(l2)
def test_geometryTypes(self):
geo = [(1, "POINT", "(0 0)"),
(2, "LINESTRING", "(0 0,1 0)"),
(3, "POLYGON", "((0 0,1 0,1 1,0 0))"),
(4, "MULTIPOINT", "((1 1))"),
(5, "MULTILINESTRING", "((0 0,1 0),(0 1,1 1))"),
(6, "MULTIPOLYGON", "(((0 0,1 0,1 1,0 0)),((2 2,3 0,3 3,2 2)))")]
for wkb_type, wkt_type, wkt in geo:
l = QgsVectorLayer("%s?crs=epsg:4326" % wkt_type, "m1", "memory", False)
self.assertEqual(l.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l)
f1 = QgsFeature(1)
g = QgsGeometry.fromWkt(wkt_type + wkt)
self.assertEqual(g is None, False)
f1.setGeometry(g)
l.dataProvider().addFeatures([f1])
l2 = QgsVectorLayer("?layer_ref=%s" % l.id(), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().featureCount(), 1)
self.assertEqual(l2.dataProvider().geometryType(), wkb_type)
QgsMapLayerRegistry.instance().removeMapLayer(l.id())
def test_embeddedLayer(self):
source = QUrl.toPercentEncoding(os.path.join(self.testDataDir, "france_parts.shp"))
l = QgsVectorLayer("?layer=ogr:%s" % source, "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
l = QgsVectorLayer("?layer=ogr:%s:nn" % source, "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
def test_filter_rect(self):
source = QUrl.toPercentEncoding(os.path.join(self.testDataDir, "france_parts.shp"))
query = QUrl.toPercentEncoding("select * from vtab where _search_frame_=BuildMbr(-2.10,49.38,-1.3,49.99,4326)")
l2 = QgsVectorLayer("?layer=ogr:%s:vtab&query=%s&uid=objectid" % (source, query), "vtab2", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().featureCount(), 1)
a = [fit.attributes()[4] for fit in l2.getFeatures()]
self.assertEqual(a, [u"Basse-Normandie"])
def test_recursiveLayer(self):
source = QUrl.toPercentEncoding(os.path.join(self.testDataDir, "france_parts.shp"))
l = QgsVectorLayer("?layer=ogr:%s" % source, "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l)
l2 = QgsVectorLayer("?layer_ref=" + l.id(), "vtab2", "virtual", False)
self.assertEqual(l2.isValid(), True)
QgsMapLayerRegistry.instance().removeMapLayer(l.id())
def test_no_geometry(self):
source = QUrl.toPercentEncoding(os.path.join(self.testDataDir, "france_parts.shp"))
l2 = QgsVectorLayer("?layer=ogr:%s:vtab&nogeometry" % source, "vtab2", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().geometryType(), 100) # NoGeometry
def test_reopen(self):
source = QUrl.toPercentEncoding(os.path.join(self.testDataDir, "france_parts.shp"))
tmp = os.path.join(tempfile.gettempdir(), "t.sqlite")
l = QgsVectorLayer("%s?layer=ogr:%s:vtab" % (tmp, source), "vtab2", "virtual", False)
self.assertEqual(l.isValid(), True)
l2 = QgsVectorLayer(tmp, "tt", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().geometryType(), 3)
self.assertEqual(l2.dataProvider().featureCount(), 4)
def test_reopen2(self):
source = QUrl.toPercentEncoding(os.path.join(self.testDataDir, "france_parts.shp"))
tmp = os.path.join(tempfile.gettempdir(), "t.sqlite")
l = QgsVectorLayer("%s?layer=ogr:%s:vtab&nogeometry" % (tmp, source), "vtab2", "virtual", False)
self.assertEqual(l.isValid(), True)
l2 = QgsVectorLayer(tmp, "tt", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().geometryType(), 100)
self.assertEqual(l2.dataProvider().featureCount(), 4)
def test_reopen3(self):
source = QUrl.toPercentEncoding(os.path.join(self.testDataDir, "france_parts.shp"))
tmp = os.path.join(tempfile.gettempdir(), "t.sqlite")
query = QUrl.toPercentEncoding("SELECT * FROM vtab")
l = QgsVectorLayer("%s?layer=ogr:%s:vtab&query=%s&uid=objectid&geometry=geometry:3:4326" % (tmp, source, query), "vtab2", "virtual", False)
self.assertEqual(l.isValid(), True)
l2 = QgsVectorLayer(tmp, "tt", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().geometryType(), 3)
self.assertEqual(l2.dataProvider().featureCount(), 4)
sumid = sum([f.id() for f in l2.getFeatures()])
self.assertEqual(sumid, 10659)
suma = sum([f.attributes()[1] for f in l2.getFeatures()])
self.assertEqual(suma, 3064.0)
def test_reopen4(self):
source = QUrl.toPercentEncoding(os.path.join(self.testDataDir, "france_parts.shp"))
tmp = os.path.join(tempfile.gettempdir(), "t.sqlite")
query = QUrl.toPercentEncoding("SELECT * FROM vtab")
l = QgsVectorLayer("%s?layer=ogr:%s:vtab&query=%s&uid=objectid&nogeometry" % (tmp, source, query), "vtab2", "virtual", False)
self.assertEqual(l.isValid(), True)
l2 = QgsVectorLayer(tmp, "tt", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().geometryType(), 100)
self.assertEqual(l2.dataProvider().featureCount(), 4)
sumid = sum([f.id() for f in l2.getFeatures()])
self.assertEqual(sumid, 10659)
suma = sum([f.attributes()[1] for f in l2.getFeatures()])
self.assertEqual(suma, 3064.0)
def test_refLayer(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "delimitedtext/test.csv") + "?type=csv&geomType=none&subsetIndex=no&watchFile=no", "test", "delimitedtext", False)
self.assertEqual(l1.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l1)
l2 = QgsVectorLayer("?layer_ref=" + l1.id(), "vtab", "virtual", False)
self.assertEqual(l2.isValid(), True)
# now delete the layer
QgsMapLayerRegistry.instance().removeMapLayer(l1.id())
# check that it does not crash
print sum([f.id() for f in l2.getFeatures()])
def test_refLayers(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "delimitedtext/test.csv") + "?type=csv&geomType=none&subsetIndex=no&watchFile=no", "test", "delimitedtext", False)
self.assertEqual(l1.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l1)
# cf qgis bug #12266
for i in range(10):
q = QUrl.toPercentEncoding("select * from t" + str(i))
l2 = QgsVectorLayer("?layer_ref=%s:t%d&query=%s&uid=id" % (l1.id(), i, q), "vtab", "virtual", False)
QgsMapLayerRegistry.instance().addMapLayer(l2)
self.assertEqual(l2.isValid(), True)
s = sum([f.id() for f in l2.dataProvider().getFeatures()])
self.assertEqual(sum([f.id() for f in l2.getFeatures()]), 21)
QgsMapLayerRegistry.instance().removeMapLayer(l2.id())
def test_refLayers2(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "delimitedtext/test.csv") + "?type=csv&geomType=none&subsetIndex=no&watchFile=no", "test", "delimitedtext", False)
self.assertEqual(l1.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l1)
# referenced layers cannot be stored !
tmp = os.path.join(tempfile.gettempdir(), "t.sqlite")
l2 = QgsVectorLayer("%s?layer_ref=%s" % (tmp, l1.id()), "tt", "virtual", False)
self.assertEqual(l2.isValid(), False)
self.assertEqual("Cannot store referenced layers" in l2.dataProvider().error().message(), True)
def test_sql(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "delimitedtext/test.csv") + "?type=csv&geomType=none&subsetIndex=no&watchFile=no", "test", "delimitedtext", False)
self.assertEqual(l1.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l1)
l3 = QgsVectorLayer("?query=SELECT * FROM test", "tt", "virtual")
self.assertEqual(l3.isValid(), True)
s = sum(f.id() for f in l3.getFeatures())
self.assertEqual(s, 15)
def test_sql2(self):
l2 = QgsVectorLayer(os.path.join(self.testDataDir, "france_parts.shp"), "france_parts", "ogr", False)
self.assertEqual(l2.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l2)
query = QUrl.toPercentEncoding("SELECT * FROM france_parts")
l4 = QgsVectorLayer("?query=%s" % query, "tt", "virtual")
self.assertEqual(l4.isValid(), True)
self.assertEqual(l4.dataProvider().geometryType(), 3)
self.assertEqual(l4.dataProvider().crs().postgisSrid(), 4326)
n = 0
r = QgsFeatureRequest(QgsRectangle(-1.677, 49.624, -0.816, 49.086))
for f in l4.getFeatures(r):
self.assertEqual(f.geometry() is not None, True)
self.assertEqual(f.attributes()[0], 2661)
n += 1
self.assertEqual(n, 1)
# use uid
query = QUrl.toPercentEncoding("SELECT * FROM france_parts")
l5 = QgsVectorLayer("?query=%s&geometry=geometry:polygon:4326&uid=ObjectId" % query, "tt", "virtual")
self.assertEqual(l5.isValid(), True)
idSum = sum(f.id() for f in l5.getFeatures())
self.assertEqual(idSum, 10659)
r = QgsFeatureRequest(2661)
idSum2 = sum(f.id() for f in l5.getFeatures(r))
self.assertEqual(idSum2, 2661)
r = QgsFeatureRequest()
r.setFilterFids([2661, 2664])
self.assertEqual(sum(f.id() for f in l5.getFeatures(r)), 2661 + 2664)
# test attribute subset
r = QgsFeatureRequest()
r.setFlags(QgsFeatureRequest.SubsetOfAttributes)
r.setSubsetOfAttributes([1])
s = [(f.id(), f.attributes()[1]) for f in l5.getFeatures(r)]
self.assertEqual(sum(map(lambda x: x[0], s)), 10659)
self.assertEqual(sum(map(lambda x: x[1], s)), 3064.0)
# test NoGeometry
# by request flag
r = QgsFeatureRequest()
r.setFlags(QgsFeatureRequest.NoGeometry)
self.assertEqual(all([f.geometry() is None for f in l5.getFeatures(r)]), True)
# test subset
self.assertEqual(l5.dataProvider().featureCount(), 4)
l5.setSubsetString("ObjectId = 2661")
idSum2 = sum(f.id() for f in l5.getFeatures(r))
self.assertEqual(idSum2, 2661)
self.assertEqual(l5.dataProvider().featureCount(), 1)
def test_sql3(self):
l2 = QgsVectorLayer(os.path.join(self.testDataDir, "france_parts.shp"), "france_parts", "ogr", False)
self.assertEqual(l2.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l2)
# unnamed column
query = QUrl.toPercentEncoding("SELECT 42")
l4 = QgsVectorLayer("?query=%s" % query, "tt", "virtual", False)
self.assertEqual(l4.isValid(), False)
self.assertEqual("Result column #1 has no name" in l4.dataProvider().error().message(), True)
def test_sql_field_types(self):
query = QUrl.toPercentEncoding("SELECT 42 as t, 'ok'||'ok' as t2, GeomFromText('') as t3, 3.14*2 as t4")
l4 = QgsVectorLayer("?query=%s" % query, "tt", "virtual", False)
self.assertEqual(l4.isValid(), True)
self.assertEqual(l4.dataProvider().fields().at(0).name(), "t")
self.assertEqual(l4.dataProvider().fields().at(0).type(), QVariant.Int)
self.assertEqual(l4.dataProvider().fields().at(1).name(), "t2")
self.assertEqual(l4.dataProvider().fields().at(1).type(), QVariant.String)
self.assertEqual(l4.dataProvider().fields().at(2).name(), "t3")
self.assertEqual(l4.dataProvider().fields().at(2).type(), QVariant.String)
self.assertEqual(l4.dataProvider().fields().at(3).name(), "t4")
self.assertEqual(l4.dataProvider().fields().at(3).type(), QVariant.Double)
# with type annotations
query = QUrl.toPercentEncoding("SELECT '42.0' as t /*:real*/, 3 as t2/*:text */, GeomFromText('') as t3 /*:multiPoInT:4326 */, 3.14*2 as t4/*:int*/")
l4 = QgsVectorLayer("?query=%s" % query, "tt", "virtual", False)
self.assertEqual(l4.isValid(), True)
self.assertEqual(l4.dataProvider().fields().at(0).name(), "t")
self.assertEqual(l4.dataProvider().fields().at(0).type(), QVariant.Double)
self.assertEqual(l4.dataProvider().fields().at(1).name(), "t2")
self.assertEqual(l4.dataProvider().fields().at(1).type(), QVariant.String)
self.assertEqual(l4.dataProvider().fields().at(2).name(), "t4")
self.assertEqual(l4.dataProvider().fields().at(2).type(), QVariant.Int)
self.assertEqual(l4.dataProvider().geometryType(), 4) # multipoint
# test value types (!= from declared column types)
for f in l4.getFeatures():
self.assertEqual(f.attributes()[0], "42.0")
self.assertEqual(f.attributes()[1], 3)
self.assertEqual(f.attributes()[2], 6.28)
# with type annotations and url options
query = QUrl.toPercentEncoding("SELECT 1 as id /*:int*/, geomfromtext('point(0 0)',4326) as geometry/*:point:4326*/")
l4 = QgsVectorLayer("?query=%s&geometry=geometry" % query, "tt", "virtual", False)
self.assertEqual(l4.isValid(), True)
self.assertEqual(l4.dataProvider().geometryType(), 1) # point
# with type annotations and url options (2)
query = QUrl.toPercentEncoding("SELECT 1 as id /*:int*/, 3.14 as f, geomfromtext('point(0 0)',4326) as geometry/*:point:4326*/")
l4 = QgsVectorLayer("?query=%s&geometry=geometry&field=id:text" % query, "tt", "virtual", False)
self.assertEqual(l4.isValid(), True)
self.assertEqual(l4.dataProvider().fields().at(0).name(), "id")
self.assertEqual(l4.dataProvider().fields().at(0).type(), QVariant.String)
self.assertEqual(l4.dataProvider().fields().at(1).name(), "f")
self.assertEqual(l4.dataProvider().fields().at(1).type(), QVariant.Double)
self.assertEqual(l4.dataProvider().geometryType(), 1) # point
def test_sql3b(self):
query = QUrl.toPercentEncoding("SELECT GeomFromText('POINT(0 0)') as geom")
l4 = QgsVectorLayer("?query=%s&geometry=geom" % query, "tt", "virtual", False)
self.assertEqual(l4.isValid(), True)
self.assertEqual(l4.dataProvider().geometryType(), 1)
# forced geometry type
query = QUrl.toPercentEncoding("SELECT GeomFromText('POINT(0 0)') as geom")
l4 = QgsVectorLayer("?query=%s&geometry=geom:point:0" % query, "tt", "virtual", False)
self.assertEqual(l4.isValid(), True)
self.assertEqual(l4.dataProvider().geometryType(), 1)
query = QUrl.toPercentEncoding("SELECT CastToPoint(GeomFromText('POINT(0 0)')) as geom")
l4 = QgsVectorLayer("?query=%s" % query, "tt", "virtual", False)
self.assertEqual(l4.isValid(), True)
self.assertEqual(l4.dataProvider().geometryType(), 1)
def test_sql4(self):
l2 = QgsVectorLayer(os.path.join(self.testDataDir, "france_parts.shp"), "france_parts", "ogr", False)
self.assertEqual(l2.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l2)
query = QUrl.toPercentEncoding("SELECT OBJECTId from france_parts")
l4 = QgsVectorLayer("?query=%s" % query, "tt", "virtual", False)
self.assertEqual(l4.isValid(), True)
s = sum(f.attributes()[0] for f in l4.getFeatures())
self.assertEqual(s, 10659)
def test_layer_name(self):
# test space and upper case
l2 = QgsVectorLayer(os.path.join(self.testDataDir, "france_parts.shp"), "FranCe parts", "ogr", False)
self.assertEqual(l2.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(l2)
query = QUrl.toPercentEncoding('SELECT OBJECTId from "FranCe parts"')
l4 = QgsVectorLayer("?query=%s" % query, "tt", "virtual", False)
self.assertEqual(l4.isValid(), True)
s = sum(f.attributes()[0] for f in l4.getFeatures())
self.assertEqual(s, 10659)
def test_encoding(self):
# changes encoding on a shapefile (the only provider supporting setEncoding)
source = QUrl.toPercentEncoding(os.path.join(self.testDataDir, "shp_latin1.dbf"))
l = QgsVectorLayer("?layer=ogr:%s:fp:latin1" % source, "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
for f in l.getFeatures():
self.assertEqual(f.attributes()[1], u"accents éàè")
# use UTF-8 now
l = QgsVectorLayer("?layer=ogr:%s:fp:UTF-8" % source, "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
for f in l.getFeatures():
self.assertEqual(f.attributes()[1], u"accents \ufffd\ufffd\ufffd") # invalid unicode characters
def test_rowid(self):
source = QUrl.toPercentEncoding(os.path.join(self.testDataDir, "france_parts.shp"))
query = QUrl.toPercentEncoding("select rowid as uid, * from vtab limit 1 offset 3")
l = QgsVectorLayer("?layer=ogr:%s:vtab&query=%s" % (source, query), "vtab2", "virtual", False)
# the last line must have a fixed rowid (not an autoincrement)
for f in l.getFeatures():
lid = f.attributes()[0]
self.assertEqual(lid, 3)
def test_geometry_conversion(self):
query = QUrl.toPercentEncoding("select geomfromtext('multipoint((0 0),(1 1))') as geom")
l = QgsVectorLayer("?query=%s&geometry=geom:multipoint:0" % query, "tt", "virtual", False)
self.assertEqual(l.isValid(), True)
for f in l.getFeatures():
self.assertEqual(f.geometry().exportToWkt().lower().startswith("multipoint"), True)
self.assertEqual("),(" in f.geometry().exportToWkt(), True) # has two points
query = QUrl.toPercentEncoding("select geomfromtext('multipolygon(((0 0,1 0,1 1,0 1,0 0)),((0 1,1 1,1 2,0 2,0 1)))') as geom")
l = QgsVectorLayer("?query=%s&geometry=geom:multipolygon:0" % query, "tt", "virtual", False)
self.assertEqual(l.isValid(), True)
for f in l.getFeatures():
self.assertEqual(f.geometry().exportToWkt().lower().startswith("multipolygon"), True)
self.assertEqual(")),((" in f.geometry().exportToWkt(), True) # has two polygons
query = QUrl.toPercentEncoding("select geomfromtext('multilinestring((0 0,1 0,1 1,0 1,0 0),(0 1,1 1,1 2,0 2,0 1))') as geom")
l = QgsVectorLayer("?query=%s&geometry=geom:multilinestring:0" % query, "tt", "virtual", False)
self.assertEqual(l.isValid(), True)
for f in l.getFeatures():
self.assertEqual(f.geometry().exportToWkt().lower().startswith("multilinestring"), True)
self.assertEqual("),(" in f.geometry().exportToWkt(), True) # has two linestrings
def test_queryOnMemoryLayer(self):
ml = QgsVectorLayer("Point?srid=EPSG:4326&field=a:int", "mem", "memory")
self.assertEqual(ml.isValid(), True)
QgsMapLayerRegistry.instance().addMapLayer(ml)
ml.startEditing()
f1 = QgsFeature(ml.fields())
f1.setGeometry(QgsGeometry.fromWkt('POINT(0 0)'))
f2 = QgsFeature(ml.fields())
f2.setGeometry(QgsGeometry.fromWkt('POINT(1 1)'))
ml.addFeatures([f1, f2])
ml.commitChanges()
vl = QgsVectorLayer("?query=select * from mem", "vl", "virtual")
self.assertEqual(vl.isValid(), True)
self.assertEqual(ml.featureCount(), vl.featureCount())
# test access to pending features as well
ml.startEditing()
f3 = QgsFeature(ml.fields())
ml.addFeatures([f3])
self.assertEqual(ml.featureCount(), vl.featureCount())
if __name__ == '__main__':
unittest.main()

78
tests/src/python/test_qgsvirtuallayerdefinition.py Normal file
View File

@ -0,0 +1,78 @@
# -*- coding: utf-8 -*-
"""QGIS Unit tests for QgsVirtualLayerDefinition
.. note:: This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
"""
__author__ = 'Hugo Mercier'
__date__ = '10/12/2015'
__copyright__ = 'Copyright 2015, The QGIS Project'
# This will get replaced with a git SHA1 when you do a git archive
__revision__ = '$Format:%H$'
import qgis
from qgis.core import (QGis,
QgsField,
QgsWKBTypes,
QgsFields,
QgsVirtualLayerDefinition
)
from utilities import (TestCase, unittest)
from PyQt4.QtCore import QVariant
class TestQgsVirtualLayerDefinition(TestCase):
def test1(self):
d = QgsVirtualLayerDefinition()
self.assertEqual(d.toString(), "")
d.setFilePath("/file")
self.assertEqual(d.toString(), "/file")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).filePath(), "/file")
d.setFilePath("C:\\file")
self.assertEqual(d.toString(), "C:%5Cfile")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).filePath(), "C:\\file")
d.setQuery("SELECT * FROM mytable")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).query(), "SELECT * FROM mytable")
q = u"SELECT * FROM tableéé /*:int*/"
d.setQuery(q)
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).query(), q)
s1 = u"file://foo&bar=okié"
d.addSource("name", s1, "provider", "utf8")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).sourceLayers()[0].source(), s1)
n1 = u"éé ok"
d.addSource(n1, s1, "provider")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).sourceLayers()[1].name(), n1)
d.addSource("ref1", "id0001")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).sourceLayers()[2].reference(), "id0001")
d.setGeometryField("geom")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).geometryField(), "geom")
d.setGeometryWkbType(QgsWKBTypes.Point)
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).geometryWkbType(), QgsWKBTypes.Point)
f = QgsFields()
f.append(QgsField("a", QVariant.Int))
f.append(QgsField("f", QVariant.Double))
f.append(QgsField("s", QVariant.String))
d.setFields(f)
f2 = QgsVirtualLayerDefinition.fromUrl(d.toUrl()).fields()
self.assertEqual(f[0].name(), f2[0].name())
self.assertEqual(f[0].type(), f2[0].type())
self.assertEqual(f[1].name(), f2[1].name())
self.assertEqual(f[1].type(), f2[1].type())
self.assertEqual(f[2].name(), f2[2].name())
self.assertEqual(f[2].type(), f2[2].type())
if __name__ == '__main__':
unittest.main()

BIN
tests/testdata/shp_latin1.dbf vendored Normal file
View File

Binary file not shown.