Improve handling of time values in attributes

- make QgsVectorFileWriter respect QTime field values and only
convert to string if required by output format
- list time types as native types for memory, postgres and OGR
providers (OGR support depends on file format)
This commit is contained in:
Nyall Dawson 2016-01-28 19:22:49 +11:00
parent 082f113197
commit 7dc5eac8bd
18 changed files with 312 additions and 19 deletions

View File

@ -79,6 +79,14 @@ class QgsExpressionSorter
else
return v1.toDate() > v2.toDate();
case QVariant::Time:
if ( v1.toTime() == v2.toTime() )
continue;
if ( orderBy.ascending() )
return v1.toTime() < v2.toTime();
else
return v1.toTime() > v2.toTime();
case QVariant::DateTime:
if ( v1.toDateTime() == v2.toDateTime() )
continue;

View File

@ -103,14 +103,13 @@ void QgsVectorFileWriter::init( QString vectorFileName, QString fileEncoding, co
return;
}
QString ogrDriverName;
if ( driverName == "MapInfo MIF" )
{
ogrDriverName = "MapInfo File";
mOgrDriverName = "MapInfo File";
}
else if ( driverName == "SpatiaLite" )
{
ogrDriverName = "SQLite";
mOgrDriverName = "SQLite";
if ( !datasourceOptions.contains( "SPATIALITE=YES" ) )
{
datasourceOptions.append( "SPATIALITE=YES" );
@ -118,7 +117,7 @@ void QgsVectorFileWriter::init( QString vectorFileName, QString fileEncoding, co
}
else if ( driverName == "DBF file" )
{
ogrDriverName = "ESRI Shapefile";
mOgrDriverName = "ESRI Shapefile";
if ( !layerOptions.contains( "SHPT=NULL" ) )
{
layerOptions.append( "SHPT=NULL" );
@ -127,14 +126,14 @@ void QgsVectorFileWriter::init( QString vectorFileName, QString fileEncoding, co
}
else
{
ogrDriverName = driverName;
mOgrDriverName = driverName;
}
// find driver in OGR
OGRSFDriverH poDriver;
QgsApplication::registerOgrDrivers();
poDriver = OGRGetDriverByName( ogrDriverName.toLocal8Bit().data() );
poDriver = OGRGetDriverByName( mOgrDriverName.toLocal8Bit().data() );
if ( !poDriver )
{
@ -145,7 +144,7 @@ void QgsVectorFileWriter::init( QString vectorFileName, QString fileEncoding, co
return;
}
if ( ogrDriverName == "ESRI Shapefile" )
if ( mOgrDriverName == "ESRI Shapefile" )
{
if ( layerOptions.join( "" ).toUpper().indexOf( "ENCODING=" ) == -1 )
{
@ -315,7 +314,7 @@ void QgsVectorFileWriter::init( QString vectorFileName, QString fileEncoding, co
if ( srs )
{
if ( ogrDriverName == "ESRI Shapefile" )
if ( mOgrDriverName == "ESRI Shapefile" )
{
QString layerName = vectorFileName.left( vectorFileName.indexOf( ".shp", Qt::CaseInsensitive ) );
QFile prjFile( layerName + ".qpj" );
@ -389,6 +388,18 @@ void QgsVectorFileWriter::init( QString vectorFileName, QString fileEncoding, co
ogrType = OFTDate;
break;
case QVariant::Time:
if ( mOgrDriverName == "ESRI Shapefile" )
{
ogrType = OFTString;
ogrWidth = 12;
}
else
{
ogrType = OFTTime;
}
break;
case QVariant::DateTime:
ogrType = OFTDateTime;
break;
@ -403,7 +414,7 @@ void QgsVectorFileWriter::init( QString vectorFileName, QString fileEncoding, co
QString name( attrField.name() );
if ( ogrDriverName == "SQLite" && name.compare( "ogc_fid", Qt::CaseInsensitive ) == 0 )
if ( mOgrDriverName == "SQLite" && name.compare( "ogc_fid", Qt::CaseInsensitive ) == 0 )
{
int i;
for ( i = 0; i < 10; i++ )
@ -1778,12 +1789,19 @@ OGRFeatureH QgsVectorFileWriter::createFeature( QgsFeature& feature )
0 );
break;
case QVariant::Time:
OGR_F_SetFieldDateTime( poFeature, ogrField,
0, 0, 0,
attrValue.toDateTime().time().hour(),
attrValue.toDateTime().time().minute(),
attrValue.toDateTime().time().second(),
0 );
if ( mOgrDriverName == "ESRI Shapefile" )
{
OGR_F_SetFieldString( poFeature, ogrField, mCodec->fromUnicode( attrValue.toString() ).data() );
}
else
{
OGR_F_SetFieldDateTime( poFeature, ogrField,
0, 0, 0,
attrValue.toTime().hour(),
attrValue.toTime().minute(),
attrValue.toTime().second(),
0 );
}
break;
case QVariant::Invalid:
break;

View File

@ -352,6 +352,8 @@ class CORE_EXPORT QgsVectorFileWriter
/** Scale for symbology export (e.g. for symbols units in map units)*/
double mSymbologyScaleDenominator;
QString mOgrDriverName;
private:
void init( QString vectorFileName, QString fileEncoding, const QgsFields& fields, QgsWKBTypes::Type geometryType, const QgsCoordinateReferenceSystem* srs, const QString& driverName, QStringList datasourceOptions, QStringList layerOptions, QString* newFilename );

View File

@ -80,6 +80,9 @@ bool QgsAttributeTableFilterModel::lessThan( const QModelIndex &left, const QMod
case QVariant::Date:
return leftData.toDate() < rightData.toDate();
case QVariant::Time:
return leftData.toTime() < rightData.toTime();
case QVariant::DateTime:
return leftData.toDateTime() < rightData.toDateTime();

View File

@ -86,6 +86,7 @@ QgsMemoryProvider::QgsMemoryProvider( const QString& uri )
// date type
<< QgsVectorDataProvider::NativeType( tr( "Date" ), "date", QVariant::Date, -1, -1, -1, -1 )
<< QgsVectorDataProvider::NativeType( tr( "Time" ), "time", QVariant::Time, -1, -1, -1, -1 )
<< QgsVectorDataProvider::NativeType( tr( "Date & Time" ), "datetime", QVariant::DateTime, -1, -1, -1, -1 )
// integer types
@ -107,7 +108,7 @@ QgsMemoryProvider::QgsMemoryProvider( const QString& uri )
{
QList<QgsField> attributes;
QRegExp reFieldDef( "\\:"
"(int|integer|real|double|string|date|datetime)" // type
"(int|integer|real|double|string|date|time|datetime)" // type
"(?:\\((\\d+)" // length
"(?:\\,(\\d+))?" // precision
"\\))?"
@ -145,6 +146,12 @@ QgsMemoryProvider::QgsMemoryProvider( const QString& uri )
typeName = "date";
length = -1;
}
else if ( typeName == "time" )
{
type = QVariant::Time;
typeName = "time";
length = -1;
}
else if ( typeName == "datetime" )
{
type = QVariant::DateTime;
@ -357,6 +364,7 @@ bool QgsMemoryProvider::addAttributes( const QList<QgsField> &attributes )
case QVariant::Double:
case QVariant::String:
case QVariant::Date:
case QVariant::Time:
case QVariant::DateTime:
case QVariant::LongLong:
break;

View File

@ -295,12 +295,15 @@ void QgsOgrFeatureIterator::getFeatureAttribute( OGRFeatureH ogrFet, QgsFeature
break;
case QVariant::Date:
case QVariant::DateTime:
case QVariant::Time:
{
int year, month, day, hour, minute, second, tzf;
OGR_F_GetFieldAsDateTime( ogrFet, attindex, &year, &month, &day, &hour, &minute, &second, &tzf );
if ( mSource->mFields.at( attindex ).type() == QVariant::Date )
value = QDate( year, month, day );
else if ( mSource->mFields.at( attindex ).type() == QVariant::Time )
value = QTime( hour, minute, second );
else
value = QDateTime( QDate( year, month, day ), QTime( hour, minute, second ) );
}

View File

@ -119,6 +119,10 @@ bool QgsOgrProvider::convertField( QgsField &field, const QTextCodec &encoding )
ogrType = OFTDate;
break;
case QVariant::Time:
ogrType = OFTTime;
break;
case QVariant::DateTime:
ogrType = OFTDateTime;
break;
@ -366,6 +370,7 @@ QgsOgrProvider::QgsOgrProvider( QString const & uri )
if ( ogrDriverName != "ESRI Shapefile" )
{
mNativeTypes
<< QgsVectorDataProvider::NativeType( tr( "Time" ), "time", QVariant::Time, -1, -1 )
<< QgsVectorDataProvider::NativeType( tr( "Date & Time" ), "datetime", QVariant::DateTime );
}
@ -721,6 +726,9 @@ void QgsOgrProvider::loadFields()
case OFTDate:
varType = QVariant::Date;
break;
case OFTTime:
varType = QVariant::Time;
break;
case OFTDateTime:
varType = QVariant::DateTime;
break;
@ -1006,6 +1014,16 @@ bool QgsOgrProvider::addFeature( QgsFeature& f )
0, 0, 0,
0 );
break;
case OFTTime:
OGR_F_SetFieldDateTime( feature, targetAttributeId,
0, 0, 0,
attrVal.toTime().hour(),
attrVal.toTime().minute(),
attrVal.toTime().second(),
0 );
break;
case OFTDateTime:
OGR_F_SetFieldDateTime( feature, targetAttributeId,
attrVal.toDateTime().date().year(),
@ -1099,6 +1117,9 @@ bool QgsOgrProvider::addAttributes( const QList<QgsField> &attributes )
case QVariant::Date:
type = OFTDate;
break;
case QVariant::Time:
type = OFTTime;
break;
case QVariant::DateTime:
type = OFTDateTime;
break;
@ -1227,6 +1248,14 @@ bool QgsOgrProvider::changeAttributeValues( const QgsChangedAttributesMap &attr_
0, 0, 0,
0 );
break;
case OFTTime:
OGR_F_SetFieldDateTime( of, f,
0, 0, 0,
it2->toTime().hour(),
it2->toTime().minute(),
it2->toTime().second(),
0 );
break;
case OFTDateTime:
OGR_F_SetFieldDateTime( of, f,
it2->toDateTime().date().year(),
@ -2285,6 +2314,10 @@ QGISEXTERN bool createEmptyDataSource( const QString &uri,
{
field = OGR_Fld_Create( codec->fromUnicode( it->first ).constData(), OFTDate );
}
else if ( fields[0] == "Time" )
{
field = OGR_Fld_Create( codec->fromUnicode( it->first ).constData(), OFTTime );
}
else if ( fields[0] == "DateTime" )
{
field = OGR_Fld_Create( codec->fromUnicode( it->first ).constData(), OFTDateTime );

View File

@ -179,6 +179,7 @@ QgsPostgresProvider::QgsPostgresProvider( QString const & uri )
// date type
<< QgsVectorDataProvider::NativeType( tr( "Date" ), "date", QVariant::Date, -1, -1, -1, -1 )
<< QgsVectorDataProvider::NativeType( tr( "Time" ), "time", QVariant::Time, -1, -1, -1, -1 )
<< QgsVectorDataProvider::NativeType( tr( "Date & Time" ), "timestamp without time zone", QVariant::DateTime, -1, -1, -1, -1 )
;
@ -875,6 +876,11 @@ bool QgsPostgresProvider::loadFields()
fieldType = QVariant::Date;
fieldSize = -1;
}
else if ( fieldTypeName == "time" )
{
fieldType = QVariant::Time;
fieldSize = -1;
}
else if ( fieldTypeName == "timestamp" )
{
fieldType = QVariant::DateTime;

View File

@ -55,6 +55,7 @@ ADD_PYTHON_TEST(PyQgsRelation test_qgsrelation.py)
ADD_PYTHON_TEST(PyQgsRulebasedRenderer test_qgsrulebasedrenderer.py)
ADD_PYTHON_TEST(PyQgsSingleSymbolRenderer test_qgssinglesymbolrenderer.py)
ADD_PYTHON_TEST(PyQgsShapefileProvider test_provider_shapefile.py)
ADD_PYTHON_TEST(PyQgsTabfileProvider test_provider_tabfile.py)
ADD_PYTHON_TEST(PyQgsSpatialIndex test_qgsspatialindex.py)
ADD_PYTHON_TEST(PyQgsSpatialiteProvider test_provider_spatialite.py)
ADD_PYTHON_TEST(PyQgsSymbolLayerV2 test_qgssymbollayerv2.py)

View File

@ -193,6 +193,7 @@ class TestPyQgsMemoryProvider(TestCase, ProviderTestCase):
QgsField('TestDbl', QVariant.Double, 'double', 8, 6),
QgsField('TestString', QVariant.String, 'string', 50, 0),
QgsField('TestDate', QVariant.Date, 'date'),
QgsField('TestTime', QVariant.Time, 'time'),
QgsField('TestDateTime', QVariant.DateTime, 'datetime')]
assert myMemoryLayer.startEditing()
for f in myFields:

View File

@ -18,7 +18,7 @@ import sys
from qgis.core import NULL
from qgis.core import QgsVectorLayer, QgsFeatureRequest, QgsFeature, QgsProviderRegistry
from PyQt4.QtCore import QSettings
from PyQt4.QtCore import QSettings, QDate, QTime, QDateTime, QVariant
from utilities import (unitTestDataPath,
getQgisTestApp,
unittest,
@ -59,6 +59,24 @@ class TestPyQgsPostgresProvider(TestCase, ProviderTestCase):
assert self.provider.defaultValue(1) == NULL
assert self.provider.defaultValue(2) == '\'qgis\'::text'
def testDateTimeTypes(self):
vl = QgsVectorLayer('%s table="qgis_test"."date_times" sql=' % (self.dbconn), "testdatetimes", "postgres")
assert(vl.isValid())
fields = vl.dataProvider().fields()
self.assertEqual(fields.at(fields.indexFromName('date_field')).type(), QVariant.Date)
self.assertEqual(fields.at(fields.indexFromName('time_field')).type(), QVariant.Time)
self.assertEqual(fields.at(fields.indexFromName('datetime_field')).type(), QVariant.DateTime)
f = vl.getFeatures(QgsFeatureRequest()).next()
date_idx = vl.fieldNameIndex('date_field')
assert isinstance(f.attributes()[date_idx], QDate)
time_idx = vl.fieldNameIndex('time_field')
assert isinstance(f.attributes()[time_idx], QTime)
datetime_idx = vl.fieldNameIndex('datetime_field')
assert isinstance(f.attributes()[datetime_idx], QDateTime)
def testQueryLayers(self):
def test_query(dbconn, query, key):
ql = QgsVectorLayer('%s srid=4326 table="%s" (geom) key=\'%s\' sql=' % (dbconn, query.replace('"', '\\"'), key), "testgeom", "postgres")

View File

@ -0,0 +1,56 @@
# -*- coding: utf-8 -*-
"""QGIS Unit tests for the OGR/MapInfo tab provider.
.. note:: This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
"""
__author__ = 'Nyall Dawson'
__date__ = '2016-01-28'
__copyright__ = 'Copyright 2016, The QGIS Project'
# This will get replaced with a git SHA1 when you do a git archive
__revision__ = '$Format:%H$'
import os
import tempfile
import shutil
import glob
from qgis.core import QgsVectorLayer, QgsFeatureRequest, QgsFeature, QgsProviderRegistry
from PyQt4.QtCore import QSettings, QDate, QTime, QDateTime, QVariant
from utilities import (unitTestDataPath,
getQgisTestApp,
unittest,
TestCase
)
QGISAPP, CANVAS, IFACE, PARENT = getQgisTestApp()
TEST_DATA_DIR = unitTestDataPath()
# Note - doesn't implement ProviderTestCase as OGR provider is tested by the shapefile provider test
class TestPyQgsTabfileProvider(TestCase):
def testDateTimeFormats(self):
# check that date and time formats are correctly interpreted
basetestfile = os.path.join(TEST_DATA_DIR, 'tab_file.tab')
vl = QgsVectorLayer(u'{}|layerid=0'.format(basetestfile), u'test', u'ogr')
fields = vl.dataProvider().fields()
self.assertEqual(fields.at(fields.indexFromName('date')).type(), QVariant.Date)
self.assertEqual(fields.at(fields.indexFromName('time')).type(), QVariant.Time)
self.assertEqual(fields.at(fields.indexFromName('date_time')).type(), QVariant.DateTime)
f = vl.getFeatures(QgsFeatureRequest()).next()
date_idx = vl.fieldNameIndex('date')
assert isinstance(f.attributes()[date_idx], QDate)
time_idx = vl.fieldNameIndex('time')
assert isinstance(f.attributes()[time_idx], QTime)
datetime_idx = vl.fieldNameIndex('date_time')
assert isinstance(f.attributes()[datetime_idx], QDateTime)
if __name__ == '__main__':
unittest.main()

View File

@ -17,9 +17,13 @@ import qgis
from qgis.core import (QgsVectorLayer,
QgsFeature,
QgsGeometry,
QgsPoint
QgsPoint,
QgsCoordinateReferenceSystem,
QgsVectorFileWriter,
QgsFeatureRequest
)
from PyQt4.QtCore import QDate, QTime, QDateTime, QVariant, QDir
import os
from utilities import (getQgisTestApp,
TestCase,
unittest,
@ -53,5 +57,113 @@ class TestQgsVectorLayer(TestCase):
writeShape(self.mMemoryLayer, 'writetest.shp')
def testDateTimeWriteShapefile(self):
"""Check writing date and time fields to an ESRI shapefile."""
ml = QgsVectorLayer(
('Point?crs=epsg:4326&field=id:int&'
'field=date_f:date&field=time_f:time&field=dt_f:datetime'),
'test',
'memory')
assert ml is not None, 'Provider not initialised'
assert ml.isValid(), 'Source layer not valid'
provider = ml.dataProvider()
assert provider is not None
ft = QgsFeature()
ft.setGeometry(QgsGeometry.fromPoint(QgsPoint(10, 10)))
ft.setAttributes([1, QDate(2014, 0o3, 0o5), QTime(13, 45, 22), QDateTime(QDate(2014, 0o3, 0o5), QTime(13, 45, 22))])
res, features = provider.addFeatures([ft])
assert res
assert len(features) > 0
dest_file_name = os.path.join(str(QDir.tempPath()), 'datetime.shp')
print(dest_file_name)
crs = QgsCoordinateReferenceSystem()
crs.createFromId(4326, QgsCoordinateReferenceSystem.EpsgCrsId)
write_result = QgsVectorFileWriter.writeAsVectorFormat(
ml,
dest_file_name,
'utf-8',
crs,
'ESRI Shapefile')
self.assertEqual(write_result, QgsVectorFileWriter.NoError)
# Open result and check
created_layer = QgsVectorLayer(u'{}|layerid=0'.format(dest_file_name), u'test', u'ogr')
fields = created_layer.dataProvider().fields()
self.assertEqual(fields.at(fields.indexFromName('date_f')).type(), QVariant.Date)
#shapefiles do not support time types, result should be string
self.assertEqual(fields.at(fields.indexFromName('time_f')).type(), QVariant.String)
#shapefiles do not support datetime types, result should be date
self.assertEqual(fields.at(fields.indexFromName('dt_f')).type(), QVariant.Date)
f = created_layer.getFeatures(QgsFeatureRequest()).next()
date_idx = created_layer.fieldNameIndex('date_f')
assert isinstance(f.attributes()[date_idx], QDate)
self.assertEqual(f.attributes()[date_idx], QDate(2014, 0o3, 0o5))
time_idx = created_layer.fieldNameIndex('time_f')
#shapefiles do not support time types
assert isinstance(f.attributes()[time_idx], basestring)
self.assertEqual(f.attributes()[time_idx], '13:45:22')
#shapefiles do not support datetime types
datetime_idx = created_layer.fieldNameIndex('dt_f')
assert isinstance(f.attributes()[datetime_idx], QDate)
self.assertEqual(f.attributes()[datetime_idx], QDate(2014, 0o3, 0o5))
def testDateTimeWriteTabfile(self):
"""Check writing date and time fields to an MapInfo tabfile."""
ml = QgsVectorLayer(
('Point?crs=epsg:4326&field=id:int&'
'field=date_f:date&field=time_f:time&field=dt_f:datetime'),
'test',
'memory')
assert ml is not None, 'Provider not initialised'
assert ml.isValid(), 'Source layer not valid'
provider = ml.dataProvider()
assert provider is not None
ft = QgsFeature()
ft.setGeometry(QgsGeometry.fromPoint(QgsPoint(10, 10)))
ft.setAttributes([1, QDate(2014, 0o3, 0o5), QTime(13, 45, 22), QDateTime(QDate(2014, 0o3, 0o5), QTime(13, 45, 22))])
res, features = provider.addFeatures([ft])
assert res
assert len(features) > 0
dest_file_name = os.path.join(str(QDir.tempPath()), 'datetime.tab')
print(dest_file_name)
crs = QgsCoordinateReferenceSystem()
crs.createFromId(4326, QgsCoordinateReferenceSystem.EpsgCrsId)
write_result = QgsVectorFileWriter.writeAsVectorFormat(
ml,
dest_file_name,
'utf-8',
crs,
'MapInfo File')
self.assertEqual(write_result, QgsVectorFileWriter.NoError)
# Open result and check
created_layer = QgsVectorLayer(u'{}|layerid=0'.format(dest_file_name), u'test', u'ogr')
fields = created_layer.dataProvider().fields()
self.assertEqual(fields.at(fields.indexFromName('date_f')).type(), QVariant.Date)
self.assertEqual(fields.at(fields.indexFromName('time_f')).type(), QVariant.Time)
self.assertEqual(fields.at(fields.indexFromName('dt_f')).type(), QVariant.DateTime)
f = created_layer.getFeatures(QgsFeatureRequest()).next()
date_idx = created_layer.fieldNameIndex('date_f')
assert isinstance(f.attributes()[date_idx], QDate)
self.assertEqual(f.attributes()[date_idx], QDate(2014, 0o3, 0o5))
time_idx = created_layer.fieldNameIndex('time_f')
assert isinstance(f.attributes()[time_idx], QTime)
self.assertEqual(f.attributes()[time_idx], QTime(13, 45, 22))
datetime_idx = created_layer.fieldNameIndex('dt_f')
assert isinstance(f.attributes()[datetime_idx], QDateTime)
self.assertEqual(f.attributes()[datetime_idx], QDateTime(QDate(2014, 0o3, 0o5), QTime(13, 45, 22)))
if __name__ == '__main__':
unittest.main()

View File

@ -73,6 +73,15 @@ ALTER TABLE ONLY qgis_test."someData"
-- PostgreSQL database dump complete
--
CREATE TABLE qgis_test.date_times(
id int,
date_field date,
time_field time,
datetime_field timestamp without time zone
);
INSERT INTO qgis_test.date_times values (1, '2004-03-04'::date, '13:41:52'::time, '2004-03-04 13:41:52'::timestamp without time zone );
CREATE TABLE qgis_test.p2d(
id int,
geom Geometry(Polygon,4326)

BIN
tests/testdata/tab_file.dat vendored Executable file

Binary file not shown.

BIN
tests/testdata/tab_file.id vendored Executable file

Binary file not shown.

BIN
tests/testdata/tab_file.map vendored Executable file

Binary file not shown.

15
tests/testdata/tab_file.tab vendored Executable file
View File

@ -0,0 +1,15 @@
!table
!version 900
!charset WindowsLatin1
Definition Table
Type NATIVE Charset "WindowsLatin1"
Fields 3
date Date ;
date_time DateTime ;
time Time ;
begin_metadata
"\IsReadOnly" = "FALSE"
"\MapInfo" = ""
"\MapInfo\TableID" = "bff4e165-c593-457c-a460-6c425cc13dc7"
end_metadata