Adding CSVT file reading - using CSVT file alongside data file to determine field types

resources/context_help/QgsDelimitedTextSourceSelect-en_US

@@ -26,9 +26,25 @@ describe points, lines, and polygons of arbitrary complexity.  The file can also
 only table, which can then be joined to other tables in QGIS.
 </p>
 <p>
-In addition to the geometry definition the file can contain text, integer, and real number fields.  QGIS
-will choose the type of field based on its contents. 
+In addition to the geometry definition the file can contain text, integer, and real number fields.  By default 
+QGIS will choose the type of field based on its the non blank values of the field.  If all can be interpreted
+as integer then the type will be integer, if all can be interpreted as real numbers then the type will
+be double, otherwise the type will be text.
 </p>
+<p>
+QGIS can also read the types from an OGR CSV driver compatible &quot;csvt&quot; file.  
+This is a file alongside the data file, but with a &quot;t&quot; appended to the file name. 
+The file should just contain one linewhich lists the type of each field. 
+Valid types are &quot;integer&quot;, &quot;real&quot;, &quot;string&quot;, &quot;date&quot;, &quot;time&quot;, and &quot;datetime&quot;. The date, time, and datetime types are treated as strings in QGIS.
+Each type may be followed by a width and precision, for example &quot;real(10.4)&quot;.
+The list of types are separated by commas, regardless of the delimiter used in the data file.  An
+example of a valid format file would be:
+</p>
+
+<pre>
+&quot;integer&quot;,&quot;string&quot;,&quot;string(20)&quot;,&quot;real(20.4)&quot;
+</pre>
+
 <h4><a name="creating">Creating a delimited text layer</a></h4>
 <p>Creating a delimited text layer involves choosing the data file, defining the format (how each record is to
 be split into fields), and defining the geometry is represented.  

src/providers/delimitedtext/qgsdelimitedtextprovider.cpp

@@ -19,6 +19,7 @@
 
 #include <QtGlobal>
 #include <QFile>
+#include <QFileInfo>
 #include <QDataStream>
 #include <QTextStream>
 #include <QStringList>
@@ -164,6 +165,83 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
   }
 }
 
+QStringList QgsDelimitedTextProvider::readCsvtFieldTypes( QString filename, QString *message )
+{
+  // Look for a file with the same name as the data file, but an extra 't' or 'T' at the end
+  QStringList types;
+  QFileInfo csvtInfo( filename + 't' );
+  if ( ! csvtInfo.exists() ) csvtInfo.setFile( filename + 'T' );
+  if ( ! csvtInfo.exists() ) return types;
+  QFile csvtFile( csvtInfo.filePath() );
+  if ( ! csvtFile.open( QIODevice::ReadOnly ) ) return types;
+
+
+  // If anything goes wrong here, just ignore it, as the file
+  // is not valid, so just ignore any exceptions.
+
+  // For it to be valid, there must be just one non blank line at the beginning of the
+  // file.
+
+  QString strTypeList;
+  try
+  {
+    QTextStream csvtStream( &csvtFile );
+    strTypeList = csvtStream.readLine();
+    if ( strTypeList.isEmpty() ) return types;
+    QString extra = csvtStream.readLine();
+    while ( ! extra.isNull() )
+    {
+      if ( ! extra.isEmpty() ) return types;
+      extra = csvtStream.readLine();
+    }
+  }
+  catch ( ... )
+  {
+    return types;
+  }
+  csvtFile.close();
+
+  // Is the type string valid?
+  // This is a slightly generous regular expression in that it allows spaces and unquoted field types
+  // not allowed in OGR CSVT files.  Also doesn't care if int and string fields have
+
+  strTypeList = strTypeList.toLower();
+  QRegExp reTypeList( "^(?:\\s*(\\\"?)(?:integer|real|string|date|datetime|time)(?:\\(\\d+(?:\\.\\d+)?\\))?\\1\\s*(?:,|$))+" );
+  if ( ! reTypeList.exactMatch( strTypeList ) )
+  {
+    // Looks like this was supposed to be a CSVT file, so report bad formatted string
+    if ( message ) { *message = tr( "File type string in %1 is not correctly formatted" ).arg( csvtInfo.fileName() ); }
+    return types;
+  }
+
+  // All good, so pull out the types from the string.  Currently only returning integer, real, and string types
+
+  QgsDebugMsg( QString( "Reading field types from %1" ).arg( csvtInfo.fileName() ) );
+  QgsDebugMsg( QString( "Field type string: %1" ).arg( strTypeList ) );
+
+  int pos = 0;
+  QRegExp reType( "(integer|real|string|date|datetime|time)" );
+
+  while (( pos = reType.indexIn( strTypeList, pos ) ) != -1 )
+  {
+    QgsDebugMsg( QString( "Found type: %1" ).arg( reType.cap( 1 ) ) );
+    types << reType.cap( 1 );
+    pos += reType.matchedLength();
+  }
+
+  if ( message )
+  {
+    // Would be a useful info message, but don't want dialog to pop up every time...
+    // *message=tr("Reading field types from %1").arg(csvtInfo.fileName());
+  }
+
+
+  return types;
+
+
+}
+
+
 void QgsDelimitedTextProvider::resetCachedSubset()
 {
   mCachedSubsetString = QString();
@@ -482,6 +560,10 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
   mFieldCount = fieldNames.size();
   attributeColumns.clear();
   attributeFields.clear();
+
+  QString csvtMessage;
+  QStringList csvtTypes = readCsvtFieldTypes( mFile->fileName(), &csvtMessage );
+
   for ( int i = 0; i < fieldNames.size(); i++ )
   {
     // Skip over WKT field ... don't want to display in attribute table
@@ -490,8 +572,21 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
     // Add the field index lookup for the column
     attributeColumns.append( i );
     QVariant::Type fieldType = QVariant::String;
-    QString typeName = "Text";
-    if ( i < couldBeInt.size() )
+    QString typeName = "text";
+    if ( i < csvtTypes.size() )
+    {
+      if ( csvtTypes[i] == "integer" )
+      {
+        fieldType = QVariant::Int;
+        typeName = "integer";
+      }
+      else if ( csvtTypes[i] == "real" )
+      {
+        fieldType = QVariant::Double;
+        typeName = "double";
+      }
+    }
+    else if ( i < couldBeInt.size() )
     {
       if ( couldBeInt[i] )
       {
@@ -513,6 +608,7 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes )
   QgsDebugMsg( "feature count is: " + QString::number( mNumberFeatures ) );
 
   QStringList warnings;
+  if ( ! csvtMessage.isEmpty() ) warnings.append( csvtMessage );
   if ( nBadFormatRecords > 0 )
     warnings.append( tr( "%1 records discarded due to invalid format" ).arg( nBadFormatRecords ) );
   if ( nEmptyGeometry > 0 )
@@ -1104,25 +1200,41 @@ void QgsDelimitedTextProvider::fetchAttribute( QgsFeature& feature, int fieldIdx
   switch ( attributeFields[fieldIdx].type() )
   {
     case QVariant::Int:
-      if ( value.isEmpty() )
-        val = QVariant( attributeFields[fieldIdx].type() );
+    {
+      int ivalue;
+      bool ok = false;
+      if ( ! value.isEmpty() ) ivalue = value.toInt( &ok );
+      if ( ok )
+        val = QVariant( ivalue );
       else
-        val = QVariant( value );
+        val = QVariant( attributeFields[fieldIdx].type() );
       break;
+    }
     case QVariant::Double:
-      if ( value.isEmpty() )
+    {
+      int dvalue;
+      bool ok = false;
+      if ( ! value.isEmpty() )
       {
-        val = QVariant( attributeFields[fieldIdx].type() );
+        if ( mDecimalPoint.isEmpty() )
+        {
+          dvalue = value.toDouble( &ok );
+        }
+        else
+        {
+          dvalue = QVariant( QString( value ).replace( mDecimalPoint, "." ) ).toDouble( &ok );
+        }
       }
-      else if ( mDecimalPoint.isEmpty() )
+      if ( ok )
       {
-        val = QVariant( value.toDouble() );
+        val = QVariant( dvalue );
       }
       else
       {
-        val = QVariant( QString( value ).replace( mDecimalPoint, "." ).toDouble() );
+        val = QVariant( attributeFields[fieldIdx].type() );
       }
       break;
+    }
     default:
       val = QVariant( value );
       break;

src/providers/delimitedtext/qgsdelimitedtextprovider.h

@@ -201,6 +201,14 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider
     */
     bool boundsCheck( QgsGeometry *geom );
 
+    /**
+     * Try to read field types from CSVT (or equialent xxxT) file.
+     * @param filename The name of the file from which to read the field types
+     * @param message  Pointer to a string to receive a status message
+     * @return A list of field type strings, empty if not found or not valid
+     */
+    QStringList readCsvtFieldTypes( QString filename, QString *message = 0 );
+
   private slots:
 
     void onFileUpdated();

tests/src/python/test_qgsdelimitedtextprovider.py

@@ -94,6 +94,7 @@ def layerData( layer, request={}, offset=0 ):
     first = True
     data = {}
     fields = []
+    fieldTypes = []
     fr = QgsFeatureRequest()
     if request:
         if 'exact' in request and request['exact']:
@@ -112,6 +113,7 @@ def layerData( layer, request={}, offset=0 ):
             first = False
             for field in f.fields():
                 fields.append(str(field.name()))
+                fieldTypes.append(str(field.typeName()))
         fielddata = dict ( (name, unicode(f[name].toString()) ) for name in fields )
         g = f.geometry()
         if g:
@@ -129,7 +131,7 @@ def layerData( layer, request={}, offset=0 ):
     if 'description' not in fields: fields.insert(1,'description')
     fields.append(fidkey)
     fields.append(geomkey)
-    return fields, data
+    return fields, fieldTypes, data
 
 # Retrieve the data for a delimited text url
 
@@ -157,6 +159,7 @@ def delimitedTextData( testname, filename, requests, verbose, **params ):
             basename='file'
         uri = re.sub(r'^file\:\/\/[^\?]*','file://'+basename,uri)
         fields = []
+        fieldTypes = []
         data = {}
         if layer.isValid():
             for nr,r in enumerate(requests):
@@ -165,14 +168,16 @@ def delimitedTextData( testname, filename, requests, verbose, **params ):
                 if callable(r):
                     r( layer )
                     continue
-                rfields,rdata = layerData(layer,r,nr*1000)
-                if len(rfields) > len(fields): fields = rfields
+                rfields,rtypes, rdata = layerData(layer,r,nr*1000)
+                if len(rfields) > len(fields): 
+                    fields = rfields
+                    fieldTypes=rtypes
                 data.update(rdata)
                 if not rdata:
                     log.append("Request "+str(nr)+" did not return any data")
         for msg in logger.messages():
             log.append(msg.replace(filepath,'file'))
-        return dict( fields=fields, data=data, log=log, uri=uri)
+        return dict( fields=fields, fieldTypes=fieldTypes, data=data, log=log, uri=uri)
 
 def printWanted( testname, result ):
     # Routine to export the result as a function definition
@@ -186,6 +191,7 @@ def printWanted( testname, result ):
     # Dump the data for a layer - used to construct unit tests
     print prefix+"wanted={}"
     print prefix+"wanted['uri']="+repr(result['uri'])
+    print prefix+"wanted['fieldTypes']="+repr(result['fieldTypes'])
     print prefix+"wanted['data']={"
     for k in sorted(data.keys()):
         row = data[k]
@@ -265,6 +271,10 @@ def runTest( file, requests, **params ):
             result['uri'],wanted['uri'])
         print '    '+msg
         failures.append(msg)
+    if result['fieldTypes'] != wanted['fieldTypes']:
+        msg = "Layer field types ({0}) doesn't match expected ({1})".format(
+            result['fieldTypes'],wanted['fieldTypes'])
+        failures.apend
     wanted_data = wanted['data']
     for id in sorted(wanted_data.keys()):
         wrec = wanted_data[id]
@@ -614,6 +624,34 @@ class TestQgsDelimitedTextProvider(TestCase):
         ]
         runTest(filename,requests,**params)
 
+    def test_034_csvt_file(self):
+        # CSVT field types
+        filename='testcsvt.csv'
+        params={'geomType': 'none', 'type': 'csv'}
+        requests=None
+        runTest(filename,requests,**params)
+
+    def test_035_csvt_file2(self):
+        # CSV field types 2
+        filename='testcsvt2.txt'
+        params={'geomType': 'none', 'type': 'csv', 'delimiter': '|'}
+        requests=None
+        runTest(filename,requests,**params)
+
+    def test_036_csvt_file_invalid_types(self):
+        # CSV field types invalid string format
+        filename='testcsvt3.csv'
+        params={'geomType': 'none', 'type': 'csv'}
+        requests=None
+        runTest(filename,requests,**params)
+
+    def test_037_csvt_file_invalid_file(self):
+        # CSV field types invalid file
+        filename='testcsvt4.csv'
+        params={'geomType': 'none', 'type': 'csv'}
+        requests=None
+        runTest(filename,requests,**params)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/src/python/test_qgsdelimitedtextprovider_wanted.py

@@ -2,6 +2,7 @@
 def test_002_load_csv_file():
     wanted={}
     wanted['uri']=u'file://test.csv?geomType=none&type=csv'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -66,6 +67,7 @@ def test_002_load_csv_file():
 def test_003_field_naming():
     wanted={}
     wanted['uri']=u'file://testfields.csv?geomType=none&type=csv'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -92,6 +94,7 @@ def test_003_field_naming():
 def test_004_max_fields():
     wanted={}
     wanted['uri']=u'file://testfields.csv?geomType=none&maxFields=7&type=csv'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -113,6 +116,7 @@ def test_004_max_fields():
 def test_005_load_whitespace():
     wanted={}
     wanted['uri']=u'file://test.space?geomType=none&type=whitespace'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -183,6 +187,7 @@ def test_005_load_whitespace():
 def test_006_quote_escape():
     wanted={}
     wanted['uri']=u'file://test.pipe?geomType=none&quote="&delimiter=|&escape=\\'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -273,6 +278,7 @@ def test_006_quote_escape():
 def test_007_multiple_quote():
     wanted={}
     wanted['uri']=u'file://test.quote?geomType=none&quote=\'"&type=csv&escape="\''
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -337,6 +343,7 @@ def test_007_multiple_quote():
 def test_008_badly_formed_quotes():
     wanted={}
     wanted['uri']=u'file://test.badquote?geomType=none&quote="&type=csv&escape="'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text']
     wanted['data']={
         4L: {
             'id': u'3',
@@ -360,6 +367,7 @@ def test_008_badly_formed_quotes():
 def test_009_skip_lines():
     wanted={}
     wanted['uri']=u'file://test2.csv?geomType=none&skipLines=2&type=csv&useHeader=no'
+    wanted['fieldTypes']=['integer', 'text', 'text']
     wanted['data']={
         3L: {
             'id': u'3',
@@ -379,6 +387,7 @@ def test_009_skip_lines():
 def test_010_read_coordinates():
     wanted={}
     wanted['uri']=u'file://testpt.csv?yField=geom_y&xField=geom_x&type=csv'
+    wanted['fieldTypes']=['integer', 'text', 'double', 'double']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -417,6 +426,7 @@ def test_010_read_coordinates():
 def test_011_read_wkt():
     wanted={}
     wanted['uri']=u'file://testwkt.csv?delimiter=|&type=csv&wktField=geom_wkt'
+    wanted['fieldTypes']=['integer', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -462,6 +472,7 @@ def test_011_read_wkt():
 def test_012_read_wkt_point():
     wanted={}
     wanted['uri']=u'file://testwkt.csv?geomType=point&delimiter=|&type=csv&wktField=geom_wkt'
+    wanted['fieldTypes']=['integer', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -507,6 +518,7 @@ def test_012_read_wkt_point():
 def test_013_read_wkt_line():
     wanted={}
     wanted['uri']=u'file://testwkt.csv?geomType=line&delimiter=|&type=csv&wktField=geom_wkt'
+    wanted['fieldTypes']=['integer', 'text']
     wanted['data']={
         4L: {
             'id': u'3',
@@ -552,6 +564,7 @@ def test_013_read_wkt_line():
 def test_014_read_wkt_polygon():
     wanted={}
     wanted['uri']=u'file://testwkt.csv?geomType=polygon&delimiter=|&type=csv&wktField=geom_wkt'
+    wanted['fieldTypes']=['integer', 'text']
     wanted['data']={
         6L: {
             'id': u'5',
@@ -579,6 +592,7 @@ def test_014_read_wkt_polygon():
 def test_015_read_dms_xy():
     wanted={}
     wanted['uri']=u'file://testdms.csv?yField=lat&xField=lon&type=csv&xyDms=yes'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text']
     wanted['data']={
         3L: {
             'id': u'1',
@@ -749,6 +763,7 @@ def test_015_read_dms_xy():
 def test_016_decimal_point():
     wanted={}
     wanted['uri']=u'file://testdp.csv?yField=geom_y&xField=geom_x&type=csv&delimiter=;&decimalPoint=,'
+    wanted['fieldTypes']=['integer', 'text', 'double', 'double', 'double', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -764,8 +779,8 @@ def test_016_decimal_point():
             'id': u'2',
             'description': u'Comma as decimal point 2',
             'geom_x': u'12',
-            'geom_y': u'25.003',
-            'other': u'-38.55',
+            'geom_y': u'25',
+            'other': u'-38',
             'text field': u'Plain text field',
             '#fid': 3L,
             '#geometry': 'POINT(12.0 25.003)',
@@ -779,6 +794,7 @@ def test_016_decimal_point():
 def test_017_regular_expression_1():
     wanted={}
     wanted['uri']=u'file://testre.txt?geomType=none&trimFields=Y&delimiter=RE(?:GEXP)?&type=regexp'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -805,6 +821,7 @@ def test_017_regular_expression_1():
 def test_018_regular_expression_2():
     wanted={}
     wanted['uri']=u'file://testre.txt?geomType=none&trimFields=Y&delimiter=(RE)(GEXP)?&type=regexp'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -843,6 +860,7 @@ def test_018_regular_expression_2():
 def test_019_regular_expression_3():
     wanted={}
     wanted['uri']=u'file://testre2.txt?geomType=none&trimFields=Y&delimiter=^(.{5})(.{30})(.{5,})&type=regexp'
+    wanted['fieldTypes']=['integer', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -871,6 +889,7 @@ def test_019_regular_expression_3():
 def test_020_regular_expression_4():
     wanted={}
     wanted['uri']=u'file://testre3.txt?geomType=none&delimiter=x?&type=regexp'
+    wanted['fieldTypes']=['text', 'text', 'text', 'text', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'f',
@@ -894,6 +913,7 @@ def test_020_regular_expression_4():
 def test_021_regular_expression_5():
     wanted={}
     wanted['uri']=u'file://testre3.txt?geomType=none&delimiter=\\b&type=regexp'
+    wanted['fieldTypes']=['text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'fi',
@@ -913,6 +933,7 @@ def test_021_regular_expression_5():
 def test_022_utf8_encoded_file():
     wanted={}
     wanted['uri']=u'file://testutf8.csv?geomType=none&delimiter=|&type=csv&encoding=utf-8'
+    wanted['fieldTypes']=['integer', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -930,6 +951,7 @@ def test_022_utf8_encoded_file():
 def test_023_latin1_encoded_file():
     wanted={}
     wanted['uri']=u'file://testlatin1.csv?geomType=none&delimiter=|&type=csv&encoding=latin1'
+    wanted['fieldTypes']=['integer', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -947,6 +969,7 @@ def test_023_latin1_encoded_file():
 def test_024_filter_rect_xy():
     wanted={}
     wanted['uri']=u'file://testextpt.txt?yField=y&delimiter=|&type=csv&xField=x'
+    wanted['fieldTypes']=['integer', 'text', 'integer', 'integer']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -990,6 +1013,7 @@ def test_024_filter_rect_xy():
 def test_025_filter_rect_wkt():
     wanted={}
     wanted['uri']=u'file://testextw.txt?delimiter=|&type=csv&wktField=wkt'
+    wanted['fieldTypes']=['integer', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -1049,6 +1073,7 @@ def test_025_filter_rect_wkt():
 def test_026_filter_fid():
     wanted={}
     wanted['uri']=u'file://test.csv?geomType=none&type=csv'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text', 'text']
     wanted['data']={
         3L: {
             'id': u'2',
@@ -1087,6 +1112,7 @@ def test_026_filter_fid():
 def test_027_filter_attributes():
     wanted={}
     wanted['uri']=u'file://test.csv?geomType=none&type=csv'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'',
@@ -1196,6 +1222,7 @@ def test_027_filter_attributes():
 def test_028_substring_test():
     wanted={}
     wanted['uri']=u'file://test.csv?geomType=none&type=csv&subset=id%20%25%202%20%3D%201'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -1233,6 +1260,7 @@ def test_028_substring_test():
 def test_029_file_watcher():
     wanted={}
     wanted['uri']=u'file://file?geomType=none&type=csv&watchFile=yes'
+    wanted['fieldTypes']=['integer', 'text']
     wanted['data']={
         3L: {
             'id': u'2',
@@ -1355,6 +1383,7 @@ def test_029_file_watcher():
 def test_030_filter_rect_xy_spatial_index():
     wanted={}
     wanted['uri']=u'file://testextpt.txt?spatialIndex=Y&yField=y&delimiter=|&type=csv&xField=x'
+    wanted['fieldTypes']=['integer', 'text', 'integer', 'integer']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -1542,6 +1571,7 @@ def test_030_filter_rect_xy_spatial_index():
 def test_031_filter_rect_wkt_spatial_index():
     wanted={}
     wanted['uri']=u'file://testextw.txt?spatialIndex=Y&delimiter=|&type=csv&wktField=wkt'
+    wanted['fieldTypes']=['integer', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -1673,6 +1703,7 @@ def test_031_filter_rect_wkt_spatial_index():
 def test_032_filter_rect_wkt_create_spatial_index():
     wanted={}
     wanted['uri']=u'file://testextw.txt?delimiter=|&type=csv&wktField=wkt'
+    wanted['fieldTypes']=['integer', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -1866,9 +1897,11 @@ def test_032_filter_rect_wkt_create_spatial_index():
         ]
     return wanted
 
+
 def test_033_reset_subset_string():
     wanted={}
     wanted['uri']=u'file://test.csv?geomType=none&type=csv'
+    wanted['fieldTypes']=['integer', 'text', 'text', 'text', 'text']
     wanted['data']={
         2L: {
             'id': u'1',
@@ -2028,3 +2061,137 @@ def test_033_reset_subset_string():
         ]
     return wanted
 
+
+def test_034_csvt_file():
+    wanted={}
+    wanted['uri']=u'file://testcsvt.csv?geomType=none&type=csv'
+    wanted['fieldTypes']=['integer', 'text', 'integer', 'double', 'text', 'text', 'text']
+    wanted['data']={
+        2L: {
+            'id': u'1',
+            'description': u'Test csvt 1',
+            'f1': u'1',
+            'f2': u'1',
+            'f3': u'01',
+            'f4': u'text',
+            'f5': u'times',
+            '#fid': 2L,
+            '#geometry': 'None',
+            },
+        3L: {
+            'id': u'2',
+            'description': u'Test csvt 2',
+            'f1': u'3',
+            'f2': u'1',
+            'f3': u'99',
+            'f4': u'23.5',
+            'f5': u'80',
+            '#fid': 3L,
+            '#geometry': 'None',
+            },
+        }
+    wanted['log']=[
+        ]
+    return wanted
+
+
+def test_035_csvt_file2():
+    wanted={}
+    wanted['uri']=u'file://testcsvt2.txt?geomType=none&delimiter=|&type=csv'
+    wanted['fieldTypes']=['integer', 'text', 'integer', 'double', 'integer', 'text', 'integer']
+    wanted['data']={
+        2L: {
+            'id': u'1',
+            'description': u'Test csvt 1',
+            'f1': u'1',
+            'f2': u'1',
+            'f3': u'1',
+            'f4': u'text',
+            'f5': u'0',
+            '#fid': 2L,
+            '#geometry': 'None',
+            },
+        3L: {
+            'id': u'2',
+            'description': u'Test csvt 2',
+            'f1': u'3',
+            'f2': u'1',
+            'f3': u'99',
+            'f4': u'23.5',
+            'f5': u'80',
+            '#fid': 3L,
+            '#geometry': 'None',
+            },
+        }
+    wanted['log']=[
+        ]
+    return wanted
+
+
+def test_036_csvt_file_invalid_types():
+    wanted={}
+    wanted['uri']=u'file://testcsvt3.csv?geomType=none&type=csv'
+    wanted['fieldTypes']=['integer', 'text', 'integer', 'double', 'integer', 'text', 'text']
+    wanted['data']={
+        2L: {
+            'id': u'1',
+            'description': u'Test csvt 1',
+            'f1': u'1',
+            'f2': u'1',
+            'f3': u'1',
+            'f4': u'text',
+            'f5': u'times',
+            '#fid': 2L,
+            '#geometry': 'None',
+            },
+        3L: {
+            'id': u'2',
+            'description': u'Test csvt 2',
+            'f1': u'3',
+            'f2': u'1',
+            'f3': u'99',
+            'f4': u'23.5',
+            'f5': u'80',
+            '#fid': 3L,
+            '#geometry': 'None',
+            },
+        }
+    wanted['log']=[
+        u'Errors in file file',
+        u'File type string in testcsvt3.csvt is not correctly formatted',
+        ]
+    return wanted
+
+
+def test_037_csvt_file_invalid_file():
+    wanted={}
+    wanted['uri']=u'file://testcsvt4.csv?geomType=none&type=csv'
+    wanted['fieldTypes']=['integer', 'text', 'integer', 'double', 'integer', 'text', 'text']
+    wanted['data']={
+        2L: {
+            'id': u'1',
+            'description': u'Test csvt 1',
+            'f1': u'1',
+            'f2': u'1',
+            'f3': u'1',
+            'f4': u'text',
+            'f5': u'times',
+            '#fid': 2L,
+            '#geometry': 'None',
+            },
+        3L: {
+            'id': u'2',
+            'description': u'Test csvt 2',
+            'f1': u'3',
+            'f2': u'1',
+            'f3': u'99',
+            'f4': u'23.5',
+            'f5': u'80',
+            '#fid': 3L,
+            '#geometry': 'None',
+            },
+        }
+    wanted['log']=[
+        ]
+    return wanted
+

tests/testdata/delimitedtext/testcsvt.csv

@@ -0,0 +1,4 @@
+id,description,f1,f2,f3,f4,f5
+1,Test csvt 1,1,1.2,01,text,times
+2,Test csvt 2,3,1.5,99,23.5,80
+

tests/testdata/delimitedtext/testcsvt.csvt

@@ -0,0 +1 @@
+integer,string,integer,real,string,string,datetime

tests/testdata/delimitedtext/testcsvt2.txt

@@ -0,0 +1,4 @@
+id|description|f1|f2|f3|f4|f5
+1|Test csvt 1|1|1.2|01|text|times
+2|Test csvt 2|3|1.5|99|23.5|80
+

tests/testdata/delimitedtext/testcsvt2.txtt

@@ -0,0 +1 @@
+"integer(5)","string(30)","integer(2)","real","integer","string(20)","integer"

tests/testdata/delimitedtext/testcsvt3.csv

@@ -0,0 +1,4 @@
+id,description,f1,f2,f3,f4,f5
+1,Test csvt 1,1,1.2,01,text,times
+2,Test csvt 2,3,1.5,99,23.5,80
+

tests/testdata/delimitedtext/testcsvt3.csvt

@@ -0,0 +1,5 @@
+integer,string,integer,gotitwrong,string,string,datetime
+
+
+
+

tests/testdata/delimitedtext/testcsvt4.csv

@@ -0,0 +1,4 @@
+id,description,f1,f2,f3,f4,f5
+1,Test csvt 1,1,1.2,01,text,times
+2,Test csvt 2,3,1.5,99,23.5,80
+

tests/testdata/delimitedtext/testcsvt4.csvt

@@ -0,0 +1,2 @@
+This is nothing like a CSVT file
+Just happened to be given the same name