sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-10-06 00:07:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

Error message fixes and more regexp testing

Browse Source
This commit is contained in:
Chris Crook 2013-04-18 19:59:34 +12:00
parent 8570646a2a
commit b501c9a133
8 changed files with 174 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
resources/context_help/QgsDelimitedTextSourceSelect-en_US
View File

@ -8,7 +8,7 @@ Loads and displays delimited text files
<a href="#wkt">How WKT text is interpreted</a><br />
<a href="#example">Example of a text file with X,Y point coordinates</a><br/>
<a href="#wkt_example">Example of a text file with WKT geometries</a><br/>
<a href="#notes">Notes</a><br/>
<a href="#python">Using delimited text layers in Python</a><br/>
</p>
<h4><a name="re">Overview</a></h4>
@ -170,3 +170,5 @@ id|wkt<br />
<li>Uses <b>|</b> as a delimiter.</li>
<li>Specifies each point using the WKT notation
</ul>
<h4><a name="python">Using delimited text layers in Python</a></h4>

15
src/providers/delimitedtext/qgsdelimitedtextfile.cpp
View File

@ -180,6 +180,10 @@ bool QgsDelimitedTextFile::setFromUrl( QUrl &url )
{
mTrimFields = ! url.queryItemValue( "trimFields" ).toUpper().startsWith( 'N' );;
}
if ( url.hasQueryItem( "maxFields" ) )
{
mMaxFields = url.queryItemValue( "maxFields" ).toInt();
}
QgsDebugMsg( "Delimited text file is: " + mFileName );
QgsDebugMsg( "Encoding is: " + mEncoding );
@ -188,6 +192,7 @@ bool QgsDelimitedTextFile::setFromUrl( QUrl &url )
QgsDebugMsg( "Quote character is: [" + quote + "]" );
QgsDebugMsg( "Escape character is: [" + escape + "]" );
QgsDebugMsg( "Skip lines: " + QString::number( mSkipLines ) );
QgsDebugMsg( "Maximum number of fields in record: " + QString::number( mMaxFields ) );
QgsDebugMsg( "Use headers: " + QString( mUseHeader ? "Yes" : "No" ) );
QgsDebugMsg( "Discard empty fields: " + QString( mDiscardEmptyFields ? "Yes" : "No" ) );
QgsDebugMsg( "Trim fields: " + QString( mTrimFields ? "Yes" : "No" ) );
@ -246,6 +251,10 @@ QUrl QgsDelimitedTextFile::url()
{
url.addQueryItem( "skipEmptyFields", "Yes" );
}
if ( mMaxFields > 0 )
{
url.addQueryItem( "maxFields", QString::number( mMaxFields ) );
}
return url;
}
@ -341,6 +350,12 @@ void QgsDelimitedTextFile::setTrimFields( bool trimFields )
mTrimFields = trimFields;
}
void QgsDelimitedTextFile::setMaxFields( int maxFields )
{
resetDefinition();
mMaxFields = maxFields;
}
void QgsDelimitedTextFile::setDiscardEmptyFields( bool discardEmptyFields )
{
resetDefinition();

37
src/providers/delimitedtext/qgsdelimitedtextfile.h
View File

@ -142,11 +142,11 @@ class QgsDelimitedTextFile
*/
void setTypeCSV( QString delim = QString( "," ), QString quote = QString( "\"" ), QString escape = QString( "\"" ) );
/* Set the number of header lines to skip
/** Set the number of header lines to skip
* @param skiplines The maximum lines to skip
*/
void setSkipLines( int skiplines );
/* Return the number of header lines to skip
/** Return the number of header lines to skip
* @return skiplines The maximum lines to skip
*/
int skipLines()
@ -154,11 +154,11 @@ class QgsDelimitedTextFile
return mSkipLines;
}
/* Set reading field names from the first record
/** Set reading field names from the first record
* @param useheaders Field names will be read if true
*/
void setUseHeader( bool useheader = true );
/* Return the option for reading field names from the first record
/** Return the option for reading field names from the first record
* @return useheaders Field names will be read if true
*/
bool useHeader()
@ -166,11 +166,11 @@ class QgsDelimitedTextFile
return mUseHeader;
}
/* Set the option for dicarding empty fields
/** Set the option for dicarding empty fields
* @param useheaders Empty fields will be discarded if true
*/
void setDiscardEmptyFields( bool discardEmptyFields = true );
/* Return the option for discarding empty fields
/** Return the option for discarding empty fields
* @return useheaders Empty fields will be discarded if true
*/
bool discardEmptyFields()
@ -178,11 +178,11 @@ class QgsDelimitedTextFile
return mDiscardEmptyFields;
}
/* Set the option for trimming whitespace from fields
/** Set the option for trimming whitespace from fields
* @param trimFields Fields will be trimmed if true
*/
void setTrimFields( bool trimFields = true );
/* Return the option for trimming empty fields
/** Return the option for trimming empty fields
* @return useheaders Empty fields will be trimmed if true
*/
bool trimFields()
@ -190,16 +190,31 @@ class QgsDelimitedTextFile
return mTrimFields;
}
/** Set the maximum number of fields that will be read from a record
* By default the maximum number is unlimited (0)
* @param maxFields The maximum number of fields that will be read
*/
void setMaxFields( int maxFields );
/** Return the maximum number of fields that will be read
* @return maxFields The maximum number of fields that will be read
*/
int maxFields() { return mMaxFields; }
/** Set the field names
* Field names are set from QStringList. Names may be modified
* to ensure that they are unique, not empty, and do not conflict
* with default field name (Field_##)
* with default field name (field_##)
* @param names A list of proposed field names
*/
void setFieldNames( const QStringList &names );
/** Return the field names read from the header, or default names
* Col## if none defined. Will open and read the head of the file
* if required, then reset..
* field_## if none defined. Will open and read the head of the file
* if required, then reset. Note that if header record record has
* not been read then the field names are empty until records have
* been read. The list may be expanded as the file is read and records
* with more fields are loaded.
* @return names A list of field names in the file
*/
QStringList &fieldNames();

32
src/providers/delimitedtext/qgsdelimitedtextprovider.cpp
View File

@ -129,6 +129,9 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( QString uri )
if ( ! mFile->isValid() )
{
// uri is invalid so the layer must be too...
QStringList messages;
messages.append( "File cannot be opened or delimiter parameters are not valid" );
reportErrors( messages );
QgsDebugMsg( "Delimited text source invalid - filename or delimiter parameters" );
return;
}
@ -539,7 +542,7 @@ void QgsDelimitedTextProvider::recordInvalidLine( QString message )
void QgsDelimitedTextProvider::reportErrors( QStringList messages )
{
if ( !mInvalidLines.isEmpty() )
if ( !mInvalidLines.isEmpty() || ! messages.isEmpty() )
{
QString tag( "DelimitedText" );
QgsMessageLog::logMessage( tr( "Errors in file %1" ).arg( mFile->fileName() ), tag );
@ -547,12 +550,14 @@ void QgsDelimitedTextProvider::reportErrors( QStringList messages )
{
QgsMessageLog::logMessage( message );
}
QgsMessageLog::logMessage( tr( "The following lines were not loaded from %1 into QGIS due to errors:\n" ).arg( mFile->fileName() ),
tag );
for ( int i = 0; i < mInvalidLines.size(); ++i )
QgsMessageLog::logMessage( mInvalidLines.at( i ), tag );
if ( mNExtraInvalidLines > 0 )
QgsMessageLog::logMessage( tr( "There are %1 additional errors in the file" ).arg( mNExtraInvalidLines ), tag );
if ( ! mInvalidLines.isEmpty() )
{
QgsMessageLog::logMessage( tr( "The following lines were not loaded into QGIS due to errors:" ), tag );
for ( int i = 0; i < mInvalidLines.size(); ++i )
QgsMessageLog::logMessage( mInvalidLines.at( i ), tag );
if ( mNExtraInvalidLines > 0 )
QgsMessageLog::logMessage( tr( "There are %1 additional errors in the file" ).arg( mNExtraInvalidLines ), tag );
}
// Display errors in a dialog...
@ -565,11 +570,14 @@ void QgsDelimitedTextProvider::reportErrors( QStringList messages )
{
output->appendMessage( message );
}
output->appendMessage( tr( "The following lines were not loaded from %1 into QGIS due to errors:\n" ).arg( mFile->fileName() ) );
for ( int i = 0; i < mInvalidLines.size(); ++i )
output->appendMessage( mInvalidLines.at( i ) );
if ( mNExtraInvalidLines > 0 )
output->appendMessage( tr( "There are %1 additional errors in the file" ).arg( mNExtraInvalidLines ) );
if ( ! mInvalidLines.isEmpty() )
{
output->appendMessage( tr( "The following lines were not loaded into QGIS due to errors:" ) );
for ( int i = 0; i < mInvalidLines.size(); ++i )
output->appendMessage( mInvalidLines.at( i ) );
if ( mNExtraInvalidLines > 0 )
output->appendMessage( tr( "There are %1 additional errors in the file" ).arg( mNExtraInvalidLines ) );
}
output->showMessage();
}

38
src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp
View File

@ -56,7 +56,7 @@ QgsDelimitedTextSourceSelect::QgsDelimitedTextSourceSelect( QWidget * parent, Qt
codecs.append( codec );
}
codecs.sort();
foreach( QString codec, codecs )
foreach ( QString codec, codecs )
{
cmbEncoding->addItem( codec );
}
@ -143,8 +143,6 @@ void QgsDelimitedTextSourceSelect::on_buttonBox_accepted()
QUrl url = mFile->url();
bool useHeader = mFile->useHeader();
if ( cbxPointIsComma->isChecked() )
{
url.addQueryItem( "decimalPoint", "," );
@ -637,19 +635,11 @@ void QgsDelimitedTextSourceSelect::updateFileName()
void QgsDelimitedTextSourceSelect::updateFieldsAndEnable()
{
// Check the regular expression is valid
lblRegexpError->setText( "" );
if ( delimiterRegexp->isChecked() )
{
QRegExp re( txtDelimiterRegexp->text() );
if ( ! re.isValid() ) lblRegexpError->setText( tr( "Expression is not valid" ) );
}
updateFieldLists();
enableAccept();
}
void QgsDelimitedTextSourceSelect::enableAccept()
bool QgsDelimitedTextSourceSelect::validate()
{
// Check that input data is valid - provide a status message if not..
@ -672,9 +662,23 @@ void QgsDelimitedTextSourceSelect::enableAccept()
{
message = tr( "At least one delimiter character must be specified" );
}
else if ( delimiterRegexp->isChecked() && ! QRegExp( txtDelimiterRegexp->text() ).isValid() )
if ( message.isEmpty() && delimiterRegexp->isChecked() )
{
message = tr( "Regular expression is not valid" );
QRegExp re( txtDelimiterRegexp->text() );
if ( ! re.isValid() )
{
message = tr( "Regular expression is not valid" );
}
else if ( re.pattern().startsWith( "^" ) && re.captureCount() == 0 )
{
message = tr( "^.. expression needs capture groups" );
}
lblRegexpError->setText( message );
}
if ( ! message.isEmpty() )
{
// continue...
}
// Hopefully won't hit this none-specific message, but just in case ...
else if ( ! mFile->isValid() )
@ -702,8 +706,12 @@ void QgsDelimitedTextSourceSelect::enableAccept()
{
enabled = true;
}
lblStatus->setText( message );
return enabled;
}
void QgsDelimitedTextSourceSelect::enableAccept()
{
bool enabled = validate();
buttonBox->button( QDialogButtonBox::Ok )->setEnabled( enabled );
}

1
src/providers/delimitedtext/qgsdelimitedtextsourceselect.h
View File

@ -66,6 +66,7 @@ class QgsDelimitedTextSourceSelect : public QDialog, private Ui::QgsDelimitedTex
void updateFileName();
void updateFieldsAndEnable();
void enableAccept();
bool validate();
signals:
void addVectorLayer( QString, QString, QString );

95
tests/src/python/test_qgsdelimitedtextprovider.py
View File

@ -362,6 +362,31 @@ class TestQgsDelimitedTextProvider(TestCase):
runTest(description,wanted,log_wanted,filename,**params)
def test_002b_max_fields(self):
description='Limiting maximum number of fields'
filename='testfields.csv'
params={'geomType': 'none', 'maxFields': '7', 'type': 'csv'}
if printTests:
createTest(description,filename,**params)
assert False,"Set printTests to False to run delimited text tests"
wanted={
u'1': {
'id': u'1',
'description': u'Generation of field names',
'data': u'Some data',
'field_4': u'Some info',
'data_1': u'',
'field_6': u'',
'field_7': u'',
'#fid': 2L,
'#geometry': 'None',
},
}
log_wanted=[
]
runTest(description,wanted,log_wanted,filename,**params)
def test_003_load_whitespace(self):
description='Whitespace file parsing'
filename='test.space'
@ -589,7 +614,7 @@ class TestQgsDelimitedTextProvider(TestCase):
}
log_wanted=[
u'Errors in file file',
u'The following lines were not loaded from file into QGIS due to errors:\n',
u'The following lines were not loaded into QGIS due to errors:',
u'Invalid record format at line 7',
u'Invalid record format at line 8',
u'Invalid record format at line 9',
@ -616,7 +641,7 @@ class TestQgsDelimitedTextProvider(TestCase):
}
log_wanted=[
u'Errors in file file',
u'The following lines were not loaded from file into QGIS due to errors:\n',
u'The following lines were not loaded into QGIS due to errors:',
u'Invalid record format at line 2',
u'Invalid record format at line 5',
]
@ -681,7 +706,7 @@ class TestQgsDelimitedTextProvider(TestCase):
}
log_wanted=[
u'Errors in file file',
u'The following lines were not loaded from file into QGIS due to errors:\n',
u'The following lines were not loaded into QGIS due to errors:',
u'Invalid X or Y fields at line 4',
]
runTest(description,wanted,log_wanted,filename,**params)
@ -728,7 +753,7 @@ class TestQgsDelimitedTextProvider(TestCase):
}
log_wanted=[
u'Errors in file file',
u'The following lines were not loaded from file into QGIS due to errors:\n',
u'The following lines were not loaded into QGIS due to errors:',
u'Invalid WKT at line 8',
]
runTest(description,wanted,log_wanted,filename,**params)
@ -775,7 +800,7 @@ class TestQgsDelimitedTextProvider(TestCase):
}
log_wanted=[
u'Errors in file file',
u'The following lines were not loaded from file into QGIS due to errors:\n',
u'The following lines were not loaded into QGIS due to errors:',
u'Invalid WKT at line 8',
]
runTest(description,wanted,log_wanted,filename,**params)
@ -822,7 +847,7 @@ class TestQgsDelimitedTextProvider(TestCase):
}
log_wanted=[
u'Errors in file file',
u'The following lines were not loaded from file into QGIS due to errors:\n',
u'The following lines were not loaded into QGIS due to errors:',
u'Invalid WKT at line 8',
]
runTest(description,wanted,log_wanted,filename,**params)
@ -851,7 +876,7 @@ class TestQgsDelimitedTextProvider(TestCase):
}
log_wanted=[
u'Errors in file file',
u'The following lines were not loaded from file into QGIS due to errors:\n',
u'The following lines were not loaded into QGIS due to errors:',
u'Invalid WKT at line 8',
]
runTest(description,wanted,log_wanted,filename,**params)
@ -1020,7 +1045,7 @@ class TestQgsDelimitedTextProvider(TestCase):
}
log_wanted=[
u'Errors in file file',
u'The following lines were not loaded from file into QGIS due to errors:\n',
u'The following lines were not loaded into QGIS due to errors:',
u'Invalid X or Y fields at line 27',
u'Invalid X or Y fields at line 28',
u'Invalid X or Y fields at line 29',
@ -1161,12 +1186,62 @@ class TestQgsDelimitedTextProvider(TestCase):
}
log_wanted=[
u'Errors in file file',
u'The following lines were not loaded from file into QGIS due to errors:\n',
u'The following lines were not loaded into QGIS due to errors:',
u'Invalid record format at line 3',
]
runTest(description,wanted,log_wanted,filename,**params)
def test_017a_regular_expression_4(self):
description='Parsing zero length re'
filename='testre3.txt'
params={'geomType': 'none', 'delimiter': 'x?', 'type': 'regexp'}
if printTests:
createTest(description,filename,**params)
assert False,"Set printTests to False to run delimited text tests"
wanted={
u'f': {
'id': u'f',
'description': u'i',
's': u'f',
'm': u'i',
'a': u'.',
'l': u'.',
'l_1': u'i',
'field_6': u'l',
'field_7': u'e',
'#fid': 2L,
'#geometry': 'None',
},
}
log_wanted=[
]
runTest(description,wanted,log_wanted,filename,**params)
def test_017a_regular_expression_5(self):
description='Parsing zero length re 2'
filename='testre3.txt'
params={'geomType': 'none', 'delimiter': '\\b', 'type': 'regexp'}
if printTests:
createTest(description,filename,**params)
assert False,"Set printTests to False to run delimited text tests"
wanted={
u'fi': {
'id': u'fi',
'description': u'..',
'small': u'fi',
'field_2': u'..',
'field_3': u'ile',
'#fid': 2L,
'#geometry': 'None',
},
}
log_wanted=[
]
runTest(description,wanted,log_wanted,filename,**params)
def test_018_utf8_encoded_file(self):
description='UTF8 encoded file test'
filename='testutf8.csv'
@ -1208,8 +1283,6 @@ class TestQgsDelimitedTextProvider(TestCase):
]
runTest(description,wanted,log_wanted,filename,**params)
#END
if __name__ == '__main__':

2
tests/testdata/delimitedtext/testre3.txt vendored Normal file
View File

@ -0,0 +1,2 @@
small
fi..ile