From 5df309447ebac64a8f58640472ffabf7d3b57179 Mon Sep 17 00:00:00 2001 From: Mathieu Pellerin Date: Fri, 6 Sep 2019 19:35:27 +0700 Subject: [PATCH] [delimiter text] Add Point{Z,M,ZM} geometry support (fixes #25645) (#31595) [FEATURE][delimiter text] Add Point{Z,M,ZM} geometry support (fixes #25645) --- .../qgsdelimitedtextfeatureiterator.cpp | 24 +++- .../qgsdelimitedtextfeatureiterator.h | 2 + .../qgsdelimitedtextprovider.cpp | 66 ++++++++++- .../delimitedtext/qgsdelimitedtextprovider.h | 7 +- .../qgsdelimitedtextsourceselect.cpp | 21 +++- src/ui/qgsdelimitedtextsourceselectbase.ui | 106 +++++++++++++++++- .../python/test_qgsdelimitedtextprovider.py | 64 ++++++++++- tests/testdata/provider/delimited_xyzm.csv | 6 + 8 files changed, 281 insertions(+), 15 deletions(-) create mode 100644 tests/testdata/provider/delimited_xyzm.csv diff --git a/src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.cpp b/src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.cpp index 8eba407c445..1d493365421 100644 --- a/src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.cpp +++ b/src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.cpp @@ -431,13 +431,25 @@ QgsGeometry QgsDelimitedTextFeatureIterator::loadGeometryXY( const QStringList & isNull = true; return QgsGeometry(); } - isNull = false; - QgsPointXY pt; - bool ok = QgsDelimitedTextProvider::pointFromXY( sX, sY, pt, mSource->mDecimalPoint, mSource->mXyDms ); - if ( ok && wantGeometry( pt ) ) + isNull = false; + QgsPoint *pt = new QgsPoint(); + bool ok = QgsDelimitedTextProvider::pointFromXY( sX, sY, *pt, mSource->mDecimalPoint, mSource->mXyDms ); + + QString sZ, sM; + if ( mSource->mZFieldIndex > -1 ) + sZ = tokens[mSource->mZFieldIndex]; + if ( mSource->mMFieldIndex > -1 ) + sM = tokens[mSource->mMFieldIndex]; + + if ( !sZ.isEmpty() || !sM.isEmpty() ) { - return QgsGeometry::fromPointXY( pt ); + QgsDelimitedTextProvider::appendZM( sZ, sM, *pt, mSource->mDecimalPoint ); + } + + if ( ok && wantGeometry( *pt ) ) + { + return QgsGeometry( pt ); } return QgsGeometry(); } @@ -511,6 +523,8 @@ QgsDelimitedTextFeatureSource::QgsDelimitedTextFeatureSource( const QgsDelimited , mFieldCount( p->mFieldCount ) , mXFieldIndex( p->mXFieldIndex ) , mYFieldIndex( p->mYFieldIndex ) + , mZFieldIndex( p->mZFieldIndex ) + , mMFieldIndex( p->mMFieldIndex ) , mWktFieldIndex( p->mWktFieldIndex ) , mWktHasPrefix( p->mWktHasPrefix ) , mGeometryType( p->mGeometryType ) diff --git a/src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.h b/src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.h index 687a50b7b49..1fabd763fab 100644 --- a/src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.h +++ b/src/providers/delimitedtext/qgsdelimitedtextfeatureiterator.h @@ -43,6 +43,8 @@ class QgsDelimitedTextFeatureSource : public QgsAbstractFeatureSource int mFieldCount; // Note: this includes field count for wkt field int mXFieldIndex; int mYFieldIndex; + int mZFieldIndex; + int mMFieldIndex; int mWktFieldIndex; bool mWktHasPrefix; QgsWkbTypes::GeometryType mGeometryType; diff --git a/src/providers/delimitedtext/qgsdelimitedtextprovider.cpp b/src/providers/delimitedtext/qgsdelimitedtextprovider.cpp index b842e173100..abb8f942107 100644 --- a/src/providers/delimitedtext/qgsdelimitedtextprovider.cpp +++ b/src/providers/delimitedtext/qgsdelimitedtextprovider.cpp @@ -101,8 +101,14 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString &uri, const Pr mGeometryType = QgsWkbTypes::PointGeometry; mXFieldName = url.queryItemValue( QStringLiteral( "xField" ) ); mYFieldName = url.queryItemValue( QStringLiteral( "yField" ) ); + if ( url.hasQueryItem( QStringLiteral( "zField" ) ) ) + mZFieldName = url.queryItemValue( QStringLiteral( "zField" ) ); + if ( url.hasQueryItem( QStringLiteral( "mField" ) ) ) + mMFieldName = url.queryItemValue( QStringLiteral( "mField" ) ); QgsDebugMsg( "xField is: " + mXFieldName ); QgsDebugMsg( "yField is: " + mYFieldName ); + QgsDebugMsg( "zField is: " + mZFieldName ); + QgsDebugMsg( "mField is: " + mMFieldName ); if ( url.hasQueryItem( QStringLiteral( "xyDms" ) ) ) { @@ -336,11 +342,27 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes ) mYFieldIndex = mFile->fieldIndex( mYFieldName ); if ( mXFieldIndex < 0 ) { - messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( QStringLiteral( "X" ), mWktFieldName ) ); + messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( QStringLiteral( "X" ), mXFieldName ) ); } if ( mYFieldIndex < 0 ) { - messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( QStringLiteral( "Y" ), mWktFieldName ) ); + messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( QStringLiteral( "Y" ), mYFieldName ) ); + } + if ( !mZFieldName.isEmpty() ) + { + mZFieldIndex = mFile->fieldIndex( mZFieldName ); + if ( mZFieldIndex < 0 ) + { + messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( QStringLiteral( "Z" ), mZFieldName ) ); + } + } + if ( !mMFieldName.isEmpty() ) + { + mMFieldIndex = mFile->fieldIndex( mMFieldName ); + if ( mMFieldIndex < 0 ) + { + messages.append( tr( "%0 field %1 is not defined in delimited text file" ).arg( QStringLiteral( "M" ), mMFieldName ) ); + } } } if ( !messages.isEmpty() ) @@ -466,6 +488,11 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes ) QString sX = mXFieldIndex < parts.size() ? parts[mXFieldIndex] : QString(); QString sY = mYFieldIndex < parts.size() ? parts[mYFieldIndex] : QString(); + QString sZ, sM; + if ( mZFieldIndex > -1 ) + sZ = mZFieldIndex < parts.size() ? parts[mZFieldIndex] : QString(); + if ( mMFieldIndex > -1 ) + sM = mMFieldIndex < parts.size() ? parts[mMFieldIndex] : QString(); if ( sX.isEmpty() && sY.isEmpty() ) { nEmptyGeometry++; @@ -473,11 +500,14 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes ) } else { - QgsPointXY pt; + QgsPoint pt; bool ok = pointFromXY( sX, sY, pt, mDecimalPoint, mXyDms ); if ( ok ) { + if ( !sZ.isEmpty() || sM.isEmpty() ) + appendZM( sZ, sM, pt, mDecimalPoint ); + if ( foundFirstGeometry ) { mExtent.combineExtentWith( pt.x(), pt.y() ); @@ -487,6 +517,10 @@ void QgsDelimitedTextProvider::scanFile( bool buildIndexes ) // Extent for the first point is just the first point mExtent.set( pt.x(), pt.y(), pt.x(), pt.y() ); mWkbType = QgsWkbTypes::Point; + if ( mZFieldIndex > -1 ) + mWkbType = QgsWkbTypes::addZ( mWkbType ); + if ( mMFieldIndex > -1 ) + mWkbType = QgsWkbTypes::addM( mWkbType ); mGeometryType = QgsWkbTypes::PointGeometry; foundFirstGeometry = true; } @@ -841,7 +875,31 @@ double QgsDelimitedTextProvider::dmsStringToDouble( const QString &sX, bool *xOk return x; } -bool QgsDelimitedTextProvider::pointFromXY( QString &sX, QString &sY, QgsPointXY &pt, const QString &decimalPoint, bool xyDms ) +void QgsDelimitedTextProvider::appendZM( QString &sZ, QString &sM, QgsPoint &point, const QString &decimalPoint ) +{ + if ( ! decimalPoint.isEmpty() ) + { + sZ.replace( decimalPoint, QLatin1String( "." ) ); + sM.replace( decimalPoint, QLatin1String( "." ) ); + } + + bool zOk, mOk; + double z, m; + if ( !sZ.isEmpty() ) + { + z = sZ.toDouble( &zOk ); + if ( zOk ) + point.addZValue( z ); + } + if ( !sM.isEmpty() ) + { + m = sM.toDouble( &mOk ); + if ( mOk ) + point.addMValue( m ); + } +} + +bool QgsDelimitedTextProvider::pointFromXY( QString &sX, QString &sY, QgsPoint &pt, const QString &decimalPoint, bool xyDms ) { if ( ! decimalPoint.isEmpty() ) { diff --git a/src/providers/delimitedtext/qgsdelimitedtextprovider.h b/src/providers/delimitedtext/qgsdelimitedtextprovider.h index 26068d4326d..7fd1f1f2ad5 100644 --- a/src/providers/delimitedtext/qgsdelimitedtextprovider.h +++ b/src/providers/delimitedtext/qgsdelimitedtextprovider.h @@ -211,7 +211,8 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider static QgsGeometry geomFromWkt( QString &sWkt, bool wktHasPrefixRegexp ); - static bool pointFromXY( QString &sX, QString &sY, QgsPointXY &point, const QString &decimalPoint, bool xyDms ); + static bool pointFromXY( QString &sX, QString &sY, QgsPoint &point, const QString &decimalPoint, bool xyDms ); + static void appendZM( QString &sZ, QString &sM, QgsPoint &point, const QString &decimalPoint ); static double dmsStringToDouble( const QString &sX, bool *xOk ); // mLayerValid defines whether the layer has been loaded as a valid layer @@ -232,10 +233,14 @@ class QgsDelimitedTextProvider : public QgsVectorDataProvider QString mWktFieldName; QString mXFieldName; QString mYFieldName; + QString mZFieldName; + QString mMFieldName; bool mDetectTypes = true; mutable int mXFieldIndex = -1; mutable int mYFieldIndex = -1; + mutable int mZFieldIndex = -1; + mutable int mMFieldIndex = -1; mutable int mWktFieldIndex = -1; // mWktPrefix regexp is used to clean up diff --git a/src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp b/src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp index 37a2db7e6e5..dff1e709106 100644 --- a/src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp +++ b/src/providers/delimitedtext/qgsdelimitedtextsourceselect.cpp @@ -152,13 +152,24 @@ void QgsDelimitedTextSourceSelect::addButtonClicked() bool haveGeom = true; if ( geomTypeXY->isChecked() ) { + QString field; if ( !cmbXField->currentText().isEmpty() && !cmbYField->currentText().isEmpty() ) { - QString field = cmbXField->currentText(); + field = cmbXField->currentText(); url.addQueryItem( QStringLiteral( "xField" ), field ); field = cmbYField->currentText(); url.addQueryItem( QStringLiteral( "yField" ), field ); } + if ( !cmbZField->currentText().isEmpty() ) + { + field = cmbZField->currentText(); + url.addQueryItem( QStringLiteral( "zField" ), field ); + } + if ( !cmbMField->currentText().isEmpty() ) + { + field = cmbMField->currentText(); + url.addQueryItem( QStringLiteral( "mField" ), field ); + } } else if ( geomTypeWKT->isChecked() ) { @@ -407,11 +418,15 @@ void QgsDelimitedTextSourceSelect::updateFieldLists() QString columnX = cmbXField->currentText(); QString columnY = cmbYField->currentText(); + QString columnZ = cmbZField->currentText(); + QString columnM = cmbMField->currentText(); QString columnWkt = cmbWktField->currentText(); // clear the field lists cmbXField->clear(); cmbYField->clear(); + cmbZField->clear(); + cmbMField->clear(); cmbWktField->clear(); // clear the sample text box @@ -532,6 +547,8 @@ void QgsDelimitedTextSourceSelect::updateFieldLists() if ( field.isEmpty() ) continue; cmbXField->addItem( field ); cmbYField->addItem( field ); + cmbZField->addItem( field ); + cmbMField->addItem( field ); cmbWktField->addItem( field ); fieldNo++; } @@ -541,6 +558,8 @@ void QgsDelimitedTextSourceSelect::updateFieldLists() cmbWktField->setCurrentIndex( cmbWktField->findText( columnWkt ) ); cmbXField->setCurrentIndex( cmbXField->findText( columnX ) ); cmbYField->setCurrentIndex( cmbYField->findText( columnY ) ); + cmbZField->setCurrentIndex( cmbYField->findText( columnZ ) ); + cmbMField->setCurrentIndex( cmbYField->findText( columnM ) ); // Now try setting optional X,Y fields - will only reset the fields if // not already set. diff --git a/src/ui/qgsdelimitedtextsourceselectbase.ui b/src/ui/qgsdelimitedtextsourceselectbase.ui index cc59ef7ddda..039ce1d4e60 100644 --- a/src/ui/qgsdelimitedtextsourceselectbase.ui +++ b/src/ui/qgsdelimitedtextsourceselectbase.ui @@ -879,7 +879,7 @@ - + X and Y coordinates are expressed in degrees/minutes/seconds @@ -957,8 +957,70 @@ + + + + true + + + + 0 + 0 + + + + + 120 + 0 + + + + Name of the field containing z values + + + Name of the field containing z values + + + Name of the field containing z values + + + false + + + + + + + true + + + + 0 + 0 + + + + + 120 + 0 + + + + Name of the field containing m values + + + Name of the field containing m values + + + Name of the field containing m values + + + false + + + - + true @@ -974,7 +1036,7 @@ - + true @@ -992,6 +1054,44 @@ + + + + true + + + + 0 + 0 + + + + <p align="left">Z field</p> + + + cmbZField + + + + + + + true + + + + 0 + 0 + + + + <p align="left">M field</p> + + + cmbMField + + + diff --git a/tests/src/python/test_qgsdelimitedtextprovider.py b/tests/src/python/test_qgsdelimitedtextprovider.py index 4658be37317..e5e47df4895 100644 --- a/tests/src/python/test_qgsdelimitedtextprovider.py +++ b/tests/src/python/test_qgsdelimitedtextprovider.py @@ -41,7 +41,8 @@ from qgis.core import ( QgsFeatureRequest, QgsRectangle, QgsApplication, - QgsFeature) + QgsFeature, + QgsWkbTypes) from qgis.testing import start_app, unittest from utilities import unitTestDataPath, compareWkt @@ -826,6 +827,67 @@ class TestQgsDelimitedTextProviderOther(unittest.TestCase): components = registry.decodeUri('delimitedtext', uri) self.assertEqual(components['path'], filename) + def test_044_ZM(self): + # Create test layer + srcpath = os.path.join(TEST_DATA_DIR, 'provider') + basetestfile = os.path.join(srcpath, 'delimited_xyzm.csv') + + url = MyUrl.fromLocalFile(basetestfile) + url.addQueryItem("crs", "epsg:4326") + url.addQueryItem("type", "csv") + url.addQueryItem("xField", "X") + url.addQueryItem("yField", "Y") + url.addQueryItem("zField", "Z") + url.addQueryItem("mField", "M") + url.addQueryItem("spatialIndex", "no") + url.addQueryItem("subsetIndex", "no") + url.addQueryItem("watchFile", "no") + + vl = QgsVectorLayer(url.toString(), 'test', 'delimitedtext') + assert vl.isValid(), "{} is invalid".format(basetestfile) + assert vl.wkbType() == QgsWkbTypes.PointZM, "wrong wkb type, should be PointZM" + assert vl.getFeature(2).geometry().asWkt() == "PointZM (-71.12300000000000466 78.23000000000000398 1 2)", "wrong PointZM geometry" + + def test_045_Z(self): + # Create test layer + srcpath = os.path.join(TEST_DATA_DIR, 'provider') + basetestfile = os.path.join(srcpath, 'delimited_xyzm.csv') + + url = MyUrl.fromLocalFile(basetestfile) + url.addQueryItem("crs", "epsg:4326") + url.addQueryItem("type", "csv") + url.addQueryItem("xField", "X") + url.addQueryItem("yField", "Y") + url.addQueryItem("zField", "Z") + url.addQueryItem("spatialIndex", "no") + url.addQueryItem("subsetIndex", "no") + url.addQueryItem("watchFile", "no") + + vl = QgsVectorLayer(url.toString(), 'test', 'delimitedtext') + assert vl.isValid(), "{} is invalid".format(basetestfile) + assert vl.wkbType() == QgsWkbTypes.PointZ, "wrong wkb type, should be PointZ" + assert vl.getFeature(2).geometry().asWkt() == "PointZ (-71.12300000000000466 78.23000000000000398 1)", "wrong PointZ geometry" + + def test_046_M(self): + # Create test layer + srcpath = os.path.join(TEST_DATA_DIR, 'provider') + basetestfile = os.path.join(srcpath, 'delimited_xyzm.csv') + + url = MyUrl.fromLocalFile(basetestfile) + url.addQueryItem("crs", "epsg:4326") + url.addQueryItem("type", "csv") + url.addQueryItem("xField", "X") + url.addQueryItem("yField", "Y") + url.addQueryItem("mField", "M") + url.addQueryItem("spatialIndex", "no") + url.addQueryItem("subsetIndex", "no") + url.addQueryItem("watchFile", "no") + + vl = QgsVectorLayer(url.toString(), 'test', 'delimitedtext') + assert vl.isValid(), "{} is invalid".format(basetestfile) + assert vl.wkbType() == QgsWkbTypes.PointM, "wrong wkb type, should be PointM" + assert vl.getFeature(2).geometry().asWkt() == "PointM (-71.12300000000000466 78.23000000000000398 2)", "wrong PointM geometry" + if __name__ == '__main__': unittest.main() diff --git a/tests/testdata/provider/delimited_xyzm.csv b/tests/testdata/provider/delimited_xyzm.csv new file mode 100644 index 00000000000..b091e4780a5 --- /dev/null +++ b/tests/testdata/provider/delimited_xyzm.csv @@ -0,0 +1,6 @@ +pk,cnt,name,name2,num_char,X,Y,Z,M +5,-200,,NuLl,5,-71.123,78.23,1,2 +3,300,Pear,PEaR,3,,,3,4 +1,100,Orange,oranGe,1,-70.332,66.33,3,4 +2,200,Apple,Apple,2,-68.2,70.8,3,4 +4,400,Honey,Honey,4,-65.32,78.3,3,4