From 16629b406b63607abf38f51ff38c49924081a90e Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 2 Aug 2017 23:05:59 +1000 Subject: [PATCH 01/23] Register QgsFeatureIds metatype Fixes warnings when using signals which use this type, like QgsVectorLayer::selectionChanged --- src/core/qgsapplication.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/qgsapplication.cpp b/src/core/qgsapplication.cpp index 49dda787965..2e5f0610ffe 100644 --- a/src/core/qgsapplication.cpp +++ b/src/core/qgsapplication.cpp @@ -147,6 +147,7 @@ void QgsApplication::init( QString profileFolder ) qRegisterMetaType( "QgsProcessingFeatureSourceDefinition" ); qRegisterMetaType( "QgsProcessingOutputLayerDefinition" ); qRegisterMetaType( "QgsUnitTypes::LayoutUnit" ); + qRegisterMetaType( "QgsFeatureIds" ); QString prefixPath( getenv( "QGIS_PREFIX_PATH" ) ? getenv( "QGIS_PREFIX_PATH" ) : applicationDirPath() ); // QgsDebugMsg( QString( "prefixPath(): %1" ).arg( prefixPath ) ); From a64d199e6f216cdaeb99d2e5cc6b56af93371452 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 2 Aug 2017 23:07:22 +1000 Subject: [PATCH 02/23] [processing] If an error occurs while running an algorith, always keep the algorithm dialog open after execution Otherwise it's hard to see the error - you have to know to check the python log. Keeping the dialog open at the log makes the error immediately visible to the user --- python/plugins/processing/gui/AlgorithmDialog.py | 8 ++++---- python/plugins/processing/gui/AlgorithmDialogBase.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/plugins/processing/gui/AlgorithmDialog.py b/python/plugins/processing/gui/AlgorithmDialog.py index 4c7d96c14ba..fad8255f9c0 100644 --- a/python/plugins/processing/gui/AlgorithmDialog.py +++ b/python/plugins/processing/gui/AlgorithmDialog.py @@ -229,7 +229,7 @@ class AlgorithmDialog(AlgorithmDialogBase): feedback.pushInfo( self.tr('Execution completed in {0:0.2f} seconds'.format(time.time() - start_time))) self.buttonCancel.setEnabled(False) - self.finish(parameters, context, feedback) + self.finish(True, parameters, context, feedback) else: self.buttonCancel.setEnabled(False) self.resetGUI() @@ -250,7 +250,7 @@ class AlgorithmDialog(AlgorithmDialogBase): feedback.pushInfo('') self.buttonCancel.setEnabled(False) - self.finish(results, context, feedback) + self.finish(ok, results, context, feedback) task = QgsProcessingAlgRunnerTask(self.alg, parameters, context, feedback) task.executed.connect(on_complete) @@ -269,8 +269,8 @@ class AlgorithmDialog(AlgorithmDialogBase): self.bar.pushMessage("", self.tr("Wrong or missing parameter value: {0}").format(e.parameter.description()), level=QgsMessageBar.WARNING, duration=5) - def finish(self, result, context, feedback): - keepOpen = ProcessingConfig.getSetting(ProcessingConfig.KEEP_DIALOG_OPEN) + def finish(self, successful, result, context, feedback): + keepOpen = not successful or ProcessingConfig.getSetting(ProcessingConfig.KEEP_DIALOG_OPEN) if self.iterateParam is None: diff --git a/python/plugins/processing/gui/AlgorithmDialogBase.py b/python/plugins/processing/gui/AlgorithmDialogBase.py index a73ba82daa1..f34f51d9502 100644 --- a/python/plugins/processing/gui/AlgorithmDialogBase.py +++ b/python/plugins/processing/gui/AlgorithmDialogBase.py @@ -238,7 +238,7 @@ class AlgorithmDialogBase(BASE, WIDGET): self._saveGeometry() super(AlgorithmDialogBase, self).reject() - def finish(self, context, feedback): + def finish(self, successful, result, context, feedback): pass def toggleCollapsed(self): From 7ab82444f1108fba8a26629821a13138a8b50a72 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 2 Aug 2017 23:09:47 +1000 Subject: [PATCH 03/23] Port random selection algorithms to new API And heavily optimise random selection within subsets alg --- .../algs/qgis/QGISAlgorithmProvider.py | 7 +- .../processing/algs/qgis/RandomSelection.py | 45 ++++----- .../algs/qgis/RandomSelectionWithinSubsets.py | 96 +++++++++---------- 3 files changed, 72 insertions(+), 76 deletions(-) diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 13c8704e439..6615f66f4c7 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -99,6 +99,8 @@ from .RandomPointsAlongLines import RandomPointsAlongLines from .RandomPointsExtent import RandomPointsExtent from .RandomPointsLayer import RandomPointsLayer from .RandomPointsPolygons import RandomPointsPolygons +from .RandomSelection import RandomSelection +from .RandomSelectionWithinSubsets import RandomSelectionWithinSubsets from .RasterLayerStatistics import RasterLayerStatistics from .RegularPoints import RegularPoints from .ReverseLineDirection import ReverseLineDirection @@ -135,8 +137,6 @@ from .VoronoiPolygons import VoronoiPolygons from .ZonalStatistics import ZonalStatistics # from .ExtractByLocation import ExtractByLocation -# from .RandomSelection import RandomSelection -# from .RandomSelectionWithinSubsets import RandomSelectionWithinSubsets # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin # from .GridLine import GridLine @@ -185,7 +185,6 @@ class QGISAlgorithmProvider(QgsProcessingProvider): def getAlgs(self): # algs = [ - # RandomSelection(), RandomSelectionWithinSubsets(), # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), @@ -270,6 +269,8 @@ class QGISAlgorithmProvider(QgsProcessingProvider): RandomPointsExtent(), RandomPointsLayer(), RandomPointsPolygons(), + RandomSelection(), + RandomSelectionWithinSubsets(), RasterLayerStatistics(), RegularPoints(), ReverseLineDirection(), diff --git a/python/plugins/processing/algs/qgis/RandomSelection.py b/python/plugins/processing/algs/qgis/RandomSelection.py index 167c3c6729f..31af20a29f8 100644 --- a/python/plugins/processing/algs/qgis/RandomSelection.py +++ b/python/plugins/processing/algs/qgis/RandomSelection.py @@ -30,13 +30,16 @@ import os import random from qgis.PyQt.QtGui import QIcon -from qgis.core import QgsFeatureSink, QgsProcessingUtils +from qgis.core import (QgsFeatureSink, + QgsProcessingException, + QgsProcessingUtils, + QgsProcessingParameterVectorLayer, + QgsProcessingParameterEnum, + QgsProcessingParameterNumber, + QgsProcessingParameterFeatureSink, + QgsProcessingOutputVectorLayer) + from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterSelection -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterNumber -from processing.core.outputs import OutputVector pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] @@ -61,13 +64,14 @@ class RandomSelection(QgisAlgorithm): self.methods = [self.tr('Number of selected features'), self.tr('Percentage of selected features')] - self.addParameter(ParameterVector(self.INPUT, - self.tr('Input layer'))) - self.addParameter(ParameterSelection(self.METHOD, - self.tr('Method'), self.methods, 0)) - self.addParameter(ParameterNumber(self.NUMBER, - self.tr('Number/percentage of selected features'), 0, None, 10)) - self.addOutput(OutputVector(self.OUTPUT, self.tr('Selection'), True)) + self.addParameter(QgsProcessingParameterVectorLayer(self.INPUT, + self.tr('Input layer'))) + self.addParameter(QgsProcessingParameterEnum(self.METHOD, + self.tr('Method'), self.methods, False, 0)) + self.addParameter(QgsProcessingParameterNumber(self.NUMBER, + self.tr('Number/percentage of selected features'), QgsProcessingParameterNumber.Integer, + 10, False, 0.0, 999999999999.0)) + self.addOutput(QgsProcessingOutputVectorLayer(self.OUTPUT, self.tr('Selected (random)'))) def name(self): return 'randomselection' @@ -76,23 +80,20 @@ class RandomSelection(QgisAlgorithm): return self.tr('Random selection') def processAlgorithm(self, parameters, context, feedback): - filename = self.getParameterValue(self.INPUT) - layer = QgsProcessingUtils.mapLayerFromString(filename, context) - method = self.getParameterValue(self.METHOD) + layer = self.parameterAsVectorLayer(parameters, self.INPUT, context) + method = self.parameterAsEnum(parameters, self.METHOD, context) featureCount = layer.featureCount() - value = int(self.getParameterValue(self.NUMBER)) - - layer.removeSelection() + value = self.parameterAsInt(parameters, self.NUMBER, context) if method == 0: if value > featureCount: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Selected number is greater than feature count. ' 'Choose a lower value and try again.')) else: if value > 100: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr("Percentage can't be greater than 100. Set a " "different value and try again.")) value = int(round(value / 100.0, 4) * featureCount) @@ -100,4 +101,4 @@ class RandomSelection(QgisAlgorithm): selran = random.sample(list(range(featureCount)), value) layer.selectByIds(selran) - self.setOutputValue(self.OUTPUT, filename) + return {self.OUTPUT: parameters[self.INPUT]} diff --git a/python/plugins/processing/algs/qgis/RandomSelectionWithinSubsets.py b/python/plugins/processing/algs/qgis/RandomSelectionWithinSubsets.py index 6086c31efcf..c72390a4976 100644 --- a/python/plugins/processing/algs/qgis/RandomSelectionWithinSubsets.py +++ b/python/plugins/processing/algs/qgis/RandomSelectionWithinSubsets.py @@ -31,15 +31,17 @@ import random from qgis.PyQt.QtGui import QIcon -from qgis.core import QgsFeature, QgsFeatureSink, QgsProcessingUtils - +from qgis.core import (QgsFeatureRequest, + QgsProcessingException, + QgsProcessingUtils, + QgsProcessingParameterVectorLayer, + QgsProcessingParameterEnum, + QgsProcessingParameterField, + QgsProcessingParameterNumber, + QgsProcessingParameterFeatureSink, + QgsProcessingOutputVectorLayer) +from collections import defaultdict from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterSelection -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterNumber -from processing.core.parameters import ParameterTableField -from processing.core.outputs import OutputVector pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] @@ -65,16 +67,17 @@ class RandomSelectionWithinSubsets(QgisAlgorithm): self.methods = [self.tr('Number of selected features'), self.tr('Percentage of selected features')] - self.addParameter(ParameterVector(self.INPUT, - self.tr('Input layer'))) - self.addParameter(ParameterTableField(self.FIELD, - self.tr('ID Field'), self.INPUT)) - self.addParameter(ParameterSelection(self.METHOD, - self.tr('Method'), self.methods, 0)) - self.addParameter(ParameterNumber(self.NUMBER, - self.tr('Number/percentage of selected features'), 1, None, 10)) - - self.addOutput(OutputVector(self.OUTPUT, self.tr('Selection stratified'), True)) + self.addParameter(QgsProcessingParameterVectorLayer(self.INPUT, + self.tr('Input layer'))) + self.addParameter(QgsProcessingParameterField(self.FIELD, + self.tr('ID field'), None, self.INPUT)) + self.addParameter(QgsProcessingParameterEnum(self.METHOD, + self.tr('Method'), self.methods, False, 0)) + self.addParameter(QgsProcessingParameterNumber(self.NUMBER, + self.tr('Number/percentage of selected features'), + QgsProcessingParameterNumber.Integer, + 10, False, 0.0, 999999999999.0)) + self.addOutput(QgsProcessingOutputVectorLayer(self.OUTPUT, self.tr('Selected (stratified random)'))) def name(self): return 'randomselectionwithinsubsets' @@ -83,61 +86,52 @@ class RandomSelectionWithinSubsets(QgisAlgorithm): return self.tr('Random selection within subsets') def processAlgorithm(self, parameters, context, feedback): - filename = self.getParameterValue(self.INPUT) + layer = self.parameterAsVectorLayer(parameters, self.INPUT, context) + method = self.parameterAsEnum(parameters, self.METHOD, context) + field = self.parameterAsString(parameters, self.FIELD, context) - layer = QgsProcessingUtils.mapLayerFromString(filename, context) - field = self.getParameterValue(self.FIELD) - method = self.getParameterValue(self.METHOD) - - layer.removeSelection() index = layer.fields().lookupField(field) - unique = QgsProcessingUtils.uniqueValues(layer, index, context) + unique = layer.uniqueValues(index) featureCount = layer.featureCount() - value = int(self.getParameterValue(self.NUMBER)) + value = self.parameterAsInt(parameters, self.NUMBER, context) if method == 0: if value > featureCount: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Selected number is greater that feature count. ' 'Choose lesser value and try again.')) else: if value > 100: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr("Percentage can't be greater than 100. Set a " "different value and try again.")) value = value / 100.0 - selran = [] - inFeat = QgsFeature() - - current = 0 total = 100.0 / (featureCount * len(unique)) if featureCount else 1 if not len(unique) == featureCount: - for i in unique: - features = QgsProcessingUtils.getFeatures(layer, context) - FIDs = [] - for inFeat in features: - attrs = inFeat.attributes() - if attrs[index] == i: - FIDs.append(inFeat.id()) - current += 1 - feedback.setProgress(int(current * total)) + classes = defaultdict(list) - if method == 1: - selValue = int(round(value * len(FIDs), 0)) - else: - selValue = value + features = layer.getFeatures(QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry).setSubsetOfAttributes([index])) - if selValue >= len(FIDs): - selFeat = FIDs - else: - selFeat = random.sample(FIDs, selValue) + for i, feature in enumerate(features): + if feedback.isCanceled(): + break + + classes[feature.attributes()[index]].append(feature.id()) + feedback.setProgress(int(i * total)) + + selran = [] + for subset in classes.values(): + if feedback.isCanceled(): + break + + selValue = value if method != 1 else int(round(value * len(subset), 0)) + selran.extend(random.sample(subset, selValue)) - selran.extend(selFeat) layer.selectByIds(selran) else: layer.selectByIds(list(range(featureCount))) # FIXME: implies continuous feature ids - self.setOutputValue(self.OUTPUT, filename) + return {self.OUTPUT: parameters[self.INPUT]} From 54be72048516896d9022da4fa24369d5ac4aeb0c Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 2 Aug 2017 23:22:30 +1000 Subject: [PATCH 04/23] Port grid lines to new API --- .../plugins/processing/algs/qgis/GridLine.py | 79 ++++++++++------- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- .../tests/testdata/qgis_algorithm_tests.yaml | 88 +++++++++---------- 3 files changed, 92 insertions(+), 80 deletions(-) diff --git a/python/plugins/processing/algs/qgis/GridLine.py b/python/plugins/processing/algs/qgis/GridLine.py index bcf5cabae4d..3bb87b8d9f5 100644 --- a/python/plugins/processing/algs/qgis/GridLine.py +++ b/python/plugins/processing/algs/qgis/GridLine.py @@ -39,14 +39,15 @@ from qgis.core import (QgsRectangle, QgsPoint, QgsLineString, QgsWkbTypes, + QgsProcessing, + QgsProcessingException, + QgsProcessingParameterEnum, + QgsProcessingParameterExtent, + QgsProcessingParameterNumber, + QgsProcessingParameterCrs, + QgsProcessingParameterFeatureSink, QgsFields) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterExtent -from processing.core.parameters import ParameterNumber -from processing.core.parameters import ParameterCrs -from processing.core.outputs import OutputVector -from processing.tools import dataobjects pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] @@ -73,19 +74,24 @@ class GridLine(QgisAlgorithm): super().__init__() def initAlgorithm(self, config=None): - self.addParameter(ParameterExtent(self.EXTENT, - self.tr('Grid extent'), optional=False)) - self.addParameter(ParameterNumber(self.HSPACING, - self.tr('Horizontal spacing'), 0.0, 1000000000.0, default=0.0001)) - self.addParameter(ParameterNumber(self.VSPACING, - self.tr('Vertical spacing'), 0.0, 1000000000.0, default=0.0001)) - self.addParameter(ParameterNumber(self.HOVERLAY, - self.tr('Horizontal overlay'), 0.0, 1000000000.0, default=0.0)) - self.addParameter(ParameterNumber(self.VOVERLAY, - self.tr('Vertical overlay'), 0.0, 1000000000.0, default=0.0)) - self.addParameter(ParameterCrs(self.CRS, 'Grid CRS', 'EPSG:4326')) + self.addParameter(QgsProcessingParameterExtent(self.EXTENT, self.tr('Grid extent'))) - self.addOutput(OutputVector(self.OUTPUT, self.tr('Grid'), datatype=[dataobjects.TYPE_VECTOR_LINE])) + self.addParameter(QgsProcessingParameterNumber(self.HSPACING, + self.tr('Horizontal spacing'), QgsProcessingParameterNumber.Double, + 0.0001, False, 0, 1000000000.0)) + self.addParameter(QgsProcessingParameterNumber(self.VSPACING, + self.tr('Vertical spacing'), QgsProcessingParameterNumber.Double, + 0.0001, False, 0, 1000000000.0)) + self.addParameter(QgsProcessingParameterNumber(self.HOVERLAY, + self.tr('Horizontal overlay'), QgsProcessingParameterNumber.Double, + 0.0, False, 0, 1000000000.0)) + self.addParameter(QgsProcessingParameterNumber(self.VOVERLAY, + self.tr('Vertical overlay'), QgsProcessingParameterNumber.Double, + 0.0, False, 0, 1000000000.0)) + + self.addParameter(QgsProcessingParameterCrs(self.CRS, 'Grid CRS', 'ProjectCrs')) + + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Grid'), type=QgsProcessing.TypeVectorLine)) def name(self): return 'creategridlines' @@ -94,33 +100,31 @@ class GridLine(QgisAlgorithm): return self.tr('Create grid (lines)') def processAlgorithm(self, parameters, context, feedback): - extent = self.getParameterValue(self.EXTENT).split(',') - hSpacing = self.getParameterValue(self.HSPACING) - vSpacing = self.getParameterValue(self.VSPACING) - hOverlay = self.getParameterValue(self.HOVERLAY) - vOverlay = self.getParameterValue(self.VOVERLAY) - crs = QgsCoordinateReferenceSystem(self.getParameterValue(self.CRS)) + hSpacing = self.parameterAsDouble(parameters, self.HSPACING, context) + vSpacing = self.parameterAsDouble(parameters, self.VSPACING, context) + hOverlay = self.parameterAsDouble(parameters, self.HOVERLAY, context) + vOverlay = self.parameterAsDouble(parameters, self.VOVERLAY, context) - bbox = QgsRectangle(float(extent[0]), float(extent[2]), - float(extent[1]), float(extent[3])) + bbox = self.parameterAsExtent(parameters, self.EXTENT, context) + crs = self.parameterAsCrs(parameters, self.CRS, context) width = bbox.width() height = bbox.height() if hSpacing <= 0 or vSpacing <= 0: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Invalid grid spacing: {0}/{1}').format(hSpacing, vSpacing)) if hSpacing <= hOverlay or vSpacing <= vOverlay: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Invalid overlay: {0}/{1}').format(hOverlay, vOverlay)) if width < hSpacing: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Horizontal spacing is too small for the covered area')) if height < vSpacing: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Vertical spacing is too small for the covered area')) fields = QgsFields() @@ -131,7 +135,8 @@ class GridLine(QgisAlgorithm): fields.append(QgsField('id', QVariant.Int, '', 10, 0)) fields.append(QgsField('coord', QVariant.Double, '', 24, 15)) - writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.LineString, crs, context) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, QgsWkbTypes.LineString, crs) if hOverlay > 0: hSpace = [hSpacing - hOverlay, hOverlay] @@ -154,6 +159,9 @@ class GridLine(QgisAlgorithm): count_update = count_max * 0.10 y = bbox.yMaximum() while y >= bbox.yMinimum(): + if feedback.isCanceled(): + break + pt1 = QgsPoint(bbox.xMinimum(), y) pt2 = QgsPoint(bbox.xMaximum(), y) line = QgsLineString() @@ -165,7 +173,7 @@ class GridLine(QgisAlgorithm): y, id, y]) - writer.addFeature(feat, QgsFeatureSink.FastInsert) + sink.addFeature(feat, QgsFeatureSink.FastInsert) y = y - vSpace[count % 2] id += 1 count += 1 @@ -181,6 +189,9 @@ class GridLine(QgisAlgorithm): count_update = count_max * 0.10 x = bbox.xMinimum() while x <= bbox.xMaximum(): + if feedback.isCanceled(): + break + pt1 = QgsPoint(x, bbox.yMaximum()) pt2 = QgsPoint(x, bbox.yMinimum()) line = QgsLineString() @@ -192,11 +203,11 @@ class GridLine(QgisAlgorithm): bbox.yMinimum(), id, x]) - writer.addFeature(feat, QgsFeatureSink.FastInsert) + sink.addFeature(feat, QgsFeatureSink.FastInsert) x = x + hSpace[count % 2] id += 1 count += 1 if int(math.fmod(count, count_update)) == 0: feedback.setProgress(50 + int(count / count_max * 50)) - del writer + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 6615f66f4c7..53397648311 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -70,6 +70,7 @@ from .ExtractSpecificNodes import ExtractSpecificNodes from .FixedDistanceBuffer import FixedDistanceBuffer from .FixGeometry import FixGeometry from .GeometryByExpression import GeometryByExpression +from .GridLine import GridLine from .GridPolygon import GridPolygon from .Heatmap import Heatmap from .Hillshade import Hillshade @@ -139,7 +140,6 @@ from .ZonalStatistics import ZonalStatistics # from .ExtractByLocation import ExtractByLocation # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin -# from .GridLine import GridLine # from .Gridify import Gridify # from .HubDistancePoints import HubDistancePoints # from .HubDistanceLines import HubDistanceLines @@ -188,7 +188,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), - # GridLine(), Gridify(), HubDistancePoints(), + # Gridify(), HubDistancePoints(), # HubDistanceLines(), HubLines(), # GeometryConvert(), FieldsCalculator(), # JoinAttributes(), @@ -240,6 +240,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): FixedDistanceBuffer(), FixGeometry(), GeometryByExpression(), + GridLine(), GridPolygon(), Heatmap(), Hillshade(), diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 1c73df8b908..64b881dd059 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -1795,21 +1795,21 @@ tests: name: expected/dropped_geometry.csv type: vector -# - algorithm: qgis:creategridlines -# name: Create grid (lines) -# params: -# CRS: EPSG:4326 -# EXTENT: -1,11.2,-4,6.5 -# HSPACING: 5.0 -# VSPACING: 3.0 -# results: -# OUTPUT: -# name: expected/grid_lines.gml -# type: vector -# compare: -# geometry: -# precision: 7 -# + - algorithm: qgis:creategridlines + name: Create grid (lines) + params: + CRS: EPSG:4326 + EXTENT: -1,11.2,-4,6.5 + HSPACING: 5.0 + VSPACING: 3.0 + results: + OUTPUT: + name: expected/grid_lines.gml + type: vector + compare: + geometry: + precision: 7 + - algorithm: qgis:creategridpolygon name: Create grid (rectangles) params: @@ -1858,20 +1858,20 @@ tests: geometry: precision: 7 -# - algorithm: qgis:creategridlines -# name: Create grid (lines with overlay) -# params: -# CRS: EPSG:4326 -# EXTENT: -1,11.2,-4,6.5 -# HOVERLAY: 2.0 -# HSPACING: 5.0 -# VOVERLAY: 1.0 -# VSPACING: 3.0 -# results: -# OUTPUT: -# name: expected/grid_lines_overlay.gml -# type: vector -# + - algorithm: qgis:creategridlines + name: Create grid (lines with overlay) + params: + CRS: EPSG:4326 + EXTENT: -1,11.2,-4,6.5 + HOVERLAY: 2.0 + HSPACING: 5.0 + VOVERLAY: 1.0 + VSPACING: 3.0 + results: + OUTPUT: + name: expected/grid_lines_overlay.gml + type: vector + - algorithm: qgis:creategridpolygon name: Create grid (rectangle with overlay) params: @@ -2586,21 +2586,21 @@ tests: # OUTPUT_LAYER: # name: expected/buffer_ovals.gml # type: vector -# -# - algorithm: qgis:creategridlines -# name: Lines grid 0.1 degree spacing -# params: -# CRS: EPSG:4326 -# EXTENT: -0.10453905405405395,8.808021567567568,-2.5010055337837844,4.058021763513514 -# HOVERLAY: 0.0 -# HSPACING: 0.1 -# VOVERLAY: 0.0 -# VSPACING: 0.1 -# results: -# OUTPUT: -# name: expected/create_grid_lines.gml -# type: vector -# + + - algorithm: qgis:creategridlines + name: Lines grid 0.1 degree spacing + params: + CRS: EPSG:4326 + EXTENT: -0.10453905405405395,8.808021567567568,-2.5010055337837844,4.058021763513514 + HOVERLAY: 0.0 + HSPACING: 0.1 + VOVERLAY: 0.0 + VSPACING: 0.1 + results: + OUTPUT: + name: expected/create_grid_lines.gml + type: vector + # - algorithm: qgis:convertgeometrytype # name: polygon to centroid # params: From 591de92b071e72466c6b78225ae1652256c5e009 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 00:33:27 +1000 Subject: [PATCH 05/23] Port gridify to new API --- .../plugins/processing/algs/qgis/Gridify.py | 92 ++++++++----------- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- 2 files changed, 43 insertions(+), 54 deletions(-) diff --git a/python/plugins/processing/algs/qgis/Gridify.py b/python/plugins/processing/algs/qgis/Gridify.py index df782427b3a..2605ccd9cd0 100644 --- a/python/plugins/processing/algs/qgis/Gridify.py +++ b/python/plugins/processing/algs/qgis/Gridify.py @@ -31,36 +31,29 @@ from qgis.core import (QgsGeometry, QgsPointXY, QgsWkbTypes, QgsApplication, - QgsMessageLog, - QgsProcessingUtils) -from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterNumber -from processing.core.outputs import OutputVector + QgsProcessingException, + QgsProcessingParameterNumber) +from processing.algs.qgis.QgisAlgorithm import QgisFeatureBasedAlgorithm -class Gridify(QgisAlgorithm): - INPUT = 'INPUT' +class Gridify(QgisFeatureBasedAlgorithm): + HSPACING = 'HSPACING' VSPACING = 'VSPACING' - OUTPUT = 'OUTPUT' def group(self): return self.tr('Vector general tools') def __init__(self): super().__init__() + self.h_spacing = None + self.v_spacing = None - def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.INPUT, - self.tr('Input Layer'))) - self.addParameter(ParameterNumber(self.HSPACING, - self.tr('Horizontal spacing'), default=0.1)) - self.addParameter(ParameterNumber(self.VSPACING, - self.tr('Vertical spacing'), default=0.1)) - - self.addOutput(OutputVector(self.OUTPUT, self.tr('Snapped'))) + def initParameters(self, config=None): + self.addParameter(QgsProcessingParameterNumber(self.HSPACING, + self.tr('Horizontal spacing'), type=QgsProcessingParameterNumber.Double, minValue=0.0, defaultValue=0.1)) + self.addParameter(QgsProcessingParameterNumber(self.VSPACING, + self.tr('Vertical spacing'), type=QgsProcessingParameterNumber.Double, minValue=0.0, defaultValue=0.1)) def name(self): return 'snappointstogrid' @@ -68,46 +61,45 @@ class Gridify(QgisAlgorithm): def displayName(self): return self.tr('Snap points to grid') - def processAlgorithm(self, parameters, context, feedback): - layer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.INPUT), context) - hSpacing = self.getParameterValue(self.HSPACING) - vSpacing = self.getParameterValue(self.VSPACING) + def outputName(self): + return self.tr('Snapped') - if hSpacing <= 0 or vSpacing <= 0: - raise GeoAlgorithmExecutionException( - self.tr('Invalid grid spacing: {0}/{1}').format(hSpacing, vSpacing)) + def prepareAlgorithm(self, parameters, context, feedback): + self.h_spacing = self.parameterAsDouble(parameters, self.HSPACING, context) + self.v_spacing = self.parameterAsDouble(parameters, self.VSPACING, context) + if self.h_spacing <= 0 or self.v_spacing <= 0: + raise QgsProcessingException( + self.tr('Invalid grid spacing: {0}/{1}').format(self.h_spacing, self.v_spacing)) - writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layer.fields(), layer.wkbType(), layer.crs(), - context) + return True - features = QgsProcessingUtils.getFeatures(layer, context) - total = 100.0 / layer.featureCount() if layer.featureCount() else 0 - - for current, f in enumerate(features): - geom = f.geometry() - geomType = geom.wkbType() + def processFeature(self, feature, feedback): + if feature.hasGeometry(): + geom = feature.geometry() + geomType = QgsWkbTypes.flatType(geom.wkbType()) + newGeom = None if geomType == QgsWkbTypes.Point: - points = self._gridify([geom.asPoint()], hSpacing, vSpacing) + points = self._gridify([geom.asPoint()], self.h_spacing, self.v_spacing) newGeom = QgsGeometry.fromPoint(points[0]) elif geomType == QgsWkbTypes.MultiPoint: - points = self._gridify(geom.aMultiPoint(), hSpacing, vSpacing) + points = self._gridify(geom.aMultiPoint(), self.h_spacing, self.v_spacing) newGeom = QgsGeometry.fromMultiPoint(points) elif geomType == QgsWkbTypes.LineString: - points = self._gridify(geom.asPolyline(), hSpacing, vSpacing) + points = self._gridify(geom.asPolyline(), self.h_spacing, self.v_spacing) if len(points) < 2: - QgsMessageLog.logMessage(self.tr('Failed to gridify feature with FID {0}').format(f.id()), self.tr('Processing'), QgsMessageLog.INFO) + feedback.reportError(self.tr('Failed to gridify feature with FID {0}').format(feature.id())) newGeom = None else: newGeom = QgsGeometry.fromPolyline(points) elif geomType == QgsWkbTypes.MultiLineString: polyline = [] for line in geom.asMultiPolyline(): - points = self._gridify(line, hSpacing, vSpacing) + points = self._gridify(line, self.h_spacing, self.v_spacing) if len(points) > 1: polyline.append(points) if len(polyline) <= 0: - QgsMessageLog.logMessage(self.tr('Failed to gridify feature with FID {0}').format(f.id()), self.tr('Processing'), QgsMessageLog.INFO) + feedback.reportError(self.tr('Failed to gridify feature with FID {0}').format(feature.id())) newGeom = None else: newGeom = QgsGeometry.fromMultiPolyline(polyline) @@ -115,11 +107,11 @@ class Gridify(QgisAlgorithm): elif geomType == QgsWkbTypes.Polygon: polygon = [] for line in geom.asPolygon(): - points = self._gridify(line, hSpacing, vSpacing) + points = self._gridify(line, self.h_spacing, self.v_spacing) if len(points) > 1: polygon.append(points) if len(polygon) <= 0: - QgsMessageLog.logMessage(self.tr('Failed to gridify feature with FID {0}').format(f.id()), self.tr('Processing'), QgsMessageLog.INFO) + feedback.reportError(self.tr('Failed to gridify feature with FID {0}').format(feature.id())) newGeom = None else: newGeom = QgsGeometry.fromPolygon(polygon) @@ -128,7 +120,7 @@ class Gridify(QgisAlgorithm): for polygon in geom.asMultiPolygon(): newPolygon = [] for line in polygon: - points = self._gridify(line, hSpacing, vSpacing) + points = self._gridify(line, self.h_spacing, self.v_spacing) if len(points) > 2: newPolygon.append(points) @@ -136,20 +128,16 @@ class Gridify(QgisAlgorithm): multipolygon.append(newPolygon) if len(multipolygon) <= 0: - QgsMessageLog.logMessage(self.tr('Failed to gridify feature with FID {0}').format(f.id()), self.tr('Processing'), QgsMessageLog.INFO) + feedback.reportError(self.tr('Failed to gridify feature with FID {0}').format(feature.id())) newGeom = None else: newGeom = QgsGeometry.fromMultiPolygon(multipolygon) if newGeom is not None: - feat = QgsFeature() - feat.setGeometry(newGeom) - feat.setAttributes(f.attributes()) - writer.addFeature(feat, QgsFeatureSink.FastInsert) - - feedback.setProgress(int(current * total)) - - del writer + feature.setGeometry(newGeom) + else: + feature.clearGeometry() + return feature def _gridify(self, points, hSpacing, vSpacing): nPoints = [] diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 53397648311..97d113597dc 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -70,6 +70,7 @@ from .ExtractSpecificNodes import ExtractSpecificNodes from .FixedDistanceBuffer import FixedDistanceBuffer from .FixGeometry import FixGeometry from .GeometryByExpression import GeometryByExpression +from .Gridify import Gridify from .GridLine import GridLine from .GridPolygon import GridPolygon from .Heatmap import Heatmap @@ -140,7 +141,6 @@ from .ZonalStatistics import ZonalStatistics # from .ExtractByLocation import ExtractByLocation # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin -# from .Gridify import Gridify # from .HubDistancePoints import HubDistancePoints # from .HubDistanceLines import HubDistanceLines # from .HubLines import HubLines @@ -188,7 +188,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), - # Gridify(), HubDistancePoints(), + # HubDistancePoints(), # HubDistanceLines(), HubLines(), # GeometryConvert(), FieldsCalculator(), # JoinAttributes(), @@ -240,6 +240,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): FixedDistanceBuffer(), FixGeometry(), GeometryByExpression(), + Gridify(), GridLine(), GridPolygon(), Heatmap(), From 0930e18bf929bd6487382c1c89ce8800299f9eff Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 01:26:42 +1000 Subject: [PATCH 06/23] Add tests for gridify --- .../tests/testdata/expected/gridify_lines.gfs | 16 ++++++ .../tests/testdata/expected/gridify_lines.gml | 46 +++++++++++++++ .../tests/testdata/expected/gridify_polys.gfs | 32 +++++++++++ .../tests/testdata/expected/gridify_polys.gml | 57 +++++++++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 26 +++++++++ 5 files changed, 177 insertions(+) create mode 100644 python/plugins/processing/tests/testdata/expected/gridify_lines.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/gridify_lines.gml create mode 100644 python/plugins/processing/tests/testdata/expected/gridify_polys.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/gridify_polys.gml diff --git a/python/plugins/processing/tests/testdata/expected/gridify_lines.gfs b/python/plugins/processing/tests/testdata/expected/gridify_lines.gfs new file mode 100644 index 00000000000..ae929a27136 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/gridify_lines.gfs @@ -0,0 +1,16 @@ + + + gridify_lines + gridify_lines + + 2 + EPSG:4326 + + 7 + 2.00000 + 12.00000 + -4.00000 + 4.00000 + + + diff --git a/python/plugins/processing/tests/testdata/expected/gridify_lines.gml b/python/plugins/processing/tests/testdata/expected/gridify_lines.gml new file mode 100644 index 00000000000..961702eaa01 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/gridify_lines.gml @@ -0,0 +1,46 @@ + + + + + 2-4 + 124 + + + + + + 6,2 8,2 8,4 12,4 + + + + + + + + + 2,0 2,2 4,2 4,4 + + + + + + + + + 8,-4 10,-4 + + + + + 6,-4 10,0 + + + + + + + diff --git a/python/plugins/processing/tests/testdata/expected/gridify_polys.gfs b/python/plugins/processing/tests/testdata/expected/gridify_polys.gfs new file mode 100644 index 00000000000..cc44b1a9047 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/gridify_polys.gfs @@ -0,0 +1,32 @@ + + + gridify_polys + gridify_polys + + 3 + EPSG:4326 + + 6 + 0.00000 + 10.00000 + -4.00000 + 6.00000 + + + name + name + String + 5 + + + intval + intval + Integer + + + floatval + floatval + Real + + + diff --git a/python/plugins/processing/tests/testdata/expected/gridify_polys.gml b/python/plugins/processing/tests/testdata/expected/gridify_polys.gml new file mode 100644 index 00000000000..c255726ba4e --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/gridify_polys.gml @@ -0,0 +1,57 @@ + + + + + -0-4 + 106 + + + + + + 0,0 0,4 4,4 4,2 2,2 2,0 0,0 + aaaaa + 33 + 44.123456 + + + + + Aaaaa + -33 + 0 + + + + + 2,4 2,6 4,6 4,4 2,4 + bbaaa + 0.123 + + + + + 6,0 10,0 10,-4 6,-4 6,0 + ASDF + 0 + + + + + 120 + -100291.43213 + + + + + 4,2 6,0 6,-4 2,0 2,2 4,2 + elim + 2 + 3.33 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 64b881dd059..ac3d7d2929a 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2158,6 +2158,32 @@ tests: name: expected/lines_to_polygon.gml type: vector + - algorithm: qgis:snappointstogrid + name: Gridify polys + params: + INPUT: + name: polys.gml + type: vector + HSPACING: 2 + VSPACING: 2 + results: + OUTPUT: + name: expected/gridify_polys.gml + type: vector + + - algorithm: qgis:snappointstogrid + name: Gridify lines + params: + INPUT: + name: lines.gml + type: vector + HSPACING: 2 + VSPACING: 2 + results: + OUTPUT: + name: expected/gridify_lines.gml + type: vector + # - algorithm: qgis:joinattributestable # name: join the attribute table by common field # params: From fc1746e7706e35ff54a75c664071a2c86732386d Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 01:34:28 +1000 Subject: [PATCH 07/23] Port Hub Distance (points) to new API Improvements: - handle different CRS between points and hubs - add unit test --- .../processing/algs/qgis/HubDistancePoints.py | 113 +++++++++--------- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- .../tests/testdata/custom/hub_points.gfs | 22 ++++ .../tests/testdata/custom/hub_points.gml | 32 +++++ .../testdata/expected/hub_distance_points.gfs | 37 ++++++ .../testdata/expected/hub_distance_points.gml | 95 +++++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 16 +++ 7 files changed, 261 insertions(+), 59 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/custom/hub_points.gfs create mode 100644 python/plugins/processing/tests/testdata/custom/hub_points.gml create mode 100644 python/plugins/processing/tests/testdata/expected/hub_distance_points.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/hub_distance_points.gml diff --git a/python/plugins/processing/algs/qgis/HubDistancePoints.py b/python/plugins/processing/algs/qgis/HubDistancePoints.py index ea670b39b4d..32effadfc14 100644 --- a/python/plugins/processing/algs/qgis/HubDistancePoints.py +++ b/python/plugins/processing/algs/qgis/HubDistancePoints.py @@ -33,34 +33,32 @@ from qgis.core import (QgsField, QgsDistanceArea, QgsFeature, QgsFeatureRequest, + QgsSpatialIndex, QgsWkbTypes, - QgsApplication, - QgsProject, - QgsProcessingUtils) + QgsUnitTypes, + QgsProcessing, + QgsProcessingUtils, + QgsProcessingParameterFeatureSource, + QgsProcessingParameterField, + QgsProcessingParameterEnum, + QgsProcessingParameterFeatureSink, + QgsProcessingException) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTableField -from processing.core.parameters import ParameterSelection -from processing.core.outputs import OutputVector - -from processing.tools import dataobjects - -from math import sqrt class HubDistancePoints(QgisAlgorithm): - POINTS = 'POINTS' + INPUT = 'INPUT' HUBS = 'HUBS' FIELD = 'FIELD' UNIT = 'UNIT' OUTPUT = 'OUTPUT' + LAYER_UNITS = 'LAYER_UNITS' - UNITS = ['Meters', - 'Feet', - 'Miles', - 'Kilometers', - 'Layer units' + UNITS = [QgsUnitTypes.DistanceMeters, + QgsUnitTypes.DistanceFeet, + QgsUnitTypes.DistanceMiles, + QgsUnitTypes.DistanceKilometers, + LAYER_UNITS ] def group(self): @@ -76,16 +74,16 @@ class HubDistancePoints(QgisAlgorithm): self.tr('Kilometers'), self.tr('Layer units')] - self.addParameter(ParameterVector(self.POINTS, - self.tr('Source points layer'))) - self.addParameter(ParameterVector(self.HUBS, - self.tr('Destination hubs layer'))) - self.addParameter(ParameterTableField(self.FIELD, - self.tr('Hub layer name attribute'), self.HUBS)) - self.addParameter(ParameterSelection(self.UNIT, - self.tr('Measurement unit'), self.units)) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Source points layer'))) + self.addParameter(QgsProcessingParameterFeatureSource(self.HUBS, + self.tr('Destination hubs layer'))) + self.addParameter(QgsProcessingParameterField(self.FIELD, + self.tr('Hub layer name attribute'), parentLayerParameterName=self.HUBS)) + self.addParameter(QgsProcessingParameterEnum(self.UNIT, + self.tr('Measurement unit'), self.units)) - self.addOutput(OutputVector(self.OUTPUT, self.tr('Hub distance'), datatype=[dataobjects.TYPE_VECTOR_POINT])) + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Hub distance'), QgsProcessing.TypeVectorPoint)) def name(self): return 'distancetonearesthubpoints' @@ -94,61 +92,62 @@ class HubDistancePoints(QgisAlgorithm): return self.tr('Distance to nearest hub (points)') def processAlgorithm(self, parameters, context, feedback): - layerPoints = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.POINTS), context) - layerHubs = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.HUBS), context) - fieldName = self.getParameterValue(self.FIELD) - - units = self.UNITS[self.getParameterValue(self.UNIT)] - - if layerPoints.source() == layerHubs.source(): - raise GeoAlgorithmExecutionException( + if parameters[self.INPUT] == parameters[self.HUBS]: + raise QgsProcessingException( self.tr('Same layer given for both hubs and spokes')) - fields = layerPoints.fields() + point_source = self.parameterAsSource(parameters, self.INPUT, context) + hub_source = self.parameterAsSource(parameters, self.HUBS, context) + fieldName = self.parameterAsString(parameters, self.FIELD, context) + + units = self.UNITS[self.parameterAsEnum(parameters, self.UNIT, context)] + + fields = point_source.fields() fields.append(QgsField('HubName', QVariant.String)) fields.append(QgsField('HubDist', QVariant.Double)) - writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.Point, layerPoints.crs(), - context) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, QgsWkbTypes.Point, point_source.sourceCrs()) - index = QgsProcessingUtils.createSpatialIndex(layerHubs, context) + index = QgsSpatialIndex(hub_source.getFeatures(QgsFeatureRequest().setSubsetOfAttributes([]).setDestinationCrs(point_source.sourceCrs()))) distance = QgsDistanceArea() - distance.setSourceCrs(layerPoints.crs()) + distance.setSourceCrs(point_source.sourceCrs()) distance.setEllipsoid(context.project().ellipsoid()) # Scan source points, find nearest hub, and write to output file - features = QgsProcessingUtils.getFeatures(layerPoints, context) - total = 100.0 / layerPoints.featureCount() if layerPoints.featureCount() else 0 + features = point_source.getFeatures() + total = 100.0 / point_source.featureCount() if point_source.featureCount() else 0 for current, f in enumerate(features): + if feedback.isCanceled(): + break + + if not f.hasGeometry(): + sink.addFeature(f, QgsFeatureSink.FastInsert) + continue + src = f.geometry().boundingBox().center() neighbors = index.nearestNeighbor(src, 1) - ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], layerHubs.fields()))) + ft = next(hub_source.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], hub_source.fields()).setDestinationCrs(point_source.sourceCrs()))) closest = ft.geometry().boundingBox().center() hubDist = distance.measureLine(src, closest) + if units != self.LAYER_UNITS: + hub_dist_in_desired_units = distance.convertLengthMeasurement(hubDist, units) + else: + hub_dist_in_desired_units = hubDist + attributes = f.attributes() attributes.append(ft[fieldName]) - if units == 'Feet': - attributes.append(hubDist * 3.2808399) - elif units == 'Miles': - attributes.append(hubDist * 0.000621371192) - elif units == 'Kilometers': - attributes.append(hubDist / 1000.0) - elif units != 'Meters': - attributes.append(sqrt( - pow(src.x() - closest.x(), 2.0) + - pow(src.y() - closest.y(), 2.0))) - else: - attributes.append(hubDist) + attributes.append(hub_dist_in_desired_units) feat = QgsFeature() feat.setAttributes(attributes) feat.setGeometry(QgsGeometry.fromPoint(src)) - writer.addFeature(feat, QgsFeatureSink.FastInsert) + sink.addFeature(feat, QgsFeatureSink.FastInsert) feedback.setProgress(int(current * total)) - del writer + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 97d113597dc..d58960a5770 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -75,6 +75,7 @@ from .GridLine import GridLine from .GridPolygon import GridPolygon from .Heatmap import Heatmap from .Hillshade import Hillshade +from .HubDistancePoints import HubDistancePoints from .ImportIntoPostGIS import ImportIntoPostGIS from .ImportIntoSpatialite import ImportIntoSpatialite from .Intersection import Intersection @@ -141,7 +142,7 @@ from .ZonalStatistics import ZonalStatistics # from .ExtractByLocation import ExtractByLocation # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin -# from .HubDistancePoints import HubDistancePoints + # from .HubDistanceLines import HubDistanceLines # from .HubLines import HubLines # from .GeometryConvert import GeometryConvert @@ -188,7 +189,6 @@ class QGISAlgorithmProvider(QgsProcessingProvider): # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), - # HubDistancePoints(), # HubDistanceLines(), HubLines(), # GeometryConvert(), FieldsCalculator(), # JoinAttributes(), @@ -245,6 +245,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): GridPolygon(), Heatmap(), Hillshade(), + HubDistancePoints(), ImportIntoPostGIS(), ImportIntoSpatialite(), Intersection(), diff --git a/python/plugins/processing/tests/testdata/custom/hub_points.gfs b/python/plugins/processing/tests/testdata/custom/hub_points.gfs new file mode 100644 index 00000000000..35cbfd91664 --- /dev/null +++ b/python/plugins/processing/tests/testdata/custom/hub_points.gfs @@ -0,0 +1,22 @@ + + + hub_points + hub_points + + 1 + EPSG:4326 + + 3 + 1.34481 + 6.29897 + -1.25947 + 2.27221 + + + name + name + String + 6 + + + diff --git a/python/plugins/processing/tests/testdata/custom/hub_points.gml b/python/plugins/processing/tests/testdata/custom/hub_points.gml new file mode 100644 index 00000000000..99d14c1d02a --- /dev/null +++ b/python/plugins/processing/tests/testdata/custom/hub_points.gml @@ -0,0 +1,32 @@ + + + + + 1.344807662693645-1.259467184083282 + 6.2989682466352512.272211648033507 + + + + + + 1.34480766269365,-1.25946718408328 + point1 + + + + + 6.29896824663525,0.138489020296281 + point2 + + + + + 3.12290985247467,2.27221164803351 + point3 + + + diff --git a/python/plugins/processing/tests/testdata/expected/hub_distance_points.gfs b/python/plugins/processing/tests/testdata/expected/hub_distance_points.gfs new file mode 100644 index 00000000000..43674410adb --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_distance_points.gfs @@ -0,0 +1,37 @@ + + + hub_distance_points + hub_distance_points + + 1 + EPSG:4326 + + 9 + 0.00000 + 8.00000 + -5.00000 + 3.00000 + + + id + id + Integer + + + id2 + id2 + Integer + + + HubName + HubName + String + 6 + + + HubDist + HubDist + Real + + + diff --git a/python/plugins/processing/tests/testdata/expected/hub_distance_points.gml b/python/plugins/processing/tests/testdata/expected/hub_distance_points.gml new file mode 100644 index 00000000000..4e767e8b117 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_distance_points.gml @@ -0,0 +1,95 @@ + + + + + 0-5 + 83 + + + + + + 1,1 + 1 + 2 + point1 + 254434.675423572 + + + + + 3,3 + 2 + 1 + point3 + 82164.2455422206 + + + + + 2,2 + 3 + 0 + point3 + 128622.227687308 + + + + + 5,2 + 4 + 2 + point3 + 211142.486929284 + + + + + 4,1 + 5 + 1 + point3 + 172016.876891364 + + + + + 0,-5 + 6 + 0 + point1 + 442487.532089586 + + + + + 8,-1 + 7 + 0 + point2 + 227856.24000978 + + + + + 7,-1 + 8 + 0 + point2 + 148835.564980152 + + + + + 0,-1 + 9 + 0 + point1 + 152464.26003518 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index ac3d7d2929a..e79b77c2cd3 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2184,6 +2184,22 @@ tests: name: expected/gridify_lines.gml type: vector + - algorithm: qgis:distancetonearesthubpoints + name: Hub distance points + params: + INPUT: + name: points.gml + type: vector + HUBS: + name: custom/hub_points.gml + type: vector + FIELD: name + UNIT: 0 + results: + OUTPUT: + name: expected/hub_distance_points.gml + type: vector + # - algorithm: qgis:joinattributestable # name: join the attribute table by common field # params: From e0354456e3240d7b2498547727c5ffc27c7610cd Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 18:55:30 +1000 Subject: [PATCH 08/23] Port Hub Distance (lines) to new API Improvements: - handle different CRS between points and hubs - add unit test --- .../processing/algs/qgis/HubDistanceLines.py | 110 +++++++++--------- .../processing/algs/qgis/HubDistancePoints.py | 1 - .../algs/qgis/QGISAlgorithmProvider.py | 6 +- .../testdata/expected/hub_distance_lines.gfs | 37 ++++++ .../testdata/expected/hub_distance_lines.gml | 95 +++++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 16 +++ 6 files changed, 206 insertions(+), 59 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/expected/hub_distance_lines.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/hub_distance_lines.gml diff --git a/python/plugins/processing/algs/qgis/HubDistanceLines.py b/python/plugins/processing/algs/qgis/HubDistanceLines.py index bb360b5a971..491a82a7ff9 100644 --- a/python/plugins/processing/algs/qgis/HubDistanceLines.py +++ b/python/plugins/processing/algs/qgis/HubDistanceLines.py @@ -34,33 +34,33 @@ from qgis.core import (QgsField, QgsFeatureSink, QgsFeatureRequest, QgsWkbTypes, - QgsApplication, - QgsProject, - QgsProcessingUtils) + QgsUnitTypes, + QgsProcessing, + QgsProcessingParameterFeatureSource, + QgsProcessingParameterField, + QgsProcessingParameterEnum, + QgsProcessingParameterFeatureSink, + QgsProcessingException, + QgsSpatialIndex) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTableField -from processing.core.parameters import ParameterSelection -from processing.core.outputs import OutputVector - -from processing.tools import dataobjects from math import sqrt class HubDistanceLines(QgisAlgorithm): - POINTS = 'POINTS' + INPUT = 'INPUT' HUBS = 'HUBS' FIELD = 'FIELD' UNIT = 'UNIT' OUTPUT = 'OUTPUT' - UNITS = ['Meters', - 'Feet', - 'Miles', - 'Kilometers', - 'Layer units' + LAYER_UNITS = 'LAYER_UNITS' + + UNITS = [QgsUnitTypes.DistanceMeters, + QgsUnitTypes.DistanceFeet, + QgsUnitTypes.DistanceMiles, + QgsUnitTypes.DistanceKilometers, + LAYER_UNITS ] def group(self): @@ -76,16 +76,16 @@ class HubDistanceLines(QgisAlgorithm): self.tr('Kilometers'), self.tr('Layer units')] - self.addParameter(ParameterVector(self.POINTS, - self.tr('Source points layer'))) - self.addParameter(ParameterVector(self.HUBS, - self.tr('Destination hubs layer'))) - self.addParameter(ParameterTableField(self.FIELD, - self.tr('Hub layer name attribute'), self.HUBS)) - self.addParameter(ParameterSelection(self.UNIT, - self.tr('Measurement unit'), self.units)) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Source points layer'))) + self.addParameter(QgsProcessingParameterFeatureSource(self.HUBS, + self.tr('Destination hubs layer'))) + self.addParameter(QgsProcessingParameterField(self.FIELD, + self.tr('Hub layer name attribute'), parentLayerParameterName=self.HUBS)) + self.addParameter(QgsProcessingParameterEnum(self.UNIT, + self.tr('Measurement unit'), self.units)) - self.addOutput(OutputVector(self.OUTPUT, self.tr('Hub distance'), datatype=[dataobjects.TYPE_VECTOR_LINE])) + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Hub distance'), QgsProcessing.TypeVectorLine)) def name(self): return 'distancetonearesthublinetohub' @@ -94,61 +94,61 @@ class HubDistanceLines(QgisAlgorithm): return self.tr('Distance to nearest hub (line to hub)') def processAlgorithm(self, parameters, context, feedback): - layerPoints = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.POINTS), context) - layerHubs = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.HUBS), context) - fieldName = self.getParameterValue(self.FIELD) - - units = self.UNITS[self.getParameterValue(self.UNIT)] - - if layerPoints.source() == layerHubs.source(): - raise GeoAlgorithmExecutionException( + if parameters[self.INPUT] == parameters[self.HUBS]: + raise QgsProcessingException( self.tr('Same layer given for both hubs and spokes')) - fields = layerPoints.fields() + point_source = self.parameterAsSource(parameters, self.INPUT, context) + hub_source = self.parameterAsSource(parameters, self.HUBS, context) + fieldName = self.parameterAsString(parameters, self.FIELD, context) + + units = self.UNITS[self.parameterAsEnum(parameters, self.UNIT, context)] + + fields = point_source.fields() fields.append(QgsField('HubName', QVariant.String)) fields.append(QgsField('HubDist', QVariant.Double)) - writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(fields, QgsWkbTypes.LineString, layerPoints.crs(), - context) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, QgsWkbTypes.LineString, point_source.sourceCrs()) - index = QgsProcessingUtils.createSpatialIndex(layerHubs, context) + index = QgsSpatialIndex(hub_source.getFeatures(QgsFeatureRequest().setSubsetOfAttributes([]).setDestinationCrs(point_source.sourceCrs()))) distance = QgsDistanceArea() - distance.setSourceCrs(layerPoints.crs()) + distance.setSourceCrs(point_source.sourceCrs()) distance.setEllipsoid(context.project().ellipsoid()) # Scan source points, find nearest hub, and write to output file - features = QgsProcessingUtils.getFeatures(layerPoints, context) - total = 100.0 / layerPoints.featureCount() if layerPoints.featureCount() else 0 + features = point_source.getFeatures() + total = 100.0 / point_source.featureCount() if point_source.featureCount() else 0 for current, f in enumerate(features): + if feedback.isCanceled(): + break + + if not f.hasGeometry(): + sink.addFeature(f, QgsFeatureSink.FastInsert) + continue src = f.geometry().boundingBox().center() neighbors = index.nearestNeighbor(src, 1) - ft = next(layerHubs.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], layerHubs.fields()))) + ft = next(hub_source.getFeatures(QgsFeatureRequest().setFilterFid(neighbors[0]).setSubsetOfAttributes([fieldName], hub_source.fields()).setDestinationCrs(point_source.sourceCrs()))) closest = ft.geometry().boundingBox().center() hubDist = distance.measureLine(src, closest) + if units != self.LAYER_UNITS: + hub_dist_in_desired_units = distance.convertLengthMeasurement(hubDist, units) + else: + hub_dist_in_desired_units = hubDist + attributes = f.attributes() attributes.append(ft[fieldName]) - if units == 'Feet': - attributes.append(hubDist * 3.2808399) - elif units == 'Miles': - attributes.append(hubDist * 0.000621371192) - elif units == 'Kilometers': - attributes.append(hubDist / 1000.0) - elif units != 'Meters': - attributes.append(sqrt( - pow(src.x() - closest.x(), 2.0) + - pow(src.y() - closest.y(), 2.0))) - else: - attributes.append(hubDist) + attributes.append(hub_dist_in_desired_units) feat = QgsFeature() feat.setAttributes(attributes) feat.setGeometry(QgsGeometry.fromPolyline([src, closest])) - writer.addFeature(feat, QgsFeatureSink.FastInsert) + sink.addFeature(feat, QgsFeatureSink.FastInsert) feedback.setProgress(int(current * total)) - del writer + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/HubDistancePoints.py b/python/plugins/processing/algs/qgis/HubDistancePoints.py index 32effadfc14..7ec153557e6 100644 --- a/python/plugins/processing/algs/qgis/HubDistancePoints.py +++ b/python/plugins/processing/algs/qgis/HubDistancePoints.py @@ -37,7 +37,6 @@ from qgis.core import (QgsField, QgsWkbTypes, QgsUnitTypes, QgsProcessing, - QgsProcessingUtils, QgsProcessingParameterFeatureSource, QgsProcessingParameterField, QgsProcessingParameterEnum, diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index d58960a5770..01f2f57d236 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -75,6 +75,7 @@ from .GridLine import GridLine from .GridPolygon import GridPolygon from .Heatmap import Heatmap from .Hillshade import Hillshade +from .HubDistanceLines import HubDistanceLines from .HubDistancePoints import HubDistancePoints from .ImportIntoPostGIS import ImportIntoPostGIS from .ImportIntoSpatialite import ImportIntoSpatialite @@ -142,8 +143,6 @@ from .ZonalStatistics import ZonalStatistics # from .ExtractByLocation import ExtractByLocation # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin - -# from .HubDistanceLines import HubDistanceLines # from .HubLines import HubLines # from .GeometryConvert import GeometryConvert # from .StatisticsByCategories import StatisticsByCategories @@ -189,7 +188,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), - # HubDistanceLines(), HubLines(), + # HubLines(), # GeometryConvert(), FieldsCalculator(), # JoinAttributes(), # FieldsPyculator(), @@ -245,6 +244,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): GridPolygon(), Heatmap(), Hillshade(), + HubDistanceLines(), HubDistancePoints(), ImportIntoPostGIS(), ImportIntoSpatialite(), diff --git a/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gfs b/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gfs new file mode 100644 index 00000000000..b5b6e761f23 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gfs @@ -0,0 +1,37 @@ + + + hub_distance_lines + hub_distance_lines + + 2 + EPSG:4326 + + 9 + 0.00000 + 8.00000 + -5.00000 + 3.00000 + + + id + id + Integer + + + id2 + id2 + Integer + + + HubName + HubName + String + 6 + + + HubDist + HubDist + Real + + + diff --git a/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gml b/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gml new file mode 100644 index 00000000000..275b34f7a0b --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_distance_lines.gml @@ -0,0 +1,95 @@ + + + + + 0-5 + 83 + + + + + + 1,1 1.34480766269365,-1.25946718408328 + 1 + 2 + point1 + 254434.675423572 + + + + + 3,3 3.12290985247467,2.27221164803351 + 2 + 1 + point3 + 82164.2455422206 + + + + + 2,2 3.12290985247467,2.27221164803351 + 3 + 0 + point3 + 128622.227687308 + + + + + 5,2 3.12290985247467,2.27221164803351 + 4 + 2 + point3 + 211142.486929284 + + + + + 4,1 3.12290985247467,2.27221164803351 + 5 + 1 + point3 + 172016.876891364 + + + + + 0,-5 1.34480766269365,-1.25946718408328 + 6 + 0 + point1 + 442487.532089586 + + + + + 8,-1 6.29896824663525,0.138489020296281 + 7 + 0 + point2 + 227856.24000978 + + + + + 7,-1 6.29896824663525,0.138489020296281 + 8 + 0 + point2 + 148835.564980152 + + + + + 0,-1 1.34480766269365,-1.25946718408328 + 9 + 0 + point1 + 152464.26003518 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index e79b77c2cd3..0b232b7d61a 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2200,6 +2200,22 @@ tests: name: expected/hub_distance_points.gml type: vector + - algorithm: qgis:distancetonearesthublinetohub + name: Hub distance lines + params: + INPUT: + name: points.gml + type: vector + HUBS: + name: custom/hub_points.gml + type: vector + FIELD: name + UNIT: 0 + results: + OUTPUT: + name: expected/hub_distance_lines.gml + type: vector + # - algorithm: qgis:joinattributestable # name: join the attribute table by common field # params: From b4b39996d23cfeae09991f3234f0c221cc9af657 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 19:49:50 +1000 Subject: [PATCH 09/23] Port hub lines algorithm to new API Improvements: - transparent reprojection to match hub/spoke CRS - keep all attributes from matched hub/spoke features - don't break after matching one hub point to spoke - instead join ALL hub/spoke points with matching id values --- python/plugins/processing/algs/help/qgis.yaml | 2 + .../plugins/processing/algs/qgis/HubLines.py | 117 +++++++++++------- .../algs/qgis/QGISAlgorithmProvider.py | 4 +- .../tests/testdata/custom/spoke_points.gfs | 27 ++++ .../tests/testdata/custom/spoke_points.gml | 63 ++++++++++ .../tests/testdata/expected/hub_lines.gfs | 43 +++++++ .../tests/testdata/expected/hub_lines.gml | 84 +++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 16 +++ 8 files changed, 306 insertions(+), 50 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/custom/spoke_points.gfs create mode 100644 python/plugins/processing/tests/testdata/custom/spoke_points.gml create mode 100644 python/plugins/processing/tests/testdata/expected/hub_lines.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/hub_lines.gml diff --git a/python/plugins/processing/algs/help/qgis.yaml b/python/plugins/processing/algs/help/qgis.yaml index f6756f0fd42..ba5ed67e8e1 100755 --- a/python/plugins/processing/algs/help/qgis.yaml +++ b/python/plugins/processing/algs/help/qgis.yaml @@ -242,7 +242,9 @@ qgis:generatepointspixelcentroidsinsidepolygons: qgis:hublines: + This algorithm creates hub and spoke diagrams with lines drawn from points on the Spoke Point layer to matching points in the Hub Point layer. + Determination of which hub goes with each point is based on a match between the Hub ID field on the hub points and the Spoke ID field on the spoke points. qgis:hypsometriccurves: > This algorithm computes hypsometric curves for an input Digital Elevation Model. Curves are produced as table files in an output folder specified by the user. diff --git a/python/plugins/processing/algs/qgis/HubLines.py b/python/plugins/processing/algs/qgis/HubLines.py index b42112a8304..70778010037 100644 --- a/python/plugins/processing/algs/qgis/HubLines.py +++ b/python/plugins/processing/algs/qgis/HubLines.py @@ -31,15 +31,15 @@ from qgis.core import (QgsFeature, QgsGeometry, QgsPointXY, QgsWkbTypes, - QgsApplication, - QgsProcessingUtils) + QgsFeatureRequest, + QgsProcessing, + QgsProcessingParameterFeatureSource, + QgsProcessingParameterField, + QgsProcessingParameterFeatureSink, + QgsProcessingException, + QgsExpression) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTableField -from processing.core.outputs import OutputVector - -from processing.tools import dataobjects +from processing.tools import vector class HubLines(QgisAlgorithm): @@ -55,17 +55,21 @@ class HubLines(QgisAlgorithm): def __init__(self): super().__init__() - def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.HUBS, - self.tr('Hub layer'))) - self.addParameter(ParameterTableField(self.HUB_FIELD, - self.tr('Hub ID field'), self.HUBS)) - self.addParameter(ParameterVector(self.SPOKES, - self.tr('Spoke layer'))) - self.addParameter(ParameterTableField(self.SPOKE_FIELD, - self.tr('Spoke ID field'), self.SPOKES)) + def tags(self): + return self.tr('join,points,lines,connect,hub,spoke').split(',') - self.addOutput(OutputVector(self.OUTPUT, self.tr('Hub lines'), datatype=[dataobjects.TYPE_VECTOR_LINE])) + def initAlgorithm(self, config=None): + + self.addParameter(QgsProcessingParameterFeatureSource(self.HUBS, + self.tr('Hub layer'))) + self.addParameter(QgsProcessingParameterField(self.HUB_FIELD, + self.tr('Hub ID field'), parentLayerParameterName=self.HUBS)) + self.addParameter(QgsProcessingParameterFeatureSource(self.SPOKES, + self.tr('Spoke layer'))) + self.addParameter(QgsProcessingParameterField(self.SPOKE_FIELD, + self.tr('Spoke ID field'), parentLayerParameterName=self.SPOKES)) + + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Hub lines'), QgsProcessing.TypeVectorLine)) def name(self): return 'hublines' @@ -74,44 +78,61 @@ class HubLines(QgisAlgorithm): return self.tr('Hub lines') def processAlgorithm(self, parameters, context, feedback): - layerHub = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.HUBS), context) - layerSpoke = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.SPOKES), context) - - fieldHub = self.getParameterValue(self.HUB_FIELD) - fieldSpoke = self.getParameterValue(self.SPOKE_FIELD) - - if layerHub.source() == layerSpoke.source(): - raise GeoAlgorithmExecutionException( + if parameters[self.SPOKES] == parameters[self.HUBS]: + raise QgsProcessingException( self.tr('Same layer given for both hubs and spokes')) - writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(layerSpoke.fields(), QgsWkbTypes.LineString, - layerSpoke.crs(), context) + hub_source = self.parameterAsSource(parameters, self.HUBS, context) + spoke_source = self.parameterAsSource(parameters, self.SPOKES, context) + field_hub = self.parameterAsString(parameters, self.HUB_FIELD, context) + field_hub_index = hub_source.fields().lookupField(field_hub) + field_spoke = self.parameterAsString(parameters, self.SPOKE_FIELD, context) + field_spoke_index = hub_source.fields().lookupField(field_spoke) - spokes = QgsProcessingUtils.getFeatures(layerSpoke, context) - hubs = QgsProcessingUtils.getFeatures(layerHub, context) - total = 100.0 / layerSpoke.featureCount() if layerSpoke.featureCount() else 0 + fields = vector.combineFields(hub_source.fields(), spoke_source.fields()) - for current, spokepoint in enumerate(spokes): - p = spokepoint.geometry().boundingBox().center() - spokeX = p.x() - spokeY = p.y() - spokeId = str(spokepoint[fieldSpoke]) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, QgsWkbTypes.LineString, hub_source.sourceCrs()) - for hubpoint in hubs: - hubId = str(hubpoint[fieldHub]) - if hubId == spokeId: - p = hubpoint.geometry().boundingBox().center() - hubX = p.x() - hubY = p.y() + hubs = hub_source.getFeatures() + total = 100.0 / hub_source.featureCount() if hub_source.featureCount() else 0 + + matching_field_types = hub_source.fields().at(field_hub_index).type() == spoke_source.fields().at(field_spoke_index).type() + + for current, hub_point in enumerate(hubs): + if feedback.isCanceled(): + break + + if not hub_point.hasGeometry(): + continue + + p = hub_point.geometry().boundingBox().center() + hub_x = p.x() + hub_y = p.y() + hub_id = str(hub_point[field_hub]) + hub_attributes = hub_point.attributes() + + request = QgsFeatureRequest().setDestinationCrs(hub_source.sourceCrs()) + if matching_field_types: + request.setFilterExpression(QgsExpression.createFieldEqualityExpression(field_spoke, hub_attributes[field_hub_index])) + + spokes = spoke_source.getFeatures() + for spoke_point in spokes: + if feedback.isCanceled(): + break + + spoke_id = str(spoke_point[field_spoke]) + if hub_id == spoke_id: + p = spoke_point.geometry().boundingBox().center() + spoke_x = p.x() + spoke_y = p.y() f = QgsFeature() - f.setAttributes(spokepoint.attributes()) + f.setAttributes(hub_attributes + spoke_point.attributes()) f.setGeometry(QgsGeometry.fromPolyline( - [QgsPointXY(spokeX, spokeY), QgsPointXY(hubX, hubY)])) - writer.addFeature(f, QgsFeatureSink.FastInsert) - - break + [QgsPointXY(hub_x, hub_y), QgsPointXY(spoke_x, spoke_y)])) + sink.addFeature(f, QgsFeatureSink.FastInsert) feedback.setProgress(int(current * total)) - del writer + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 01f2f57d236..e6da61cf343 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -77,6 +77,7 @@ from .Heatmap import Heatmap from .Hillshade import Hillshade from .HubDistanceLines import HubDistanceLines from .HubDistancePoints import HubDistancePoints +from .HubLines import HubLines from .ImportIntoPostGIS import ImportIntoPostGIS from .ImportIntoSpatialite import ImportIntoSpatialite from .Intersection import Intersection @@ -143,7 +144,6 @@ from .ZonalStatistics import ZonalStatistics # from .ExtractByLocation import ExtractByLocation # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin -# from .HubLines import HubLines # from .GeometryConvert import GeometryConvert # from .StatisticsByCategories import StatisticsByCategories # from .FieldsCalculator import FieldsCalculator @@ -188,7 +188,6 @@ class QGISAlgorithmProvider(QgsProcessingProvider): # SelectByLocation(), # ExtractByLocation(), # SpatialJoin(), - # HubLines(), # GeometryConvert(), FieldsCalculator(), # JoinAttributes(), # FieldsPyculator(), @@ -246,6 +245,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): Hillshade(), HubDistanceLines(), HubDistancePoints(), + HubLines(), ImportIntoPostGIS(), ImportIntoSpatialite(), Intersection(), diff --git a/python/plugins/processing/tests/testdata/custom/spoke_points.gfs b/python/plugins/processing/tests/testdata/custom/spoke_points.gfs new file mode 100644 index 00000000000..5166cf9a29f --- /dev/null +++ b/python/plugins/processing/tests/testdata/custom/spoke_points.gfs @@ -0,0 +1,27 @@ + + + spoke_points + spoke_points + + 1 + EPSG:4326 + + 7 + 1.27875 + 6.82625 + -4.16750 + 3.88250 + + + id + id + Integer + + + name + name + String + 8 + + + diff --git a/python/plugins/processing/tests/testdata/custom/spoke_points.gml b/python/plugins/processing/tests/testdata/custom/spoke_points.gml new file mode 100644 index 00000000000..3896f58f81a --- /dev/null +++ b/python/plugins/processing/tests/testdata/custom/spoke_points.gml @@ -0,0 +1,63 @@ + + + + + 1.27875-4.1675 + 6.8262499999999993.882499999999999 + + + + + + 5.07625,-2.1725 + 1 + point 1 + + + + + 5.82,3.8825 + 2 + point 2 + + + + + 1.62,1.4675 + 3 + point 3 + + + + + 6.68625,1.23125 + 4 + point 4 + + + + + 1.27875,-3.66875 + 4 + point 4a + + + + + 3.81625,-4.1675 + 4 + point 4b + + + + + 6.82625,-2.79375 + 8 + point 8 + + + diff --git a/python/plugins/processing/tests/testdata/expected/hub_lines.gfs b/python/plugins/processing/tests/testdata/expected/hub_lines.gfs new file mode 100644 index 00000000000..553c3cbf06c --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_lines.gfs @@ -0,0 +1,43 @@ + + + hub_lines + hub_lines + + 2 + EPSG:4326 + + 7 + 1.00000 + 7.00000 + -4.16750 + 3.88250 + + + id + id + Integer + + + id2 + id2 + Integer + + + fid_2 + fid_2 + String + 14 + + + id_2 + id_2 + Integer + + + name + name + String + 8 + + + diff --git a/python/plugins/processing/tests/testdata/expected/hub_lines.gml b/python/plugins/processing/tests/testdata/expected/hub_lines.gml new file mode 100644 index 00000000000..b530cce0641 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/hub_lines.gml @@ -0,0 +1,84 @@ + + + + + 1-4.1675 + 73.8825 + + + + + + 1,1 5.07625,-2.1725 + 1 + 2 + spoke_points.0 + 1 + point 1 + + + + + 3,3 5.82,3.8825 + 2 + 1 + spoke_points.1 + 2 + point 2 + + + + + 2,2 1.62,1.4675 + 3 + 0 + spoke_points.2 + 3 + point 3 + + + + + 5,2 6.68625,1.23125 + 4 + 2 + spoke_points.3 + 4 + point 4 + + + + + 5,2 1.27875,-3.66875 + 4 + 2 + spoke_points.4 + 4 + point 4a + + + + + 5,2 3.81625,-4.1675 + 4 + 2 + spoke_points.5 + 4 + point 4b + + + + + 7,-1 6.82625,-2.79375 + 8 + 0 + spoke_points.6 + 8 + point 8 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 0b232b7d61a..18fd8566b39 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2216,6 +2216,22 @@ tests: name: expected/hub_distance_lines.gml type: vector + - algorithm: qgis:hublines + name: Hub lines + params: + HUBS: + name: points.gml + type: vector + SPOKES: + name: custom/spoke_points.gml + type: vector + HUB_FIELD: id + SPOKE_FIELD: id + results: + OUTPUT: + name: expected/hub_lines.gml + type: vector + # - algorithm: qgis:joinattributestable # name: join the attribute table by common field # params: From ec4df6c019efddc05e9555c43d58a155b811d078 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 3 Aug 2017 23:26:39 +1000 Subject: [PATCH 10/23] Port points to path to new API Improvements: - Maintain Z/M values - Keep original data type for group/order fields - Group field is optional - Added unit tests - Don't export text files for features by default --- python/plugins/processing/algs/help/qgis.yaml | 2 + .../processing/algs/qgis/PointsToPaths.py | 179 ++++++++++-------- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- .../testdata/expected/points_to_path.gfs | 26 +++ .../testdata/expected/points_to_path.gml | 21 ++ .../expected/points_to_path_grouped.gfs | 31 +++ .../expected/points_to_path_grouped.gml | 38 ++++ .../tests/testdata/qgis_algorithm_tests.yaml | 25 +++ 8 files changed, 250 insertions(+), 77 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path.gml create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gml diff --git a/python/plugins/processing/algs/help/qgis.yaml b/python/plugins/processing/algs/help/qgis.yaml index ba5ed67e8e1..a60b0af4a27 100755 --- a/python/plugins/processing/algs/help/qgis.yaml +++ b/python/plugins/processing/algs/help/qgis.yaml @@ -370,7 +370,9 @@ qgis:pointslayerfromtable: > The attributes table of the resulting layer will be the input table. qgis:pointstopath: + Converts a point layer to a line layer, by joining points in a defined order. + Points can be grouped by a field to output individual line features per group. qgis:polarplot: > This algorithm generates a polar plot based on the value of an input vector layer. diff --git a/python/plugins/processing/algs/qgis/PointsToPaths.py b/python/plugins/processing/algs/qgis/PointsToPaths.py index e73c57a9fd5..df2105b33c0 100644 --- a/python/plugins/processing/algs/qgis/PointsToPaths.py +++ b/python/plugins/processing/algs/qgis/PointsToPaths.py @@ -29,36 +29,34 @@ __revision__ = '$Format:%H$' import os from datetime import datetime -from qgis.PyQt.QtCore import QVariant -from qgis.core import (QgsApplication, - QgsFeature, +from qgis.core import (QgsFeature, QgsFeatureSink, QgsFields, QgsField, QgsGeometry, QgsDistanceArea, - QgsProject, + QgsPointXY, + QgsLineString, QgsWkbTypes, - QgsProcessingUtils) + QgsFeatureRequest, + QgsProcessingParameterFeatureSource, + QgsProcessingParameterField, + QgsProcessingParameterString, + QgsProcessing, + QgsProcessingParameterFeatureSink, + QgsProcessingParameterFolderDestination) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTableField -from processing.core.parameters import ParameterString -from processing.core.outputs import OutputVector -from processing.core.outputs import OutputDirectory -from processing.tools import dataobjects class PointsToPaths(QgisAlgorithm): - VECTOR = 'VECTOR' + INPUT = 'INPUT' GROUP_FIELD = 'GROUP_FIELD' ORDER_FIELD = 'ORDER_FIELD' DATE_FORMAT = 'DATE_FORMAT' - #GAP_PERIOD = 'GAP_PERIOD' - OUTPUT_LINES = 'OUTPUT_LINES' - OUTPUT_TEXT = 'OUTPUT_TEXT' + OUTPUT = 'OUTPUT' + OUTPUT_TEXT_DIR = 'OUTPUT_TEXT_DIR' def group(self): return self.tr('Vector creation tools') @@ -66,20 +64,23 @@ class PointsToPaths(QgisAlgorithm): def __init__(self): super().__init__() + def tags(self): + return self.tr('join,points,lines,connect').split(',') + def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.VECTOR, - self.tr('Input point layer'), [dataobjects.TYPE_VECTOR_POINT])) - self.addParameter(ParameterTableField(self.GROUP_FIELD, - self.tr('Group field'), self.VECTOR)) - self.addParameter(ParameterTableField(self.ORDER_FIELD, - self.tr('Order field'), self.VECTOR)) - self.addParameter(ParameterString(self.DATE_FORMAT, - self.tr('Date format (if order field is DateTime)'), '', optional=True)) - #self.addParameter(ParameterNumber( - # self.GAP_PERIOD, - # 'Gap period (if order field is DateTime)', 0, 60, 0)) - self.addOutput(OutputVector(self.OUTPUT_LINES, self.tr('Paths'), datatype=[dataobjects.TYPE_VECTOR_LINE])) - self.addOutput(OutputDirectory(self.OUTPUT_TEXT, self.tr('Directory'))) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Input point layer'), [QgsProcessing.TypeVectorPoint])) + self.addParameter(QgsProcessingParameterField(self.ORDER_FIELD, + self.tr('Order field'), parentLayerParameterName=self.INPUT)) + self.addParameter(QgsProcessingParameterField(self.GROUP_FIELD, + self.tr('Group field'), parentLayerParameterName=self.INPUT, optional=True)) + self.addParameter(QgsProcessingParameterString(self.DATE_FORMAT, + self.tr('Date format (if order field is DateTime)'), optional=True)) + + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Paths'), QgsProcessing.TypeVectorLine)) + output_dir_param = QgsProcessingParameterFolderDestination(self.OUTPUT_TEXT_DIR, self.tr('Directory for text output'), optional=True) + output_dir_param.setCreateByDefault(False) + self.addParameter(output_dir_param) def name(self): return 'pointstopath' @@ -88,29 +89,58 @@ class PointsToPaths(QgisAlgorithm): return self.tr('Points to path') def processAlgorithm(self, parameters, context, feedback): - layer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.VECTOR), context) - groupField = self.getParameterValue(self.GROUP_FIELD) - orderField = self.getParameterValue(self.ORDER_FIELD) - dateFormat = str(self.getParameterValue(self.DATE_FORMAT)) - #gap = int(self.getParameterValue(self.GAP_PERIOD)) - dirName = self.getOutputValue(self.OUTPUT_TEXT) + source = self.parameterAsSource(parameters, self.INPUT, context) + group_field_name = self.parameterAsString(parameters, self.GROUP_FIELD, context) + order_field_name = self.parameterAsString(parameters, self.ORDER_FIELD, context) + date_format = self.parameterAsString(parameters, self.DATE_FORMAT, context) + text_dir = self.parameterAsString(parameters, self.OUTPUT_TEXT_DIR, context) + + group_field_index = source.fields().lookupField(group_field_name) + order_field_index = source.fields().lookupField(order_field_name) + + if group_field_index >= 0: + group_field_def = source.fields().at(group_field_index) + else: + group_field_def = None + order_field_def = source.fields().at(order_field_index) fields = QgsFields() - fields.append(QgsField('group', QVariant.String, '', 254, 0)) - fields.append(QgsField('begin', QVariant.String, '', 254, 0)) - fields.append(QgsField('end', QVariant.String, '', 254, 0)) - writer = self.getOutputFromName(self.OUTPUT_LINES).getVectorWriter(fields, QgsWkbTypes.LineString, layer.crs(), - context) + if group_field_def is not None: + fields.append(group_field_def) + begin_field = QgsField(order_field_def) + begin_field.setName('begin') + fields.append(begin_field) + end_field = QgsField(order_field_def) + end_field.setName('end') + fields.append(end_field) + + output_wkb = QgsWkbTypes.LineString + if QgsWkbTypes.hasM(source.wkbType()): + output_wkb = QgsWkbTypes.addM(output_wkb) + if QgsWkbTypes.hasZ(source.wkbType()): + output_wkb = QgsWkbTypes.addZ(output_wkb) + + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, output_wkb, source.sourceCrs()) points = dict() - features = QgsProcessingUtils.getFeatures(layer, context) - total = 100.0 / layer.featureCount() if layer.featureCount() else 0 + features = source.getFeatures(QgsFeatureRequest().setSubsetOfAttributes([group_field_index, order_field_index])) + total = 100.0 / source.featureCount() if source.featureCount() else 0 for current, f in enumerate(features): - point = f.geometry().asPoint() - group = f[groupField] - order = f[orderField] - if dateFormat != '': - order = datetime.strptime(str(order), dateFormat) + if feedback.isCanceled(): + break + + if not f.hasGeometry(): + continue + + point = f.geometry().geometry().clone() + if group_field_index >= 0: + group = f.attributes()[group_field_index] + else: + group = 1 + order = f.attributes()[order_field_index] + if date_format != '': + order = datetime.strptime(str(order), date_format) if group in points: points[group].append((order, point)) else: @@ -121,46 +151,45 @@ class PointsToPaths(QgisAlgorithm): feedback.setProgress(0) da = QgsDistanceArea() - da.setSourceCrs(layer.sourceCrs()) + da.setSourceCrs(source.sourceCrs()) da.setEllipsoid(context.project().ellipsoid()) current = 0 total = 100.0 / len(points) if points else 1 for group, vertices in list(points.items()): + if feedback.isCanceled(): + break + vertices.sort() f = QgsFeature() - f.initAttributes(len(fields)) - f.setFields(fields) - f['group'] = group - f['begin'] = vertices[0][0] - f['end'] = vertices[-1][0] + attributes = [] + if group_field_index >= 0: + attributes.append(group) + attributes.extend([vertices[0][0], vertices[-1][0]]) + f.setAttributes(attributes) + line = [node[1] for node in vertices] - fileName = os.path.join(dirName, '%s.txt' % group) + if text_dir: + fileName = os.path.join(text_dir, '%s.txt' % group) - with open(fileName, 'w') as fl: - fl.write('angle=Azimuth\n') - fl.write('heading=Coordinate_System\n') - fl.write('dist_units=Default\n') + with open(fileName, 'w') as fl: + fl.write('angle=Azimuth\n') + fl.write('heading=Coordinate_System\n') + fl.write('dist_units=Default\n') - line = [] - i = 0 - for node in vertices: - line.append(node[1]) + for i in range(len(line)): + if i == 0: + fl.write('startAt=%f;%f;90\n' % (line[i].x(), line[i].y())) + fl.write('survey=Polygonal\n') + fl.write('[data]\n') + else: + angle = line[i - 1].azimuth(line[i]) + distance = da.measureLine(QgsPointXY(line[i - 1]), QgsPointXY(line[i])) + fl.write('%f;%f;90\n' % (angle, distance)) - if i == 0: - fl.write('startAt=%f;%f;90\n' % (node[1].x(), node[1].y())) - fl.write('survey=Polygonal\n') - fl.write('[data]\n') - else: - angle = line[i - 1].azimuth(line[i]) - distance = da.measureLine(line[i - 1], line[i]) - fl.write('%f;%f;90\n' % (angle, distance)) - - i += 1 - - f.setGeometry(QgsGeometry.fromPolyline(line)) - writer.addFeature(f, QgsFeatureSink.FastInsert) + f.setGeometry(QgsGeometry(QgsLineString(line))) + sink.addFeature(f, QgsFeatureSink.FastInsert) current += 1 feedback.setProgress(int(current * total)) - del writer + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index e6da61cf343..c9f11fabbd2 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -94,6 +94,7 @@ from .PointOnSurface import PointOnSurface from .PointsAlongGeometry import PointsAlongGeometry from .PointsInPolygon import PointsInPolygon from .PointsLayerFromTable import PointsLayerFromTable +from .PointsToPaths import PointsToPaths from .PoleOfInaccessibility import PoleOfInaccessibility from .Polygonize import Polygonize from .PolygonsToLines import PolygonsToLines @@ -152,7 +153,6 @@ from .ZonalStatistics import ZonalStatistics # from .PointsDisplacement import PointsDisplacement # from .PointsFromPolygons import PointsFromPolygons # from .PointsFromLines import PointsFromLines -# from .PointsToPaths import PointsToPaths # from .SetVectorStyle import SetVectorStyle # from .SetRasterStyle import SetRasterStyle # from .SelectByAttributeSum import SelectByAttributeSum @@ -194,7 +194,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): # StatisticsByCategories(), # RasterLayerStatistics(), PointsDisplacement(), # PointsFromPolygons(), - # PointsFromLines(), PointsToPaths(), + # PointsFromLines(), # SetVectorStyle(), SetRasterStyle(), # HypsometricCurves(), # FieldsMapper(), SelectByAttributeSum(), Datasources2Vrt(), @@ -262,6 +262,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): PointsAlongGeometry(), PointsInPolygon(), PointsLayerFromTable(), + PointsToPaths(), PoleOfInaccessibility(), Polygonize(), PolygonsToLines(), diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path.gfs b/python/plugins/processing/tests/testdata/expected/points_to_path.gfs new file mode 100644 index 00000000000..89a8bf0693f --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path.gfs @@ -0,0 +1,26 @@ + + + points_to_path + points_to_path + + 2 + EPSG:4326 + + 1 + 0.00000 + 8.00000 + -5.00000 + 3.00000 + + + begin + begin + Integer + + + end + end + Integer + + + diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path.gml b/python/plugins/processing/tests/testdata/expected/points_to_path.gml new file mode 100644 index 00000000000..5057497c90d --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path.gml @@ -0,0 +1,21 @@ + + + + + 0-5 + 83 + + + + + + 1,1 3,3 2,2 5,2 4,1 0,-5 8,-1 7,-1 0,-1 + 1 + 9 + + + diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gfs b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gfs new file mode 100644 index 00000000000..076d497b096 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gfs @@ -0,0 +1,31 @@ + + + points_to_path_grouped + points_to_path_grouped + + 2 + EPSG:4326 + + 3 + 0.00000 + 8.00000 + -5.00000 + 3.00000 + + + id2 + id2 + Integer + + + begin + begin + Integer + + + end + end + Integer + + + diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gml b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gml new file mode 100644 index 00000000000..e82344ba8ef --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped.gml @@ -0,0 +1,38 @@ + + + + + 0-5 + 83 + + + + + + 1,1 5,2 + 2 + 1 + 4 + + + + + 3,3 4,1 + 1 + 2 + 5 + + + + + 2,2 0,-5 8,-1 7,-1 0,-1 + 0 + 3 + 9 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 18fd8566b39..a0adf00e432 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2232,6 +2232,31 @@ tests: name: expected/hub_lines.gml type: vector + - algorithm: qgis:pointstopath + name: Points to path (non grouped) + params: + INPUT: + name: points.gml + type: vector + ORDER_FIELD: id + results: + OUTPUT: + name: expected/points_to_path.gml + type: vector + + - algorithm: qgis:pointstopath + name: Points to path (grouped) + params: + INPUT: + name: points.gml + type: vector + ORDER_FIELD: id + GROUP_FIELD: id2 + results: + OUTPUT: + name: expected/points_to_path_grouped.gml + type: vector + # - algorithm: qgis:joinattributestable # name: join the attribute table by common field # params: From 7132faa97470866baf09759bb17a2ecc005c5680 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 00:03:41 +1000 Subject: [PATCH 11/23] Port Topocolor algorithm to new API --- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- .../processing/algs/qgis/TopoColors.py | 73 +++++++++++-------- .../testdata/expected/topocolor_polys.gml | 4 +- .../tests/testdata/qgis_algorithm_tests.yaml | 53 +++++++------- 4 files changed, 76 insertions(+), 59 deletions(-) diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index c9f11fabbd2..e1e1cc7e47a 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -133,6 +133,7 @@ from .SplitWithLines import SplitWithLines from .SumLines import SumLines from .SymmetricalDifference import SymmetricalDifference from .TextToFloat import TextToFloat +from .TopoColors import TopoColor from .Translate import Translate from .TruncateTable import TruncateTable from .Union import Union @@ -169,7 +170,6 @@ from .ZonalStatistics import ZonalStatistics # from .RasterCalculator import RasterCalculator # from .ExecuteSQL import ExecuteSQL # from .FindProjection import FindProjection -# from .TopoColors import TopoColor # from .EliminateSelection import EliminateSelection pluginPath = os.path.normpath(os.path.join( @@ -206,7 +206,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): # IdwInterpolation(), TinInterpolation(), # RasterCalculator(), # ExecuteSQL(), FindProjection(), - # TopoColor(), EliminateSelection() + # EliminateSelection() # ] algs = [AddTableField(), Aspect(), @@ -301,6 +301,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): SumLines(), SymmetricalDifference(), TextToFloat(), + TopoColor(), Translate(), TruncateTable(), Union(), diff --git a/python/plugins/processing/algs/qgis/TopoColors.py b/python/plugins/processing/algs/qgis/TopoColors.py index b50625b4e75..33c078cce05 100644 --- a/python/plugins/processing/algs/qgis/TopoColors.py +++ b/python/plugins/processing/algs/qgis/TopoColors.py @@ -31,33 +31,31 @@ import sys from collections import defaultdict -from qgis.core import (QgsApplication, - QgsField, +from qgis.core import (QgsField, QgsFeatureSink, QgsGeometry, QgsSpatialIndex, QgsPointXY, NULL, - QgsProcessingUtils) + QgsProcessing, + QgsProcessingParameterFeatureSource, + QgsProcessingParameterNumber, + QgsProcessingParameterEnum, + QgsProcessingParameterFeatureSink) from qgis.PyQt.QtCore import (QVariant) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.parameters import (ParameterVector, - ParameterSelection, - ParameterNumber) -from processing.core.outputs import OutputVector -from processing.tools import dataobjects pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] class TopoColor(QgisAlgorithm): - INPUT_LAYER = 'INPUT_LAYER' + INPUT = 'INPUT' MIN_COLORS = 'MIN_COLORS' MIN_DISTANCE = 'MIN_DISTANCE' BALANCE = 'BALANCE' - OUTPUT_LAYER = 'OUTPUT_LAYER' + OUTPUT = 'OUTPUT' def tags(self): return self.tr('topocolor,colors,graph,adjacent,assign').split(',') @@ -69,21 +67,23 @@ class TopoColor(QgisAlgorithm): super().__init__() def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.INPUT_LAYER, - self.tr('Input layer'), [dataobjects.TYPE_VECTOR_POLYGON])) - self.addParameter(ParameterNumber(self.MIN_COLORS, - self.tr('Minimum number of colors'), 1, 1000, 4)) - self.addParameter(ParameterNumber(self.MIN_DISTANCE, - self.tr('Minimum distance between features'), 0.0, 999999999.0, 0.0)) + + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Input layer'), [QgsProcessing.TypeVectorPolygon])) + self.addParameter(QgsProcessingParameterNumber(self.MIN_COLORS, + self.tr('Minimum number of colors'), minValue=1, maxValue=1000, defaultValue=4)) + self.addParameter(QgsProcessingParameterNumber(self.MIN_DISTANCE, + self.tr('Minimum distance between features'), type=QgsProcessingParameterNumber.Double, + minValue=0.0, maxValue=999999999.0, defaultValue=0.0)) balance_by = [self.tr('By feature count'), self.tr('By assigned area'), self.tr('By distance between colors')] - self.addParameter(ParameterSelection( + self.addParameter(QgsProcessingParameterEnum( self.BALANCE, self.tr('Balance color assignment'), - balance_by, default=0)) + options=balance_by, defaultValue=0)) - self.addOutput(OutputVector(self.OUTPUT_LAYER, self.tr('Colored'), datatype=[dataobjects.TYPE_VECTOR_POLYGON])) + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Colored'), QgsProcessing.TypeVectorPolygon)) def name(self): return 'topologicalcoloring' @@ -92,18 +92,18 @@ class TopoColor(QgisAlgorithm): return self.tr('Topological coloring') def processAlgorithm(self, parameters, context, feedback): - layer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.INPUT_LAYER), context) - min_colors = self.getParameterValue(self.MIN_COLORS) - balance_by = self.getParameterValue(self.BALANCE) - min_distance = self.getParameterValue(self.MIN_DISTANCE) + source = self.parameterAsSource(parameters, self.INPUT, context) + min_colors = self.parameterAsInt(parameters, self.MIN_COLORS, context) + balance_by = self.parameterAsEnum(parameters, self.BALANCE, context) + min_distance = self.parameterAsDouble(parameters, self.MIN_DISTANCE, context) - fields = layer.fields() + fields = source.fields() fields.append(QgsField('color_id', QVariant.Int)) - writer = self.getOutputFromName( - self.OUTPUT_LAYER).getVectorWriter(fields, layer.wkbType(), layer.crs(), context) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, source.wkbType(), source.sourceCrs()) - features = {f.id(): f for f in QgsProcessingUtils.getFeatures(layer, context)} + features = {f.id(): f for f in source.getFeatures()} topology, id_graph = self.compute_graph(features, feedback, min_distance=min_distance) feature_colors = ColoringAlgorithm.balanced(features, @@ -118,6 +118,9 @@ class TopoColor(QgisAlgorithm): total = 20.0 / len(features) current = 0 for feature_id, input_feature in features.items(): + if feedback.isCanceled(): + break + output_feature = input_feature attributes = input_feature.attributes() if feature_id in feature_colors: @@ -126,11 +129,11 @@ class TopoColor(QgisAlgorithm): attributes.append(NULL) output_feature.setAttributes(attributes) - writer.addFeature(output_feature, QgsFeatureSink.FastInsert) + sink.addFeature(output_feature, QgsFeatureSink.FastInsert) current += 1 feedback.setProgress(80 + int(current * total)) - del writer + return {self.OUTPUT: dest_id} @staticmethod def compute_graph(features, feedback, create_id_graph=False, min_distance=0): @@ -148,6 +151,9 @@ class TopoColor(QgisAlgorithm): i = 0 for feature_id, f in features_with_geometry.items(): + if feedback.isCanceled(): + break + g = f.geometry() if min_distance > 0: g = g.buffer(min_distance, 5) @@ -172,6 +178,9 @@ class TopoColor(QgisAlgorithm): feedback.setProgress(int(i * total)) for feature_id, f in features_with_geometry.items(): + if feedback.isCanceled(): + break + if feature_id not in s.node_edge: s.add_edge(feature_id, None) @@ -206,6 +215,9 @@ class ColoringAlgorithm: i = 0 for (feature_id, n) in sorted_by_count: + if feedback.isCanceled(): + break + # first work out which already assigned colors are adjacent to this feature adjacent_colors = set() for neighbour in graph.node_edge[feature_id]: @@ -240,6 +252,9 @@ class ColoringAlgorithm: # loop through these, and calculate the minimum distance from this feature to the nearest # feature with each assigned color for other_feature_id, c in other_features.items(): + if feedback.isCanceled(): + break + other_geometry = features[other_feature_id].geometry() other_centroid = QgsPointXY(other_geometry.centroid().geometry()) diff --git a/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml b/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml index 09ad337285b..c16afe08ef6 100644 --- a/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml +++ b/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml @@ -41,7 +41,7 @@ 8.23935 -3.11331 11 - 4 + 5 @@ -52,7 +52,7 @@ 8.23935 -6.11331 12 - 5 + 4 diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index a0adf00e432..aa182b10f96 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2598,32 +2598,33 @@ tests: name: expected/polygon_from_extent.gml type: vector -# - algorithm: qgis:topologicalcoloring -# name: Topological coloring -# params: -# INPUT_LAYER: -# name: custom/adjacent_polys.gml -# type: vector -# MIN_COLORS: 4 -# results: -# OUTPUT_LAYER: -# name: expected/topocolor_polys.gml -# type: vector -# -# - algorithm: qgis:topologicalcoloring -# name: Topological coloring w/ min distance -# params: -# BALANCE: '0' -# INPUT_LAYER: -# name: custom/adjacent_polys.gml -# type: vector -# MIN_COLORS: 4 -# MIN_DISTANCE: 4.0 -# results: -# OUTPUT_LAYER: -# name: expected/topocolor_polys_min_dist.gml -# type: vector -# + - algorithm: qgis:topologicalcoloring + name: Topological coloring + params: + BALANCE: 0 + INPUT: + name: custom/adjacent_polys.gml + type: vector + MIN_COLORS: 4 + results: + OUTPUT: + name: expected/topocolor_polys.gml + type: vector + + - algorithm: qgis:topologicalcoloring + name: Topological coloring w/ min distance + params: + BALANCE: 0 + INPUT: + name: custom/adjacent_polys.gml + type: vector + MIN_COLORS: 4 + MIN_DISTANCE: 4.0 + results: + OUTPUT: + name: expected/topocolor_polys_min_dist.gml + type: vector + - algorithm: qgis:regularpoints name: Regular point with standard extent params: From 03bae593bb3dfac77ef9f11ec8839f2d58a92ff2 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 00:37:14 +1000 Subject: [PATCH 12/23] Port Eliminate Selection to new API --- .../algs/qgis/EliminateSelection.py | 68 +++++++++++-------- .../algs/qgis/QGISAlgorithmProvider.py | 4 +- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/python/plugins/processing/algs/qgis/EliminateSelection.py b/python/plugins/processing/algs/qgis/EliminateSelection.py index b051d34545f..ae54df30e0d 100644 --- a/python/plugins/processing/algs/qgis/EliminateSelection.py +++ b/python/plugins/processing/algs/qgis/EliminateSelection.py @@ -34,15 +34,14 @@ from qgis.core import (QgsFeatureRequest, QgsFeature, QgsFeatureSink, QgsGeometry, - QgsMessageLog, - QgsProcessingUtils) + QgsProcessingException, + QgsProcessingUtils, + QgsProcessingParameterVectorLayer, + QgsProcessingParameterEnum, + QgsProcessing, + QgsProcessingParameterFeatureSink) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterSelection -from processing.core.outputs import OutputVector -from processing.tools import dataobjects pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] @@ -67,16 +66,17 @@ class EliminateSelection(QgisAlgorithm): super().__init__() def initAlgorithm(self, config=None): - self.modes = [self.tr('Largest area'), + self.modes = [self.tr('Largest Area'), self.tr('Smallest Area'), - self.tr('Largest common boundary')] + self.tr('Largest Common Boundary')] - self.addParameter(ParameterVector(self.INPUT, - self.tr('Input layer'), [dataobjects.TYPE_VECTOR_POLYGON])) - self.addParameter(ParameterSelection(self.MODE, - self.tr('Merge selection with the neighbouring polygon with the'), - self.modes)) - self.addOutput(OutputVector(self.OUTPUT, self.tr('Eliminated'), datatype=[dataobjects.TYPE_VECTOR_POLYGON])) + self.addParameter(QgsProcessingParameterVectorLayer(self.INPUT, + self.tr('Input layer'), [QgsProcessing.TypeVectorPolygon])) + self.addParameter(QgsProcessingParameterEnum(self.MODE, + self.tr('Merge selection with the neighbouring polygon with the'), + options=self.modes)) + + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Eliminated'), QgsProcessing.TypeVectorPolygon)) def name(self): return 'eliminateselectedpolygons' @@ -85,29 +85,32 @@ class EliminateSelection(QgisAlgorithm): return self.tr('Eliminate selected polygons') def processAlgorithm(self, parameters, context, feedback): - inLayer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.INPUT), context) - boundary = self.getParameterValue(self.MODE) == self.MODE_BOUNDARY - smallestArea = self.getParameterValue(self.MODE) == self.MODE_SMALLEST_AREA + inLayer = self.parameterAsVectorLayer(parameters, self.INPUT, context) + boundary = self.parameterAsEnum(parameters, self.MODE, context) == self.MODE_BOUNDARY + smallestArea = self.parameterAsEnum(parameters, self.MODE, context) == self.MODE_SMALLEST_AREA if inLayer.selectedFeatureCount() == 0: - QgsMessageLog.logMessage(self.tr('{0}: (No selection in input layer "{1}")').format(self.displayName(), self.getParameterValue(self.INPUT)), - self.tr('Processing'), QgsMessageLog.WARNING) + feedback.reportError(self.tr('{0}: (No selection in input layer "{1}")').format(self.displayName(), parameters[self.INPUT])) featToEliminate = [] selFeatIds = inLayer.selectedFeatureIds() - output = self.getOutputFromName(self.OUTPUT) - writer = output.getVectorWriter(inLayer.fields(), inLayer.wkbType(), inLayer.crs(), context) + + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + inLayer.fields(), inLayer.wkbType(), inLayer.sourceCrs()) for aFeat in inLayer.getFeatures(): + if feedback.isCanceled(): + break + if aFeat.id() in selFeatIds: # Keep references to the features to eliminate featToEliminate.append(aFeat) else: # write the others to output - writer.addFeature(aFeat, QgsFeatureSink.FastInsert) + sink.addFeature(aFeat, QgsFeatureSink.FastInsert) # Delete all features to eliminate in processLayer - processLayer = output.layer + processLayer = QgsProcessingUtils.mapLayerFromString(dest_id, context) processLayer.startEditing() # ANALYZE @@ -129,6 +132,9 @@ class EliminateSelection(QgisAlgorithm): # Iterate over the polygons to eliminate for i in range(len(featToEliminate)): + if feedback.isCanceled(): + break + feat = featToEliminate.pop() geom2Eliminate = feat.geometry() bbox = geom2Eliminate.boundingBox() @@ -145,6 +151,9 @@ class EliminateSelection(QgisAlgorithm): engine.prepareGeometry() while fit.nextFeature(selFeat): + if feedback.isCanceled(): + break + selGeom = selFeat.geometry() if engine.intersects(selGeom.geometry()): @@ -193,7 +202,7 @@ class EliminateSelection(QgisAlgorithm): if processLayer.changeGeometry(mergeWithFid, newGeom): madeProgress = True else: - raise GeoAlgorithmExecutionException( + raise QgsProcessingException( self.tr('Could not replace geometry of feature with id {0}').format(mergeWithFid)) start = start + add @@ -207,7 +216,12 @@ class EliminateSelection(QgisAlgorithm): # End while if not processLayer.commitChanges(): - raise GeoAlgorithmExecutionException(self.tr('Could not commit changes')) + raise QgsProcessingException(self.tr('Could not commit changes')) for feature in featNotEliminated: - writer.addFeature(feature, QgsFeatureSink.FastInsert) + if feedback.isCanceled(): + break + + sink.addFeature(feature, QgsFeatureSink.FastInsert) + + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index e1e1cc7e47a..5a04391fa90 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -60,6 +60,7 @@ from .DensifyGeometriesInterval import DensifyGeometriesInterval from .Difference import Difference from .DropGeometry import DropGeometry from .DropMZValues import DropMZValues +from .EliminateSelection import EliminateSelection from .EquivalentNumField import EquivalentNumField from .Explode import Explode from .ExportGeometryInfo import ExportGeometryInfo @@ -170,7 +171,6 @@ from .ZonalStatistics import ZonalStatistics # from .RasterCalculator import RasterCalculator # from .ExecuteSQL import ExecuteSQL # from .FindProjection import FindProjection -# from .EliminateSelection import EliminateSelection pluginPath = os.path.normpath(os.path.join( os.path.split(os.path.dirname(__file__))[0], os.pardir)) @@ -206,7 +206,6 @@ class QGISAlgorithmProvider(QgsProcessingProvider): # IdwInterpolation(), TinInterpolation(), # RasterCalculator(), # ExecuteSQL(), FindProjection(), - # EliminateSelection() # ] algs = [AddTableField(), Aspect(), @@ -228,6 +227,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): Difference(), DropGeometry(), DropMZValues(), + EliminateSelection(), EquivalentNumField(), Explode(), ExportGeometryInfo(), From 5d635d190df39e4061f16b825290ec43fcf804f4 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 18:45:48 +1000 Subject: [PATCH 13/23] Allow list of acceptable raster hashes for processing algorithm tests Differences in gdal libraries mean the hash value may differ between platforms. Allow multiple acceptable hashes to be listed for expected test results --- .../plugins/processing/tests/AlgorithmsTestBase.py | 5 ++++- .../tests/testdata/qgis_algorithm_tests.yaml | 12 +++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/python/plugins/processing/tests/AlgorithmsTestBase.py b/python/plugins/processing/tests/AlgorithmsTestBase.py index 65c4b119687..c4a2f437b39 100644 --- a/python/plugins/processing/tests/AlgorithmsTestBase.py +++ b/python/plugins/processing/tests/AlgorithmsTestBase.py @@ -279,7 +279,10 @@ class AlgorithmsTest(object): dataArray = nan_to_num(dataset.ReadAsArray(0)) strhash = hashlib.sha224(dataArray.data).hexdigest() - self.assertEqual(strhash, expected_result['hash']) + if not isinstance(expected_result['hash'], str): + self.assertTrue(strhash in expected_result['hash']) + else: + self.assertEqual(strhash, expected_result['hash']) elif 'file' == expected_result['type']: expected_filepath = self.filepath_from_param(expected_result) result_filepath = results[id] diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index aa182b10f96..297f4123497 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -1148,7 +1148,9 @@ tests: Z_FACTOR: 1.0 results: OUTPUT: - hash: 762865ee485a6736d188402aa10e6fd38a812a9e45a7dd2d4885a63a + hash: + - 762865ee485a6736d188402aa10e6fd38a812a9e45a7dd2d4885a63a + - f6a8e64647ae93a94f2a4945add8986526a7a07bc85849f3690d15b2 type: rasterhash - algorithm: qgis:slope @@ -1160,7 +1162,9 @@ tests: Z_FACTOR: 1.0 results: OUTPUT: - hash: 151ea76a21b286c16567eb6b4b692925a84145b65561a0017effb1a1 + hash: + - 151ea76a21b286c16567eb6b4b692925a84145b65561a0017effb1a1 + - 177475642c57428b395bc0a1e7e86fc1cfd4d86ffc19f31ff8bc964d type: rasterhash - algorithm: qgis:ruggednessindex @@ -1186,7 +1190,9 @@ tests: Z_FACTOR: 1.0 results: OUTPUT: - hash: 58365b3715b925d6286e7f082ebd9c2a20f09fa1c922176d3f238002 + hash: + - 58365b3715b925d6286e7f082ebd9c2a20f09fa1c922176d3f238002 + - 75cca4c1a870a1e21185a2d85b33b6d9958a69fc6ebb04e4d6ceb8a3 type: rasterhash # - algorithm: qgis:relief From e8d667cac33f5992c9f1a325c5bebda773cf98ea Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 19:07:03 +1000 Subject: [PATCH 14/23] Allow testing of layer equality without throwing asserts Sometimes in tests it's required to check for layer equality without aborting in case of mismatches --- python/testing/__init__.py | 79 ++++++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/python/testing/__init__.py b/python/testing/__init__.py index 5e2ae736cf0..a495a335681 100644 --- a/python/testing/__init__.py +++ b/python/testing/__init__.py @@ -55,6 +55,21 @@ class TestCase(_TestCase): { fields: { a: skip, b: { precision: 2 }, geometry: { precision: 5 } } { fields: { __all__: cast( str ) } } """ + self.checkLayersEqual(layer_expected, layer_result, True, **kwargs) + + def checkLayersEqual(self, layer_expected, layer_result, use_asserts=False, **kwargs): + """ + :param layer_expected: The first layer to compare + :param layer_result: The second layer to compare + :param use_asserts: If true, asserts are used to test conditions, if false, asserts + are not used and the function will only return False if the test fails + :param request: Optional, A feature request. This can be used to specify + an order by clause to make sure features are compared in + a given sequence if they don't match by default. + :keyword compare: A map of comparison options. e.g. + { fields: { a: skip, b: { precision: 2 }, geometry: { precision: 5 } } + { fields: { __all__: cast( str ) } } + """ try: request = kwargs['request'] @@ -67,10 +82,16 @@ class TestCase(_TestCase): compare = {} # Compare CRS - _TestCase.assertEqual(self, layer_expected.dataProvider().crs().authid(), layer_result.dataProvider().crs().authid()) + if use_asserts: + _TestCase.assertEqual(self, layer_expected.dataProvider().crs().authid(), layer_result.dataProvider().crs().authid()) + elif not layer_expected.dataProvider().crs().authid() == layer_result.dataProvider().crs().authid(): + return False # Compare features - _TestCase.assertEqual(self, layer_expected.featureCount(), layer_result.featureCount()) + if use_asserts: + _TestCase.assertEqual(self, layer_expected.featureCount(), layer_result.featureCount()) + elif layer_expected.featureCount() != layer_result.featureCount(): + return False try: precision = compare['geometry']['precision'] @@ -89,17 +110,20 @@ class TestCase(_TestCase): geom1 = feats[1].geometry().geometry().asWkt(precision) else: geom1 = None - _TestCase.assertEqual( - self, - geom0, - geom1, - 'Features {}/{} differ in geometry: \n\n {}\n\n vs \n\n {}'.format( - feats[0].id(), - feats[1].id(), + if use_asserts: + _TestCase.assertEqual( + self, geom0, - geom1 + geom1, + 'Features {}/{} differ in geometry: \n\n {}\n\n vs \n\n {}'.format( + feats[0].id(), + feats[1].id(), + geom0, + geom1 + ) ) - ) + elif geom0 != geom1: + return False for attr_expected, field_expected in zip(feats[0].attributes(), layer_expected.fields().toList()): try: @@ -134,21 +158,26 @@ class TestCase(_TestCase): attr_expected = round(attr_expected, cmp['precision']) attr_result = round(attr_result, cmp['precision']) - _TestCase.assertEqual( - self, - attr_expected, - attr_result, - 'Features {}/{} differ in attributes\n\n * Field1: {} ({})\n * Field2: {} ({})\n\n * {} != {}'.format( - feats[0].id(), - feats[1].id(), - field_expected.name(), - field_expected.typeName(), - field_result.name(), - field_result.typeName(), - repr(attr_expected), - repr(attr_result) + if use_asserts: + _TestCase.assertEqual( + self, + attr_expected, + attr_result, + 'Features {}/{} differ in attributes\n\n * Field1: {} ({})\n * Field2: {} ({})\n\n * {} != {}'.format( + feats[0].id(), + feats[1].id(), + field_expected.name(), + field_expected.typeName(), + field_result.name(), + field_result.typeName(), + repr(attr_expected), + repr(attr_result) + ) ) - ) + elif attr_expected != attr_result: + return False + + return True def assertFilesEqual(self, filepath_expected, filepath_result): with open(filepath_expected, 'r') as file_expected: From 9968962ab925474da97218eede9162106eb84d41 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 21:39:40 +1000 Subject: [PATCH 15/23] Allow specifying multiple possible vector layer results for processing tests Some algorithms are non-deterministic and the results may vary from run to run. In this case we allow specifying multiple possible valid results, and the test will pass if the result layer matches any of these. --- .../processing/tests/AlgorithmsTestBase.py | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/python/plugins/processing/tests/AlgorithmsTestBase.py b/python/plugins/processing/tests/AlgorithmsTestBase.py index c4a2f437b39..2ae126ccbf6 100644 --- a/python/plugins/processing/tests/AlgorithmsTestBase.py +++ b/python/plugins/processing/tests/AlgorithmsTestBase.py @@ -43,6 +43,7 @@ import tempfile from osgeo.gdalconst import GA_ReadOnly from numpy import nan_to_num +from copy import deepcopy import processing @@ -186,7 +187,10 @@ class AlgorithmsTest(object): if param['type'] in ['vector', 'file', 'table', 'regex']: outdir = tempfile.mkdtemp() self.cleanup_paths.append(outdir) - basename = os.path.basename(param['name']) + if isinstance(param['name'], str): + basename = os.path.basename(param['name']) + else: + basename = os.path.basename(param['name'][0]) filepath = os.path.join(outdir, basename) return filepath elif param['type'] == 'rasterhash': @@ -198,6 +202,19 @@ class AlgorithmsTest(object): raise KeyError("Unknown type '{}' specified for parameter".format(param['type'])) + def load_layers(self, id, param): + layers = [] + if param['type'] in ('vector', 'table') and isinstance(param['name'], str): + layers.append(self.load_layer(id, param)) + elif param['type'] in ('vector', 'table'): + for n in param['name']: + layer_param = deepcopy(param) + layer_param['name'] = n + layers.append(self.load_layer(id, layer_param)) + else: + layers.append(self.load_layer(id, param)) + return layers + def load_layer(self, id, param): """ Loads a layer which was specified as parameter. @@ -253,7 +270,7 @@ class AlgorithmsTest(object): self.assertTrue(result_lyr.isValid()) continue - expected_lyr = self.load_layer(id, expected_result) + expected_lyrs = self.load_layers(id, expected_result) if 'in_place_result' in expected_result: result_lyr = QgsProcessingUtils.mapLayerFromString(self.in_place_layers[id], context) self.assertTrue(result_lyr.isValid(), self.in_place_layers[id]) @@ -271,7 +288,15 @@ class AlgorithmsTest(object): compare = expected_result.get('compare', {}) - self.assertLayersEqual(expected_lyr, result_lyr, compare=compare) + if len(expected_lyrs) == 1: + self.assertLayersEqual(expected_lyrs[0], result_lyr, compare=compare) + else: + res = False + for l in expected_lyrs: + if self.checkLayersEqual(l, result_lyr, compare=compare): + res = True + break + self.assertTrue(res, 'Could not find matching layer in expected results') elif 'rasterhash' == expected_result['type']: print("id:{} result:{}".format(id, results[id])) From c2559d827317fa201d1e1211b41bef00150ff6cd Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 21:40:49 +1000 Subject: [PATCH 16/23] Add second reference layer for topocolor algorithm --- .../testdata/expected/topocolor_polys.gml | 4 +- .../testdata/expected/topocolor_polys2.gfs | 46 ++++++ .../testdata/expected/topocolor_polys2.gml | 133 ++++++++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 4 +- 4 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/expected/topocolor_polys2.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/topocolor_polys2.gml diff --git a/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml b/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml index c16afe08ef6..09ad337285b 100644 --- a/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml +++ b/python/plugins/processing/tests/testdata/expected/topocolor_polys.gml @@ -41,7 +41,7 @@ 8.23935 -3.11331 11 - 5 + 4 @@ -52,7 +52,7 @@ 8.23935 -6.11331 12 - 4 + 5 diff --git a/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gfs b/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gfs new file mode 100644 index 00000000000..eb07beba751 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gfs @@ -0,0 +1,46 @@ + + + topocolor_polys + topocolor_polys + + 3 + EPSG:4326 + + 11 + -0.76065 + 14.23935 + -6.11331 + 5.88669 + + + left + left + Real + + + top + top + Real + + + right + right + Real + + + bottom + bottom + Real + + + id + id + Integer + + + color_id + color_id + Integer + + + diff --git a/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gml b/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gml new file mode 100644 index 00000000000..c16afe08ef6 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/topocolor_polys2.gml @@ -0,0 +1,133 @@ + + + + + -0.760650357995228-6.11330548926014 + 14.23934964200485.88669451073986 + + + + + + -0.760650357995228,-0.113305489260142 2.23934964200477,-0.113305489260142 2.23934964200477,-3.11330548926014 -0.760650357995228,-3.11330548926014 -0.760650357995228,-0.113305489260142 + -0.76065 + -0.11331 + 2.23935 + -3.11331 + 3 + 1 + + + + + -0.760650357995228,-3.11330548926014 2.23934964200477,-3.11330548926014 2.23934964200477,-6.11330548926014 -0.760650357995228,-6.11330548926014 -0.760650357995228,-3.11330548926014 + -0.76065 + -3.11331 + 2.23935 + -6.11331 + 4 + 3 + + + + + 5.23934964200477,-0.113305489260142 8.23934964200477,-0.113305489260142 8.23934964200477,-3.11330548926014 5.23934964200477,-3.11330548926014 5.23934964200477,-0.113305489260142 + 5.23935 + -0.11331 + 8.23935 + -3.11331 + 11 + 5 + + + + + 5.23934964200477,-3.11330548926014 8.23934964200477,-3.11330548926014 8.23934964200477,-6.11330548926014 5.23934964200477,-6.11330548926014 5.23934964200477,-3.11330548926014 + 5.23935 + -3.11331 + 8.23935 + -6.11331 + 12 + 4 + + + + + 8.23934964200477,-3.11330548926014 11.2393496420048,-3.11330548926014 11.2393496420048,-6.11330548926014 8.23934964200477,-6.11330548926014 8.23934964200477,-3.11330548926014 + 8.23935 + -3.11331 + 11.23935 + -6.11331 + 16 + 3 + + + + + 11.2393496420048,-0.113305489260142 14.2393496420048,-0.113305489260142 14.2393496420048,-3.11330548926014 11.2393496420048,-3.11330548926014 11.2393496420048,-0.113305489260142 + 11.23935 + -0.11331 + 14.23935 + -3.11331 + 19 + 2 + + + + + 11.2393496420048,-3.11330548926014 14.2393496420048,-3.11330548926014 14.2393496420048,-6.11330548926014 11.2393496420048,-6.11330548926014 11.2393496420048,-3.11330548926014 + 11.23935 + -3.11331 + 14.23935 + -6.11331 + 20 + 5 + + + + + 2.23934964200477,5.88669451073986 5.23934964200477,5.88669451073986 5.23934964200477,2.88669451073986 2.23934964200477,2.88669451073986 2.23934964200477,-0.113305489260142 -0.760650357995228,-0.113305489260142 -0.760650357995228,2.88669451073986 -0.760650357995228,5.88669451073986 2.23934964200477,5.88669451073986 + 2.23935 + 5.88669 + 5.23935 + 2.88669 + 5 + 3 + + + + + 5.23934964200477,2.88669451073986 8.23934964200477,2.88669451073986 11.2393496420048,2.88669451073986 11.2393496420048,-0.113305489260142 11.2393496420048,-3.11330548926014 8.23934964200477,-3.11330548926014 8.23934964200477,-0.113305489260142 5.23934964200477,-0.113305489260142 5.23934964200477,2.88669451073986 + 5.23935 + 2.88669 + 8.23935 + 10 + 1 + + + + + 2.23934964200477,2.88669451073986 5.23934964200477,2.88669451073986 5.23934964200477,-0.113305489260142 5.23934964200477,-3.11330548926014 5.23934964200477,-6.11330548926014 2.23934964200477,-6.11330548926014 2.23934964200477,-3.11330548926014 2.23934964200477,-0.113305489260142 2.23934964200477,2.88669451073986 + 2.23935 + 2.88669 + 5.23935 + 6 + 2 + + + + + 5.23934964200477,5.88669451073986 8.23934964200477,5.88669451073986 11.2393496420048,5.88669451073986 14.2393496420048,5.88669451073986 14.2393496420048,2.88669451073986 14.2393496420048,-0.113305489260142 11.2393496420048,-0.113305489260142 11.2393496420048,2.88669451073986 8.23934964200477,2.88669451073986 5.23934964200477,2.88669451073986 5.23934964200477,5.88669451073986 + 5.23935 + 5.88669 + 8.23935 + 2.88669 + 9 + 4 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 297f4123497..846d2c65b1f 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2614,8 +2614,10 @@ tests: MIN_COLORS: 4 results: OUTPUT: - name: expected/topocolor_polys.gml type: vector + name: + - expected/topocolor_polys.gml + - expected/topocolor_polys2.gml - algorithm: qgis:topologicalcoloring name: Topological coloring w/ min distance From a1f487d679ed180a34a6949b24c9eb2a208ef462 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 4 Aug 2017 21:45:16 +1000 Subject: [PATCH 17/23] Add alternate reference layer for points to path algorithm --- .../expected/points_to_path_grouped2.gfs | 31 +++++++++++++++ .../expected/points_to_path_grouped2.gml | 39 +++++++++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 4 +- 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gml diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gfs b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gfs new file mode 100644 index 00000000000..076d497b096 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gfs @@ -0,0 +1,31 @@ + + + points_to_path_grouped + points_to_path_grouped + + 2 + EPSG:4326 + + 3 + 0.00000 + 8.00000 + -5.00000 + 3.00000 + + + id2 + id2 + Integer + + + begin + begin + Integer + + + end + end + Integer + + + diff --git a/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gml b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gml new file mode 100644 index 00000000000..68ec6408c30 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/points_to_path_grouped2.gml @@ -0,0 +1,39 @@ + + + + + 0-5 + 83 + + + + + + 2,2 0,-5 8,-1 7,-1 0,-1 + 0 + 3 + 9 + + + + + 3,3 4,1 + 1 + 2 + 5 + + + + + 1,1 5,2 + 2 + 1 + 4 + + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 846d2c65b1f..83b6a0b3dd4 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2260,7 +2260,9 @@ tests: GROUP_FIELD: id2 results: OUTPUT: - name: expected/points_to_path_grouped.gml + name: + - expected/points_to_path_grouped.gml + - expected/points_to_path_grouped2.gml type: vector # - algorithm: qgis:joinattributestable From 572dadab0157e60ea1d75d2cc854ee7ba85cb436 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 05:06:46 +1000 Subject: [PATCH 18/23] Remember window geometry in multi input dialogs --- python/plugins/processing/gui/MultipleFileInputDialog.py | 8 ++++++++ python/plugins/processing/gui/MultipleInputDialog.py | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/python/plugins/processing/gui/MultipleFileInputDialog.py b/python/plugins/processing/gui/MultipleFileInputDialog.py index 43762ac8708..2b448056f55 100644 --- a/python/plugins/processing/gui/MultipleFileInputDialog.py +++ b/python/plugins/processing/gui/MultipleFileInputDialog.py @@ -32,6 +32,7 @@ import os from qgis.core import QgsSettings from qgis.PyQt import uic +from qgis.PyQt.QtCore import QByteArray from qgis.PyQt.QtWidgets import QDialog, QAbstractItemView, QPushButton, QDialogButtonBox, QFileDialog from qgis.PyQt.QtGui import QStandardItemModel, QStandardItem @@ -65,7 +66,14 @@ class MultipleFileInputDialog(BASE, WIDGET): self.btnRemove.clicked.connect(lambda: self.removeRows()) self.btnRemoveAll.clicked.connect(lambda: self.removeRows(True)) + self.settings = QgsSettings() + self.restoreGeometry(self.settings.value("/Processing/multipleFileInputDialogGeometry", QByteArray())) + self.populateList() + self.finished.connect(self.saveWindowGeometry) + + def saveWindowGeometry(self): + self.settings.setValue("/Processing/multipleInputDialogGeometry", self.saveGeometry()) def populateList(self): model = QStandardItemModel() diff --git a/python/plugins/processing/gui/MultipleInputDialog.py b/python/plugins/processing/gui/MultipleInputDialog.py index e609af1959f..a8fd860f02d 100644 --- a/python/plugins/processing/gui/MultipleInputDialog.py +++ b/python/plugins/processing/gui/MultipleInputDialog.py @@ -29,8 +29,10 @@ __revision__ = '$Format:%H$' import os +from qgis.core import QgsSettings from qgis.PyQt import uic from qgis.PyQt.QtCore import Qt +from qgis.PyQt.QtCore import QByteArray from qgis.PyQt.QtWidgets import QDialog, QAbstractItemView, QPushButton, QDialogButtonBox from qgis.PyQt.QtGui import QStandardItemModel, QStandardItem @@ -71,7 +73,14 @@ class MultipleInputDialog(BASE, WIDGET): self.btnClearSelection.clicked.connect(lambda: self.selectAll(False)) self.btnToggleSelection.clicked.connect(self.toggleSelection) + self.settings = QgsSettings() + self.restoreGeometry(self.settings.value("/Processing/multipleInputDialogGeometry", QByteArray())) + self.populateList() + self.finished.connect(self.saveWindowGeometry) + + def saveWindowGeometry(self): + self.settings.setValue("/Processing/multipleInputDialogGeometry", self.saveGeometry()) def populateList(self): model = QStandardItemModel() From adda744576be7722377a0fc81dc18658b600c9e7 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 05:41:15 +1000 Subject: [PATCH 19/23] Port Join Attributes alg to new API Improvements: - don't fetch unused geometry for joined table --- .../processing/algs/qgis/JoinAttributes.py | 81 ++++++++++--------- .../algs/qgis/QGISAlgorithmProvider.py | 4 +- .../tests/testdata/qgis_algorithm_tests.yaml | 30 +++---- 3 files changed, 59 insertions(+), 56 deletions(-) diff --git a/python/plugins/processing/algs/qgis/JoinAttributes.py b/python/plugins/processing/algs/qgis/JoinAttributes.py index 1354d5a7a79..5898bd89186 100644 --- a/python/plugins/processing/algs/qgis/JoinAttributes.py +++ b/python/plugins/processing/algs/qgis/JoinAttributes.py @@ -30,14 +30,13 @@ import os from qgis.core import (QgsFeature, QgsFeatureSink, - QgsApplication, - QgsProcessingUtils) + QgsFeatureRequest, + QgsProcessingParameterFeatureSource, + QgsProcessingUtils, + QgsProcessingParameterField, + QgsProcessingParameterFeatureSink) from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTable -from processing.core.parameters import ParameterTableField -from processing.core.outputs import OutputVector from processing.tools import vector pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] @@ -45,11 +44,11 @@ pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] class JoinAttributes(QgisAlgorithm): - OUTPUT_LAYER = 'OUTPUT_LAYER' - INPUT_LAYER = 'INPUT_LAYER' - INPUT_LAYER_2 = 'INPUT_LAYER_2' - TABLE_FIELD = 'TABLE_FIELD' - TABLE_FIELD_2 = 'TABLE_FIELD_2' + OUTPUT = 'OUTPUT' + INPUT = 'INPUT' + INPUT_2 = 'INPUT_2' + FIELD = 'FIELD' + FIELD_2 = 'FIELD_2' def group(self): return self.tr('Vector general tools') @@ -58,16 +57,15 @@ class JoinAttributes(QgisAlgorithm): super().__init__() def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.INPUT_LAYER, - self.tr('Input layer'))) - self.addParameter(ParameterTable(self.INPUT_LAYER_2, - self.tr('Input layer 2'), False)) - self.addParameter(ParameterTableField(self.TABLE_FIELD, - self.tr('Table field'), self.INPUT_LAYER)) - self.addParameter(ParameterTableField(self.TABLE_FIELD_2, - self.tr('Table field 2'), self.INPUT_LAYER_2)) - self.addOutput(OutputVector(self.OUTPUT_LAYER, - self.tr('Joined layer'))) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Input layer'))) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT_2, + self.tr('Input layer 2'))) + self.addParameter(QgsProcessingParameterField(self.FIELD, + self.tr('Table field'), parentLayerParameterName=self.INPUT)) + self.addParameter(QgsProcessingParameterField(self.FIELD_2, + self.tr('Table field 2'), parentLayerParameterName=self.INPUT_2)) + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Joined layer'))) def name(self): return 'joinattributestable' @@ -76,26 +74,27 @@ class JoinAttributes(QgisAlgorithm): return self.tr('Join attributes table') def processAlgorithm(self, parameters, context, feedback): - input = self.getParameterValue(self.INPUT_LAYER) - input2 = self.getParameterValue(self.INPUT_LAYER_2) - output = self.getOutputFromName(self.OUTPUT_LAYER) - field = self.getParameterValue(self.TABLE_FIELD) - field2 = self.getParameterValue(self.TABLE_FIELD_2) + input = self.parameterAsSource(parameters, self.INPUT, context) + input2 = self.parameterAsSource(parameters, self.INPUT_2, context) + field = self.parameterAsString(parameters, self.FIELD, context) + field2 = self.parameterAsString(parameters, self.FIELD_2, context) - layer = QgsProcessingUtils.mapLayerFromString(input, context) - joinField1Index = layer.fields().lookupField(field) + joinField1Index = input.fields().lookupField(field) + joinField2Index = input2.fields().lookupField(field2) - layer2 = QgsProcessingUtils.mapLayerFromString(input2, context) - joinField2Index = layer2.fields().lookupField(field2) + outFields = vector.combineFields(input.fields(), input2.fields()) - outFields = vector.combineVectorFields(layer, layer2) - writer = output.getVectorWriter(outFields, layer.wkbType(), layer.crs(), context) + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + outFields, input.wkbType(), input.sourceCrs()) - # Cache attributes of Layer 2 + # Cache attributes of input2 cache = {} - features = QgsProcessingUtils.getFeatures(layer2, context) - total = 100.0 / layer2.featureCount() if layer2.featureCount() else 0 + features = input2.getFeatures(QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry)) + total = 100.0 / input2.featureCount() if input2.featureCount() else 0 for current, feat in enumerate(features): + if feedback.isCanceled(): + break + attrs = feat.attributes() joinValue2 = str(attrs[joinField2Index]) if joinValue2 not in cache: @@ -104,14 +103,18 @@ class JoinAttributes(QgisAlgorithm): # Create output vector layer with additional attribute outFeat = QgsFeature() - features = QgsProcessingUtils.getFeatures(layer, context) - total = 100.0 / layer.featureCount() if layer.featureCount() else 0 + features = input.getFeatures() + total = 100.0 / input.featureCount() if input.featureCount() else 0 for current, feat in enumerate(features): + if feedback.isCanceled(): + break + outFeat.setGeometry(feat.geometry()) attrs = feat.attributes() joinValue1 = str(attrs[joinField1Index]) attrs.extend(cache.get(joinValue1, [])) outFeat.setAttributes(attrs) - writer.addFeature(outFeat, QgsFeatureSink.FastInsert) + sink.addFeature(outFeat, QgsFeatureSink.FastInsert) feedback.setProgress(int(current * total)) - del writer + + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 5a04391fa90..3773bc80779 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -82,6 +82,7 @@ from .HubLines import HubLines from .ImportIntoPostGIS import ImportIntoPostGIS from .ImportIntoSpatialite import ImportIntoSpatialite from .Intersection import Intersection +from .JoinAttributes import JoinAttributes from .LinesIntersection import LinesIntersection from .LinesToPolygons import LinesToPolygons from .MeanCoords import MeanCoords @@ -151,7 +152,6 @@ from .ZonalStatistics import ZonalStatistics # from .StatisticsByCategories import StatisticsByCategories # from .FieldsCalculator import FieldsCalculator # from .FieldPyculator import FieldsPyculator -# from .JoinAttributes import JoinAttributes # from .PointsDisplacement import PointsDisplacement # from .PointsFromPolygons import PointsFromPolygons # from .PointsFromLines import PointsFromLines @@ -189,7 +189,6 @@ class QGISAlgorithmProvider(QgsProcessingProvider): # ExtractByLocation(), # SpatialJoin(), # GeometryConvert(), FieldsCalculator(), - # JoinAttributes(), # FieldsPyculator(), # StatisticsByCategories(), # RasterLayerStatistics(), PointsDisplacement(), @@ -249,6 +248,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): ImportIntoPostGIS(), ImportIntoSpatialite(), Intersection(), + JoinAttributes(), LinesIntersection(), LinesToPolygons(), MeanCoords(), diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 83b6a0b3dd4..64d8729b470 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2265,21 +2265,21 @@ tests: - expected/points_to_path_grouped2.gml type: vector -# - algorithm: qgis:joinattributestable -# name: join the attribute table by common field -# params: -# INPUT_LAYER: -# name: points.gml -# type: vector -# INPUT_LAYER_2: -# name: table.dbf -# type: table -# TABLE_FIELD: id -# TABLE_FIELD_2: ID -# results: -# OUTPUT_LAYER: -# name: expected/join_attribute_table.gml -# type: vector + - algorithm: qgis:joinattributestable + name: join the attribute table by common field + params: + INPUT: + name: points.gml + type: vector + INPUT_2: + name: table.dbf + type: table + FIELD: id + FIELD_2: ID + results: + OUTPUT: + name: expected/join_attribute_table.gml + type: vector - algorithm: qgis:convexhull name: Simple convex hull From b93be39c2469edb776ad898a4693c92a9800ecbe Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 07:01:06 +1000 Subject: [PATCH 20/23] Port Stats by Category to new API Improvements: - keep original field type and name for category field - add unit test --- .../algs/qgis/QGISAlgorithmProvider.py | 5 +- .../algs/qgis/StatisticsByCategories.py | 80 ++++++++++++------- .../testdata/expected/stats_by_category.gfs | 45 +++++++++++ .../testdata/expected/stats_by_category.gml | 42 ++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 13 +++ 5 files changed, 153 insertions(+), 32 deletions(-) create mode 100644 python/plugins/processing/tests/testdata/expected/stats_by_category.gfs create mode 100644 python/plugins/processing/tests/testdata/expected/stats_by_category.gml diff --git a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py index 3773bc80779..c93138350e5 100644 --- a/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py +++ b/python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py @@ -132,6 +132,7 @@ from .SnapGeometries import SnapGeometriesToLayer from .SpatialiteExecuteSQL import SpatialiteExecuteSQL from .SpatialIndex import SpatialIndex from .SplitWithLines import SplitWithLines +from .StatisticsByCategories import StatisticsByCategories from .SumLines import SumLines from .SymmetricalDifference import SymmetricalDifference from .TextToFloat import TextToFloat @@ -149,7 +150,6 @@ from .ZonalStatistics import ZonalStatistics # from .SelectByLocation import SelectByLocation # from .SpatialJoin import SpatialJoin # from .GeometryConvert import GeometryConvert -# from .StatisticsByCategories import StatisticsByCategories # from .FieldsCalculator import FieldsCalculator # from .FieldPyculator import FieldsPyculator # from .PointsDisplacement import PointsDisplacement @@ -190,7 +190,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): # SpatialJoin(), # GeometryConvert(), FieldsCalculator(), # FieldsPyculator(), - # StatisticsByCategories(), + # # RasterLayerStatistics(), PointsDisplacement(), # PointsFromPolygons(), # PointsFromLines(), @@ -298,6 +298,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider): SpatialiteExecuteSQL(), SpatialIndex(), SplitWithLines(), + StatisticsByCategories(), SumLines(), SymmetricalDifference(), TextToFloat(), diff --git a/python/plugins/processing/algs/qgis/StatisticsByCategories.py b/python/plugins/processing/algs/qgis/StatisticsByCategories.py index 1b6e9a5a277..8257d67c53c 100644 --- a/python/plugins/processing/algs/qgis/StatisticsByCategories.py +++ b/python/plugins/processing/algs/qgis/StatisticsByCategories.py @@ -26,19 +26,24 @@ __copyright__ = '(C) 2012, Victor Olaya' __revision__ = '$Format:%H$' -from qgis.core import (QgsApplication, - QgsFeatureSink, +from qgis.core import (QgsProcessingParameterFeatureSource, QgsStatisticalSummary, - QgsProcessingUtils) -from processing.core.outputs import OutputTable + QgsFeatureRequest, + QgsProcessingParameterField, + QgsProcessingParameterFeatureSink, + QgsFields, + QgsField, + QgsWkbTypes, + QgsCoordinateReferenceSystem, + QgsFeature, + QgsFeatureSink) +from qgis.PyQt.QtCore import QVariant from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm -from processing.core.parameters import ParameterVector -from processing.core.parameters import ParameterTableField class StatisticsByCategories(QgisAlgorithm): - INPUT_LAYER = 'INPUT_LAYER' + INPUT = 'INPUT' VALUES_FIELD_NAME = 'VALUES_FIELD_NAME' CATEGORIES_FIELD_NAME = 'CATEGORIES_FIELD_NAME' OUTPUT = 'OUTPUT' @@ -50,16 +55,16 @@ class StatisticsByCategories(QgisAlgorithm): super().__init__() def initAlgorithm(self, config=None): - self.addParameter(ParameterVector(self.INPUT_LAYER, - self.tr('Input vector layer'))) - self.addParameter(ParameterTableField(self.VALUES_FIELD_NAME, - self.tr('Field to calculate statistics on'), - self.INPUT_LAYER, ParameterTableField.DATA_TYPE_NUMBER)) - self.addParameter(ParameterTableField(self.CATEGORIES_FIELD_NAME, - self.tr('Field with categories'), - self.INPUT_LAYER, ParameterTableField.DATA_TYPE_ANY)) + self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, + self.tr('Input vector layer'))) + self.addParameter(QgsProcessingParameterField(self.VALUES_FIELD_NAME, + self.tr('Field to calculate statistics on'), + parentLayerParameterName=self.INPUT, type=QgsProcessingParameterField.Numeric)) + self.addParameter(QgsProcessingParameterField(self.CATEGORIES_FIELD_NAME, + self.tr('Field with categories'), + parentLayerParameterName=self.INPUT, type=QgsProcessingParameterField.Any)) - self.addOutput(OutputTable(self.OUTPUT, self.tr('Statistics by category'))) + self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Statistics by category'))) def name(self): return 'statisticsbycategories' @@ -68,36 +73,51 @@ class StatisticsByCategories(QgisAlgorithm): return self.tr('Statistics by categories') def processAlgorithm(self, parameters, context, feedback): - layer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.INPUT_LAYER), context) - valuesFieldName = self.getParameterValue(self.VALUES_FIELD_NAME) - categoriesFieldName = self.getParameterValue(self.CATEGORIES_FIELD_NAME) + source = self.parameterAsSource(parameters, self.INPUT, context) + value_field_name = self.parameterAsString(parameters, self.VALUES_FIELD_NAME, context) + category_field_name = self.parameterAsString(parameters, self.CATEGORIES_FIELD_NAME, context) - output = self.getOutputFromName(self.OUTPUT) - valuesField = layer.fields().lookupField(valuesFieldName) - categoriesField = layer.fields().lookupField(categoriesFieldName) + value_field_index = source.fields().lookupField(value_field_name) + category_field_index = source.fields().lookupField(category_field_name) - features = QgsProcessingUtils.getFeatures(layer, context) - total = 100.0 / layer.featureCount() if layer.featureCount() else 0 + features = source.getFeatures(QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry)) + total = 100.0 / source.featureCount() if source.featureCount() else 0 values = {} for current, feat in enumerate(features): + if feedback.isCanceled(): + break + feedback.setProgress(int(current * total)) attrs = feat.attributes() try: - value = float(attrs[valuesField]) - cat = str(attrs[categoriesField]) + value = float(attrs[value_field_index]) + cat = attrs[category_field_index] if cat not in values: values[cat] = [] values[cat].append(value) except: pass - fields = ['category', 'min', 'max', 'mean', 'stddev', 'sum', 'count'] - writer = output.getTableWriter(fields) + fields = QgsFields() + fields.append(source.fields().at(category_field_index)) + fields.append(QgsField('min', QVariant.Double)) + fields.append(QgsField('max', QVariant.Double)) + fields.append(QgsField('mean', QVariant.Double)) + fields.append(QgsField('stddev', QVariant.Double)) + fields.append(QgsField('sum', QVariant.Double)) + fields.append(QgsField('count', QVariant.Int)) + + (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, + fields, QgsWkbTypes.NoGeometry, QgsCoordinateReferenceSystem()) + stat = QgsStatisticalSummary(QgsStatisticalSummary.Min | QgsStatisticalSummary.Max | QgsStatisticalSummary.Mean | QgsStatisticalSummary.StDevSample | QgsStatisticalSummary.Sum | QgsStatisticalSummary.Count) for (cat, v) in list(values.items()): stat.calculate(v) - record = [cat, stat.min(), stat.max(), stat.mean(), stat.sampleStDev(), stat.sum(), stat.count()] - writer.addRecord(record) + f = QgsFeature() + f.setAttributes([cat, stat.min(), stat.max(), stat.mean(), stat.sampleStDev(), stat.sum(), stat.count()]) + sink.addFeature(f, QgsFeatureSink.FastInsert) + + return {self.OUTPUT: dest_id} diff --git a/python/plugins/processing/tests/testdata/expected/stats_by_category.gfs b/python/plugins/processing/tests/testdata/expected/stats_by_category.gfs new file mode 100644 index 00000000000..ba663caa0c0 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/stats_by_category.gfs @@ -0,0 +1,45 @@ + + + stats_by_category + stats_by_category + 100 + + 3 + + + id2 + id2 + Integer + + + min + min + Integer + + + max + max + Integer + + + mean + mean + Real + + + stddev + stddev + Real + + + sum + sum + Integer + + + count + count + Integer + + + diff --git a/python/plugins/processing/tests/testdata/expected/stats_by_category.gml b/python/plugins/processing/tests/testdata/expected/stats_by_category.gml new file mode 100644 index 00000000000..4647a986f75 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/stats_by_category.gml @@ -0,0 +1,42 @@ + + + missing + + + + 2 + 1 + 4 + 2.5 + 2.12132034355964 + 5 + 2 + + + + + 1 + 2 + 5 + 3.5 + 2.12132034355964 + 7 + 2 + + + + + 0 + 3 + 9 + 6.6 + 2.30217288664427 + 33 + 5 + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 64d8729b470..215160793c6 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2489,6 +2489,19 @@ tests: name: expected/single_to_multi.gml type: vector + - algorithm: qgis:statisticsbycategories + name: stats by category + params: + VALUES_FIELD_NAME: id + CATEGORIES_FIELD_NAME: id2 + INPUT: + name: points.gml + type: vector + results: + OUTPUT: + name: expected/stats_by_category.gml + type: vector + # - algorithm: qgis:zonalstatistics # name: simple zonal statistics # params: From 6aa672d9e0ee4a4833b5ee41cd122ae9a7cd14c3 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 17:51:34 +1000 Subject: [PATCH 21/23] Fix typo in gridify alg --- python/plugins/processing/algs/qgis/Gridify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/plugins/processing/algs/qgis/Gridify.py b/python/plugins/processing/algs/qgis/Gridify.py index 2605ccd9cd0..06f99983bf8 100644 --- a/python/plugins/processing/algs/qgis/Gridify.py +++ b/python/plugins/processing/algs/qgis/Gridify.py @@ -83,7 +83,7 @@ class Gridify(QgisFeatureBasedAlgorithm): points = self._gridify([geom.asPoint()], self.h_spacing, self.v_spacing) newGeom = QgsGeometry.fromPoint(points[0]) elif geomType == QgsWkbTypes.MultiPoint: - points = self._gridify(geom.aMultiPoint(), self.h_spacing, self.v_spacing) + points = self._gridify(geom.asMultiPoint(), self.h_spacing, self.v_spacing) newGeom = QgsGeometry.fromMultiPoint(points) elif geomType == QgsWkbTypes.LineString: points = self._gridify(geom.asPolyline(), self.h_spacing, self.v_spacing) From d4ad063f45b32191d320620d2bf1c4b54c1a1359 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 22:10:18 +1000 Subject: [PATCH 22/23] Allow specifying a 'primary key' field when comparing layers for processing tests Some algorithms will return results in different orders, e.g. due to the use of dicts or other methods which do not guarantee a fixed return order. Using a primary key to do the feature match allows us to flexibly handle these situations and provide tests for these algorithms. --- .../processing/tests/AlgorithmsTestBase.py | 5 +++-- .../tests/testdata/qgis_algorithm_tests.yaml | 4 ++++ python/testing/__init__.py | 18 ++++++++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/python/plugins/processing/tests/AlgorithmsTestBase.py b/python/plugins/processing/tests/AlgorithmsTestBase.py index 2ae126ccbf6..2e19146a7b7 100644 --- a/python/plugins/processing/tests/AlgorithmsTestBase.py +++ b/python/plugins/processing/tests/AlgorithmsTestBase.py @@ -287,13 +287,14 @@ class AlgorithmsTest(object): self.assertTrue(result_lyr, results[id]) compare = expected_result.get('compare', {}) + pk = expected_result.get('pk', None) if len(expected_lyrs) == 1: - self.assertLayersEqual(expected_lyrs[0], result_lyr, compare=compare) + self.assertLayersEqual(expected_lyrs[0], result_lyr, compare=compare, pk=pk) else: res = False for l in expected_lyrs: - if self.checkLayersEqual(l, result_lyr, compare=compare): + if self.checkLayersEqual(l, result_lyr, compare=compare, pk=pk): res = True break self.assertTrue(res, 'Could not find matching layer in expected results') diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 215160793c6..79378a18d39 100644 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -2501,6 +2501,10 @@ tests: OUTPUT: name: expected/stats_by_category.gml type: vector + pk: id2 + compare: + fields: + fid: skip # - algorithm: qgis:zonalstatistics # name: simple zonal statistics diff --git a/python/testing/__init__.py b/python/testing/__init__.py index a495a335681..fc3eafe1fa0 100644 --- a/python/testing/__init__.py +++ b/python/testing/__init__.py @@ -54,6 +54,10 @@ class TestCase(_TestCase): :keyword compare: A map of comparison options. e.g. { fields: { a: skip, b: { precision: 2 }, geometry: { precision: 5 } } { fields: { __all__: cast( str ) } } + :keyword pk: "Primary key" type field - used to match features + from the expected table to their corresponding features in the result table. If not specified + features are compared by their order in the layer (e.g. first feature compared with first feature, + etc) """ self.checkLayersEqual(layer_expected, layer_result, True, **kwargs) @@ -69,6 +73,10 @@ class TestCase(_TestCase): :keyword compare: A map of comparison options. e.g. { fields: { a: skip, b: { precision: 2 }, geometry: { precision: 5 } } { fields: { __all__: cast( str ) } } + :keyword pk: "Primary key" type field - used to match features + from the expected table to their corresponding features in the result table. If not specified + features are compared by their order in the layer (e.g. first feature compared with first feature, + etc) """ try: @@ -98,8 +106,14 @@ class TestCase(_TestCase): except KeyError: precision = 14 - expected_features = sorted(layer_expected.getFeatures(request), key=lambda f: f.id()) - result_features = sorted(layer_result.getFeatures(request), key=lambda f: f.id()) + def sort_by_pk_or_fid(f): + if 'pk' in kwargs and kwargs['pk'] is not None: + return f[kwargs['pk']] + else: + return f.id() + + expected_features = sorted(layer_expected.getFeatures(request), key=sort_by_pk_or_fid) + result_features = sorted(layer_result.getFeatures(request), key=sort_by_pk_or_fid) for feats in zip(expected_features, result_features): if feats[0].hasGeometry(): From 470afbebbe7063828cb0e2222c7e6ba4c16ae692 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Sat, 5 Aug 2017 23:19:03 +1000 Subject: [PATCH 23/23] Use correct file filters for processing vector/raster input selectors --- .../processing/gui/ParameterGuiUtils.py | 21 ++++++++++++------- python/plugins/processing/gui/wrappers.py | 3 +-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/python/plugins/processing/gui/ParameterGuiUtils.py b/python/plugins/processing/gui/ParameterGuiUtils.py index 3694daf8e8a..2931926e7e9 100644 --- a/python/plugins/processing/gui/ParameterGuiUtils.py +++ b/python/plugins/processing/gui/ParameterGuiUtils.py @@ -27,7 +27,7 @@ __copyright__ = '(C) 2017, Nyall Dawson' __revision__ = '$Format:%H$' from qgis.core import (QgsProcessing, - QgsProcessingParameterDefinition, + QgsProviderRegistry, QgsProcessingFeatureSourceDefinition, QgsVectorFileWriter) from qgis.PyQt.QtCore import QCoreApplication @@ -55,22 +55,29 @@ def getFileFilter(param): exts = QgsVectorFileWriter.supportedFormatExtensions() for i in range(len(exts)): exts[i] = tr('{0} files (*.{1})', 'QgsProcessingParameterMultipleLayers').format(exts[i].upper(), exts[i].lower()) - return ';;'.join(exts) - elif param.type() in ('raster', 'rasterDestination'): + return tr('All files (*.*)') + ';;' + ';;'.join(exts) + elif param.type() == 'raster': + return QgsProviderRegistry.instance().fileRasterFilters() + elif param.type() == 'rasterDestination': exts = dataobjects.getSupportedOutputRasterLayerExtensions() for i in range(len(exts)): exts[i] = tr('{0} files (*.{1})', 'QgsProcessingParameterRasterDestination').format(exts[i].upper(), exts[i].lower()) - return ';;'.join(exts) + return tr('All files (*.*)') + ';;' + ';;'.join(exts) elif param.type() == 'table': exts = ['csv', 'dbf'] for i in range(len(exts)): exts[i] = tr('{0} files (*.{1})', 'ParameterTable').format(exts[i].upper(), exts[i].lower()) - return ';;'.join(exts) + return tr('All files (*.*)') + ';;' + ';;'.join(exts) elif param.type() == 'sink': exts = QgsVectorFileWriter.supportedFormatExtensions() for i in range(len(exts)): exts[i] = tr('{0} files (*.{1})', 'ParameterVector').format(exts[i].upper(), exts[i].lower()) - return ';;'.join(exts) + return tr('All files (*.*)') + ';;' + ';;'.join(exts) + elif param.type() == 'source': + return QgsProviderRegistry.instance().fileVectorFilters() + elif param.type() == 'vector': + return QgsProviderRegistry.instance().fileVectorFilters() elif param.type() == 'fileOut': - return param.fileFilter() + return tr('All files (*.*)') + ';;' + param.fileFilter() + return '' diff --git a/python/plugins/processing/gui/wrappers.py b/python/plugins/processing/gui/wrappers.py index 2908bae1a3d..7635ecc300d 100644 --- a/python/plugins/processing/gui/wrappers.py +++ b/python/plugins/processing/gui/wrappers.py @@ -218,8 +218,7 @@ class WidgetWrapper(QObject): path = '' filename, selected_filter = QFileDialog.getOpenFileName(self.widget, self.tr('Select file'), - path, self.tr( - 'All files (*.*);;') + getFileFilter(self.param)) + path, getFileFilter(self.param)) if filename: settings.setValue('/Processing/LastInputPath', os.path.dirname(str(filename)))