From 5b1da988baf93c988108f687abc311adb35aaac9 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 14 Dec 2017 12:34:59 +1000 Subject: [PATCH] [processing] List unique values improvements - allow running on non-spatial tables - allow choice of more than one field --- python/plugins/processing/algs/help/qgis.yaml | 2 +- .../processing/algs/qgis/UniqueValues.py | 53 ++++++++++++++----- .../tests/testdata/expected/unique_values.gfs | 4 +- .../tests/testdata/expected/unique_values.gml | 6 +-- .../expected/unique_values_multiple.gml | 45 ++++++++++++++++ .../expected/unique_values_multiple.xsd | 36 +++++++++++++ .../tests/testdata/qgis_algorithm_tests.yaml | 24 ++++++++- 7 files changed, 150 insertions(+), 20 deletions(-) create mode 100755 python/plugins/processing/tests/testdata/expected/unique_values_multiple.gml create mode 100755 python/plugins/processing/tests/testdata/expected/unique_values_multiple.xsd diff --git a/python/plugins/processing/algs/help/qgis.yaml b/python/plugins/processing/algs/help/qgis.yaml index 0a5023ee5e0..1863dd80403 100644 --- a/python/plugins/processing/algs/help/qgis.yaml +++ b/python/plugins/processing/algs/help/qgis.yaml @@ -267,7 +267,7 @@ qgis:linestopolygons: > The attribute table of the output layer is the same as the one from of the input line layer. qgis:listuniquevalues: > - This algorithm generates a report with information about the categories found in a given attribute of a vector layer. + This algorithm generates a report with information about the unique values found in a given attribute (or attributes) of a vector layer. qgis:meanandstandarddeviationplot: diff --git a/python/plugins/processing/algs/qgis/UniqueValues.py b/python/plugins/processing/algs/qgis/UniqueValues.py index 2096ed29a40..7b82f55ad7c 100644 --- a/python/plugins/processing/algs/qgis/UniqueValues.py +++ b/python/plugins/processing/algs/qgis/UniqueValues.py @@ -34,7 +34,9 @@ from qgis.core import (QgsCoordinateReferenceSystem, QgsWkbTypes, QgsFeature, QgsFeatureSink, + QgsFeatureRequest, QgsFields, + QgsProcessing, QgsProcessingParameterField, QgsProcessingParameterFeatureSource, QgsProcessingParameterFeatureSink, @@ -51,7 +53,7 @@ pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0] class UniqueValues(QgisAlgorithm): INPUT = 'INPUT' - FIELD_NAME = 'FIELD_NAME' + FIELDS = 'FIELDS' TOTAL_VALUES = 'TOTAL_VALUES' UNIQUE_VALUES = 'UNIQUE_VALUES' OUTPUT = 'OUTPUT' @@ -71,10 +73,10 @@ class UniqueValues(QgisAlgorithm): def initAlgorithm(self, config=None): self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, - self.tr('Input layer'))) - self.addParameter(QgsProcessingParameterField(self.FIELD_NAME, - self.tr('Target field'), - parentLayerParameterName=self.INPUT, type=QgsProcessingParameterField.Any)) + self.tr('Input layer'), types=[QgsProcessing.TypeVector])) + self.addParameter(QgsProcessingParameterField(self.FIELDS, + self.tr('Target field(s)'), + parentLayerParameterName=self.INPUT, type=QgsProcessingParameterField.Any, allowMultiple=True)) self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT, self.tr('Unique values'), optional=True, defaultValue='')) @@ -91,23 +93,48 @@ class UniqueValues(QgisAlgorithm): def processAlgorithm(self, parameters, context, feedback): source = self.parameterAsSource(parameters, self.INPUT, context) - field_name = self.parameterAsString(parameters, self.FIELD_NAME, context) - values = source.uniqueValues(source.fields().lookupField(field_name)) + field_names = self.parameterAsFields(parameters, self.FIELDS, context) fields = QgsFields() - field = source.fields()[source.fields().lookupField(field_name)] - field.setName('VALUES') - fields.append(field) + field_indices = [] + for field_name in field_names: + field_index = source.fields().lookupField(field_name) + if field_index < 0: + feedback.reportError(self.tr('Invalid field name {}').format(field_name)) + continue + field = source.fields()[field_index] + fields.append(field) + field_indices.append(field_index) (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, fields, QgsWkbTypes.NoGeometry, QgsCoordinateReferenceSystem()) + results = {} + values = set() + if len(field_indices) == 1: + # one field, can use provider optimised method + values = tuple([v] for v in source.uniqueValues(field_indices[0])) + else: + # have to scan whole table + # TODO - add this support to QgsVectorDataProvider so we can run it on + # the backend + request = QgsFeatureRequest().setFlags(QgsFeatureRequest.NoGeometry) + request.setSubsetOfAttributes(field_indices) + total = 100.0 / source.featureCount() if source.featureCount() else 0 + for current, f in enumerate(source.getFeatures(request)): + if feedback.isCanceled(): + break + + value = tuple(f.attribute(i) for i in field_indices) + values.add(value) + feedback.setProgress(int(current * total)) + if sink: for value in values: if feedback.isCanceled(): break f = QgsFeature() - f.setAttributes([value]) + f.setAttributes([attr for attr in value]) sink.addFeature(f, QgsFeatureSink.FastInsert) results[self.OUTPUT] = dest_id @@ -117,7 +144,7 @@ class UniqueValues(QgisAlgorithm): results[self.OUTPUT_HTML_FILE] = output_file results[self.TOTAL_VALUES] = len(values) - results[self.UNIQUE_VALUES] = ';'.join([str(v) for v in + results[self.UNIQUE_VALUES] = ';'.join([','.join([str(attr) for attr in v]) for v in values]) return results @@ -130,5 +157,5 @@ class UniqueValues(QgisAlgorithm): f.write(self.tr('

Unique values:

')) f.write('') diff --git a/python/plugins/processing/tests/testdata/expected/unique_values.gfs b/python/plugins/processing/tests/testdata/expected/unique_values.gfs index 038b13d0664..3cb46a02336 100644 --- a/python/plugins/processing/tests/testdata/expected/unique_values.gfs +++ b/python/plugins/processing/tests/testdata/expected/unique_values.gfs @@ -7,8 +7,8 @@ 3 - VALUES - VALUES + id2 + id2 Integer diff --git a/python/plugins/processing/tests/testdata/expected/unique_values.gml b/python/plugins/processing/tests/testdata/expected/unique_values.gml index 8045252b037..02716775717 100644 --- a/python/plugins/processing/tests/testdata/expected/unique_values.gml +++ b/python/plugins/processing/tests/testdata/expected/unique_values.gml @@ -8,17 +8,17 @@ - 0 + 0 - 1 + 1 - 2 + 2 diff --git a/python/plugins/processing/tests/testdata/expected/unique_values_multiple.gml b/python/plugins/processing/tests/testdata/expected/unique_values_multiple.gml new file mode 100755 index 00000000000..05504a1c784 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/unique_values_multiple.gml @@ -0,0 +1,45 @@ + + + missing + + + + aa + 1 + + + + + bb + 2 + + + + + cc + + + + + + + 120 + + + + + bb + 1 + + + + + dd + + + + diff --git a/python/plugins/processing/tests/testdata/expected/unique_values_multiple.xsd b/python/plugins/processing/tests/testdata/expected/unique_values_multiple.xsd new file mode 100755 index 00000000000..d8c85284d22 --- /dev/null +++ b/python/plugins/processing/tests/testdata/expected/unique_values_multiple.xsd @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml index 64ceba7b2ad..731af2796da 100755 --- a/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml +++ b/python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml @@ -408,11 +408,33 @@ tests: INPUT: name: points.gml type: vector - FIELD_NAME: id2 + FIELDS: id2 results: OUTPUT: name: expected/unique_values.gml type: vector + pk: + - id2 + + - algorithm: qgis:listuniquevalues + name: Unique values (multiple fields) + params: + FIELDS: + - name + - intval + INPUT: + name: dissolve_polys.gml + type: vector + results: + OUTPUT: + name: expected/unique_values_multiple.gml + type: vector + compare: + fields: + fid: skip + pk: + - name + - intval - algorithm: native:addautoincrementalfield name: Add autoincremental field