Resurrect Python Field Calculator algorithm, add test

python/plugins/processing/algs/qgis/FieldPyculator.py

@@ -29,30 +29,27 @@ __revision__ = '$Format:%H$'
 import sys
 
 from qgis.PyQt.QtCore import QVariant
-from qgis.core import (QgsFeature,
+from qgis.core import (QgsProcessingException,
                        QgsField,
                        QgsFeatureSink,
-                       QgsApplication,
+                       QgsProcessingParameterFeatureSource,
-                       QgsProcessingUtils)
+                       QgsProcessingParameterString,
+                       QgsProcessingParameterEnum,
+                       QgsProcessingParameterNumber,
+                       QgsProcessingParameterFeatureSink)
 from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm
-from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException
-from processing.core.parameters import ParameterVector
-from processing.core.parameters import ParameterString
-from processing.core.parameters import ParameterNumber
-from processing.core.parameters import ParameterSelection
-from processing.core.outputs import OutputVector
 
 
 class FieldsPyculator(QgisAlgorithm):
 
-    INPUT_LAYER = 'INPUT_LAYER'
+    INPUT = 'INPUT'
     FIELD_NAME = 'FIELD_NAME'
     FIELD_TYPE = 'FIELD_TYPE'
     FIELD_LENGTH = 'FIELD_LENGTH'
     FIELD_PRECISION = 'FIELD_PRECISION'
     GLOBAL = 'GLOBAL'
     FORMULA = 'FORMULA'
-    OUTPUT_LAYER = 'OUTPUT_LAYER'
+    OUTPUT = 'OUTPUT'
     RESULT_VAR_NAME = 'value'
 
     TYPES = [QVariant.Int, QVariant.Double, QVariant.String]
@@ -68,21 +65,21 @@ class FieldsPyculator(QgisAlgorithm):
                            self.tr('Float'),
                            self.tr('String')]
 
-        self.addParameter(ParameterVector(self.INPUT_LAYER,
+        self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT, self.tr('Input layer')))
-                                          self.tr('Input layer')))
+        self.addParameter(QgsProcessingParameterString(self.FIELD_NAME,
-        self.addParameter(ParameterString(self.FIELD_NAME,
+                                                       self.tr('Result field name'), defaultValue='NewField'))
-                                          self.tr('Result field name'), 'NewField'))
+        self.addParameter(QgsProcessingParameterEnum(self.FIELD_TYPE,
-        self.addParameter(ParameterSelection(self.FIELD_TYPE,
+                                                     self.tr('Field type'), options=self.type_names))
-                                             self.tr('Field type'), self.type_names))
+        self.addParameter(QgsProcessingParameterNumber(self.FIELD_LENGTH,
-        self.addParameter(ParameterNumber(self.FIELD_LENGTH,
+                                                       self.tr('Field length'), minValue=1, maxValue=255, defaultValue=10))
-                                          self.tr('Field length'), 1, 255, 10))
+        self.addParameter(QgsProcessingParameterNumber(self.FIELD_PRECISION,
-        self.addParameter(ParameterNumber(self.FIELD_PRECISION,
+                                                       self.tr('Field precision'), minValue=0, maxValue=15, defaultValue=3))
-                                          self.tr('Field precision'), 0, 10, 0))
+        self.addParameter(QgsProcessingParameterString(self.GLOBAL,
-        self.addParameter(ParameterString(self.GLOBAL,
+                                                       self.tr('Global expression'), multiLine=True, optional=True))
-                                          self.tr('Global expression'), multiline=True, optional=True))
+        self.addParameter(QgsProcessingParameterString(self.FORMULA,
-        self.addParameter(ParameterString(self.FORMULA,
+                                                       self.tr('Formula'), defaultValue='value = ', multiLine=True))
-                                          self.tr('Formula'), 'value = ', multiline=True))
+        self.addParameter(QgsProcessingParameterFeatureSink(self.OUTPUT,
-        self.addOutput(OutputVector(self.OUTPUT_LAYER, self.tr('Calculated')))
+                                                            self.tr('Calculated')))
 
     def name(self):
         return 'advancedpythonfieldcalculator'
@@ -91,33 +88,33 @@ class FieldsPyculator(QgisAlgorithm):
         return self.tr('Advanced Python field calculator')
 
     def processAlgorithm(self, parameters, context, feedback):
-        fieldName = self.getParameterValue(self.FIELD_NAME)
+        source = self.parameterAsSource(parameters, self.INPUT, context)
-        fieldType = self.getParameterValue(self.FIELD_TYPE)
+        field_name = self.parameterAsString(parameters, self.FIELD_NAME, context)
-        fieldLength = self.getParameterValue(self.FIELD_LENGTH)
+        field_type = self.TYPES[self.parameterAsEnum(parameters, self.FIELD_TYPE, context)]
-        fieldPrecision = self.getParameterValue(self.FIELD_PRECISION)
+        width = self.parameterAsInt(parameters, self.FIELD_LENGTH, context)
-        code = self.getParameterValue(self.FORMULA)
+        precision = self.parameterAsInt(parameters, self.FIELD_PRECISION, context)
-        globalExpression = self.getParameterValue(self.GLOBAL)
+        code = self.parameterAsString(parameters, self.FORMULA, context)
-        output = self.getOutputFromName(self.OUTPUT_LAYER)
+        globalExpression = self.parameterAsString(parameters, self.GLOBAL, context)
 
-        layer = QgsProcessingUtils.mapLayerFromString(self.getParameterValue(self.INPUT_LAYER), context)
+        fields = source.fields()
-        fields = layer.fields()
+        fields.append(QgsField(field_name, self.TYPES[field_type], '',
-        fields.append(QgsField(fieldName, self.TYPES[fieldType], '',
+                               width, precision))
-                               fieldLength, fieldPrecision))
-        writer = output.getVectorWriter(fields, layer.wkbType(), layer.crs(), context)
-        outFeat = QgsFeature()
         new_ns = {}
 
+        (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context,
+                                               fields, source.wkbType(), source.sourceCrs())
+
         # Run global code
         if globalExpression.strip() != '':
             try:
                 bytecode = compile(globalExpression, '<string>', 'exec')
                 exec(bytecode, new_ns)
             except:
-                raise GeoAlgorithmExecutionException(
+                raise QgsProcessingException(
                     self.tr("FieldPyculator code execute error.Global code block can't be executed!\n{0}\n{1}").format(str(sys.exc_info()[0].__name__), str(sys.exc_info()[1])))
 
         # Replace all fields tags
-        fields = layer.fields()
+        fields = source.fields()
         num = 0
         for field in fields:
             field_name = str(field.name())
@@ -136,13 +133,17 @@ class FieldsPyculator(QgisAlgorithm):
         try:
             bytecode = compile(code, '<string>', 'exec')
         except:
-            raise GeoAlgorithmExecutionException(
+            raise QgsProcessingException(
                 self.tr("FieldPyculator code execute error. Field code block can't be executed!\n{0}\n{1}").format(str(sys.exc_info()[0].__name__), str(sys.exc_info()[1])))
 
         # Run
-        features = QgsProcessingUtils.getFeatures(layer, context)
+        features = source.getFeatures()
-        total = 100.0 / layer.featureCount() if layer.featureCount() else 0
+        total = 100.0 / source.featureCount() if source.featureCount() else 0
+
         for current, feat in enumerate(features):
+            if feedback.isCanceled():
+                break
+
             feedback.setProgress(int(current * total))
             attrs = feat.attributes()
             feat_id = feat.id()
@@ -168,18 +169,17 @@ class FieldsPyculator(QgisAlgorithm):
 
             # Check result
             if self.RESULT_VAR_NAME not in new_ns:
-                raise GeoAlgorithmExecutionException(
+                raise QgsProcessingException(
                     self.tr("FieldPyculator code execute error\n"
                             "Field code block does not return '{0}' variable! "
                             "Please declare this variable in your code!").format(self.RESULT_VAR_NAME))
 
             # Write feature
-            outFeat.setGeometry(feat.geometry())
             attrs.append(new_ns[self.RESULT_VAR_NAME])
-            outFeat.setAttributes(attrs)
+            feat.setAttributes(attrs)
-            writer.addFeature(outFeat, QgsFeatureSink.FastInsert)
+            sink.addFeature(feat, QgsFeatureSink.FastInsert)
 
-        del writer
+        return {self.OUTPUT: dest_id}
 
     def checkParameterValues(self, parameters, context):
         # TODO check that formula is correct and fields exist

python/plugins/processing/algs/qgis/QGISAlgorithmProvider.py

@@ -70,6 +70,7 @@ from .ExtendLines import ExtendLines
 from .ExtentFromLayer import ExtentFromLayer
 from .ExtractNodes import ExtractNodes
 from .ExtractSpecificNodes import ExtractSpecificNodes
+from .FieldPyculator import FieldsPyculator
 from .FieldsCalculator import FieldsCalculator
 from .FieldsMapper import FieldsMapper
 from .FixedDistanceBuffer import FixedDistanceBuffer
@@ -167,7 +168,6 @@ from .ZonalStatistics import ZonalStatistics
 # from .SelectByLocation import SelectByLocation
 # from .SpatialJoin import SpatialJoin
 # from .GeometryConvert import GeometryConvert
-# from .FieldPyculator import FieldsPyculator
 # from .SelectByAttributeSum import SelectByAttributeSum
 # from .DefineProjection import DefineProjection
 # from .RasterCalculator import RasterCalculator
@@ -191,8 +191,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider):
         #         ExtractByLocation(),
         #         SpatialJoin(),
         #         GeometryConvert(),
-        #         FieldsPyculator(),
+        #         SelectByAttributeSum()
-        #         FieldsMapper(), SelectByAttributeSum()
         #         DefineProjection(),
         #         RasterCalculator(),
         #          ExecuteSQL(), FindProjection(),
@@ -229,6 +228,7 @@ class QGISAlgorithmProvider(QgsProcessingProvider):
                 ExtractSpecificNodes(),
                 FieldsCalculator(),
                 FieldsMapper(),
+                FieldsPyculator(),
                 FixedDistanceBuffer(),
                 FixGeometry(),
                 GeometryByExpression(),

python/plugins/processing/tests/testdata/expected/pycalculator_points.gfs

@@ -0,0 +1,31 @@
+<GMLFeatureClassList>
+  <GMLFeatureClass>
+    <Name>pycalculator_points</Name>
+    <ElementPath>pycalculator_points</ElementPath>
+    <!--POINT-->
+    <GeometryType>1</GeometryType>
+    <SRSName>EPSG:4326</SRSName>
+    <DatasetSpecificInfo>
+      <FeatureCount>9</FeatureCount>
+      <ExtentXMin>0.00000</ExtentXMin>
+      <ExtentXMax>8.00000</ExtentXMax>
+      <ExtentYMin>-5.00000</ExtentYMin>
+      <ExtentYMax>3.00000</ExtentYMax>
+    </DatasetSpecificInfo>
+    <PropertyDefn>
+      <Name>id</Name>
+      <ElementPath>id</ElementPath>
+      <Type>Integer</Type>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>id2</Name>
+      <ElementPath>id2</ElementPath>
+      <Type>Integer</Type>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>new_field</Name>
+      <ElementPath>new_field</ElementPath>
+      <Type>Integer</Type>
+    </PropertyDefn>
+  </GMLFeatureClass>
+</GMLFeatureClassList>

python/plugins/processing/tests/testdata/expected/pycalculator_points.gml

@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<ogr:FeatureCollection
+     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xsi:schemaLocation=""
+     xmlns:ogr="http://ogr.maptools.org/"
+     xmlns:gml="http://www.opengis.net/gml">
+  <gml:boundedBy>
+    <gml:Box>
+      <gml:coord><gml:X>0</gml:X><gml:Y>-5</gml:Y></gml:coord>
+      <gml:coord><gml:X>8</gml:X><gml:Y>3</gml:Y></gml:coord>
+    </gml:Box>
+  </gml:boundedBy>
+                                                                                                                                                               
+  <gml:featureMember>
+    <ogr:pycalculator_points fid="points.0">
+      <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>1,1</gml:coordinates></gml:Point></ogr:geometryProperty>
+      <ogr:id>1</ogr:id>
+      <ogr:id2>2</ogr:id2>
+      <ogr:new_field>4</ogr:new_field>
+    </ogr:pycalculator_points>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:pycalculator_points fid="points.1">
+      <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>3,3</gml:coordinates></gml:Point></ogr:geometryProperty>
+      <ogr:id>2</ogr:id>
+      <ogr:id2>1</ogr:id2>
+      <ogr:new_field>2</ogr:new_field>
+    </ogr:pycalculator_points>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:pycalculator_points fid="points.2">
+      <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>2,2</gml:coordinates></gml:Point></ogr:geometryProperty>
+      <ogr:id>3</ogr:id>
+      <ogr:id2>0</ogr:id2>
+      <ogr:new_field>0</ogr:new_field>
+    </ogr:pycalculator_points>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:pycalculator_points fid="points.3">
+      <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>5,2</gml:coordinates></gml:Point></ogr:geometryProperty>
+      <ogr:id>4</ogr:id>
+      <ogr:id2>2</ogr:id2>
+      <ogr:new_field>4</ogr:new_field>
+    </ogr:pycalculator_points>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:pycalculator_points fid="points.4">
+      <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>4,1</gml:coordinates></gml:Point></ogr:geometryProperty>
+      <ogr:id>5</ogr:id>
+      <ogr:id2>1</ogr:id2>
+      <ogr:new_field>2</ogr:new_field>
+    </ogr:pycalculator_points>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:pycalculator_points fid="points.5">
+      <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>0,-5</gml:coordinates></gml:Point></ogr:geometryProperty>
+      <ogr:id>6</ogr:id>
+      <ogr:id2>0</ogr:id2>
+      <ogr:new_field>0</ogr:new_field>
+    </ogr:pycalculator_points>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:pycalculator_points fid="points.6">
+      <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>8,-1</gml:coordinates></gml:Point></ogr:geometryProperty>
+      <ogr:id>7</ogr:id>
+      <ogr:id2>0</ogr:id2>
+      <ogr:new_field>0</ogr:new_field>
+    </ogr:pycalculator_points>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:pycalculator_points fid="points.7">
+      <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>7,-1</gml:coordinates></gml:Point></ogr:geometryProperty>
+      <ogr:id>8</ogr:id>
+      <ogr:id2>0</ogr:id2>
+      <ogr:new_field>0</ogr:new_field>
+    </ogr:pycalculator_points>
+  </gml:featureMember>
+  <gml:featureMember>
+    <ogr:pycalculator_points fid="points.8">
+      <ogr:geometryProperty><gml:Point srsName="EPSG:4326"><gml:coordinates>0,-1</gml:coordinates></gml:Point></ogr:geometryProperty>
+      <ogr:id>9</ogr:id>
+      <ogr:id2>0</ogr:id2>
+      <ogr:new_field>0</ogr:new_field>
+    </ogr:pycalculator_points>
+  </gml:featureMember>
+</ogr:FeatureCollection>

python/plugins/processing/tests/testdata/qgis_algorithm_tests.yaml

@@ -3193,3 +3193,20 @@ tests:
       OUTPUT:
         name: expected/field_calculator_points.gml
         type: vector
+
+  - algorithm: qgis:advancedpythonfieldcalculator
+    name: Test advanced python calculator
+    params:
+      FIELD_LENGTH: 10
+      FIELD_NAME: new_field
+      FIELD_PRECISION: 3
+      FIELD_TYPE: 0
+      FORMULA: value = __attr[2]*2
+      GLOBAL: ''
+      INPUT:
+        name: points.gml
+        type: vector
+    results:
+      OUTPUT:
+        name: expected/pycalculator_points.gml
+        type: vector