[processing] support for expressions in numerical values in modeler

includes cleanup of modeler, to adapt to latest changes in parameters architecture
This commit is contained in:
volaya 2016-09-12 06:17:23 +02:00
parent fe5d0166cc
commit e08fdaa444
10 changed files with 127 additions and 380 deletions

View File

@ -54,7 +54,6 @@ from processing.tools import dataobjects
from processing.core.alglist import algList
from processing.modeler.ModelerAlgorithmProvider import ModelerAlgorithmProvider
from processing.modeler.ModelerOnlyAlgorithmProvider import ModelerOnlyAlgorithmProvider
from processing.algs.qgis.QGISAlgorithmProvider import QGISAlgorithmProvider
from processing.algs.grass.GrassAlgorithmProvider import GrassAlgorithmProvider
from processing.algs.grass7.Grass7AlgorithmProvider import Grass7AlgorithmProvider

View File

@ -32,6 +32,7 @@ import sys
import os
from inspect import isclass
from copy import deepcopy
import numbers
from qgis.utils import iface
from qgis.PyQt.QtCore import QCoreApplication
@ -40,6 +41,7 @@ from qgis.core import (QgsRasterLayer, QgsVectorLayer, QgsMapLayer, QgsCoordinat
from processing.tools.vector import resolveFieldIndex, features
from processing.tools import dataobjects
from processing.core.outputs import OutputNumber
def parseBool(s):
if s is None or s == str(None).lower():
@ -192,6 +194,9 @@ class Parameter:
def evaluate(self, alg):
pass
def evaluateForModeler(self, value, model):
return value
class ParameterBoolean(Parameter):
@ -810,6 +815,7 @@ class ParameterNumber(Parameter):
self.value = float(v)
return True
except:
raise
return False
else:
try:
@ -826,6 +832,7 @@ class ParameterNumber(Parameter):
self.value = value
return True
except:
raise
return False
def getAsScriptCode(self):
@ -843,8 +850,8 @@ class ParameterNumber(Parameter):
default = definition.strip()[len('number') + 1:] or None
return ParameterNumber(name, descName, default=default, optional=isOptional)
def _evaluate(self):
exp = QgsExpression(self.value)
def _evaluate(self, value):
exp = QgsExpression(value)
if exp.hasParserError():
raise ValueError(self.tr("Error in parameter expression: ") + exp.parserErrorString())
result = exp.evaluate(_expressionContext())
@ -853,7 +860,26 @@ class ParameterNumber(Parameter):
return result
def evaluate(self, alg):
self.value = self._evaluate(self.value)
if isinstance(self.value, basestring):
self.value = self._evaluate(self.value)
def evaluateForModeler(self, value, model):
if isinstance(value, numbers.Number):
return value
variables = {}
for param in model.parameters:
if isinstance(param, ParameterNumber):
variables["@" + param.name] = param.value
for alg in model.algs.values():
for out in alg.algorithm.outputs:
if isinstance(out, OutputNumber):
variables["@%s_%s" % (alg.name, out.name)] = out.value
for k,v in variables.iteritems():
print k,v
value = value.replace(k,unicode(v))
print value
return value
def expressionContext(self):
return _expressionContext()
@ -1427,9 +1453,12 @@ def getParameterFromString(s):
isAdvanced = True
tokens = s.split("|")
params = [t if unicode(t) != unicode(None) else None for t in tokens[1:]]
clazz = getattr(sys.modules[__name__], tokens[0])
param = clazz(*params)
param.isAdvanced = isAdvanced
try:
clazz = getattr(sys.modules[__name__], tokens[0])
param = clazz(*params)
param.isAdvanced = isAdvanced
except:
return None
else: # try script syntax
for paramClass in paramClasses:
try:

View File

@ -35,9 +35,13 @@ from qgis.PyQt.QtWidgets import QDialog
from qgis.core import (QgsDataSourceUri,
QgsCredentials,
QgsExpression,
QgsRasterLayer)
QgsRasterLayer,
QgsExpressionContextScope)
from qgis.gui import QgsEncodingFileDialog, QgsExpressionBuilderDialog
from qgis.utils import iface
from processing.core.parameters import ParameterNumber
from processing.core.outputs import OutputNumber
from processing.modeler.ModelerAlgorithm import ValueFromInput, ValueFromOutput, CompoundValue
pluginPath = os.path.split(os.path.dirname(__file__))[0]
WIDGET, BASE = uic.loadUiType(
@ -48,18 +52,27 @@ class NumberInputPanel(BASE, WIDGET):
hasChanged = pyqtSignal()
def __init__(self, param):
def __init__(self, param, modelParametersDialog=None):
super(NumberInputPanel, self).__init__(None)
self.setupUi(self)
self.param = param
self.text = param.default
self.modelParametersDialog = modelParametersDialog
if param.default:
self.setValue(param.default)
self.btnSelect.clicked.connect(self.showExpressionsBuilder)
self.leText.textChanged.connect(lambda: self.hasChanged.emit())
def showExpressionsBuilder(self):
context = self.param.expressionContext()
if self.modelParametersDialog is not None:
context.popScope()
values = self.modelParametersDialog.getAvailableValuesOfType(ParameterNumber, OutputNumber)
modelerScope = QgsExpressionContextScope()
for value in values:
name = value.name if isinstance(value, ValueFromInput) else "%s_%s" % (value.alg, value.output)
modelerScope.setVariable(name, 1)
context.appendScope(modelerScope)
dlg = QgsExpressionBuilderDialog(None, self.leText.text(), self, 'generic', context)
dlg.setWindowTitle(self.tr('Expression based input'))
if dlg.exec_() == QDialog.Accepted:
@ -69,7 +82,23 @@ class NumberInputPanel(BASE, WIDGET):
def getValue(self):
return self.leText.text()
if self.modelParametersDialog:
value = self.leText.text()
values = []
for param in self.modelParametersDialog.model.parameters:
if isinstance(param, ParameterNumber):
if "@" + param.name in value:
values.append(ValueFromInput(param.name))
for alg in self.modelParametersDialog.model.algs.values():
for out in alg.algorithm.outputs:
if isinstance(out, OutputNumber) and "@%s_%s" % (alg.name, out.name) in value:
values.append(ValueFromOutput(alg.name, out.name))
if values:
return CompoundValue(values, value)
else:
return value
else:
return self.leText.text()
def setValue(self, value):
self.leText.setText(unicode(value))

View File

@ -455,39 +455,15 @@ class NumberWidgetWrapper(WidgetWrapper):
def createWidget(self):
if self.dialogType in (DIALOG_STANDARD, DIALOG_BATCH):
return NumberInputPanel(self.param)
return NumberInputPanel(self.param, None)
else:
widget = QComboBox()
widget.setEditable(True)
files = self.dialog.getAvailableValuesOfType(ParameterNumber, OutputNumber)
for f in files:
widget.addItem(self.dialog.resolveValueDescription(f), f)
return widget
return NumberInputPanel(self.param, self.dialog)
def setValue(self, value):
if self.dialogType in (DIALOG_STANDARD, DIALOG_BATCH):
self.widget.setValue(value)
else:
self.setComboValue(value)
self.widget.setValue(value)
def value(self):
if self.dialogType in (DIALOG_STANDARD, DIALOG_BATCH):
return self.widget.getValue()
else:
def validator(v):
if str(v).strip():
try:
if self.param.isInteger:
int(v)
else:
float(v)
return True
except:
return False
else:
return self.param.optional
return self.comboValue(validator)
return self.widget.getValue()
class RasterWidgetWrapper(WidgetWrapper):

View File

@ -1,145 +0,0 @@
# -*- coding: utf-8 -*-
"""
***************************************************************************
CalculatorModelerAlgorithm.py
---------------------
Date : August 2012
Copyright : (C) 2012 by Victor Olaya
Email : volayaf at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************
"""
from builtins import chr
from builtins import str
from builtins import range
__author__ = 'Victor Olaya'
__date__ = 'August 2012'
__copyright__ = '(C) 2012, Victor Olaya'
# This will get replaced with a git SHA1 when you do a git archive
__revision__ = '$Format:%H$'
from qgis.PyQt.QtCore import Qt, QMetaObject
from qgis.PyQt.QtWidgets import QDialogButtonBox, QTextEdit, QLineEdit, QVBoxLayout
from processing.core.GeoAlgorithm import GeoAlgorithm
from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException
from processing.core.parameters import ParameterString
from processing.core.parameters import ParameterNumber
from processing.core.outputs import OutputNumber
from processing.modeler.ModelerParametersDialog import ModelerParametersDialog
from processing.modeler.ModelerAlgorithm import Algorithm
FORMULA = 'FORMULA'
NUMBER = 'NUMBER'
RESULT = 'RESULT'
AVAILABLE_VARIABLES = 10
class CalculatorModelerAlgorithm(GeoAlgorithm):
def defineCharacteristics(self):
self.showInModeler = True
self.showInToolbox = False
self.name = self.tr('Calculator', 'CalculatorModelerAlgorithm')
self.group = self.tr('Modeler-only tools', 'CalculatorModelerAlgorithm')
self.addParameter(ParameterString(FORMULA,
self.tr('Formula', 'CalculatorModelerAlgorithm'), ''))
for i in range(AVAILABLE_VARIABLES):
self.addParameter(ParameterNumber(NUMBER
+ str(i), 'dummy', optional=True))
self.addOutput(OutputNumber(RESULT,
self.tr('Result', 'CalculatorModelerAlgorithm')))
def processAlgorithm(self, progress):
formula = self.getParameterValue(FORMULA)
for i in range(AVAILABLE_VARIABLES):
name = NUMBER + str(i)
num = self.getParameterValue(name)
formula = formula.replace(chr(97 + i), str(num))
try:
result = eval(formula)
self.setOutputValue(RESULT, result)
except:
raise GeoAlgorithmExecutionException(
self.tr('Wrong formula: %s', 'CalculatorModelerAlgorithm') % formula)
def getCustomModelerParametersDialog(self, modelAlg, algName=None):
return CalculatorModelerParametersDialog(self, modelAlg, algName)
class CalculatorModelerParametersDialog(ModelerParametersDialog):
def setupUi(self):
self.valueItems = {}
self.dependentItems = {}
self.resize(650, 450)
self.buttonBox = QDialogButtonBox()
self.buttonBox.setOrientation(Qt.Horizontal)
self.buttonBox.setStandardButtons(QDialogButtonBox.Cancel
| QDialogButtonBox.Ok)
self.infoText = QTextEdit()
numbers = self.getAvailableValuesOfType(ParameterNumber, OutputNumber)
text = self.tr('You can refer to model values in your formula, using '
'single-letter variables, as follows:\n', 'CalculatorModelerParametersDialog')
ichar = 97
if numbers:
for number in numbers:
text += chr(ichar) + '->' + self.resolveValueDescription(number) + '\n'
ichar += 1
else:
text += self.tr('\n - No numerical variables are available.', 'CalculatorModelerParametersDialog')
self.infoText.setText(text)
self.infoText.setEnabled(False)
self.formulaText = QLineEdit()
if hasattr(self.formulaText, 'setPlaceholderText'):
self.formulaText.setPlaceholderText(self.tr('[Enter your formula here]', 'CalculatorModelerParametersDialog'))
if self._algName is not None:
alg = self.model.algs[self._algName]
self.formulaText.setText(alg.params[FORMULA])
self.setWindowTitle(self.tr('Calculator', 'CalculatorModelerParametersDialog'))
self.verticalLayout = QVBoxLayout()
self.verticalLayout.setSpacing(2)
self.verticalLayout.setMargin(0)
self.verticalLayout.addWidget(self.infoText)
self.verticalLayout.addWidget(self.formulaText)
self.verticalLayout.addWidget(self.buttonBox)
self.setLayout(self.verticalLayout)
self.buttonBox.accepted.connect(self.okPressed)
self.buttonBox.rejected.connect(self.cancelPressed)
QMetaObject.connectSlotsByName(self)
def createAlgorithm(self):
alg = Algorithm('modelertools:calculator')
alg.setName(self.model)
alg.description = self.tr('Calculator', 'CalculatorModelerParametersDialog')
formula = self.formulaText.text()
alg.params[FORMULA] = formula
for i in range(AVAILABLE_VARIABLES):
paramname = NUMBER + str(i)
alg.params[paramname] = None
numbers = self.getAvailableValuesOfType(ParameterNumber, OutputNumber)
used = []
for i in range(len(numbers)):
if str(chr(i + 97)) in formula:
used.append(numbers[i])
for i, variable in enumerate(used):
paramname = NUMBER + str(i)
alg.params[paramname] = variable
# TODO check formula is correct
return alg

View File

@ -56,7 +56,7 @@ from processing.core.parameters import (getParameterFromString,
ParameterCrs,
ParameterDataObject,
ParameterMultipleInput)
from processing.tools import dataobjects
from processing.gui.Help2Html import getHtmlFromDescriptionsDict
from processing.core.alglist import algList
@ -204,6 +204,27 @@ class ValueFromOutput(object):
def asPythonParameter(self):
return "outputs_%s['%s']" % (self.alg, self.output)
class CompoundValue():
def __init__(self, values = [], definition=""):
self.values = values
self.definition = definition
def todict(self):
return self.__dict__
def __eq__(self, other):
try:
return self.values == other.values and self.definition == other.definition
except:
return False
def __str__(self):
return self.definition
def asPythonParameter(self):
return "" #TODO
class ModelerAlgorithm(GeoAlgorithm):
@ -343,6 +364,12 @@ class ModelerAlgorithm(GeoAlgorithm):
for value in list(alg.params.values()):
if value is None:
continue
if isinstance(value, CompoundValue):
print value
for v in value.values:
if isinstance(v, ValueFromOutput):
algs.add(v.alg)
algs.update(self.getDependsOnAlgorithms(v.alg))
if isinstance(value, list):
for v in value:
if isinstance(v, ValueFromOutput):
@ -387,7 +414,7 @@ class ModelerAlgorithm(GeoAlgorithm):
for param in algInstance.parameters:
if not param.hidden:
if param.name in alg.params:
value = self.resolveValue(alg.params[param.name])
value = self.resolveValue(alg.params[param.name], param)
else:
if iface is not None:
iface.messageBar().pushMessage(self.tr("Warning"),
@ -430,17 +457,20 @@ class ModelerAlgorithm(GeoAlgorithm):
def getSafeNameForOutput(self, algName, outName):
return outName + '_ALG' + algName
def resolveValue(self, value):
def resolveValue(self, value, param):
if value is None:
return None
v = None
if isinstance(value, list):
return ";".join([self.resolveValue(v) for v in value])
if isinstance(value, ValueFromInput):
return self.getParameterFromName(value.name).value
v = ";".join([self.resolveValue(v, param) for v in value])
elif isinstance(value, CompoundValue):
v = self.resolveValue(value.definition, param)
elif isinstance(value, ValueFromInput):
v = self.getParameterFromName(value.name).value
elif isinstance(value, ValueFromOutput):
return self.algs[value.alg].algorithm.getOutputFromName(value.output).value
v = self.algs[value.alg].algorithm.getOutputFromName(value.output).value
else:
return value
v = value
return param.evaluateForModeler(v, self)
def processAlgorithm(self, progress):
executed = []
@ -571,128 +601,13 @@ class ModelerAlgorithm(GeoAlgorithm):
return model
@staticmethod
def fromJsonFile(filename):
def fromFile(filename):
with open(filename) as f:
s = f.read()
alg = ModelerAlgorithm.fromJson(s)
alg.descriptionFile = filename
return alg
############LEGACY METHOD TO SUPPORT OLD FORMAT###########
LINE_BREAK_STRING = '%%%'
@staticmethod
def fromFile(filename):
try:
alg = ModelerAlgorithm.fromJsonFile(filename)
return alg
except WrongModelException:
alg = ModelerAlgorithm.fromOldFormatFile(filename)
return alg
@staticmethod
def fromOldFormatFile(filename):
def _tr(s):
return QCoreApplication.translate('ModelerAlgorithm', s)
hardcodedValues = {}
modelParameters = []
modelAlgs = []
model = ModelerAlgorithm()
model.descriptionFile = filename
lines = codecs.open(filename, 'r', encoding='utf-8')
line = lines.readline().strip('\n').strip('\r')
try:
while line != '':
if line.startswith('PARAMETER:'):
paramLine = line[len('PARAMETER:'):]
param = getParameterFromString(paramLine)
if param:
pass
else:
raise WrongModelException(
_tr('Error in parameter line: %s', 'ModelerAlgorithm') % line)
line = lines.readline().strip('\n')
tokens = line.split(',')
model.addParameter(ModelerParameter(param,
QPointF(float(tokens[0]), float(tokens[1]))))
modelParameters.append(param.name)
elif line.startswith('VALUE:'):
valueLine = line[len('VALUE:'):]
tokens = valueLine.split('===')
name = tokens[0]
value = tokens[1].replace(ModelerAlgorithm.LINE_BREAK_STRING, '\n')
hardcodedValues[name] = value
elif line.startswith('NAME:'):
model.name = line[len('NAME:'):]
elif line.startswith('GROUP:'):
model.group = line[len('GROUP:'):]
elif line.startswith('ALGORITHM:'):
algLine = line[len('ALGORITHM:'):]
alg = algList.getAlgorithm(algLine)
if alg is not None:
modelAlg = Algorithm(alg.commandLineName())
modelAlg.description = alg.name
posline = lines.readline().strip('\n').strip('\r')
tokens = posline.split(',')
modelAlg.pos = QPointF(float(tokens[0]), float(tokens[1]))
# dependenceline = lines.readline().strip('\n').strip('\r')
for param in alg.parameters:
if not param.hidden:
line = lines.readline().strip('\n').strip('\r')
if line == str(None):
modelAlg.params[param.name] = None
else:
tokens = line.split('|')
try:
algIdx = int(tokens[0])
except:
raise WrongModelException(
_tr('Number of parameters in the '
'{} algorithm does not match '
'current Processing '
'implementation'.format(alg.name)))
if algIdx == -1:
if tokens[1] in modelParameters:
modelAlg.params[param.name] = ValueFromInput(tokens[1])
else:
modelAlg.params[param.name] = hardcodedValues[tokens[1]]
else:
modelAlg.params[param.name] = ValueFromOutput(algIdx, tokens[1])
for out in alg.outputs:
if not out.hidden:
line = lines.readline().strip('\n').strip('\r')
if str(None) != line:
if '|' in line:
tokens = line.split('|')
name = tokens[0]
tokens = tokens[1].split(',')
pos = QPointF(float(tokens[0]), float(tokens[1]))
else:
name = line
pos = None
modelerOutput = ModelerOutput(name)
modelerOutput.pos = pos
modelAlg.outputs[out.name] = modelerOutput
model.addAlgorithm(modelAlg)
modelAlgs.append(modelAlg.name)
else:
raise WrongModelException(
_tr('Error in algorithm name: %s',) % algLine)
line = lines.readline().strip('\n').strip('\r')
for modelAlg in list(model.algs.values()):
for name, value in modelAlg.params.items():
if isinstance(value, ValueFromOutput):
value.alg = modelAlgs[value.alg]
return model
except Exception as e:
if isinstance(e, WrongModelException):
raise e
else:
raise WrongModelException(_tr('Error in model definition line: ') + '%s\n%s' % (line.strip(), traceback.format_exc()))
def toPython(self):
s = ['##%s=name' % self.name]
for param in list(self.inputs.values()):

View File

@ -1,53 +0,0 @@
# -*- coding: utf-8 -*-
"""
***************************************************************************
ModelerOnlyAlgorithmProvider.py
---------------------
Date : August 2012
Copyright : (C) 2012 by Victor Olaya
Email : volayaf at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************
"""
__author__ = 'Victor Olaya'
__date__ = 'August 2012'
__copyright__ = '(C) 2012, Victor Olaya'
# This will get replaced with a git SHA1 when you do a git archive
__revision__ = '$Format:%H$'
import os.path
from qgis.PyQt.QtGui import QIcon
from processing.core.AlgorithmProvider import AlgorithmProvider
from processing.modeler.CalculatorModelerAlgorithm import CalculatorModelerAlgorithm
pluginPath = os.path.split(os.path.dirname(__file__))[0]
class ModelerOnlyAlgorithmProvider(AlgorithmProvider):
def __init__(self):
AlgorithmProvider.__init__(self)
def getName(self):
return 'modelertools'
def getDescription(self):
return self.tr('Modeler-only tools', 'ModelerOnlyAlgorithmProvider')
def getIcon(self):
return QIcon(os.path.join(pluginPath, 'images', 'model.png'))
def _loadAlgorithms(self):
self.algs = [CalculatorModelerAlgorithm()]
for alg in self.algs:
alg.provider = self

View File

@ -212,7 +212,8 @@ class ModelerParameterDefinitionDialog(QDialog):
default = self.param.default
if self.param.isInteger:
default = int(math.floor(default))
self.defaultTextBox.setText(str(default))
if default:
self.defaultTextBox.setText(str(default))
self.horizontalLayoutDefault.addWidget(self.defaultTextBox)
self.verticalLayout.addLayout(self.horizontalLayoutDefault)
elif self.paramType == ModelerParameterDefinitionDialog.PARAMETER_STRING or \
@ -294,7 +295,7 @@ class ModelerParameterDefinitionDialog(QDialog):
or isinstance(self.param, ParameterBoolean):
self.param = ParameterBoolean(name, description,
self.state.isChecked())
elif self.paramType in ModelerParameterDefinitionDialog.PARAMETER_TABLE_FIELD \
elif self.paramType == ModelerParameterDefinitionDialog.PARAMETER_TABLE_FIELD \
or isinstance(self.param, ParameterTableField):
if self.parentCombo.currentIndex() < 0:
QMessageBox.warning(self, self.tr('Unable to define parameter'),

View File

@ -52,6 +52,7 @@ from processing.core.outputs import (OutputRaster,
OutputString,
OutputExtent,
OutputCrs)
from processing.core.parameters import ParameterPoint, ParameterExtent
from processing.modeler.ModelerAlgorithm import (ValueFromInput,
ValueFromOutput,
@ -261,20 +262,15 @@ class ModelerParametersDialog(QDialog):
self.labels[param.name].setVisible(self.showAdvanced)
self.widgets[param.name].setVisible(self.showAdvanced)
def getAvailableValuesForParam(self, param):
outputType = None
if isinstance(param, ParameterCrs):
outputType = OutputCrs
return self.getAvailableValuesOfType(param.__class__, outputType)
def getAvailableValuesOfType(self, paramType, outType=None, dataType=None):
values = []
inputs = self.model.inputs
for i in list(inputs.values()):
param = i.param
if isinstance(param, paramType):
if dataType is not None and param.datatype in dataType:
values.append(ValueFromInput(param.name))
if dataType is not None:
if param.datatype in dataType:
values.append(ValueFromInput(param.name))
else:
values.append(ValueFromInput(param.name))
if outType is None:
@ -340,12 +336,9 @@ class ModelerParametersDialog(QDialog):
value = alg.params[param.name]
else:
value = param.default
wrapper = self.wrappers[param.name]
wrapper.setValue(value)
self.wrappers[param.name].setValue(value)
for name, out in alg.outputs.items():
widget = self.valueItems[name].setText(out.description)
self.valueItems[name].setText(out.description)
selected = []
dependencies = self.getAvailableDependencies()

View File

@ -30,7 +30,7 @@ from qgis.PyQt.QtCore import QPointF, Qt
from qgis.PyQt.QtWidgets import QGraphicsItem, QGraphicsScene
from processing.modeler.ModelerGraphicItem import ModelerGraphicItem
from processing.modeler.ModelerArrowItem import ModelerArrowItem
from processing.modeler.ModelerAlgorithm import ValueFromInput, ValueFromOutput
from processing.modeler.ModelerAlgorithm import ValueFromInput, ValueFromOutput, CompoundValue
class ModelerScene(QGraphicsScene):
@ -65,6 +65,9 @@ class ModelerScene(QGraphicsScene):
if isinstance(value, list):
for v in value:
items.extend(self.getItemsFromParamValue(v))
elif isinstance(value, CompoundValue):
for v in value.values:
items.extend(self.getItemsFromParamValue(v))
elif isinstance(value, ValueFromInput):
items.append((self.paramItems[value.name], 0))
elif isinstance(value, ValueFromOutput):