# -*- coding: utf-8 -*-
"""
***************************************************************************
ModelerAlgorithm.py
---------------------
Date : August 2012
Copyright : (C) 2012 by Victor Olaya
Email : volayaf at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************
"""
from builtins import str
from builtins import object
__author__ = 'Victor Olaya'
__date__ = 'August 2012'
__copyright__ = '(C) 2012, Victor Olaya'
# This will get replaced with a git SHA1 when you do a git archive
__revision__ = '$Format:%H$'
import os.path
import sys
import copy
import time
import json
from qgis.PyQt.QtCore import QPointF
from operator import attrgetter
from qgis.core import QgsApplication
from qgis.gui import QgsMessageBar
from qgis.utils import iface
from processing.core.GeoAlgorithm import GeoAlgorithm
from processing.modeler.WrongModelException import WrongModelException
from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException
from processing.core.parameters import (ParameterRaster,
ParameterVector,
ParameterTable,
ParameterTableField,
ParameterBoolean,
ParameterString,
ParameterNumber,
ParameterDataObject)
from processing.gui.Help2Html import getHtmlFromDescriptionsDict
from processing.core.alglist import algList
pluginPath = os.path.split(os.path.dirname(__file__))[0]
class ModelerParameter(object):
def __init__(self, param=None, pos=None):
self.param = param
self.pos = pos
def todict(self):
return self.__dict__
@staticmethod
def fromdict(d):
return ModelerParameter(d["param"], d["pos"])
class ModelerOutput(object):
def __init__(self, description=""):
self.description = description
self.pos = None
def todict(self):
return self.__dict__
class Algorithm(object):
def __init__(self, consoleName=""):
self.name = None
self.description = ""
# The type of the algorithm, indicated as a string, which corresponds
# to the string used to refer to it in the python console
self.consoleName = consoleName
self._algInstance = None
# A dict of Input object. keys are param names
self.params = {}
# A dict of ModelerOutput with final output descriptions. Keys are output names.
# Outputs not final are not stored in this dict
self.outputs = {}
self.pos = None
self.dependencies = []
self.paramsFolded = True
self.outputsFolded = True
self.active = True
def todict(self):
return {k: v for k, v in list(self.__dict__.items()) if not k.startswith("_")}
@property
def algorithm(self):
if self._algInstance is None:
self._algInstance = algList.getAlgorithm(self.consoleName).getCopy()
return self._algInstance
def setName(self, model):
if self.name is None:
i = 1
name = self.consoleName + "_" + str(i)
while name in model.algs:
i += 1
name = self.consoleName + "_" + str(i)
self.name = name
def getOutputType(self, outputName):
output = self.algorithm.getOutputFromName(outputName)
return "output " + output.__class__.__name__.split(".")[-1][6:].lower()
def toPython(self):
s = []
params = []
for param in self.algorithm.parameters:
value = self.params[param.name]
def _toString(v):
if isinstance(v, (ValueFromInput, ValueFromOutput)):
return v.asPythonParameter()
elif isinstance(v, str):
return "\\n".join(("'%s'" % v).splitlines())
elif isinstance(v, list):
return "[%s]" % ",".join([_toString(val) for val in v])
else:
return str(value)
params.append(_toString(value))
for out in self.algorithm.outputs:
if not out.hidden:
if out.name in self.outputs:
params.append(safeName(self.outputs[out.name].description).lower())
else:
params.append(str(None))
s.append("outputs_%s=processing.run('%s', %s)" % (self.name, self.consoleName, ",".join(params)))
return s
class ValueFromInput(object):
def __init__(self, name=""):
self.name = name
def todict(self):
return self.__dict__
def __str__(self):
return self.name
def __eq__(self, other):
try:
return self.name == other.name
except:
return False
def asPythonParameter(self):
return self.name
class ValueFromOutput(object):
def __init__(self, alg="", output=""):
self.alg = alg
self.output = output
def todict(self):
return self.__dict__
def __eq__(self, other):
try:
return self.alg == other.alg and self.output == other.output
except:
return False
def __str__(self):
return self.alg + ":" + self.output
def asPythonParameter(self):
return "outputs_%s['%s']" % (self.alg, self.output)
class CompoundValue(object):
def __init__(self, values=[], definition=""):
self.values = values
self.definition = definition
def todict(self):
return self.__dict__
def __eq__(self, other):
try:
return self.values == other.values and self.definition == other.definition
except:
return False
def __str__(self):
return self.definition
def asPythonParameter(self):
return "" # TODO
class ModelerAlgorithm(GeoAlgorithm):
CANVAS_SIZE = 4000
def getCopy(self):
newone = ModelerAlgorithm()
newone.provider = self.provider
newone.algs = {}
for algname, alg in self.algs.items():
newone.algs[algname] = Algorithm()
newone.algs[algname].__dict__.update(copy.deepcopy(alg.todict()))
newone.inputs = copy.deepcopy(self.inputs)
newone.defineCharacteristics()
newone.name = self.name
newone.group = self.group
newone.descriptionFile = self.descriptionFile
newone.helpContent = copy.deepcopy(self.helpContent)
return newone
def __init__(self):
self.name = self.tr('Model', 'ModelerAlgorithm')
# The dialog where this model is being edited
self.modelerdialog = None
self.descriptionFile = None
self.helpContent = {}
# Geoalgorithms in this model. A dict of Algorithm objects, with names as keys
self.algs = {}
# Input parameters. A dict of Input objects, with names as keys
self.inputs = {}
GeoAlgorithm.__init__(self)
def getIcon(self):
return QgsApplication.getThemeIcon("/processingModel.svg")
def defineCharacteristics(self):
classes = [ParameterRaster, ParameterVector, ParameterTable, ParameterTableField,
ParameterBoolean, ParameterString, ParameterNumber]
self.parameters = []
for c in classes:
for inp in list(self.inputs.values()):
if isinstance(inp.param, c):
self.parameters.append(inp.param)
for inp in list(self.inputs.values()):
if inp.param not in self.parameters:
self.parameters.append(inp.param)
self.parameters.sort(key=attrgetter("description"))
self.outputs = []
for alg in list(self.algs.values()):
if alg.active:
for out in alg.outputs:
modelOutput = copy.deepcopy(alg.algorithm.getOutputFromName(out))
modelOutput.name = self.getSafeNameForOutput(alg.name, out)
modelOutput.description = alg.outputs[out].description
self.outputs.append(modelOutput)
self.outputs.sort(key=attrgetter("description"))
def addParameter(self, param):
self.inputs[param.param.name] = param
def updateParameter(self, param):
self.inputs[param.name].param = param
def addAlgorithm(self, alg):
name = self.getNameForAlgorithm(alg)
alg.name = name
self.algs[name] = alg
def getNameForAlgorithm(self, alg):
i = 1
while alg.consoleName.upper().replace(":", "") + "_" + str(i) in list(self.algs.keys()):
i += 1
return alg.consoleName.upper().replace(":", "") + "_" + str(i)
def updateAlgorithm(self, alg):
alg.pos = self.algs[alg.name].pos
alg.paramsFolded = self.algs[alg.name].paramsFolded
alg.outputsFolded = self.algs[alg.name].outputsFolded
self.algs[alg.name] = alg
from processing.modeler.ModelerGraphicItem import ModelerGraphicItem
for i, out in enumerate(alg.outputs):
alg.outputs[out].pos = (alg.outputs[out].pos or
alg.pos + QPointF(
ModelerGraphicItem.BOX_WIDTH,
(i + 1.5) * ModelerGraphicItem.BOX_HEIGHT))
def removeAlgorithm(self, name):
"""Returns True if the algorithm could be removed, False if
others depend on it and could not be removed.
"""
if self.hasDependencies(name):
return False
del self.algs[name]
self.modelerdialog.hasChanged = True
return True
def removeParameter(self, name):
"""Returns True if the parameter could be removed, False if
others depend on it and could not be removed.
"""
if self.hasDependencies(name):
return False
del self.inputs[name]
self.modelerdialog.hasChanged = True
return True
def hasDependencies(self, name):
"""This method returns True if some other element depends on
the passed one.
"""
for alg in list(self.algs.values()):
for value in list(alg.params.values()):
if value is None:
continue
if isinstance(value, list):
for v in value:
if isinstance(v, ValueFromInput):
if v.name == name:
return True
elif isinstance(v, ValueFromOutput):
if v.alg == name:
return True
if isinstance(value, ValueFromInput):
if value.name == name:
return True
elif isinstance(value, ValueFromOutput):
if value.alg == name:
return True
if alg.name != name:
for dep in alg.dependencies:
if (dep == name):
return True
return False
def getDependsOnAlgorithms(self, name):
"""This method returns a list with names of algorithms
a given one depends on.
"""
alg = self.algs[name]
algs = set()
algs.update(set(alg.dependencies))
for value in list(alg.params.values()):
if value is None:
continue
if isinstance(value, CompoundValue):
for v in value.values:
if isinstance(v, ValueFromOutput):
algs.add(v.alg)
algs.update(self.getDependsOnAlgorithms(v.alg))
if isinstance(value, list):
for v in value:
if isinstance(v, ValueFromOutput):
algs.add(v.alg)
algs.update(self.getDependsOnAlgorithms(v.alg))
elif isinstance(value, ValueFromOutput):
algs.add(value.alg)
algs.update(self.getDependsOnAlgorithms(value.alg))
return algs
def getDependentAlgorithms(self, name):
"""This method returns a list with the names of algorithms
depending on a given one. It includes the algorithm itself
"""
algs = set()
algs.add(name)
for alg in list(self.algs.values()):
for value in list(alg.params.values()):
if value is None:
continue
if isinstance(value, list):
for v in value:
if isinstance(v, ValueFromOutput) and v.alg == name:
algs.update(self.getDependentAlgorithms(alg.name))
elif isinstance(value, ValueFromOutput) and value.alg == name:
algs.update(self.getDependentAlgorithms(alg.name))
return algs
def setPositions(self, paramPos, algPos, outputsPos):
for param, pos in list(paramPos.items()):
self.inputs[param].pos = pos
for alg, pos in list(algPos.items()):
self.algs[alg].pos = pos
for alg, positions in list(outputsPos.items()):
for output, pos in list(positions.items()):
self.algs[alg].outputs[output].pos = pos
def prepareAlgorithm(self, alg):
algInstance = alg.algorithm
for param in algInstance.parameters:
if not param.hidden:
if param.name in alg.params:
value = self.resolveValue(alg.params[param.name], param)
else:
if iface is not None:
iface.messageBar().pushMessage(self.tr("Warning"),
self.tr("Parameter {0} in algorithm {1} in the model is run with default value! Edit the model to make sure that this is correct.").format(param.name, alg.name),
QgsMessageBar.WARNING, 4)
value = param.default
# We allow unexistent filepaths, since that allows
# algorithms to skip some conversion routines
if not param.setValue(value) and not isinstance(param,
ParameterDataObject):
raise GeoAlgorithmExecutionException(
self.tr('Wrong value {0} for {1} {2}', 'ModelerAlgorithm').format(
value, param.__class__.__name__, param.name
)
)
for out in algInstance.outputs:
if not out.hidden:
if out.name in alg.outputs:
name = self.getSafeNameForOutput(alg.name, out.name)
modelOut = self.getOutputFromName(name)
if modelOut:
out.value = modelOut.value
else:
out.value = None
return algInstance
def deactivateAlgorithm(self, algName):
dependent = self.getDependentAlgorithms(algName)
for alg in dependent:
self.algs[alg].active = False
def activateAlgorithm(self, algName):
parents = self.getDependsOnAlgorithms(algName)
for alg in parents:
if not self.algs[alg].active:
return False
self.algs[algName].active = True
return True
def getSafeNameForOutput(self, algName, outName):
return outName + '_ALG' + algName
def resolveValue(self, value, param):
if value is None:
v = None
if isinstance(value, list):
v = ";".join([self.resolveValue(v, param) for v in value])
elif isinstance(value, CompoundValue):
v = self.resolveValue(value.definition, param)
elif isinstance(value, ValueFromInput):
v = self.getParameterFromName(value.name).value
elif isinstance(value, ValueFromOutput):
v = self.algs[value.alg].algorithm.getOutputFromName(value.output).value
else:
v = value
return param.evaluateForModeler(v, self)
def processAlgorithm(self, feedback):
executed = []
toExecute = [alg for alg in list(self.algs.values()) if alg.active]
while len(executed) < len(toExecute):
for alg in toExecute:
if alg.name not in executed:
canExecute = True
required = self.getDependsOnAlgorithms(alg.name)
for requiredAlg in required:
if requiredAlg != alg.name and requiredAlg not in executed:
canExecute = False
break
if canExecute:
try:
feedback.pushDebugInfo(
self.tr('Prepare algorithm: {0}', 'ModelerAlgorithm').format(alg.name))
self.prepareAlgorithm(alg)
feedback.setProgressText(
self.tr('Running {0} [{1}/{2}]', 'ModelerAlgorithm').format(alg.description, len(executed) + 1, len(toExecute)))
feedback.pushDebugInfo('Parameters: ' + ', '.join([str(p).strip() +
'=' + str(p.value) for p in alg.algorithm.parameters]))
t0 = time.time()
alg.algorithm.execute(feedback, self)
dt = time.time() - t0
# copy algorithm output value(s) back to model in case the algorithm modified those
for out in alg.algorithm.outputs:
if not out.hidden:
if out.name in alg.outputs:
modelOut = self.getOutputFromName(self.getSafeNameForOutput(alg.name, out.name))
if modelOut:
modelOut.value = out.value
executed.append(alg.name)
feedback.pushDebugInfo(
self.tr('OK. Execution took %{0:.3f} ms ({1} outputs).', 'ModelerAlgorithm').format(dt, len(alg.algorithm.outputs)))
except GeoAlgorithmExecutionException as e:
feedback.pushDebugInfo(self.tr('Failed', 'ModelerAlgorithm'))
raise GeoAlgorithmExecutionException(
self.tr('Error executing algorithm {0}\n{1}', 'ModelerAlgorithm').format(alg.description, e.msg))
feedback.pushDebugInfo(
self.tr('Model processed ok. Executed {0} algorithms total', 'ModelerAlgorithm').format(len(executed)))
def getAsCommand(self):
if self.descriptionFile:
return GeoAlgorithm.getAsCommand(self)
else:
return None
def commandLineName(self):
if self.descriptionFile is None:
return ''
else:
return 'modeler:' + os.path.basename(self.descriptionFile)[:-6].lower()
def checkBeforeOpeningParametersDialog(self):
for alg in list(self.algs.values()):
algInstance = algList.getAlgorithm(alg.consoleName)
if algInstance is None:
return self.tr("The model you are trying to run contains an algorithm that is not available: <i>{0}</i>").format(alg.consoleName)
def setModelerView(self, dialog):
self.modelerdialog = dialog
def updateModelerView(self):
if self.modelerdialog:
self.modelerdialog.repaintModel()
def help(self):
try:
return True, getHtmlFromDescriptionsDict(self, self.helpContent)
except:
return False, None
def shortHelp(self):
if 'ALG_DESC' in self.helpContent:
return self._formatHelp(str(self.helpContent['ALG_DESC']))
return None
def getParameterDescriptions(self):
descs = {}
descriptions = self.helpContent
for param in self.parameters:
if param.name in descriptions:
descs[param.name] = str(descriptions[param.name])
return descs
def todict(self):
keys = ["inputs", "group", "name", "algs", "helpContent"]
return {k: v for k, v in list(self.__dict__.items()) if k in keys}
def toJson(self):
def todict(o):
if isinstance(o, QPointF):
return {"class": "point", "values": {"x": o.x(), "y": o.y()}}
try:
d = o.todict()
return {"class": o.__class__.__module__ + "." + o.__class__.__name__, "values": d}
except Exception:
pass
return json.dumps(self, default=todict, indent=4)
@staticmethod
def fromJson(s):
def fromdict(d):
try:
fullClassName = d["class"]
if isinstance(fullClassName, str):
tokens = fullClassName.split(".")
else:
tokens = fullClassName.__class__.__name__.split(".")
className = tokens[-1]
moduleName = ".".join(tokens[:-1])
values = d["values"]
if className == "point":
return QPointF(values["x"], values["y"])
def _import(name):
__import__(name)
return sys.modules[name]
if moduleName.startswith("processing.parameters"):
moduleName = "processing.core.parameters"
module = _import(moduleName)
clazz = getattr(module, className)
instance = clazz()
for k, v in list(values.items()):
instance.__dict__[k] = v
return instance
except KeyError:
return d
except Exception as e:
raise e
try:
model = json.loads(s, object_hook=fromdict)
except Exception as e:
raise WrongModelException(e.args[0])
return model
@staticmethod
def fromFile(filename):
with open(filename) as f:
s = f.read()
alg = ModelerAlgorithm.fromJson(s)
alg.descriptionFile = filename
return alg
def toPython(self):
s = ['##%s=name' % self.name]
for param in list(self.inputs.values()):
s.append(param.param.getAsScriptCode())
for alg in list(self.algs.values()):
for name, out in list(alg.outputs.items()):
s.append('##%s=%s' % (safeName(out.description).lower(), alg.getOutputType(name)))
executed = []
toExecute = [alg for alg in list(self.algs.values()) if alg.active]
while len(executed) < len(toExecute):
for alg in toExecute:
if alg.name not in executed:
canExecute = True
required = self.getDependsOnAlgorithms(alg.name)
for requiredAlg in required:
if requiredAlg != alg.name and requiredAlg not in executed:
canExecute = False
break
if canExecute:
s.extend(alg.toPython())
executed.append(alg.name)
return '\n'.join(s)
def safeName(name):
validChars = 'abcdefghijklmnopqrstuvwxyz'
return ''.join(c for c in name.lower() if c in validChars)