mirror of
https://github.com/qgis/QGIS.git
synced 2025-02-25 00:58:06 -05:00
658 lines
24 KiB
Python
658 lines
24 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
"""
|
|
***************************************************************************
|
|
ModelerAlgorithm.py
|
|
---------------------
|
|
Date : August 2012
|
|
Copyright : (C) 2012 by Victor Olaya
|
|
Email : volayaf at gmail dot com
|
|
***************************************************************************
|
|
* *
|
|
* This program is free software; you can redistribute it and/or modify *
|
|
* it under the terms of the GNU General Public License as published by *
|
|
* the Free Software Foundation; either version 2 of the License, or *
|
|
* (at your option) any later version. *
|
|
* *
|
|
***************************************************************************
|
|
"""
|
|
from builtins import str
|
|
from builtins import object
|
|
|
|
__author__ = 'Victor Olaya'
|
|
__date__ = 'August 2012'
|
|
__copyright__ = '(C) 2012, Victor Olaya'
|
|
|
|
# This will get replaced with a git SHA1 when you do a git archive
|
|
|
|
__revision__ = '$Format:%H$'
|
|
|
|
import os.path
|
|
import sys
|
|
import copy
|
|
import time
|
|
import json
|
|
from qgis.PyQt.QtCore import QPointF
|
|
from operator import attrgetter
|
|
|
|
from qgis.core import QgsApplication
|
|
from qgis.gui import QgsMessageBar
|
|
from qgis.utils import iface
|
|
from processing.core.GeoAlgorithm import GeoAlgorithm
|
|
from processing.modeler.WrongModelException import WrongModelException
|
|
from processing.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException
|
|
from processing.core.parameters import (ParameterRaster,
|
|
ParameterVector,
|
|
ParameterTable,
|
|
ParameterTableField,
|
|
ParameterBoolean,
|
|
ParameterString,
|
|
ParameterNumber,
|
|
ParameterDataObject)
|
|
|
|
from processing.gui.Help2Html import getHtmlFromDescriptionsDict
|
|
from processing.core.alglist import algList
|
|
|
|
pluginPath = os.path.split(os.path.dirname(__file__))[0]
|
|
|
|
|
|
class ModelerParameter(object):
|
|
|
|
def __init__(self, param=None, pos=None):
|
|
self.param = param
|
|
self.pos = pos
|
|
|
|
def todict(self):
|
|
return self.__dict__
|
|
|
|
@staticmethod
|
|
def fromdict(d):
|
|
return ModelerParameter(d["param"], d["pos"])
|
|
|
|
|
|
class ModelerOutput(object):
|
|
|
|
def __init__(self, description=""):
|
|
self.description = description
|
|
self.pos = None
|
|
|
|
def todict(self):
|
|
return self.__dict__
|
|
|
|
|
|
class Algorithm(object):
|
|
|
|
def __init__(self, consoleName=""):
|
|
|
|
self.name = None
|
|
self.description = ""
|
|
|
|
# The type of the algorithm, indicated as a string, which corresponds
|
|
# to the string used to refer to it in the python console
|
|
self.consoleName = consoleName
|
|
|
|
self._algInstance = None
|
|
|
|
# A dict of Input object. keys are param names
|
|
self.params = {}
|
|
|
|
# A dict of ModelerOutput with final output descriptions. Keys are output names.
|
|
# Outputs not final are not stored in this dict
|
|
self.outputs = {}
|
|
|
|
self.pos = None
|
|
|
|
self.dependencies = []
|
|
|
|
self.paramsFolded = True
|
|
self.outputsFolded = True
|
|
self.active = True
|
|
|
|
def todict(self):
|
|
return {k: v for k, v in list(self.__dict__.items()) if not k.startswith("_")}
|
|
|
|
@property
|
|
def algorithm(self):
|
|
if self._algInstance is None:
|
|
self._algInstance = algList.getAlgorithm(self.consoleName).getCopy()
|
|
return self._algInstance
|
|
|
|
def setName(self, model):
|
|
if self.name is None:
|
|
i = 1
|
|
name = self.consoleName + "_" + str(i)
|
|
while name in model.algs:
|
|
i += 1
|
|
name = self.consoleName + "_" + str(i)
|
|
self.name = name
|
|
|
|
def getOutputType(self, outputName):
|
|
output = self.algorithm.getOutputFromName(outputName)
|
|
return "output " + output.__class__.__name__.split(".")[-1][6:].lower()
|
|
|
|
def toPython(self):
|
|
s = []
|
|
params = []
|
|
for param in self.algorithm.parameters:
|
|
value = self.params[param.name]
|
|
|
|
def _toString(v):
|
|
if isinstance(v, (ValueFromInput, ValueFromOutput)):
|
|
return v.asPythonParameter()
|
|
elif isinstance(v, str):
|
|
return "\\n".join(("'%s'" % v).splitlines())
|
|
elif isinstance(v, list):
|
|
return "[%s]" % ",".join([_toString(val) for val in v])
|
|
else:
|
|
return str(value)
|
|
params.append(_toString(value))
|
|
for out in self.algorithm.outputs:
|
|
if not out.hidden:
|
|
if out.name in self.outputs:
|
|
params.append(safeName(self.outputs[out.name].description).lower())
|
|
else:
|
|
params.append(str(None))
|
|
s.append("outputs_%s=processing.runalg('%s', %s)" % (self.name, self.consoleName, ",".join(params)))
|
|
return s
|
|
|
|
|
|
class ValueFromInput(object):
|
|
|
|
def __init__(self, name=""):
|
|
self.name = name
|
|
|
|
def todict(self):
|
|
return self.__dict__
|
|
|
|
def __str__(self):
|
|
return self.name
|
|
|
|
def __eq__(self, other):
|
|
try:
|
|
return self.name == other.name
|
|
except:
|
|
return False
|
|
|
|
def asPythonParameter(self):
|
|
return self.name
|
|
|
|
|
|
class ValueFromOutput(object):
|
|
|
|
def __init__(self, alg="", output=""):
|
|
self.alg = alg
|
|
self.output = output
|
|
|
|
def todict(self):
|
|
return self.__dict__
|
|
|
|
def __eq__(self, other):
|
|
try:
|
|
return self.alg == other.alg and self.output == other.output
|
|
except:
|
|
return False
|
|
|
|
def __str__(self):
|
|
return self.alg + ":" + self.output
|
|
|
|
def asPythonParameter(self):
|
|
return "outputs_%s['%s']" % (self.alg, self.output)
|
|
|
|
|
|
class CompoundValue(object):
|
|
|
|
def __init__(self, values=[], definition=""):
|
|
self.values = values
|
|
self.definition = definition
|
|
|
|
def todict(self):
|
|
return self.__dict__
|
|
|
|
def __eq__(self, other):
|
|
try:
|
|
return self.values == other.values and self.definition == other.definition
|
|
except:
|
|
return False
|
|
|
|
def __str__(self):
|
|
return self.definition
|
|
|
|
def asPythonParameter(self):
|
|
return "" # TODO
|
|
|
|
|
|
class ModelerAlgorithm(GeoAlgorithm):
|
|
|
|
CANVAS_SIZE = 4000
|
|
|
|
def getCopy(self):
|
|
newone = ModelerAlgorithm()
|
|
newone.provider = self.provider
|
|
|
|
newone.algs = {}
|
|
for algname, alg in self.algs.items():
|
|
newone.algs[algname] = Algorithm()
|
|
newone.algs[algname].__dict__.update(copy.deepcopy(alg.todict()))
|
|
newone.inputs = copy.deepcopy(self.inputs)
|
|
newone.defineCharacteristics()
|
|
newone.name = self.name
|
|
newone.group = self.group
|
|
newone.descriptionFile = self.descriptionFile
|
|
newone.helpContent = copy.deepcopy(self.helpContent)
|
|
return newone
|
|
|
|
def __init__(self):
|
|
self.name = self.tr('Model', 'ModelerAlgorithm')
|
|
# The dialog where this model is being edited
|
|
self.modelerdialog = None
|
|
self.descriptionFile = None
|
|
self.helpContent = {}
|
|
|
|
# Geoalgorithms in this model. A dict of Algorithm objects, with names as keys
|
|
self.algs = {}
|
|
|
|
# Input parameters. A dict of Input objects, with names as keys
|
|
self.inputs = {}
|
|
GeoAlgorithm.__init__(self)
|
|
|
|
def getIcon(self):
|
|
return QgsApplication.getThemeIcon("/processingModel.svg")
|
|
|
|
def defineCharacteristics(self):
|
|
classes = [ParameterRaster, ParameterVector, ParameterTable, ParameterTableField,
|
|
ParameterBoolean, ParameterString, ParameterNumber]
|
|
self.parameters = []
|
|
for c in classes:
|
|
for inp in list(self.inputs.values()):
|
|
if isinstance(inp.param, c):
|
|
self.parameters.append(inp.param)
|
|
for inp in list(self.inputs.values()):
|
|
if inp.param not in self.parameters:
|
|
self.parameters.append(inp.param)
|
|
self.parameters.sort(key=attrgetter("description"))
|
|
|
|
self.outputs = []
|
|
for alg in list(self.algs.values()):
|
|
if alg.active:
|
|
for out in alg.outputs:
|
|
modelOutput = copy.deepcopy(alg.algorithm.getOutputFromName(out))
|
|
modelOutput.name = self.getSafeNameForOutput(alg.name, out)
|
|
modelOutput.description = alg.outputs[out].description
|
|
self.outputs.append(modelOutput)
|
|
self.outputs.sort(key=attrgetter("description"))
|
|
|
|
def addParameter(self, param):
|
|
self.inputs[param.param.name] = param
|
|
|
|
def updateParameter(self, param):
|
|
self.inputs[param.name].param = param
|
|
|
|
def addAlgorithm(self, alg):
|
|
name = self.getNameForAlgorithm(alg)
|
|
alg.name = name
|
|
self.algs[name] = alg
|
|
|
|
def getNameForAlgorithm(self, alg):
|
|
i = 1
|
|
while alg.consoleName.upper().replace(":", "") + "_" + str(i) in list(self.algs.keys()):
|
|
i += 1
|
|
return alg.consoleName.upper().replace(":", "") + "_" + str(i)
|
|
|
|
def updateAlgorithm(self, alg):
|
|
alg.pos = self.algs[alg.name].pos
|
|
alg.paramsFolded = self.algs[alg.name].paramsFolded
|
|
alg.outputsFolded = self.algs[alg.name].outputsFolded
|
|
self.algs[alg.name] = alg
|
|
|
|
from processing.modeler.ModelerGraphicItem import ModelerGraphicItem
|
|
for i, out in enumerate(alg.outputs):
|
|
alg.outputs[out].pos = (alg.outputs[out].pos or
|
|
alg.pos + QPointF(
|
|
ModelerGraphicItem.BOX_WIDTH,
|
|
(i + 1.5) * ModelerGraphicItem.BOX_HEIGHT))
|
|
|
|
def removeAlgorithm(self, name):
|
|
"""Returns True if the algorithm could be removed, False if
|
|
others depend on it and could not be removed.
|
|
"""
|
|
if self.hasDependencies(name):
|
|
return False
|
|
del self.algs[name]
|
|
self.modelerdialog.hasChanged = True
|
|
return True
|
|
|
|
def removeParameter(self, name):
|
|
"""Returns True if the parameter could be removed, False if
|
|
others depend on it and could not be removed.
|
|
"""
|
|
if self.hasDependencies(name):
|
|
return False
|
|
del self.inputs[name]
|
|
self.modelerdialog.hasChanged = True
|
|
return True
|
|
|
|
def hasDependencies(self, name):
|
|
"""This method returns True if some other element depends on
|
|
the passed one.
|
|
"""
|
|
for alg in list(self.algs.values()):
|
|
for value in list(alg.params.values()):
|
|
if value is None:
|
|
continue
|
|
if isinstance(value, list):
|
|
for v in value:
|
|
if isinstance(v, ValueFromInput):
|
|
if v.name == name:
|
|
return True
|
|
elif isinstance(v, ValueFromOutput):
|
|
if v.alg == name:
|
|
return True
|
|
if isinstance(value, ValueFromInput):
|
|
if value.name == name:
|
|
return True
|
|
elif isinstance(value, ValueFromOutput):
|
|
if value.alg == name:
|
|
return True
|
|
if alg.name != name:
|
|
for dep in alg.dependencies:
|
|
if (dep == name):
|
|
return True
|
|
return False
|
|
|
|
def getDependsOnAlgorithms(self, name):
|
|
"""This method returns a list with names of algorithms
|
|
a given one depends on.
|
|
"""
|
|
alg = self.algs[name]
|
|
algs = set()
|
|
algs.update(set(alg.dependencies))
|
|
for value in list(alg.params.values()):
|
|
if value is None:
|
|
continue
|
|
if isinstance(value, CompoundValue):
|
|
for v in value.values:
|
|
if isinstance(v, ValueFromOutput):
|
|
algs.add(v.alg)
|
|
algs.update(self.getDependsOnAlgorithms(v.alg))
|
|
if isinstance(value, list):
|
|
for v in value:
|
|
if isinstance(v, ValueFromOutput):
|
|
algs.add(v.alg)
|
|
algs.update(self.getDependsOnAlgorithms(v.alg))
|
|
elif isinstance(value, ValueFromOutput):
|
|
algs.add(value.alg)
|
|
algs.update(self.getDependsOnAlgorithms(value.alg))
|
|
|
|
return algs
|
|
|
|
def getDependentAlgorithms(self, name):
|
|
"""This method returns a list with the names of algorithms
|
|
depending on a given one. It includes the algorithm itself
|
|
"""
|
|
algs = set()
|
|
algs.add(name)
|
|
for alg in list(self.algs.values()):
|
|
for value in list(alg.params.values()):
|
|
if value is None:
|
|
continue
|
|
if isinstance(value, list):
|
|
for v in value:
|
|
if isinstance(v, ValueFromOutput) and v.alg == name:
|
|
algs.update(self.getDependentAlgorithms(alg.name))
|
|
elif isinstance(value, ValueFromOutput) and value.alg == name:
|
|
algs.update(self.getDependentAlgorithms(alg.name))
|
|
|
|
return algs
|
|
|
|
def setPositions(self, paramPos, algPos, outputsPos):
|
|
for param, pos in list(paramPos.items()):
|
|
self.inputs[param].pos = pos
|
|
for alg, pos in list(algPos.items()):
|
|
self.algs[alg].pos = pos
|
|
for alg, positions in list(outputsPos.items()):
|
|
for output, pos in list(positions.items()):
|
|
self.algs[alg].outputs[output].pos = pos
|
|
|
|
def prepareAlgorithm(self, alg):
|
|
algInstance = alg.algorithm
|
|
for param in algInstance.parameters:
|
|
if not param.hidden:
|
|
if param.name in alg.params:
|
|
value = self.resolveValue(alg.params[param.name], param)
|
|
else:
|
|
if iface is not None:
|
|
iface.messageBar().pushMessage(self.tr("Warning"),
|
|
self.tr("Parameter {0} in algorithm {1} in the model is run with default value! Edit the model to make sure that this is correct.").format(param.name, alg.name),
|
|
QgsMessageBar.WARNING, 4)
|
|
value = param.default
|
|
# We allow unexistent filepaths, since that allows
|
|
# algorithms to skip some conversion routines
|
|
if not param.setValue(value) and not isinstance(param,
|
|
ParameterDataObject):
|
|
raise GeoAlgorithmExecutionException(
|
|
self.tr('Wrong value {0} for {1} {2}', 'ModelerAlgorithm').format(
|
|
value, param.__class__.__name__, param.name
|
|
)
|
|
)
|
|
|
|
for out in algInstance.outputs:
|
|
if not out.hidden:
|
|
if out.name in alg.outputs:
|
|
name = self.getSafeNameForOutput(alg.name, out.name)
|
|
modelOut = self.getOutputFromName(name)
|
|
if modelOut:
|
|
out.value = modelOut.value
|
|
else:
|
|
out.value = None
|
|
|
|
return algInstance
|
|
|
|
def deactivateAlgorithm(self, algName):
|
|
dependent = self.getDependentAlgorithms(algName)
|
|
for alg in dependent:
|
|
self.algs[alg].active = False
|
|
|
|
def activateAlgorithm(self, algName):
|
|
parents = self.getDependsOnAlgorithms(algName)
|
|
for alg in parents:
|
|
if not self.algs[alg].active:
|
|
return False
|
|
self.algs[algName].active = True
|
|
return True
|
|
|
|
def getSafeNameForOutput(self, algName, outName):
|
|
return outName + '_ALG' + algName
|
|
|
|
def resolveValue(self, value, param):
|
|
if value is None:
|
|
v = None
|
|
if isinstance(value, list):
|
|
v = ";".join([self.resolveValue(v, param) for v in value])
|
|
elif isinstance(value, CompoundValue):
|
|
v = self.resolveValue(value.definition, param)
|
|
elif isinstance(value, ValueFromInput):
|
|
v = self.getParameterFromName(value.name).value
|
|
elif isinstance(value, ValueFromOutput):
|
|
v = self.algs[value.alg].algorithm.getOutputFromName(value.output).value
|
|
else:
|
|
v = value
|
|
return param.evaluateForModeler(v, self)
|
|
|
|
def processAlgorithm(self, feedback):
|
|
executed = []
|
|
toExecute = [alg for alg in list(self.algs.values()) if alg.active]
|
|
while len(executed) < len(toExecute):
|
|
for alg in toExecute:
|
|
if alg.name not in executed:
|
|
canExecute = True
|
|
required = self.getDependsOnAlgorithms(alg.name)
|
|
for requiredAlg in required:
|
|
if requiredAlg != alg.name and requiredAlg not in executed:
|
|
canExecute = False
|
|
break
|
|
if canExecute:
|
|
try:
|
|
feedback.pushDebugInfo(
|
|
self.tr('Prepare algorithm: {0}', 'ModelerAlgorithm').format(alg.name))
|
|
self.prepareAlgorithm(alg)
|
|
feedback.setProgressText(
|
|
self.tr('Running {0} [{1}/{2}]', 'ModelerAlgorithm').format(alg.description, len(executed) + 1, len(toExecute)))
|
|
feedback.pushDebugInfo('Parameters: ' + ', '.join([str(p).strip() +
|
|
'=' + str(p.value) for p in alg.algorithm.parameters]))
|
|
t0 = time.time()
|
|
alg.algorithm.execute(feedback, self)
|
|
dt = time.time() - t0
|
|
|
|
# copy algorithm output value(s) back to model in case the algorithm modified those
|
|
for out in alg.algorithm.outputs:
|
|
if not out.hidden:
|
|
if out.name in alg.outputs:
|
|
modelOut = self.getOutputFromName(self.getSafeNameForOutput(alg.name, out.name))
|
|
if modelOut:
|
|
modelOut.value = out.value
|
|
|
|
executed.append(alg.name)
|
|
feedback.pushDebugInfo(
|
|
self.tr('OK. Execution took %{0:.3f} ms ({1} outputs).', 'ModelerAlgorithm').format(dt, len(alg.algorithm.outputs)))
|
|
except GeoAlgorithmExecutionException as e:
|
|
feedback.pushDebugInfo(self.tr('Failed', 'ModelerAlgorithm'))
|
|
raise GeoAlgorithmExecutionException(
|
|
self.tr('Error executing algorithm {0}\n{1}', 'ModelerAlgorithm').format(alg.description, e.msg))
|
|
|
|
feedback.pushDebugInfo(
|
|
self.tr('Model processed ok. Executed {0} algorithms total', 'ModelerAlgorithm').format(len(executed)))
|
|
|
|
def getAsCommand(self):
|
|
if self.descriptionFile:
|
|
return GeoAlgorithm.getAsCommand(self)
|
|
else:
|
|
return None
|
|
|
|
def commandLineName(self):
|
|
if self.descriptionFile is None:
|
|
return ''
|
|
else:
|
|
return 'modeler:' + os.path.basename(self.descriptionFile)[:-6].lower()
|
|
|
|
def checkBeforeOpeningParametersDialog(self):
|
|
for alg in list(self.algs.values()):
|
|
algInstance = algList.getAlgorithm(alg.consoleName)
|
|
if algInstance is None:
|
|
return self.tr("The model you are trying to run contains an algorithm that is not available: <i>{0}</i>").format(alg.consoleName)
|
|
|
|
def setModelerView(self, dialog):
|
|
self.modelerdialog = dialog
|
|
|
|
def updateModelerView(self):
|
|
if self.modelerdialog:
|
|
self.modelerdialog.repaintModel()
|
|
|
|
def help(self):
|
|
try:
|
|
return True, getHtmlFromDescriptionsDict(self, self.helpContent)
|
|
except:
|
|
return False, None
|
|
|
|
def shortHelp(self):
|
|
if 'ALG_DESC' in self.helpContent:
|
|
return self._formatHelp(str(self.helpContent['ALG_DESC']))
|
|
return None
|
|
|
|
def getParameterDescriptions(self):
|
|
descs = {}
|
|
descriptions = self.helpContent
|
|
for param in self.parameters:
|
|
if param.name in descriptions:
|
|
descs[param.name] = str(descriptions[param.name])
|
|
return descs
|
|
|
|
def todict(self):
|
|
keys = ["inputs", "group", "name", "algs", "helpContent"]
|
|
return {k: v for k, v in list(self.__dict__.items()) if k in keys}
|
|
|
|
def toJson(self):
|
|
def todict(o):
|
|
if isinstance(o, QPointF):
|
|
return {"class": "point", "values": {"x": o.x(), "y": o.y()}}
|
|
try:
|
|
d = o.todict()
|
|
return {"class": o.__class__.__module__ + "." + o.__class__.__name__, "values": d}
|
|
except Exception:
|
|
pass
|
|
return json.dumps(self, default=todict, indent=4)
|
|
|
|
@staticmethod
|
|
def fromJson(s):
|
|
def fromdict(d):
|
|
try:
|
|
fullClassName = d["class"]
|
|
if isinstance(fullClassName, str):
|
|
tokens = fullClassName.split(".")
|
|
else:
|
|
tokens = fullClassName.__class__.__name__.split(".")
|
|
className = tokens[-1]
|
|
moduleName = ".".join(tokens[:-1])
|
|
values = d["values"]
|
|
if className == "point":
|
|
return QPointF(values["x"], values["y"])
|
|
|
|
def _import(name):
|
|
__import__(name)
|
|
return sys.modules[name]
|
|
|
|
if moduleName.startswith("processing.parameters"):
|
|
moduleName = "processing.core.parameters"
|
|
module = _import(moduleName)
|
|
clazz = getattr(module, className)
|
|
instance = clazz()
|
|
for k, v in list(values.items()):
|
|
instance.__dict__[k] = v
|
|
return instance
|
|
except KeyError:
|
|
return d
|
|
except Exception as e:
|
|
raise e
|
|
try:
|
|
model = json.loads(s, object_hook=fromdict)
|
|
except Exception as e:
|
|
raise WrongModelException(e.args[0])
|
|
return model
|
|
|
|
@staticmethod
|
|
def fromFile(filename):
|
|
with open(filename) as f:
|
|
s = f.read()
|
|
alg = ModelerAlgorithm.fromJson(s)
|
|
alg.descriptionFile = filename
|
|
return alg
|
|
|
|
def toPython(self):
|
|
s = ['##%s=name' % self.name]
|
|
for param in list(self.inputs.values()):
|
|
s.append(param.param.getAsScriptCode())
|
|
for alg in list(self.algs.values()):
|
|
for name, out in list(alg.outputs.items()):
|
|
s.append('##%s=%s' % (safeName(out.description).lower(), alg.getOutputType(name)))
|
|
|
|
executed = []
|
|
toExecute = [alg for alg in list(self.algs.values()) if alg.active]
|
|
while len(executed) < len(toExecute):
|
|
for alg in toExecute:
|
|
if alg.name not in executed:
|
|
canExecute = True
|
|
required = self.getDependsOnAlgorithms(alg.name)
|
|
for requiredAlg in required:
|
|
if requiredAlg != alg.name and requiredAlg not in executed:
|
|
canExecute = False
|
|
break
|
|
if canExecute:
|
|
s.extend(alg.toPython())
|
|
executed.append(alg.name)
|
|
|
|
return '\n'.join(s)
|
|
|
|
|
|
def safeName(name):
|
|
validChars = 'abcdefghijklmnopqrstuvwxyz'
|
|
return ''.join(c for c in name.lower() if c in validChars)
|