# -*- coding: utf-8 -*- """ *************************************************************************** ModelerAlgorithm.py --------------------- Date : August 2012 Copyright : (C) 2012 by Victor Olaya Email : volayaf at gmail dot com *************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * *************************************************************************** """ __author__ = 'Victor Olaya' __date__ = 'August 2012' __copyright__ = '(C) 2012, Victor Olaya' # This will get replaced with a git SHA1 when you do a git archive __revision__ = '$Format:%H$' import traceback import copy import os.path import codecs import time from PyQt4 import QtCore, QtGui from qgis.core import * from processing.core.GeoAlgorithm import GeoAlgorithm from processing.core.GeoAlgorithmExecutionException import \ GeoAlgorithmExecutionException from processing.gui.Help2Html import Help2Html from processing.modeler.WrongModelException import WrongModelException from processing.modeler.ModelerUtils import ModelerUtils from processing.parameters.ParameterFactory import ParameterFactory from processing.parameters.ParameterRaster import ParameterRaster from processing.parameters.ParameterDataObject import ParameterDataObject from processing.parameters.ParameterExtent import ParameterExtent from processing.parameters.ParameterMultipleInput import ParameterMultipleInput from processing.parameters.Parameter import Parameter from processing.parameters.ParameterVector import ParameterVector from processing.parameters.ParameterTableField import ParameterTableField from processing.outputs.OutputRaster import OutputRaster from processing.outputs.OutputHTML import OutputHTML from processing.outputs.OutputTable import OutputTable from processing.outputs.OutputVector import OutputVector from processing.outputs.OutputNumber import OutputNumber from processing.outputs.OutputString import OutputString from processing.tools import dataobjects class ModelerAlgorithm(GeoAlgorithm): CANVAS_SIZE = 4000 LINE_BREAK_STRING = '%%%' def getCopy(self): newone = ModelerAlgorithm() newone.openModel(self.descriptionFile) newone.provider = self.provider newone.deactivated = self.deactivated return newone def __init__(self): GeoAlgorithm.__init__(self) # The dialog where this model is being edited self.modelerdialog = None self.descriptionFile = None # Geoalgorithms in this model self.algs = [] # Parameters of Geoalgorithms in self.algs. Each entry is a # map with (paramname, paramvalue) values for algs[i]. # paramvalues are instances of AlgorithmAndParameter. self.algParameters = [] # Algorithms that each algorithm depends on. This is just a # list of dependencies not set by outputs and inputs but # explicitly entered instead, meaning that an algorithm must # 'wait' for another to finish. Each entry is a list with # algorithm indexes self.dependencies = [] # Outputs of Geoalgorithms in self.algs. Each entry is a map # with (output, outputvalue) values for algs[i]. outputvalue # is the name of the output if final. None if is an # intermediate output self.algOutputs = [] # Hardcoded parameter values entered by the user when defining # the model. Keys are value names. self.paramValues = {} # Position of items in canvas self.algPos = [] self.paramPos = [] self.outputPos = [] # same structure as algOutputs # Deactivated algorithms that should not be executed self.deactivated = [] def getIcon(self): return QtGui.QIcon(os.path.dirname(__file__) + '/../images/model.png') def openModel(self, filename): self.algPos = [] self.paramPos = [] self.outputOutputs = [] self.algs = [] self.algParameters = [] self.algOutputs = [] self.paramValues = {} self.dependencies = [] self.descriptionFile = filename lines = codecs.open(filename, 'r', encoding='utf-8') line = lines.readline().strip('\n').strip('\r') iAlg = 0 try: while line != '': if line.startswith('PARAMETER:'): paramLine = line[len('PARAMETER:'):] param = ParameterFactory.getFromString(paramLine) if param: self.parameters.append(param) else: raise WrongModelException('Error in parameter line: ' + line) line = lines.readline().strip('\n') tokens = line.split(',') self.paramPos.append(QtCore.QPointF(float(tokens[0]), float(tokens[1]))) elif line.startswith('VALUE:'): valueLine = line[len('VALUE:'):] tokens = valueLine.split('===') self.paramValues[tokens[0]] = \ tokens[1].replace(ModelerAlgorithm.LINE_BREAK_STRING, '\n') elif line.startswith('NAME:'): self.name = line[len('NAME:'):] elif line.startswith('GROUP:'): self.group = line[len('GROUP:'):] if self.group == '[Test models]': self.showInModeler = False self.showInToolbox = False elif line.startswith('ALGORITHM:'): algParams = {} algOutputs = {} algLine = line[len('ALGORITHM:'):] alg = ModelerUtils.getAlgorithm(algLine) if alg is not None: posline = lines.readline().strip('\n').strip('\r') tokens = posline.split(',') self.algPos.append(QtCore.QPointF(float(tokens[0]), float(tokens[1]))) self.algs.append(alg) dependenceline = lines.readline().strip('\n' ).strip('\r') dependencies = [] if dependenceline != str(None): for index in dependenceline.split(','): try: dependencies.append(int(index)) except: # A quick fix while I figure out # how to solve problems when # parsing this pass for param in alg.parameters: line = lines.readline().strip('\n').strip('\r') if line == str(None): algParams[param.name] = None else: tokens = line.split('|') algParams[param.name] = \ AlgorithmAndParameter(int(tokens[0]), tokens[1]) outputPos = {} for out in alg.outputs: line = lines.readline().strip('\n').strip('\r') if str(None) != line: if '|' in line: tokens = line.split('|') name = tokens[0] tokens = tokens[1].split(',') outputPos[out.name] = QtCore.QPointF( float(tokens[0]), float(tokens[1])) else: name = line outputPos[out.name] = None algOutputs[out.name] = name # We add the output to the algorithm, # with a name indicating where it comes # from that guarantees that the name is # unique output = copy.deepcopy(out) output.description = name output.name = self.getSafeNameForOutput(iAlg, output) self.addOutput(output) else: algOutputs[out.name] = None self.outputPos.append(outputPos) self.algOutputs.append(algOutputs) self.algParameters.append(algParams) self.dependencies.append(dependencies) iAlg += 1 else: raise WrongModelException('Error in algorithm name: ' + algLine) line = lines.readline().strip('\n').strip('\r') except Exception, e: if isinstance(e, WrongModelException): raise e else: raise WrongModelException('Error in model definition line:' + line.strip() + ' : ' + traceback.format_exc()) def addParameter(self, param): self.parameters.append(param) self.paramPos.append(self.getPositionForParameterItem()) def updateParameter(self, paramIndex, param): self.parameters[paramIndex] = param def addAlgorithm(self, alg, parametersMap, valuesMap, outputsMap, dependencies): self.algs.append(alg) self.algParameters.append(parametersMap) self.algOutputs.append(outputsMap) self.dependencies.append(dependencies) for value in valuesMap.keys(): self.paramValues[value] = valuesMap[value] algPos = self.getPositionForAlgorithmItem() self.algPos.append(algPos) pos = {} i = 0 from processing.modeler.ModelerGraphicItem import ModelerGraphicItem for out in outputsMap: pos[out] = algPos + QtCore.QPointF(ModelerGraphicItem.BOX_WIDTH, i * ModelerGraphicItem.BOX_HEIGHT) i += 1 self.outputPos.append(pos) def updateAlgorithm(self, algIndex, parametersMap, valuesMap, outputsMap, dependencies): self.algParameters[algIndex] = parametersMap self.algOutputs[algIndex] = outputsMap self.dependencies[algIndex] = dependencies for value in valuesMap.keys(): self.paramValues[value] = valuesMap[value] self.updateModelerView() algPos = self.algPos[algIndex] pos = {} i = 0 from processing.modeler.ModelerGraphicItem import ModelerGraphicItem for out in outputsMap: pos[out] = algPos + QtCore.QPointF(ModelerGraphicItem.BOX_WIDTH, i * ModelerGraphicItem.BOX_HEIGHT) i += 1 self.outputPos[algIndex] = pos def removeAlgorithm(self, index): """Returns True if the algorithm could be removed, False if others depend on it and could not be removed. """ if self.hasDependencies(self.algs[index], index): return False for out in self.algs[index].outputs: val = self.algOutputs[index][out.name] if val: name = self.getSafeNameForOutput(index, out) self.removeOutputFromName(name) del self.algs[index] del self.algParameters[index] del self.algOutputs[index] del self.algPos[index] del self.outputPos[index] i = -1 for paramValues in self.algParameters: i += 1 newValues = {} for (name, value) in paramValues.iteritems(): if value: if value.alg > index: newValues[name] = AlgorithmAndParameter(value.alg - 1, value.param, value.algName, value.paramName) else: newValues[name] = value else: newValues[name] = value self.algParameters[i] = newValues self.updateModelerView() return True def removeParameter(self, index): """Returns True if the parameter could be removed, False if others depend on it and could not be removed. """ if self.hasDependencies(self.parameters[index], index): return False del self.parameters[index] del self.paramPos[index] self.updateModelerView() return True def hasDependencies(self, element, elementIndex): """This method returns True if some other element depends on the passed one. """ if isinstance(element, Parameter): for alg in self.algParameters: for aap in alg.values(): if aap: if aap.alg == \ AlgorithmAndParameter.PARENT_MODEL_ALGORITHM: if aap.param == element.name: return True elif aap.param in self.paramValues: # Check for multiple inputs aap2 = self.paramValues[aap.param] if element.name in aap2: return True if isinstance(element, ParameterVector): for param in self.parameters: if isinstance(param, ParameterTableField): if param.parent == element.name: return True else: for alg in self.algParameters: for aap in alg.values(): if aap: if aap.alg == elementIndex: return True return False def deactivateAlgorithm(self, algIndex, update=False): if algIndex not in self.deactivated: dependent = self.getDependentAlgorithms(algIndex) self.deactivated.extend(dependent) if update: self.updateModelerView() def activateAlgorithm(self, algIndex, update=False): if algIndex in self.deactivated: dependsOn = self.getDependsOnAlgorithms(algIndex) for alg in dependsOn: if alg in self.deactivated and alg != algIndex: return False self.deactivated.remove(algIndex) dependent = self.getDependentAlgorithms(algIndex) for alg in dependent: if alg in self.deactivated: self.deactivated.remove(alg) if update: self.updateModelerView() return True def getDependsOnAlgorithms(self, algIndex): """This method returns a list with the indexes of algorithms a given one depends on. """ algs = [] algs.extend(self.dependencies[algIndex]) index = -1 for aap in self.algParameters[algIndex].values(): index += 1 if aap is not None: if aap.alg != AlgorithmAndParameter.PARENT_MODEL_ALGORITHM \ and aap.alg not in algs: algs.append(aap.alg) dep = self.getDependsOnAlgorithms(aap.alg) for alg in dep: if alg not in algs: algs.append(alg) return algs def getDependentAlgorithms(self, algIndex): """This method returns a list with the indexes of algorithms depending on a given one. """ dependent = [algIndex] index = -1 for alg in self.algParameters: index += 1 if index in dependent: continue for aap in alg.values(): if aap is not None: if aap.alg == algIndex: dep = self.getDependentAlgorithms(index) for alg in dep: if alg not in dependent: dependent.append(alg) break index = -1 for dep in self.dependencies: index += 1 if algIndex in dep: dep = self.getDependentAlgorithms(index) for alg in dep: if alg not in dependent: dependent.append(alg) return dependent def getPositionForAlgorithmItem(self): MARGIN = 20 BOX_WIDTH = 200 BOX_HEIGHT = 80 if len(self.algPos) != 0: maxX = max([pos.x() for pos in self.algPos]) maxY = max([pos.y() for pos in self.algPos]) newX = min(MARGIN + BOX_WIDTH + maxX, self.CANVAS_SIZE - BOX_WIDTH) newY = min(MARGIN + BOX_HEIGHT + maxY, self.CANVAS_SIZE - BOX_HEIGHT) else: newX = MARGIN + BOX_WIDTH / 2 newY = MARGIN * 2 + BOX_HEIGHT + BOX_HEIGHT / 2 return QtCore.QPointF(newX, newY) def getPositionForParameterItem(self): MARGIN = 20 BOX_WIDTH = 200 BOX_HEIGHT = 80 if len(self.paramPos) != 0: maxX = max([pos.x() for pos in self.paramPos]) newX = min(MARGIN + BOX_WIDTH + maxX, self.CANVAS_SIZE - BOX_WIDTH) else: newX = MARGIN + BOX_WIDTH / 2 return QtCore.QPointF(newX, MARGIN + BOX_HEIGHT / 2) def serialize(self): s = 'NAME:' + unicode(self.name) + '\n' s += 'GROUP:' + unicode(self.group) + '\n' i = 0 for param in self.parameters: s += 'PARAMETER:' + param.serialize() + '\n' pt = self.paramPos[i] s += str(pt.x()) + ',' + str(pt.y()) + '\n' i += 1 for key in self.paramValues.keys(): s += 'VALUE:' + key + '===' \ + str(self.paramValues[key]).replace('\n', ModelerAlgorithm.LINE_BREAK_STRING) + '\n' for i in range(len(self.algs)): alg = self.algs[i] s += 'ALGORITHM:' + alg.commandLineName() + '\n' pt = self.algPos[i] s += str(pt.x()) + ',' + str(pt.y()) + '\n' if len(self.dependencies[i]) != 0: s += ','.join([str(index) for index in self.dependencies[i]]) \ + '\n' else: s += str(None) + '\n' for param in alg.parameters: value = self.algParameters[i][param.name] if value: s += value.serialize() + '\n' else: s += str(None) + '\n' for out in alg.outputs: value = self.algOutputs[i][out.name] s += unicode(value) if value is not None: pt = self.outputPos[i][out.name] s += '|' + str(pt.x()) + ',' + str(pt.y()) s += '\n' return s def setPositions(self, paramPos, algPos, outputPos): self.paramPos = paramPos self.algPos = algPos self.outputPos = outputPos def prepareAlgorithm(self, alg, iAlg): for param in alg.parameters: aap = self.algParameters[iAlg][param.name] if aap is None: if isinstance(param, ParameterExtent): value = self.getMinCoveringExtent() if not param.setValue(value): raise GeoAlgorithmExecutionException('Wrong value: ' + str(value)) else: param.setValue(None) continue if isinstance(param, ParameterMultipleInput): value = self.getValueFromAlgorithmAndParameter(aap) tokens = value.split(';') layerslist = [] for token in tokens: (i, paramname) = token.split('|') aap = AlgorithmAndParameter(int(i), paramname) value = self.getValueFromAlgorithmAndParameter(aap) layerslist.append(str(value)) value = ';'.join(layerslist) else: value = self.getValueFromAlgorithmAndParameter(aap) # We allow unexistent filepaths, since that allows # algorithms to skip some conversion routines if not param.setValue(value) and not isinstance(param, ParameterDataObject): raise GeoAlgorithmExecutionException('Wrong value: ' + str(value)) for out in alg.outputs: val = self.algOutputs[iAlg][out.name] if val: name = self.getSafeNameForOutput(iAlg, out) out.value = self.getOutputFromName(name).value else: out.value = None def getMinCoveringExtent(self): first = True found = False for param in self.parameters: if param.value: if isinstance(param, (ParameterRaster, ParameterVector)): found = True if isinstance(param.value, (QgsRasterLayer, QgsVectorLayer)): layer = param.value else: layer = dataobjects.getObjectFromUri(param.value) self.addToRegion(layer, first) first = False elif isinstance(param, ParameterMultipleInput): found = True layers = param.value.split(';') for layername in layers: layer = dataobjects.getObjectFromUri(layername) self.addToRegion(layer, first) first = False if found: return str(self.xmin) + ',' + str(self.xmax) + ',' \ + str(self.ymin) + ',' + str(self.ymax) else: return None def addToRegion(self, layer, first): if first: self.xmin = layer.extent().xMinimum() self.xmax = layer.extent().xMaximum() self.ymin = layer.extent().yMinimum() self.ymax = layer.extent().yMaximum() else: self.xmin = min(self.xmin, layer.extent().xMinimum()) self.xmax = max(self.xmax, layer.extent().xMaximum()) self.ymin = min(self.ymin, layer.extent().yMinimum()) self.ymax = max(self.ymax, layer.extent().yMaximum()) def getSafeNameForOutput(self, ialg, out): return out.name + '_ALG' + str(ialg) def getValueFromAlgorithmAndParameter(self, aap): if aap is None: return None if float(aap.alg) \ == float(AlgorithmAndParameter.PARENT_MODEL_ALGORITHM): for key in self.paramValues.keys(): if aap.param == key: return self.paramValues[key] for param in self.parameters: if aap.param == param.name: return param.value else: return self.producedOutputs[int(aap.alg)][aap.param] def processAlgorithm(self, progress): self.producedOutputs = {} executed = [] while len(executed) < len(self.algs) - len(self.deactivated): iAlg = 0 for alg in self.algs: if iAlg not in self.deactivated and iAlg not in executed: canExecute = True required = self.getDependsOnAlgorithms(iAlg) for requiredAlg in required: if requiredAlg != iAlg and requiredAlg not in executed: canExecute = False break if canExecute: try: alg = alg.getCopy() progress.setDebugInfo('Prepare algorithm %i: %s' % (iAlg, alg.name)) self.prepareAlgorithm(alg, iAlg) progress.setText('Running ' + alg.name + ' [' + str(iAlg + 1) + '/' + str(len(self.algs) - len(self.deactivated)) + ']') outputs = {} progress.setDebugInfo('Parameters: ' + ', '.join([unicode(p).strip() + '=' + unicode(p.value) for p in alg.parameters])) t0 = time.time() alg.execute(progress, self) dt = time.time() - t0 for out in alg.outputs: outputs[out.name] = out.value progress.setDebugInfo('Outputs: ' + ', '.join([unicode(out).strip() + '=' + unicode(outputs[out.name]) for out in alg.outputs])) self.producedOutputs[iAlg] = outputs executed.append(iAlg) progress.setDebugInfo( 'OK. Execution took %0.3f ms (%i outputs).' % (dt, len(outputs))) except GeoAlgorithmExecutionException, e: progress.setDebugInfo('Failed') raise GeoAlgorithmExecutionException( 'Error executing algorithm ' + str(iAlg) + '\n' + e.msg) else: pass iAlg += 1 progress.setDebugInfo( 'Model processed ok. Executed %i algorithms total' % iAlg) def getOutputType(self, i, outname): for out in self.algs[i].outputs: if out.name == outname: if isinstance(out, OutputRaster): return 'output raster' elif isinstance(out, OutputVector): return 'output vector' elif isinstance(out, OutputTable): return 'output table' elif isinstance(out, OutputHTML): return 'output html' elif isinstance(out, OutputNumber): return 'output number' elif isinstance(out, OutputString): return 'output string' def getAsPythonCode(self): s = [] for param in self.parameters: s.append(str(param.getAsScriptCode().lower())) i = 0 for outs in self.algOutputs: for out in outs.keys(): if outs[out]: s.append('##' + out.lower() + '_alg' + str(i) + '=' + self.getOutputType(i, out)) i += 1 i = 0 iMultiple = 0 for alg in self.algs: multiple = [] runline = 'outputs_' + str(i) + '=Processing.runalg("' \ + alg.commandLineName() + '"' for param in alg.parameters: aap = self.algParameters[i][param.name] if aap is None: runline += ', None' elif isinstance(param, ParameterMultipleInput): value = self.paramValues[aap.param] tokens = value.split(';') layerslist = [] for token in tokens: (iAlg, paramname) = token.split('|') if float(iAlg) == float( AlgorithmAndParameter.PARENT_MODEL_ALGORITHM): if self.ismodelparam(paramname): value = paramname.lower() else: value = self.paramValues[paramname] else: value = 'outputs_' + str(iAlg) + "['" + paramname \ + "']" layerslist.append(str(value)) multiple.append('multiple_' + str(iMultiple) + '=[' + ','.join(layerslist) + ']') runline += ', ";".join(multiple_' + str(iMultiple) + ') ' else: if float(aap.alg) == float( AlgorithmAndParameter.PARENT_MODEL_ALGORITHM): if self.ismodelparam(aap.param): runline += ', ' + aap.param.lower() else: runline += ', ' + str(self.paramValues[aap.param]) else: runline += ', outputs_' + str(aap.alg) + "['" \ + aap.param + "']" for out in alg.outputs: value = self.algOutputs[i][out.name] if value: name = out.name.lower() + '_alg' + str(i) else: name = str(None) runline += ', ' + name i += 1 s += multiple s.append(str(runline + ')')) return '\n'.join(s) def ismodelparam(self, paramname): for modelparam in self.parameters: if modelparam.name == paramname: return True return False def getAsCommand(self): if self.descriptionFile: return GeoAlgorithm.getAsCommand(self) else: return None def commandLineName(self): return 'modeler:' + os.path.basename(self.descriptionFile)[:-6].lower() def setModelerView(self, dialog): self.modelerdialog = dialog def updateModelerView(self): if self.modelerdialog: self.modelerdialog.repaintModel() def helpFile(self): helpfile = self.descriptionFile + '.help' if os.path.exists(helpfile): h2h = Help2Html() return h2h.getHtmlFile(self, helpfile) else: return None class AlgorithmAndParameter: PARENT_MODEL_ALGORITHM = -1 # alg is the index of the algorithm in the list in # ModelerAlgorithm.algs. # -1 if the value is not taken from the output of an algorithm, # but from an input of the model or a hardcoded value. # Names are just used for decoration, and are not needed to # create a hardcoded value. def __init__(self, alg, param, algName='', paramName=''): self.alg = alg self.param = param self.algName = algName self.paramName = paramName def serialize(self): return str(self.alg) + '|' + str(self.param) def name(self): if self.alg != AlgorithmAndParameter.PARENT_MODEL_ALGORITHM: return self.paramName + ' from algorithm ' + str(self.alg) + '(' \ + self.algName + ')' else: return self.paramName def __str__(self): return str(self.alg) + '|' + str(self.param)