QGIS/python/plugins/processing/gui/BatchPanel.py
Nyall Dawson 08e850db38 [processing] In batch mode, don't try to run rows with invalid parameter values
Previously we would show a warning about these, but then go ahead and try
to run the row anyway (using an empty set of parameters), resulting in
the log being filled with confusing error messages.

Instead, keep the existing warning advising about which values are
invalid, but skip the affected row and don't try to run it at all.
2020-03-19 15:42:13 +10:00

628 lines
25 KiB
Python

# -*- coding: utf-8 -*-
"""
***************************************************************************
BatchPanel.py
---------------------
Date : November 2014
Copyright : (C) 2014 by Alexander Bruy
Email : alexander dot bruy at gmail dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************
"""
__author__ = 'Alexander Bruy'
__date__ = 'November 2014'
__copyright__ = '(C) 2014, Alexander Bruy'
import os
import json
import warnings
from qgis.PyQt import uic
from qgis.PyQt.QtWidgets import (
QTableWidgetItem,
QComboBox,
QHeaderView,
QFileDialog,
QMessageBox,
QToolButton,
QMenu,
QAction
)
from qgis.PyQt.QtGui import QPalette
from qgis.PyQt.QtCore import (
QDir,
QFileInfo,
QCoreApplication
)
from qgis.core import (
Qgis,
QgsApplication,
QgsSettings,
QgsProperty, # NOQA - must be here for saved file evaluation
QgsProject,
QgsProcessingFeatureSourceDefinition, # NOQA - must be here for saved file evaluation
QgsCoordinateReferenceSystem, # NOQA - must be here for saved file evaluation
QgsProcessingParameterDefinition,
QgsProcessingModelAlgorithm,
QgsProcessingParameterFile,
QgsProcessingParameterMapLayer,
QgsProcessingParameterRasterLayer,
QgsProcessingParameterMeshLayer,
QgsProcessingParameterVectorLayer,
QgsProcessingParameterFeatureSource,
QgsProcessingParameterRasterDestination,
QgsProcessingParameterVectorDestination,
QgsProcessingParameterFeatureSink,
QgsProcessingOutputLayerDefinition,
QgsExpressionContextUtils,
QgsExpression
)
from qgis.gui import (
QgsProcessingParameterWidgetContext,
QgsProcessingContextGenerator,
QgsFindFilesByPatternDialog,
QgsExpressionBuilderDialog
)
from qgis.utils import iface
from processing.gui.wrappers import WidgetWrapperFactory, WidgetWrapper
from processing.gui.BatchOutputSelectionPanel import BatchOutputSelectionPanel
from processing.tools import dataobjects
from processing.tools.dataobjects import createContext
pluginPath = os.path.split(os.path.dirname(__file__))[0]
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
WIDGET, BASE = uic.loadUiType(
os.path.join(pluginPath, 'ui', 'widgetBatchPanel.ui'))
class BatchPanelFillWidget(QToolButton):
def __init__(self, parameterDefinition, column, panel, parent=None):
super().__init__(parent)
self.setBackgroundRole(QPalette.Button)
self.setAutoFillBackground(True)
self.parameterDefinition = parameterDefinition
self.column = column
self.panel = panel
self.setText(QCoreApplication.translate('BatchPanel', 'Autofill…'))
f = self.font()
f.setItalic(True)
self.setFont(f)
self.setPopupMode(QToolButton.InstantPopup)
self.setAutoRaise(True)
self.menu = QMenu()
self.menu.aboutToShow.connect(self.createMenu)
self.setMenu(self.menu)
def createMenu(self):
self.menu.clear()
self.menu.setMinimumWidth(self.width())
fill_down_action = QAction(self.tr('Fill Down'), self.menu)
fill_down_action.triggered.connect(self.fillDown)
fill_down_action.setToolTip(self.tr('Copy the first value down to all other rows'))
self.menu.addAction(fill_down_action)
calculate_by_expression = QAction(QCoreApplication.translate('BatchPanel', 'Calculate by Expression…'),
self.menu)
calculate_by_expression.setIcon(QgsApplication.getThemeIcon('/mActionCalculateField.svg'))
calculate_by_expression.triggered.connect(self.calculateByExpression)
calculate_by_expression.setToolTip(self.tr('Calculates parameter values by evaluating an expression'))
self.menu.addAction(calculate_by_expression)
add_by_expression = QAction(QCoreApplication.translate('BatchPanel', 'Add Values by Expression…'),
self.menu)
add_by_expression.triggered.connect(self.addByExpression)
add_by_expression.setToolTip(self.tr('Adds new parameter values by evaluating an expression'))
self.menu.addAction(add_by_expression)
if isinstance(self.parameterDefinition, (QgsProcessingParameterFile,
QgsProcessingParameterMapLayer,
QgsProcessingParameterRasterLayer,
QgsProcessingParameterMeshLayer,
QgsProcessingParameterVectorLayer,
QgsProcessingParameterFeatureSource)):
self.menu.addSeparator()
find_by_pattern_action = QAction(QCoreApplication.translate('BatchPanel', 'Add Files by Pattern…'),
self.menu)
find_by_pattern_action.triggered.connect(self.addFilesByPattern)
find_by_pattern_action.setToolTip(self.tr('Adds files by a file pattern match'))
self.menu.addAction(find_by_pattern_action)
def fillDown(self):
"""
Copy the top value down
"""
context = dataobjects.createContext()
wrapper = self.panel.wrappers[0][self.column]
if wrapper is None:
# e.g. double clicking on a destination header
widget = self.panel.tblParameters.cellWidget(1, self.column)
value = widget.getValue()
else:
value = wrapper.parameterValue()
for row in range(1, self.panel.batchRowCount()):
self.setRowValue(row, value, context)
def setRowValue(self, row, value, context):
"""
Sets the value for a row, in the current column
"""
if self.panel.batchRowCount() <= row:
self.panel.addRow()
wrapper = self.panel.wrappers[row][self.column]
if wrapper is None:
# e.g. destination header
self.panel.tblParameters.cellWidget(row + 1, self.column).setValue(str(value))
else:
wrapper.setParameterValue(value, context)
def addFilesByPattern(self):
"""
Populates the dialog using a file pattern match
"""
dlg = QgsFindFilesByPatternDialog()
dlg.setWindowTitle(self.tr("Add Files by Pattern"))
if dlg.exec_():
files = dlg.files()
context = dataobjects.createContext()
first_row = self.panel.batchRowCount() if self.panel.batchRowCount() > 1 else 0
for row, file in enumerate(files):
self.setRowValue(first_row + row, file, context)
def calculateByExpression(self):
"""
Calculates parameter values by evaluating expressions.
"""
self.populateByExpression(adding=False)
def addByExpression(self):
"""
Adds new parameter values by evaluating an expression
"""
self.populateByExpression(adding=True)
def populateByExpression(self, adding=False):
"""
Populates the panel using an expression
"""
context = dataobjects.createContext()
expression_context = context.expressionContext()
# use the first row parameter values as a preview during expression creation
params, ok = self.panel.parametersForRow(0, warnOnInvalid=False)
alg_scope = QgsExpressionContextUtils.processingAlgorithmScope(self.panel.alg, params, context)
# create explicit variables corresponding to every parameter
for k, v in params.items():
alg_scope.setVariable(k, v, True)
# add batchCount in the alg scope to be used in the expressions. 0 is only an example value
alg_scope.setVariable('row_number', 0, False)
expression_context.appendScope(alg_scope)
# mark the parameter variables as highlighted for discoverability
highlighted_vars = expression_context.highlightedVariables()
highlighted_vars.extend(list(params.keys()))
highlighted_vars.append('row_number')
expression_context.setHighlightedVariables(highlighted_vars)
dlg = QgsExpressionBuilderDialog(layer=None, context=context.expressionContext())
if adding:
dlg.setExpectedOutputFormat(self.tr('An array of values corresponding to each new row to add'))
if not dlg.exec_():
return
if adding:
exp = QgsExpression(dlg.expressionText())
res = exp.evaluate(expression_context)
if type(res) is not list:
res = [res]
first_row = self.panel.batchRowCount() if self.panel.batchRowCount() > 1 else 0
for row, value in enumerate(res):
self.setRowValue(row + first_row, value, context)
else:
for row in range(self.panel.batchRowCount()):
params, ok = self.panel.parametersForRow(row, warnOnInvalid=False)
# remove previous algorithm scope -- we need to rebuild this completely, using the
# other parameter values from the current row
expression_context.popScope()
alg_scope = QgsExpressionContextUtils.processingAlgorithmScope(self.panel.alg, params, context)
for k, v in params.items():
alg_scope.setVariable(k, v, True)
# add batch row number as evaluable variable in algorithm scope
alg_scope.setVariable('row_number', row, False)
expression_context.appendScope(alg_scope)
# rebuild a new expression every time -- we don't want the expression compiler to replace
# variables with precompiled values
exp = QgsExpression(dlg.expressionText())
value = exp.evaluate(expression_context)
self.setRowValue(row, value, context)
class BatchPanel(BASE, WIDGET):
PARAMETERS = "PARAMETERS"
OUTPUTS = "OUTPUTS"
def __init__(self, parent, alg):
super(BatchPanel, self).__init__(None)
self.setupUi(self)
self.wrappers = []
self.btnAdvanced.hide()
# Set icons
self.btnAdd.setIcon(QgsApplication.getThemeIcon('/symbologyAdd.svg'))
self.btnRemove.setIcon(QgsApplication.getThemeIcon('/symbologyRemove.svg'))
self.btnOpen.setIcon(QgsApplication.getThemeIcon('/mActionFileOpen.svg'))
self.btnSave.setIcon(QgsApplication.getThemeIcon('/mActionFileSave.svg'))
self.btnAdvanced.setIcon(QgsApplication.getThemeIcon("/processingAlgorithm.svg"))
self.alg = alg
self.parent = parent
self.btnAdd.clicked.connect(self.addRow)
self.btnRemove.clicked.connect(self.removeRows)
self.btnOpen.clicked.connect(self.load)
self.btnSave.clicked.connect(self.save)
self.btnAdvanced.toggled.connect(self.toggleAdvancedMode)
self.tblParameters.horizontalHeader().resizeSections(QHeaderView.ResizeToContents)
self.tblParameters.horizontalHeader().setDefaultSectionSize(250)
self.tblParameters.horizontalHeader().setMinimumSectionSize(150)
self.processing_context = createContext()
class ContextGenerator(QgsProcessingContextGenerator):
def __init__(self, context):
super().__init__()
self.processing_context = context
def processingContext(self):
return self.processing_context
self.context_generator = ContextGenerator(self.processing_context)
self.column_to_parameter_definition = {}
self.parameter_to_column = {}
self.initWidgets()
def layerRegistryChanged(self):
pass
def initWidgets(self):
# If there are advanced parameters — show corresponding button
for param in self.alg.parameterDefinitions():
if param.flags() & QgsProcessingParameterDefinition.FlagAdvanced:
self.btnAdvanced.show()
break
# Determine column count
self.tblParameters.setColumnCount(
self.alg.countVisibleParameters())
# Table headers
column = 0
for param in self.alg.parameterDefinitions():
if param.isDestination():
continue
self.tblParameters.setHorizontalHeaderItem(
column, QTableWidgetItem(param.description()))
if param.flags() & QgsProcessingParameterDefinition.FlagAdvanced:
self.tblParameters.setColumnHidden(column, True)
self.column_to_parameter_definition[column] = param.name()
self.parameter_to_column[param.name()] = column
column += 1
for out in self.alg.destinationParameterDefinitions():
if not out.flags() & QgsProcessingParameterDefinition.FlagHidden:
self.tblParameters.setHorizontalHeaderItem(
column, QTableWidgetItem(out.description()))
self.column_to_parameter_definition[column] = out.name()
self.parameter_to_column[out.name()] = column
column += 1
self.addFillRow()
# Add an empty row to begin
self.addRow()
self.tblParameters.horizontalHeader().resizeSections(QHeaderView.ResizeToContents)
self.tblParameters.verticalHeader().setSectionResizeMode(QHeaderView.ResizeToContents)
self.tblParameters.horizontalHeader().setStretchLastSection(True)
def batchRowCount(self):
"""
Returns the number of rows corresponding to execution iterations
"""
return len(self.wrappers)
def clear(self):
self.tblParameters.setRowCount(1)
self.wrappers = []
self.column_to_parameter_definition = {}
self.parameter_to_column = {}
def load(self):
context = dataobjects.createContext()
settings = QgsSettings()
last_path = settings.value("/Processing/LastBatchPath", QDir.homePath())
filename, selected_filter = QFileDialog.getOpenFileName(self,
self.tr('Open Batch'), last_path,
self.tr('JSON files (*.json)'))
if filename:
last_path = QFileInfo(filename).path()
settings.setValue('/Processing/LastBatchPath', last_path)
with open(filename) as f:
values = json.load(f)
else:
# If the user clicked on the cancel button.
return
self.clear()
try:
for row, alg in enumerate(values):
self.addRow()
params = alg[self.PARAMETERS]
outputs = alg[self.OUTPUTS]
column = 0
for param in self.alg.parameterDefinitions():
if param.flags() & QgsProcessingParameterDefinition.FlagHidden:
continue
if param.isDestination():
continue
if param.name() in params:
value = eval(params[param.name()])
wrapper = self.wrappers[row][column]
wrapper.setParameterValue(value, context)
column += 1
for out in self.alg.destinationParameterDefinitions():
if out.flags() & QgsProcessingParameterDefinition.FlagHidden:
continue
if out.name() in outputs:
value = outputs[out.name()].strip("'")
widget = self.tblParameters.cellWidget(row + 1, column)
widget.setValue(value)
column += 1
except TypeError:
QMessageBox.critical(
self,
self.tr('Error'),
self.tr('An error occurred while reading your file.'))
def save(self):
toSave = []
context = dataobjects.createContext()
for row in range(self.batchRowCount()):
algParams = {}
algOutputs = {}
col = 0
alg = self.alg
for param in alg.parameterDefinitions():
if param.flags() & QgsProcessingParameterDefinition.FlagHidden:
continue
if param.isDestination():
continue
wrapper = self.wrappers[row][col]
# For compatibility with 3.x API, we need to check whether the wrapper is
# the deprecated WidgetWrapper class. If not, it's the newer
# QgsAbstractProcessingParameterWidgetWrapper class
# TODO QGIS 4.0 - remove
if issubclass(wrapper.__class__, WidgetWrapper):
widget = wrapper.widget
else:
widget = wrapper.wrappedWidget()
value = wrapper.parameterValue()
if not param.checkValueIsAcceptable(value, context):
self.parent.messageBar().pushMessage("", self.tr(
'Wrong or missing parameter value: {0} (row {1})').format(
param.description(), row + 1),
level=Qgis.Warning, duration=5)
return
algParams[param.name()] = param.valueAsPythonString(value, context)
col += 1
for out in alg.destinationParameterDefinitions():
if out.flags() & QgsProcessingParameterDefinition.FlagHidden:
continue
widget = self.tblParameters.cellWidget(row + 1, col)
text = widget.getValue()
if text.strip() != '':
algOutputs[out.name()] = text.strip()
col += 1
else:
self.parent.messageBar().pushMessage("",
self.tr('Wrong or missing output value: {0} (row {1})').format(
out.description(), row + 1),
level=Qgis.Warning, duration=5)
return
toSave.append({self.PARAMETERS: algParams, self.OUTPUTS: algOutputs})
settings = QgsSettings()
last_path = settings.value("/Processing/LastBatchPath", QDir.homePath())
filename, __ = QFileDialog.getSaveFileName(self,
self.tr('Save Batch'),
last_path,
self.tr('JSON files (*.json)'))
if filename:
if not filename.endswith('.json'):
filename += '.json'
last_path = QFileInfo(filename).path()
settings.setValue('/Processing/LastBatchPath', last_path)
with open(filename, 'w') as f:
json.dump(toSave, f)
def setCellWrapper(self, row, column, wrapper, context):
self.wrappers[row - 1][column] = wrapper
widget_context = QgsProcessingParameterWidgetContext()
widget_context.setProject(QgsProject.instance())
if iface is not None:
widget_context.setMapCanvas(iface.mapCanvas())
widget_context.setMessageBar(self.parent.messageBar())
if isinstance(self.alg, QgsProcessingModelAlgorithm):
widget_context.setModel(self.alg)
wrapper.setWidgetContext(widget_context)
wrapper.registerProcessingContextGenerator(self.context_generator)
# For compatibility with 3.x API, we need to check whether the wrapper is
# the deprecated WidgetWrapper class. If not, it's the newer
# QgsAbstractProcessingParameterWidgetWrapper class
# TODO QGIS 4.0 - remove
is_cpp_wrapper = not issubclass(wrapper.__class__, WidgetWrapper)
if is_cpp_wrapper:
widget = wrapper.createWrappedWidget(context)
else:
widget = wrapper.widget
self.tblParameters.setCellWidget(row, column, widget)
def addFillRow(self):
self.tblParameters.setRowCount(1)
for col, name in self.column_to_parameter_definition.items():
param_definition = self.alg.parameterDefinition(self.column_to_parameter_definition[col])
self.tblParameters.setCellWidget(0, col, BatchPanelFillWidget(param_definition, col, self))
def addRow(self):
self.wrappers.append([None] * self.tblParameters.columnCount())
self.tblParameters.setRowCount(self.tblParameters.rowCount() + 1)
context = dataobjects.createContext()
wrappers = {}
row = self.tblParameters.rowCount() - 1
column = 0
for param in self.alg.parameterDefinitions():
if param.flags() & QgsProcessingParameterDefinition.FlagHidden or param.isDestination():
continue
wrapper = WidgetWrapperFactory.create_wrapper(param, self.parent, row, column)
wrappers[param.name()] = wrapper
self.setCellWrapper(row, column, wrapper, context)
column += 1
for out in self.alg.destinationParameterDefinitions():
if out.flags() & QgsProcessingParameterDefinition.FlagHidden:
continue
self.tblParameters.setCellWidget(
row, column, BatchOutputSelectionPanel(
out, self.alg, row, column, self))
column += 1
if len(self.alg.destinationParameterDefinitions()) > 0:
item = QComboBox()
item.addItem(self.tr('Yes'))
item.addItem(self.tr('No'))
item.setCurrentIndex(0)
self.tblParameters.setCellWidget(row, column, item)
for wrapper in list(wrappers.values()):
wrapper.postInitialize(list(wrappers.values()))
def removeRows(self):
rows = set()
for index in self.tblParameters.selectedIndexes():
if index.row() == 0:
continue
rows.add(index.row())
for row in sorted(rows, reverse=True):
if self.tblParameters.rowCount() <= 2:
break
del self.wrappers[row - 1]
self.tblParameters.removeRow(row)
def toggleAdvancedMode(self, checked):
for param in self.alg.parameterDefinitions():
if param.flags() & QgsProcessingParameterDefinition.FlagAdvanced:
self.tblParameters.setColumnHidden(self.parameter_to_column[param.name()], not checked)
def valueForParameter(self, row, parameter_name):
"""
Returns the current value for a parameter in a row
"""
wrapper = self.wrappers[row][self.parameter_to_column[parameter_name]]
return wrapper.parameterValue()
def parametersForRow(self, row, destinationProject=None, warnOnInvalid=True):
"""
Returns the parameters dictionary corresponding to a row in the batch table
"""
col = 0
parameters = {}
for param in self.alg.parameterDefinitions():
if param.flags() & QgsProcessingParameterDefinition.FlagHidden or param.isDestination():
continue
wrapper = self.wrappers[row][col]
parameters[param.name()] = wrapper.parameterValue()
if warnOnInvalid and not param.checkValueIsAcceptable(wrapper.parameterValue()):
self.parent.messageBar().pushMessage("",
self.tr('Wrong or missing parameter value: {0} (row {1})').format(
param.description(), row + 1),
level=Qgis.Warning, duration=5)
return {}, False
col += 1
count_visible_outputs = 0
for out in self.alg.destinationParameterDefinitions():
if out.flags() & QgsProcessingParameterDefinition.FlagHidden:
continue
count_visible_outputs += 1
widget = self.tblParameters.cellWidget(row + 1, col)
text = widget.getValue()
if not warnOnInvalid or out.checkValueIsAcceptable(text):
if isinstance(out, (QgsProcessingParameterRasterDestination,
QgsProcessingParameterVectorDestination,
QgsProcessingParameterFeatureSink)):
# load rasters and sinks on completion
parameters[out.name()] = QgsProcessingOutputLayerDefinition(text, destinationProject)
else:
parameters[out.name()] = text
col += 1
else:
self.parent.messageBar().pushMessage("", self.tr('Wrong or missing output value: {0} (row {1})').format(
out.description(), row + 1),
level=Qgis.Warning, duration=5)
return {}, False
return parameters, True