[processing] update script provider to use QgsProcessingAlgorithm

subclasses instead of custom script code

python/plugins/processing/algs/qgis/QgisAlgorithmProvider.py

@@ -36,7 +36,7 @@ except:
 from qgis.core import (QgsApplication,
                        QgsProcessingProvider)
 
-from processing.script.ScriptUtils import ScriptUtils
+from processing.script import ScriptUtils
 
 from .QgisAlgorithm import QgisAlgorithm
 
@@ -288,11 +288,11 @@ class QgisAlgorithmProvider(QgsProcessingProvider):
                          VectorLayerScatterplot3D()])
 
         # to store algs added by 3rd party plugins as scripts
-        folder = os.path.join(os.path.dirname(__file__), 'scripts')
+        #folder = os.path.join(os.path.dirname(__file__), 'scripts')
-        scripts = ScriptUtils.loadFromFolder(folder)
+        #scripts = ScriptUtils.loadFromFolder(folder)
-        for script in scripts:
+        #for script in scripts:
-            script.allowEdit = False
+        #    script.allowEdit = False
-        algs.extend(scripts)
+        #algs.extend(scripts)
 
         return algs
 

python/plugins/processing/core/Processing.py

@@ -45,12 +45,12 @@ from qgis.core import (QgsMessageLog,
                        QgsProcessingOutputMapLayer)
 
 import processing
-from processing.script.ScriptUtils import ScriptUtils
 from processing.core.ProcessingConfig import ProcessingConfig
 from processing.gui.MessageBarProgress import MessageBarProgress
 from processing.gui.RenderingStyles import RenderingStyles
 from processing.gui.Postprocessing import handleAlgorithmResults
 from processing.gui.AlgorithmExecutor import execute
+from processing.script import ScriptUtils
 from processing.tools import dataobjects
 
 from processing.algs.qgis.QgisAlgorithmProvider import QgisAlgorithmProvider  # NOQA
@@ -105,25 +105,25 @@ class Processing(object):
 
     @staticmethod
     def addScripts(folder):
-        Processing.initialize()
+        #Processing.initialize()
-        provider = QgsApplication.processingRegistry().providerById("qgis")
+        #provider = QgsApplication.processingRegistry().providerById("qgis")
-        scripts = ScriptUtils.loadFromFolder(folder)
+        #scripts = ScriptUtils.loadFromFolder(folder)
-        # fix_print_with_import
+        #for script in scripts:
-        print(scripts)
+        #    script.allowEdit = False
-        for script in scripts:
+        #    script._icon = provider.icon()
-            script.allowEdit = False
+        #provider.externalAlgs.extend(scripts)
-            script._icon = provider.icon()
+        #provider.refreshAlgorithms()
-        provider.externalAlgs.extend(scripts)
+        pass
-        provider.refreshAlgorithms()
 
     @staticmethod
     def removeScripts(folder):
-        provider = QgsApplication.processingRegistry().providerById("qgis")
+        #provider = QgsApplication.processingRegistry().providerById("qgis")
-        for alg in provider.externalAlgs[::-1]:
+        #for alg in provider.externalAlgs[::-1]:
-            path = os.path.dirname(alg.descriptionFile)
+        #    path = os.path.dirname(alg.descriptionFile)
-            if path == folder:
+        #    if path == folder:
-                provider.externalAlgs.remove(alg)
+        #        provider.externalAlgs.remove(alg)
-        provider.refreshAlgorithms()
+        #provider.refreshAlgorithms()
+        pass
 
     @staticmethod
     def runAlgorithm(algOrName, parameters, onFinish=None, feedback=None, context=None):

python/plugins/processing/script/AddScriptFromFileAction.py

@@ -36,7 +36,7 @@ from processing.gui.ToolboxAction import ToolboxAction
 
 from processing.script.ScriptAlgorithm import ScriptAlgorithm
 from processing.script.WrongScriptException import WrongScriptException
-from processing.script.ScriptUtils import ScriptUtils
+from processing.script import ScriptUtils
 
 pluginPath = os.path.split(os.path.dirname(__file__))[0]
 

python/plugins/processing/script/ScriptAlgorithmProvider.py

@@ -31,6 +31,7 @@ from qgis.core import (QgsApplication,
                        QgsProcessingProvider)
 
 from processing.core.ProcessingConfig import ProcessingConfig, Setting
+
 from processing.gui.ProviderActions import (ProviderActions,
                                             ProviderContextMenuActions)
 
@@ -39,9 +40,7 @@ from processing.script.CreateNewScriptAction import CreateNewScriptAction
 from processing.script.DeleteScriptAction import DeleteScriptAction
 from processing.script.EditScriptAction import EditScriptAction
 from processing.script.CreateScriptCollectionPluginAction import CreateScriptCollectionPluginAction
-from processing.script.ScriptUtils import ScriptUtils
+from processing.script import ScriptUtils
-
-pluginPath = os.path.split(os.path.dirname(__file__))[0]
 
 
 class ScriptAlgorithmProvider(QgsProcessingProvider):
@@ -59,17 +58,22 @@ class ScriptAlgorithmProvider(QgsProcessingProvider):
     def load(self):
         ProcessingConfig.settingIcons[self.name()] = self.icon()
         ProcessingConfig.addSetting(Setting(self.name(),
-                                            ScriptUtils.SCRIPTS_FOLDER,
+                                            ScriptUtils.SCRIPTS_FOLDERS,
-                                            self.tr('Scripts folder', 'ScriptAlgorithmProvider'),
+                                            self.tr("Scripts folder(s)"),
-                                            ScriptUtils.defaultScriptsFolder(), valuetype=Setting.MULTIPLE_FOLDERS))
+                                            ScriptUtils.defaultScriptsFolder(),
+                                            valuetype=Setting.MULTIPLE_FOLDERS))
+
         ProviderActions.registerProviderActions(self, self.actions)
         ProviderContextMenuActions.registerProviderContextMenuActions(self.contextMenuActions)
+
         ProcessingConfig.readSettings()
         self.refreshAlgorithms()
+
         return True
 
     def unload(self):
-        ProcessingConfig.removeSetting(ScriptUtils.SCRIPTS_FOLDER)
+        ProcessingConfig.removeSetting(ScriptUtils.SCRIPTS_FOLDERS)
+
         ProviderActions.deregisterProviderActions(self)
         ProviderContextMenuActions.deregisterProviderContextMenuActions(self.contextMenuActions)
 
@@ -80,22 +84,10 @@ class ScriptAlgorithmProvider(QgsProcessingProvider):
         return QgsApplication.iconPath("processingScript.svg")
 
     def id(self):
-        return 'script'
+        return "script"
 
     def name(self):
-        return self.tr('Scripts', 'ScriptAlgorithmProvider')
+        return self.tr("Scripts")
-
-    def loadAlgorithms(self):
-        self.algs = []
-        folders = ScriptUtils.scriptsFolders()
-        for f in folders:
-            self.algs.extend(ScriptUtils.loadFromFolder(f))
-        self.algs.extend(self.folder_algorithms)
-        for a in self.algs:
-            self.addAlgorithm(a)
-
-    def addAlgorithmsFromFolder(self, folder):
-        self.folder_algorithms.extend(ScriptUtils.loadFromFolder(folder))
 
     def supportsNonFileBasedOutput(self):
         # TODO - this may not be strictly true. We probably need a way for scripts
@@ -103,3 +95,19 @@ class ScriptAlgorithmProvider(QgsProcessingProvider):
         # but for now allow it. At best we expose nice features to users, at worst
         # they'll get an error if they use them with incompatible outputs...
         return True
+
+    def loadAlgorithms(self):
+        self.algs = []
+        folders = ScriptUtils.scriptsFolders()
+        for folder in folders:
+            items = os.scandir(folder)
+            for entry in items:
+                if entry.name.lower().endswith("py") and entry.is_file():
+                    algName = os.path.splitext(entry.name)[0]
+                    filePath = os.path.abspath(os.path.join(folder, entry.name))
+                    alg = ScriptUtils.loadAlgorithm(algName, filePath)
+                    if alg is not None:
+                        self.algs.append(alg)
+
+        for a in self.algs:
+            self.addAlgorithm(a)

python/plugins/processing/script/ScriptEditorDialog.py

@@ -44,7 +44,7 @@ from processing.gui.AlgorithmDialog import AlgorithmDialog
 from processing.gui.HelpEditionDialog import HelpEditionDialog
 
 from processing.script.ScriptAlgorithm import ScriptAlgorithm
-from processing.script.ScriptUtils import ScriptUtils
+from processing.script import ScriptUtils
 
 pluginPath = os.path.split(os.path.dirname(__file__))[0]
 WIDGET, BASE = uic.loadUiType(

python/plugins/processing/script/ScriptUtils.py

@@ -26,53 +26,48 @@ __copyright__ = '(C) 2012, Victor Olaya'
 __revision__ = '$Format:%H$'
 
 import os
-from qgis.core import (QgsProcessingUtils,
+import inspect
-                       QgsMessageLog)
+import importlib
-from processing.core.ProcessingConfig import ProcessingConfig
-from processing.script.ScriptAlgorithm import ScriptAlgorithm
-from processing.script.WrongScriptException import WrongScriptException
-from processing.tools.system import mkdir, userFolder
 
 from qgis.PyQt.QtCore import QCoreApplication
 
+from qgis.core import QgsProcessingAlgorithm, QgsMessageLog
 
-class ScriptUtils:
+from processing.core.ProcessingConfig import ProcessingConfig
 
-    SCRIPTS_FOLDER = 'SCRIPTS_FOLDER'
+from processing.script.ScriptAlgorithm import ScriptAlgorithm
+from processing.script.WrongScriptException import WrongScriptException
 
-    @staticmethod
+from processing.tools.system import mkdir, userFolder
-    def defaultScriptsFolder():
+
-        folder = str(os.path.join(userFolder(), 'scripts'))
+
+SCRIPTS_FOLDERS = "SCRIPTS_FOLDERS"
+
+
+def defaultScriptsFolder():
+    folder = str(os.path.join(userFolder(), "scripts"))
     mkdir(folder)
     return os.path.abspath(folder)
 
-    @staticmethod
+
-    def scriptsFolders():
+def scriptsFolders():
-        folder = ProcessingConfig.getSetting(ScriptUtils.SCRIPTS_FOLDER)
+    folder = ProcessingConfig.getSetting(SCRIPTS_FOLDERS)
     if folder is not None:
-            return folder.split(';')
+        return folder.split(";")
     else:
         return [ScriptUtils.defaultScriptsFolder()]
 
-    @staticmethod
+
-    def loadFromFolder(folder):
+def loadAlgorithm(moduleName, filePath):
-        if not os.path.exists(folder):
-            return []
-        algs = []
-        for path, subdirs, files in os.walk(folder):
-            for descriptionFile in files:
-                if descriptionFile.endswith('py'):
     try:
-                        fullpath = os.path.join(path, descriptionFile)
+        spec = importlib.util.spec_from_file_location(moduleName, filePath)
-                        alg = ScriptAlgorithm(fullpath)
+        module = importlib.util.module_from_spec(spec)
-                        if alg.name().strip() != '':
+        spec.loader.exec_module(module)
-                            algs.append(alg)
+        for x in dir(module):
-                    except WrongScriptException as e:
+            obj = getattr(module, x)
-                        QgsMessageLog.logMessage(e.msg, QCoreApplication.translate('Processing', 'Processing'), QgsMessageLog.CRITICAL)
+            if inspect.isclass(obj) and issubclass(obj, QgsProcessingAlgorithm) and obj.__name__ == moduleName:
-                    except Exception as e:
+                return obj()
-                        QgsMessageLog.logMessage(
+    except ImportError as e:
-                            QCoreApplication.translate('Processing', 'Could not load script: {0}\n{1}').format(descriptionFile, str(e)),
+        QgsMessageLog.logMessage("Could not import script algorithm '{}' from '{}'\n{}".format(moduleName, filePath, str(e)),
-                            QCoreApplication.translate('Processing', 'Processing'),
+                                 "Processing",
-                            QgsMessageLog.CRITICAL
+                                 QgsMessageLog.CRITICAL)
-                        )
-        return algs