Don't require manual execution of description_to_json, run during cmake

instead

python/plugins/grassprovider/CMakeLists.txt

@@ -1,6 +1,16 @@
 file(GLOB PY_FILES *.py)
 file(GLOB OTHER_FILES grass7.txt metadata.txt)
-file(GLOB DESCR_FILES description/algorithms.json)
+
+execute_process(
+  COMMAND ${Python_EXECUTABLE} -m grassprovider.description_to_json ${CMAKE_CURRENT_SOURCE_DIR}/description ${CMAKE_CURRENT_SOURCE_DIR}/description/algorithms.json
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..
+  RESULT_VARIABLE result
+  ERROR_VARIABLE error_output
+)
+if(NOT "${result}" STREQUAL "0")
+    message(FATAL_ERROR "Create grass provider algorithm descriptions failed with error: ${error_output}")
+endif()
+set(DESCR_FILES ${CMAKE_CURRENT_SOURCE_DIR}/description/algorithms.json)
 
 add_subdirectory(ext)
 add_subdirectory(tests)

python/plugins/grassprovider/Grass7Algorithm.py

@@ -25,7 +25,6 @@ from typing import (
 )
 import sys
 import os
-import re
 import uuid
 import math
 import importlib
@@ -77,11 +76,10 @@ with warnings.catch_warnings():
     from osgeo import ogr
 
 from processing.core.ProcessingConfig import ProcessingConfig
+from processing.core.parameters import getParameterFromString
 
-from grassprovider.Grass7Utils import (
+from grassprovider.parsed_description import ParsedDescription
-    Grass7Utils,
+from grassprovider.Grass7Utils import Grass7Utils
-    ParsedDescription
-)
 
 from processing.tools.system import isWindows, getTempFilename
 
@@ -263,7 +261,51 @@ class Grass7Algorithm(QgsProcessingAlgorithm):
         self._group = description.group
         self._groupId = description.group_id
         self.hardcodedStrings = description.hardcoded_strings[:]
-        self.params = description.params
+
+        self.params = []
+
+        has_raster_input: bool = False
+        has_vector_input: bool = False
+
+        has_raster_output: bool = False
+        has_vector_outputs: bool = False
+
+        for param_string in description.param_strings:
+            try:
+                parameter = getParameterFromString(param_string, "GrassAlgorithm")
+            except Exception as e:
+                QgsMessageLog.logMessage(
+                    QCoreApplication.translate("GrassAlgorithm",
+                                               'Could not open GRASS GIS 7 algorithm: {0}').format(
+                        self._name),
+                    QCoreApplication.translate("GrassAlgorithm",
+                                               'Processing'),
+                    Qgis.Critical)
+                raise e
+
+            if parameter is None:
+                continue
+
+            self.params.append(parameter)
+            if isinstance(parameter, (
+                    QgsProcessingParameterVectorLayer,
+                    QgsProcessingParameterFeatureSource)):
+                has_vector_input = True
+            elif isinstance(parameter,
+                            QgsProcessingParameterRasterLayer):
+                has_raster_input = True
+            elif isinstance(parameter,
+                            QgsProcessingParameterMultipleLayers):
+                if parameter.layerType() < 3 or parameter.layerType() == 5:
+                    has_vector_input = True
+                elif parameter.layerType() == 3:
+                    has_raster_input = True
+            elif isinstance(parameter,
+                            QgsProcessingParameterVectorDestination):
+                has_vector_outputs = True
+            elif isinstance(parameter,
+                            QgsProcessingParameterRasterDestination):
+                has_raster_output = True
 
         param = QgsProcessingParameterExtent(
             self.GRASS_REGION_EXTENT_PARAMETER,
@@ -273,7 +315,7 @@ class Grass7Algorithm(QgsProcessingAlgorithm):
         param.setFlags(param.flags() | QgsProcessingParameterDefinition.FlagAdvanced)
         self.params.append(param)
 
-        if description.has_raster_output or description.has_raster_input:
+        if has_raster_output or has_raster_input:
             # Add a cellsize parameter
             param = QgsProcessingParameterNumber(
                 self.GRASS_REGION_CELLSIZE_PARAMETER,
@@ -284,7 +326,7 @@ class Grass7Algorithm(QgsProcessingAlgorithm):
             param.setFlags(param.flags() | QgsProcessingParameterDefinition.FlagAdvanced)
             self.params.append(param)
 
-        if description.has_raster_output:
+        if has_raster_output:
             # Add a createopt parameter for format export
             param = QgsProcessingParameterString(
                 self.GRASS_RASTER_FORMAT_OPT,
@@ -305,7 +347,7 @@ class Grass7Algorithm(QgsProcessingAlgorithm):
             param.setHelp(self.tr('Metadata options should be comma separated'))
             self.params.append(param)
 
-        if description.has_vector_input:
+        if has_vector_input:
             param = QgsProcessingParameterNumber(self.GRASS_SNAP_TOLERANCE_PARAMETER,
                                                  self.tr('v.in.ogr snap tolerance (-1 = no snap)'),
                                                  type=QgsProcessingParameterNumber.Double,
@@ -321,7 +363,7 @@ class Grass7Algorithm(QgsProcessingAlgorithm):
             param.setFlags(param.flags() | QgsProcessingParameterDefinition.FlagAdvanced)
             self.params.append(param)
 
-        if description.has_vector_outputs:
+        if has_vector_outputs:
             # Add an optional output type
             param = QgsProcessingParameterEnum(self.GRASS_OUTPUT_TYPE_PARAMETER,
                                                self.tr('v.out.ogr output type'),

python/plugins/grassprovider/Grass7Utils.py

@@ -44,190 +44,13 @@ from qgis.core import (
     QgsMessageLog,
     QgsCoordinateReferenceSystem,
     QgsProcessingContext,
-    QgsProcessingParameterDefinition,
-    QgsProcessingParameterVectorLayer,
-    QgsProcessingParameterFeatureSource,
-    QgsProcessingParameterRasterLayer,
-    QgsProcessingParameterMultipleLayers,
-    QgsProcessingParameterVectorDestination,
-    QgsProcessingParameterRasterDestination
 )
 
 from processing.algs.gdal.GdalUtils import GdalUtils
 from processing.core.ProcessingConfig import ProcessingConfig
-from processing.core.parameters import getParameterFromString
 from processing.tools.system import userFolder, isWindows, isMac, mkdir
 
 
-@dataclass
-class ParsedDescription:
-    """
-    Results of parsing a description file
-    """
-
-    GROUP_ID_REGEX = re.compile(r'^[^\s(]+')
-
-    grass_command: Optional[str] = None
-    short_description: Optional[str] = None
-    name: Optional[str] = None
-    display_name: Optional[str] = None
-    group: Optional[str] = None
-    group_id: Optional[str] = None
-
-    ext_path: Optional[str] = None
-
-    has_raster_input: bool = False
-    has_vector_input: bool = False
-
-    has_raster_output: bool = False
-    has_vector_outputs: bool = False
-
-    hardcoded_strings: List[str] = field(default_factory=list)
-    params: List[QgsProcessingParameterDefinition] = field(
-        default_factory=list)
-    param_strings: List[str] = field(default_factory=list)
-
-    def as_dict(self) -> Dict:
-        """
-        Returns a JSON serializable dictionary representing the parsed
-        description
-        """
-        return {
-            'name': self.name,
-            'display_name': self.display_name,
-            'command': self.grass_command,
-            'short_description': self.short_description,
-            'group': self.group,
-            'group_id': self.group_id,
-            'ext_path': self.ext_path,
-            'inputs': {
-                'has_raster': self.has_raster_input,
-                'has_vector': self.has_vector_input
-            },
-            'outputs':
-                {
-                    'has_raster': self.has_raster_output,
-                    'has_vector': self.has_vector_outputs
-            },
-            'hardcoded_strings': self.hardcoded_strings,
-            'parameters': self.param_strings
-        }
-
-    @staticmethod
-    def from_dict(description: Dict) -> 'ParsedDescription':
-        """
-        Parses a dictionary as a description and returns the result
-        """
-        result = ParsedDescription()
-        result.name = description.get('name')
-        result.display_name = description.get('display_name')
-        result.grass_command = description.get('command')
-        result.short_description = QCoreApplication.translate(
-            "GrassAlgorithm",
-            description.get('short_description')
-        )
-        result.group = QCoreApplication.translate("GrassAlgorithm",
-                                                  description.get('group'))
-        result.group_id = description.get('group_id')
-        result.ext_path = description.get('ext_path')
-        result.has_raster_input = description.get('inputs', {}).get(
-            'has_raster', False)
-        result.has_vector_input = description.get('inputs', {}).get(
-            'has_vector', False)
-        result.has_raster_output = description.get('outputs', {}).get(
-            'has_raster', False)
-        result.has_vector_outputs = description.get('outputs', {}).get(
-            'has_vector', False)
-        result.hardcoded_strings = description.get('hardcoded_strings', [])
-        result.param_strings = description.get('parameters', [])
-        for param in result.param_strings:
-            result.params.append(
-                getParameterFromString(param, "GrassAlgorithm"))
-
-        return result
-
-    @staticmethod
-    def parse_description_file(
-            description_file: Path,
-            translate: bool = True) -> 'ParsedDescription':
-        """
-        Parses a description file and returns the result
-        """
-        result = ParsedDescription()
-
-        with description_file.open() as lines:
-            # First line of the file is the Grass algorithm name
-            line = lines.readline().strip('\n').strip()
-            result.grass_command = line
-            # Second line if the algorithm name in Processing
-            line = lines.readline().strip('\n').strip()
-            result.short_description = line
-            if " - " not in line:
-                result.name = result.grass_command
-            else:
-                result.name = line[:line.find(' ')].lower()
-            if translate:
-                result.short_description = QCoreApplication.translate(
-                    "GrassAlgorithm", line)
-            else:
-                result.short_description = line
-
-            result.display_name = result.name
-            # Read the grass group
-            line = lines.readline().strip('\n').strip()
-            if translate:
-                result.group = QCoreApplication.translate("GrassAlgorithm",
-                                                          line)
-            else:
-                result.group = line
-
-            result.group_id = ParsedDescription.GROUP_ID_REGEX.search(
-                line).group(0).lower()
-
-            # Then you have parameters/output definition
-            line = lines.readline().strip('\n').strip()
-            while line != '':
-                try:
-                    line = line.strip('\n').strip()
-                    if line.startswith('Hardcoded'):
-                        result.hardcoded_strings.append(
-                            line[len('Hardcoded|'):])
-                    result.param_strings.append(line)
-                    parameter = getParameterFromString(line, "GrassAlgorithm")
-                    if parameter is not None:
-                        result.params.append(parameter)
-                        if isinstance(parameter, (
-                                QgsProcessingParameterVectorLayer,
-                                QgsProcessingParameterFeatureSource)):
-                            result.has_vector_input = True
-                        elif isinstance(parameter,
-                                        QgsProcessingParameterRasterLayer):
-                            result.has_raster_input = True
-                        elif isinstance(parameter,
-                                        QgsProcessingParameterMultipleLayers):
-                            if parameter.layerType() < 3 or parameter.layerType() == 5:
-                                result.has_vector_input = True
-                            elif parameter.layerType() == 3:
-                                result.has_raster_input = True
-                        elif isinstance(parameter,
-                                        QgsProcessingParameterVectorDestination):
-                            result.has_vector_outputs = True
-                        elif isinstance(parameter,
-                                        QgsProcessingParameterRasterDestination):
-                            result.has_raster_output = True
-                    line = lines.readline().strip('\n').strip()
-                except Exception as e:
-                    QgsMessageLog.logMessage(
-                        QCoreApplication.translate("GrassAlgorithm",
-                                                   'Could not open GRASS GIS 7 algorithm: {0}\n{1}').format(
-                            description_file, line),
-                        QCoreApplication.translate("GrassAlgorithm",
-                                                   'Processing'),
-                        Qgis.Critical)
-                    raise e
-        return result
-
-
 class Grass7Utils:
     GRASS_REGION_XMIN = 'GRASS7_REGION_XMIN'
     GRASS_REGION_YMIN = 'GRASS7_REGION_YMIN'

python/plugins/grassprovider/description_to_json.py

@@ -12,27 +12,45 @@ Parses .txt algorithm description files and builds aggregated .json
 description
 """
 
+import argparse
 import json
+import os
+from pathlib import Path
 
-from grassprovider.Grass7Utils import (
+
-    Grass7Utils,
+def main(description_folder: str, output_file: str):
+    from .parsed_description import (
         ParsedDescription
     )
 
-base_description_folders = [f for f in Grass7Utils.grassDescriptionFolders()
-                            if f != Grass7Utils.userDescriptionFolder()]
-
-for folder in base_description_folders:
     algorithms = []
+    folder = Path(description_folder)
     for description_file in folder.glob('*.txt'):
+
         description = ParsedDescription.parse_description_file(
             description_file, translate=False)
 
-        extpath = description_file.parents[1].joinpath('ext', description.name.replace('.', '_') + '.py')
+        ext_path = description_file.parents[1].joinpath(
-        if extpath.exists():
+            'ext', description.name.replace('.', '_') + '.py')
+        if ext_path.exists():
             description.ext_path = description.name.replace('.', '_')
 
         algorithms.append(description.as_dict())
 
-    with open(folder / 'algorithms.json', 'wt', encoding='utf8') as f_out:
+    Path(output_file).parent.mkdir(parents=True, exist_ok=True)
+    with open(output_file, 'wt', encoding='utf8') as f_out:
         f_out.write(json.dumps(algorithms, indent=2))
+
+
+parser = argparse.ArgumentParser(description="Parses GRASS .txt algorithm "
+                                             "description files and builds "
+                                             "aggregated .json description")
+
+parser.add_argument("input", help="Path to the description directory")
+parser.add_argument("output", help="Path to the output algorithms.json file")
+args = parser.parse_args()
+
+if not os.path.isdir(args.input):
+    raise ValueError(f"Input directory '{args.input}' is not a directory.")
+
+main(args.input, args.output)

python/plugins/grassprovider/parsed_description.py

@@ -0,0 +1,137 @@
+"""
+***************************************************************************
+*                                                                         *
+*   This program is free software; you can redistribute it and/or modify  *
+*   it under the terms of the GNU General Public License as published by  *
+*   the Free Software Foundation; either version 2 of the License, or     *
+*   (at your option) any later version.                                   *
+*                                                                         *
+***************************************************************************
+"""
+
+import re
+from dataclasses import (
+    dataclass,
+    field
+)
+from pathlib import Path
+from typing import (
+    Optional,
+    List,
+    Dict
+)
+
+
+@dataclass
+class ParsedDescription:
+    """
+    Results of parsing a description file
+    """
+
+    GROUP_ID_REGEX = re.compile(r'^[^\s(]+')
+
+    grass_command: Optional[str] = None
+    short_description: Optional[str] = None
+    name: Optional[str] = None
+    display_name: Optional[str] = None
+    group: Optional[str] = None
+    group_id: Optional[str] = None
+
+    ext_path: Optional[str] = None
+
+    hardcoded_strings: List[str] = field(default_factory=list)
+    param_strings: List[str] = field(default_factory=list)
+
+    def as_dict(self) -> Dict:
+        """
+        Returns a JSON serializable dictionary representing the parsed
+        description
+        """
+        return {
+            'name': self.name,
+            'display_name': self.display_name,
+            'command': self.grass_command,
+            'short_description': self.short_description,
+            'group': self.group,
+            'group_id': self.group_id,
+            'ext_path': self.ext_path,
+            'hardcoded_strings': self.hardcoded_strings,
+            'parameters': self.param_strings
+        }
+
+    @staticmethod
+    def from_dict(description: Dict) -> 'ParsedDescription':
+        """
+        Parses a dictionary as a description and returns the result
+        """
+
+        from qgis.PyQt.QtCore import QCoreApplication
+
+        result = ParsedDescription()
+        result.name = description.get('name')
+        result.display_name = description.get('display_name')
+        result.grass_command = description.get('command')
+        result.short_description = QCoreApplication.translate(
+            "GrassAlgorithm",
+            description.get('short_description')
+        )
+        result.group = QCoreApplication.translate("GrassAlgorithm",
+                                                  description.get('group'))
+        result.group_id = description.get('group_id')
+        result.ext_path = description.get('ext_path')
+        result.hardcoded_strings = description.get('hardcoded_strings', [])
+        result.param_strings = description.get('parameters', [])
+
+        return result
+
+    @staticmethod
+    def parse_description_file(
+            description_file: Path,
+            translate: bool = True) -> 'ParsedDescription':
+        """
+        Parses a description file and returns the result
+        """
+
+        result = ParsedDescription()
+
+        with description_file.open() as lines:
+            # First line of the file is the Grass algorithm name
+            line = lines.readline().strip('\n').strip()
+            result.grass_command = line
+            # Second line if the algorithm name in Processing
+            line = lines.readline().strip('\n').strip()
+            result.short_description = line
+            if " - " not in line:
+                result.name = result.grass_command
+            else:
+                result.name = line[:line.find(' ')].lower()
+            if translate:
+                from qgis.PyQt.QtCore import QCoreApplication
+                result.short_description = QCoreApplication.translate(
+                    "GrassAlgorithm", line)
+            else:
+                result.short_description = line
+
+            result.display_name = result.name
+            # Read the grass group
+            line = lines.readline().strip('\n').strip()
+            if translate:
+                from qgis.PyQt.QtCore import QCoreApplication
+                result.group = QCoreApplication.translate("GrassAlgorithm",
+                                                          line)
+            else:
+                result.group = line
+
+            result.group_id = ParsedDescription.GROUP_ID_REGEX.search(
+                line).group(0).lower()
+
+            # Then you have parameters/output definition
+            line = lines.readline().strip('\n').strip()
+            while line != '':
+                line = line.strip('\n').strip()
+                if line.startswith('Hardcoded'):
+                    result.hardcoded_strings.append(
+                        line[len('Hardcoded|'):])
+                result.param_strings.append(line)
+                line = lines.readline().strip('\n').strip()
+        return result