Merge pull request #59949 from nyalldawson/grass7im

Fix grass imagery tests on newer GRASS versions

.ci/test_blocklist_qt6.txt

@@ -58,6 +58,5 @@ PyQgsServerWFS
 ProcessingQgisAlgorithmsTestPt2
 ProcessingQgisAlgorithmsTestPt4
 ProcessingGdalAlgorithmsVectorTest
-ProcessingGrassAlgorithmsImageryTest
 # PyQgsProviderRegistry runs fine locally on Fedora:rawhide but not on CI
 PyQgsProviderRegistry

python/plugins/grassprovider/tests/AlgorithmsTestBase.py

@@ -34,12 +34,15 @@ from osgeo.gdalconst import GA_ReadOnly
 from numpy import nan_to_num
 from copy import deepcopy
 
+from qgis.PyQt.QtCore import QT_VERSION
 from qgis.core import (
+    Qgis,
     QgsVectorLayer,
     QgsRasterLayer,
     QgsCoordinateReferenceSystem,
     QgsFeatureRequest,
     QgsMapLayer,
+    QgsMeshLayer,
     QgsProject,
     QgsApplication,
     QgsProcessingContext,
@@ -48,14 +51,23 @@ from qgis.core import (
 )
 from qgis.analysis import QgsNativeAlgorithms
 from qgis.testing import _UnexpectedSuccess, QgisTestCase, start_app
-
 from utilities import unitTestDataPath
+from grassprovider.grass_utils import GrassUtils
 
 import processing
 
 gdal.UseExceptions()
 
 
+def GDAL_COMPUTE_VERSION(maj, min, rev):
+    return (maj) * 1000000 + (min) * 10000 + (rev) * 100
+
+
+def GRASS_VERSION_INT(version: str) -> int:
+    parts = version.split(".")
+    return int(parts[0]) * 1000000 + int(parts[1]) * 10000 + int(parts[2]) * 100
+
+
 def processingTestDataPath():
     return os.path.join(os.path.dirname(__file__), "testdata")
 
@@ -73,6 +85,108 @@ class AlgorithmsTest:
 
         if "tests" in algorithm_tests and algorithm_tests["tests"] is not None:
             for idx, algtest in enumerate(algorithm_tests["tests"]):
+                condition = algtest.get("condition")
+                if condition:
+                    grass_condition = condition.get("grass")
+                    if grass_condition:
+                        grass_version = GRASS_VERSION_INT(GrassUtils.installedVersion())
+                        less_than_condition = grass_condition.get("less_than")
+                        if less_than_condition:
+                            if grass_version >= less_than_condition:
+                                print(
+                                    "!!! Skipping {}, requires GRASS < {}, have version {}".format(
+                                        algtest["name"],
+                                        less_than_condition,
+                                        grass_version,
+                                    )
+                                )
+                                continue
+                        at_least_condition = grass_condition.get("at_least")
+                        if at_least_condition:
+                            if grass_version < at_least_condition:
+                                print(
+                                    "!!! Skipping {}, requires GRASS >= {}, have version {}".format(
+                                        algtest["name"],
+                                        at_least_condition,
+                                        grass_version,
+                                    )
+                                )
+                                continue
+
+                    geos_condition = condition.get("geos")
+                    if geos_condition:
+                        less_than_condition = geos_condition.get("less_than")
+                        if less_than_condition:
+                            if Qgis.geosVersionInt() >= less_than_condition:
+                                print(
+                                    "!!! Skipping {}, requires GEOS < {}, have version {}".format(
+                                        algtest["name"],
+                                        less_than_condition,
+                                        Qgis.geosVersionInt(),
+                                    )
+                                )
+                                continue
+                        at_least_condition = geos_condition.get("at_least")
+                        if at_least_condition:
+                            if Qgis.geosVersionInt() < at_least_condition:
+                                print(
+                                    "!!! Skipping {}, requires GEOS >= {}, have version {}".format(
+                                        algtest["name"],
+                                        at_least_condition,
+                                        Qgis.geosVersionInt(),
+                                    )
+                                )
+                                continue
+                    gdal_condition = condition.get("gdal")
+                    if gdal_condition:
+                        less_than_condition = gdal_condition.get("less_than")
+                        if less_than_condition:
+                            if (
+                                int(gdal.VersionInfo("VERSION_NUM"))
+                                >= less_than_condition
+                            ):
+                                print(
+                                    "!!! Skipping {}, requires GDAL < {}, have version {}".format(
+                                        algtest["name"],
+                                        less_than_condition,
+                                        gdal.VersionInfo("VERSION_NUM"),
+                                    )
+                                )
+                                continue
+                        at_least_condition = gdal_condition.get("at_least")
+                        if at_least_condition:
+                            if (
+                                int(gdal.VersionInfo("VERSION_NUM"))
+                                < at_least_condition
+                            ):
+                                print(
+                                    "!!! Skipping {}, requires GDAL >= {}, have version {}".format(
+                                        algtest["name"],
+                                        at_least_condition,
+                                        gdal.VersionInfo("VERSION_NUM"),
+                                    )
+                                )
+                                continue
+                    qt_condition = condition.get("qt")
+                    if qt_condition:
+                        less_than_condition = qt_condition.get("less_than")
+                        if less_than_condition:
+                            if QT_VERSION >= less_than_condition:
+                                print(
+                                    "!!! Skipping {}, requires Qt < {}, have version {}".format(
+                                        algtest["name"], less_than_condition, QT_VERSION
+                                    )
+                                )
+                                continue
+                        at_least_condition = qt_condition.get("at_least")
+                        if at_least_condition:
+                            if QT_VERSION < at_least_condition:
+                                print(
+                                    "!!! Skipping {}, requires Qt >= {}, have version {}".format(
+                                        algtest["name"], at_least_condition, QT_VERSION
+                                    )
+                                )
+                                continue
                 print(
                     'About to start {} of {}: "{}"'.format(
                         idx, len(algorithm_tests["tests"]), algtest["name"]
@@ -136,6 +250,12 @@ class AlgorithmsTest:
         # ignore user setting for invalid geometry handling
         context = QgsProcessingContext()
         context.setProject(QgsProject.instance())
+        if "ellipsoid" in defs:
+            # depending on the project settings, we can't always rely
+            # on QgsProject.ellipsoid() returning the same ellipsoid as was
+            # specified in the test definition. So just force ensure that the
+            # context's ellipsoid is the desired one
+            context.setEllipsoid(defs["ellipsoid"])
 
         if "skipInvalid" in defs and defs["skipInvalid"]:
             context.setInvalidGeometryCheck(
@@ -180,7 +300,7 @@ class AlgorithmsTest:
         parameter based on its key `type` and return the appropriate parameter to pass to the algorithm.
         """
         try:
-            if param["type"] in ("vector", "raster", "table"):
+            if param["type"] in ("vector", "raster", "table", "mesh"):
                 return self.load_layer(id, param).id()
             elif param["type"] == "vrtlayers":
                 vals = []
@@ -290,6 +410,8 @@ class AlgorithmsTest:
             options = QgsRasterLayer.LayerOptions()
             options.loadDefaultStyle = False
             lyr = QgsRasterLayer(filepath, param["name"], "gdal", options)
+        elif param["type"] == "mesh":
+            lyr = QgsMeshLayer(filepath, param["name"], "mdal")
 
         self.assertTrue(
             lyr.isValid(), f'Could not load layer "{filepath}" from param {param}'
@@ -440,10 +562,16 @@ class GenericAlgorithmsTest(QgisTestCase):
     @classmethod
     def setUpClass(cls):
         start_app()
+        from processing.core.Processing import Processing
+
+        Processing.initialize()
         cls.cleanup_paths = []
 
     @classmethod
     def tearDownClass(cls):
+        from processing.core.Processing import Processing
+
+        Processing.deinitialize()
         for path in cls.cleanup_paths:
             shutil.rmtree(path)
 

python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml

@@ -381,8 +381,13 @@ tests:
         type: raster
     results:
       signaturefile:
-        type: file
         name: expected/i.gensig.txt
+        type: regex
+        rules:
+          - '300\s*\n(?:\d+\s*\n)?7043\.06\s*\n1\.89772e\+07\s*'
+          - '1700\s*\n(?:\d+\s*\n)?7405\.93\s*\n2\.01921e\+07\s*'
+          - '6000\s*\n(?:\d+\s*\n)?7287\.21\s*\n1\.90221e\+07\s*'
+          - '7000\s*\n(?:\d+\s*\n)?7246\.81\s*\n1\.90226e\+07\s*'
 
   - algorithm: grass7:i.gensigset
     name: GRASS7 i.gensigset
@@ -399,8 +404,26 @@ tests:
         type: raster
     results:
       signaturefile:
-        type: file
+        type: regex
         name: expected/i.gensigset.txt
+        rules:
+          - 'subclass:\s*\n\s*pi:\s*0\.22802\s*\n\s*means:\s*9764\.5\s*\n\s*covar:\s*\n\s*3\.02234e\+06'
+          - 'subclass:\s*\n\s*pi:\s*0\.495197\s*\n\s*means:\s*5208\.5\s*\n\s*covar:\s*\n\s*7\.97435e\+06'
+          - 'subclass:\s*\n\s*pi:\s*0\.118668\s*\n\s*means:\s*997\.979\s*\n\s*covar:\s*\n\s*686411'
+          - 'subclass:\s*\n\s*pi:\s*0\.158115\s*\n\s*means:\s*13401\s*\n\s*covar:\s*\n\s*702077'
+          - 'subclass:\s*\n\s*pi:\s*0\.177696\s*\n\s*means:\s*13267\.4\s*\n\s*covar:\s*\n\s*1\.06971e\+06'
+          - 'subclass:\s*\n\s*pi:\s*0\.172485\s*\n\s*means:\s*3861\.81\s*\n\s*covar:\s*\n\s*2\.35556e\+06'
+          - 'subclass:\s*\n\s*pi:\s*0\.48383\s*\n\s*means:\s*8696\.48\s*\n\s*covar:\s*\n\s*7\.62681e\+06'
+          - 'subclass:\s*\n\s*pi:\s*0\.165989\s*\n\s*means:\s*1052\.05\s*\n\s*covar:\s*\n\s*692627'
+          - 'subclass:\s*\n\s*pi:\s*0\.135816\s*\n\s*means:\s*1158\.36\s*\n\s*covar:\s*\n\s*934515'
+          - 'subclass:\s*\n\s*pi:\s*0\.187835\s*\n\s*means:\s*11847\.9\s*\n\s*covar:\s*\n\s*1\.74438e\+06'
+          - 'subclass:\s*\n\s*pi:\s*0\.0829488\s*\n\s*means:\s*14066\.2\s*\n\s*covar:\s*\n\s*268579'
+          - 'subclass:\s*\n\s*pi:\s*0\.5934\s*\n\s*means:\s*6298\.74\s*\n\s*covar:\s*\n\s*8\.66516e\+06'
+          - 'subclass:\s*\n\s*pi:\s*0\.278456\s*\n\s*means:\s*3394\.59\s*\n\s*covar:\s*\n\s*2\.33337e\+06'
+          - 'subclass:\s*\n\s*pi:\s*0\.0849227\s*\n\s*means:\s*13998\.4\s*\n\s*covar:\s*\n\s*256746'
+          - 'subclass:\s*\n\s*pi:\s*0\.102818\s*\n\s*means:\s*686\.099\s*\n\s*covar:\s*\n\s*379342'
+          - 'subclass:\s*\n\s*pi:\s*0\.222279\s*\n\s*means:\s*11683\.6\s*\n\s*covar:\s*\n\s*1\.75208e\+06'
+          - 'subclass:\s*\n\s*pi:\s*0\.311523\s*\n\s*means:\s*7849\.26\s*\n\s*covar:\s*\n\s*3\.21781e\+06'
 
   - algorithm: grass7:i.rgb.his
     name: GRASS7 i.rgb.his
@@ -426,43 +449,53 @@ tests:
         hash: 07578ad38cf948473a519f040acb0235f60a592904ac8ef46aa607e1
         type: rasterhash
 
-# Triggers a Python exception on the GRASS version used on the CI - disabling until fixed in GRASS
+  - algorithm: grass7:i.pansharpen
-#  - algorithm: grass7:i.pansharpen
+    name: GRASS7 i.pansharpen
-#    name: GRASS7 i.pansharpen
+    condition:
-#    params:
+      grass:
-#      GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0'
+        # This test used to segfault on older GRASS releases
-#      blue:
+        at_least: 8040000
-#        name: raster_4class.tif
+    params:
-#        type: raster
+      GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0'
-#      green:
+      blue:
-#        name: raster_5class.tif
+        name: raster_4class.tif
-#        type: raster
+        type: raster
-#      method: 2
+      green:
-#      pan:
+        name: raster_5class.tif
-#        name: float_raster.tif
+        type: raster
-#        type: raster
+      method: 2
-#      red:
+      pan:
-#        name: raster_6class.tif
+        name: float_raster.tif
-#        type: raster
+        type: raster
-#    results:
+      red:
-#      blueoutput:
+        name: raster_6class.tif
-#        hash:
+        type: raster
-#        - 906de8be89e302057ed849d00eaf49332ecca73ffaba1374994f1a17
+    results:
-#        - 6abf8904844b902dfb567246ad906ff51673fa61a821633eadc480c4
+      blueoutput:
-#        type: rasterhash
+        hash:
-#      greenoutput:
+        - 906de8be89e302057ed849d00eaf49332ecca73ffaba1374994f1a17
-#        hash:
+        - 6abf8904844b902dfb567246ad906ff51673fa61a821633eadc480c4
-#        - 588ad1ef8360ce903fc2defb1a1728a1dc8335d737d5fa77797605ed
+        type: rasterhash
-#        - 5c3ac77c7ac4a57b7a985541ce0c5f76c1709d8de7aa66b3acfa45d6
+      greenoutput:
-#        type: rasterhash
+        hash:
-#      redoutput:
+        - 588ad1ef8360ce903fc2defb1a1728a1dc8335d737d5fa77797605ed
-#        hash:
+        - 5c3ac77c7ac4a57b7a985541ce0c5f76c1709d8de7aa66b3acfa45d6
-#        - ad80c0007faa1b0dc15c0b0c21ff4e0045ff5e67b454df0f65e68899
+        type: rasterhash
-#        - b488227a86c3ed58f5b2328238d109e69c55024f86efb1d4876de1f6
+      redoutput:
-#        type: rasterhash
+        hash:
-#
+        - ad80c0007faa1b0dc15c0b0c21ff4e0045ff5e67b454df0f65e68899
+        - b488227a86c3ed58f5b2328238d109e69c55024f86efb1d4876de1f6
+        type: rasterhash
+
+
   - algorithm: grass7:i.smap
     name: GRASS7 i.smap
+    condition:
+      grass:
+        # we can only run this test on older GRASS versions -- newer ones
+        # have the raster paths hardcoded into the signature file, and we can't handle
+        # that currently in the test suite
+        less_than: 8000000
     params:
       GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0'
       blocksize: 1024
@@ -484,6 +517,12 @@ tests:
 
   - algorithm: grass7:i.maxlik
     name: GRASS7 i.maxlik
+    condition:
+      grass:
+        # we can only run this test on older GRASS versions -- newer ones
+        # have the raster paths hardcoded into the signature file, and we can't handle
+        # that currently in the test suite
+        less_than: 8000000
     params:
       GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0'
       input: