From e9a0bd332e05fbff845703b77aa1fee1026144c3 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 16 May 2025 11:00:53 +1000 Subject: [PATCH] [ogr] Handle auto addition of vsizip prefix for vsicurl archives Fixes #61561 --- .../providers/ogr/qgsogrprovidermetadata.cpp | 3 +- src/core/qgsgdalutils.cpp | 9 +- tests/src/python/test_provider_ogr.py | 90 +++++++++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/src/core/providers/ogr/qgsogrprovidermetadata.cpp b/src/core/providers/ogr/qgsogrprovidermetadata.cpp index b05a7e27d2c..8c70e7179c8 100644 --- a/src/core/providers/ogr/qgsogrprovidermetadata.cpp +++ b/src/core/providers/ogr/qgsogrprovidermetadata.cpp @@ -790,7 +790,8 @@ QList QgsOgrProviderMetadata::querySublayers( const // Try to open using VSIFileHandler const QString vsiPrefix = QgsGdalUtils::vsiPrefixForPath( uriParts.value( QStringLiteral( "path" ) ).toString() ); - if ( !vsiPrefix.isEmpty() && uriParts.value( QStringLiteral( "vsiPrefix" ) ).toString().isEmpty() ) + if ( !vsiPrefix.isEmpty() && ( uriParts.value( QStringLiteral( "vsiPrefix" ) ).toString().isEmpty() + || ( QgsGdalUtils::isVsiArchivePrefix( vsiPrefix ) && uriParts.value( QStringLiteral( "vsiPrefix" ) ).toString() != vsiPrefix ) ) ) { if ( !uri.startsWith( vsiPrefix ) ) { diff --git a/src/core/qgsgdalutils.cpp b/src/core/qgsgdalutils.cpp index 6106c743abb..5441fe688cd 100644 --- a/src/core/qgsgdalutils.cpp +++ b/src/core/qgsgdalutils.cpp @@ -952,7 +952,14 @@ QList QgsGdalUtils::vsiNetworkFileSys bool QgsGdalUtils::isVsiArchivePrefix( const QString &prefix ) { - return vsiArchivePrefixes().contains( prefix ); + const QStringList prefixes = vsiArchivePrefixes(); + for ( const QString &archivePrefix : prefixes ) + { + // catch chained prefixes, eg "/vsizip/vsicurl" + if ( prefix.contains( archivePrefix ) ) + return true; + } + return false; } QStringList QgsGdalUtils::vsiArchiveFileExtensions() diff --git a/tests/src/python/test_provider_ogr.py b/tests/src/python/test_provider_ogr.py index 6345ff6a536..3d09f1b786a 100644 --- a/tests/src/python/test_provider_ogr.py +++ b/tests/src/python/test_provider_ogr.py @@ -17,6 +17,12 @@ import sys import tempfile import math from datetime import datetime +import http.server +import os +import socketserver +import threading +import time +import shutil from osgeo import gdal, ogr # NOQA from qgis.PyQt.QtCore import QByteArray, QTemporaryDir, QVariant @@ -119,6 +125,19 @@ class PyQgsOGRProvider(QgisTestCase): cls.dirs_to_cleanup = [cls.basetestpath] + # Bring up a simple HTTP server, for vsicurl tests + os.chdir(unitTestDataPath() + "") + + cls.httpd = socketserver.TCPServer( + ("localhost", 0), http.server.SimpleHTTPRequestHandler + ) + cls.port = cls.httpd.server_address[1] + cls.port = cls.httpd.server_address[1] + + cls.httpd_thread = threading.Thread(target=cls.httpd.serve_forever) + cls.httpd_thread.daemon = True + cls.httpd_thread.start() + @classmethod def tearDownClass(cls): """Run after all tests""" @@ -661,6 +680,11 @@ class PyQgsOGRProvider(QgisTestCase): self.assertEqual(gdal.GetConfigOption("GDAL_HTTP_PROXY"), "myproxyhostname.com") self.assertEqual(gdal.GetConfigOption("GDAL_HTTP_PROXYUSERPWD"), "username") + settings.setValue("proxy/proxyEnabled", False) + QgsNetworkAccessManager.instance().setupDefaultProxyAndCache() + gdal.SetConfigOption("GDAL_HTTP_PROXY", "") + gdal.SetConfigOption("GDAL_HTTP_PROXYUSERPWD", "") + def testEditGeoJsonRemoveField(self): """Test bugfix of https://github.com/qgis/QGIS/issues/26484 (deleting an existing field)""" @@ -3271,6 +3295,72 @@ class PyQgsOGRProvider(QgisTestCase): for feature in layer.getFeatures(): self.assertEqual(feature.geometry().wkbType(), QgsWkbTypes.MultiPolygon) + # vsicurl + res = metadata.querySublayers( + f"/vsicurl/http://localhost:{self.port}/polys.shp" + ) + self.assertEqual(len(res), 1) + self.assertEqual(res[0].layerNumber(), 0) + self.assertEqual(res[0].name(), "polys") + self.assertEqual(res[0].description(), "") + self.assertEqual( + res[0].uri(), + f"/vsicurl/http://localhost:{self.port}/polys.shp|layername=polys", + ) + self.assertEqual(res[0].providerKey(), "ogr") + self.assertEqual(res[0].type(), QgsMapLayerType.VectorLayer) + self.assertEqual(res[0].featureCount(), Qgis.FeatureCountState.Uncounted) + self.assertEqual(res[0].wkbType(), QgsWkbTypes.Type.Polygon) + self.assertEqual(res[0].geometryColumnName(), "") + self.assertEqual(res[0].driverName(), "ESRI Shapefile") + vl = res[0].toLayer(options) + self.assertTrue(vl.isValid()) + self.assertEqual(vl.wkbType(), QgsWkbTypes.Type.MultiPolygon) + + # vsicurl with zip + res = metadata.querySublayers( + f"/vsicurl/http://localhost:{self.port}/zip/points2.zip" + ) + self.assertEqual(len(res), 1) + self.assertEqual(res[0].layerNumber(), 0) + self.assertEqual(res[0].name(), "points.shp") + self.assertEqual(res[0].description(), "") + self.assertEqual( + res[0].uri(), + f"/vsizip//vsicurl/http://localhost:{self.port}/zip/points2.zip/points.shp|layername=points", + ) + self.assertEqual(res[0].providerKey(), "ogr") + self.assertEqual(res[0].type(), QgsMapLayerType.VectorLayer) + self.assertEqual(res[0].featureCount(), Qgis.FeatureCountState.Uncounted) + self.assertEqual(res[0].wkbType(), QgsWkbTypes.Type.Point) + self.assertEqual(res[0].geometryColumnName(), "") + self.assertEqual(res[0].driverName(), "ESRI Shapefile") + vl = res[0].toLayer(options) + self.assertTrue(vl.isValid()) + self.assertEqual(vl.wkbType(), QgsWkbTypes.Type.Point) + + # vsicurl with zip, explicit vsizip prefix + res = metadata.querySublayers( + f"/vsizip//vsicurl/http://localhost:{self.port}/zip/points2.zip" + ) + self.assertEqual(len(res), 1) + self.assertEqual(res[0].layerNumber(), 0) + self.assertEqual(res[0].name(), "points") + self.assertEqual(res[0].description(), "") + self.assertEqual( + res[0].uri(), + f"/vsizip//vsicurl/http://localhost:{self.port}/zip/points2.zip|layername=points", + ) + self.assertEqual(res[0].providerKey(), "ogr") + self.assertEqual(res[0].type(), QgsMapLayerType.VectorLayer) + self.assertEqual(res[0].featureCount(), Qgis.FeatureCountState.Uncounted) + self.assertEqual(res[0].wkbType(), QgsWkbTypes.Type.Point) + self.assertEqual(res[0].geometryColumnName(), "") + self.assertEqual(res[0].driverName(), "ESRI Shapefile") + vl = res[0].toLayer(options) + self.assertTrue(vl.isValid()) + self.assertEqual(vl.wkbType(), QgsWkbTypes.Type.Point) + @unittest.skipIf( int(gdal.VersionInfo("VERSION_NUM")) < GDAL_COMPUTE_VERSION(3, 4, 0), "GDAL 3.4 required",