Add option to limit the number of expanded features returned

This helps reduce the load on backend servers, as feature expansion
can quickly balloon out to a huge number of features. Default
to a very conservative expansion limit, requiring users to "opt in"
to larger limits which may be inappropriate for a service.
This commit is contained in:
Nyall Dawson 2024-03-25 15:43:27 +10:00
parent 742fa6b486
commit e471a0bd1d
7 changed files with 501 additions and 46 deletions

View File

@ -394,6 +394,12 @@ QVariantMap QgsSensorThingsProviderMetadata::decodeUri( const QString &uri ) con
{
components.insert( QStringLiteral( "featureLimit" ), featureLimitParam );
}
ok = false;
const int expansionLimitParam = dsUri.param( QStringLiteral( "expansionLimit" ) ).toInt( &ok );
if ( ok )
{
components.insert( QStringLiteral( "expansionLimit" ), expansionLimitParam );
}
switch ( QgsWkbTypes::geometryType( dsUri.wkbType() ) )
{
@ -487,6 +493,12 @@ QString QgsSensorThingsProviderMetadata::encodeUri( const QVariantMap &parts ) c
{
dsUri.setParam( QStringLiteral( "featureLimit" ), QString::number( featureLimitParam ) );
}
ok = false;
const int expansionLimitParam = parts.value( QStringLiteral( "expansionLimit" ) ).toInt( &ok );
if ( ok )
{
dsUri.setParam( QStringLiteral( "expansionLimit" ), QString::number( expansionLimitParam ) );
}
const QString geometryType = parts.value( QStringLiteral( "geometryType" ) ).toString();
if ( geometryType.compare( QLatin1String( "point" ), Qt::CaseInsensitive ) == 0 )

View File

@ -47,8 +47,13 @@ QgsSensorThingsSharedData::QgsSensorThingsSharedData( const QString &uri )
expandQueryParts.append( QgsSensorThingsUtils::entityToSetString( expandToEntityType ) );
}
}
mExpansionLimit = uriParts.value( QStringLiteral( "expansionLimit" ) ).toInt();
if ( !expandQueryParts.empty() )
{
mExpandQueryString = QStringLiteral( "$expand=" ) + expandQueryParts.join( '/' );
if ( mExpansionLimit > 0 )
mExpandQueryString += QStringLiteral( "($top=%1)" ).arg( mExpansionLimit );
}
mFields = QgsSensorThingsUtils::fieldsForExpandedEntityType( mEntityType, mExpandTo );

View File

@ -85,6 +85,7 @@ class QgsSensorThingsSharedData
QList< Qgis::SensorThingsEntity > mExpandTo;
int mFeatureLimit = 0;
int mExpansionLimit = 0;
Qgis::WkbType mGeometryType = Qgis::WkbType::Unknown;
QString mGeometryField;
QgsFields mFields;

View File

@ -40,6 +40,9 @@ class CORE_EXPORT QgsSensorThingsUtils
//! Default limit on number of features fetched
static constexpr int DEFAULT_FEATURE_LIMIT = 10000; SIP_SKIP
//! Default limit on number of expanded features fetched
static constexpr int DEFAULT_EXPANSION_LIMIT = 100; SIP_SKIP
/**
* Converts a string value to a Qgis::SensorThingsEntity type.
*

View File

@ -42,10 +42,16 @@ QgsSensorThingsSourceWidget::QgsSensorThingsSourceWidget( QWidget *parent )
vl->addWidget( mExtentWidget );
mExtentLimitFrame->setLayout( vl );
mSpinExpansionLimit->setEnabled( false );
mSpinPageSize->setClearValue( 0, tr( "Default (%1)" ).arg( QgsSensorThingsUtils::DEFAULT_PAGE_SIZE ) );
mSpinFeatureLimit->setClearValue( 0, tr( "No limit" ) );
mSpinExpansionLimit->setClearValue( 0, tr( "No limit" ) );
mSpinExpansionLimit->setToolTip( tr( "Limits the maximum number of related features to fetch when expanding child features" ) );
// set a relatively conservative feature limit by default, to make it so they have to opt-in to shoot themselves in the foot!
mSpinFeatureLimit->setValue( QgsSensorThingsUtils::DEFAULT_FEATURE_LIMIT );
mSpinExpansionLimit->setValue( QgsSensorThingsUtils::DEFAULT_EXPANSION_LIMIT );
for ( Qgis::SensorThingsEntity type :
{
@ -72,7 +78,11 @@ QgsSensorThingsSourceWidget::QgsSensorThingsSourceWidget( QWidget *parent )
connect( mRetrieveTypesButton, &QToolButton::clicked, this, &QgsSensorThingsSourceWidget::retrieveTypes );
mRetrieveTypesButton->setEnabled( false );
connect( mExtentWidget, &QgsExtentWidget::extentChanged, this, &QgsSensorThingsSourceWidget::validate );
connect( mComboExpandTo, qOverload< int >( &QComboBox::currentIndexChanged ), this, &QgsSensorThingsSourceWidget::validate );
connect( mComboExpandTo, qOverload< int >( &QComboBox::currentIndexChanged ), this, [ = ]( int )
{
mSpinExpansionLimit->setEnabled( mComboExpandTo->currentData().isValid() );
validate();
} );
validate();
}
@ -124,6 +134,23 @@ void QgsSensorThingsSourceWidget::setSourceUri( const QString &uri )
mSpinFeatureLimit->setValue( QgsSensorThingsUtils::DEFAULT_FEATURE_LIMIT );
}
ok = false;
const int expansionLimitParam = mSourceParts.value( QStringLiteral( "expansionLimit" ) ).toInt( &ok );
if ( ok )
{
mSpinExpansionLimit->setValue( expansionLimitParam );
}
else if ( type != Qgis::SensorThingsEntity::Invalid )
{
// if not setting an initial uri for a new layer, use "no limit" if it's not present in the uri
mSpinExpansionLimit->clear();
}
else
{
// when setting an initial uri, use the default, not "no limit"
mSpinExpansionLimit->setValue( QgsSensorThingsUtils::DEFAULT_EXPANSION_LIMIT );
}
const QgsRectangle bounds = mSourceParts.value( QStringLiteral( "bounds" ) ).value< QgsRectangle >();
if ( !bounds.isNull() )
{
@ -226,6 +253,15 @@ QString QgsSensorThingsSourceWidget::updateUriFromGui( const QString &connection
parts.remove( QStringLiteral( "featureLimit" ) );
}
if ( mSpinExpansionLimit->value() > 0 )
{
parts.insert( QStringLiteral( "expansionLimit" ), QString::number( mSpinExpansionLimit->value() ) );
}
else
{
parts.remove( QStringLiteral( "expansionLimit" ) );
}
if ( !mComboExpandTo->currentData().isValid() )
{
parts.remove( QStringLiteral( "expandTo" ) );

View File

@ -7,7 +7,7 @@
<x>0</x>
<y>0</y>
<width>537</width>
<height>180</height>
<height>213</height>
</rect>
</property>
<property name="windowTitle">
@ -26,6 +26,13 @@
<property name="bottomMargin">
<number>0</number>
</property>
<item row="1" column="0">
<widget class="QLabel" name="label_2">
<property name="text">
<string>Geometry type</string>
</property>
</widget>
</item>
<item row="1" column="2">
<widget class="QToolButton" name="mRetrieveTypesButton">
<property name="minimumSize">
@ -53,37 +60,30 @@
</property>
</widget>
</item>
<item row="5" column="0">
<item row="7" column="0">
<widget class="QLabel" name="label_5">
<property name="text">
<string>Extent limit</string>
</property>
</widget>
</item>
<item row="5" column="1" colspan="2">
<item row="2" column="1" colspan="2">
<widget class="QgsSpinBox" name="mSpinPageSize">
<property name="maximum">
<number>9999999</number>
</property>
</widget>
</item>
<item row="1" column="1">
<widget class="QComboBox" name="mComboGeometryType"/>
</item>
<item row="7" column="1" colspan="2">
<widget class="QWidget" name="mExtentLimitFrame" native="true">
<property name="focusPolicy">
<enum>Qt::StrongFocus</enum>
</property>
</widget>
</item>
<item row="0" column="1" colspan="2">
<widget class="QComboBox" name="mComboEntityType"/>
</item>
<item row="0" column="0">
<widget class="QLabel" name="label">
<property name="text">
<string>Entity type</string>
</property>
</widget>
</item>
<item row="3" column="0">
<widget class="QLabel" name="label_4">
<property name="text">
<string>Expand to</string>
</property>
</widget>
</item>
<item row="4" column="0">
<widget class="QLabel" name="label_6">
<property name="text">
@ -91,26 +91,9 @@
</property>
</widget>
</item>
<item row="4" column="1" colspan="2">
<widget class="QgsSpinBox" name="mSpinFeatureLimit">
<property name="maximum">
<number>999999999</number>
</property>
</widget>
</item>
<item row="3" column="1" colspan="2">
<item row="5" column="1" colspan="2">
<widget class="QComboBox" name="mComboExpandTo"/>
</item>
<item row="1" column="0">
<widget class="QLabel" name="label_2">
<property name="text">
<string>Geometry type</string>
</property>
</widget>
</item>
<item row="1" column="1">
<widget class="QComboBox" name="mComboGeometryType"/>
</item>
<item row="2" column="0">
<widget class="QLabel" name="label_3">
<property name="text">
@ -118,10 +101,41 @@
</property>
</widget>
</item>
<item row="2" column="1" colspan="2">
<widget class="QgsSpinBox" name="mSpinPageSize">
<item row="5" column="0">
<widget class="QLabel" name="label_4">
<property name="text">
<string>Expand to</string>
</property>
</widget>
</item>
<item row="0" column="0">
<widget class="QLabel" name="label">
<property name="text">
<string>Entity type</string>
</property>
</widget>
</item>
<item row="4" column="1" colspan="2">
<widget class="QgsSpinBox" name="mSpinFeatureLimit">
<property name="maximum">
<number>9999999</number>
<number>999999999</number>
</property>
</widget>
</item>
<item row="0" column="1" colspan="2">
<widget class="QComboBox" name="mComboEntityType"/>
</item>
<item row="6" column="0">
<widget class="QLabel" name="label_7">
<property name="text">
<string>Expansion limit</string>
</property>
</widget>
</item>
<item row="6" column="1" colspan="2">
<widget class="QgsSpinBox" name="mSpinExpansionLimit">
<property name="maximum">
<number>999999999</number>
</property>
</widget>
</item>
@ -139,8 +153,9 @@
<tabstop>mComboGeometryType</tabstop>
<tabstop>mRetrieveTypesButton</tabstop>
<tabstop>mSpinPageSize</tabstop>
<tabstop>mComboExpandTo</tabstop>
<tabstop>mSpinFeatureLimit</tabstop>
<tabstop>mComboExpandTo</tabstop>
<tabstop>mSpinExpansionLimit</tabstop>
<tabstop>mExtentLimitFrame</tabstop>
</tabstops>
<resources/>

View File

@ -4128,6 +4128,387 @@ class TestPyQgsSensorThingsProvider(QgisTestCase): # , ProviderTestCase):
'Polygon ((103 0, 104 0, 104 1, 103 1, 103 0))'],
)
def test_feature_expansion_with_limit(self):
"""
Test a layer using feature expansion with limited child features
"""
with tempfile.TemporaryDirectory() as temp_dir:
base_path = temp_dir.replace("\\", "/")
endpoint = base_path + "/fake_qgis_http_endpoint"
with open(sanitize(endpoint, ""), "wt", encoding="utf8") as f:
f.write(
"""
{
"value": [
{
"name": "Locations",
"url": "endpoint/Locations"
}
],
"serverSettings": {
}
}""".replace(
"endpoint", "http://" + endpoint
)
)
with open(
sanitize(endpoint,
"/Locations?$top=0&$count=true&$filter=location/type eq 'Polygon' or location/geometry/type eq 'Polygon'"),
"wt",
encoding="utf8",
) as f:
f.write("""{"@iot.count":3,"value":[]}""")
with open(
sanitize(endpoint,
"/Locations?$top=2&$count=false&$expand=Things/Datastreams($top=1)&$filter=location/type eq 'Polygon' or location/geometry/type eq 'Polygon'"),
"wt",
encoding="utf8",
) as f:
f.write(
"""
{
"value": [
{
"@iot.selfLink": "endpoint/Locations(1)",
"@iot.id": 1,
"name": "Location 1",
"description": "Desc 1",
"unitOfMeasurements": [
{
"name": "ug.m-3",
"symbol": "ug.m-3",
"definition": "http://dd.eionet.europa.eu/vocabulary/uom/concentration/ug.m-3"
}
],
"observationType": "http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement",
"multiObservationDataTypes": ["http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement"],
"phenomenonTime": "2017-12-31T23:00:00Z/2018-01-12T04:00:00Z",
"resultTime": "2017-12-31T23:30:00Z/2017-12-31T23:31:00Z",
"properties": {
"owner": "owner 1"
},
"Things": [
{
"@iot.selfLink": "endpoint/Things(1)",
"@iot.id": 1,
"name": "Thing 1",
"description": "Description Thing 1",
"properties": {
"countryCode": "AT"
},
"Datastreams": [
{
"@iot.selfLink": "endpoint/Datastreams(45)",
"@iot.id": 45,
"name": "Datastream 45",
"description": "Description datastream 45",
"observationType": "http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement",
"unitOfMeasurement": {
"name": "ug.m-3",
"symbol": "ug.m-3",
"definition": "http://dd.eionet.europa.eu/vocabulary/uom/concentration/ug.m-3"
},
"phenomenonTime": "2017-12-31T23:00:00Z/2024-03-25T04:00:00Z",
"properties": {
"owner": "someone"
}
}
]
}
],
"location": {
"type": "Polygon",
"coordinates": [
[
[100, 0], [101, 0], [101, 1], [100, 1], [100, 0]
]
]
},
"Things@iot.navigationLink": "endpoint/Locations(1)/Things",
"HistoricalLocations@iot.navigationLink": "endpoint/Locations(1)/HistoricalLocations"
},
{
"@iot.selfLink": "endpoint/Locations(2)",
"@iot.id": 2,
"name": "Location 2",
"description": "Desc 2",
"unitOfMeasurements": [
{
"name": "ug.m-3",
"symbol": "ug.m-3",
"definition": "http://dd.eionet.europa.eu/vocabulary/uom/concentration/ug.m-3"
}],
"observationType": "http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement",
"multiObservationDataTypes": ["http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement"],
"phenomenonTime": "2018-12-31T23:00:00Z/2019-01-12T04:00:00Z",
"resultTime": "2018-12-31T23:30:00Z/2018-12-31T23:31:00Z",
"properties": {
"owner": "owner 2"
},
"Things": [
{
"@iot.selfLink": "endpoint/Things(2)",
"@iot.id": 2,
"name": "Thing 2",
"description": "Description Thing 2",
"properties": {
"countryCode": "AT"
},
"Datastreams": [
{
"@iot.selfLink": "endpoint/Datastreams(51)",
"@iot.id": 51,
"name": "Datastream 51",
"description": "Description datastream 51",
"observationType": "http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement",
"unitOfMeasurement": {
"name": "ug.m-3",
"symbol": "ug.m-3",
"definition": "http://dd.eionet.europa.eu/vocabulary/uom/concentration/ug.m-3"
},
"phenomenonTime": "2017-12-31T23:00:00Z/2024-03-25T04:00:00Z",
"properties": {
"owner": "someone"
}
}
]
}
],
"location": {
"type": "Polygon",
"coordinates": [
[
[102, 0], [103, 0], [103, 1], [102, 1], [102, 0]
]
]
},
"Things@iot.navigationLink": "endpoint/Locations(2)/Things",
"HistoricalLocations@iot.navigationLink": "endpoint/Locations(2)/HistoricalLocations"
}
],
"@iot.nextLink": "endpoint/Locations?$top=2&$skip=2&$expand=Things/Datastreams($top=1)&$filter=location/type eq 'Polygon' or location/geometry/type eq 'Polygon'"
}
""".replace(
"endpoint", "http://" + endpoint
)
)
with open(
sanitize(endpoint,
"/Locations?$top=2&$skip=2&$expand=Things/Datastreams($top=1)&$filter=location/type eq 'Polygon' or location/geometry/type eq 'Polygon'"),
"wt",
encoding="utf8",
) as f:
f.write(
"""
{
"value": [
{
"@iot.selfLink": "endpoint/Locations(3)",
"@iot.id": 3,
"name": "Location 3",
"description": "Desc 3",
"unitOfMeasurements": [{
"name": "ug.m-3",
"symbol": "ug.m-3",
"definition": "http://dd.eionet.europa.eu/vocabulary/uom/concentration/ug.m-3"
}],
"observationType": "http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement",
"multiObservationDataTypes": ["http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement"],
"phenomenonTime": "2020-12-31T23:00:00Z/2021-01-12T04:00:00Z",
"resultTime": "2020-12-31T23:30:00Z/2020-12-31T23:31:00Z",
"properties": {
"owner": "owner 3"
},
"Things": [
{
"@iot.selfLink": "endpoint/Things(8)",
"@iot.id": 8,
"name": "Thing 8",
"description": "Description Thing 8",
"properties": {
"countryCode": "AT"
},
"Datastreams": [
{
"@iot.selfLink": "endpoint/Datastreams(59)",
"@iot.id": 59,
"name": "Datastream 59",
"description": "Description datastream 59",
"observationType": "http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement",
"unitOfMeasurement": {
"name": "ug.m-3",
"symbol": "ug.m-3",
"definition": "http://dd.eionet.europa.eu/vocabulary/uom/concentration/ug.m-3"
},
"phenomenonTime": "2017-12-31T23:00:00Z/2024-03-25T04:00:00Z",
"properties": {
"owner": "someone"
}
}
]
}
],
"location": {
"type": "Polygon",
"coordinates": [
[
[103, 0], [104, 0], [104, 1], [103, 1], [103, 0]
]
]
},
"Things@iot.navigationLink": "endpoint/Locations(3)/Things",
"HistoricalLocations@iot.navigationLink": "endpoint/Locations(3)/HistoricalLocations"
}
]
}
""".replace(
"endpoint", "http://" + endpoint
)
)
vl = QgsVectorLayer(
f"url='http://{endpoint}' pageSize=2 type=MultiPolygonZ entity='Location' expansionLimit=1 expandTo='Thing,Datastream'",
"test",
"sensorthings",
)
self.assertTrue(vl.isValid())
# basic layer properties tests
self.assertEqual(vl.storageType(), "OGC SensorThings API")
self.assertEqual(vl.wkbType(), Qgis.WkbType.MultiPolygonZ)
self.assertEqual(vl.featureCount(), -1)
self.assertEqual(vl.crs().authid(), 'EPSG:4326')
self.assertIn("Entity Type</td><td>Location</td>",
vl.htmlMetadata())
self.assertIn(f'href="http://{endpoint}/Locations"',
vl.htmlMetadata())
self.assertEqual(
[f.name() for f in vl.fields()],
['id', 'selfLink', 'name', 'description', 'properties',
'Thing_id', 'Thing_selfLink', 'Thing_name',
'Thing_description', 'Thing_properties',
'Thing_Datastream_id', 'Thing_Datastream_selfLink',
'Thing_Datastream_name', 'Thing_Datastream_description',
'Thing_Datastream_unitOfMeasurement',
'Thing_Datastream_observationType',
'Thing_Datastream_properties',
'Thing_Datastream_phenomenonTimeStart',
'Thing_Datastream_phenomenonTimeEnd',
'Thing_Datastream_resultTimeStart',
'Thing_Datastream_resultTimeEnd'],
)
self.assertEqual(
[f.type() for f in vl.fields()],
[
QVariant.String,
QVariant.String,
QVariant.String,
QVariant.String,
QVariant.Map,
QVariant.String,
QVariant.String,
QVariant.String,
QVariant.String,
QVariant.Map,
QVariant.String,
QVariant.String,
QVariant.String,
QVariant.String,
QVariant.Map,
QVariant.String,
QVariant.Map,
QVariant.DateTime,
QVariant.DateTime,
QVariant.DateTime,
QVariant.DateTime,
],
)
# test retrieving all features from layer
features = list(vl.getFeatures())
self.assertEqual([f.id() for f in features],
[0, 1, 2])
self.assertEqual([f["id"] for f in features],
["1", "2", "3"])
self.assertEqual(
[f["selfLink"][-13:] for f in features],
["/Locations(1)", "/Locations(2)",
"/Locations(3)"],
)
self.assertEqual(
[f["name"] for f in features],
["Location 1", "Location 2",
"Location 3"],
)
self.assertEqual(
[f["description"] for f in features],
["Desc 1", "Desc 2",
"Desc 3"]
)
self.assertEqual(
[f["properties"] for f in features],
[{'owner': 'owner 1'},
{'owner': 'owner 2'},
{'owner': 'owner 3'}]
)
self.assertEqual(
[f["Thing_id"] for f in features],
['1', '2', '8']
)
self.assertEqual(
[f["Thing_selfLink"][-10:] for f in features],
['/Things(1)', '/Things(2)', '/Things(8)']
)
self.assertEqual(
[f["Thing_name"] for f in features],
['Thing 1', 'Thing 2', 'Thing 8']
)
self.assertEqual(
[f["Thing_description"] for f in features],
['Description Thing 1', 'Description Thing 2',
'Description Thing 8']
)
self.assertEqual(
[f["Thing_properties"] for f in features],
[{'countryCode': 'AT'}, {'countryCode': 'AT'},
{'countryCode': 'AT'}]
)
self.assertEqual(
[f["Thing_Datastream_id"] for f in features],
['45', '51', '59']
)
self.assertEqual(
[f["Thing_Datastream_selfLink"][-16:] for f in features],
['/Datastreams(45)', '/Datastreams(51)',
'/Datastreams(59)']
)
self.assertEqual(
[f["Thing_Datastream_name"] for f in features],
['Datastream 45', 'Datastream 51', 'Datastream 59']
)
self.assertEqual(
[f["Thing_Datastream_description"] for f in features],
['Description datastream 45', 'Description datastream 51',
'Description datastream 59']
)
self.assertEqual(
[f["Thing_Datastream_properties"] for f in features],
[{'owner': 'someone'}, {'owner': 'someone'},
{'owner': 'someone'}]
)
self.assertEqual(
[f.geometry().asWkt() for f in features],
['Polygon ((100 0, 101 0, 101 1, 100 1, 100 0))',
'Polygon ((102 0, 103 0, 103 1, 102 1, 102 0))',
'Polygon ((103 0, 104 0, 104 1, 103 1, 103 0))'],
)
def testDecodeUri(self):
"""
Test decoding a SensorThings uri
@ -4222,7 +4603,7 @@ class TestPyQgsSensorThingsProvider(QgisTestCase): # , ProviderTestCase):
},
)
uri = "url='https://sometest.com/api' type=MultiPolygonZ authcfg='abc' expandTo='Thing,Datastream' entity='Location'"
uri = "url='https://sometest.com/api' type=MultiPolygonZ authcfg='abc' expandTo='Thing,Datastream' expansionLimit=30 entity='Location'"
parts = QgsProviderRegistry.instance().decodeUri("sensorthings", uri)
self.assertEqual(
parts,
@ -4231,6 +4612,7 @@ class TestPyQgsSensorThingsProvider(QgisTestCase): # , ProviderTestCase):
"entity": "Location",
"geometryType": "polygon",
"authcfg": "abc",
'expansionLimit': 30,
"expandTo": ['Thing', 'Datastream']
},
)
@ -4332,12 +4714,13 @@ class TestPyQgsSensorThingsProvider(QgisTestCase): # , ProviderTestCase):
"authcfg": "aaaaa",
"entity": "location",
"geometryType": "polygon",
"expandTo": ["Thing", "Datastream"]
"expandTo": ["Thing", "Datastream"],
'expansionLimit': 30
}
uri = QgsProviderRegistry.instance().encodeUri("sensorthings", parts)
self.assertEqual(
uri,
"authcfg=aaaaa type=MultiPolygonZ entity='Location' expandTo='Thing,Datastream' url='http://blah.com'",
"authcfg=aaaaa type=MultiPolygonZ entity='Location' expandTo='Thing,Datastream' expansionLimit='30' url='http://blah.com'",
)