Improvements to the vector tile writer

- filtering of input layers by expressions and min/max zoom level
- custom layer names in the output
- writing of custom metadata for MBTiles output
- auto-calculate output extent (instead of defaulting to the whole world's extent)
- passing transform context to the encoder

python/core/auto_generated/vectortile/qgsvectortilewriter.sip.in

@@ -31,6 +31,24 @@ be ordinary file system path, e.g.: /home/qgis/output.mbtiles
 
 Currently the writer only support MVT encoding of data.
 
+Metadata support: it is possible to pass a QVariantMap with metadata. This
+is backend dependent. Currently only "mbtiles" source type supports writing
+of metadata. The key-value pairs will be written to the "metadata" table
+in the MBTiles file. Some useful metadata keys listed here, but see MBTiles spec
+for more details:
+- "name" - human-readable name of the tileset
+- "bounds" - extent in WGS 84: "minlon,minlat,maxlon,maxlat"
+- "center" - default view of the map: "longitude,latitude,zoomlevel"
+- "minzoom" - lowest zoom level
+- "maxzoom" - highest zoom level
+- "attribution" - string that explains the sources of data
+- "description" - descriptions of the content
+- "type" - whether this is an overlay or a basemap
+- "version" - version of the tileset
+Vector tile writer always writes "format" and "json" metadata. If not specified
+in metadata by the client, tile writer also writes "name", "bounds", "minzoom"
+and "maxzoom".
+
 .. versionadded:: 3.14
 %End
 
@@ -60,6 +78,42 @@ Constructs an entry for a vector layer
         QgsVectorLayer *layer() const;
 %Docstring
 Returns vector layer of this entry
+%End
+
+        QString filterExpression() const;
+%Docstring
+Returns filter expression. If not empty, only features matching the expression will be used
+%End
+        void setFilterExpression( const QString &expr );
+%Docstring
+Sets filter expression. If not empty, only features matching the expression will be used
+%End
+
+        QString layerName() const;
+%Docstring
+Returns layer name in the output. If not set, layer()->name() will be used.
+%End
+        void setLayerName( const QString &name );
+%Docstring
+Sets layer name in the output. If not set, layer()->name() will be used.
+%End
+
+        int minZoom() const;
+%Docstring
+Returns minimum zoom level at which this layer will be used. Negative value means no min. zoom level
+%End
+        void setMinZoom( int minzoom );
+%Docstring
+Sets minimum zoom level at which this layer will be used. Negative value means no min. zoom level
+%End
+
+        int maxZoom() const;
+%Docstring
+Returns maximum zoom level at which this layer will be used. Negative value means no max. zoom level
+%End
+        void setMaxZoom( int maxzoom );
+%Docstring
+Sets maximum zoom level at which this layer will be used. Negative value means no max. zoom level
 %End
 
     };
@@ -73,7 +127,7 @@ See the class description about the syntax of destination URIs.
     void setExtent( const QgsRectangle &extent );
 %Docstring
 Sets extent of vector tile output. Currently always in EPSG:3857.
-If unset, it will use the standard range of the Web Mercator system.
+If unset, it will use the full extent of all input layers combined
 %End
 
     void setMinZoom( int minZoom );
@@ -88,6 +142,16 @@ Sets the maximum zoom level of tiles. Allowed values are in interval [0,24]
     void setLayers( const QList<QgsVectorTileWriter::Layer> &layers );
 %Docstring
 Sets vector layers and their configuration for output of vector tiles
+%End
+
+    void setMetadata( const QVariantMap &metadata );
+%Docstring
+Sets that will be written to the output dataset. See class description for more on metadata support
+%End
+
+    void setTransformContext( const QgsCoordinateTransformContext &transformContext );
+%Docstring
+Sets coordinate transform context for transforms between layers and tile matrix CRS
 %End
 
     bool writeTiles( QgsFeedback *feedback = 0 );
@@ -103,6 +167,11 @@ provide cancellation functionality.
 %Docstring
 Returns error message related to the previous call to writeTiles(). Will return
 an empty string if writing was successful.
+%End
+
+    QgsRectangle fullExtent() const;
+%Docstring
+Returns calculated extent that combines extent of all input layers
 %End
 
 };

src/core/vectortile/qgsvectortilewriter.cpp

@@ -18,6 +18,7 @@
 #include "qgsdatasourceuri.h"
 #include "qgsfeedback.h"
 #include "qgsjsonutils.h"
+#include "qgslogger.h"
 #include "qgsmbtiles.h"
 #include "qgstiles.h"
 #include "qgsvectorlayer.h"
@@ -34,7 +35,6 @@
 
 QgsVectorTileWriter::QgsVectorTileWriter()
 {
-  mExtent = QgsTileMatrix::fromWebMercator( 0 ).tileExtent( QgsTileXYZ( 0, 0, 0 ) );
 }
 
 
@@ -73,13 +73,24 @@ bool QgsVectorTileWriter::writeTiles( QgsFeedback *feedback )
     return false;
   }
 
+  QgsRectangle outputExtent = mExtent;
+  if ( outputExtent.isEmpty() )
+  {
+    outputExtent = fullExtent();
+    if ( outputExtent.isEmpty() )
+    {
+      mErrorMessage = tr( "Failed to calculate output extent" );
+      return false;
+    }
+  }
+
   // figure out how many tiles we will need to do
   int tilesToCreate = 0;
   for ( int zoomLevel = mMinZoom; zoomLevel <= mMaxZoom; ++zoomLevel )
   {
     QgsTileMatrix tileMatrix = QgsTileMatrix::fromWebMercator( zoomLevel );
 
-    QgsTileRange tileRange = tileMatrix.tileRangeFromExtent( mExtent );
+    QgsTileRange tileRange = tileMatrix.tileRangeFromExtent( outputExtent );
     tilesToCreate += ( tileRange.endRow() - tileRange.startRow() + 1 ) *
                      ( tileRange.endColumn() - tileRange.startColumn() + 1 );
   }
@@ -99,17 +110,39 @@ bool QgsVectorTileWriter::writeTiles( QgsFeedback *feedback )
     }
 
     // required metadata
-    mbtiles->setMetadataValue( "name", "???" );  // TODO: custom name?
     mbtiles->setMetadataValue( "format", "pbf" );
-
-    // optional metadata
-    mbtiles->setMetadataValue( "bounds", "-180.0,-85,180,85" );
-    mbtiles->setMetadataValue( "minzoom", QString::number( mMinZoom ) );
-    mbtiles->setMetadataValue( "maxzoom", QString::number( mMaxZoom ) );
-    // TODO: "center"? initial view with [lon,lat,zoom]
-
-    // required metadata for vector tiles: "json" with schema of layers
     mbtiles->setMetadataValue( "json", mbtilesJsonSchema() );
+
+    // metadata specified by the client
+    const QStringList metaKeys = mMetadata.keys();
+    for ( const QString &key : metaKeys )
+    {
+      mbtiles->setMetadataValue( key, mMetadata[key].toString() );
+    }
+
+    // default metadata that we always write (if not written by the client)
+    if ( !mMetadata.contains( "name" ) )
+      mbtiles->setMetadataValue( "name",  "unnamed" );  // required by the spec
+    if ( !mMetadata.contains( "minzoom" ) )
+      mbtiles->setMetadataValue( "minzoom", QString::number( mMinZoom ) );
+    if ( !mMetadata.contains( "maxzoom" ) )
+      mbtiles->setMetadataValue( "maxzoom", QString::number( mMaxZoom ) );
+    if ( !mMetadata.contains( "bounds" ) )
+    {
+      QString boundsStr = "-180,-85,180,85";  // fallback value - whole world except for poles
+      try
+      {
+        QgsCoordinateTransform ct( QgsCoordinateReferenceSystem( "EPSG:3857" ), QgsCoordinateReferenceSystem( "EPSG:4326" ), mTransformContext );
+        QgsRectangle wgsExtent = ct.transform( outputExtent );
+        boundsStr = QString( "%1,%2,%3,%4" )
+                    .arg( wgsExtent.xMinimum() ).arg( wgsExtent.yMinimum() )
+                    .arg( wgsExtent.xMaximum() ).arg( wgsExtent.yMaximum() );
+      }
+      catch ( const QgsCsException & )
+      {
+      }
+      mbtiles->setMetadataValue( "bounds", boundsStr );
+    }
   }
 
   int tilesCreated = 0;
@@ -117,17 +150,22 @@ bool QgsVectorTileWriter::writeTiles( QgsFeedback *feedback )
   {
     QgsTileMatrix tileMatrix = QgsTileMatrix::fromWebMercator( zoomLevel );
 
-    QgsTileRange tileRange = tileMatrix.tileRangeFromExtent( mExtent );
+    QgsTileRange tileRange = tileMatrix.tileRangeFromExtent( outputExtent );
     for ( int row = tileRange.startRow(); row <= tileRange.endRow(); ++row )
     {
       for ( int col = tileRange.startColumn(); col <= tileRange.endColumn(); ++col )
       {
         QgsTileXYZ tileID( col, row, zoomLevel );
         QgsVectorTileMVTEncoder encoder( tileID );
+        encoder.setTransformContext( mTransformContext );
 
         for ( const Layer &layer : qgis::as_const( mLayers ) )
         {
-          encoder.addLayer( layer.layer(), feedback );
+          if ( ( layer.minZoom() >= 0 && zoomLevel < layer.minZoom() ) ||
+               ( layer.maxZoom() >= 0 && zoomLevel > layer.maxZoom() ) )
+            continue;
+
+          encoder.addLayer( layer.layer(), feedback, layer.filterExpression(), layer.layerName() );
         }
 
         if ( feedback && feedback->isCanceled() )
@@ -169,6 +207,28 @@ bool QgsVectorTileWriter::writeTiles( QgsFeedback *feedback )
   return true;
 }
 
+QgsRectangle QgsVectorTileWriter::fullExtent() const
+{
+  QgsRectangle extent;
+  QgsCoordinateReferenceSystem destCrs( "EPSG:3857" );
+
+  for ( const Layer &layer : mLayers )
+  {
+    QgsVectorLayer *vl = layer.layer();
+    QgsCoordinateTransform ct( vl->crs(), destCrs, mTransformContext );
+    try
+    {
+      QgsRectangle r = ct.transformBoundingBox( vl->extent() );
+      extent.combineExtentWith( r );
+    }
+    catch ( const QgsCsException & )
+    {
+      QgsDebugMsg( "Failed to reproject layer extent to destination CRS" );
+    }
+  }
+  return extent;
+}
+
 bool QgsVectorTileWriter::writeTileFileXYZ( const QString &sourcePath, QgsTileXYZ tileID, const QByteArray &tileData )
 {
   QString filePath = QgsVectorTileUtils::formatXYZUrlTemplate( sourcePath, tileID );

src/core/vectortile/qgsvectortilewriter.h

@@ -18,6 +18,7 @@
 
 #include <QCoreApplication>
 #include "qgsrectangle.h"
+#include "qgscoordinatetransformcontext.h"
 
 class QgsFeedback;
 class QgsTileXYZ;
@@ -44,6 +45,25 @@ class QgsVectorLayer;
  *
  * Currently the writer only support MVT encoding of data.
  *
+ * Metadata support: it is possible to pass a QVariantMap with metadata. This
+ * is backend dependent. Currently only "mbtiles" source type supports writing
+ * of metadata. The key-value pairs will be written to the "metadata" table
+ * in the MBTiles file. Some useful metadata keys listed here, but see MBTiles spec
+ * for more details:
+ * - "name" - human-readable name of the tileset
+ * - "bounds" - extent in WGS 84: "minlon,minlat,maxlon,maxlat"
+ * - "center" - default view of the map: "longitude,latitude,zoomlevel"
+ * - "minzoom" - lowest zoom level
+ * - "maxzoom" - highest zoom level
+ * - "attribution" - string that explains the sources of data
+ * - "description" - descriptions of the content
+ * - "type" - whether this is an overlay or a basemap
+ * - "version" - version of the tileset
+ * Vector tile writer always writes "format" and "json" metadata. If not specified
+ * in metadata by the client, tile writer also writes "name", "bounds", "minzoom"
+ * and "maxzoom".
+ *
+ *
  * \since QGIS 3.14
  */
 class CORE_EXPORT QgsVectorTileWriter
@@ -70,10 +90,32 @@ class CORE_EXPORT QgsVectorTileWriter
         //! Returns vector layer of this entry
         QgsVectorLayer *layer() const { return mLayer; }
 
+        //! Returns filter expression. If not empty, only features matching the expression will be used
+        QString filterExpression() const { return mFilterExpression; }
+        //! Sets filter expression. If not empty, only features matching the expression will be used
+        void setFilterExpression( const QString &expr ) { mFilterExpression = expr; }
+
+        //! Returns layer name in the output. If not set, layer()->name() will be used.
+        QString layerName() const { return mLayerName; }
+        //! Sets layer name in the output. If not set, layer()->name() will be used.
+        void setLayerName( const QString &name ) { mLayerName = name; }
+
+        //! Returns minimum zoom level at which this layer will be used. Negative value means no min. zoom level
+        int minZoom() const { return mMinZoom; }
+        //! Sets minimum zoom level at which this layer will be used. Negative value means no min. zoom level
+        void setMinZoom( int minzoom ) { mMinZoom = minzoom; }
+
+        //! Returns maximum zoom level at which this layer will be used. Negative value means no max. zoom level
+        int maxZoom() const { return mMaxZoom; }
+        //! Sets maximum zoom level at which this layer will be used. Negative value means no max. zoom level
+        void setMaxZoom( int maxzoom ) { mMaxZoom = maxzoom; }
+
       private:
         QgsVectorLayer *mLayer;
-        //QString mFilterExpression;
-        //QString mLayerName;
+        QString mFilterExpression;
+        QString mLayerName;
+        int mMinZoom = -1;
+        int mMaxZoom = -1;
     };
 
     /**
@@ -84,7 +126,7 @@ class CORE_EXPORT QgsVectorTileWriter
 
     /**
      * Sets extent of vector tile output. Currently always in EPSG:3857.
-     * If unset, it will use the standard range of the Web Mercator system.
+     * If unset, it will use the full extent of all input layers combined
      */
     void setExtent( const QgsRectangle &extent ) { mExtent = extent; }
 
@@ -96,6 +138,12 @@ class CORE_EXPORT QgsVectorTileWriter
     //! Sets vector layers and their configuration for output of vector tiles
     void setLayers( const QList<QgsVectorTileWriter::Layer> &layers ) { mLayers = layers; }
 
+    //! Sets that will be written to the output dataset. See class description for more on metadata support
+    void setMetadata( const QVariantMap &metadata ) { mMetadata = metadata; }
+
+    //! Sets coordinate transform context for transforms between layers and tile matrix CRS
+    void setTransformContext( const QgsCoordinateTransformContext &transformContext ) { mTransformContext = transformContext; }
+
     /**
      * Writes vector tiles according to the configuration.
      * Returns true on success (upon failure one can get error cause using errorMessage())
@@ -111,6 +159,9 @@ class CORE_EXPORT QgsVectorTileWriter
      */
     QString errorMessage() const { return mErrorMessage; }
 
+    //! Returns calculated extent that combines extent of all input layers
+    QgsRectangle fullExtent() const;
+
   private:
     bool writeTileFileXYZ( const QString &sourcePath, QgsTileXYZ tileID, const QByteArray &tileData );
     QString mbtilesJsonSchema();
@@ -121,6 +172,8 @@ class CORE_EXPORT QgsVectorTileWriter
     int mMaxZoom = 4;
     QList<Layer> mLayers;
     QString mDestinationUri;
+    QVariantMap mMetadata;
+    QgsCoordinateTransformContext mTransformContext;
 
     QString mErrorMessage;
 };

tests/src/core/testqgsvectortilewriter.cpp

@@ -19,6 +19,7 @@
 
 //qgis includes...
 #include "qgsapplication.h"
+#include "qgsmbtiles.h"
 #include "qgsproject.h"
 #include "qgstiles.h"
 #include "qgsvectorlayer.h"
@@ -51,6 +52,8 @@ class TestQgsVectorTileWriter : public QObject
 
     void test_basic();
     void test_mbtiles();
+    void test_mbtiles_metadata();
+    void test_filtering();
 };
 
 
@@ -210,6 +213,108 @@ void TestQgsVectorTileWriter::test_mbtiles()
   delete vtLayer;
 }
 
+void TestQgsVectorTileWriter::test_mbtiles_metadata()
+{
+  // here we test that the metadata we pass to the writer get stored properly
+
+  QString fileName = QDir::tempPath() + "/test_qgsvectortilewriter_metadata.mbtiles";
+  if ( QFile::exists( fileName ) )
+    QFile::remove( fileName );
+
+  QgsDataSourceUri ds;
+  ds.setParam( "type", "mbtiles" );
+  ds.setParam( "url", fileName );
+
+  QgsVectorLayer *vlPoints = new QgsVectorLayer( mDataDir + "/points.shp", "points", "ogr" );
+
+  QVariantMap meta;
+  meta["name"] = "QGIS rocks!";
+  meta["attribution"] = "QGIS sample data";
+
+  QgsVectorTileWriter writer;
+  writer.setDestinationUri( ds.encodedUri() );
+  writer.setMaxZoom( 1 );
+  writer.setLayers( QList<QgsVectorTileWriter::Layer>() << QgsVectorTileWriter::Layer( vlPoints ) );
+  writer.setMetadata( meta );
+
+  bool res = writer.writeTiles();
+  QVERIFY( res );
+  QVERIFY( writer.errorMessage().isEmpty() );
+
+  delete vlPoints;
+
+  // do some checks on the output
+
+  QgsMbTiles reader( fileName );
+  QVERIFY( reader.open() );
+  QCOMPARE( reader.metadataValue( "name" ), QStringLiteral( "QGIS rocks!" ) );
+  QCOMPARE( reader.metadataValue( "attribution" ), QStringLiteral( "QGIS sample data" ) );
+  QCOMPARE( reader.metadataValue( "description" ), QString() );  // was not specified
+  QCOMPARE( reader.metadataValue( "minzoom" ).toInt(), 0 );
+  QCOMPARE( reader.metadataValue( "maxzoom" ).toInt(), 1 );
+}
+
+void TestQgsVectorTileWriter::test_filtering()
+{
+  // test filtering of layers by expression and by min/max zoom level
+
+  QString fileName = QDir::tempPath() + "/test_qgsvectortilewriter_filtering.mbtiles";
+  if ( QFile::exists( fileName ) )
+    QFile::remove( fileName );
+
+  QgsDataSourceUri ds;
+  ds.setParam( "type", "mbtiles" );
+  ds.setParam( "url", fileName );
+
+  QgsVectorLayer *vlPoints = new QgsVectorLayer( mDataDir + "/points.shp", "points", "ogr" );
+  QgsVectorLayer *vlLines = new QgsVectorLayer( mDataDir + "/lines.shp", "lines", "ogr" );
+  QgsVectorLayer *vlPolys = new QgsVectorLayer( mDataDir + "/polys.shp", "polys", "ogr" );
+
+  QList<QgsVectorTileWriter::Layer> layers;
+  layers << QgsVectorTileWriter::Layer( vlPoints );
+  layers << QgsVectorTileWriter::Layer( vlLines );
+  layers << QgsVectorTileWriter::Layer( vlPolys );
+
+  layers[0].setLayerName( "b52" );
+  layers[0].setFilterExpression( "Class = 'B52'" );
+  layers[1].setMaxZoom( 1 ); // lines only [0,1]
+  layers[2].setMinZoom( 1 ); // polys only [1,2,3]
+
+  QgsVectorTileWriter writer;
+  writer.setDestinationUri( ds.encodedUri() );
+  writer.setMaxZoom( 3 );
+  writer.setLayers( layers );
+
+  bool res = writer.writeTiles();
+  QVERIFY( res );
+  QVERIFY( writer.errorMessage().isEmpty() );
+
+  delete vlPoints;
+  delete vlLines;
+  delete vlPolys;
+
+  // do some checks on the output
+
+  QgsVectorTileLayer *vtLayer = new QgsVectorTileLayer( ds.encodedUri(), "output" );
+
+  QByteArray tile0 = vtLayer->getRawTile( QgsTileXYZ( 0, 0, 0 ) );
+  QgsVectorTileMVTDecoder decoder;
+  bool resDecode0 = decoder.decode( QgsTileXYZ( 0, 0, 0 ), tile0 );
+  QVERIFY( resDecode0 );
+  QStringList layerNames = decoder.layers();
+  QCOMPARE( layerNames, QStringList() << "b52" << "lines" );
+
+  QMap<QString, QgsFields> perLayerFields;
+  perLayerFields["polys"] = QgsFields();
+  perLayerFields["lines"] = QgsFields();
+  perLayerFields["b52"] = QgsFields();
+
+  QgsVectorTileFeatures features0 = decoder.layerFeatures( perLayerFields, QgsCoordinateTransform() );
+  QCOMPARE( features0["b52"].count(), 4 );
+  QCOMPARE( features0["lines"].count(), 6 );
+  QCOMPARE( features0["polys"].count(), 0 );
+}
+
 
 QGSTEST_MAIN( TestQgsVectorTileWriter )
 #include "testqgsvectortilewriter.moc"