Review fixes

python/core/auto_generated/processing/qgsprocessingparametervectortilewriterlayers.sip.in

@@ -0,0 +1,117 @@
+/************************************************************************
+ * This file has been generated automatically from                      *
+ *                                                                      *
+ * src/core/processing/qgsprocessingparametervectortilewriterlayers.h   *
+ *                                                                      *
+ * Do not edit manually ! Edit header and run scripts/sipify.pl again   *
+ ************************************************************************/
+
+
+
+
+class QgsProcessingParameterVectorTileWriterLayers : QgsProcessingParameterDefinition
+{
+%Docstring
+A parameter for processing algorithms that need a list of input vector layers for writing
+of vector tiles - this parameter provides processing framework's adapter for QList<QgsVectorTileWriter.Layer>.
+
+A valid value for this parameter is a list (QVariantList), where each item is a map (QVariantMap) in this form:
+{
+'layer':  <string> or <QgsMapLayer>,
+// key-value pairs below are optional
+'layerName': <string>,
+'filterExpression': <string>,
+'minZoom': <int>,
+'maxZoom': <int>
+}
+
+Static functions parametersAsLayers(), variantMapAsLayer(), layerAsVariantMap() provide conversion between
+QgsVectorTileWriter.Layer representation and QVariant representation.
+
+.. note::
+
+   This class is not a part of public API.
+
+.. versionadded:: 3.14
+%End
+
+%TypeHeaderCode
+#include "qgsprocessingparametervectortilewriterlayers.h"
+%End
+  public:
+    QgsProcessingParameterVectorTileWriterLayers( const QString &name, const QString &description = QString() );
+%Docstring
+Constructor for QgsProcessingParameterVectorTileWriterLayers.
+%End
+
+    virtual QgsProcessingParameterDefinition *clone() const;
+
+    virtual QString type() const;
+
+    virtual bool checkValueIsAcceptable( const QVariant &input, QgsProcessingContext *context ) const;
+
+    virtual QString valueAsPythonString( const QVariant &value, QgsProcessingContext &context ) const;
+
+    virtual QString asPythonString( QgsProcessing::PythonOutputType outputType = QgsProcessing::PythonQgsProcessingAlgorithmSubclass ) const;
+
+
+    static QString typeName();
+%Docstring
+Returns the type name for the parameter class.
+%End
+
+    static QList<QgsVectorTileWriter::Layer> parameterAsLayers( const QVariant &layersVariant, QgsProcessingContext &context );
+%Docstring
+Converts a QVariant value (a QVariantList) to a list of input layers
+%End
+    static QgsVectorTileWriter::Layer variantMapAsLayer( const QVariantMap &layerVariantMap, QgsProcessingContext &context );
+%Docstring
+Converts a QVariant value (a QVariantMap) to a single input layer
+%End
+    static QVariantMap layerAsVariantMap( const QgsVectorTileWriter::Layer &layer );
+%Docstring
+Converts a single input layer to QVariant representation (a QVariantMap)
+%End
+
+};
+
+
+class QgsProcessingParameterTypeVectorTileWriterLayers : QgsProcessingParameterType
+{
+%Docstring
+Parameter type definition for QgsProcessingParameterVectorTileWriterLayers.
+
+.. note::
+
+   This class is not a part of public API.
+
+.. versionadded:: 3.14
+%End
+
+%TypeHeaderCode
+#include "qgsprocessingparametervectortilewriterlayers.h"
+%End
+  public:
+    virtual QgsProcessingParameterDefinition *create( const QString &name ) const /Factory/;
+
+    virtual QString description() const;
+
+    virtual QString name() const;
+
+    virtual QString id() const;
+
+    virtual QString pythonImportString() const;
+
+    virtual QString className() const;
+
+    virtual QStringList acceptedPythonTypes() const;
+};
+
+
+/************************************************************************
+ * This file has been generated automatically from                      *
+ *                                                                      *
+ * src/core/processing/qgsprocessingparametervectortilewriterlayers.h   *
+ *                                                                      *
+ * Do not edit manually ! Edit header and run scripts/sipify.pl again   *
+ ************************************************************************/

python/core/core_auto.sip

@@ -453,6 +453,7 @@
 %Include auto_generated/processing/qgsprocessingoutputs.sip
 %Include auto_generated/processing/qgsprocessingparameters.sip
 %Include auto_generated/processing/qgsprocessingparametertype.sip
+%Include auto_generated/processing/qgsprocessingparametervectortilewriterlayers.sip
 %Include auto_generated/processing/qgsprocessingprovider.sip
 %Include auto_generated/processing/qgsprocessingregistry.sip
 %Include auto_generated/processing/qgsprocessingutils.sip

src/core/processing/qgsprocessingparametervectortilewriterlayers.cpp

@@ -74,17 +74,17 @@ QString QgsProcessingParameterVectorTileWriterLayers::valueAsPythonString( const
   for ( const QgsVectorTileWriter::Layer &layer : layers )
   {
     QStringList layerDefParts;
-    layerDefParts << "'layer': " + QgsProcessingUtils::stringToPythonLiteral( QgsProcessingUtils::normalizeLayerSource( layer.layer()->source() ) );
+    layerDefParts << QStringLiteral( "'layer': " ) + QgsProcessingUtils::stringToPythonLiteral( QgsProcessingUtils::normalizeLayerSource( layer.layer()->source() ) );
     if ( !layer.filterExpression().isEmpty() )
-      layerDefParts << "'filterExpression': " + QgsProcessingUtils::variantToPythonLiteral( layer.filterExpression() );
+      layerDefParts << QStringLiteral( "'filterExpression': " ) + QgsProcessingUtils::variantToPythonLiteral( layer.filterExpression() );
     if ( !layer.layerName().isEmpty() )
-      layerDefParts << "'layerName': " + QgsProcessingUtils::variantToPythonLiteral( layer.layerName() );
+      layerDefParts << QStringLiteral( "'layerName': " ) + QgsProcessingUtils::variantToPythonLiteral( layer.layerName() );
     if ( layer.minZoom() >= 0 )
-      layerDefParts << "'minZoom': " + QgsProcessingUtils::variantToPythonLiteral( layer.minZoom() );
+      layerDefParts << QStringLiteral( "'minZoom': " ) + QgsProcessingUtils::variantToPythonLiteral( layer.minZoom() );
     if ( layer.maxZoom() >= 0 )
-      layerDefParts << "'maxZoom': " + QgsProcessingUtils::variantToPythonLiteral( layer.maxZoom() );
+      layerDefParts << QStringLiteral( "'maxZoom': " ) + QgsProcessingUtils::variantToPythonLiteral( layer.maxZoom() );
 
-    QString layerDef = QString( "{ %1 }" ).arg( layerDefParts.join( ',' ) );
+    QString layerDef = QStringLiteral( "{ %1 }" ).arg( layerDefParts.join( ',' ) );
     parts << layerDef;
   }
   return parts.join( ',' ).prepend( '[' ).append( ']' );
@@ -96,8 +96,7 @@ QString QgsProcessingParameterVectorTileWriterLayers::asPythonString( QgsProcess
   {
     case QgsProcessing::PythonQgsProcessingAlgorithmSubclass:
     {
-      QString code = QStringLiteral( "QgsProcessingParameterVectorTileWriterLayers('%1', '%2'" ).arg( name(), description() );
-      code += QStringLiteral( ")" );
+      QString code = QStringLiteral( "QgsProcessingParameterVectorTileWriterLayers('%1', '%2')" ).arg( name(), description() );
       return code;
     }
   }

src/core/processing/qgsprocessingparametervectortilewriterlayers.h

@@ -16,8 +16,6 @@
 #ifndef QGSPROCESSINGPARAMETERVECTORTILEWRITERLAYERS_H
 #define QGSPROCESSINGPARAMETERVECTORTILEWRITERLAYERS_H
 
-#define SIP_NO_FILE
-
 #include "qgsprocessingparameters.h"
 #include "qgsprocessingparametertype.h"
 #include "qgsvectortilewriter.h"
@@ -76,7 +74,7 @@ class CORE_EXPORT QgsProcessingParameterVectorTileWriterLayers : public QgsProce
  * \note This class is not a part of public API.
  * \since QGIS 3.14
  */
-class QgsProcessingParameterTypeVectorTileWriterLayers : public QgsProcessingParameterType
+class CORE_EXPORT QgsProcessingParameterTypeVectorTileWriterLayers : public QgsProcessingParameterType
 {
   public:
     QgsProcessingParameterDefinition *create( const QString &name ) const override SIP_FACTORY
@@ -98,6 +96,21 @@ class QgsProcessingParameterTypeVectorTileWriterLayers : public QgsProcessingPar
     {
       return QgsProcessingParameterVectorTileWriterLayers::typeName();
     }
+
+    QString pythonImportString() const override
+    {
+      return QStringLiteral( "from qgis.core import QgsProcessingParameterVectorTileWriterLayers" );
+    }
+
+    QString className() const override
+    {
+      return QStringLiteral( "QgsProcessingParameterVectorTileWriterLayers" );
+    }
+
+    QStringList acceptedPythonTypes() const override
+    {
+      return QStringList() << QObject::tr( "list[dict]: list of input layers as dictionaries, see QgsProcessingParameterVectorTileWriterLayers docs" );
+    }
 };
 
 

src/core/vectortile/qgsvectortiledownloadmanager.cpp

@@ -0,0 +1,157 @@
+
+#include "qgsnetworkaccessmanager.h"
+
+class TileReply : public QObject
+{
+    QOBJECT
+  public:
+    TileReply( QString url ): mUrl( url ) {}
+    ~TileReply()
+    {
+      // TODO: notify manager
+      //if ( active )
+      //  DownloadManager::cancelRequest( this );
+    }
+    QString url() const { return mUrl; }
+
+    //void setParentReply( QNetworkReply *networkReply ) { mNetworkReply = networkReply; }
+    //QNetworkReply *parentReply() const { return mNetworkReply; }
+
+    void setData( const QByteArray &data ) { mData = data; }
+    QByteArray data() const { return mData; }
+
+  signals:
+    void finished();
+
+  private:
+    QString mUrl;
+    //QNetworkReply *mNetworkReply = nullptr;
+    QByteArray mData;
+};
+
+class DownloadManager;
+
+class Worker : public QObject
+{
+    QOBJECT
+
+  public slots:
+
+    void request( QString url )
+    {
+      // this is only called if such request does not exist already
+
+      QNetworkRequest request( url );
+      request.setAttribute( ( QNetworkRequest::Attribute )( QNetworkRequest::User + 1 ), url );
+      QNetworkReply *networkReply = QgsNetworkAccessManager::instance()->get( request );
+      connect( networkReply, &QNetworkReply::finished, this, &Worker::tileReplyFinished );
+    }
+
+    void cancelRequest( QString url )
+    {
+      // this is only called if such request has been requested here
+
+      QNetworkReply *r = mNetworkReplies.take( url );
+      r->abort();
+    }
+
+  private slots:
+    void tileReplyFinished()
+    {
+      QNetworkReply *reply = qobject_cast<QNetworkReply *>( sender() );
+
+      QString url = reply->request().attribute( ( QNetworkRequest::Attribute )( QNetworkRequest::User + 1 ) ).toString();
+      QByteArray data = reply->readAll();
+
+      mNetworkReplies.remove( url );
+      reply->deleteLater();
+
+      // there may be multiple replies waiting for the same network request
+      QMutexLocker locker( &DownloadManager::mutex );
+      const QList<TileReply *> tileReplies = DownloadManager::replies[url];
+      for ( TileReply *rr : tileReplies )
+      {
+        rr->setData( data );
+        QMetaObject::invokeMethod( rr, "finished" );
+      }
+    }
+
+  private:
+    QMap<QString, QNetworkReply *> mNetworkReplies;
+
+};
+
+//
+// TODO:
+// - DownloadManager - singleton?
+// - auto-kill thread after some time?
+// - auto-start thread when needed
+// - in-memory caching
+//
+
+/*
+ * singleton that lives in the main thread.
+ */
+class DownloadManager
+{
+  public:
+
+    static void startThread()
+    {
+      worker = new Worker;
+      worker->moveToThread( &workerThread );
+      QObject::connect( &workerThread, &QThread::finished, worker, &QObject::deleteLater );
+      workerThread.start();
+    }
+
+    static void stopThread()
+    {
+      workerThread.quit();
+      workerThread.wait();
+    }
+
+    //! thread-safe
+    static TileReply *requestTile( const QString &url )
+    {
+      QMutexLocker locker( &mutex );
+
+      TileReply *reply = new TileReply( url ); // lives in the same thread as the caller
+
+      if ( replies.contains( url ) )
+      {
+        // no extra work to do - previous reply is not finished yet
+        replies[url].append( reply );
+      }
+      else
+      {
+        replies[url] = QList<TileReply *>() << reply;
+        // asynchronously request network request
+        QMetaObject::invokeMethod( worker, "request", Qt::QueuedConnection, Q_ARG( QString, url ) );
+      }
+
+      return reply;
+    }
+
+    //! thread-safe
+    static void cancelTileRequest( TileReply *reply )
+    {
+      QMutexLocker locker( &mutex );
+
+      QString url = reply->url();
+
+      replies[url].removeOne( reply );
+
+      if ( replies[url].isEmpty() )
+      {
+        replies.remove( url );
+        // also make sure the underlying request is cancelled
+        QMetaObject::invokeMethod( worker, "cancelRequest", Qt::QueuedConnection, Q_ARG( QString, url ) );
+      }
+    }
+
+  private:
+    static QThread workerThread;  // will run its event loop
+    static Worker *worker;
+    static QMutex mutex;  // protecting data structures with replies
+    static QMap<QString, QList<TileReply *> > replies;
+};

src/core/vectortile/qgsvectortileutils.cpp

@@ -134,6 +134,13 @@ QString QgsVectorTileUtils::formatXYZUrlTemplate( const QString &url, QgsTileXYZ
   return turl;
 }
 
+bool QgsVectorTileUtils::checkXYZUrlTemplate( const QString &url )
+{
+  return url.contains( QStringLiteral( "{x}" ) ) &&
+         url.contains( QStringLiteral( "{y}" ) ) &&
+         url.contains( QStringLiteral( "{z}" ) );
+}
+
 //! a helper class for ordering tile requests according to the distance from view center
 struct LessThanTileRequest
 {

src/core/vectortile/qgsvectortileutils.h

@@ -61,6 +61,8 @@ class CORE_EXPORT QgsVectorTileUtils
     static QgsVectorLayer *makeVectorLayerForTile( QgsVectorTileLayer *mvt, QgsTileXYZ tileID, const QString &layerName );
     //! Returns formatted tile URL string replacing {x}, {y}, {z} placeholders
     static QString formatXYZUrlTemplate( const QString &url, QgsTileXYZ tile );
+    //! Checks whether the URL template string is correct (contains {x}, {y}, {z} placeholders)
+    static bool checkXYZUrlTemplate( const QString &url );
 };
 
 #endif // QGSVECTORTILEUTILS_H

src/core/vectortile/qgsvectortilewriter.cpp

@@ -62,6 +62,12 @@ bool QgsVectorTileWriter::writeTiles( QgsFeedback *feedback )
   {
     // remove the initial file:// scheme
     sourcePath = QUrl( sourcePath ).toLocalFile();
+
+    if ( !QgsVectorTileUtils::checkXYZUrlTemplate( sourcePath ) )
+    {
+      mErrorMessage = tr( "Invalid template for XYZ: " ) + sourcePath;
+      return false;
+    }
   }
   else if ( sourceType == QStringLiteral( "mbtiles" ) )
   {

src/gui/processing/qgsprocessingvectortilewriterlayerswidgetwrapper.cpp

@@ -29,6 +29,7 @@
 
 #include "qgsprocessingparametervectortilewriterlayers.h"
 
+/// @cond private
 
 //
 // QgsProcessingVectorTileWriteLayerDetailsWidget
@@ -362,3 +363,5 @@ QStringList QgsProcessingVectorTileWriterLayersWidgetWrapper::compatibleOutputTy
 {
   return QStringList();
 }
+
+/// @endcond

src/gui/processing/qgsprocessingvectortilewriterlayerswidgetwrapper.h

@@ -120,11 +120,6 @@ class QgsProcessingVectorTileWriterLayersWidgetWrapper : public QgsAbstractProce
     // QgsProcessingParameterWidgetFactoryInterface
     QString parameterType() const override;
     QgsAbstractProcessingParameterWidgetWrapper *createWidgetWrapper( const QgsProcessingParameterDefinition *parameter, QgsProcessingGui::WidgetType type ) override;
-//    QgsProcessingAbstractParameterDefinitionWidget *createParameterDefinitionWidget(
-//      QgsProcessingContext &context,
-//      const QgsProcessingParameterWidgetContext &widgetContext,
-//      const QgsProcessingParameterDefinition *definition = nullptr,
-//      const QgsProcessingAlgorithm *algorithm = nullptr ) override;
 
     // QgsProcessingParameterWidgetWrapper interface
     QWidget *createWidget() override SIP_FACTORY;
@@ -137,7 +132,6 @@ class QgsProcessingVectorTileWriterLayersWidgetWrapper : public QgsAbstractProce
 
     QStringList compatibleParameterTypes() const override;
     QStringList compatibleOutputTypes() const override;
-//    QString modelerExpressionFormatString() const override;
 
   private: