Introduced QgsProjectStorage and QgsProjectStorageRegistry

This is going to be used as an abstraction of how/where project
files are stored.

python/core/core_auto.sip

@@ -91,6 +91,8 @@
 %Include qgsprojectbadlayerhandler.sip
 %Include qgsprojectfiletransform.sip
 %Include qgsprojectproperty.sip
+%Include qgsprojectstorage.sip
+%Include qgsprojectstorageregistry.sip
 %Include qgsprojectversion.sip
 %Include qgsproperty.sip
 %Include qgspropertycollection.sip

python/core/qgsapplication.sip.in

@@ -735,6 +735,13 @@ Returns registry of available 3D renderers.
    not available in Python bindings
 
 .. versionadded:: 3.0
+%End
+
+    static QgsProjectStorageRegistry *projectStorageRegistry();
+%Docstring
+Returns registry of available project storage implementations.
+
+.. versionadded:: 3.2
 %End
 
     static QString nullRepresentation();

python/core/qgsprojectstorage.sip.in

@@ -0,0 +1,64 @@
+/************************************************************************
+ * This file has been generated automatically from                      *
+ *                                                                      *
+ * src/core/qgsprojectstorage.h                                         *
+ *                                                                      *
+ * Do not edit manually ! Edit header and run scripts/sipify.pl again   *
+ ************************************************************************/
+
+
+
+
+
+class QgsProjectStorage
+{
+%Docstring
+Abstract interface for project storage - to be implemented by various backends
+and registered in QgsProjectStorageRegistry.
+
+.. versionadded:: 3.2
+%End
+
+%TypeHeaderCode
+#include "qgsprojectstorage.h"
+%End
+  public:
+    virtual ~QgsProjectStorage();
+
+    virtual QString type() = 0;
+%Docstring
+Unique identifier of the project storage type. If type() returns "memory", all project file names
+starting with "memory:" will have read/write redirected through that storage implementation.
+%End
+
+    virtual QStringList listProjects( const QString &uri ) = 0;
+%Docstring
+Returns list of all projects for given URI (specific to each storage backend)
+%End
+
+    virtual bool readProject( const QString &uri, QIODevice *device, QgsReadWriteContext &context ) = 0;
+%Docstring
+Reads project file content stored in the backend at the specified URI to the given device
+(could be e.g. a temporary file or a memory buffer). The device is expected to be empty
+when passed to readProject() so that the method can write all data to it and then rewind
+it using seek(0) to make it ready for reading in :py:class:`QgsProject`.
+%End
+
+    virtual bool writeProject( const QString &uri, QIODevice *device, QgsReadWriteContext &context ) = 0;
+%Docstring
+Writes project file content stored in given device (could be e.g. a temporary file or a memory buffer)
+using the backend to the specified URI. The device is expected to contain all project file data
+and having position at the start of the content when passed to writeProject() so that the method
+can read all data from it until it reaches its end.
+%End
+
+
+};
+
+/************************************************************************
+ * This file has been generated automatically from                      *
+ *                                                                      *
+ * src/core/qgsprojectstorage.h                                         *
+ *                                                                      *
+ * Do not edit manually ! Edit header and run scripts/sipify.pl again   *
+ ************************************************************************/

python/core/qgsprojectstorageregistry.sip.in

@@ -0,0 +1,57 @@
+/************************************************************************
+ * This file has been generated automatically from                      *
+ *                                                                      *
+ * src/core/qgsprojectstorageregistry.h                                 *
+ *                                                                      *
+ * Do not edit manually ! Edit header and run scripts/sipify.pl again   *
+ ************************************************************************/
+
+
+
+
+
+class QgsProjectStorageRegistry
+{
+%Docstring
+Registry of storage backends that QgsProject may use.
+This is a singleton that should be accessed through :py:func:`QgsApplication.projectStorageRegistry()`
+
+.. versionadded:: 3.2
+%End
+
+%TypeHeaderCode
+#include "qgsprojectstorageregistry.h"
+%End
+  public:
+    QgsProjectStorageRegistry();
+    ~QgsProjectStorageRegistry();
+
+    QgsProjectStorage *projectStorageFromUri( const QString &uri );
+%Docstring
+Returns storage implementation if the URI matches one. Returns null pointer otherwise (it is a normal file)
+%End
+
+    QList<QgsProjectStorage *> projectStorages() const;
+%Docstring
+Returns a list of registered project storage implementations
+%End
+
+    void registerProjectStorage( QgsProjectStorage *storage /Transfer/ );
+%Docstring
+Registers a storage backend and takes ownership of it
+%End
+
+    void unregisterProjectStorage( QgsProjectStorage *storage );
+%Docstring
+Unregisters a storage backend and destroys its instance
+%End
+
+};
+
+/************************************************************************
+ * This file has been generated automatically from                      *
+ *                                                                      *
+ * src/core/qgsprojectstorageregistry.h                                 *
+ *                                                                      *
+ * Do not edit manually ! Edit header and run scripts/sipify.pl again   *
+ ************************************************************************/

src/core/CMakeLists.txt

@@ -259,6 +259,8 @@ SET(QGIS_CORE_SRCS
   qgsprojectfiletransform.cpp
   qgssnappingconfig.cpp
   qgsprojectproperty.cpp
+  qgsprojectstorage.cpp
+  qgsprojectstorageregistry.cpp
   qgsprojectversion.cpp
   qgsproperty.cpp
   qgspropertycollection.cpp
@@ -889,6 +891,8 @@ SET(QGIS_CORE_HDRS
   qgsprojectbadlayerhandler.h
   qgsprojectfiletransform.h
   qgsprojectproperty.h
+  qgsprojectstorage.h
+  qgsprojectstorageregistry.h
   qgsprojectversion.h
   qgsproperty.h
   qgsproperty_p.h

src/core/qgsapplication.cpp

@@ -31,6 +31,7 @@
 #include "qgssvgcache.h"
 #include "qgscolorschemeregistry.h"
 #include "qgspainteffectregistry.h"
+#include "qgsprojectstorageregistry.h"
 #include "qgsrasterrendererregistry.h"
 #include "qgsrendererregistry.h"
 #include "qgssymbollayerregistry.h"
@@ -1710,6 +1711,11 @@ Qgs3DRendererRegistry *QgsApplication::renderer3DRegistry()
   return members()->m3DRendererRegistry;
 }
 
+QgsProjectStorageRegistry *QgsApplication::projectStorageRegistry()
+{
+  return members()->mProjectStorageRegistry;
+}
+
 QgsApplication::ApplicationMembers::ApplicationMembers()
 {
   // don't use initializer lists or scoped pointers - as more objects are added here we
@@ -1733,6 +1739,7 @@ QgsApplication::ApplicationMembers::ApplicationMembers()
   mLayoutItemRegistry->populate();
   mAnnotationRegistry = new QgsAnnotationRegistry();
   m3DRendererRegistry = new Qgs3DRendererRegistry();
+  mProjectStorageRegistry = new QgsProjectStorageRegistry();
 }
 
 QgsApplication::ApplicationMembers::~ApplicationMembers()
@@ -1747,6 +1754,7 @@ QgsApplication::ApplicationMembers::~ApplicationMembers()
   delete mPaintEffectRegistry;
   delete mPluginLayerRegistry;
   delete mProcessingRegistry;
+  delete mProjectStorageRegistry;
   delete mPageSizeRegistry;
   delete mLayoutItemRegistry;
   delete mProfiler;

src/core/qgsapplication.h

@@ -30,6 +30,7 @@ class QgsTaskManager;
 class QgsFieldFormatterRegistry;
 class QgsColorSchemeRegistry;
 class QgsPaintEffectRegistry;
+class QgsProjectStorageRegistry;
 class QgsRendererRegistry;
 class QgsSvgCache;
 class QgsSymbolLayerRegistry;
@@ -668,6 +669,12 @@ class CORE_EXPORT QgsApplication : public QApplication
      */
     static Qgs3DRendererRegistry *renderer3DRegistry();
 
+    /**
+     * Returns registry of available project storage implementations.
+     * \since QGIS 3.2
+     */
+    static QgsProjectStorageRegistry *projectStorageRegistry();
+
     /**
      * This string is used to represent the value `NULL` throughout QGIS.
      *
@@ -799,6 +806,7 @@ class CORE_EXPORT QgsApplication : public QApplication
       QgsPaintEffectRegistry *mPaintEffectRegistry = nullptr;
       QgsPluginLayerRegistry *mPluginLayerRegistry = nullptr;
       QgsProcessingRegistry *mProcessingRegistry = nullptr;
+      QgsProjectStorageRegistry *mProjectStorageRegistry = nullptr;
       QgsPageSizeRegistry *mPageSizeRegistry = nullptr;
       QgsRasterRendererRegistry *mRasterRendererRegistry = nullptr;
       QgsRendererRegistry *mRendererRegistry = nullptr;

src/core/qgsproject.cpp

@@ -29,6 +29,8 @@
 #include "qgsprojectfiletransform.h"
 #include "qgssnappingconfig.h"
 #include "qgspathresolver.h"
+#include "qgsprojectstorage.h"
+#include "qgsprojectstorageregistry.h"
 #include "qgsprojectversion.h"
 #include "qgsrasterlayer.h"
 #include "qgsreadwritecontext.h"
@@ -822,6 +824,26 @@ bool QgsProject::read()
   QString filename = mFile.fileName();
   bool rc;
 
+  QgsProjectStorage *storage = QgsApplication::projectStorageRegistry()->projectStorageFromUri( filename );
+  if ( storage )
+  {
+    QTemporaryFile inDevice;
+    if ( !inDevice.open() )
+    {
+      setError( tr( "Unable to open %1" ).arg( inDevice.fileName() ) );
+      return false;
+    }
+
+    QgsReadWriteContext context;
+    if ( !storage->readProject( filename, &inDevice, context ) )
+    {
+      setError( tr( "Unable to open %1" ).arg( filename ) );
+      return false;
+    }
+
+    return unzip( inDevice.fileName() );  // calls setError() if returning false
+  }
+
   if ( QgsZipUtils::isZipFile( mFile.fileName() ) )
   {
     rc = unzip( mFile.fileName() );
@@ -1374,6 +1396,39 @@ bool QgsProject::write( const QString &filename )
 
 bool QgsProject::write()
 {
+  QgsProjectStorage *projectStorage = QgsApplication::projectStorageRegistry()->projectStorageFromUri( mFile.fileName() );
+  if ( projectStorage )
+  {
+    // for projects stored in a custom storage, we cannot use relative paths since the storage most likely
+    // will not be in a file system
+    writeEntry( QStringLiteral( "Paths" ), QStringLiteral( "/Absolute" ), true );
+
+    QString tempPath = QStandardPaths::standardLocations( QStandardPaths::TempLocation ).at( 0 );
+    QString tmpZipFilename( tempPath + QDir::separator() + QUuid::createUuid().toString() );
+
+    if ( !zip( tmpZipFilename ) )
+      return false;  // zip() already calls setError() when returning false
+
+    QFile tmpZipFile( tmpZipFilename );
+    if ( !tmpZipFile.open( QIODevice::ReadOnly ) )
+    {
+      setError( tr( "Unable to read file %1" ).arg( tmpZipFilename ) );
+      return false;
+    }
+
+    QgsReadWriteContext context;
+    if ( !projectStorage->writeProject( mFile.fileName(), &tmpZipFile, context ) )
+    {
+      setError( tr( "Unable to save project to storage %1" ).arg( mFile.fileName() ) );
+      return false;
+    }
+
+    tmpZipFile.close();
+    QFile::remove( tmpZipFilename );
+
+    return true;
+  }
+
   if ( QgsZipUtils::isZipFile( mFile.fileName() ) )
   {
     return zip( mFile.fileName() );

src/core/qgsproject.h

@@ -56,6 +56,7 @@ class QgsMapLayer;
 class QgsMapThemeCollection;
 class QgsPathResolver;
 class QgsProjectBadLayerHandler;
+class QgsProjectStorage;
 class QgsRelationManager;
 class QgsTolerance;
 class QgsTransactionGroup;

src/core/qgsprojectstorage.cpp

@@ -0,0 +1,21 @@
+/***************************************************************************
+  qgsprojectstorage.cpp
+  --------------------------------------
+  Date                 : March 2018
+  Copyright            : (C) 2018 by Martin Dobias
+  Email                : wonder dot sk at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsprojectstorage.h"
+
+
+QgsProjectStorage::~QgsProjectStorage()
+{
+}

src/core/qgsprojectstorage.h

@@ -0,0 +1,68 @@
+/***************************************************************************
+  qgsprojectstorage.h
+  --------------------------------------
+  Date                 : March 2018
+  Copyright            : (C) 2018 by Martin Dobias
+  Email                : wonder dot sk at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSPROJECTSTORAGE_H
+#define QGSPROJECTSTORAGE_H
+
+#include "qgis_core.h"
+
+class QIODevice;
+class QString;
+class QStringList;
+
+class QgsReadWriteContext;
+
+/**
+ * Abstract interface for project storage - to be implemented by various backends
+ * and registered in QgsProjectStorageRegistry.
+ *
+ * \since QGIS 3.2
+ */
+class CORE_EXPORT QgsProjectStorage
+{
+  public:
+    virtual ~QgsProjectStorage();
+
+    /**
+     * Unique identifier of the project storage type. If type() returns "memory", all project file names
+     * starting with "memory:" will have read/write redirected through that storage implementation.
+     */
+    virtual QString type() = 0;
+
+    //! Returns list of all projects for given URI (specific to each storage backend)
+    virtual QStringList listProjects( const QString &uri ) = 0;
+
+    /**
+     * Reads project file content stored in the backend at the specified URI to the given device
+     * (could be e.g. a temporary file or a memory buffer). The device is expected to be empty
+     * when passed to readProject() so that the method can write all data to it and then rewind
+     * it using seek(0) to make it ready for reading in QgsProject.
+     */
+    virtual bool readProject( const QString &uri, QIODevice *device, QgsReadWriteContext &context ) = 0;
+
+    /**
+     * Writes project file content stored in given device (could be e.g. a temporary file or a memory buffer)
+     * using the backend to the specified URI. The device is expected to contain all project file data
+     * and having position at the start of the content when passed to writeProject() so that the method
+     * can read all data from it until it reaches its end.
+     */
+    virtual bool writeProject( const QString &uri, QIODevice *device, QgsReadWriteContext &context ) = 0;
+
+    // TODO: rename / remove ?
+
+    // TODO: load/save GUI
+};
+
+#endif // QGSPROJECTSTORAGE_H

src/core/qgsprojectstorageregistry.cpp

@@ -0,0 +1,55 @@
+/***************************************************************************
+  qgsprojectstorageregistry.cpp
+  --------------------------------------
+  Date                 : March 2018
+  Copyright            : (C) 2018 by Martin Dobias
+  Email                : wonder dot sk at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsprojectstorageregistry.h"
+
+#include "qgsprojectstorage.h"
+
+QgsProjectStorageRegistry::QgsProjectStorageRegistry()
+{
+
+}
+
+QgsProjectStorageRegistry::~QgsProjectStorageRegistry()
+{
+  qDeleteAll( mBackends.values() );
+}
+
+QgsProjectStorage *QgsProjectStorageRegistry::projectStorageFromUri( const QString &uri )
+{
+  for ( auto it = mBackends.constBegin(); it != mBackends.constEnd(); ++it )
+  {
+    QgsProjectStorage *storage = it.value();
+    QString scheme = storage->type() + ":";
+    if ( uri.startsWith( scheme ) )
+      return storage;
+  }
+  return nullptr;
+}
+
+QList<QgsProjectStorage *> QgsProjectStorageRegistry::projectStorages() const
+{
+  return mBackends.values();
+}
+
+void QgsProjectStorageRegistry::registerProjectStorage( QgsProjectStorage *storage )
+{
+  mBackends.insert( storage->type(), storage );
+}
+
+void QgsProjectStorageRegistry::unregisterProjectStorage( QgsProjectStorage *storage )
+{
+  delete mBackends.take( storage->type() );
+}

src/core/qgsprojectstorageregistry.h

@@ -0,0 +1,54 @@
+/***************************************************************************
+  qgsprojectstorageregistry.h
+  --------------------------------------
+  Date                 : March 2018
+  Copyright            : (C) 2018 by Martin Dobias
+  Email                : wonder dot sk at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSPROJECTSTORAGEREGISTRY_H
+#define QGSPROJECTSTORAGEREGISTRY_H
+
+#include "qgis_core.h"
+#include "qgis_sip.h"
+
+#include <QHash>
+
+class QgsProjectStorage;
+
+/**
+ * Registry of storage backends that QgsProject may use.
+ * This is a singleton that should be accessed through QgsApplication::projectStorageRegistry().
+ *
+ * \since QGIS 3.2
+ */
+class CORE_EXPORT QgsProjectStorageRegistry
+{
+  public:
+    QgsProjectStorageRegistry();
+    ~QgsProjectStorageRegistry();
+
+    //! Returns storage implementation if the URI matches one. Returns null pointer otherwise (it is a normal file)
+    QgsProjectStorage *projectStorageFromUri( const QString &uri );
+
+    //! Returns a list of registered project storage implementations
+    QList<QgsProjectStorage *> projectStorages() const;
+
+    //! Registers a storage backend and takes ownership of it
+    void registerProjectStorage( QgsProjectStorage *storage SIP_TRANSFER );
+
+    //! Unregisters a storage backend and destroys its instance
+    void unregisterProjectStorage( QgsProjectStorage *storage );
+
+  private:
+    QHash<QString, QgsProjectStorage *> mBackends;
+};
+
+#endif // QGSPROJECTSTORAGEREGISTRY_H

tests/src/core/CMakeLists.txt

@@ -155,6 +155,7 @@ SET(TESTS
  testqgspointpatternfillsymbol.cpp
  testqgspoint.cpp
  testqgsproject.cpp
+ testqgsprojectstorage.cpp
  testqgsproperty.cpp
  testqgis.cpp
  testqgsrasterfilewriter.cpp

tests/src/core/testqgsprojectstorage.cpp

@@ -0,0 +1,160 @@
+/***************************************************************************
+     testqgsprojectstorage.cpp
+     --------------------------------------
+    Date                 : March 2018
+    Copyright            : (C) 2018 by Martin Dobias
+    Email                : wonder.sk at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgstest.h"
+
+#include "qgsproject.h"
+#include "qgsprojectstorage.h"
+#include "qgsprojectstorageregistry.h"
+#include "qgsvectorlayer.h"
+
+
+class TestQgsProjectStorage : public QObject
+{
+    Q_OBJECT
+  private slots:
+    void initTestCase();// will be called before the first testfunction is executed.
+    void cleanupTestCase();// will be called after the last testfunction was executed.
+    void init();// will be called before each testfunction is executed.
+    void cleanup();// will be called after every testfunction.
+
+    void testMemoryStorage();
+};
+
+void TestQgsProjectStorage::init()
+{
+}
+
+void TestQgsProjectStorage::cleanup()
+{
+  // will be called after every testfunction.
+}
+
+void TestQgsProjectStorage::initTestCase()
+{
+  // Runs once before any tests are run
+
+  // Set up the QgsSettings environment
+  QCoreApplication::setOrganizationName( QStringLiteral( "QGIS" ) );
+  QCoreApplication::setOrganizationDomain( QStringLiteral( "qgis.org" ) );
+  QCoreApplication::setApplicationName( QStringLiteral( "QGIS-TEST" ) );
+
+  QgsApplication::init();
+  QgsApplication::initQgis();
+}
+
+
+void TestQgsProjectStorage::cleanupTestCase()
+{
+  // Runs once after all tests are run
+  QgsApplication::exitQgis();
+}
+
+
+//! A simple storage implementation that stores projects in memory
+class MemoryStorage : public QgsProjectStorage
+{
+  public:
+    virtual QString type() override
+    {
+      return QStringLiteral( "memory" );
+    }
+
+    virtual QStringList listProjects( const QString &uri ) override
+    {
+      Q_UNUSED( uri );
+      return mProjects.keys();
+    }
+
+    virtual bool readProject( const QString &uri, QIODevice *ioDevice, QgsReadWriteContext &context ) override
+    {
+      QgsReadWriteContextCategoryPopper popper( context.enterCategory( "memory storage" ) );
+
+      QStringList lst = uri.split( ":" );
+      Q_ASSERT( lst.count() == 2 );
+      QString projectName = lst[1];
+
+      if ( !mProjects.contains( projectName ) )
+      {
+        context.pushMessage( "project not found", Qgis::Critical );
+        return false;
+      }
+
+      QByteArray content = mProjects[projectName];
+      ioDevice->write( content );
+      ioDevice->seek( 0 );
+      return true;
+    }
+
+    virtual bool writeProject( const QString &uri, QIODevice *ioDevice, QgsReadWriteContext &context ) override
+    {
+      Q_UNUSED( context );
+      QStringList lst = uri.split( ":" );
+      Q_ASSERT( lst.count() == 2 );
+      QString projectName = lst[1];
+
+      mProjects[projectName] = ioDevice->readAll();
+      return true;
+    }
+
+  private:
+    QHash<QString, QByteArray> mProjects;
+};
+
+
+void TestQgsProjectStorage::testMemoryStorage()
+{
+  QString dataDir( TEST_DATA_DIR ); // defined in CmakeLists.txt
+  QString layerPath = dataDir + "/points.shp";
+  QgsVectorLayer *layer1 = new QgsVectorLayer( layerPath, QStringLiteral( "points" ), QStringLiteral( "ogr" ) );
+  QVERIFY( layer1->isValid() );
+
+  MemoryStorage *memStorage = new MemoryStorage;
+
+  QgsApplication::projectStorageRegistry()->registerProjectStorage( memStorage );
+
+  QVERIFY( QgsApplication::projectStorageRegistry()->projectStorages().contains( memStorage ) );
+
+  QCOMPARE( memStorage->listProjects( QString() ).count(), 0 );
+
+  QgsProject prj1;
+  prj1.setTitle( "best project ever" );
+  prj1.addMapLayer( layer1 );
+  prj1.setFileName( "memory:project1" );
+  bool writeOk = prj1.write();
+
+  QVERIFY( writeOk );
+  QCOMPARE( memStorage->listProjects( QString() ).count(), 1 );
+
+  QgsProject prj2;
+  prj2.setFileName( "memory:project1" );
+  bool readOk = prj2.read();
+
+  QVERIFY( readOk );
+  QCOMPARE( prj2.mapLayers().count(), 1 );
+  QCOMPARE( prj2.title(), QString( "best project ever" ) );
+
+  QgsProject prj3;
+  prj3.setFileName( "memory:nooooooooo!" );
+  bool readInvalidOk = prj3.read();
+
+  QVERIFY( !readInvalidOk );
+
+  QgsApplication::projectStorageRegistry()->unregisterProjectStorage( memStorage );
+}
+
+
+QGSTEST_MAIN( TestQgsProjectStorage )
+#include "testqgsprojectstorage.moc"