[FEATURE] Settings migration framework (#5080)

Only run for default profile and only if
not run before. Moves settings and symbols from
QGIS 2.x to QGIS 3 default profile

* --version-migration flag to force migration

resources/2to3migration.txt

@@ -0,0 +1,19 @@
+# version=1
+# If you update this file make sure you bump the above version number it must be higher then the last run.
+#oldkey;newkey
+
+# Connections
+MSSQL/connections/*;*
+Qgis/connections-xyz/*;qgis/connections-xyz/*
+Qgis/connections-wms/*;qgis/connections-wms/*
+Qgis/connections-wfs/*;qgis/connections-wfs/*
+
+# random stuff
+browser/favourites;*
+svg/searchPathsForSVG;*
+Qgis/compileExpressions;*
+
+# variables
+variables/names;*
+variables/values;*
+

resources/CMakeLists.txt

@@ -1,4 +1,4 @@
-INSTALL(FILES srs.db qgis.db symbology-style.xml spatialite.db customization.xml
+INSTALL(FILES srs.db qgis.db symbology-style.xml spatialite.db customization.xml 2to3migration.txt
         DESTINATION ${QGIS_DATA_DIR}/resources)
 INSTALL(FILES qgis_global_settings.ini
         DESTINATION ${QGIS_DATA_DIR})

src/app/CMakeLists.txt

@@ -52,6 +52,7 @@ SET(QGIS_APP_SRCS
   qgsmapcanvasdockwidget.cpp
   qgsmaplayerstyleguiutils.cpp
   qgsmapsavedialog.cpp
+  qgsversionmigration.cpp
   qgsrulebasedlabelingwidget.cpp
   qgssavestyletodbdialog.cpp
   qgssnappinglayertreemodel.cpp

src/app/main.cpp

@@ -99,6 +99,7 @@ typedef SInt32 SRefCon;
 #include "qgis_app.h"
 #include "qgscrashhandler.h"
 #include "qgsziputils.h"
+#include "qgsversionmigration.h"
 
 #include "qgsuserprofilemanager.h"
 #include "qgsuserprofile.h"
@@ -139,6 +140,7 @@ void usage( const QString &appName )
       << QStringLiteral( "\t[--dxf-preset maptheme]\tmap theme to use for dxf output\n" )
       << QStringLiteral( "\t[--profile name]\tload a named profile from the users profiles folder.\n" )
       << QStringLiteral( "\t[--profiles-path path]\tpath to store user profile folders. Will create profiles inside a {path}\\profiles folder \n" )
+      << QStringLiteral( "\t[--version-migration]\tforce the settings migration from older version if found\n" )
       << QStringLiteral( "\t[--help]\t\tthis text\n" )
       << QStringLiteral( "\t[--]\t\ttreat all following arguments as FILEs\n\n" )
       << QStringLiteral( "  FILE:\n" )
@@ -501,6 +503,7 @@ int main( int argc, char *argv[] )
   int mySnapshotHeight = 600;
 
   bool myHideSplash = false;
+  bool mySettingsMigrationForce = false;
   bool mySkipVersionCheck = false;
 #if defined(ANDROID)
   QgsDebugMsg( QString( "Android: Splash hidden" ) );
@@ -570,6 +573,10 @@ int main( int argc, char *argv[] )
       {
         myHideSplash = true;
       }
+      else if ( arg == QLatin1String( "--version-migration" ) )
+      {
+        mySettingsMigrationForce = true;
+      }
       else if ( arg == QLatin1String( "--noversioncheck" ) || arg == QLatin1String( "-V" ) )
       {
         mySkipVersionCheck = true;
@@ -849,6 +856,18 @@ int main( int argc, char *argv[] )
     }
   }
 
+  // Settings migration is only supported on the default profile for now.
+  if ( profileName == "default" )
+  {
+    QgsVersionMigration *migration = QgsVersionMigration::canMigrate( 20000, Qgis::QGIS_VERSION_INT );
+    if ( migration && ( mySettingsMigrationForce || migration->requiresMigration() ) )
+    {
+      QgsDebugMsg( "RUNNING MIGRATION" );
+      migration->runMigration();
+      delete migration;
+    }
+  }
+
 #ifdef Q_OS_MAC
   // Set hidpi icons; use SVG icons, as PNGs will be relatively too small
   QCoreApplication::setAttribute( Qt::AA_UseHighDpiPixmaps );

src/app/qgsversionmigration.cpp

@@ -0,0 +1,293 @@
+/***************************************************************************
+  qgsversionmigration.cpp - QgsVersionMigration
+
+ ---------------------
+ begin                : 30.7.2017
+ copyright            : (C) 2017 by nathan
+ email                : woodrow.nathan at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+#include "qgsversionmigration.h"
+#include "qgssettings.h"
+#include "qgslogger.h"
+#include "qsettings.h"
+#include "qgsmessagelog.h"
+#include "qgsapplication.h"
+#include "qgssymbol.h"
+#include "qgsstyle.h"
+#include "qgssymbollayerutils.h"
+#include "qgsreadwritecontext.h"
+
+#include <QFile>
+#include <QTextStream>
+#include <QDir>
+#include <QSqlDatabase>
+#include <QSqlQuery>
+#include <QSqlError>
+#include <QDomDocument>
+
+QgsVersionMigration::QgsVersionMigration()
+{
+
+}
+
+QgsVersionMigration::~QgsVersionMigration()
+{
+
+}
+
+QgsVersionMigration *QgsVersionMigration::canMigrate( int fromVersion, int toVersion )
+{
+  if ( fromVersion == 20000 && toVersion >= 29900 )
+  {
+    return new Qgs2To3Migration();
+  }
+  return nullptr;
+}
+
+QgsError Qgs2To3Migration::runMigration()
+{
+  QgsError error;
+  QgsError settingsErrors = migrateSettings();
+  if ( !settingsErrors.isEmpty() )
+  {
+    // TODO Merge error messages
+  }
+  QgsError stylesError = migrateStyles();
+  return error;
+}
+
+bool Qgs2To3Migration::requiresMigration()
+{
+  QgsSettings settings;
+  bool alreadyMigrated = settings.value( QStringLiteral( "migration/settings" ), false ).toBool();
+  int  settingsMigrationVersion = settings.value( QStringLiteral( "migration/fileVersion" ), 0 ).toInt();
+  QFile migrationFile( migrationFilePath() );
+  if ( migrationFile.open( QIODevice::ReadOnly | QIODevice::Text ) )
+  {
+    QTextStream in( &migrationFile );
+    QString line = in.readLine();
+    if ( line.startsWith( "#" ) && line.contains( QStringLiteral( "version=" ) ) )
+    {
+      QStringList parts = line.split( '=' );
+      mMigrationFileVersion = parts.at( 1 ).toInt();
+      QgsDebugMsg( QString( "File version is=%1" ).arg( mMigrationFileVersion ) );
+    }
+    migrationFile.close();
+  }
+  else
+  {
+    QString msg = QString( "Can not open %1" ).arg( migrationFile.fileName() );
+    QgsDebugMsg( msg );
+    mMigrationFileVersion = settingsMigrationVersion;
+  }
+
+  return ( !alreadyMigrated || settingsMigrationVersion != mMigrationFileVersion );
+}
+
+QgsError Qgs2To3Migration::migrateStyles()
+{
+  QgsError error;
+  QString oldHome = QStringLiteral( "%1/.qgis2" ).arg( QDir::homePath() );
+  QString oldStyleFile = QStringLiteral( "%1/symbology-ng-style.db" ).arg( oldHome );
+  QgsDebugMsg( QString( "OLD STYLE FILE %1" ).arg( oldStyleFile ) );
+  QSqlDatabase db = QSqlDatabase::addDatabase( "QSQLITE", "migration" );
+  db.setDatabaseName( oldStyleFile );
+  if ( !db.open() )
+  {
+    error.append( db.lastError().text() );
+    QgsDebugMsg( db.lastError().text() );
+    return error;
+  }
+
+  QSqlQuery query( db );
+  QSqlQuery tagQuery( "SELECT name FROM tag"
+                      "JOIN tagmap ON tagmap.tag_id = tag.id"
+                      "WHERE tagmap.symbol_id = :symbol_id", db );
+
+  QgsStyle *style = QgsStyle::defaultStyle();
+  if ( query.exec( "SELECT id, name, xml FROM symbol" ) )
+  {
+    while ( query.next() )
+    {
+      QString symbol_id = query.value( 0 ).toString();
+      QString name = query.value( 1 ).toString();
+      QString xml = query.value( 2 ).toString();
+      QDomDocument doc;
+      if ( !doc.setContent( xml ) )
+      {
+        QgsDebugMsg( "Cannot open symbol " + name );
+        continue;
+      }
+
+      tagQuery.bindValue( ":symbol_id", symbol_id );
+
+      QStringList tags;
+      if ( tagQuery.exec() )
+      {
+        while ( query.next() )
+        {
+          QString tagname = query.value( 0 ).toString();
+          tags << tagname;
+        }
+      }
+
+      QDomElement symElement = doc.documentElement();
+      QgsDebugMsg( QString( "MIGRATION: Importing %1" ).arg( name ) );
+      QgsSymbol *symbol = QgsSymbolLayerUtils::loadSymbol( symElement, QgsReadWriteContext() );
+      tags << "QGIS 2";
+      if ( style->symbolId( name ) == 0 )
+      {
+        style->saveSymbol( name, symbol, false, tags );
+      }
+    }
+  }
+
+  QgsDebugMsg( oldStyleFile );
+  return error;
+}
+
+QgsError Qgs2To3Migration::migrateSettings()
+{
+  QgsError error;
+
+  QgsSettings newSettings;
+
+  // The platform default location for the settings from 2.x
+  mOldSettings = new QSettings( "QGIS", "QGIS2" );
+
+  QFile inputFile( migrationFilePath() );
+  std::map<QString, QgsSettings::Section> sections;
+  sections["none"] = QgsSettings::NoSection;
+  sections["core"] = QgsSettings::Core;
+  sections["gui"] = QgsSettings::Gui;
+  sections["server"] = QgsSettings::Server;
+  sections["plugins"] = QgsSettings::Plugins;
+  sections["auth"] = QgsSettings::Auth;
+  sections["app"] = QgsSettings::App;
+  sections["providers"] = QgsSettings::Providers;
+  sections["misc"] = QgsSettings::Misc;
+
+  QList<QPair<QString, QString>> keys;
+
+  if ( inputFile.open( QIODevice::ReadOnly | QIODevice::Text ) )
+  {
+    QTextStream in( &inputFile );
+    while ( !in.atEnd() )
+    {
+      QString line = in.readLine();
+
+      if ( line.startsWith( "#" ) )
+        continue;
+
+      if ( line.isEmpty() )
+        continue;
+
+      QStringList parts = line.split( ";" );
+
+      Q_ASSERT_X( parts.count() == 2, "QgsVersionMigration::migrateSettings()", "Can't split line in 2 parts." );
+
+      QString oldKey = parts.at( 0 );
+      QString newKey = parts.at( 1 );
+
+      if ( oldKey.endsWith( "/*" ) )
+      {
+        oldKey = oldKey.replace( "/*", "" );
+        QList<QPair<QString, QString>> keyList = walk( oldKey, newKey );
+        keys.append( keyList );
+      }
+      else
+      {
+        QPair<QString, QString> key = transformKey( oldKey, newKey );
+        keys.append( key );
+      }
+
+    }
+    inputFile.close();
+    newSettings.setValue( QStringLiteral( "migration/settings" ), true );
+    // Set the dev gen so we can force a migration.
+    newSettings.setValue( QStringLiteral( "migration/fileVersion" ), mMigrationFileVersion );
+  }
+  else
+  {
+    QString msg = QString( "Can not open %1" ).arg( inputFile.fileName() );
+    QgsDebugMsg( msg );
+    error.append( msg );
+  }
+
+  if ( keys.count() > 0 )
+  {
+    QgsDebugMsg( "MIGRATION: Translating settings keys" );
+    QList<QPair<QString, QString>>::iterator i;
+    for ( i = keys.begin(); i != keys.end(); ++i )
+    {
+      QPair<QString, QString> pair = *i;
+
+      QString oldKey = pair.first;
+      QString newKey = pair.second;
+
+      if ( oldKey.contains( oldKey ) )
+      {
+        QgsDebugMsg( QString( " -> %1 -> %2" ).arg( oldKey, newKey ) );
+        newSettings.setValue( newKey, mOldSettings->value( oldKey ) );
+      }
+    }
+  }
+  return error;
+}
+
+QList<QPair<QString, QString> > Qgs2To3Migration::walk( QString group, QString newkey )
+{
+  mOldSettings->beginGroup( group );
+  QList<QPair<QString, QString> > foundKeys;
+  Q_FOREACH ( const QString &group, mOldSettings->childGroups() )
+  {
+    QList<QPair<QString, QString> > data = walk( group, newkey );
+    foundKeys.append( data );
+  }
+
+  Q_FOREACH ( const QString &key, mOldSettings->childKeys() )
+  {
+    QString fullKey = mOldSettings->group() + "/" + key;
+    foundKeys.append( transformKey( fullKey, newkey ) );
+  }
+  mOldSettings->endGroup();
+  return foundKeys;
+}
+
+QPair<QString, QString> Qgs2To3Migration::transformKey( QString fullOldKey, QString newKeyPart )
+{
+  QString newKey = newKeyPart;
+  QString oldKey = fullOldKey;
+
+  if ( newKeyPart == QStringLiteral( "*" ) )
+  {
+    newKey = fullOldKey;
+  }
+
+  if ( newKeyPart.endsWith( "/*" ) )
+  {
+    QStringList newKeyparts = newKeyPart.split( "/" );
+    // Throw away the *
+    newKeyparts.removeLast();
+    QStringList oldKeyParts = fullOldKey.split( "/" );
+    for ( int i = 0; i < newKeyparts.count(); ++i )
+    {
+      oldKeyParts.replace( i, newKeyparts.at( i ) );
+    }
+    newKey = oldKeyParts.join( "/" );
+  }
+
+  return qMakePair( oldKey, newKey );
+}
+
+QString Qgs2To3Migration::migrationFilePath()
+{
+  return QgsApplication::pkgDataPath() +  "/resources/2to3migration.txt";
+}

src/app/qgsversionmigration.h

@@ -0,0 +1,73 @@
+/***************************************************************************
+  qgsversionmigration.h - QgsVersionMigration
+
+ ---------------------
+ begin                : 30.7.2017
+ copyright            : (C) 2017 by nathan
+ email                : woodrow.nathan at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+#ifndef QGSVERSIONMIGRATION_H
+#define QGSVERSIONMIGRATION_H
+
+#include "qgis_app.h"
+#include "qgserror.h"
+
+#include <QSettings>
+
+class QgsSettings;
+
+/**
+ * Version migration class used to transfer settings, etc between major versions.
+ * \note Not everything can be translated and depends on the from and to versions.
+ */
+class APP_EXPORT QgsVersionMigration
+{
+  public:
+    QgsVersionMigration();
+    virtual ~QgsVersionMigration();
+
+    /**
+     * Check if two version has a migration options.
+     * @param fromVersion The version migrating from.
+     * @param toVersion The version migrating to.
+     * @return
+     */
+    static QgsVersionMigration *canMigrate( int fromVersion, int toVersion );
+
+    /**
+     * Run the version migration to convert between versions.
+     * @return QgsError containing any error messages when running the conversion.
+     */
+    virtual QgsError runMigration() = 0;
+    virtual bool requiresMigration() = 0;
+};
+
+
+class Qgs2To3Migration : public QgsVersionMigration
+{
+  public:
+    virtual QgsError runMigration() override;
+    virtual bool requiresMigration() override;
+  private:
+    QgsError migrateStyles();
+    QgsError migrateSettings();
+
+    QList<QPair<QString, QString>> walk( QString group, QString newkey );
+    QPair<QString, QString> transformKey( QString fullOldKey, QString newKeyPart );
+
+    QString migrationFilePath();
+
+    int mMigrationFileVersion;
+
+    QSettings *mOldSettings;
+
+};
+
+#endif // QGSVERSIONMIGRATION_H

src/core/symbology/qgsstyle.cpp

@@ -362,7 +362,7 @@ bool QgsStyle::load( const QString &filename )
   // Make sure there are no Null fields in parenting symbols and groups
   char *query = sqlite3_mprintf( "UPDATE symbol SET favorite=0 WHERE favorite IS NULL;"
                                  "UPDATE colorramp SET favorite=0 WHERE favorite IS NULL;"
-                                 "UPDATE symgroup SET parent=0 WHERE parent IS NULL;" );
+                               );
   runEmptyQuery( query );
 
   // First create all the main symbols