QGIS/scripts/pyqt5_to_pyqt6/pyqt5_to_pyqt6.py
2024-02-13 08:13:43 +10:00

623 lines
26 KiB
Python
Executable File

#!/usr/bin/env python3
"""
***************************************************************************
3to4.py
---------------------
Date : 2023 December
Copyright : (C) 2023 by Julien Cabieces
Email : julien dot cabieces at oslandia dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************
"""
"""
Migrate a folder containing python files from QGIS3/Qt5 to QGIS4/Qt6
Highly inspired from this video https://www.youtube.com/watch?v=G1omxo5pphw
needed tools:
pip install astpretty tokenize-rt
Few useful commands:
- display python file ast
astpretty --no-show-offsets myfile.py
- display file tokens
tokenize-rt myfile.py
"""
__author__ = 'Julien Cabieces'
__date__ = '2023 December'
__copyright__ = '(C) 2023, Julien Cabieces'
import argparse
import ast
import glob
import os
import inspect
import sys
from enum import Enum
from collections import defaultdict
from tokenize_rt import Offset, src_to_tokens, tokens_to_src, reversed_enumerate, Token
from typing import Sequence
from PyQt6 import QtCore, QtGui, QtWidgets, QtTest, QtSql, QtSvg, QtXml, QtNetwork, QtPrintSupport, Qsci
from PyQt6.QtCore import * # noqa: F403
from PyQt6.QtGui import * # noqa: F403
from PyQt6.QtWidgets import * # noqa: F403
from PyQt6.QtTest import * # noqa: F403
from PyQt6.QtSql import * # noqa: F403
from PyQt6.QtXml import * # noqa: F403
from PyQt6.QtNetwork import * # noqa: F403
from PyQt6.QtPrintSupport import * # noqa: F403
from PyQt6.Qsci import * # noqa: F403
try:
import qgis.core as qgis_core # noqa: F403
import qgis.gui as qgis_gui # noqa: F403
import qgis.analysis as qgis_analysis # noqa: F403
import qgis._3d as qgis_3d # noqa: F403
from qgis.core import * # noqa: F403
from qgis.gui import * # noqa: F403
from qgis.analysis import * # noqa: F403
from qgis._3d import * # noqa: F403
except ImportError:
qgis_core = None
qgis_gui = None
qgis_analysis = None
qgis_3d = None
print('QGIS classes not available for introspection, only a partial upgrade will be performed')
target_modules = [QtCore,
QtGui,
QtWidgets,
QtTest,
QtSql,
QtSvg,
QtXml,
QtNetwork,
QtPrintSupport,
Qsci]
if qgis_core is not None:
target_modules.extend([
qgis_core,
qgis_gui,
qgis_analysis,
qgis_3d
]
)
# qmetatype which have been renamed
qmetatype_mapping = {
"Invalid": "UnknownType",
"BitArray": "QBitArray",
"Bitmap": "QBitmap",
"Brush": "QBrush",
"ByteArray": "QByteArray",
"Char": "QChar",
"Color": "QColor",
"Cursor": "QCursor",
"Date": "QDate",
"DateTime": "QDateTime",
"EasingCurve": "QEasingCurve",
"Uuid": "QUuid",
"ModelIndex": "QModelIndex",
"PersistentModelIndex": "QPersistentModelIndex",
"Font": "QFont",
"Hash": "QVariantHash",
"Icon": "QIcon",
"Image": "QImage",
"KeySequence": "QKeySequence",
"Line": "QLine",
"LineF": "QLineF",
"List": "QVariantList",
"Locale": "QLocale",
"Map": "QVariantMap",
"Transform": "QTransform",
"Matrix4x4": "QMatrix4x4",
"Palette": "QPalette",
"Pen": "QPen",
"Pixmap": "QPixmap",
"Point": "QPoint",
"PointF": "QPointF",
"Polygon": "QPolygon",
"PolygonF": "QPolygonF",
"Quaternion": "QQuaternion",
"Rect": "QRect",
"RectF": "QRectF",
"RegularExpression": "QRegularExpression",
"Region": "QRegion",
"Size": "QSize",
"SizeF": "QSizeF",
"SizePolicy": "QSizePolicy",
"String": "QString",
"StringList": "QStringList",
"TextFormat": "QTextFormat",
"TextLength": "QTextLength",
"Time": "QTime",
"Url": "QUrl",
"Vector2D": "QVector2D",
"Vector3D": "QVector3D",
"Vector4D": "QVector4D",
"UserType": "User",
}
deprecated_renamed_enums = {
('Qt', 'MidButton'): ('MouseButton', 'MiddleButton'),
('Qt', 'TextColorRole'): ('ItemDataRole', 'ForegroundRole'),
('Qt', 'BackgroundColorRole'): ('ItemDataRole', 'BackgroundRole'),
}
rename_function_attributes = {
'exec_': 'exec'
}
rename_function_definitions = {
'exec_': 'exec'
}
import_warnings = {
'QRegExp': 'QRegExp is removed in Qt6, please use QRegularExpression for Qt5/Qt6 compatibility'
}
# { (class, enum_value) : enum_name }
qt_enums = {}
ambiguous_enums = defaultdict(set)
def fix_file(filename: str, qgis3_compat: bool) -> int:
with open(filename, encoding='UTF-8') as f:
contents = f.read()
fix_qvariant_type = [] # QVariant.Int, QVariant.Double ...
fix_pyqt_import = [] # from PyQt5.QtXXX
fix_qt_enums = {} # Unscoping of enums
member_renames = {}
token_renames = {}
function_def_renames = {}
rename_qt_enums = [] # Renaming deprecated removed enums
custom_updates = {}
imported_modules = set()
extra_imports = defaultdict(set)
removed_imports = defaultdict(set)
import_offsets = {}
def visit_call(_node: ast.Call, _parent):
if isinstance(_node.func, ast.Attribute):
if _node.func.attr in rename_function_attributes:
attr_node = _node.func
member_renames[
Offset(_node.func.lineno, attr_node.end_col_offset - len(
_node.func.attr) - 1)] = rename_function_attributes[
_node.func.attr]
if _node.func.attr == 'addAction':
if len(_node.args) >= 4:
sys.stderr.write(
f'{filename}:{_node.lineno}:{_node.col_offset} WARNING: fragile call to addAction. Use my_action = QAction(...), obj.addAction(my_action) instead.\n')
if _node.func.attr == 'desktop':
if len(_node.args) == 0:
sys.stderr.write(
f'{filename}:{_node.lineno}:{_node.col_offset} WARNING: QDesktopWidget is deprecated and removed in Qt6. Replace with alternative approach instead.\n')
if isinstance(_node.func, ast.Name) and _node.func.id == 'QVariant':
if len(_node.args) == 1 and isinstance(_node.args[0], ast.Attribute) and isinstance(_node.args[0].value, ast.Name) and _node.args[0].value.id == 'QVariant':
extra_imports['qgis.core'].update({'NULL'})
def _fix_null_qvariant(start_index: int, tokens):
assert tokens[start_index].src == 'QVariant'
assert tokens[start_index + 1].src == '('
assert tokens[start_index + 2].src == 'QVariant'
assert tokens[start_index + 3].src == '.'
assert tokens[start_index + 5].src == ')'
tokens[start_index] = tokens[start_index]._replace(src='NULL')
for i in range(start_index + 1, start_index + 6):
tokens[i] = tokens[i]._replace(src='')
custom_updates[Offset(_node.lineno,
_node.col_offset)] = _fix_null_qvariant
elif isinstance(_node.func, ast.Name) and _node.func.id == 'QDateTime':
if len(_node.args) == 8:
# QDateTime(yyyy, mm, dd, hh, MM, ss, ms, ts) doesn't work anymore,
# so port to more reliable QDateTime(QDate, QTime, ts) form
extra_imports['qgis.PyQt.QtCore'].update({'QDate', 'QTime'})
def _fix_qdatetime_construct(start_index: int, tokens):
i = start_index + 1
assert tokens[i].src == '('
tokens[i] = tokens[i]._replace(src='(QDate(')
while tokens[i].offset < Offset(_node.args[2].lineno, _node.args[2].col_offset):
i += 1
assert tokens[i + 1].src == ','
i += 1
tokens[i] = tokens[i]._replace(src='), QTime(')
i += 1
while not tokens[i].src.strip():
tokens[i] = tokens[i]._replace(src='')
i += 1
while tokens[i].offset < Offset(_node.args[6].lineno, _node.args[6].col_offset):
i += 1
i += 1
assert tokens[i].src == ','
tokens[i] = tokens[i]._replace(src='),')
custom_updates[Offset(_node.lineno, _node.col_offset)] = _fix_qdatetime_construct
elif len(_node.args) == 1 and isinstance(_node.args[0], ast.Call) and _node.args[0].func.id == 'QDate':
# QDateTime(QDate(..)) doesn't work anymore,
# so port to more reliable QDateTime(QDate(...), QTime(0,0,0)) form
extra_imports['qgis.PyQt.QtCore'].update({'QTime'})
def _fix_qdatetime_construct(start_index: int, tokens):
assert tokens[start_index].src == 'QDateTime'
assert tokens[start_index + 1].src == '('
assert tokens[start_index + 2].src == 'QDate'
assert tokens[start_index + 3].src == '('
i = start_index + 4
while tokens[i].offset < Offset(_node.args[0].end_lineno,
_node.args[0].end_col_offset):
i += 1
assert tokens[i - 1].src == ')'
tokens[i - 1] = tokens[i - 1]._replace(src='), QTime(0, 0, 0)')
custom_updates[Offset(_node.lineno,
_node.col_offset)] = _fix_qdatetime_construct
def visit_attribute(_node: ast.Attribute, _parent):
if isinstance(_node.value, ast.Name):
if _node.value.id == 'qApp':
token_renames[Offset(_node.value.lineno, _node.value.col_offset)] = 'QApplication.instance()'
extra_imports['qgis.PyQt.QtWidgets'].update({'QApplication'})
removed_imports['qgis.PyQt.QtWidgets'].update({'qApp'})
if _node.value.id == 'QVariant' and _node.attr == 'Type':
def _replace_qvariant_type(start_index: int, tokens):
# QVariant.Type.XXX doesn't exist, it should be QVariant.XXX
assert tokens[start_index].src == 'QVariant'
assert tokens[start_index + 1].src == '.'
assert tokens[start_index + 2].src == 'Type'
assert tokens[start_index + 3].src == '.'
tokens[start_index + 2] = tokens[start_index + 2]._replace(src='')
tokens[start_index + 3] = tokens[start_index + 3]._replace(
src='')
custom_updates[Offset(node.lineno, node.col_offset)] = _replace_qvariant_type
elif isinstance(_node.value, ast.Call):
if (isinstance(_node.value.func, ast.Attribute) and
_node.value.func.attr == 'fontMetrics' and
_node.attr == 'width'):
sys.stderr.write(
f'{filename}:{_node.lineno}:{_node.col_offset} WARNING: QFontMetrics.width() '
'has been removed in Qt6. Use QFontMetrics.horizontalAdvance() if plugin can '
'safely require Qt >= 5.11, or QFontMetrics.boundingRect().width() otherwise.\n')
def visit_import(_node: ast.ImportFrom, _parent):
import_offsets[Offset(node.lineno, node.col_offset)] = (
node.module, set(name.name for name in node.names), node.end_lineno,
node.end_col_offset)
imported_modules.add(node.module)
for name in node.names:
if name.name in import_warnings:
print(f'{filename}: {import_warnings[name.name]}')
if _node.module == 'qgis.PyQt.Qt':
extra_imports['qgis.PyQt.QtCore'].update({'Qt'})
removed_imports['qgis.PyQt.Qt'].update({'Qt'})
tree = ast.parse(contents, filename=filename)
for parent in ast.walk(tree):
for node in ast.iter_child_nodes(parent):
if isinstance(node, ast.ImportFrom):
visit_import(node, parent)
if (not qgis3_compat and isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name)
and node.value.id == "QVariant"):
fix_qvariant_type.append(Offset(node.lineno, node.col_offset))
if isinstance(node, ast.Call):
visit_call(node, parent)
if isinstance(node, ast.Attribute):
visit_attribute(node, parent)
if isinstance(node, ast.FunctionDef) and node.name in rename_function_definitions:
function_def_renames[
Offset(node.lineno, node.col_offset)] = rename_function_definitions[node.name]
if (isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name)
and (node.value.id, node.attr) in ambiguous_enums):
disambiguated = False
try:
actual = eval(f'{node.value.id}.{node.attr}')
obj = globals()[node.value.id]
if isinstance(obj, type):
for attr_name in dir(obj):
attr = getattr(obj, attr_name)
if attr is actual.__class__:
# print(f'Found alias {node.value.id}.{attr_name}')
disambiguated = True
fix_qt_enums[
Offset(node.lineno, node.col_offset)] = (
node.value.id, attr_name, node.attr
)
break
except AttributeError:
pass
if not disambiguated:
possible_values = [f'{node.value.id}.{e}.{node.attr}' for e
in ambiguous_enums[
(node.value.id, node.attr)]]
sys.stderr.write(f'{filename}:{node.lineno}:{node.col_offset} WARNING: ambiguous enum, cannot fix: {node.value.id}.{node.attr}. Could be: {", ".join(possible_values)}\n')
elif (isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and not isinstance(parent, ast.Attribute)
and (node.value.id, node.attr) in qt_enums):
fix_qt_enums[Offset(node.lineno, node.col_offset)] = (node.value.id, qt_enums[(node.value.id, node.attr)], node.attr)
if (isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name)
and (node.value.id, node.attr) in deprecated_renamed_enums):
rename_qt_enums.append(Offset(node.lineno, node.col_offset))
elif (isinstance(node, ast.ImportFrom) and node.module and node.module.startswith("PyQt5.")):
fix_pyqt_import.append(Offset(node.lineno, node.col_offset))
for module, classes in extra_imports.items():
if module not in imported_modules:
class_import = ', '.join(classes)
import_statement = f'from {module} import {class_import}'
print(f'{filename}: Missing import, manually add \n\t{import_statement}')
if not any([fix_qvariant_type,
fix_pyqt_import,
fix_qt_enums,
rename_qt_enums,
member_renames,
function_def_renames,
custom_updates,
extra_imports,
removed_imports,
token_renames]):
return 0
tokens = src_to_tokens(contents)
for i, token in reversed_enumerate(tokens):
if token.offset in import_offsets:
end_import_offset = Offset(*import_offsets[token.offset][-2:])
assert tokens[i].src == 'from'
token_index = i + 1
while not tokens[token_index].src.strip():
token_index += 1
module = ''
while tokens[token_index].src.strip():
module += tokens[token_index].src
token_index += 1
if extra_imports.get(module) or removed_imports.get(module):
current_imports = set()
while True:
token_index += 1
if tokens[token_index].offset == end_import_offset:
break
if tokens[token_index].src.strip() in ('', ',', 'import', '(', ')'):
continue
import_ = tokens[token_index].src
if import_ in removed_imports.get(module, set()):
tokens[token_index] = tokens[token_index]._replace(src='')
prev_token_index = token_index - 1
while True:
if tokens[prev_token_index].src.strip() in ('', ','):
tokens[prev_token_index] = tokens[
prev_token_index]._replace(src='')
prev_token_index -= 1
else:
break
none_forward = True
current_index = prev_token_index + 1
while True:
if tokens[current_index].src in ('\n', ')'):
break
elif tokens[current_index].src.strip():
none_forward = False
break
current_index += 1
none_backward = True
current_index = prev_token_index
while True:
if tokens[current_index].src in ('import',):
break
elif tokens[current_index].src.strip():
none_backward = False
break
current_index -= 1
if none_backward and none_forward:
# no more imports from this module, remove whole import
while True:
if tokens[current_index].src in ('from',):
break
current_index -= 1
while True:
if tokens[current_index].src in ('\n',):
tokens[current_index] = tokens[
current_index]._replace(src='')
break
tokens[current_index] = tokens[current_index]._replace(src='')
current_index += 1
else:
current_imports.add(import_)
imports_to_add = extra_imports.get(module, set()) - current_imports
if imports_to_add:
additional_import_string = ', '.join(imports_to_add)
if tokens[token_index - 1].src == ')':
token_index -= 1
while tokens[token_index].src.strip() in ('', ',', ')'):
tokens[token_index] = tokens[token_index]._replace(
src='')
token_index -= 1
tokens[token_index + 1] = tokens[token_index + 1]._replace(src=f", {additional_import_string})")
else:
tokens[token_index] = tokens[token_index]._replace(src=f", {additional_import_string}{tokens[token_index].src}")
if token.offset in fix_qvariant_type:
assert tokens[i].src == "QVariant"
assert tokens[i + 1].src == "."
tokens[i] = tokens[i]._replace(src="QMetaType.Type")
attr = tokens[i + 2].src
if attr in qmetatype_mapping:
tokens[i + 2] = tokens[i + 2]._replace(src=qmetatype_mapping[attr])
if token.offset in custom_updates:
custom_updates[token.offset](i, tokens)
if token.offset in fix_pyqt_import:
assert tokens[i + 2].src == "PyQt5"
tokens[i + 2] = tokens[i + 2]._replace(src="qgis.PyQt")
if token.offset in function_def_renames and tokens[i].src == "def":
tokens[i + 2] = tokens[i + 2]._replace(src=function_def_renames[token.offset])
if token.offset in token_renames:
tokens[i] = tokens[i]._replace(src=token_renames[token.offset])
if token.offset in member_renames:
counter = i
while tokens[counter].src != '.':
counter += 1
tokens[counter + 1] = tokens[counter + 1]._replace(src=member_renames[token.offset])
if token.offset in fix_qt_enums:
assert tokens[i + 1].src == "."
_class, enum_name, value = fix_qt_enums[token.offset]
# make sure we CAN import enum!
try:
eval(f'{_class}.{enum_name}.{value}')
tokens[i + 2] = tokens[i + 2]._replace(
src=f"{enum_name}.{tokens[i + 2].src}")
except AttributeError:
# let's see if we can find what the replacement should be automatically...
# print(f'Trying to find {_class}.{value}.')
actual = eval(f'{_class}.{value}')
# print(f'Trying to find aliases for {actual.__class__}.')
obj = globals()[_class]
recovered = False
if isinstance(obj, type):
for attr_name in dir(obj):
try:
attr = getattr(obj, attr_name)
if attr is actual.__class__:
# print(f'Found alias {_class}.{attr_name}')
recovered = True
tokens[i + 2] = tokens[i + 2]._replace(
src=f"{attr_name}.{tokens[i + 2].src}")
except AttributeError:
continue
if not recovered:
sys.stderr.write(
f'{filename}:{token.line}:{token.utf8_byte_offset} ERROR: wanted to replace with {_class}.{enum_name}.{value}, but does not exist\n')
continue
if token.offset in rename_qt_enums:
assert tokens[i + 1].src == "."
enum_name = deprecated_renamed_enums[(tokens[i].src, tokens[i + 2].src)]
assert enum_name
tokens[i + 2] = tokens[i + 2]._replace(src=f"{enum_name[0]}.{enum_name[1]}")
new_contents = tokens_to_src(tokens)
with open(filename, 'w') as f:
f.write(new_contents)
return new_contents != contents
def get_class_enums(item):
if not inspect.isclass(item):
return
# enums might be referenced using a subclass instead of their
# parent class, so we need to loop through all those too...
def all_subclasses(cls):
if cls is object:
return set()
return {cls}.union(
s for c in cls.__subclasses__() for s in all_subclasses(c))
matched_classes = {item}.union(all_subclasses(item))
for key, value in item.__dict__.items():
if inspect.isclass(value) and type(value).__name__ == 'EnumType':
for ekey, evalue in value.__dict__.items():
for matched_class in matched_classes:
if isinstance(evalue, value):
try:
test_value = getattr(item, str(ekey))
if not issubclass(type(test_value), Enum):
# There's a naming clash between an enum value (Eg QgsAggregateMappingModel.ColumnDataIndex.Aggregate)
# and a class (QgsAggregateMappingModel.Aggregate)
# So don't do any upgrades for these values, as current code will always be referring
# to the CLASS
continue
except AttributeError:
pass
if (matched_class.__name__, ekey) in ambiguous_enums:
if value.__name__ not in ambiguous_enums[(matched_class.__name__, ekey)]:
ambiguous_enums[(matched_class.__name__, ekey)].add(value.__name__)
continue
existing_entry = qt_enums.get((matched_class.__name__, ekey))
if existing_entry != value.__name__ and existing_entry:
ambiguous_enums[(matched_class.__name__, ekey)].add(existing_entry)
ambiguous_enums[(matched_class.__name__, ekey)].add(
value.__name__)
del qt_enums[(matched_class.__name__, ekey)]
else:
qt_enums[(matched_class.__name__, ekey)] = f"{value.__name__}"
elif inspect.isclass(value):
get_class_enums(value)
def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser()
parser.add_argument('directory')
parser.add_argument('--qgis3-incompatible-changes', action='store_true',
help='Apply modifications that would break behavior on QGIS 3, hence code may not work on QGIS 3')
args = parser.parse_args(argv)
# get all scope for all qt enum
for module in target_modules:
for key, value in module.__dict__.items():
get_class_enums(value)
ret = 0
for filename in glob.glob(os.path.join(args.directory, "**/*.py"), recursive=True):
# print(f'Processing {filename}')
if 'auto_additions' in filename:
continue
ret |= fix_file(filename, not args.qgis3_incompatible_changes)
return ret
if __name__ == '__main__':
raise SystemExit(main())