QgsDataItem::createChildren(): fix SIP generated C++ method

The SIP_FACTORY annotation didn't do the job for a Python implementation
of createChildren(). We must write a custom VirtualCatcherCode to make
sure that if the returned QgsDataItem elements are Python objects they
aren't killed too early.

Fixes the new added test, which uses to crash before the fix,
and corresponds to the use case of a plugin defining its own data items.

Has been checked not to cause memory leaks.

The change of SIP_FACTORY to SIP_TRANSFERBACK is to avoid a memory
leak when Python calls a C++ createChildren() implementation. Was found
by making test_provider_ogr.py::testDataItems() loop on its call to
createChildren()

python/core/auto_generated/qgsdataitem.sip.in

@@ -81,7 +81,7 @@ The default implementation returns ``False``, subclasses must implement this met
 
     int rowCount();
 
-    virtual QVector<QgsDataItem *> createChildren() /Factory/;
+    virtual QVector<QgsDataItem *> createChildren() /TransferBack/;
 %Docstring
 Create children. Children are not expected to have parent set.
 
@@ -89,6 +89,28 @@ Create children. Children are not expected to have parent set.
 
    This method MUST BE THREAD SAFE.
 %End
+%VirtualCatcherCode
+    PyObject *sipResObj = sipCallMethod( 0, sipMethod, "" );
+    // H = Convert a Python object to a mapped type instance.
+    // 5 = 1 (disallows the conversion of Py_None to NULL) + 4 (returns a copy of the C/C++ instance)
+    sipIsErr = !sipResObj || sipParseResult( 0, sipMethod, sipResObj, "H5", sipType_QVector_0101QgsDataItem, &sipRes ) < 0;
+    if ( !sipIsErr )
+    {
+      for ( QgsDataItem *item : sipRes )
+      {
+        PyObject *pyItem = sipGetPyObject( item, sipType_QgsDataItem );
+        if ( pyItem != NULL )
+        {
+          // pyItem is given an extra reference which is removed when the C++ instance’s destructor is called.
+          sipTransferTo( pyItem, Py_None );
+        }
+      }
+    }
+    if ( sipResObj != NULL )
+    {
+      Py_DECREF( sipResObj );
+    }
+%End
 
     enum State
     {

src/core/qgsdataitem.h

@@ -119,7 +119,31 @@ class CORE_EXPORT QgsDataItem : public QObject
      * Create children. Children are not expected to have parent set.
      * \warning This method MUST BE THREAD SAFE.
     */
-    virtual QVector<QgsDataItem *> createChildren() SIP_FACTORY;
+    virtual QVector<QgsDataItem *> createChildren() SIP_TRANSFERBACK;
+#ifdef SIP_RUN
+    SIP_VIRTUAL_CATCHER_CODE
+    PyObject *sipResObj = sipCallMethod( 0, sipMethod, "" );
+    // H = Convert a Python object to a mapped type instance.
+    // 5 = 1 (disallows the conversion of Py_None to NULL) + 4 (returns a copy of the C/C++ instance)
+    sipIsErr = !sipResObj || sipParseResult( 0, sipMethod, sipResObj, "H5", sipType_QVector_0101QgsDataItem, &sipRes ) < 0;
+    if ( !sipIsErr )
+    {
+      for ( QgsDataItem *item : sipRes )
+      {
+        PyObject *pyItem = sipGetPyObject( item, sipType_QgsDataItem );
+        if ( pyItem != NULL )
+        {
+          // pyItem is given an extra reference which is removed when the C++ instance’s destructor is called.
+          sipTransferTo( pyItem, Py_None );
+        }
+      }
+    }
+    if ( sipResObj != NULL )
+    {
+      Py_DECREF( sipResObj );
+    }
+    SIP_END
+#endif
 
     enum State
     {

tests/src/python/CMakeLists.txt

@@ -55,6 +55,7 @@ ADD_PYTHON_TEST(PyQgsDefaultValue test_qgsdefaultvalue.py)
 ADD_PYTHON_TEST(PyQgsXmlUtils test_qgsxmlutils.py)
 ADD_PYTHON_TEST(PyQgsCore test_qgscore.py)
 ADD_PYTHON_TEST(PyQgsCoordinateTransform test_qgscoordinatetransform.py)
+ADD_PYTHON_TEST(PyQgsDataItem test_qgsdataitem.py)
 ADD_PYTHON_TEST(PyQgsDataItemGuiProviderRegistry test_qgsdataitemguiproviderregistry.py)
 ADD_PYTHON_TEST(PyQgsDataItemProviderRegistry test_qgsdataitemproviderregistry.py)
 ADD_PYTHON_TEST(PyQgsDateTimeEdit test_qgsdatetimeedit.py)

tests/src/python/test_qgsdataitem.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+"""QGIS Unit tests for QgsDataItem
+
+.. note:: This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+"""
+__author__ = 'Even Rouault'
+__date__ = '14/11/2020 late in the night'
+__copyright__ = 'Copyright 2020, The QGIS Project'
+
+from qgis.PyQt.QtCore import QEventLoop
+from qgis.core import QgsDataCollectionItem, QgsLayerItem
+
+from qgis.testing import start_app, unittest
+
+app = start_app()
+
+
+class PyQgsLayerItem(QgsLayerItem):
+
+    def __del__(self):
+        self.tabSetDestroyedFlag[0] = True
+
+
+class PyQgsDataConnectionItem(QgsDataCollectionItem):
+
+    def createChildren(self):
+        children = []
+
+        # Add a Python object as child
+        pyQgsLayerItem = PyQgsLayerItem(None, "name", "", "uri", QgsLayerItem.Vector, "my_provider")
+        pyQgsLayerItem.tabSetDestroyedFlag = self.tabSetDestroyedFlag
+        children.append(pyQgsLayerItem)
+
+        # Add a C++ object as child
+        children.append(QgsLayerItem(None, "name2", "", "uri", QgsLayerItem.Vector, "my_provider"))
+
+        return children
+
+
+class TestQgsDataItem(unittest.TestCase):
+
+    def testPythonCreateChildrenCalledFromCplusplus(self):
+        """ test createChildren() method implemented in Python, called from C++ """
+
+        loop = QEventLoop()
+        NUM_ITERS = 10  # set more to detect memory leaks
+        for i in range(NUM_ITERS):
+            tabSetDestroyedFlag = [False]
+
+            item = PyQgsDataConnectionItem(None, "name", "", "my_provider")
+            item.tabSetDestroyedFlag = tabSetDestroyedFlag
+
+            # Causes PyQgsDataConnectionItem.createChildren() to be called
+            item.populate()
+
+            # wait for populate() to have done its job
+            item.stateChanged.connect(loop.quit)
+            loop.exec_()
+
+            # Python object PyQgsLayerItem should still be alive
+            self.assertFalse(tabSetDestroyedFlag[0])
+
+            self.assertEqual(len(item.children()), 2)
+            self.assertEqual(item.children()[0].name(), "name")
+            self.assertEqual(item.children()[1].name(), "name2")
+
+            # Delete the object and make sure all deferred deletions are processed
+            item.destroyed.connect(loop.quit)
+            item.deleteLater()
+            loop.exec_()
+
+            # Check that the PyQgsLayerItem Python object is now destroyed
+            self.assertTrue(tabSetDestroyedFlag[0])
+            tabSetDestroyedFlag[0] = False
+
+
+if __name__ == '__main__':
+    unittest.main()