do not insert layers in embedded groups (#10004)

fixes #29678

python/core/auto_generated/layertree/qgslayertreeutils.sip.in

@@ -126,6 +126,17 @@ of a map layer within a layer tree and only connecting/disconnecting when
 there is only one occurrence of that layer.
 
 .. versionadded:: 3.4
+%End
+
+    static QgsLayerTreeGroup *firstGroupWithoutCustomProperty( QgsLayerTreeGroup *group, const QString &property );
+%Docstring
+Returns the first parent which doesn't have the given custom property
+or the group itself if it doesn't hold the property
+
+:param group: the layer tree group
+:param property: the property
+
+.. versionadded:: 3.8
 %End
 };
 

src/app/qgisapp.cpp

@@ -4066,31 +4066,39 @@ void QgisApp::setupLayerTreeViewFromSettings()
 void QgisApp::updateNewLayerInsertionPoint()
 {
   // defaults
-  QgsLayerTreeGroup *parentGroup = mLayerTreeView->layerTreeModel()->rootGroup();
-  int index = 0;
+  QgsLayerTreeGroup *insertGroup = mLayerTreeView->layerTreeModel()->rootGroup();
   QModelIndex current = mLayerTreeView->currentIndex();
+  int index = 0;
 
   if ( current.isValid() )
   {
+    index = current.row();
+
     if ( QgsLayerTreeNode *currentNode = mLayerTreeView->currentNode() )
     {
       // if the insertion point is actually a group, insert new layers into the group
       if ( QgsLayerTree::isGroup( currentNode ) )
       {
-        QgsProject::instance()->layerTreeRegistryBridge()->setLayerInsertionPoint( QgsLayerTree::toGroup( currentNode ), 0 );
+        // if the group is embedded go to the first non-embedded group, at worst the top level item
+        QgsLayerTreeGroup *insertGroup = QgsLayerTreeUtils::firstGroupWithoutCustomProperty( QgsLayerTree::toGroup( currentNode ), QStringLiteral( "embedded" ) );
+        QgsProject::instance()->layerTreeRegistryBridge()->setLayerInsertionPoint( insertGroup, 0 );
         return;
       }
 
       // otherwise just set the insertion point in front of the current node
       QgsLayerTreeNode *parentNode = currentNode->parent();
       if ( QgsLayerTree::isGroup( parentNode ) )
-        parentGroup = QgsLayerTree::toGroup( parentNode );
+      {
+        // if the group is embedded go to the first non-embedded group, at worst the top level item
+        QgsLayerTreeGroup *parentGroup = QgsLayerTree::toGroup( parentNode );
+        insertGroup = QgsLayerTreeUtils::firstGroupWithoutCustomProperty( parentGroup, QStringLiteral( "embedded" ) );
+        if ( parentGroup != insertGroup )
+          index = 0;
+      }
     }
-
-    index = current.row();
   }
 
-  QgsProject::instance()->layerTreeRegistryBridge()->setLayerInsertionPoint( parentGroup, index );
+  QgsProject::instance()->layerTreeRegistryBridge()->setLayerInsertionPoint( insertGroup, index );
 }
 
 void QgisApp::autoSelectAddedLayer( QList<QgsMapLayer *> layers )

src/core/layertree/qgslayertreeutils.cpp

@@ -512,3 +512,19 @@ int QgsLayerTreeUtils::countMapLayerInTree( QgsLayerTreeNode *tree, QgsMapLayer
     cnt += countMapLayerInTree( child, layer );
   return cnt;
 }
+
+QgsLayerTreeGroup *QgsLayerTreeUtils::firstGroupWithoutCustomProperty( QgsLayerTreeGroup *group, const QString &property )
+{
+  // if the group is embedded go to the first non-embedded group, at worst the top level item
+  while ( group->customProperty( property ).toInt() )
+  {
+    if ( !group->parent() )
+      break;
+
+    if ( QgsLayerTree::isGroup( group->parent() ) )
+      group = QgsLayerTree::toGroup( group->parent() );
+    else
+      Q_ASSERT( false );
+  }
+  return group;
+}

src/core/layertree/qgslayertreeutils.h

@@ -115,6 +115,15 @@ class CORE_EXPORT QgsLayerTreeUtils
      * \since QGIS 3.4
      */
     static int countMapLayerInTree( QgsLayerTreeNode *tree, QgsMapLayer *layer );
+
+    /**
+     * Returns the first parent which doesn't have the given custom property
+     * or the group itself if it doesn't hold the property
+     * \param group the layer tree group
+     * \param property the property
+     * \since QGIS 3.8
+     */
+    static QgsLayerTreeGroup *firstGroupWithoutCustomProperty( QgsLayerTreeGroup *group, const QString &property );
 };
 
 #endif // QGSLAYERTREEUTILS_H