[callouts] Ensure simple callouts are rendered below all map labels

...instead of being rendered on top of some. For this callout style,
we don't want callouts overlapping labels (rather the opposite). But
leave API in place to allow other callout styles to render below
their associated labels only, as this may be wanted for some styles
(e.g. balloon style callouts)

python/core/auto_generated/callouts/qgscallout.sip.in

@@ -46,6 +46,12 @@ relevant symbology elements to render them.
       MinimumCalloutLength,
     };
 
+    enum DrawOrder
+    {
+      OrderBelowAllLabels,
+      OrderBelowIndividualLabels,
+    };
+
     QgsCallout();
 %Docstring
 Constructor for QgsCallout.
@@ -143,6 +149,13 @@ required by any data defined properties associated with the callout.
    the same render ``context``.
 %End
 
+    virtual DrawOrder drawOrder() const;
+%Docstring
+Returns the desired drawing order (stacking) to use while rendering this callout.
+
+The default order is QgsCallout.OrderBelowIndividualLabels.
+%End
+
     void render( QgsRenderContext &context, QRectF rect, const double angle, const QgsGeometry &anchor );
 %Docstring
 Renders the callout onto the specified render ``context``.

src/core/callouts/qgscallout.cpp

@@ -94,6 +94,11 @@ QSet<QString> QgsCallout::referencedFields( const QgsRenderContext &context ) co
   return mDataDefinedProperties.referencedFields( context.expressionContext() );
 }
 
+QgsCallout::DrawOrder QgsCallout::drawOrder() const
+{
+  return OrderBelowAllLabels;
+}
+
 void QgsCallout::render( QgsRenderContext &context, QRectF rect, const double angle, const QgsGeometry &anchor )
 {
   if ( !mEnabled )

src/core/callouts/qgscallout.h

@@ -71,6 +71,13 @@ class CORE_EXPORT QgsCallout
       MinimumCalloutLength, //!< Minimum length of callouts
     };
 
+    //! Options for draw order (stacking) of callouts
+    enum DrawOrder
+    {
+      OrderBelowAllLabels, //!< Render callouts below all labels
+      OrderBelowIndividualLabels, //!< Render callouts below their individual associated labels, some callouts may be drawn over other labels
+    };
+
     /**
      * Constructor for QgsCallout.
      */
@@ -159,6 +166,13 @@ class CORE_EXPORT QgsCallout
      */
     virtual QSet< QString > referencedFields( const QgsRenderContext &context ) const;
 
+    /**
+     * Returns the desired drawing order (stacking) to use while rendering this callout.
+     *
+     * The default order is QgsCallout::OrderBelowIndividualLabels.
+     */
+    virtual DrawOrder drawOrder() const;
+
     /**
      * Renders the callout onto the specified render \a context.
      *

src/core/qgslabelingengine.cpp

@@ -357,6 +357,23 @@ void QgsLabelingEngine::run( QgsRenderContext &context )
     provider->startRender( context );
   }
 
+  // draw label backgrounds
+  for ( pal::LabelPosition *label : qgis::as_const( labels ) )
+  {
+    if ( context.renderingStopped() )
+      break;
+
+    QgsLabelFeature *lf = label->getFeaturePart()->feature();
+    if ( !lf )
+    {
+      continue;
+    }
+
+    context.expressionContext().setFeature( lf->feature() );
+    context.expressionContext().setFields( lf->feature().fields() );
+    lf->provider()->drawLabelBackground( context, label );
+  }
+
   // draw the labels
   for ( pal::LabelPosition *label : qgis::as_const( labels ) )
   {
@@ -412,6 +429,11 @@ QgsAbstractLabelProvider::QgsAbstractLabelProvider( QgsMapLayer *layer, const QS
 {
 }
 
+void QgsAbstractLabelProvider::drawLabelBackground( QgsRenderContext &, pal::LabelPosition * ) const
+{
+
+}
+
 void QgsAbstractLabelProvider::startRender( QgsRenderContext &context )
 {
   const auto subproviders = subProviders();

src/core/qgslabelingengine.h

@@ -73,6 +73,20 @@ class CORE_EXPORT QgsAbstractLabelProvider
      */
     virtual void drawLabel( QgsRenderContext &context, pal::LabelPosition *label ) const = 0;
 
+    /**
+     * Draw the background for the specified \a label.
+     *
+     * This is called in turn for each label provider before any actual labels are rendered,
+     * and allows the provider to render content which should be drawn below ALL map labels
+     * (such as background rectangles or callout lines).
+     *
+     * Before any calls to drawLabelBackground(), a provider should be prepared for rendering by a call to
+     * startRender() and a corresponding call to stopRender().
+     *
+     * \since QGIS 3.10
+     */
+    virtual void drawLabelBackground( QgsRenderContext &context, pal::LabelPosition *label ) const;
+
     /**
      * To be called before rendering of labels begins. Must be accompanied by
      * a corresponding call to stopRender()

src/core/qgsvectorlayerlabelprovider.cpp

@@ -278,6 +278,36 @@ QgsGeometry QgsVectorLayerLabelProvider::getPointObstacleGeometry( QgsFeature &f
   return QgsGeometry( std::move( obstacleGeom ) );
 }
 
+void QgsVectorLayerLabelProvider::drawLabelBackground( QgsRenderContext &context, LabelPosition *label ) const
+{
+  if ( !mSettings.drawLabels )
+    return;
+
+  // render callout
+  if ( mSettings.callout() && mSettings.callout()->drawOrder() == QgsCallout::OrderBelowAllLabels )
+  {
+    drawCallout( context, label );
+  }
+}
+
+void QgsVectorLayerLabelProvider::drawCallout( QgsRenderContext &context, pal::LabelPosition *label ) const
+{
+  context.expressionContext().setOriginalValueVariable( mSettings.callout()->enabled() );
+  const bool enabled = mSettings.dataDefinedProperties().valueAsBool( QgsPalLayerSettings::CalloutDraw, context.expressionContext(), mSettings.callout()->enabled() );
+  if ( enabled )
+  {
+    QgsMapToPixel xform = context.mapToPixel();
+    xform.setMapRotation( 0, 0, 0 );
+    QPointF outPt = xform.transform( label->getX(), label->getY() ).toQPointF();
+    QgsPointXY outPt2 = xform.transform( label->getX() + label->getWidth(), label->getY() + label->getHeight() );
+    QRectF rect( outPt.x(), outPt.y(), outPt2.x() - outPt.x(), outPt2.y() - outPt.y() );
+
+    QgsGeometry g( QgsGeos::fromGeos( label->getFeaturePart()->feature()->geometry() ) );
+    g.transform( xform.transform() );
+    mSettings.callout()->render( context, rect, label->getAlpha() * 180 / M_PI, g );
+  }
+}
+
 void QgsVectorLayerLabelProvider::drawLabel( QgsRenderContext &context, pal::LabelPosition *label ) const
 {
   if ( !mSettings.drawLabels )
@@ -349,22 +379,9 @@ void QgsVectorLayerLabelProvider::drawLabel( QgsRenderContext &context, pal::Lab
   //   (backgrounds -> text)
 
   // render callout
-  if ( mSettings.callout() )// && label->getFeaturePart()->hasFixedPosition() )
+  if ( mSettings.callout() && mSettings.callout()->drawOrder() == QgsCallout::OrderBelowIndividualLabels )
   {
-    context.expressionContext().setOriginalValueVariable( mSettings.callout()->enabled() );
-    const bool enabled = mSettings.dataDefinedProperties().valueAsBool( QgsPalLayerSettings::CalloutDraw, context.expressionContext(), mSettings.callout()->enabled() );
-    if ( enabled )
-    {
-      QgsMapToPixel xform = context.mapToPixel();
-      xform.setMapRotation( 0, 0, 0 );
-      QPointF outPt = xform.transform( label->getX(), label->getY() ).toQPointF();
-      QgsPointXY outPt2 = xform.transform( label->getX() + label->getWidth(), label->getY() + label->getHeight() );
-      QRectF rect( outPt.x(), outPt.y(), outPt2.x() - outPt.x(), outPt2.y() - outPt.y() );
-
-      QgsGeometry g( QgsGeos::fromGeos( label->getFeaturePart()->feature()->geometry() ) );
-      g.transform( xform.transform() );
-      mSettings.callout()->render( context, rect, label->getAlpha() * 180 / M_PI, g );
-    }
+    drawCallout( context, label );
   }
 
   if ( tmpLyr.format().shadow().enabled() && tmpLyr.format().shadow().shadowPlacement() == QgsTextShadowSettings::ShadowLowest )

src/core/qgsvectorlayerlabelprovider.h

@@ -51,6 +51,7 @@ class CORE_EXPORT QgsVectorLayerLabelProvider : public QgsAbstractLabelProvider
 
     QList<QgsLabelFeature *> labelFeatures( QgsRenderContext &context ) override;
 
+    void drawLabelBackground( QgsRenderContext &context, pal::LabelPosition *label ) const override;
     void drawLabel( QgsRenderContext &context, pal::LabelPosition *label ) const override;
     void startRender( QgsRenderContext &context ) override;
     void stopRender( QgsRenderContext &context ) override;
@@ -118,6 +119,7 @@ class CORE_EXPORT QgsVectorLayerLabelProvider : public QgsAbstractLabelProvider
   private:
 
     friend class TestQgsLabelingEngine;
+    void drawCallout( QgsRenderContext &context, pal::LabelPosition *label ) const;
 };
 
 #endif // QGSVECTORLAYERLABELPROVIDER_H

tests/src/core/testqgscallout.cpp

@@ -105,6 +105,7 @@ class TestQgsCallout: public QObject
     void calloutDataDefinedSymbol();
     void calloutMinimumDistance();
     void calloutDataDefinedMinimumDistance();
+    void calloutBehindLabel();
     void manhattan();
     void manhattanRotated();
 
@@ -639,6 +640,62 @@ void TestQgsCallout::calloutDataDefinedMinimumDistance()
   QVERIFY( imageCheck( "callout_data_defined_minimum_length", img, 20 ) );
 }
 
+void TestQgsCallout::calloutBehindLabel()
+{
+  // test that callouts are rendered below labels
+  QSize size( 640, 480 );
+  QgsMapSettings mapSettings;
+  mapSettings.setOutputSize( size );
+  mapSettings.setExtent( vl->extent() );
+  mapSettings.setLayers( QList<QgsMapLayer *>() << vl );
+  mapSettings.setOutputDpi( 96 );
+  mapSettings.setRotation( 45 );
+
+  QgsMapRendererSequentialJob job( mapSettings );
+  job.start();
+  job.waitForFinished();
+
+  QImage img = job.renderedImage();
+
+  QPainter p( &img );
+  QgsRenderContext context = QgsRenderContext::fromMapSettings( mapSettings );
+  context.setPainter( &p );
+
+  QgsPalLayerSettings settings;
+  settings.fieldName = QStringLiteral( "Class" );
+  settings.placement = QgsPalLayerSettings::AroundPoint;
+  settings.dataDefinedProperties().setProperty( QgsPalLayerSettings::PositionX, QgsProperty::fromExpression( QStringLiteral( "case when $id = 1 then %1 end" ).arg( mapSettings.extent().center().x() ) ) );
+  settings.dataDefinedProperties().setProperty( QgsPalLayerSettings::PositionY, QgsProperty::fromExpression( QStringLiteral( "case when $id = 1 then %1 end" ).arg( mapSettings.extent().center().y() ) ) );
+  settings.dataDefinedProperties().setProperty( QgsPalLayerSettings::ZIndex, QgsProperty::fromExpression( QStringLiteral( "100 - $id" ) ) );
+
+  QgsTextFormat format;
+  format.setFont( QgsFontUtils::getStandardTestFont( QStringLiteral( "Bold" ) ).family() );
+  format.setSize( 12 );
+  format.setNamedStyle( QStringLiteral( "Bold" ) );
+  format.setColor( QColor( 0, 0, 0 ) );
+  settings.setFormat( format );
+
+  QgsSimpleLineCallout *callout = new QgsSimpleLineCallout();
+  callout->setEnabled( true );
+  callout->lineSymbol()->setWidth( 2 );
+  callout->lineSymbol()->setColor( QColor( 255, 100, 100 ) );
+
+  settings.setCallout( callout );
+
+  vl->setLabeling( new QgsVectorLayerSimpleLabeling( settings ) );  // TODO: this should not be necessary!
+  vl->setLabelsEnabled( true );
+
+  QgsLabelingEngine engine;
+  engine.setMapSettings( mapSettings );
+  engine.addProvider( new QgsVectorLayerLabelProvider( vl, QString(), true, &settings ) );
+  //engine.setFlags( QgsLabelingEngine::RenderOutlineLabels | QgsLabelingEngine::DrawLabelRectOnly );
+  engine.run( context );
+
+  p.end();
+
+  QVERIFY( imageCheck( "callout_behind_labels", img, 20 ) );
+}
+
 void TestQgsCallout::manhattan()
 {
   QSize size( 640, 480 );