[FEATURE][API] Add option to specify a custom boundary geometry

in QgsMapSettings to restrict where labels are allowed to be placed
within.

If set, this overrides the default behavior of allowing labels to
be placed anywhere inside the rendered map extent.

python/core/auto_generated/qgsmapsettings.sip.in

@@ -529,6 +529,34 @@ Returns the global configuration of the labeling engine.
 .. seealso:: :py:func:`setLabelingEngineSettings`
 
 .. versionadded:: 3.0
+%End
+
+    QgsGeometry labelBoundaryGeometry() const;
+%Docstring
+Returns the label boundary geometry, which restricts where in the rendered map labels are permitted to be
+placed. By default this is a null geometry, which indicates that labels can be placed anywhere within
+the map's visiblePolygon().
+
+The geometry is specified using the map's destinationCrs().
+
+.. seealso:: :py:func:`setLabelBoundaryGeometry`
+
+.. versionadded:: 3.6
+%End
+
+    void setLabelBoundaryGeometry( const QgsGeometry &boundary );
+%Docstring
+Sets the label ``boundary`` geometry, which restricts where in the rendered map labels are permitted to be
+placed.
+
+A null ``boundary`` geometry (the default) indicates that labels can be placed anywhere within
+the map's visiblePolygon().
+
+The geometry is specified using the map's destinationCrs().
+
+.. seealso:: :py:func:`labelBoundaryGeometry`
+
+.. versionadded:: 3.6
 %End
 
   protected:

src/core/qgslabelingengine.cpp

@@ -25,7 +25,7 @@
 #include "problem.h"
 #include "qgsrendercontext.h"
 #include "qgsmaplayer.h"
-
+#include "qgssymbol.h"
 
 // helper function for checking for job cancelation within PAL
 static bool _palIsCanceled( void *ctx )
@@ -244,7 +244,25 @@ void QgsLabelingEngine::run( QgsRenderContext &context )
   QgsGeometry extentGeom = QgsGeometry::fromRect( mMapSettings.visibleExtent() );
   QPolygonF visiblePoly = mMapSettings.visiblePolygon();
   visiblePoly.append( visiblePoly.at( 0 ) ); //close polygon
-  QgsGeometry mapBoundaryGeom = QgsGeometry::fromQPolygonF( visiblePoly );
+
+  // get map label boundary geometry - if one hasn't been explicitly set, we use the whole of the map's visible polygon
+  QgsGeometry mapBoundaryGeom = !mMapSettings.labelBoundaryGeometry().isNull() ? mMapSettings.labelBoundaryGeometry() : QgsGeometry::fromQPolygonF( visiblePoly );
+  if ( settings.flags() & QgsLabelingEngineSettings::DrawCandidates )
+  {
+    // draw map boundary
+    QgsFeature f;
+    f.setGeometry( mapBoundaryGeom );
+    QgsStringMap properties;
+    properties.insert( QStringLiteral( "style" ), QStringLiteral( "no" ) );
+    properties.insert( QStringLiteral( "style_border" ), QStringLiteral( "solid" ) );
+    properties.insert( QStringLiteral( "color_border" ), QStringLiteral( "#0000ff" ) );
+    properties.insert( QStringLiteral( "width_border" ), QStringLiteral( "0.3" ) );
+    properties.insert( QStringLiteral( "joinstyle" ), QStringLiteral( "miter" ) );
+    std::unique_ptr< QgsFillSymbol > boundarySymbol( QgsFillSymbol::createSimple( properties ) );
+    boundarySymbol->startRender( context );
+    boundarySymbol->renderFeature( f, context );
+    boundarySymbol->stopRender( context );
+  }
 
   if ( !qgsDoubleNear( mMapSettings.rotation(), 0.0 ) )
   {

src/core/qgsmapsettings.cpp

@@ -655,3 +655,13 @@ void QgsMapSettings::writeXml( QDomNode &node, QDomDocument &doc )
   renderMapTileElem.appendChild( renderMapTileText );
   node.appendChild( renderMapTileElem );
 }
+
+QgsGeometry QgsMapSettings::labelBoundaryGeometry() const
+{
+  return mLabelBoundaryGeometry;
+}
+
+void QgsMapSettings::setLabelBoundaryGeometry( const QgsGeometry &boundary )
+{
+  mLabelBoundaryGeometry = boundary;
+}

src/core/qgsmapsettings.h

@@ -463,6 +463,32 @@ class CORE_EXPORT QgsMapSettings
      */
     const QgsLabelingEngineSettings &labelingEngineSettings() const { return mLabelingEngineSettings; }
 
+    /**
+     * Returns the label boundary geometry, which restricts where in the rendered map labels are permitted to be
+     * placed. By default this is a null geometry, which indicates that labels can be placed anywhere within
+     * the map's visiblePolygon().
+     *
+     * The geometry is specified using the map's destinationCrs().
+     *
+     * \see setLabelBoundaryGeometry()
+     * \since QGIS 3.6
+     */
+    QgsGeometry labelBoundaryGeometry() const;
+
+    /**
+     * Sets the label \a boundary geometry, which restricts where in the rendered map labels are permitted to be
+     * placed.
+     *
+     * A null \a boundary geometry (the default) indicates that labels can be placed anywhere within
+     * the map's visiblePolygon().
+     *
+     * The geometry is specified using the map's destinationCrs().
+     *
+     * \see labelBoundaryGeometry()
+     * \since QGIS 3.6
+     */
+    void setLabelBoundaryGeometry( const QgsGeometry &boundary );
+
   protected:
 
     double mDpi;
@@ -513,6 +539,8 @@ class CORE_EXPORT QgsMapSettings
 
     QgsRenderContext::TextRenderFormat mTextRenderFormat = QgsRenderContext::TextFormatAlwaysOutlines;
 
+    QgsGeometry mLabelBoundaryGeometry;
+
 #ifdef QGISDEBUG
     bool mHasTransformContext = false;
 #endif

tests/src/core/testqgslabelingengine.cpp

@@ -52,6 +52,7 @@ class TestQgsLabelingEngine : public QObject
     void testRegisterFeatureUnprojectible();
     void testRotateHidePartial();
     void testParallelLabelSmallFeature();
+    void testLabelBoundary();
 
   private:
     QgsVectorLayer *vl = nullptr;
@@ -805,5 +806,73 @@ void TestQgsLabelingEngine::testParallelLabelSmallFeature()
   //  QVERIFY( imageCheck( "label_rotate_hide_partial", img, 20 ) );
 }
 
+void TestQgsLabelingEngine::testLabelBoundary()
+{
+  // test that no labels are drawn outside of the specified label boundary
+  QgsPalLayerSettings settings;
+  setDefaultLabelParams( settings );
+
+  QgsTextFormat format = settings.format();
+  format.setSize( 20 );
+  format.setColor( QColor( 0, 0, 0 ) );
+  settings.setFormat( format );
+
+  settings.fieldName = QStringLiteral( "'X'" );
+  settings.isExpression = true;
+  settings.placement = QgsPalLayerSettings::OverPoint;
+
+  std::unique_ptr< QgsVectorLayer> vl2( new QgsVectorLayer( QStringLiteral( "Point?crs=epsg:4326&field=id:integer" ), QStringLiteral( "vl" ), QStringLiteral( "memory" ) ) );
+  vl2->setRenderer( new QgsNullSymbolRenderer() );
+
+  QgsFeature f( vl2->fields(), 1 );
+
+  for ( int x = 0; x < 15; x++ )
+  {
+    for ( int y = 0; y < 12; y++ )
+    {
+      f.setGeometry( qgis::make_unique< QgsPoint >( x, y ) );
+      vl2->dataProvider()->addFeature( f );
+    }
+  }
+
+  vl2->setLabeling( new QgsVectorLayerSimpleLabeling( settings ) );  // TODO: this should not be necessary!
+  vl2->setLabelsEnabled( true );
+
+  // make a fake render context
+  QSize size( 640, 480 );
+  QgsMapSettings mapSettings;
+  QgsCoordinateReferenceSystem tgtCrs;
+  tgtCrs.createFromString( QStringLiteral( "EPSG:4326" ) );
+  mapSettings.setDestinationCrs( tgtCrs );
+
+  mapSettings.setOutputSize( size );
+  mapSettings.setExtent( vl2->extent() );
+  mapSettings.setLayers( QList<QgsMapLayer *>() << vl2.get() );
+  mapSettings.setOutputDpi( 96 );
+
+  mapSettings.setLabelBoundaryGeometry( QgsGeometry::fromWkt( QStringLiteral( "Polygon((3 1, 12 1, 12 9, 3 9, 3 1),(8 4, 10 4, 10 7, 8 7, 8 4))" ) ) );
+
+  QgsLabelingEngineSettings engineSettings = mapSettings.labelingEngineSettings();
+  engineSettings.setFlag( QgsLabelingEngineSettings::UsePartialCandidates, false );
+  engineSettings.setFlag( QgsLabelingEngineSettings::DrawLabelRectOnly, true );
+  mapSettings.setLabelingEngineSettings( engineSettings );
+
+  QgsMapRendererSequentialJob job( mapSettings );
+  job.start();
+  job.waitForFinished();
+
+  QImage img = job.renderedImage();
+  QVERIFY( imageCheck( QStringLiteral( "label_boundary_geometry" ), img, 20 ) );
+
+  // with rotation
+  mapSettings.setRotation( 45 );
+  QgsMapRendererSequentialJob job2( mapSettings );
+  job2.start();
+  job2.waitForFinished();
+
+  img = job2.renderedImage();
+  QVERIFY( imageCheck( QStringLiteral( "rotated_label_boundary_geometry" ), img, 20 ) );
+}
+
 QGSTEST_MAIN( TestQgsLabelingEngine )
 #include "testqgslabelingengine.moc"

tests/src/core/testqgsmapsettings.cpp

@@ -45,6 +45,7 @@ class TestQgsMapSettings: public QObject
     void testMapLayerListUtils();
     void testXmlReadWrite();
     void testSetLayers();
+    void testLabelBoundary();
 
   private:
     QString toString( const QPolygonF &p, int decimalPlaces = 2 ) const;
@@ -367,5 +368,13 @@ void TestQgsMapSettings::testSetLayers()
   QCOMPARE( ms.layers(), QList< QgsMapLayer * >() << vlA.get() << vlB.get() );
 }
 
+void TestQgsMapSettings::testLabelBoundary()
+{
+  QgsMapSettings ms;
+  QVERIFY( ms.labelBoundaryGeometry().isNull() );
+  ms.setLabelBoundaryGeometry( QgsGeometry::fromWkt( QStringLiteral( "Polygon(( 0 0, 1 0, 1 1, 0 1, 0 0 ))" ) ) );
+  QCOMPARE( ms.labelBoundaryGeometry().asWkt(), QStringLiteral( "Polygon ((0 0, 1 0, 1 1, 0 1, 0 0))" ) );
+}
+
 QGSTEST_MAIN( TestQgsMapSettings )
 #include "testqgsmapsettings.moc"