diff --git a/src/core/pal/pal.cpp b/src/core/pal/pal.cpp
index 4ee512a1344..cdc5c6dffcf 100644
--- a/src/core/pal/pal.cpp
+++ b/src/core/pal/pal.cpp
@@ -389,15 +389,26 @@ std::unique_ptr<Problem> Pal::extract( const QgsRectangle &extent, const QgsGeom
         {
           // v2 placement rips out candidates where the candidate cost is too high when compared to
           // their inactive cost
-          feat->candidates.erase( std::remove_if( feat->candidates.begin(), feat->candidates.end(), [ & ]( std::unique_ptr< LabelPosition > &candidate )
+
+          // note, we start this at the SECOND candidate (you'll see why after this loop)
+          feat->candidates.erase( std::remove_if( feat->candidates.begin() + 1, feat->candidates.end(), [ & ]( std::unique_ptr< LabelPosition > &candidate )
           {
             if ( candidate->hasHardObstacleConflict() )
             {
-              feat->candidates.back()->removeFromIndex( prob->mAllCandidatesIndex );
+              candidate->removeFromIndex( prob->mAllCandidatesIndex );
               return true;
             }
             return false;
           } ), feat->candidates.end() );
+
+          if ( feat->candidates.size() == 1 && feat->candidates[ 0 ]->hasHardObstacleConflict() )
+          {
+            // we've ended up removing ALL candidates for this label. Oh well, that's allowed. We just need to
+            // make sure we move this last candidate to the unplaced labels list
+            feat->candidates.front()->removeFromIndex( prob->mAllCandidatesIndex );
+            prob->positionsWithNoCandidates()->emplace_back( std::move( feat->candidates.front() ) );
+            feat->candidates.clear();
+          }
         }
       }
 
diff --git a/src/core/pal/problem.cpp b/src/core/pal/problem.cpp
index df9f4d36e39..1d417e405bc 100644
--- a/src/core/pal/problem.cpp
+++ b/src/core/pal/problem.cpp
@@ -752,14 +752,16 @@ QList<LabelPosition *> Problem::getSolution( bool returnInactive, QList<LabelPos
       solList.push_back( mLabelPositions[ mSol.activeLabelIds[i] ].get() ); // active labels
     }
     else if ( returnInactive
-              || mLabelPositions.at( mFeatStartId[i] )->getFeaturePart()->layer()->displayAll()
-              || mLabelPositions.at( mFeatStartId[i] )->getFeaturePart()->alwaysShow() )
+              || ( mFeatStartId[i] < static_cast< int >( mLabelPositions.size() ) &&
+                   ( mLabelPositions.at( mFeatStartId[i] )->getFeaturePart()->layer()->displayAll()
+                     || mLabelPositions.at( mFeatStartId[i] )->getFeaturePart()->alwaysShow() ) ) )
     {
       solList.push_back( mLabelPositions[ mFeatStartId[i] ].get() ); // unplaced label
     }
     else if ( unlabeled )
     {
-      unlabeled->push_back( mLabelPositions[ mFeatStartId[i] ].get() );
+      if ( mFeatStartId[i] < static_cast< int >( mLabelPositions.size() ) )
+        unlabeled->push_back( mLabelPositions[ mFeatStartId[i] ].get() );
     }
   }