Make extractPointPositions() return x,y,z rather than x,z,-y

We just flip axes at the transform where needed back to world coordinates.
Make instanced renderer shader respect any QTransform and clean up its code

src/3d/qgs3dutils.cpp

@@ -513,7 +513,7 @@ void Qgs3DUtils::extractPointPositions( const QgsFeature &f, const Qgs3DRenderCo
         h = terrainZ + geomZ;
         break;
     }
-    positions.append( QVector3D( pt.x() - chunkOrigin.x(), h, -( pt.y() - chunkOrigin.y() ) ) );
+    positions.append( QVector3D( pt.x() - chunkOrigin.x(), pt.y() - chunkOrigin.y(), h ) );
     QgsDebugMsgLevel( QStringLiteral( "%1 %2 %3" ).arg( positions.last().x() ).arg( positions.last().y() ).arg( positions.last().z() ), 2 );
   }
 }

src/3d/shaders/instanced.vert

@@ -8,9 +8,8 @@ out vec3 worldPosition;
 out vec3 worldNormal;
 
 uniform mat4 modelMatrix;
-uniform mat4 modelView;
-uniform mat3 modelViewNormal;
-uniform mat4 modelViewProjection;
+uniform mat3 modelNormalMatrix;
+uniform mat4 mvp;
 
 uniform mat4 inst;  // transform of individual object instance
 uniform mat4 instNormal;  // should be mat3 but Qt3D only supports mat4...
@@ -21,15 +20,26 @@ uniform mat4 instNormal;  // should be mat3 but Qt3D only supports mat4...
 
 void main()
 {
-    // TODO: i think this is not entirely correct: the translation by "pos" works
-    // like this only because we assume that "inst" matrix only does translation/scale/rotation
-    // which all keep "w" set to 1. correctly we should use translation matrix...
-    vec4 offsetPos = inst * vec4(vertexPosition, 1.0) + vec4(pos, 0.0);
+    // vertexPosition uses XZ plane as the base plane, with Y going upwards
+    // and the coordinates are local to the object
 
-    worldNormal = normalize(mat3(instNormal) * vertexNormal);
-    worldPosition = vec3(modelMatrix * offsetPos);
+    // first let's apply user defined transform for each object (translation, rotation, scaling)
+    vec3 vertexPositionObject = vec3(inst * vec4(vertexPosition, 1.0));
+    vec3 vertexNormalObject = mat3(instNormal) * vertexNormal;
 
-    gl_Position = modelViewProjection * offsetPos;
+    // next let's flip axes, so we have XY plane as the base plane (like in map coordinates)
+    vertexPositionObject = vec3(vertexPositionObject.x, -vertexPositionObject.z, vertexPositionObject.y);
+    vertexNormalObject = vec3(vertexNormalObject.x, -vertexNormalObject.z, vertexNormalObject.y);
+
+    // add offset of the object relative to the chunk's origin
+    vec3 vertexPositionChunk = vertexPositionObject + pos;
+
+    // Transform position and normal to world space
+    worldPosition = vec3(modelMatrix * vec4(vertexPositionObject, 1.0));
+    worldNormal = normalize(modelNormalMatrix * vertexNormalObject);
+
+    // Calculate vertex position in clip coordinates
+    gl_Position = mvp * vec4(vertexPositionChunk, 1.0);
 
 #ifdef CLIPPING
     setClipDistance(worldPosition);

src/3d/symbols/qgspoint3dsymbol.cpp

@@ -272,12 +272,9 @@ QVariant QgsPoint3DSymbol::shapeProperty( const QString &property ) const
   return mShapeProperties.value( property );
 }
 
-QMatrix4x4 QgsPoint3DSymbol::billboardTransform() const
+float QgsPoint3DSymbol::billboardHeight() const
 {
-  QMatrix4x4 billboardTransformMatrix;
-  billboardTransformMatrix.translate( QVector3D( 0, mTransform.data()[13], 0 ) );
-
-  return billboardTransformMatrix;
+  return mTransform.data()[13];
 }
 
 QgsAbstractMaterialSettings *QgsPoint3DSymbol::materialSettings() const

src/3d/symbols/qgspoint3dsymbol.h

@@ -164,8 +164,8 @@ class _3D_EXPORT QgsPoint3DSymbol : public QgsAbstract3DSymbol SIP_NODEFAULTCTOR
     //! Sets transform for individual objects represented by the symbol
     void setTransform( const QMatrix4x4 &transform ) { mTransform = transform; }
 
-    //! Returns transform for billboards
-    QMatrix4x4 billboardTransform() const;
+    //! Returns how nuch the billboard should be elevated upwards
+    float billboardHeight() const;
 
     /**
      * Exports the geometries contained within the hierarchy of entity.

src/3d/symbols/qgspoint3dsymbol_p.cpp

@@ -215,6 +215,7 @@ void QgsInstancedPoint3DSymbolHandler::makeEntity( Qt3DCore::QEntity *parent, co
 
   // add transform (our geometry has coordinates relative to mChunkOrigin)
   Qt3DCore::QTransform *tr = new Qt3DCore::QTransform;
+  tr->setRotation( QQuaternion::fromAxisAndAngle( QVector3D( 1, 0, 0 ), -90 ) ); // flip map (x,y,z) to world (x,z,-y)
   QVector3D nodeTranslation = ( mChunkOrigin - context.origin() ).toVector3D();
   tr->setTranslation( QVector3D( nodeTranslation.x(), nodeTranslation.z(), -nodeTranslation.y() ) );
 
@@ -568,7 +569,9 @@ Qt3DCore::QTransform *QgsModelPoint3DSymbolHandler::transform( QVector3D positio
   tr->setMatrix( symbol->transform() );
   // position is relative to chunkOrigin
   QVector3D nodeTranslation = ( chunkOrigin - contextOrigin ).toVector3D();
-  tr->setTranslation( position + tr->translation() + QVector3D( nodeTranslation.x(), nodeTranslation.z(), -nodeTranslation.y() ) );
+  tr->setTranslation( tr->translation() +
+                      QVector3D( position.x(), position.z(), -position.y() ) +
+                      QVector3D( nodeTranslation.x(), nodeTranslation.z(), -nodeTranslation.y() ) );
   return tr;
 }
 
@@ -672,10 +675,11 @@ void QgsPoint3DBillboardSymbolHandler::makeEntity( Qt3DCore::QEntity *parent, co
 
   // Billboard Transform
   Qt3DCore::QTransform *billboardTransform = new Qt3DCore::QTransform();
-  billboardTransform->setMatrix( mSymbol->billboardTransform() );
+  billboardTransform->setRotation( QQuaternion::fromAxisAndAngle( QVector3D( 1, 0, 0 ), -90 ) ); // flip map (x,y,z) to world (x,z,-y)
+  QVector3D billboardHeightTranslation( 0, mSymbol->billboardHeight(), 0 );
   // our geometry has coordinates relative to mChunkOrigin
   QVector3D nodeTranslation = ( mChunkOrigin - context.origin() ).toVector3D();
-  billboardTransform->setTranslation( billboardTransform->translation() + QVector3D( nodeTranslation.x(), nodeTranslation.z(), -nodeTranslation.y() ) );
+  billboardTransform->setTranslation( billboardHeightTranslation + QVector3D( nodeTranslation.x(), nodeTranslation.z(), -nodeTranslation.y() ) );
 
   // Build the entity
   Qt3DCore::QEntity *entity = new Qt3DCore::QEntity;