sharpetronics/QGIS
1
0
Fork 0
mirror of https://github.com/qgis/QGIS.git synced 2025-02-25 00:58:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote and local QgsEptPointCloudIndex classes

Browse Source
This commit is contained in:
David Koňařík 2024-11-14 15:23:31 +01:00 committed by Martin Dobias
parent 643a7a2491
commit ed4a681b22
10 changed files with 345 additions and 514 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/core/CMakeLists.txt
View File

@ -2236,13 +2236,11 @@ if (WITH_EPT)
set(QGIS_CORE_SRCS ${QGIS_CORE_SRCS}
providers/ept/qgseptprovider.cpp
pointcloud/qgseptpointcloudindex.cpp
pointcloud/qgsremoteeptpointcloudindex.cpp
pointcloud/qgseptpointcloudblockrequest.cpp
)
set(QGIS_CORE_HDRS ${QGIS_CORE_HDRS}
providers/ept/qgseptprovider.h
pointcloud/qgseptpointcloudindex.h
pointcloud/qgsremoteeptpointcloudindex.h
pointcloud/qgseptpointcloudblockrequest.h
)

437
src/core/pointcloud/qgseptpointcloudindex.cpp
View File

@ -26,21 +26,31 @@
#include <QTime>
#include <QtDebug>
#include <QQueue>
#include <QNetworkRequest>
#include "qgsapplication.h"
#include "qgsblockingnetworkrequest.h"
#include "qgscachedpointcloudblockrequest.h"
#include "qgseptdecoder.h"
#include "qgseptpointcloudblockrequest.h"
#include "qgslazdecoder.h"
#include "qgscoordinatereferencesystem.h"
#include "qgspointcloudblockrequest.h"
#include "qgspointcloudrequest.h"
#include "qgspointcloudattribute.h"
#include "qgslogger.h"
#include "qgspointcloudexpression.h"
#include "qgssetrequestinitiator_p.h"
///@cond PRIVATE
#define PROVIDER_KEY QStringLiteral( "ept" )
#define PROVIDER_DESCRIPTION QStringLiteral( "EPT point cloud provider" )
QgsEptPointCloudIndex::QgsEptPointCloudIndex() = default;
QgsEptPointCloudIndex::QgsEptPointCloudIndex()
{
mHierarchyNodes.insert( IndexedPointCloudNode( 0, 0, 0, 0 ) );
}
QgsEptPointCloudIndex::~QgsEptPointCloudIndex() = default;
@ -52,37 +62,79 @@ std::unique_ptr<QgsPointCloudIndex> QgsEptPointCloudIndex::clone() const
return std::unique_ptr<QgsPointCloudIndex>( clone );
}
void QgsEptPointCloudIndex::load( const QString &fileName )
void QgsEptPointCloudIndex::load( const QString &urlString )
{
mUri = fileName;
QFile f( fileName );
if ( !f.open( QIODevice::ReadOnly ) )
QUrl url = urlString;
// Treat non-URLs as local files
if ( url.isValid() && ( url.scheme() == "http" || url.scheme() == "https" ) )
mAccessType = Remote;
else
mAccessType = Local;
mUri = urlString;
QStringList splitUrl = mUri.split( '/' );
splitUrl.pop_back();
mUrlDirectoryPart = splitUrl.join( '/' );
QByteArray content;
if ( mAccessType == Remote )
{
mError = tr( "Unable to open %1 for reading" ).arg( fileName );
mIsValid = false;
return;
QNetworkRequest nr = QNetworkRequest( QUrl( mUri ) );
QgsSetRequestInitiatorClass( nr, QStringLiteral( "QgsEptPointCloudIndex" ) );
QgsBlockingNetworkRequest req;
if ( req.get( nr ) != QgsBlockingNetworkRequest::NoError )
{
QgsDebugError( QStringLiteral( "Request failed: " ) + mUri );
mIsValid = false;
mError = req.errorMessage();
return;
}
content = req.reply().content();
}
else
{
QFile f( mUri );
if ( !f.open( QIODevice::ReadOnly ) )
{
mError = tr( "Unable to open %1 for reading" ).arg( mUri );
mIsValid = false;
return;
}
content = f.readAll();
}
const QDir directory = QFileInfo( fileName ).absoluteDir();
mDirectory = directory.absolutePath();
const QByteArray dataJson = f.readAll();
bool success = loadSchema( dataJson );
bool success = loadSchema( content );
if ( success )
{
// try to import the metadata too!
QFile manifestFile( mDirectory + QStringLiteral( "/ept-sources/manifest.json" ) );
if ( manifestFile.open( QIODevice::ReadOnly ) )
const QString manifestPath = mUrlDirectoryPart + QStringLiteral( "/ept-sources/manifest.json" );
QByteArray manifestJson;
if ( mAccessType == Remote )
{
const QByteArray manifestJson = manifestFile.readAll();
loadManifest( manifestJson );
QUrl manifestUrl( manifestPath );
QNetworkRequest nr = QNetworkRequest( QUrl( manifestUrl ) );
QgsSetRequestInitiatorClass( nr, QStringLiteral( "QgsEptPointCloudIndex" ) );
QgsBlockingNetworkRequest req;
if ( req.get( nr ) == QgsBlockingNetworkRequest::NoError )
manifestJson = req.reply().content();
}
else
{
QFile manifestFile( manifestPath );
if ( manifestFile.open( QIODevice::ReadOnly ) )
manifestJson = manifestFile.readAll();
}
if ( !manifestJson.isEmpty() )
loadManifest( manifestJson );
}
if ( success )
if ( !loadNodeHierarchy( IndexedPointCloudNode( 0, 0, 0, 0 ) ) )
{
success = loadHierarchy();
QgsDebugError( QStringLiteral( "Failed to load root EPT node" ) );
success = false;
}
mIsValid = success;
@ -93,31 +145,47 @@ void QgsEptPointCloudIndex::loadManifest( const QByteArray &manifestJson )
QJsonParseError err;
// try to import the metadata too!
const QJsonDocument manifestDoc = QJsonDocument::fromJson( manifestJson, &err );
if ( err.error == QJsonParseError::NoError )
if ( err.error != QJsonParseError::NoError )
return;
const QJsonArray manifestArray = manifestDoc.array();
if ( manifestArray.empty() )
return;
// TODO how to handle multiple?
const QJsonObject sourceObject = manifestArray.at( 0 ).toObject();
const QString metadataPath = sourceObject.value( QStringLiteral( "metadataPath" ) ).toString();
const QString fullMetadataPath = mUrlDirectoryPart + QStringLiteral( "/ept-sources/" ) + metadataPath;
QByteArray metadataJson;
if ( mAccessType == Remote )
{
const QJsonArray manifestArray = manifestDoc.array();
// TODO how to handle multiple?
if ( ! manifestArray.empty() )
{
const QJsonObject sourceObject = manifestArray.at( 0 ).toObject();
const QString metadataPath = sourceObject.value( QStringLiteral( "metadataPath" ) ).toString();
QFile metadataFile( mDirectory + QStringLiteral( "/ept-sources/" ) + metadataPath );
if ( metadataFile.open( QIODevice::ReadOnly ) )
{
const QByteArray metadataJson = metadataFile.readAll();
const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err );
if ( err.error == QJsonParseError::NoError )
{
const QJsonObject metadataObject = metadataDoc.object().value( QStringLiteral( "metadata" ) ).toObject();
if ( !metadataObject.empty() )
{
const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject();
mOriginalMetadata = sourceMetadata.toVariantMap();
}
}
}
}
QUrl metadataUrl( fullMetadataPath );
QNetworkRequest nr = QNetworkRequest( QUrl( metadataUrl ) );
QgsSetRequestInitiatorClass( nr, QStringLiteral( "QgsEptPointCloudIndex" ) );
QgsBlockingNetworkRequest req;
if ( req.get( nr ) != QgsBlockingNetworkRequest::NoError )
return;
metadataJson = req.reply().content();
}
else
{
QFile metadataFile( fullMetadataPath );
if ( ! metadataFile.open( QIODevice::ReadOnly ) )
return;
metadataJson = metadataFile.readAll();
}
const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err );
if ( err.error != QJsonParseError::NoError )
return;
const QJsonObject metadataObject = metadataDoc.object().value( QStringLiteral( "metadata" ) ).toObject();
if ( metadataObject.empty() )
return;
const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject();
mOriginalMetadata = sourceMetadata.toVariantMap();
}
bool QgsEptPointCloudIndex::loadSchema( const QByteArray &dataJson )
@ -315,47 +383,98 @@ std::unique_ptr<QgsPointCloudBlock> QgsEptPointCloudIndex::nodeData( const Index
return std::unique_ptr<QgsPointCloudBlock>( cached );
}
mHierarchyMutex.lock();
const bool found = mHierarchy.contains( n );
mHierarchyMutex.unlock();
if ( !found )
return nullptr;
std::unique_ptr<QgsPointCloudBlock> block;
if ( mAccessType == Remote )
{
std::unique_ptr<QgsPointCloudBlockRequest> blockRequest( asyncNodeData( n, request ) );
if ( !blockRequest )
return nullptr;
// we need to create a copy of the expression to pass to the decoder
// as the same QgsPointCloudExpression object mighgt be concurrently
// used on another thread, for example in a 3d view
QgsPointCloudExpression filterExpression = mFilterExpression;
QgsPointCloudAttributeCollection requestAttributes = request.attributes();
requestAttributes.extend( attributes(), filterExpression.referencedAttributes() );
QgsRectangle filterRect = request.filterRect();
QEventLoop loop;
connect( blockRequest.get(), &QgsPointCloudBlockRequest::finished, &loop, &QEventLoop::quit );
loop.exec();
std::unique_ptr<QgsPointCloudBlock> decoded;
if ( mDataType == QLatin1String( "binary" ) )
{
const QString filename = QStringLiteral( "%1/ept-data/%2.bin" ).arg( mDirectory, n.toString() );
decoded = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect );
block = blockRequest->takeBlock();
if ( !block )
{
QgsDebugError( QStringLiteral( "Error downloading node %1 data, error : %2 " ).arg( n.toString(), blockRequest->errorStr() ) );
}
}
else if ( mDataType == QLatin1String( "zstandard" ) )
else
{
const QString filename = QStringLiteral( "%1/ept-data/%2.zst" ).arg( mDirectory, n.toString() );
decoded = QgsEptDecoder::decompressZStandard( filename, attributes(), request.attributes(), scale(), offset(), filterExpression, filterRect );
}
else if ( mDataType == QLatin1String( "laszip" ) )
{
const QString filename = QStringLiteral( "%1/ept-data/%2.laz" ).arg( mDirectory, n.toString() );
decoded = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect );
// we need to create a copy of the expression to pass to the decoder
// as the same QgsPointCloudExpression object mighgt be concurrently
// used on another thread, for example in a 3d view
QgsPointCloudExpression filterExpression = mFilterExpression;
QgsPointCloudAttributeCollection requestAttributes = request.attributes();
requestAttributes.extend( attributes(), filterExpression.referencedAttributes() );
QgsRectangle filterRect = request.filterRect();
if ( mDataType == QLatin1String( "binary" ) )
{
const QString filename = QStringLiteral( "%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.toString() );
block = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect );
}
else if ( mDataType == QLatin1String( "zstandard" ) )
{
const QString filename = QStringLiteral( "%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.toString() );
block = QgsEptDecoder::decompressZStandard( filename, attributes(), request.attributes(), scale(), offset(), filterExpression, filterRect );
}
else if ( mDataType == QLatin1String( "laszip" ) )
{
const QString filename = QStringLiteral( "%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.toString() );
block = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect );
}
}
storeNodeDataToCache( decoded.get(), n, request );
return decoded;
storeNodeDataToCache( block.get(), n, request );
return block;
}
QgsPointCloudBlockRequest *QgsEptPointCloudIndex::asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request )
{
Q_UNUSED( n );
Q_UNUSED( request );
Q_ASSERT( false );
return nullptr; // unsupported
if ( QgsPointCloudBlock *cached = getNodeDataFromCache( n, request ) )
{
return new QgsCachedPointCloudBlockRequest( cached, n, mUri, attributes(), request.attributes(),
scale(), offset(), mFilterExpression, request.filterRect() );
}
if ( mAccessType != Remote )
return nullptr;
if ( !loadNodeHierarchy( n ) )
return nullptr;
QString fileUrl;
if ( mDataType == QLatin1String( "binary" ) )
{
fileUrl = QStringLiteral( "%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.toString() );
}
else if ( mDataType == QLatin1String( "zstandard" ) )
{
fileUrl = QStringLiteral( "%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.toString() );
}
else if ( mDataType == QLatin1String( "laszip" ) )
{
fileUrl = QStringLiteral( "%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.toString() );
}
else
{
return nullptr;
}
// we need to create a copy of the expression to pass to the decoder
// as the same QgsPointCloudExpression object might be concurrently
// used on another thread, for example in a 3d view
QgsPointCloudExpression filterExpression = mFilterExpression;
QgsPointCloudAttributeCollection requestAttributes = request.attributes();
requestAttributes.extend( attributes(), filterExpression.referencedAttributes() );
return new QgsEptPointCloudBlockRequest( n, fileUrl, mDataType, attributes(), requestAttributes, scale(), offset(), filterExpression, request.filterRect() );
}
bool QgsEptPointCloudIndex::hasNode( const IndexedPointCloudNode &n ) const
{
return loadNodeHierarchy( n );
}
QgsCoordinateReferenceSystem QgsEptPointCloudIndex::crs() const
@ -368,6 +487,32 @@ qint64 QgsEptPointCloudIndex::pointCount() const
return mPointCount;
}
qint64 QgsEptPointCloudIndex::nodePointCount( const IndexedPointCloudNode &nodeId ) const
{
// First try loading our cached value
{
QMutexLocker locker( &mHierarchyMutex );
qint64 pointCount = mHierarchy.value( nodeId, -1 );
if ( pointCount != -1 )
return pointCount;
}
// Try loading all nodes' hierarchy files on the path from root and stop when
// one contains the point count for nodeId
QVector<IndexedPointCloudNode> pathToRoot = nodePathToRoot( nodeId );
for ( int i = pathToRoot.size() - 1; i >= 0; --i )
{
loadSingleNodeHierarchy( pathToRoot[i] );
QMutexLocker locker( &mHierarchyMutex );
qint64 pointCount = mHierarchy.value( nodeId, -1 );
if ( pointCount != -1 )
return pointCount;
}
return -1;
}
bool QgsEptPointCloudIndex::hasStatisticsMetadata() const
{
return !mMetadataStats.isEmpty();
@ -439,57 +584,126 @@ QVariant QgsEptPointCloudIndex::metadataClassStatistic( const QString &attribute
return values.value( value.toInt() );
}
bool QgsEptPointCloudIndex::loadHierarchy()
bool QgsEptPointCloudIndex::loadSingleNodeHierarchy( const IndexedPointCloudNode &nodeId ) const
{
QQueue<QString> queue;
queue.enqueue( QStringLiteral( "0-0-0-0" ) );
while ( !queue.isEmpty() )
mHierarchyMutex.lock();
const bool foundInHierarchy = mHierarchy.contains( nodeId );
const bool foundInHierarchyNodes = mHierarchyNodes.contains( nodeId );
mHierarchyMutex.unlock();
// The hierarchy of the node is found => No need to load its file
if ( foundInHierarchy )
return true;
// We don't know that this node has a hierarchy file => We have nothing to load
if ( !foundInHierarchyNodes )
return true;
const QString filePath = QStringLiteral( "%1/ept-hierarchy/%2.json" ).arg( mUrlDirectoryPart, nodeId.toString() );
QByteArray dataJsonH;
if ( mAccessType == Remote )
{
const QString filename = QStringLiteral( "%1/ept-hierarchy/%2.json" ).arg( mDirectory, queue.dequeue() );
QFile fH( filename );
if ( !fH.open( QIODevice::ReadOnly ) )
QNetworkRequest nr( filePath );
QgsSetRequestInitiatorClass( nr, QStringLiteral( "QgsEptPointCloudIndex" ) );
nr.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
nr.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
std::unique_ptr<QgsTileDownloadManagerReply> reply( QgsApplication::tileDownloadManager()->get( nr ) );
QEventLoop loop;
connect( reply.get(), &QgsTileDownloadManagerReply::finished, &loop, &QEventLoop::quit );
loop.exec();
if ( reply->error() != QNetworkReply::NoError )
{
QgsDebugMsgLevel( QStringLiteral( "unable to read hierarchy from file %1" ).arg( filename ), 2 );
mError = QStringLiteral( "unable to read hierarchy from file %1" ).arg( filename );
QgsDebugError( QStringLiteral( "Request failed: " ) + filePath );
return false;
}
const QByteArray dataJsonH = fH.readAll();
QJsonParseError errH;
const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH );
if ( errH.error != QJsonParseError::NoError )
{
QgsDebugMsgLevel( QStringLiteral( "QJsonParseError when reading hierarchy from file %1" ).arg( filename ), 2 );
mError = QStringLiteral( "QJsonParseError when reading hierarchy from file %1" ).arg( filename );
return false;
}
const QJsonObject rootHObj = docH.object();
for ( auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it )
{
const QString nodeIdStr = it.key();
const int nodePointCount = it.value().toInt();
if ( nodePointCount < 0 )
{
queue.enqueue( nodeIdStr );
}
else
{
const IndexedPointCloudNode nodeId = IndexedPointCloudNode::fromString( nodeIdStr );
mHierarchyMutex.lock();
mHierarchy[nodeId] = nodePointCount;
mHierarchyMutex.unlock();
}
}
dataJsonH = reply->data();
}
else
{
QFile file( filePath );
if ( ! file.open( QIODevice::ReadOnly ) )
{
QgsDebugError( QStringLiteral( "Loading file failed: " ) + filePath );
return false;
}
dataJsonH = file.readAll();
}
QJsonParseError errH;
const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH );
if ( errH.error != QJsonParseError::NoError )
{
QgsDebugMsgLevel( QStringLiteral( "QJsonParseError when reading hierarchy from file %1" ).arg( filePath ), 2 );
return false;
}
QMutexLocker locker( &mHierarchyMutex );
const QJsonObject rootHObj = docH.object();
for ( auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it )
{
const QString nodeIdStr = it.key();
const int nodePointCount = it.value().toInt();
const IndexedPointCloudNode nodeId = IndexedPointCloudNode::fromString( nodeIdStr );
if ( nodePointCount >= 0 )
mHierarchy[nodeId] = nodePointCount;
else if ( nodePointCount == -1 )
mHierarchyNodes.insert( nodeId );
}
return true;
}
QVector<IndexedPointCloudNode> QgsEptPointCloudIndex::nodePathToRoot( const IndexedPointCloudNode &nodeId ) const
{
QVector<IndexedPointCloudNode> path;
IndexedPointCloudNode currentNode = nodeId;
do
{
path.push_back( currentNode );
currentNode = currentNode.parentNode();
}
while ( currentNode.d() >= 0 );
return path;
}
bool QgsEptPointCloudIndex::loadNodeHierarchy( const IndexedPointCloudNode &nodeId ) const
{
mHierarchyMutex.lock();
bool found = mHierarchy.contains( nodeId );
mHierarchyMutex.unlock();
if ( found )
return true;
QVector<IndexedPointCloudNode> pathToRoot = nodePathToRoot( nodeId );
for ( int i = pathToRoot.size() - 1; i >= 0 && !mHierarchy.contains( nodeId ); --i )
{
const IndexedPointCloudNode node = pathToRoot[i];
if ( !loadSingleNodeHierarchy( node ) )
return false;
}
mHierarchyMutex.lock();
found = mHierarchy.contains( nodeId );
mHierarchyMutex.unlock();
return found;
}
bool QgsEptPointCloudIndex::isValid() const
{
return mIsValid;
}
QgsPointCloudIndex::AccessType QgsEptPointCloudIndex::accessType() const
{
return mAccessType;
}
void QgsEptPointCloudIndex::copyCommonProperties( QgsEptPointCloudIndex *destination ) const
{
QgsPointCloudIndex::copyCommonProperties( destination );
@ -497,8 +711,9 @@ void QgsEptPointCloudIndex::copyCommonProperties( QgsEptPointCloudIndex *destina
// QgsEptPointCloudIndex specific fields
destination->mIsValid = mIsValid;
destination->mDataType = mDataType;
destination->mDirectory = mDirectory;
destination->mUrlDirectoryPart = mUrlDirectoryPart;
destination->mWkt = mWkt;
destination->mHierarchyNodes = mHierarchyNodes;
destination->mPointCount = mPointCount;
destination->mMetadataStats = mMetadataStats;
destination->mAttributeClasses = mAttributeClasses;

15
src/core/pointcloud/qgseptpointcloudindex.h
View File

@ -48,9 +48,11 @@ class CORE_EXPORT QgsEptPointCloudIndex: public QgsPointCloudIndex
std::unique_ptr<QgsPointCloudBlock> nodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) override;
QgsPointCloudBlockRequest *asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) override;
bool hasNode( const IndexedPointCloudNode &n ) const override;
QgsCoordinateReferenceSystem crs() const override;
qint64 pointCount() const override;
virtual qint64 nodePointCount( const IndexedPointCloudNode &n ) const override;
bool hasStatisticsMetadata() const override;
QVariant metadataStatistic( const QString &attribute, Qgis::Statistic statistic ) const override;
QVariantList metadataClasses( const QString &attribute ) const override;
@ -58,7 +60,7 @@ class CORE_EXPORT QgsEptPointCloudIndex: public QgsPointCloudIndex
QVariantMap originalMetadata() const override { return mOriginalMetadata; }
bool isValid() const override;
QgsPointCloudIndex::AccessType accessType() const override { return QgsPointCloudIndex::Local; };
QgsPointCloudIndex::AccessType accessType() const override;
/**
* Copies common properties to the \a destination index
@ -70,13 +72,20 @@ class CORE_EXPORT QgsEptPointCloudIndex: public QgsPointCloudIndex
bool loadSchema( const QByteArray &dataJson );
void loadManifest( const QByteArray &manifestJson );
bool loadSchema( QFile &f );
bool loadHierarchy();
bool loadSingleNodeHierarchy( const IndexedPointCloudNode &nodeId ) const;
QVector<IndexedPointCloudNode> nodePathToRoot( const IndexedPointCloudNode &nodeId ) const;
bool loadNodeHierarchy( const IndexedPointCloudNode &nodeId ) const;
bool mIsValid = false;
QgsPointCloudIndex::AccessType mAccessType = Local;
QString mDataType;
QString mDirectory;
QString mWkt;
QString mUrlDirectoryPart;
//! Contains the nodes that will have */ept-hierarchy/d-x-y-z.json file
mutable QSet<IndexedPointCloudNode> mHierarchyNodes;
qint64 mPointCount = 0;
struct AttributeStatistics

13
src/core/pointcloud/qgspointcloudindex.cpp
View File

@ -201,8 +201,7 @@ qint64 QgsPointCloudIndex::nodePointCount( const IndexedPointCloudNode &n ) cons
QList<IndexedPointCloudNode> QgsPointCloudIndex::nodeChildren( const IndexedPointCloudNode &n ) const
{
QMutexLocker locker( &mHierarchyMutex );
Q_ASSERT( mHierarchy.contains( n ) );
Q_ASSERT( hasNode( n ) );
QList<IndexedPointCloudNode> lst;
const int d = n.d() + 1;
const int x = n.x() * 2;
@ -213,7 +212,7 @@ QList<IndexedPointCloudNode> QgsPointCloudIndex::nodeChildren( const IndexedPoin
{
int dx = i & 1, dy = !!( i & 2 ), dz = !!( i & 4 );
const IndexedPointCloudNode n2( d, x + dx, y + dy, z + dz );
if ( mHierarchy.contains( n2 ) )
if ( hasNode( n2 ) )
lst.append( n2 );
}
return lst;
@ -278,14 +277,6 @@ int QgsPointCloudIndex::span() const
return mSpan;
}
int QgsPointCloudIndex::nodePointCount( const IndexedPointCloudNode &n )
{
mHierarchyMutex.lock();
const int count = mHierarchy.value( n, -1 );
mHierarchyMutex.unlock();
return count;
}
bool QgsPointCloudIndex::setSubsetString( const QString &subset )
{
const QString lastExpression = mFilterExpression;

5
src/core/pointcloud/qgspointcloudindex.h
View File

@ -344,11 +344,6 @@ class CORE_EXPORT QgsPointCloudIndex: public QObject
*/
int span() const;
/**
* Returns the number of points of indexed point cloud node \a n
*/
int nodePointCount( const IndexedPointCloudNode &n );
/**
* Sets the string used to define a subset of the point cloud.
* \param subset The subset string to be used in a \a QgsPointCloudExpression

275
src/core/pointcloud/qgsremoteeptpointcloudindex.cpp
View File

@ -1,275 +0,0 @@
/***************************************************************************
qgsremoteeptpointcloudindex.cpp
--------------------
begin : March 2021
copyright : (C) 2021 by Belgacem Nedjima
email : belgacem dot nedjima at gmail dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#include "qgsremoteeptpointcloudindex.h"
#include "moc_qgsremoteeptpointcloudindex.cpp"
#include <QFile>
#include <QFileInfo>
#include <QDir>
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QTime>
#include <QtDebug>
#include <QQueue>
#include <QTimer>
#include "qgsapplication.h"
#include "qgspointcloudrequest.h"
#include "qgspointcloudattribute.h"
#include "qgslogger.h"
#include "qgsfeedback.h"
#include "qgstiledownloadmanager.h"
#include "qgsblockingnetworkrequest.h"
#include "qgseptpointcloudblockrequest.h"
#include "qgscachedpointcloudblockrequest.h"
#include "qgspointcloudexpression.h"
#include "qgsnetworkaccessmanager.h"
#include "qgssetrequestinitiator_p.h"
///@cond PRIVATE
QgsRemoteEptPointCloudIndex::QgsRemoteEptPointCloudIndex() : QgsEptPointCloudIndex()
{
mHierarchyNodes.insert( IndexedPointCloudNode( 0, 0, 0, 0 ) );
}
QgsRemoteEptPointCloudIndex::~QgsRemoteEptPointCloudIndex() = default;
std::unique_ptr<QgsPointCloudIndex> QgsRemoteEptPointCloudIndex::clone() const
{
QgsRemoteEptPointCloudIndex *clone = new QgsRemoteEptPointCloudIndex;
QMutexLocker locker( &mHierarchyMutex );
copyCommonProperties( clone );
return std::unique_ptr<QgsPointCloudIndex>( clone );
}
QList<IndexedPointCloudNode> QgsRemoteEptPointCloudIndex::nodeChildren( const IndexedPointCloudNode &n ) const
{
QList<IndexedPointCloudNode> lst;
if ( !loadNodeHierarchy( n ) )
return lst;
const int d = n.d() + 1;
const int x = n.x() * 2;
const int y = n.y() * 2;
const int z = n.z() * 2;
lst.reserve( 8 );
for ( int i = 0; i < 8; ++i )
{
int dx = i & 1, dy = !!( i & 2 ), dz = !!( i & 4 );
const IndexedPointCloudNode n2( d, x + dx, y + dy, z + dz );
if ( loadNodeHierarchy( n2 ) )
lst.append( n2 );
}
return lst;
}
void QgsRemoteEptPointCloudIndex::load( const QString &uri )
{
mUri = uri;
QStringList splitUrl = uri.split( '/' );
mUrlFileNamePart = splitUrl.back();
splitUrl.pop_back();
mUrlDirectoryPart = splitUrl.join( '/' );
QNetworkRequest nr = QNetworkRequest( QUrl( mUri ) );
QgsBlockingNetworkRequest req;
const QgsBlockingNetworkRequest::ErrorCode errCode = req.get( nr );
if ( errCode != QgsBlockingNetworkRequest::NoError )
{
QgsDebugError( QStringLiteral( "Request failed: " ) + uri );
mIsValid = false;
mError = req.errorMessage();
return;
}
const QgsNetworkReplyContent reply = req.reply();
mIsValid = loadSchema( reply.content() );
}
std::unique_ptr<QgsPointCloudBlock> QgsRemoteEptPointCloudIndex::nodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request )
{
if ( QgsPointCloudBlock *cached = getNodeDataFromCache( n, request ) )
{
return std::unique_ptr<QgsPointCloudBlock>( cached );
}
std::unique_ptr<QgsPointCloudBlockRequest> blockRequest( asyncNodeData( n, request ) );
if ( !blockRequest )
return nullptr;
QEventLoop loop;
connect( blockRequest.get(), &QgsPointCloudBlockRequest::finished, &loop, &QEventLoop::quit );
loop.exec();
std::unique_ptr<QgsPointCloudBlock> block = blockRequest->takeBlock();
if ( !block )
{
QgsDebugError( QStringLiteral( "Error downloading node %1 data, error : %2 " ).arg( n.toString(), blockRequest->errorStr() ) );
}
storeNodeDataToCache( block.get(), n, request );
return block;
}
QgsPointCloudBlockRequest *QgsRemoteEptPointCloudIndex::asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request )
{
if ( QgsPointCloudBlock *cached = getNodeDataFromCache( n, request ) )
{
return new QgsCachedPointCloudBlockRequest( cached, n, mUri, attributes(), request.attributes(),
scale(), offset(), mFilterExpression, request.filterRect() );
}
if ( !loadNodeHierarchy( n ) )
return nullptr;
QString fileUrl;
if ( mDataType == QLatin1String( "binary" ) )
{
fileUrl = QStringLiteral( "%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.toString() );
}
else if ( mDataType == QLatin1String( "zstandard" ) )
{
fileUrl = QStringLiteral( "%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.toString() );
}
else if ( mDataType == QLatin1String( "laszip" ) )
{
fileUrl = QStringLiteral( "%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.toString() );
}
else
{
return nullptr;
}
// we need to create a copy of the expression to pass to the decoder
// as the same QgsPointCloudExpression object might be concurrently
// used on another thread, for example in a 3d view
QgsPointCloudExpression filterExpression = mFilterExpression;
QgsPointCloudAttributeCollection requestAttributes = request.attributes();
requestAttributes.extend( attributes(), filterExpression.referencedAttributes() );
return new QgsEptPointCloudBlockRequest( n, fileUrl, mDataType, attributes(), requestAttributes, scale(), offset(), filterExpression, request.filterRect() );
}
bool QgsRemoteEptPointCloudIndex::hasNode( const IndexedPointCloudNode &n ) const
{
return loadNodeHierarchy( n );
}
bool QgsRemoteEptPointCloudIndex::loadNodeHierarchy( const IndexedPointCloudNode &nodeId ) const
{
mHierarchyMutex.lock();
bool found = mHierarchy.contains( nodeId );
mHierarchyMutex.unlock();
if ( found )
return true;
QVector<IndexedPointCloudNode> nodePathToRoot;
{
IndexedPointCloudNode currentNode = nodeId;
do
{
nodePathToRoot.push_back( currentNode );
currentNode = currentNode.parentNode();
}
while ( currentNode.d() >= 0 );
}
for ( int i = nodePathToRoot.size() - 1; i >= 0 && !mHierarchy.contains( nodeId ); --i )
{
const IndexedPointCloudNode node = nodePathToRoot[i];
//! The hierarchy of the node is found => No need to load its file
mHierarchyMutex.lock();
const bool foundInHierarchy = mHierarchy.contains( node );
const bool foundInHierarchyNodes = mHierarchyNodes.contains( node );
mHierarchyMutex.unlock();
if ( foundInHierarchy )
continue;
if ( !foundInHierarchyNodes )
continue;
const QString fileUrl = QStringLiteral( "%1/ept-hierarchy/%2.json" ).arg( mUrlDirectoryPart, node.toString() );
QNetworkRequest nr( fileUrl );
QgsSetRequestInitiatorClass( nr, QStringLiteral( "QgsRemoteEptPointCloudIndex" ) );
nr.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
nr.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
std::unique_ptr<QgsTileDownloadManagerReply> reply( QgsApplication::tileDownloadManager()->get( nr ) );
QEventLoop loop;
connect( reply.get(), &QgsTileDownloadManagerReply::finished, &loop, &QEventLoop::quit );
loop.exec();
if ( reply->error() != QNetworkReply::NoError )
{
QgsDebugError( QStringLiteral( "Request failed: " ) + mUri );
return false;
}
const QByteArray dataJsonH = reply->data();
QJsonParseError errH;
const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH );
if ( errH.error != QJsonParseError::NoError )
{
QgsDebugMsgLevel( QStringLiteral( "QJsonParseError when reading hierarchy from file %1" ).arg( fileUrl ), 2 );
return false;
}
const QJsonObject rootHObj = docH.object();
for ( auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it )
{
const QString nodeIdStr = it.key();
const int nodePointCount = it.value().toInt();
const IndexedPointCloudNode nodeId = IndexedPointCloudNode::fromString( nodeIdStr );
mHierarchyMutex.lock();
if ( nodePointCount >= 0 )
mHierarchy[nodeId] = nodePointCount;
else if ( nodePointCount == -1 )
mHierarchyNodes.insert( nodeId );
mHierarchyMutex.unlock();
}
}
mHierarchyMutex.lock();
found = mHierarchy.contains( nodeId );
mHierarchyMutex.unlock();
return found;
}
bool QgsRemoteEptPointCloudIndex::isValid() const
{
return mIsValid;
}
void QgsRemoteEptPointCloudIndex::copyCommonProperties( QgsRemoteEptPointCloudIndex *destination ) const
{
QgsEptPointCloudIndex::copyCommonProperties( destination );
// QgsRemoteEptPointCloudIndex specific fields
destination->mUrlDirectoryPart = mUrlDirectoryPart;
destination->mUrlFileNamePart = mUrlFileNamePart;
destination->mHierarchyNodes = mHierarchyNodes;
}
///@endcond

86
src/core/pointcloud/qgsremoteeptpointcloudindex.h
View File

@ -1,86 +0,0 @@
/***************************************************************************
qgsremoteeptpointcloudindex.h
--------------------
begin : March 2021
copyright : (C) 2021 by Belgacem Nedjima
email : belgacem dot nedjima at gmail dot com
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef QGSREMOTEEPTPOINTCLOUDINDEX_H
#define QGSREMOTEEPTPOINTCLOUDINDEX_H
#include <QObject>
#include <QString>
#include <QHash>
#include <QStringList>
#include <QVector>
#include <QList>
#include <QFile>
#include <QUrl>
#include <QSet>
#include "qgspointcloudindex.h"
#include "qgspointcloudattribute.h"
#include "qgsstatisticalsummary.h"
#include "qgis_sip.h"
#include "qgseptpointcloudindex.h"
///@cond PRIVATE
#define SIP_NO_FILE
class QgsCoordinateReferenceSystem;
class QgsTileDownloadManager;
class CORE_EXPORT QgsRemoteEptPointCloudIndex: public QgsEptPointCloudIndex
{
Q_OBJECT
public:
explicit QgsRemoteEptPointCloudIndex();
~QgsRemoteEptPointCloudIndex();
std::unique_ptr<QgsPointCloudIndex> clone() const override;
QList<IndexedPointCloudNode> nodeChildren( const IndexedPointCloudNode &n ) const override;
void load( const QString &fileName ) override;
std::unique_ptr<QgsPointCloudBlock> nodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) override;
QgsPointCloudBlockRequest *asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request ) override;
bool hasNode( const IndexedPointCloudNode &n ) const override;
bool isValid() const override;
QgsPointCloudIndex::AccessType accessType() const override { return QgsPointCloudIndex::Remote; }
/**
* Copies common properties to the \a destination index
* \since QGIS 3.26
*/
void copyCommonProperties( QgsRemoteEptPointCloudIndex *destination ) const;
private:
bool loadNodeHierarchy( const IndexedPointCloudNode &nodeId ) const;
QString mUrlDirectoryPart;
QString mUrlFileNamePart;
QUrl mUrl;
//! Contains the nodes that will have */ept-hierarchy/d-x-y-z.json file
mutable QSet<IndexedPointCloudNode> mHierarchyNodes;
};
///@endcond
#endif // QGSREMOTEEPTPOINTCLOUDINDEX_H

6
src/core/providers/ept/qgseptprovider.cpp
View File

@ -19,7 +19,6 @@
#include "qgseptprovider.h"
#include "moc_qgseptprovider.cpp"
#include "qgseptpointcloudindex.h"
#include "qgsremoteeptpointcloudindex.h"
#include "qgsruntimeprofiler.h"
#include "qgsapplication.h"
#include "qgsprovidersublayerdetails.h"
@ -40,10 +39,7 @@ QgsEptProvider::QgsEptProvider(
Qgis::DataProviderReadFlags flags )
: QgsPointCloudDataProvider( uri, options, flags )
{
if ( uri.startsWith( QStringLiteral( "http" ), Qt::CaseSensitivity::CaseInsensitive ) )
mIndex.reset( new QgsRemoteEptPointCloudIndex );
else
mIndex.reset( new QgsEptPointCloudIndex );
mIndex.reset( new QgsEptPointCloudIndex );
std::unique_ptr< QgsScopedRuntimeProfile > profile;
if ( QgsApplication::profiler()->groupIsActive( QStringLiteral( "projectload" ) ) )

1
src/core/providers/ept/qgseptprovider.h
View File

@ -27,7 +27,6 @@
#define SIP_NO_FILE
class QgsEptPointCloudIndex;
class QgsRemoteEptPointCloudIndex;
class QgsEptProvider: public QgsPointCloudDataProvider
{

19
src/core/providers/vpc/qgsvirtualpointcloudprovider.cpp
View File

@ -22,7 +22,6 @@
#include "moc_qgsvirtualpointcloudprovider.cpp"
#include "qgscopcpointcloudindex.h"
#include "qgseptpointcloudindex.h"
#include "qgsremoteeptpointcloudindex.h"
#include "qgspointcloudsubindex.h"
#include "qgspointcloudclassifiedrenderer.h"
#include "qgspointcloudextentrenderer.h"
@ -391,20 +390,10 @@ void QgsVirtualPointCloudProvider::loadSubIndex( int i )
if ( sl.index() )
return;
if ( sl.uri().startsWith( QStringLiteral( "http" ), Qt::CaseSensitivity::CaseInsensitive ) )
{
if ( sl.uri().endsWith( QStringLiteral( "copc.laz" ), Qt::CaseSensitivity::CaseInsensitive ) )
sl.setIndex( new QgsCopcPointCloudIndex() );
else if ( sl.uri().endsWith( QStringLiteral( "ept.json" ), Qt::CaseSensitivity::CaseInsensitive ) )
sl.setIndex( new QgsRemoteEptPointCloudIndex() );
}
else
{
if ( sl.uri().endsWith( QStringLiteral( "copc.laz" ), Qt::CaseSensitivity::CaseInsensitive ) )
sl.setIndex( new QgsCopcPointCloudIndex() );
else if ( sl.uri().endsWith( QStringLiteral( "ept.json" ), Qt::CaseSensitivity::CaseInsensitive ) )
sl.setIndex( new QgsEptPointCloudIndex() );
}
if ( sl.uri().endsWith( QStringLiteral( "copc.laz" ), Qt::CaseSensitivity::CaseInsensitive ) )
sl.setIndex( new QgsCopcPointCloudIndex() );
else if ( sl.uri().endsWith( QStringLiteral( "ept.json" ), Qt::CaseSensitivity::CaseInsensitive ) )
sl.setIndex( new QgsEptPointCloudIndex() );
if ( !sl.index() )
return;