diff --git a/src/Data/ObfMapSectionInfo_P.h b/src/Data/ObfMapSectionInfo_P.h index 13efa4ead..316a4f449 100644 --- a/src/Data/ObfMapSectionInfo_P.h +++ b/src/Data/ObfMapSectionInfo_P.h @@ -7,6 +7,7 @@ #include "QtExtensions.h" #include "ignore_warnings_on_external_includes.h" #include +#include #include #include #include @@ -44,6 +45,8 @@ namespace OsmAnd bool hasChildrenDataBoxes; uint32_t firstDataBoxInnerOffset; + mutable std::shared_ptr> subNodeIds; + mutable QMutex _subNodeIdsMutex; friend class OsmAnd::ObfMapSectionReader_P; }; @@ -57,6 +60,8 @@ namespace OsmAnd mutable std::shared_ptr< const QList< std::shared_ptr > > _rootNodes; mutable QAtomicInt _rootNodesLoaded; mutable QMutex _rootNodesLoadMutex; + mutable QHash> _nodeCache; + mutable QReadWriteLock _nodeCacheAccessMutex; public: virtual ~ObfMapSectionLevel_P(); diff --git a/src/Data/ObfMapSectionReader_P.cpp b/src/Data/ObfMapSectionReader_P.cpp index 4df90a3f8..5e2eeae1f 100644 --- a/src/Data/ObfMapSectionReader_P.cpp +++ b/src/Data/ObfMapSectionReader_P.cpp @@ -20,6 +20,8 @@ #include "Logging.h" #include "Utilities.h" +const int MAX_TREE_DEPTH_IN_CACHE = 7; + using google::protobuf::internal::WireFormatLite; OsmAnd::ObfMapSectionReader_P::ObfMapSectionReader_P() @@ -396,39 +398,139 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren( const ObfReader_P& reader, const std::shared_ptr& section, const std::shared_ptr& treeNode, + QHash>& nodeCache, + QReadWriteLock& nodeCacheAccessMutex, + int treeDepth, MapSurfaceType& outChildrenSurfaceType, QList< std::shared_ptr >* nodesWithData, const AreaI* bbox31, const std::shared_ptr& queryController, ObfMapSectionReader_Metrics::Metric_loadMapObjects* const metric) { - const auto cis = reader.getCodedInputStream().get(); + QList> childNodes; - outChildrenSurfaceType = MapSurfaceType::Undefined; - for (;;) + bool fromCache = false; + if (!treeNode->subNodeIds) { - const auto tag = cis->ReadTag(); - switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) + QMutexLocker scopedLocker(&treeNode->_subNodeIdsMutex); + + if (!treeNode->subNodeIds) { - case 0: - if (!ObfReaderUtilities::reachedDataEnd(cis)) - return; + // Read all child nodes and make a lookup table + const std::shared_ptr> subNodeIds(new QList); + const auto cis = reader.getCodedInputStream().get(); + cis->Seek(treeNode->offset); + const auto oldLimit = cis->PushLimit(treeNode->length); + cis->Skip(treeNode->firstDataBoxInnerOffset); + outChildrenSurfaceType = MapSurfaceType::Undefined; + bool keepReading = true; + while (keepReading) + { + const auto tag = cis->ReadTag(); + switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) + { + case 0: + keepReading = false; + if (!ObfReaderUtilities::reachedDataEnd(cis)) + break; - return; - case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber: + break; + case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber: + { + const auto length = ObfReaderUtilities::readBigEndianInt(cis); + const auto offset = cis->CurrentPosition(); + const auto prevLimit = cis->PushLimit(length); + + std::shared_ptr childNode( + new ObfMapSectionLevelTreeNode(treeNode->level)); + childNode->surfaceType = treeNode->surfaceType; + childNode->offset = offset; + childNode->length = length; + readTreeNode(reader, section, treeNode->area31, childNode); + + ObfReaderUtilities::ensureAllDataWasRead(cis); + cis->PopLimit(prevLimit); + + // Update metric + if (metric) + metric->visitedNodes++; + + ObfMapSectionDataBlockId subNodeId; + subNodeId.sectionRuntimeGeneratedId = section->runtimeGeneratedId; + subNodeId.offset = childNode->offset; + + // Register node in lookup table + subNodeIds->append(subNodeId); + + // Store node in cache + if (treeDepth < MAX_TREE_DEPTH_IN_CACHE) + { + QWriteLocker scopedLocker(&nodeCacheAccessMutex); + + nodeCache.insert(subNodeId, childNode); + } + + if (bbox31) + { + const auto shouldSkip = + !bbox31->contains(childNode->area31) && + !childNode->area31.contains(*bbox31) && + !bbox31->intersects(childNode->area31); + if (shouldSkip) + break; + } + + // Update metric + if (metric) + metric->acceptedNodes++; + + if (nodesWithData && childNode->dataOffset > 0) + nodesWithData->push_back(childNode); + + if (childNode->hasChildrenDataBoxes) + childNodes.append(childNode); + + if (childNode->surfaceType != MapSurfaceType::Undefined) + { + if (outChildrenSurfaceType == MapSurfaceType::Undefined) + outChildrenSurfaceType = childNode->surfaceType; + else if (outChildrenSurfaceType != childNode->surfaceType) + outChildrenSurfaceType = MapSurfaceType::Mixed; + } + + break; + } + default: + ObfReaderUtilities::skipUnknownField(cis, tag); + break; + } + } + ObfReaderUtilities::ensureAllDataWasRead(cis); + cis->PopLimit(oldLimit); + + // Store lookup table + if (treeDepth < MAX_TREE_DEPTH_IN_CACHE) { - const auto length = ObfReaderUtilities::readBigEndianInt(cis); - const auto offset = cis->CurrentPosition(); - const auto oldLimit = cis->PushLimit(length); + treeNode->subNodeIds = subNodeIds; + } + } + else + fromCache = true; + } + else + fromCache = true; - const std::shared_ptr childNode(new ObfMapSectionLevelTreeNode(treeNode->level)); - childNode->surfaceType = treeNode->surfaceType; - childNode->offset = offset; - childNode->length = length; - readTreeNode(reader, section, treeNode->area31, childNode); + if (fromCache) + { + // Use lookup table to access child nodes stored in cache + QReadLocker scopedLocker(&nodeCacheAccessMutex); - ObfReaderUtilities::ensureAllDataWasRead(cis); - cis->PopLimit(oldLimit); + for (const auto& subNodeId : constOf(*treeNode->subNodeIds)) + { + const auto& citChildNode = nodeCache.constFind(subNodeId); + if (citChildNode != nodeCache.cend()) + { + const auto& childNode = citChildNode.value(); // Update metric if (metric) @@ -441,7 +543,7 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren( !childNode->area31.contains(*bbox31) && !bbox31->intersects(childNode->area31); if (shouldSkip) - break; + continue; } // Update metric @@ -451,33 +553,43 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren( if (nodesWithData && childNode->dataOffset > 0) nodesWithData->push_back(childNode); - auto subchildrenSurfaceType = MapSurfaceType::Undefined; if (childNode->hasChildrenDataBoxes) - { - cis->Seek(childNode->offset); - const auto oldLimit = cis->PushLimit(childNode->length); - - cis->Skip(childNode->firstDataBoxInnerOffset); - readTreeNodeChildren(reader, section, childNode, subchildrenSurfaceType, nodesWithData, bbox31, queryController, metric); + childNodes.append(childNode); - ObfReaderUtilities::ensureAllDataWasRead(cis); - cis->PopLimit(oldLimit); - } - - const auto surfaceTypeToMerge = (subchildrenSurfaceType != MapSurfaceType::Undefined) ? subchildrenSurfaceType : childNode->surfaceType; - if (surfaceTypeToMerge != MapSurfaceType::Undefined) + if (childNode->surfaceType != MapSurfaceType::Undefined) { if (outChildrenSurfaceType == MapSurfaceType::Undefined) - outChildrenSurfaceType = surfaceTypeToMerge; - else if (outChildrenSurfaceType != surfaceTypeToMerge) + outChildrenSurfaceType = childNode->surfaceType; + else if (outChildrenSurfaceType != childNode->surfaceType) outChildrenSurfaceType = MapSurfaceType::Mixed; } - - break; } - default: - ObfReaderUtilities::skipUnknownField(cis, tag); - break; + } + } + + for (const auto& childNode : constOf(childNodes)) + { + auto subchildrenSurfaceType = MapSurfaceType::Undefined; + readTreeNodeChildren( + reader, + section, + childNode, + nodeCache, + nodeCacheAccessMutex, + treeDepth + 1, + subchildrenSurfaceType, + nodesWithData, + bbox31, + queryController, + metric); + const auto surfaceTypeToMerge = + (subchildrenSurfaceType != MapSurfaceType::Undefined) ? subchildrenSurfaceType : childNode->surfaceType; + if (surfaceTypeToMerge != MapSurfaceType::Undefined) + { + if (outChildrenSurfaceType == MapSurfaceType::Undefined) + outChildrenSurfaceType = surfaceTypeToMerge; + else if (outChildrenSurfaceType != surfaceTypeToMerge) + outChildrenSurfaceType = MapSurfaceType::Mixed; } } } @@ -1102,17 +1214,22 @@ void OsmAnd::ObfMapSectionReader_P::loadMapObjects( auto rootSubnodesSurfaceType = MapSurfaceType::Undefined; if (rootNode->hasChildrenDataBoxes) { - cis->Seek(rootNode->offset); - auto oldLimit = cis->PushLimit(rootNode->length); - - cis->Skip(rootNode->firstDataBoxInnerOffset); - readTreeNodeChildren(reader, section, rootNode, rootSubnodesSurfaceType, &treeNodesWithData, bbox31, queryController, metric); - - ObfReaderUtilities::ensureAllDataWasRead(cis); - cis->PopLimit(oldLimit); + readTreeNodeChildren( + reader, + section, + rootNode, + mapLevel->_p->_nodeCache, + mapLevel->_p->_nodeCacheAccessMutex, + 0, + rootSubnodesSurfaceType, + &treeNodesWithData, + bbox31, + queryController, + metric); } - const auto surfaceTypeToMerge = (rootSubnodesSurfaceType != MapSurfaceType::Undefined) ? rootSubnodesSurfaceType : rootNode->surfaceType; + const auto surfaceTypeToMerge = (rootSubnodesSurfaceType != MapSurfaceType::Undefined) + ? rootSubnodesSurfaceType : rootNode->surfaceType; if (surfaceTypeToMerge != MapSurfaceType::Undefined) { if (bboxOrSectionSurfaceType == MapSurfaceType::Undefined) diff --git a/src/Data/ObfMapSectionReader_P.h b/src/Data/ObfMapSectionReader_P.h index cebd81f0f..d28badc01 100644 --- a/src/Data/ObfMapSectionReader_P.h +++ b/src/Data/ObfMapSectionReader_P.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "restore_internal_warnings.h" #include "OsmAndCore.h" @@ -81,6 +82,9 @@ namespace OsmAnd const ObfReader_P& reader, const std::shared_ptr& section, const std::shared_ptr& treeNode, + QHash>& nodeCache, + QReadWriteLock& nodeCacheAccessMutex, + int treeDepth, MapSurfaceType& outChildrenSurfaceType, QList< std::shared_ptr >* nodesWithData, const AreaI* bbox31,