Skip to content

Commit

Permalink
Use cache when reading OBF sections (#797)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Andrewschenko
Andrewschenko authored and vshcherb committed Nov 4, 2024
1 parent 28ac27f commit 7a043bd
Show file tree
Hide file tree
Showing 3 changed files with 176 additions and 50 deletions.
5 changes: 5 additions & 0 deletions src/Data/ObfMapSectionInfo_P.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "QtExtensions.h"
#include "ignore_warnings_on_external_includes.h"
#include <QMutex>
#include <QReadWriteLock>
#include <QSet>
#include <QHash>
#include <QMap>
Expand Down Expand Up @@ -44,6 +45,8 @@ namespace OsmAnd

bool hasChildrenDataBoxes;
uint32_t firstDataBoxInnerOffset;
mutable std::shared_ptr<const QList<ObfMapSectionDataBlockId>> subNodeIds;
mutable QMutex _subNodeIdsMutex;

friend class OsmAnd::ObfMapSectionReader_P;
};
Expand All @@ -57,6 +60,8 @@ namespace OsmAnd
mutable std::shared_ptr< const QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> > > _rootNodes;
mutable QAtomicInt _rootNodesLoaded;
mutable QMutex _rootNodesLoadMutex;
mutable QHash<ObfMapSectionDataBlockId, std::shared_ptr<const ObfMapSectionLevelTreeNode>> _nodeCache;
mutable QReadWriteLock _nodeCacheAccessMutex;
public:
virtual ~ObfMapSectionLevel_P();

Expand Down
217 changes: 167 additions & 50 deletions src/Data/ObfMapSectionReader_P.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
#include "Logging.h"
#include "Utilities.h"

const int MAX_TREE_DEPTH_IN_CACHE = 7;

using google::protobuf::internal::WireFormatLite;

OsmAnd::ObfMapSectionReader_P::ObfMapSectionReader_P()
Expand Down Expand Up @@ -396,39 +398,139 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
const ObfReader_P& reader,
const std::shared_ptr<const ObfMapSectionInfo>& section,
const std::shared_ptr<const ObfMapSectionLevelTreeNode>& treeNode,
QHash<ObfMapSectionDataBlockId, std::shared_ptr<const ObfMapSectionLevelTreeNode>>& nodeCache,
QReadWriteLock& nodeCacheAccessMutex,
int treeDepth,
MapSurfaceType& outChildrenSurfaceType,
QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> >* nodesWithData,
const AreaI* bbox31,
const std::shared_ptr<const IQueryController>& queryController,
ObfMapSectionReader_Metrics::Metric_loadMapObjects* const metric)
{
const auto cis = reader.getCodedInputStream().get();
QList<std::shared_ptr<const ObfMapSectionLevelTreeNode>> childNodes;

outChildrenSurfaceType = MapSurfaceType::Undefined;
for (;;)
bool fromCache = false;
if (!treeNode->subNodeIds)
{
const auto tag = cis->ReadTag();
switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag))
QMutexLocker scopedLocker(&treeNode->_subNodeIdsMutex);

if (!treeNode->subNodeIds)
{
case 0:
if (!ObfReaderUtilities::reachedDataEnd(cis))
return;
// Read all child nodes and make a lookup table
const std::shared_ptr<QList<ObfMapSectionDataBlockId>> subNodeIds(new QList<ObfMapSectionDataBlockId>);
const auto cis = reader.getCodedInputStream().get();
cis->Seek(treeNode->offset);
const auto oldLimit = cis->PushLimit(treeNode->length);
cis->Skip(treeNode->firstDataBoxInnerOffset);
outChildrenSurfaceType = MapSurfaceType::Undefined;
bool keepReading = true;
while (keepReading)
{
const auto tag = cis->ReadTag();
switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag))
{
case 0:
keepReading = false;
if (!ObfReaderUtilities::reachedDataEnd(cis))
break;

return;
case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber:
break;
case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber:
{
const auto length = ObfReaderUtilities::readBigEndianInt(cis);
const auto offset = cis->CurrentPosition();
const auto prevLimit = cis->PushLimit(length);

std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(
new ObfMapSectionLevelTreeNode(treeNode->level));
childNode->surfaceType = treeNode->surfaceType;
childNode->offset = offset;
childNode->length = length;
readTreeNode(reader, section, treeNode->area31, childNode);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(prevLimit);

// Update metric
if (metric)
metric->visitedNodes++;

ObfMapSectionDataBlockId subNodeId;
subNodeId.sectionRuntimeGeneratedId = section->runtimeGeneratedId;
subNodeId.offset = childNode->offset;

// Register node in lookup table
subNodeIds->append(subNodeId);

// Store node in cache
if (treeDepth < MAX_TREE_DEPTH_IN_CACHE)
{
QWriteLocker scopedLocker(&nodeCacheAccessMutex);

nodeCache.insert(subNodeId, childNode);
}

if (bbox31)
{
const auto shouldSkip =
!bbox31->contains(childNode->area31) &&
!childNode->area31.contains(*bbox31) &&
!bbox31->intersects(childNode->area31);
if (shouldSkip)
break;
}

// Update metric
if (metric)
metric->acceptedNodes++;

if (nodesWithData && childNode->dataOffset > 0)
nodesWithData->push_back(childNode);

if (childNode->hasChildrenDataBoxes)
childNodes.append(childNode);

if (childNode->surfaceType != MapSurfaceType::Undefined)
{
if (outChildrenSurfaceType == MapSurfaceType::Undefined)
outChildrenSurfaceType = childNode->surfaceType;
else if (outChildrenSurfaceType != childNode->surfaceType)
outChildrenSurfaceType = MapSurfaceType::Mixed;
}

break;
}
default:
ObfReaderUtilities::skipUnknownField(cis, tag);
break;
}
}
ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);

// Store lookup table
if (treeDepth < MAX_TREE_DEPTH_IN_CACHE)
{
const auto length = ObfReaderUtilities::readBigEndianInt(cis);
const auto offset = cis->CurrentPosition();
const auto oldLimit = cis->PushLimit(length);
treeNode->subNodeIds = subNodeIds;
}
}
else
fromCache = true;
}
else
fromCache = true;

const std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(new ObfMapSectionLevelTreeNode(treeNode->level));
childNode->surfaceType = treeNode->surfaceType;
childNode->offset = offset;
childNode->length = length;
readTreeNode(reader, section, treeNode->area31, childNode);
if (fromCache)
{
// Use lookup table to access child nodes stored in cache
QReadLocker scopedLocker(&nodeCacheAccessMutex);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
for (const auto& subNodeId : constOf(*treeNode->subNodeIds))
{
const auto& citChildNode = nodeCache.constFind(subNodeId);
if (citChildNode != nodeCache.cend())
{
const auto& childNode = citChildNode.value();

// Update metric
if (metric)
Expand All @@ -441,7 +543,7 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
!childNode->area31.contains(*bbox31) &&
!bbox31->intersects(childNode->area31);
if (shouldSkip)
break;
continue;
}

// Update metric
Expand All @@ -451,33 +553,43 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
if (nodesWithData && childNode->dataOffset > 0)
nodesWithData->push_back(childNode);

auto subchildrenSurfaceType = MapSurfaceType::Undefined;
if (childNode->hasChildrenDataBoxes)
{
cis->Seek(childNode->offset);
const auto oldLimit = cis->PushLimit(childNode->length);

cis->Skip(childNode->firstDataBoxInnerOffset);
readTreeNodeChildren(reader, section, childNode, subchildrenSurfaceType, nodesWithData, bbox31, queryController, metric);
childNodes.append(childNode);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
}

const auto surfaceTypeToMerge = (subchildrenSurfaceType != MapSurfaceType::Undefined) ? subchildrenSurfaceType : childNode->surfaceType;
if (surfaceTypeToMerge != MapSurfaceType::Undefined)
if (childNode->surfaceType != MapSurfaceType::Undefined)
{
if (outChildrenSurfaceType == MapSurfaceType::Undefined)
outChildrenSurfaceType = surfaceTypeToMerge;
else if (outChildrenSurfaceType != surfaceTypeToMerge)
outChildrenSurfaceType = childNode->surfaceType;
else if (outChildrenSurfaceType != childNode->surfaceType)
outChildrenSurfaceType = MapSurfaceType::Mixed;
}

break;
}
default:
ObfReaderUtilities::skipUnknownField(cis, tag);
break;
}
}

for (const auto& childNode : constOf(childNodes))
{
auto subchildrenSurfaceType = MapSurfaceType::Undefined;
readTreeNodeChildren(
reader,
section,
childNode,
nodeCache,
nodeCacheAccessMutex,
treeDepth + 1,
subchildrenSurfaceType,
nodesWithData,
bbox31,
queryController,
metric);
const auto surfaceTypeToMerge =
(subchildrenSurfaceType != MapSurfaceType::Undefined) ? subchildrenSurfaceType : childNode->surfaceType;
if (surfaceTypeToMerge != MapSurfaceType::Undefined)
{
if (outChildrenSurfaceType == MapSurfaceType::Undefined)
outChildrenSurfaceType = surfaceTypeToMerge;
else if (outChildrenSurfaceType != surfaceTypeToMerge)
outChildrenSurfaceType = MapSurfaceType::Mixed;
}
}
}
Expand Down Expand Up @@ -1102,17 +1214,22 @@ void OsmAnd::ObfMapSectionReader_P::loadMapObjects(
auto rootSubnodesSurfaceType = MapSurfaceType::Undefined;
if (rootNode->hasChildrenDataBoxes)
{
cis->Seek(rootNode->offset);
auto oldLimit = cis->PushLimit(rootNode->length);

cis->Skip(rootNode->firstDataBoxInnerOffset);
readTreeNodeChildren(reader, section, rootNode, rootSubnodesSurfaceType, &treeNodesWithData, bbox31, queryController, metric);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
readTreeNodeChildren(
reader,
section,
rootNode,
mapLevel->_p->_nodeCache,
mapLevel->_p->_nodeCacheAccessMutex,
0,
rootSubnodesSurfaceType,
&treeNodesWithData,
bbox31,
queryController,
metric);
}

const auto surfaceTypeToMerge = (rootSubnodesSurfaceType != MapSurfaceType::Undefined) ? rootSubnodesSurfaceType : rootNode->surfaceType;
const auto surfaceTypeToMerge = (rootSubnodesSurfaceType != MapSurfaceType::Undefined)
? rootSubnodesSurfaceType : rootNode->surfaceType;
if (surfaceTypeToMerge != MapSurfaceType::Undefined)
{
if (bboxOrSectionSurfaceType == MapSurfaceType::Undefined)
Expand Down
4 changes: 4 additions & 0 deletions src/Data/ObfMapSectionReader_P.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <QHash>
#include <QMap>
#include <QSet>
#include <QReadWriteLock>
#include "restore_internal_warnings.h"

#include "OsmAndCore.h"
Expand Down Expand Up @@ -81,6 +82,9 @@ namespace OsmAnd
const ObfReader_P& reader,
const std::shared_ptr<const ObfMapSectionInfo>& section,
const std::shared_ptr<const ObfMapSectionLevelTreeNode>& treeNode,
QHash<ObfMapSectionDataBlockId, std::shared_ptr<const ObfMapSectionLevelTreeNode>>& nodeCache,
QReadWriteLock& nodeCacheAccessMutex,
int treeDepth,
MapSurfaceType& outChildrenSurfaceType,
QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> >* nodesWithData,
const AreaI* bbox31,
Expand Down

0 comments on commit 7a043bd

Please sign in to comment.