Skip to content

[SharedCache] Use m_exportInfos as an export list cache #6197

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Feb 10, 2025
Merged

[SharedCache] Use m_exportInfos as an export list cache #6197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8953660
[SharedCache] Use `m_exportInfos` as an export list cache
Nov 25, 2024
e90a08f
[SharedCache] Add parameter to `SharedCache::InitializeHeader` to che…
Nov 26, 2024
28aef35
[SharedCache] Improve the types for `m_exportInfos`
Nov 26, 2024
876a5e2
[SharedCache] Only setup undo actions and bulk modify in `SharedCache…
Nov 26, 2024
0f0801a
[SharedCache] Make `m_exportInfos` map's values a `shared_ptr`
Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 121 additions & 82 deletions view/sharedcache/core/SharedCache.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ int count_trailing_zeros(uint64_t value) {

struct SharedCache::State
{
std::unordered_map<uint64_t, std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>>>
std::unordered_map<uint64_t, std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>>>
exportInfos;
std::unordered_map<uint64_t, std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>>>
symbolInfos;
Expand Down Expand Up @@ -2665,48 +2665,51 @@ void SharedCache::InitializeHeader(

if (header.exportTriePresent && header.linkeditPresent && vm->AddressIsMapped(header.linkeditSegment.vmaddr))
{
auto symbols = SharedCache::ParseExportTrie(vm->MappingAtAddress(header.linkeditSegment.vmaddr).first.fileAccessor->lock(), header);
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> exportMapping;
for (const auto& symbol : symbols)
auto symbols = GetExportListForHeader(header, [&]() {
return vm->MappingAtAddress(header.linkeditSegment.vmaddr).first.fileAccessor->lock();
});
if (symbols)
{
exportMapping.push_back({symbol->GetAddress(), {symbol->GetType(), symbol->GetRawName()}});
if (typeLib)
for (const auto& [symbolAddress, symbol] : *symbols)
{
auto type = m_dscView->ImportTypeLibraryObject(typeLib, {symbol->GetFullName()});

if (type)
if (typeLib)
{
view->DefineAutoSymbolAndVariableOrFunction(view->GetDefaultPlatform(), symbol, type);
}
else
view->DefineAutoSymbol(symbol);
auto type = m_dscView->ImportTypeLibraryObject(typeLib, symbol->GetRawName());

if (view->GetAnalysisFunction(view->GetDefaultPlatform(), symbol->GetAddress()))
{
auto func = view->GetAnalysisFunction(view->GetDefaultPlatform(), symbol->GetAddress());
if (symbol->GetFullName() == "_objc_msgSend")
if (type)
{
func->SetHasVariableArguments(false);
view->DefineAutoSymbolAndVariableOrFunction(view->GetDefaultPlatform(), symbol, type);
}
else if (symbol->GetFullName().find("_objc_retain_x") != std::string::npos || symbol->GetFullName().find("_objc_release_x") != std::string::npos)
else
view->DefineAutoSymbol(symbol);

if (view->GetAnalysisFunction(view->GetDefaultPlatform(), symbolAddress))
{
auto x = symbol->GetFullName().rfind("x");
auto num = symbol->GetFullName().substr(x + 1);
auto func = view->GetAnalysisFunction(view->GetDefaultPlatform(), symbolAddress);
auto name = symbol->GetFullName();
if (name == "_objc_msgSend")
{
func->SetHasVariableArguments(false);
}
else if (name.find("_objc_retain_x") != std::string::npos || name.find("_objc_release_x") != std::string::npos)
{
auto x = name.rfind("x");
auto num = name.substr(x + 1);

std::vector<BinaryNinja::FunctionParameter> callTypeParams;
auto cc = m_dscView->GetDefaultArchitecture()->GetCallingConventionByName("apple-arm64-objc-fast-arc-" + num);
std::vector<BinaryNinja::FunctionParameter> callTypeParams;
auto cc = m_dscView->GetDefaultArchitecture()->GetCallingConventionByName("apple-arm64-objc-fast-arc-" + num);

callTypeParams.push_back({"obj", m_dscView->GetTypeByName({ "id" }), true, BinaryNinja::Variable()});
callTypeParams.push_back({"obj", m_dscView->GetTypeByName({ "id" }), true, BinaryNinja::Variable()});

auto funcType = BinaryNinja::Type::FunctionType(m_dscView->GetTypeByName({ "id" }), cc, callTypeParams);
func->SetUserType(funcType);
auto funcType = BinaryNinja::Type::FunctionType(m_dscView->GetTypeByName({ "id" }), cc, callTypeParams);
func->SetUserType(funcType);
}
}
}
else
view->DefineAutoSymbol(symbol);
}
else
view->DefineAutoSymbol(symbol);
}
MutableState().exportInfos[header.textBase] = std::move(exportMapping);
}
view->EndBulkModifySymbols();

Expand Down Expand Up @@ -2794,6 +2797,7 @@ std::vector<Ref<Symbol>> SharedCache::ParseExportTrie(std::shared_ptr<MMappedFil

try
{
// FIXME we can absolutely predict this size
std::vector<Ref<Symbol>> symbols;
auto [begin, end] = linkeditFile->ReadSpan(header.exportTrie.dataoff, header.exportTrie.datasize);
ReadExportNode(symbols, header, begin, end, begin, header.textBase, "");
Expand All @@ -2806,6 +2810,40 @@ std::vector<Ref<Symbol>> SharedCache::ParseExportTrie(std::shared_ptr<MMappedFil
}
}


std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>> SharedCache::GetExportListForHeader(SharedCacheMachOHeader header, std::function<std::shared_ptr<MMappedFileAccessor>()> provideLinkeditFile, bool* didModifyExportList)
{
if (auto it = m_state->exportInfos.find(header.textBase); it != m_state->exportInfos.end())
{
if (didModifyExportList)
*didModifyExportList = false;
return it->second;
}
else
{
// TODO does this have to be a functor? can't we just pass the accessor? if not, why?
std::shared_ptr<MMappedFileAccessor> linkeditFile = provideLinkeditFile();
if (!linkeditFile)
{
if (didModifyExportList)
*didModifyExportList = false;
return nullptr;
}

auto exportList = SharedCache::ParseExportTrie(linkeditFile, header);
auto exportMapping = std::make_shared<std::unordered_map<uint64_t, Ref<Symbol>>>(exportList.size());
for (const auto& sym : exportList)
{
exportMapping->insert_or_assign(sym->GetAddress(), sym);
}
MutableState().exportInfos.emplace(header.textBase, exportMapping);
if (didModifyExportList)
*didModifyExportList = true;
return m_state->exportInfos[header.textBase];
}
}


std::vector<std::string> SharedCache::GetAvailableImages()
{
std::vector<std::string> installNames;
Expand All @@ -2823,30 +2861,33 @@ std::vector<std::pair<std::string, Ref<Symbol>>> SharedCache::LoadAllSymbolsAndW

std::lock_guard initialLoadBlock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex);

bool doSave = false;
std::vector<std::pair<std::string, Ref<Symbol>>> symbols;
for (const auto& img : State().images)
{
auto header = HeaderForAddress(img.headerLocation);
std::shared_ptr<MMappedFileAccessor> mapping;
try {
mapping = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie %s for %s", header->exportTriePath.c_str(), header->installName.c_str());
auto exportList = GetExportListForHeader(*header, [&]() {
try {
auto mapping = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
return mapping;
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie %s for %s", header->exportTriePath.c_str(), header->installName.c_str());
return std::shared_ptr<MMappedFileAccessor>(nullptr);
}
}, &doSave);
if (!exportList)
continue;
}
auto exportList = SharedCache::ParseExportTrie(mapping, *header);
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> exportMapping;
for (const auto& sym : exportList)
for (const auto& [_, symbol] : *exportList)
{
exportMapping.push_back({sym->GetAddress(), {sym->GetType(), sym->GetRawName()}});
symbols.push_back({img.installName, sym});
symbols.push_back({img.installName, symbol});
}
MutableState().exportInfos[header->textBase] = std::move(exportMapping);
}

SaveToDSCView();
// Only save to DSC view if a header was actually loaded
if (doSave)
SaveToDSCView();

return symbols;
}
Expand Down Expand Up @@ -2926,58 +2967,55 @@ void SharedCache::FindSymbolAtAddrAndApplyToAddr(
auto header = HeaderForAddress(symbolLocation);
if (header)
{
std::shared_ptr<MMappedFileAccessor> mapping;
try {
mapping = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie for %s", header->installName.c_str());
return;
}
auto exportList = SharedCache::ParseExportTrie(mapping, *header);
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> exportMapping;
auto typeLib = TypeLibraryForImage(header->installName);
id = m_dscView->BeginUndoActions();
m_dscView->BeginBulkModifySymbols();
for (const auto& sym : exportList)

auto exportList = GetExportListForHeader(*header, [&]() {
try {
return MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie %s for %s", header->exportTriePath.c_str(), header->installName.c_str());
return std::shared_ptr<MMappedFileAccessor>(nullptr);
}
});

if (exportList)
{
exportMapping.push_back({sym->GetAddress(), {sym->GetType(), sym->GetRawName()}});
if (sym->GetAddress() == symbolLocation)
if (auto it = exportList->find(symbolLocation); it != exportList->end())
{
if (auto func = m_dscView->GetAnalysisFunction(m_dscView->GetDefaultPlatform(), targetLocation))
auto typeLib = TypeLibraryForImage(header->installName);
id = m_dscView->BeginUndoActions();
m_dscView->BeginBulkModifySymbols();

auto func = m_dscView->GetAnalysisFunction(m_dscView->GetDefaultPlatform(), targetLocation);
if (func)
{
m_dscView->DefineUserSymbol(
new Symbol(FunctionSymbol, prefix + sym->GetFullName(), targetLocation));
new Symbol(FunctionSymbol, prefix + it->second->GetFullName(), targetLocation));

if (typeLib)
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {sym->GetFullName()}))
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {it->second->GetFullName()}))
func->SetUserType(type);
}
else
{
m_dscView->DefineUserSymbol(
new Symbol(sym->GetType(), prefix + sym->GetFullName(), targetLocation));
new Symbol(it->second->GetType(), prefix + it->second->GetFullName(), targetLocation));

if (typeLib)
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {sym->GetFullName()}))
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {it->second->GetFullName()}))
m_dscView->DefineUserDataVariable(targetLocation, type);
}
if (triggerReanalysis)
{
auto func = m_dscView->GetAnalysisFunction(m_dscView->GetDefaultPlatform(), targetLocation);
if (func)
func->Reanalyze();
}
break;

m_dscView->EndBulkModifySymbols();
m_dscView->ForgetUndoActions(id);
}
}
{
std::lock_guard lock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex);
MutableState().exportInfos[header->textBase] = std::move(exportMapping);
}
m_dscView->EndBulkModifySymbols();
m_dscView->ForgetUndoActions(id);
}
}

Expand Down Expand Up @@ -3437,12 +3475,12 @@ void SharedCache::Store(SerializationContext& context) const
Serialize(context, "key", pair1.first);
Serialize(context, "value");
context.writer.StartArray();
for (const auto& pair2 : pair1.second)
for (const auto& pair2 : *pair1.second)
{
context.writer.StartObject();
Serialize(context, "key", pair2.first);
Serialize(context, "val1", pair2.second.first);
Serialize(context, "val2", pair2.second.second);
Serialize(context, "val1", pair2.second->GetType());
Serialize(context, "val2", pair2.second->GetRawName());
context.writer.EndObject();
}
context.writer.EndArray();
Expand Down Expand Up @@ -3513,15 +3551,16 @@ void SharedCache::Load(DeserializationContext& context)

for (const auto& obj1 : context.doc["exportInfos"].GetArray())
{
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> innerVec;
std::unordered_map<uint64_t, Ref<Symbol>> innerVec;
for (const auto& obj2 : obj1["value"].GetArray())
{
std::pair<BNSymbolType, std::string> innerPair = {
(BNSymbolType)obj2["val1"].GetUint64(), obj2["val2"].GetString()};
innerVec.push_back({obj2["key"].GetUint64(), innerPair});
std::string raw = obj2["val2"].GetString();
uint64_t addr = obj2["key"].GetUint64();
innerVec[addr] = new Symbol(BNCreateSymbol((BNSymbolType)obj2["val1"].GetUint64(), raw.c_str(),
raw.c_str(), raw.c_str(), addr, NoBinding, nullptr, 0));
}

MutableState().exportInfos[obj1["key"].GetUint64()] = std::move(innerVec);
MutableState().exportInfos[obj1["key"].GetUint64()] = std::make_shared<std::unordered_map<uint64_t, Ref<Symbol>>>(innerVec);
}

for (auto& symbolInfo : context.doc["symbolInfos"].GetArray())
Expand Down
4 changes: 4 additions & 0 deletions view/sharedcache/core/SharedCache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,7 @@ namespace SharedCacheCore {

struct ViewSpecificState;


private:
Ref<Logger> m_logger;
/* VIEW STATE BEGIN -- SERIALIZE ALL OF THIS AND STORE IT IN RAW VIEW */
Expand Down Expand Up @@ -641,6 +642,9 @@ namespace SharedCacheCore {
const uint8_t *end, const uint8_t* current, uint64_t textBase, const std::string& currentText);
std::vector<Ref<Symbol>> ParseExportTrie(
std::shared_ptr<MMappedFileAccessor> linkeditFile, const SharedCacheMachOHeader& header);
std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>> GetExportListForHeader(SharedCacheMachOHeader header,
std::function<std::shared_ptr<MMappedFileAccessor>()> provideLinkeditFile, bool* didModifyExportList = nullptr);


Ref<TypeLibrary> TypeLibraryForImage(const std::string& installName);

Expand Down