Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SharedCache] Cache type libraries #6196

Merged
merged 2 commits into from
Jan 10, 2025
+119 −96
Merged

[SharedCache] Cache type libraries #6196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1dc9179
[SharedCache] Split view-specific state into a separate struct
bdash Nov 25, 2024
1f0be23
[SharedCache] Cache type libraries in the view-specific state
bdash Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
209 changes: 113 additions & 96 deletions view/sharedcache/core/SharedCache.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
#include "SharedCache.h"
#include "ObjC.h"
#include <filesystem>
#include <mutex>
#include <unordered_map>
#include <utility>
#include <fcntl.h>
#include <memory>
Expand Down Expand Up @@ -78,19 +80,53 @@ struct SharedCache::State
DSCViewState viewState = DSCViewStateUnloaded;
};

static std::recursive_mutex viewStateMutex;
static std::unordered_map<uint64_t, std::shared_ptr<struct SharedCache::State>> viewStateCache;
struct SharedCache::ViewSpecificState {
std::mutex typeLibraryMutex;
std::unordered_map<std::string, Ref<TypeLibrary>> typeLibraries;

std::mutex progressMutex;
std::unordered_map<uint64_t, BNDSCViewLoadProgress> progressMap;

struct ViewSpecificMutexes {
std::mutex viewOperationsThatInfluenceMetadataMutex;
std::mutex typeLibraryLookupAndApplicationMutex;

std::atomic<BNDSCViewLoadProgress> progress;

std::mutex stateMutex;
std::shared_ptr<struct SharedCache::State> cachedState;
};

static std::unordered_map<uint64_t, ViewSpecificMutexes> viewSpecificMutexes;

std::shared_ptr<SharedCache::ViewSpecificState> ViewSpecificStateForId(uint64_t viewIdentifier, bool insertIfNeeded = true) {
static std::mutex viewSpecificStateMutex;
static std::unordered_map<uint64_t, std::weak_ptr<SharedCache::ViewSpecificState>> viewSpecificState;

std::lock_guard lock(viewSpecificStateMutex);

if (auto it = viewSpecificState.find(viewIdentifier); it != viewSpecificState.end()) {
if (auto statePtr = it->second.lock()) {
return statePtr;
}
}

if (!insertIfNeeded) {
return nullptr;
}

auto statePtr = std::make_shared<SharedCache::ViewSpecificState>();
viewSpecificState[viewIdentifier] = statePtr;

// Prune entries for any views that are no longer in use.
for (auto it = viewSpecificState.begin(); it != viewSpecificState.end(); ) {
if (it->second.expired()) {
it = viewSpecificState.erase(it);
} else {
++it;
}
}

return statePtr;
}

std::shared_ptr<SharedCache::ViewSpecificState> ViewSpecificStateForView(Ref<BinaryNinja::BinaryView> view) {
return ViewSpecificStateForId(view->GetFile()->GetSessionId());
}

std::string base_name(std::string const& path)
{
Expand Down Expand Up @@ -220,9 +256,7 @@ void SharedCache::PerformInitialLoad()
auto path = m_dscView->GetFile()->GetOriginalFilename();
auto baseFile = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), path)->lock();

progressMutex.lock();
progressMap[m_dscView->GetFile()->GetSessionId()] = LoadProgressLoadingCaches;
progressMutex.unlock();
m_viewSpecificState->progress = LoadProgressLoadingCaches;

WillMutateState();

Expand Down Expand Up @@ -737,9 +771,8 @@ void SharedCache::PerformInitialLoad()
}
}
baseFile.reset();
progressMutex.lock();
progressMap[m_dscView->GetFile()->GetSessionId()] = LoadProgressLoadingImages;
progressMutex.unlock();

m_viewSpecificState->progress = LoadProgressLoadingImages;

// We have set up enough metadata to map VM now.

Expand Down Expand Up @@ -952,9 +985,7 @@ void SharedCache::PerformInitialLoad()

m_logger->LogDebug("Finished initial load of Shared Cache");

progressMutex.lock();
progressMap[m_dscView->GetFile()->GetSessionId()] = LoadProgressFinished;
progressMutex.unlock();
m_viewSpecificState->progress = LoadProgressFinished;
}

std::shared_ptr<VM> SharedCache::GetVMMap(bool mapPages)
Expand Down Expand Up @@ -983,10 +1014,10 @@ void SharedCache::DeserializeFromRawView()
{
if (m_dscView->QueryMetadata(SharedCacheMetadataTag))
{
std::unique_lock<std::recursive_mutex> viewStateCacheLock(viewStateMutex);
if (auto it = viewStateCache.find(m_dscView->GetFile()->GetSessionId()); it != viewStateCache.end())
std::lock_guard lock(m_viewSpecificState->stateMutex);
if (m_viewSpecificState->cachedState)
{
m_state = it->second;
m_state = m_viewSpecificState->cachedState;
m_stateIsShared = true;
m_metadataValid = true;
}
Expand Down Expand Up @@ -1367,7 +1398,7 @@ void SharedCache::ParseAndApplySlideInfoForFile(std::shared_ptr<MMappedFileAcces
}


SharedCache::SharedCache(BinaryNinja::Ref<BinaryNinja::BinaryView> dscView) : m_dscView(dscView)
SharedCache::SharedCache(BinaryNinja::Ref<BinaryNinja::BinaryView> dscView) : m_dscView(dscView), m_viewSpecificState(ViewSpecificStateForView(dscView))
{
if (dscView->GetTypeName() != VIEW_NAME)
{
Expand All @@ -1381,46 +1412,43 @@ SharedCache::SharedCache(BinaryNinja::Ref<BinaryNinja::BinaryView> dscView) : m_
DeserializeFromRawView();
if (!m_metadataValid)
return;
if (State().viewState == DSCViewStateUnloaded)

if (State().viewState != DSCViewStateUnloaded) {
m_viewSpecificState->progress = LoadProgressFinished;
return;
}

std::unique_lock lock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex);
try {
PerformInitialLoad();
}
catch (...)
{
std::unique_lock<std::mutex> lock(viewSpecificMutexes[m_dscView->GetFile()->GetSessionId()].viewOperationsThatInfluenceMetadataMutex);
try {
PerformInitialLoad();
}
catch (...)
{
m_logger->LogError("Failed to perform initial load of Shared Cache");
}
m_logger->LogError("Failed to perform initial load of Shared Cache");
}

auto settings = m_dscView->GetLoadSettings(VIEW_NAME);
bool autoLoadLibsystem = true;
if (settings && settings->Contains("loader.dsc.autoLoadLibSystem"))
{
autoLoadLibsystem = settings->Get<bool>("loader.dsc.autoLoadLibSystem", m_dscView);
}
if (autoLoadLibsystem)
auto settings = m_dscView->GetLoadSettings(VIEW_NAME);
bool autoLoadLibsystem = true;
if (settings && settings->Contains("loader.dsc.autoLoadLibSystem"))
{
autoLoadLibsystem = settings->Get<bool>("loader.dsc.autoLoadLibSystem", m_dscView);
}
if (autoLoadLibsystem)
{
for (const auto& [_, header] : State().headers)
{
for (const auto& [_, header] : State().headers)
if (header.installName.find("libsystem_c.dylib") != std::string::npos)
{
if (header.installName.find("libsystem_c.dylib") != std::string::npos)
{
lock.unlock();
m_logger->LogInfo("Loading core libsystem_c.dylib library");
LoadImageWithInstallName(header.installName, false);
lock.lock();
break;
}
lock.unlock();
m_logger->LogInfo("Loading core libsystem_c.dylib library");
LoadImageWithInstallName(header.installName, false);
break;
}
}
MutableState().viewState = DSCViewStateLoaded;
SaveToDSCView();
}
else
{
progressMutex.lock();
progressMap[m_dscView->GetFile()->GetSessionId()] = LoadProgressFinished;
progressMutex.unlock();
}

MutableState().viewState = DSCViewStateLoaded;
SaveToDSCView();
}

SharedCache::~SharedCache() {
Expand Down Expand Up @@ -1536,7 +1564,7 @@ bool SharedCache::LoadImageContainingAddress(uint64_t address, bool skipObjC)

bool SharedCache::LoadSectionAtAddress(uint64_t address)
{
std::unique_lock<std::mutex> lock(viewSpecificMutexes[m_dscView->GetFile()->GetSessionId()].viewOperationsThatInfluenceMetadataMutex);
std::unique_lock lock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex);
DeserializeFromRawView();
WillMutateState();

Expand Down Expand Up @@ -1806,7 +1834,7 @@ bool SharedCache::LoadImageWithInstallName(std::string installName, bool skipObj
{
auto settings = m_dscView->GetLoadSettings(VIEW_NAME);

std::unique_lock<std::mutex> lock(viewSpecificMutexes[m_dscView->GetFile()->GetSessionId()].viewOperationsThatInfluenceMetadataMutex);
std::unique_lock lock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex);

DeserializeFromRawView();
WillMutateState();
Expand Down Expand Up @@ -1880,21 +1908,7 @@ bool SharedCache::LoadImageWithInstallName(std::string installName, bool skipObj
return false;
}

std::unique_lock<std::mutex> typelibLock(viewSpecificMutexes[m_dscView->GetFile()->GetSessionId()].typeLibraryLookupAndApplicationMutex);
auto typeLib = m_dscView->GetTypeLibrary(header.installName);

if (!typeLib)
{
auto typeLibs = m_dscView->GetDefaultPlatform()->GetTypeLibrariesByName(header.installName);
if (!typeLibs.empty())
{
typeLib = typeLibs[0];
m_dscView->AddTypeLibrary(typeLib);
m_logger->LogInfo("shared-cache: adding type library for '%s': %s (%s)",
targetImage->installName.c_str(), typeLib->GetName().c_str(), typeLib->GetGuid().c_str());
}
}
typelibLock.unlock();
auto typeLib = TypeLibraryForImage(header.installName);

SaveToDSCView();

Expand Down Expand Up @@ -2875,7 +2889,7 @@ std::vector<std::pair<std::string, Ref<Symbol>>> SharedCache::LoadAllSymbolsAndW
{
WillMutateState();

std::unique_lock<std::mutex> initialLoadBlock(viewSpecificMutexes[m_dscView->GetFile()->GetSessionId()].viewOperationsThatInfluenceMetadataMutex);
std::lock_guard initialLoadBlock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex);

std::vector<std::pair<std::string, Ref<Symbol>>> symbols;
for (const auto& img : State().images)
Expand Down Expand Up @@ -2929,6 +2943,24 @@ std::string SharedCache::SerializedImageHeaderForName(std::string name)
return "";
}

Ref<TypeLibrary> SharedCache::TypeLibraryForImage(const std::string& installName) {
std::lock_guard lock(m_viewSpecificState->typeLibraryMutex);
if (auto it = m_viewSpecificState->typeLibraries.find(installName); it != m_viewSpecificState->typeLibraries.end()) {
return it->second;
}

auto typeLib = m_dscView->GetTypeLibrary(installName);
if (!typeLib) {
auto typeLibs = m_dscView->GetDefaultPlatform()->GetTypeLibrariesByName(installName);
if (!typeLibs.empty()) {
typeLib = typeLibs[0];
m_dscView->AddTypeLibrary(typeLib);
}
}

m_viewSpecificState->typeLibraries[installName] = typeLib;
return typeLib;
}

void SharedCache::FindSymbolAtAddrAndApplyToAddr(
uint64_t symbolLocation, uint64_t targetLocation, bool triggerReanalysis)
Expand Down Expand Up @@ -2973,18 +3005,7 @@ void SharedCache::FindSymbolAtAddrAndApplyToAddr(
}
auto exportList = SharedCache::ParseExportTrie(mapping, *header);
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> exportMapping;
std::unique_lock<std::mutex> lock(viewSpecificMutexes[m_dscView->GetFile()->GetSessionId()].typeLibraryLookupAndApplicationMutex);
auto typeLib = m_dscView->GetTypeLibrary(header->installName);
if (!typeLib)
{
auto typeLibs = m_dscView->GetDefaultPlatform()->GetTypeLibrariesByName(header->installName);
if (!typeLibs.empty())
{
typeLib = typeLibs[0];
m_dscView->AddTypeLibrary(typeLib);
}
}
lock.unlock();
auto typeLib = TypeLibraryForImage(header->installName);
id = m_dscView->BeginUndoActions();
m_dscView->BeginBulkModifySymbols();
for (const auto& sym : exportList)
Expand Down Expand Up @@ -3020,7 +3041,7 @@ void SharedCache::FindSymbolAtAddrAndApplyToAddr(
}
}
{
std::unique_lock<std::mutex> _lock(viewSpecificMutexes[m_dscView->GetFile()->GetSessionId()].viewOperationsThatInfluenceMetadataMutex);
std::lock_guard lock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex);
MutableState().exportInfos[header->textBase] = std::move(exportMapping);
}
m_dscView->EndBulkModifySymbols();
Expand All @@ -3044,8 +3065,8 @@ bool SharedCache::SaveToDSCView()
m_state = cachedState;
m_stateIsShared = true;

std::unique_lock<std::recursive_mutex> viewStateCacheLock(viewStateMutex);
viewStateCache[m_dscView->GetFile()->GetSessionId()] = std::move(cachedState);
std::lock_guard lock(m_viewSpecificState->stateMutex);
m_viewSpecificState->cachedState = std::move(cachedState);

m_metadataValid = true;

Expand All @@ -3055,7 +3076,7 @@ bool SharedCache::SaveToDSCView()
}
std::vector<MemoryRegion> SharedCache::GetMappedRegions() const
{
std::unique_lock<std::mutex> lock(viewSpecificMutexes[m_dscView->GetFile()->GetSessionId()].viewOperationsThatInfluenceMetadataMutex);
std::lock_guard lock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex);
return State().regionsMappedIntoMemory;
}

Expand Down Expand Up @@ -3392,15 +3413,11 @@ extern "C"

BNDSCViewLoadProgress BNDSCViewGetLoadProgress(uint64_t sessionID)
{
progressMutex.lock();
if (progressMap.find(sessionID) == progressMap.end())
{
progressMutex.unlock();
return LoadProgressNotStarted;
if (auto viewSpecificState = ViewSpecificStateForId(sessionID, false)) {
return viewSpecificState->progress;
}
auto progress = progressMap[sessionID];
progressMutex.unlock();
return progress;

return LoadProgressNotStarted;
}

uint64_t BNDSCViewFastGetBackingCacheCount(BNBinaryView* data)
Expand Down
6 changes: 6 additions & 0 deletions view/sharedcache/core/SharedCache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,8 @@ namespace SharedCacheCore {

struct State;

struct ViewSpecificState;

private:
Ref<Logger> m_logger;
/* VIEW STATE BEGIN -- SERIALIZE ALL OF THIS AND STORE IT IN RAW VIEW */
Expand All @@ -552,6 +554,7 @@ namespace SharedCacheCore {
bool m_metadataValid = false;

/* VIEWSTATE END -- NOTHING PAST THIS IS SERIALIZED */
std::shared_ptr<ViewSpecificState> m_viewSpecificState;

/* API VIEW START */
BinaryNinja::Ref<BinaryNinja::BinaryView> m_dscView;
Expand Down Expand Up @@ -600,6 +603,7 @@ namespace SharedCacheCore {
explicit SharedCache(BinaryNinja::Ref<BinaryNinja::BinaryView> rawView);
virtual ~SharedCache();

private:
std::optional<SharedCacheMachOHeader> LoadHeaderForAddress(
std::shared_ptr<VM> vm, uint64_t address, std::string installName);
void InitializeHeader(
Expand All @@ -609,6 +613,8 @@ namespace SharedCacheCore {
std::vector<Ref<Symbol>> ParseExportTrie(
std::shared_ptr<MMappedFileAccessor> linkeditFile, SharedCacheMachOHeader header);

Ref<TypeLibrary> TypeLibraryForImage(const std::string& installName);

const State& State() const { return *m_state; }
struct State& MutableState() { AssertMutable(); return *m_state; }

Expand Down