Skip to content

Commit

Permalink
Fix fetching CCDB metadata, extend ccdbRunDependent behaviour
Browse files Browse the repository at this point in the history 
ccdbParamSpec run-dependence flag changed from bool to int. The new convension is:
ccdbParamSpec(path) : not run-depndent
ccdbParamSpec(path, 1) : run-dependent object with usual timestamp and runNumber requested via metadata
ccdbParamSpec(path, 2) : run-dependent object with runNumber used instead of timestamp.
So, the entry like RCT/Info/RunInformation should be requested by passing 2.
  • Loading branch information
@shahor02
shahor02 committed Oct 11, 2024
1 parent 1ff58c4 commit 3fba187
Show file tree
Hide file tree
Showing 4 changed files with 24 additions and 17 deletions.
23 changes: 15 additions & 8 deletions Framework/CCDBSupport/src/CCDBHelpers.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
auto sid = _o2_signpost_id_t{(int64_t)timingInfo.timeslice};
O2_SIGNPOST_START(ccdb, sid, "populateCacheWith", "Starting to populate cache with CCDB objects");
for (auto& route : helper->routes) {
int64_t timestampToUse = timestamp;
O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Fetching object for route %{public}s", DataSpecUtils::describe(route.matcher).data());
objCnt++;
auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher);
Expand All @@ -212,8 +213,14 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
for (auto& meta : route.matcher.metadata) {
if (meta.name == "ccdb-path") {
path = meta.defaultValue.get<std::string>();
} else if (meta.name == "ccdb-run-dependent" && meta.defaultValue.get<bool>() == true) {
metadata["runNumber"] = dtc.runNumber;
} else if (meta.name == "ccdb-run-dependent" && meta.defaultValue.get<int>() > 0) {
if (meta.defaultValue.get<int>() == 1) {
metadata["runNumber"] = dtc.runNumber;
} else if (meta.defaultValue.get<int>() == 2) {
timestampToUse = std::stoi(dtc.runNumber);
} else {
LOGP(fatal, "Undefined run-dependent option {} for spec {}/{}/{}", meta.defaultValue.get<int>(), concrete.origin.as<std::string>(), concrete.description.as<std::string>(), int(concrete.subSpec));
}
} else if (isPrefix(ccdbMetadataPrefix, meta.name)) {
std::string key = meta.name.substr(ccdbMetadataPrefix.size());
auto value = meta.defaultValue.get<std::string>();
Expand All @@ -232,7 +239,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
uint64_t cachePopulatedAt = url2uuid->second.cachePopulatedAt;
// If timestamp is before the time the element was cached or after the claimed validity, we need to check validity, again
// when online.
bool cacheExpired = (validUntil <= timestamp) || (timestamp <= cachePopulatedAt);
bool cacheExpired = (validUntil <= timestampToUse) || (timestamp < cachePopulatedAt);
checkValidity = (std::abs(int(timingInfo.tfCounter - url2uuid->second.lastCheckedTF)) >= chRate) && (isOnline || cacheExpired);
} else {
checkValidity = true; // never skip check if the cache is empty
Expand All @@ -242,10 +249,10 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,

const auto& api = helper->getAPI(path);
if (checkValidity && (!api.isSnapshotMode() || etag.empty())) { // in the snapshot mode the object needs to be fetched only once
LOGP(detail, "Loading {} for timestamp {}", path, timestamp);
api.loadFileToMemory(v, path, metadata, timestamp, &headers, etag, helper->createdNotAfter, helper->createdNotBefore);
LOGP(detail, "Loading {} for timestamp {}", path, timestampToUse);
api.loadFileToMemory(v, path, metadata, timestampToUse, &headers, etag, helper->createdNotAfter, helper->createdNotBefore);
if ((headers.count("Error") != 0) || (etag.empty() && v.empty())) {
LOGP(fatal, "Unable to find object {}/{}", path, timestamp);
LOGP(fatal, "Unable to find object {}/{}", path, timestampToUse);
// FIXME: I should send a dummy message.
continue;
}
Expand All @@ -256,7 +263,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
helper->mapURL2UUID[path].lastCheckedTF = timingInfo.tfCounter;
if (etag.empty()) {
helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
helper->mapURL2UUID[path].cachePopulatedAt = timestamp;
helper->mapURL2UUID[path].cachePopulatedAt = timestampToUse;
helper->mapURL2UUID[path].cacheMiss++;
helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
Expand All @@ -269,7 +276,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
if (v.size()) { // but should be overridden by fresh object
// somewhere here pruneFromCache should be called
helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
helper->mapURL2UUID[path].cachePopulatedAt = timestamp;
helper->mapURL2UUID[path].cachePopulatedAt = timestampToUse;
helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
helper->mapURL2UUID[path].cacheMiss++;
helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
Expand Down
4 changes: 2 additions & 2 deletions Framework/Core/include/Framework/CCDBParamSpec.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ struct CCDBMetadata {
};

ConfigParamSpec ccdbPathSpec(std::string const& path);
ConfigParamSpec ccdbRunDependent(bool defaultValue = true);
ConfigParamSpec ccdbRunDependent(int defaultValue = 1); // <1: not run-dependent, 1: run-dependent object with usual timestamp, 2: run-dependent object with runNumber used instead of timestamp

std::vector<ConfigParamSpec> ccdbParamSpec(std::string const& path, bool runDependent, std::vector<CCDBMetadata> metadata = {}, int qrate = 0);
std::vector<ConfigParamSpec> ccdbParamSpec(std::string const& path, int runDependent, std::vector<CCDBMetadata> metadata = {}, int qrate = 0);
/// Helper to create an InputSpec which will read from a CCDB
/// Notice that those input specs have some convetions for their metadata:
///
Expand Down
6 changes: 3 additions & 3 deletions Framework/Core/include/Framework/InputRecord.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -493,7 +493,7 @@ class InputRecord
auto cacheEntry = cache.matcherToMetadataId.find(path);
if (cacheEntry == cache.matcherToMetadataId.end()) {
cache.matcherToMetadataId.insert(std::make_pair(path, id));
cache.idToMetadata[id] = extractCCDBHeaders(ref);
cache.idToMetadata[id] = DataRefUtils::extractCCDBHeaders(ref);
LOGP(info, "Caching CCDB metadata {}: {}", id.value, path);
return cache.idToMetadata[id];
}
Expand All @@ -506,9 +506,9 @@ class InputRecord
// The id in the cache is different. Let's destroy the old cached entry
// and create a new one.
LOGP(info, "Replacing cached entry {} with {} for {}", oldId.value, id.value, path);
cache.idToObject[id] = extracCCDBMetadata(ref);
cache.idToMetadata[id] = DataRefUtils::extractCCDBHeaders(ref);
oldId.value = id.value;
return cache.idToObject[id];
return cache.idToMetadata[id];
}

/// Helper method to be used to check if a given part of the InputRecord is present.
Expand Down
8 changes: 4 additions & 4 deletions Framework/Core/src/CCDBParamSpec.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ ConfigParamSpec ccdbPathSpec(std::string const& path)
return ConfigParamSpec{"ccdb-path", VariantType::String, path, {fmt::format("Path in CCDB ({})", path)}, ConfigParamKind::kGeneric};
}

ConfigParamSpec ccdbRunDependent(bool defaultValue)
ConfigParamSpec ccdbRunDependent(int defaultValue)
{
return ConfigParamSpec{"ccdb-run-dependent", VariantType::Bool, defaultValue, {"Give object for specific run number"}, ConfigParamKind::kGeneric};
return ConfigParamSpec{"ccdb-run-dependent", VariantType::Int, defaultValue, {"Give object for specific run number"}, ConfigParamKind::kGeneric};
}

ConfigParamSpec ccdbQueryRateSpec(int r)
Expand All @@ -45,11 +45,11 @@ std::vector<ConfigParamSpec> ccdbParamSpec(std::string const& path, std::vector<
return ccdbParamSpec(path, false, metadata, qrate);
}

std::vector<ConfigParamSpec> ccdbParamSpec(std::string const& path, bool runDependent, std::vector<CCDBMetadata> metadata, int qrate)
std::vector<ConfigParamSpec> ccdbParamSpec(std::string const& path, int runDependent, std::vector<CCDBMetadata> metadata, int qrate)
{
// Add here CCDB objecs which should be considered run dependent
std::vector<ConfigParamSpec> result{ccdbPathSpec(path)};
if (runDependent) {
if (runDependent > 0) {
result.push_back(ccdbRunDependent(runDependent));
}
if (qrate != 0) {
Expand Down

0 comments on commit 3fba187

Please sign in to comment.