Skip to content

Commit

Permalink
Merge branch 'AliceO2Group:dev' into new-detector4
Browse files Browse the repository at this point in the history
  • Loading branch information
pkurash authored Oct 10, 2024
2 parents 358535a + 9a6e661 commit 70ab460
Show file tree
Hide file tree
Showing 412 changed files with 9,260 additions and 8,864 deletions.
15 changes: 13 additions & 2 deletions CCDB/include/CCDB/BasicCCDBManager.h
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,10 @@ class CCDBManagerInstance
return getForTimeStamp<T>(path, timestamp);
}

/// retrieve an object of type T from CCDB as stored under path and using the timestamp in the middle of the run + metadata. The run number is provided separately to conform to typical analysis use (in which case metadata does not include runNumber)
template <typename T>
T* getSpecificForRun(std::string const& path, int runNumber, MD metaData = MD());

/// detect online processing modes (i.e. CCDB objects may be updated in the lifetime of the manager)
bool isOnline() const { return mDeplMode == o2::framework::DeploymentMode::OnlineAUX || mDeplMode == o2::framework::DeploymentMode::OnlineDDS || mDeplMode == o2::framework::DeploymentMode::OnlineECS; }

Expand Down Expand Up @@ -317,6 +321,14 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)

template <typename T>
T* CCDBManagerInstance::getForRun(std::string const& path, int runNumber, bool setRunMetadata)
{
auto metaData = setRunMetadata ? MD{{"runNumber", std::to_string(runNumber)}} : MD{};
mMetaData = metaData;
return getSpecificForRun<T>(path, runNumber, metaData);
}

template <typename T>
T* CCDBManagerInstance::getSpecificForRun(std::string const& path, int runNumber, MD metaData)
{
auto [start, stop] = getRunDuration(runNumber);
if (start < 0 || stop < 0) {
Expand All @@ -325,8 +337,7 @@ T* CCDBManagerInstance::getForRun(std::string const& path, int runNumber, bool s
}
return nullptr;
}
mMetaData = setRunMetadata ? MD{{"runNumber", std::to_string(runNumber)}} : MD{};
return getForTimeStamp<T>(path, start / 2 + stop / 2);
return getSpecific<T>(path, start / 2 + stop / 2, metaData);
}

class BasicCCDBManager : public CCDBManagerInstance
Expand Down
15 changes: 15 additions & 0 deletions CCDB/include/CCDB/CcdbApi.h
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,9 @@ class CcdbApi //: public DatabaseInterface
// Loads files from alien and cvmfs into given destination.
bool loadLocalContentToMemory(o2::pmr::vector<char>& dest, std::string& url) const;

// add annotated flattened headers in the end of the blob
static void appendFlatHeader(o2::pmr::vector<char>& dest, const std::map<std::string, std::string>& headers);

// the failure to load the file to memory is signaled by 0 size and non-0 capacity
static bool isMemoryFileInvalid(const o2::pmr::vector<char>& v) { return v.size() == 0 && v.capacity() > 0; }
template <typename T>
Expand Down Expand Up @@ -610,6 +613,16 @@ class CcdbApi //: public DatabaseInterface
return getSnapshotDir(topdir, path) + '/' + sfile;
}

template <typename MAP> // can be either std::map or std::multimap
static size_t getFlatHeaderSize(const MAP& Headers)
{
size_t hsize = sizeof(int) + sizeof(FlatHeaderAnnot); // annotation size
for (auto& h : Headers) {
hsize += h.first.length() + h.second.length() + 2; // 2*(string_buffer + terminating null character)
}
return hsize;
}

// tmp helper and single point of entry for a CURL perform call
// helps to switch between easy handle perform and multi handles in a single place
CURLcode CURL_perform(CURL* handle) const;
Expand All @@ -632,6 +645,8 @@ class CcdbApi //: public DatabaseInterface
size_t mCurlTimeoutDownload = 15; // download timeout in seconds, can be configured via ALICEO2_CCDB_CURL_TIMEOUT_DOWNLOAD, updated according to the deployment mode
size_t mCurlTimeoutUpload = 15; // upload timeout in seconds, can be configured via ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD, updated according to the deployment mode

static constexpr char FlatHeaderAnnot[] = "$HEADER$"; // annotation for flat header

ClassDefNV(CcdbApi, 1);
};

Expand Down
26 changes: 24 additions & 2 deletions CCDB/src/CcdbApi.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -1687,12 +1687,15 @@ void CcdbApi::scheduleDownload(RequestContext& requestContext, size_t* requestCo
ho.counter++;
try {
if (chunk.capacity() < chunk.size() + realsize) {
// estimate headers size when converted to annotated text string
const char hannot[] = "header";
size_t hsize = getFlatHeaderSize(ho.header);
auto cl = ho.header.find("Content-Length");
if (cl != ho.header.end()) {
size_t sizeFromHeader = std::stol(cl->second);
sz = std::max(chunk.size() * (sizeFromHeader ? 1 : 2) + realsize, sizeFromHeader);
sz = hsize + std::max(chunk.size() * (sizeFromHeader ? 1 : 2) + realsize, sizeFromHeader);
} else {
sz = std::max(chunk.size() * 2, chunk.size() + realsize);
sz = hsize + std::max(chunk.size() * 2, chunk.size() + realsize);
// LOGP(debug, "SIZE IS NOT IN HEADER, allocate {}", sz);
}
chunk.reserve(sz);
Expand Down Expand Up @@ -1885,6 +1888,25 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector<char>& dest, std::string const& p
vectoredLoadFileToMemory(contexts);
}

void CcdbApi::appendFlatHeader(o2::pmr::vector<char>& dest, const std::map<std::string, std::string>& headers)
{
size_t hsize = getFlatHeaderSize(headers), cnt = dest.size();
dest.resize(cnt + hsize);
auto addString = [&dest, &cnt](const std::string& s) {
for (char c : s) {
dest[cnt++] = c;
}
dest[cnt++] = 0;
};

for (auto& h : headers) {
addString(h.first);
addString(h.second);
}
*reinterpret_cast<int*>(&dest[cnt]) = hsize; // store size
std::memcpy(&dest[cnt + sizeof(int)], FlatHeaderAnnot, sizeof(FlatHeaderAnnot)); // annotate the flattened headers map
}

void CcdbApi::navigateSourcesAndLoadFile(RequestContext& requestContext, int& fromSnapshot, size_t* requestCounter) const
{
LOGP(debug, "loadFileToMemory {} ETag=[{}]", requestContext.path, requestContext.etag);
Expand Down
13 changes: 13 additions & 0 deletions DataFormats/Detectors/CTP/include/DataFormatsCTP/Scalers.h
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,19 @@ class CTPRunScalers
/// same with absolute timestamp (not orbit) as argument
std::pair<double, double> getRateGivenT(double timestamp, int classindex, int type) const;

/// retrieves integral for class
std::array<uint64_t, 7> getIntegralForClass(int i) const
{
return {
mScalerRecordO2[0].scalers[i].classIndex,
mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].lmBefore - mScalerRecordO2[0].scalers[i].lmBefore,
mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].lmAfter - mScalerRecordO2[0].scalers[i].lmAfter,
mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l0Before - mScalerRecordO2[0].scalers[i].l0Before,
mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l0After - mScalerRecordO2[0].scalers[i].l0After,
mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l1Before - mScalerRecordO2[0].scalers[i].l1Before,
mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l1After - mScalerRecordO2[0].scalers[i].l1After,
};
}
/// retrieves time boundaries of this scaler object from O2 scalers
std::pair<unsigned long, unsigned long> getTimeLimit() const
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ inline void RecoInputContainer::fillGPUIOPtr(o2::gpu::GPUTrackingInOutPointers*
ptrs->nTRDTriggerRecords = mNTriggerRecords;
ptrs->trdTriggerTimes = &(trdTriggerTimes[0]);
ptrs->trdTrackletIdxFirst = &(trdTriggerIndices[0]);
ptrs->trdTrigRecMask = reinterpret_cast<const char*>(mTrigRecMask.data());
ptrs->trdTrigRecMask = reinterpret_cast<const uint8_t*>(mTrigRecMask.data());
ptrs->nTRDTracklets = mNTracklets;
ptrs->trdTracklets = reinterpret_cast<const o2::gpu::GPUTRDTrackletWord*>(mTracklets.data());
ptrs->trdSpacePoints = reinterpret_cast<const o2::gpu::GPUTRDSpacePoint*>(mSpacePoints.data());
Expand Down
4 changes: 3 additions & 1 deletion DataFormats/Parameters/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ o2_add_library(DataFormatsParameters
src/GRPLHCIFData.cxx
src/GRPECSObject.cxx
src/GRPMagField.cxx
PUBLIC_LINK_LIBRARIES FairRoot::Base O2::CommonConstants
src/AggregatedRunInfo.cxx
PUBLIC_LINK_LIBRARIES FairRoot::Base O2::CommonConstants
O2::CommonTypes O2::CCDB
O2::DetectorsCommonDataFormats)

Expand All @@ -24,6 +25,7 @@ o2_target_root_dictionary(DataFormatsParameters
include/DataFormatsParameters/GRPLHCIFData.h
include/DataFormatsParameters/GRPECSObject.h
include/DataFormatsParameters/GRPMagField.h
include/DataFormatsParameters/AggregatedRunInfo.h
LINKDEF src/ParametersDataLinkDef.h)

o2_add_executable(simgrp-tool
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
// All rights not expressly granted are reserved.
//
// This software is distributed under the terms of the GNU General Public
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
//
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.

/// \file GRPECSObject.h
/// \brief Header of the AggregatedRunInfo struct
/// \author [email protected] [email protected]

#ifndef ALICEO2_DATA_AGGREGATEDRUNINFO_H_
#define ALICEO2_DATA_AGGREGATEDRUNINFO_H_

#include <cstdint>
#include "CCDB/BasicCCDBManager.h"

namespace o2::parameters
{

/// Composite struct where one may collect important global properties of data "runs"
/// aggregated from various sources (GRPECS, RunInformation CCDB entries, etc.).
/// Also offers the authoritative algorithms to collect these information for easy reuse
/// across various algorithms (anchoredMC, analysis, ...)
struct AggregatedRunInfo {
int runNumber; // run number
int64_t sor; // best known timestamp for the start of run
int64_t eor; // best known timestamp for end of run
int64_t orbitsPerTF; // number of orbits per TF
int64_t orbitReset; // timestamp of orbit reset before run
int64_t orbitSOR; // orbit when run starts after orbit reset
int64_t orbitEOR; // orbit when run ends after orbit reset

// we may have pointers to actual data source objects GRPECS, ...

// fills and returns AggregatedRunInfo for a given run number.
static AggregatedRunInfo buildAggregatedRunInfo(o2::ccdb::CCDBManagerInstance& ccdb, int runnumber);
};

} // namespace o2::parameters

#endif
86 changes: 86 additions & 0 deletions DataFormats/Parameters/src/AggregatedRunInfo.cxx
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
// All rights not expressly granted are reserved.
//
// This software is distributed under the terms of the GNU General Public
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
//
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.

/// \file AggregatedRunInfo.cxx
/// \author [email protected]

#include "DataFormatsParameters/AggregatedRunInfo.h"
#include "CCDB/BasicCCDBManager.h"
#include "DataFormatsParameters/GRPECSObject.h"
#include "CommonConstants/LHCConstants.h"
#include "Framework/Logger.h"

using namespace o2::parameters;

o2::parameters::AggregatedRunInfo AggregatedRunInfo::buildAggregatedRunInfo(o2::ccdb::CCDBManagerInstance& ccdb, int runnumber)
{
// TODO: could think about caching results per runnumber to
// avoid going to CCDB multiple times ---> but should be done inside the CCDBManagerInstance

// we calculate the first orbit of a run based on sor (start-of-run) and eor
// we obtain these by calling getRunDuration
auto [sor, eor] = ccdb.getRunDuration(runnumber);

// determine a good timestamp to query OrbitReset for this run
// --> the middle of the run is very appropriate and safer than just sor
auto run_mid_timestamp = sor + (eor - sor) / 2;

// query the time of the orbit reset (when orbit is defined to be 0)
auto ctpx = ccdb.getForTimeStamp<std::vector<Long64_t>>("CTP/Calib/OrbitReset", run_mid_timestamp);
int64_t tsOrbitReset = (*ctpx)[0]; // us

// get timeframe length from GRPECS
std::map<std::string, std::string> metadata;
metadata["runNumber"] = Form("%d", runnumber);
auto grpecs = ccdb.getSpecific<o2::parameters::GRPECSObject>("GLO/Config/GRPECS", run_mid_timestamp, metadata);
auto nOrbitsPerTF = grpecs->getNHBFPerTF();

// calculate SOR orbit
int64_t orbitSOR = (sor * 1000 - tsOrbitReset) / o2::constants::lhc::LHCOrbitMUS;
int64_t orbitEOR = (eor * 1000 - tsOrbitReset) / o2::constants::lhc::LHCOrbitMUS;

// adjust to the nearest TF edge to satisfy condition (orbitSOR % nOrbitsPerTF == 0)
orbitSOR = (orbitSOR / nOrbitsPerTF + 1) * nOrbitsPerTF; // +1 to choose the safe boundary ... towards run middle
orbitEOR = orbitEOR / nOrbitsPerTF * nOrbitsPerTF;

// fetch SOR directly from CTP entry on CCDB
bool oldFatalState = ccdb.getFatalWhenNull();
ccdb.setFatalWhenNull(false);
auto ctp_first_run_orbit = ccdb.getForTimeStamp<std::vector<int64_t>>("CTP/Calib/FirstRunOrbit", run_mid_timestamp);
ccdb.setFatalWhenNull(oldFatalState);
if (ctp_first_run_orbit && ctp_first_run_orbit->size() >= 3) {
// if we have CTP first run orbit available, we should use it

// int64_t creation_time = (*ctp_first_run_orbit)[0];
int64_t ctp_run_number = (*ctp_first_run_orbit)[1];
int64_t ctp_orbitSOR = (*ctp_first_run_orbit)[2];

if (ctp_run_number != runnumber) {
LOG(error) << "AggregatedRunInfo: run number inconsistency found (asked: " << runnumber << " vs CTP found: " << ctp_run_number << ")";
}

// overwrite orbitSOR
if (ctp_orbitSOR != orbitSOR) {
LOG(warn) << "The calculated orbitSOR " << orbitSOR << " differs from CTP orbitSOR " << ctp_orbitSOR;
// reasons for this is different unit of time storage in RunInformation (ms) and orbitReset (us), etc.

// so we need to adjust the SOR timings to be consistent
auto sor_new = (int64_t)((tsOrbitReset + ctp_orbitSOR * o2::constants::lhc::LHCOrbitMUS) / 1000.);
if (sor_new != sor) {
LOG(warn) << "Adjusting SOR from " << sor << " to " << sor_new;
sor = sor_new;
}
}
orbitSOR = ctp_orbitSOR;
}

return AggregatedRunInfo{runnumber, sor, eor, nOrbitsPerTF, tsOrbitReset, orbitSOR, orbitEOR};
}
1 change: 1 addition & 0 deletions DataFormats/Parameters/src/ParametersDataLinkDef.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,6 @@
#pragma link C++ class o2::parameters::GRPMagField + ;
#pragma link C++ class std::unordered_map < unsigned int, unsigned int> + ;
#pragma link C++ class std::pair < unsigned long, std::string> + ;
#pragma link C++ struct o2::parameters::AggregatedRunInfo + ;

#endif
4 changes: 4 additions & 0 deletions Detectors/CTP/macro/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -69,4 +69,8 @@ o2_add_test_root_macro(TestFetcher.C
PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
O2::CCDB
LABELS ctp)
o2_add_test_root_macro(CreateBKForRun.C
PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
O2::CCDB
LABELS ctp)

74 changes: 74 additions & 0 deletions Detectors/CTP/macro/CreateBKForRun.C
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
// All rights not expressly granted are reserved.
//
// This software is distributed under the terms of the GNU General Public
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
//
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.

#if !defined(__CLING__) || defined(__ROOTCLING__)
#include <iomanip>
#include <TMath.h>
#include <CCDB/BasicCCDBManager.h>
#include <DataFormatsCTP/Configuration.h>
#include <DataFormatsParameters/GRPLHCIFData.h>
#endif
using namespace o2::ctp;

void CreateBKForRun()
{
std::vector<int> runs = {558124, 558126, 558215, 558217, 558221, 558244, 558247};
std::string mCCDBPathCTPScalers = "CTP/Calib/Scalers";
std::string mCCDBPathCTPConfig = "CTP/Config/Config";
//
std::string filename = "BKcounters.txt";
std::ofstream outfile(filename);
if (!outfile) {
Error("", "Failed to open file %s", filename.c_str());
return;
}
auto& ccdbMgr = o2::ccdb::BasicCCDBManager::instance();
for (auto const& runNumber : runs) {
auto soreor = ccdbMgr.getRunDuration(runNumber);
uint64_t timeStamp = (soreor.second - soreor.first) / 2 + soreor.first;
std::cout << runNumber << " Timestamp:" << timeStamp << std::endl;
//
std::string srun = std::to_string(runNumber);
std::map<string, string> metadata;
metadata["runNumber"] = srun;
auto ctpscalers = ccdbMgr.getSpecific<CTPRunScalers>(mCCDBPathCTPScalers, timeStamp, metadata);
if (ctpscalers == nullptr) {
LOG(info) << "CTPRunScalers not in database, timestamp:" << timeStamp;
}
auto ctpcfg = ccdbMgr.getSpecific<CTPConfiguration>(mCCDBPathCTPConfig, timeStamp, metadata);
if (ctpcfg == nullptr) {
LOG(info) << "CTPRunConfig not in database, timestamp:" << timeStamp;
}
//
ctpscalers->convertRawToO2();
std::vector<CTPClass>& ctpcls = ctpcfg->getCTPClasses();
std::vector<int> clslist = ctpcfg->getTriggerClassList();
auto times = ctpscalers->getTimeLimit();
for (size_t i = 0; i < clslist.size(); i++) {
// std::cout << i << " " << ctpcls[i].name ;
std::array<uint64_t, 7> cnts = ctpscalers->getIntegralForClass(i);
if (clslist[i] != (int)cnts[0]) {
LOG(fatal) << "cls list incompatible with counters";
}
std::cout << std::setw(21) << ctpcls[cnts[0]].name;
outfile << runNumber << ", " << ctpcls[i].name << ", " << std::get<1>(times) / 1000;
for (int j = 1; j < 7; j++) {
// std::cout << std::setw(21) << " " << cnts[j];
std::cout << ", " << cnts[j];
outfile << ", " << cnts[j];
}
std::cout << std::endl;
outfile << std::endl;
}
}
// ctpscalers->printFromZero(std::cout);
outfile.close();
}
Loading

0 comments on commit 70ab460

Please sign in to comment.