Skip to content

Please consider the following formatting changes to #14004 #46

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alibuild wants to merge 2 commits into from
Closed

Please consider the following formatting changes to #14004 #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7f15062
Restructure pedestal file creation, add DCS CCDB publishing
wiechula Feb 25, 2025
e646315
Please consider the following formatting changes
alibuild Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions Detectors/TPC/base/include/TPCBase/CRUCalibHelpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,8 @@ constexpr float fixedSizeToFloat(uint32_t value)
/// write values of map to fileName
///
template <typename DataMap>
void writeValues(const std::string_view fileName, const DataMap& map, bool onlyFilled = false)
void writeValues(std::ostream& str, const DataMap& map, bool onlyFilled = false)
{
std::ofstream str(fileName.data(), std::ofstream::out);

for (const auto& [linkInfo, data] : map) {
if (onlyFilled) {
if (!std::accumulate(data.begin(), data.end(), uint32_t(0))) {
Expand All @@ -117,6 +115,13 @@ void writeValues(const std::string_view fileName, const DataMap& map, bool onlyF
}
}

template <typename DataMap>
void writeValues(const std::string_view fileName, const DataMap& map, bool onlyFilled = false)
{
std::ofstream str(fileName.data(), std::ofstream::out);
writeValues(str, map, onlyFilled);
}

template <class T>
struct is_map {
static constexpr bool value = false;
Expand All @@ -126,7 +131,8 @@ template <class Key, class Value>
struct is_map<std::map<Key, Value>> {
static constexpr bool value = true;
};
/// fill cal pad object from HV data map

/// fill cal pad object from HW data map
/// TODO: Function to be tested
template <typename DataMap, uint32_t SignificantBitsT = 0>
typename std::enable_if_t<is_map<DataMap>::value, void>
Expand Down Expand Up @@ -251,6 +257,7 @@ o2::tpc::CalDet<float> getCalPad(const std::string_view fileName, const std::str
/// \param minADCROCType can be either one value for all ROC types, or {IROC, OROC}, or {IROC, OROC1, OROC2, OROC3}
std::unordered_map<std::string, CalPad> preparePedestalFiles(const CalPad& pedestals, const CalPad& noise, std::vector<float> sigmaNoiseROCType = {3, 3, 3, 3}, std::vector<float> minADCROCType = {2, 2, 2, 2}, float pedestalOffset = 0, bool onlyFilled = false, bool maskBad = true, float noisyChannelThreshold = 1.5, float sigmaNoiseNoisyChannels = 4, float badChannelThreshold = 6, bool fixedSize = false);

DataMapU32 getDataMap(const CalPad& calPad);
} // namespace o2::tpc::cru_calib_helpers

#endif
50 changes: 46 additions & 4 deletions Detectors/TPC/base/src/CRUCalibHelpers.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,6 @@ std::unordered_map<std::string, CalPad> cru_calib_helpers::preparePedestalFiles(
pedestalsThreshold["PedestalsPhys"] = CalPad("Pedestals");
pedestalsThreshold["ThresholdMapPhys"] = CalPad("ThresholdMap");

auto& pedestalsCRU = pedestalsThreshold["Pedestals"];
auto& thresholdCRU = pedestalsThreshold["ThresholdMap"];

// ===| prepare values |===
for (size_t iroc = 0; iroc < pedestals.getData().size(); ++iroc) {
const ROC roc(iroc);
Expand Down Expand Up @@ -179,7 +176,7 @@ std::unordered_map<std::string, CalPad> cru_calib_helpers::preparePedestalFiles(
}

float noise = std::abs(rocNoise.getValue(ipad)); // it seems with the new fitting procedure, the noise can also be negative, since in gaus sigma is quadratic
float noiseCorr = noise - (0.847601 + 0.031514 * traceLength);
const float noiseCorr = noise - (0.847601 + 0.031514 * traceLength);
if ((pedestal <= 0) || (pedestal > 150) || (noise <= 0) || (noise > 50)) {
LOGP(info, "Bad pedestal or noise value in ROC {:2}, CRU {:3}, fec in CRU: {:2}, SAMPA: {}, channel: {:2}, pedestal: {:.4f}, noise {:.4f}", iroc, cruID, fecInPartition, sampa, sampaChannel, pedestal, noise);
if (maskBad) {
Expand Down Expand Up @@ -230,3 +227,48 @@ std::unordered_map<std::string, CalPad> cru_calib_helpers::preparePedestalFiles(

return pedestalsThreshold;
}

cru_calib_helpers::DataMapU32 cru_calib_helpers::getDataMap(const CalPad& calPad)
{
const auto& mapper = Mapper::instance();

DataMapU32 dataMap;

for (size_t iroc = 0; iroc < calPad.getData().size(); ++iroc) {
const ROC roc(iroc);

const auto& calRoc = calPad.getCalArray(iroc);

const int padOffset = roc.isOROC() ? mapper.getPadsInIROC() : 0;

// skip empty ROCs
if (!(std::abs(calRoc.getSum()) > 0)) {
continue;
}

// loop over pads
for (size_t ipad = 0; ipad < calRoc.getData().size(); ++ipad) {
const int globalPad = ipad + padOffset;
const FECInfo& fecInfo = mapper.fecInfo(globalPad);
const CRU cru = mapper.getCRU(roc.getSector(), globalPad);
const uint32_t region = cru.region();
const int cruID = cru.number();
const int sampa = fecInfo.getSampaChip();
const int sampaChannel = fecInfo.getSampaChannel();

const PartitionInfo& partInfo = mapper.getMapPartitionInfo()[cru.partition()];
const int nFECs = partInfo.getNumberOfFECs();
const int fecOffset = (nFECs + 1) / 2;
const int fecInPartition = fecInfo.getIndex() - partInfo.getSectorFECOffset();
const int dataWrapperID = fecInPartition >= fecOffset;
const int globalLinkID = (fecInPartition % fecOffset) + dataWrapperID * 12;

const int hwChannel = getHWChannel(sampa, sampaChannel, region % 2);

const auto value = calRoc.getValue(ipad);
dataMap[LinkInfo(cruID, globalLinkID)][hwChannel] = floatToFixedSize(value);
}
}

return dataMap;
}
71 changes: 8 additions & 63 deletions Detectors/TPC/calibration/macro/preparePedestalFiles.C
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,72 +60,17 @@ void preparePedestalFiles(const std::string_view pedestalFile, std::string outpu
f.GetObject("Noise", calNoise);
}

DataMapU32 pedestalValues;
DataMapU32 thresholdlValues;
DataMapU32 pedestalValuesPhysics;
DataMapU32 thresholdlValuesPhysics;

auto pedestalsThreshold = preparePedestalFiles(*calPedestal, *calNoise, sigmaNoiseROCType, minADCROCType, pedestalOffset, onlyFilled, maskBad, noisyChannelThreshold, sigmaNoiseNoisyChannels, badChannelThreshold);

// ===| prepare values |===
for (size_t iroc = 0; iroc < calPedestal->getData().size(); ++iroc) {
const ROC roc(iroc);

const auto& rocPedestal = calPedestal->getCalArray(iroc);
const auto& rocNoise = calNoise->getCalArray(iroc);
auto& rocOut = output.getCalArray(iroc);
const auto& pedestals = pedestalsThreshold["Pedestals"];
const auto& thresholds = pedestalsThreshold["ThresholdMap"];
const auto& pedestalsPhys = pedestalsThreshold["PedestalsPhys"];
const auto& thresholdsPhys = pedestalsThreshold["ThresholdMapPhys"];

const int padOffset = roc.isOROC() ? mapper.getPadsInIROC() : 0;

// skip empty
if (!(std::abs(rocPedestal.getSum() + rocNoise.getSum()) > 0)) {
continue;
}

// loop over pads
for (size_t ipad = 0; ipad < rocPedestal.getData().size(); ++ipad) {
const int globalPad = ipad + padOffset;
const FECInfo& fecInfo = mapper.fecInfo(globalPad);
const CRU cru = mapper.getCRU(roc.getSector(), globalPad);
const uint32_t region = cru.region();
const int cruID = cru.number();
const int sampa = fecInfo.getSampaChip();
const int sampaChannel = fecInfo.getSampaChannel();
// int globalLinkID = fecInfo.getIndex();

const PartitionInfo& partInfo = mapper.getMapPartitionInfo()[cru.partition()];
const int nFECs = partInfo.getNumberOfFECs();
const int fecOffset = (nFECs + 1) / 2;
const int fecInPartition = fecInfo.getIndex() - partInfo.getSectorFECOffset();
const int dataWrapperID = fecInPartition >= fecOffset;
const int globalLinkID = (fecInPartition % fecOffset) + dataWrapperID * 12;

const auto pedestal = pedestalsThreshold["Pedestals"].getCalArray(iroc).getValue(ipad);
const auto threshold = pedestalsThreshold["ThresholdMap"].getCalArray(iroc).getValue(ipad);
const auto pedestalHighNoise = pedestalsThreshold["PedestalsPhys"].getCalArray(iroc).getValue(ipad);
const auto thresholdHighNoise = pedestalsThreshold["ThresholdMapPhys"].getCalArray(iroc).getValue(ipad);

const int hwChannel = getHWChannel(sampa, sampaChannel, region % 2);
// for debugging
// printf("%4d %4d %4d %4d %4d: %u\n", cru.number(), globalLinkID, hwChannel, fecInfo.getSampaChip(), fecInfo.getSampaChannel(), getADCValue(pedestal));

// default thresholds
const auto adcPedestal = floatToFixedSize(pedestal);
const auto adcThreshold = floatToFixedSize(threshold);
pedestalValues[LinkInfo(cruID, globalLinkID)][hwChannel] = adcPedestal;
thresholdlValues[LinkInfo(cruID, globalLinkID)][hwChannel] = adcThreshold;

// higher thresholds for physics data taking
const auto adcPedestalPhysics = floatToFixedSize(pedestalHighNoise);
const auto adcThresholdPhysics = floatToFixedSize(thresholdHighNoise);
pedestalValuesPhysics[LinkInfo(cruID, globalLinkID)][hwChannel] = adcPedestalPhysics;
thresholdlValuesPhysics[LinkInfo(cruID, globalLinkID)][hwChannel] = adcThresholdPhysics;
// for debugging
// if(!(std::abs(pedestal - fixedSizeToFloat(adcPedestal)) <= 0.5 * 0.25)) {
// printf("%4d %4d %4d %4d %4d: %u %.2f %.4f %.4f\n", cru.number(), globalLinkID, hwChannel, sampa, sampaChannel, adcPedestal, fixedSizeToFloat(adcPedestal), pedestal, pedestal - fixedSizeToFloat(adcPedestal));
//}
}
}
auto pedestalValues = getDataMap(pedestals);
auto thresholdlValues = getDataMap(thresholds);
auto pedestalValuesPhysics = getDataMap(pedestalsPhys);
auto thresholdlValuesPhysics = getDataMap(thresholdsPhys);

// text files
const auto outFilePedestalTXT(outputDir + "/pedestal_values.txt");
Expand Down
2 changes: 1 addition & 1 deletion Detectors/TPC/workflow/include/TPCWorkflow/CalDetMergerPublisherSpec.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ namespace o2
namespace tpc
{

o2::framework::DataProcessorSpec getCalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool dumpAfterComplete = false);
o2::framework::DataProcessorSpec getCalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool sendToDCS, bool dumpAfterComplete = false);

} // namespace tpc
} // namespace o2
Expand Down
76 changes: 71 additions & 5 deletions Detectors/TPC/workflow/src/CalDetMergerPublisherSpec.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include <vector>
#include <string>
#include <algorithm>
#include <sstream>

#include <fmt/format.h>

Expand All @@ -37,6 +38,7 @@
#include "CCDB/CcdbObjectInfo.h"
#include "TPCBase/CDBInterface.h"
#include "TPCBase/CalDet.h"
#include "TPCBase/CRUCalibHelpers.h"
#include "TPCWorkflow/CalibRawPartInfo.h"
#include "TPCWorkflow/CalDetMergerPublisherSpec.h"
#include "TPCWorkflow/ProcessingHelpers.h"
Expand All @@ -52,7 +54,7 @@ class CalDetMergerPublisherSpec : public o2::framework::Task
using CcdbObjectInfo = o2::ccdb::CcdbObjectInfo;

public:
CalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool dumpAfterComplete = false) : mLanesToExpect(lanes), mCalibInfos(lanes), mSkipCCDB(skipCCDB), mPublishAfterComplete(dumpAfterComplete) {}
CalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool sendToDCS, bool dumpAfterComplete = false) : mLanesToExpect(lanes), mCalibInfos(lanes), mSkipCCDB(skipCCDB), mSendToDCS(sendToDCS), mPublishAfterComplete(dumpAfterComplete) {}

void init(o2::framework::InitContext& ic) final
{
Expand Down Expand Up @@ -154,10 +156,12 @@ class CalDetMergerPublisherSpec : public o2::framework::Task
CDBType mCalDetMapType; ///< calibration type of CalDetMap object
uint64_t mRunNumber{0}; ///< processed run number
uint32_t mLanesToExpect{0}; ///< number of expected lanes sending data
uint32_t mDCSSpecOffset{32768}; ///< offset for DCS specs
bool mForceQuit{false}; ///< for quit after processing finished
bool mDirectFileDump{false}; ///< directly dump the calibration data to file
bool mPublishAfterComplete{false}; ///< dump calibration directly after data from all lanes received
bool mSkipCCDB{false}; ///< skip sending of calibration data
bool mSendToDCS{false}; ///< skip sending of calibration data
bool mCheckCalibInfos{false}; ///< check calib infos

//____________________________________________________________________________
Expand All @@ -170,7 +174,6 @@ class CalDetMergerPublisherSpec : public o2::framework::Task
}

// perhaps should be changed to time of the run
const auto now = std::chrono::system_clock::now();
const long timeStart = mCalibInfos[0].tfIDInfo.creation + mCalibInfos[0].publishCycle;
const long timeEnd = o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP;

Expand All @@ -193,6 +196,11 @@ class CalDetMergerPublisherSpec : public o2::framework::Task
o2::header::DataHeader::SubSpecificationType subSpec{(o2::header::DataHeader::SubSpecificationType)mCalDetMapType};
output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "TPC_CALIB", subSpec}, *image.get());
output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "TPC_CALIB", subSpec}, w);

// for pedestal calibration send to DCS if requested
if (mSendToDCS && (mCalDetMapType == CDBType::CalPedestalNoise)) {
sendPedestalNoiseToDCS(output);
}
}

for (auto& [type, object] : mMergedCalDets) {
Expand Down Expand Up @@ -238,16 +246,74 @@ class CalDetMergerPublisherSpec : public o2::framework::Task
}
}
}

void sendPedestalNoiseToDCS(DataAllocator& output)
{
auto sendObject = [this, &output](const CalPad& data, const std::string& path, const std::string& fileNameBase = "") {
const long timeStart = mCalibInfos[0].tfIDInfo.creation + mCalibInfos[0].publishCycle;
const long timeEnd = o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP;

const auto dataMap = cru_calib_helpers::getDataMap(data);
std::ostringstream dataStr;
cru_calib_helpers::writeValues(dataStr, dataMap);

std::vector<char> dataVec;
const auto& str = dataStr.str();
std::copy(str.begin(), str.end(), std::back_inserter(dataVec));

o2::ccdb::CcdbObjectInfo w;

w.setPath(path);
w.setFileName(fmt::format("{}_{}_{}.txt", fileNameBase, mRunNumber, timeStart));
w.setStartValidityTimestamp(timeStart);
w.setEndValidityTimestamp(timeEnd);

auto md = w.getMetaData();
md[o2::base::NameConf::CCDBRunTag.data()] = std::to_string(mRunNumber);
w.setMetaData(md);

LOGP(info, "Sending object to DCS DB {}/{} of size {} ({}) bytes, valid for {} : {}", w.getPath(), w.getFileName(), dataVec.size(), dataStr.str().size(), w.getStartValidityTimestamp(), w.getEndValidityTimestamp());

o2::header::DataHeader::SubSpecificationType subSpec{(o2::header::DataHeader::SubSpecificationType)mCalDetMapType + mDCSSpecOffset};
output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "TPC_CALIB_DCS", subSpec}, dataVec);
output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "TPC_CALIB_DCS", subSpec}, w);
};

const auto& pedestals = mMergedCalDetsMap.at("Pedestals");
const auto& noise = mMergedCalDetsMap.at("Noise");

bool first = true;
for (auto threshold : {2.5f, 3.f, 3.5f}) {
auto pedestalsThreshold = cru_calib_helpers::preparePedestalFiles(pedestals, noise, {threshold});

// pedestals don't depend on threshold, publish on first iteration only
if (first) {
const auto& pedestalsPhys = pedestalsThreshold["PedestalsPhys"];
sendObject(pedestalsPhys, "TPC/Calib/PedestalsPhys", "Pedestals");
}

const auto& thresholdsPhys = pedestalsThreshold["ThresholdMapPhys"];
const auto fileNameBase = fmt::format("ThresholdsPhys-{:.0f}", threshold * 10);
sendObject(thresholdsPhys, "TPC/Calib/" + fileNameBase, fileNameBase);

first = false;
}
}
};

o2::framework::DataProcessorSpec o2::tpc::getCalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool dumpAfterComplete)
o2::framework::DataProcessorSpec o2::tpc::getCalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool sendToDCS, bool dumpAfterComplete)
{
std::vector<OutputSpec> outputs;
if (!skipCCDB) {
outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCDBPayload, "TPC_CALIB"}, Lifetime::Sporadic);
outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCDBWrapper, "TPC_CALIB"}, Lifetime::Sporadic);
}

if (sendToDCS) {
outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCDBPayload, "TPC_CALIB_DCS"}, Lifetime::Sporadic);
outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCDBWrapper, "TPC_CALIB_DCS"}, Lifetime::Sporadic);
}

std::vector<InputSpec> inputs;
inputs.emplace_back("clbPayload", ConcreteDataTypeMatcher{gDataOriginTPC, "CLBPART"}, Lifetime::Sporadic);
inputs.emplace_back("clbInfo", ConcreteDataTypeMatcher{gDataOriginTPC, "CLBPARTINFO"}, Lifetime::Sporadic);
Expand All @@ -258,11 +324,11 @@ o2::framework::DataProcessorSpec o2::tpc::getCalDetMergerPublisherSpec(uint32_t
id.data(),
inputs,
outputs,
AlgorithmSpec{adaptFromTask<CalDetMergerPublisherSpec>(lanes, skipCCDB, dumpAfterComplete)},
AlgorithmSpec{adaptFromTask<CalDetMergerPublisherSpec>(lanes, skipCCDB, sendToDCS, dumpAfterComplete)},
Options{
{"force-quit", VariantType::Bool, false, {"force quit after max-events have been reached"}},
{"direct-file-dump", VariantType::Bool, false, {"directly dump calibration to file"}},
{"check-calib-infos", VariantType::Bool, false, {"make consistency check of calib infos"}},
} // end Options
}; // end DataProcessorSpec
}; // end DataProcessorSpec
}
4 changes: 3 additions & 1 deletion Detectors/TPC/workflow/src/tpc-calib-pad-raw.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ void customize(std::vector<ConfigParamSpec>& workflowOptions)
{"configFile", VariantType::String, "", {"configuration file for configurable parameters"}},
{"calib-type", VariantType::String, "pedestal", {"Calibration type to run: pedestal, pulser, ce"}},
{"no-write-ccdb", VariantType::Bool, false, {"skip sending the calibration output to CCDB"}},
{"send-to-dcs-ccdb", VariantType::Bool, false, {"Send values to DCS DB"}},
{"lanes", VariantType::Int, defaultlanes, {"Number of parallel processing lanes."}},
{"sectors", VariantType::String, sectorDefault.c_str(), {"List of TPC sectors, comma separated ranges, e.g. 0-3,7,9-15"}},
};
Expand All @@ -83,6 +84,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)

std::string inputSpec = config.options().get<std::string>("input-spec");
const auto skipCCDB = config.options().get<bool>("no-write-ccdb");
const auto sendToDCS = config.options().get<bool>("send-to-dcs-ccdb");
const auto publishAfterTFs = config.options().get<uint32_t>("publish-after-tfs");

const auto tpcsectors = o2::RangeTokenizer::tokenize<int>(config.options().get<std::string>("sectors"));
Expand Down Expand Up @@ -121,7 +123,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
workflow.emplace_back(getTPCCalibPadRawSpec(inputSpec, ilane, range, publishAfterTFs, rawType));
}

workflow.emplace_back(getCalDetMergerPublisherSpec(nLanes, skipCCDB, publishAfterTFs > 0));
workflow.emplace_back(getCalDetMergerPublisherSpec(nLanes, skipCCDB, sendToDCS, publishAfterTFs > 0));

return workflow;
}