Skip to content

Commit c0a6940

Browse files
committed
GPU: Reactivate option to process only individual TPC sectors
1 parent 4f83bca commit c0a6940

File tree

9 files changed

+101
-72
lines changed

9 files changed

+101
-72
lines changed

DataFormats/Detectors/TPC/include/DataFormatsTPC/ClusterNativeHelper.h

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -216,24 +216,24 @@ class ClusterNativeHelper
216216
// @param mcBuffer
217217
// @param inputs data arrays, fixed array, one per sector
218218
// @param mcinputs vectors mc truth container, fixed array, one per sector
219-
// @param checkFct check whether a sector index is valid
220-
template <typename DataArrayType, typename MCArrayType, typename CheckFct = std::function<bool(size_t&)>>
219+
// @param sectorMask Bitmask with tpc sectors to process
220+
template <typename DataArrayType, typename MCArrayType>
221221
static int fillIndex(
222222
ClusterNativeAccess& clusterIndex, std::unique_ptr<ClusterNative[]>& clusterBuffer,
223223
ConstMCLabelContainerViewWithBuffer& mcBuffer, DataArrayType& inputs, MCArrayType const& mcinputs,
224-
CheckFct checkFct = [](auto const&) { return true; });
224+
unsigned long sectorMask = 0xFFFFFFFFF);
225225

226-
template <typename DataArrayType, typename CheckFct = std::function<bool(size_t&)>>
226+
template <typename DataArrayType>
227227
static int fillIndex(
228228
ClusterNativeAccess& clusterIndex, std::unique_ptr<ClusterNative[]>& clusterBuffer,
229-
DataArrayType& inputs, CheckFct checkFct = [](auto const&) { return true; })
229+
DataArrayType& inputs, unsigned long sectorMask = 0xFFFFFFFFF)
230230
{
231231
// just use a dummy parameter with empty vectors
232232
// TODO: maybe do in one function with conditional template parameter
233233
std::vector<std::unique_ptr<MCLabelContainer>> dummy;
234234
// another default, nothing will be added to the container
235235
ConstMCLabelContainerViewWithBuffer mcBuffer;
236-
return fillIndex(clusterIndex, clusterBuffer, mcBuffer, inputs, dummy, checkFct);
236+
return fillIndex(clusterIndex, clusterBuffer, mcBuffer, inputs, dummy, sectorMask);
237237
}
238238

239239
// Process data for one sector.
@@ -348,10 +348,10 @@ class ClusterNativeHelper
348348
static void copySectorData(ClusterNativeAccess const& index, int sector, BufferType& target, MCArrayType& mcTarget);
349349
};
350350

351-
template <typename DataArrayType, typename MCArrayType, typename CheckFct>
351+
template <typename DataArrayType, typename MCArrayType>
352352
int ClusterNativeHelper::Reader::fillIndex(ClusterNativeAccess& clusterIndex,
353353
std::unique_ptr<ClusterNative[]>& clusterBuffer, ConstMCLabelContainerViewWithBuffer& mcBuffer,
354-
DataArrayType& inputs, MCArrayType const& mcinputs, CheckFct checkFct)
354+
DataArrayType& inputs, MCArrayType const& mcinputs, unsigned long sectorMask)
355355
{
356356
if (mcinputs.size() > 0 && mcinputs.size() != inputs.size()) {
357357
std::runtime_error("inconsistent size of MC label array " + std::to_string(mcinputs.size()) + ", expected " + std::to_string(inputs.size()));
@@ -371,16 +371,20 @@ int ClusterNativeHelper::Reader::fillIndex(ClusterNativeAccess& clusterIndex,
371371
if (sizeof(ClusterCountIndex) + clusterIndex.nClustersTotal * sizeof(ClusterNative) > inputs[0].size()) {
372372
throw std::runtime_error("inconsistent input buffer, expecting size " + std::to_string(sizeof(ClusterCountIndex) + clusterIndex.nClustersTotal * sizeof(ClusterNative)) + " got " + std::to_string(inputs[0].size()));
373373
}
374+
if (sectorMask != 0xFFFFFFFFF) {
375+
for (unsigned int sector = 0; sector < NSectors; sector++) {
376+
if (!(sectorMask & (1ul << sector)) && clusterIndex.nClustersSector[sector]) {
377+
throw std::runtime_error("TPC sector mask provided, but received more sectors than set. (A filter could be implemented here if needed.)");
378+
}
379+
}
380+
}
374381
return clusterIndex.nClustersTotal;
375382
}
376383

377384
// multiple data blocks need to be merged into the single block
378385
const ConstMCLabelContainerView* clustersMCTruth[NSectors] = {nullptr};
379386
int result = 0;
380387
for (size_t index = 0, end = inputs.size(); index < end; index++) {
381-
if (!checkFct(index)) {
382-
continue;
383-
}
384388
o2::dataformats::ConstMCTruthContainerView<o2::MCCompLabel> const* labelsptr = nullptr;
385389
#ifdef MS_GSL_V3
386390
std::size_t extent = 0;

DataFormats/Detectors/TPC/include/DataFormatsTPC/WorkflowHelper.h

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -102,10 +102,14 @@ static auto getWorkflowTPCInput(o2::framework::ProcessingContext& pc, int verbos
102102
}
103103
if (do_digits) {
104104
for (unsigned int i = 0; i < constants::MAXSECTOR; i++) {
105-
if (verbosity >= 1) {
106-
LOG(INFO) << "GOT MC LABELS FOR SECTOR " << i << " -> " << retVal->internal.inputDigitsMC[retVal->internal.inputDigitsMCIndex[i]].getNElements();
105+
if (tpcSectorMask & (1ul << i)) {
106+
if (verbosity >= 1) {
107+
LOG(INFO) << "GOT MC LABELS FOR SECTOR " << i << " -> " << retVal->internal.inputDigitsMC[retVal->internal.inputDigitsMCIndex[i]].getNElements();
108+
}
109+
retVal->inputDigitsMCPtrs[i] = &retVal->internal.inputDigitsMC[retVal->internal.inputDigitsMCIndex[i]];
110+
} else {
111+
retVal->inputDigitsMCPtrs[i] = nullptr;
107112
}
108-
retVal->inputDigitsMCPtrs[i] = &retVal->internal.inputDigitsMC[retVal->internal.inputDigitsMCIndex[i]];
109113
}
110114
}
111115
}
@@ -131,9 +135,11 @@ static auto getWorkflowTPCInput(o2::framework::ProcessingContext& pc, int verbos
131135
recvMask |= sectorHeader->sectorBits;
132136
retVal->internal.inputrefs[sector].data = ref;
133137
if (do_digits) {
134-
retVal->inputDigits[sector] = pc.inputs().get<gsl::span<o2::tpc::Digit>>(ref);
135-
if (verbosity >= 1) {
136-
LOG(INFO) << "GOT DIGITS SPAN FOR SECTOR " << sector << " -> " << retVal->inputDigits[sector].size();
138+
if (tpcSectorMask & (1ul << sector)) {
139+
retVal->inputDigits[sector] = pc.inputs().get<gsl::span<o2::tpc::Digit>>(ref);
140+
if (verbosity >= 1) {
141+
LOG(INFO) << "GOT DIGITS SPAN FOR SECTOR " << sector << " -> " << retVal->inputDigits[sector].size();
142+
}
137143
}
138144
}
139145
}
@@ -149,6 +155,9 @@ static auto getWorkflowTPCInput(o2::framework::ProcessingContext& pc, int verbos
149155
// skip zero-length message
150156
continue;
151157
}
158+
if (!(tpcSectorMask & (1ul << sector))) {
159+
continue;
160+
}
152161
if (refentry.second.labels.header != nullptr && refentry.second.labels.payload != nullptr) {
153162
retVal->internal.mcInputs.emplace_back(o2::dataformats::ConstMCLabelContainerView(pc.inputs().get<gsl::span<char>>(refentry.second.labels)));
154163
}
@@ -162,7 +171,7 @@ static auto getWorkflowTPCInput(o2::framework::ProcessingContext& pc, int verbos
162171

163172
if (do_clusters) {
164173
memset(&retVal->clusterIndex, 0, sizeof(retVal->clusterIndex));
165-
ClusterNativeHelper::Reader::fillIndex(retVal->clusterIndex, retVal->internal.clusterBuffer, retVal->internal.clustersMCBuffer, retVal->internal.inputs, retVal->internal.mcInputs, [&tpcSectorMask](auto& index) { return tpcSectorMask & (1ul << index); });
174+
ClusterNativeHelper::Reader::fillIndex(retVal->clusterIndex, retVal->internal.clusterBuffer, retVal->internal.clustersMCBuffer, retVal->internal.inputs, retVal->internal.mcInputs, tpcSectorMask);
166175
}
167176

168177
return retVal;

DataFormats/Detectors/TPC/src/ClusterNativeHelper.cxx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ int ClusterNativeHelper::Reader::fillIndex(ClusterNativeAccess& clusterIndex, st
189189
}
190190
}
191191

192-
int result = fillIndex(clusterIndex, clusterBuffer, mcBuffer, clustersTPC, constMCLabelContainerViews, [](auto&) { return true; });
192+
int result = fillIndex(clusterIndex, clusterBuffer, mcBuffer, clustersTPC, constMCLabelContainerViews);
193193
return result;
194194
}
195195

Detectors/TPC/reconstruction/src/GPUCATracking.cxx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ int GPUCATracking::runTracking(GPUO2InterfaceIOPtrs* data, GPUInterfaceOutputs*
7171
const float zsThreshold = mTrackingCAO2Interface->getConfig().configReconstruction.tpcZSthreshold;
7272
const int maxContTimeBin = mTrackingCAO2Interface->getConfig().configGRP.continuousMaxTimeBin;
7373
for (int i = 0; i < Sector::MAXSECTOR; i++) {
74-
const auto& d = (*(data->o2Digits))[i];
74+
const auto& d = (*data->o2Digits)[i];
7575
if (zsThreshold > 0 && data->tpcZS == nullptr) {
7676
gpuDigits[i].reserve(d.size());
7777
}

Detectors/TPC/workflow/include/TPCWorkflow/ZSSpec.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ namespace tpc
1616
{
1717

1818
/// create a processor spec
19-
framework::DataProcessorSpec getZSEncoderSpec(std::vector<int> const& inputIds, bool zs12bit, float threshold, bool outRaw);
19+
framework::DataProcessorSpec getZSEncoderSpec(std::vector<int> const& tpcSectors, bool zs12bit, float threshold, bool outRaw);
2020

21-
framework::DataProcessorSpec getZStoDigitsSpec(std::vector<int> const& inputIds);
21+
framework::DataProcessorSpec getZStoDigitsSpec(std::vector<int> const& tpcSectors);
2222

2323
} // end namespace tpc
2424
} // end namespace o2

Detectors/TPC/workflow/src/CATrackerSpec.cxx

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,9 @@ DataProcessorSpec getCATrackerSpec(CompletionPolicyData* policyData, ca::Config
214214
config.configWorkflow.inputs.set(GPUDataTypes::InOutType::TPCCompressedClusters);
215215
config.configWorkflow.outputs.setBits(GPUDataTypes::InOutType::TPCClusters, true);
216216
config.configWorkflow.outputs.setBits(GPUDataTypes::InOutType::TPCCompressedClusters, false);
217+
if (processAttributes->tpcSectorMask != 0xFFFFFFFFF) {
218+
throw std::invalid_argument("Cannot run TPC decompression with a sector mask");
219+
}
217220
}
218221
if (specconfig.outputSharedClusterMap) {
219222
config.configProcessing.outputSharedClusterMap = true;
@@ -344,6 +347,14 @@ DataProcessorSpec getCATrackerSpec(CompletionPolicyData* policyData, ca::Config
344347
getWorkflowTPCInput_digits = true;
345348
}
346349

350+
if (specconfig.zsOnTheFly || specconfig.zsDecoder) {
351+
for (unsigned int i = 0; i < GPUTrackingInOutZS::NSLICES; i++) {
352+
for (unsigned int j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
353+
tpcZSmetaPointers[i][j].clear();
354+
tpcZSmetaSizes[i][j].clear();
355+
}
356+
}
357+
}
347358
if (specconfig.zsOnTheFly) {
348359
tpcZSonTheFlySizes = {0};
349360
// tpcZSonTheFlySizes: #zs pages per endpoint:
@@ -368,12 +379,6 @@ DataProcessorSpec getCATrackerSpec(CompletionPolicyData* policyData, ca::Config
368379
if (!recv || !recvsizes) {
369380
throw std::runtime_error("TPC ZS data not received");
370381
}
371-
for (unsigned int i = 0; i < GPUTrackingInOutZS::NSLICES; i++) {
372-
for (unsigned int j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
373-
tpcZSmetaPointers[i][j].clear();
374-
tpcZSmetaSizes[i][j].clear();
375-
}
376-
}
377382

378383
unsigned int offset = 0;
379384
for (unsigned int i = 0; i < NSectors; i++) {
@@ -388,12 +393,6 @@ DataProcessorSpec getCATrackerSpec(CompletionPolicyData* policyData, ca::Config
388393
}
389394
}
390395
if (specconfig.zsDecoder) {
391-
for (unsigned int i = 0; i < GPUTrackingInOutZS::NSLICES; i++) {
392-
for (unsigned int j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
393-
tpcZSmetaPointers[i][j].clear();
394-
tpcZSmetaSizes[i][j].clear();
395-
}
396-
}
397396
std::vector<InputSpec> filter = {{"check", ConcreteDataTypeMatcher{gDataOriginTPC, "RAWDATA"}, Lifetime::Timeframe}};
398397
for (auto const& ref : InputRecordWalker(pc.inputs(), filter)) {
399398
const DataHeader* dh = DataRefUtils::getHeader<DataHeader*>(ref);
@@ -593,6 +592,21 @@ DataProcessorSpec getCATrackerSpec(CompletionPolicyData* policyData, ca::Config
593592
tfSettings.hasTfStartOrbit = 1;
594593
ptrs.settingsTF = &tfSettings;
595594

595+
if (processAttributes->tpcSectorMask != 0xFFFFFFFFF) {
596+
// Clean out the unused sectors, such that if they were present by chance, they are not processed, and if the values are uninitialized, we should not crash
597+
for (int i = 0; i < NSectors; i++) {
598+
if (!(processAttributes->tpcSectorMask & (1ul << i))) {
599+
if (ptrs.tpcZS) {
600+
for (unsigned int j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) {
601+
tpcZS.slice[i].zsPtr[j] = nullptr;
602+
tpcZS.slice[i].nZSPtr[j] = nullptr;
603+
tpcZS.slice[i].count[j] = 0;
604+
}
605+
}
606+
}
607+
}
608+
}
609+
596610
int retVal = tracker->runTracking(&ptrs, &outputRegions);
597611
if (processAttributes->suppressOutput) {
598612
return;

Detectors/TPC/workflow/src/ZSSpec.cxx

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ namespace o2
5656
namespace tpc
5757
{
5858

59-
DataProcessorSpec getZSEncoderSpec(std::vector<int> const& inputIds, bool zs10bit, float threshold = 2.f, bool outRaw = false)
59+
DataProcessorSpec getZSEncoderSpec(std::vector<int> const& tpcSectors, bool zs10bit, float threshold = 2.f, bool outRaw = false)
6060
{
6161
std::string processorName = "tpc-zsEncoder";
6262
constexpr static size_t NSectors = o2::tpc::Sector::MAXSECTOR;
@@ -66,21 +66,25 @@ DataProcessorSpec getZSEncoderSpec(std::vector<int> const& inputIds, bool zs10bi
6666
struct ProcessAttributes {
6767
std::unique_ptr<unsigned long long int[]> zsoutput;
6868
std::vector<unsigned int> sizes;
69-
std::vector<int> inputIds;
69+
std::vector<int> tpcSectors;
7070
bool verify = false;
7171
int verbosity = 1;
7272
bool finished = false;
7373
};
7474

75-
auto initFunction = [inputIds, zs10bit, threshold, outRaw](InitContext& ic) {
75+
auto initFunction = [tpcSectors, zs10bit, threshold, outRaw](InitContext& ic) {
7676
auto processAttributes = std::make_shared<ProcessAttributes>();
7777
auto& zsoutput = processAttributes->zsoutput;
78-
processAttributes->inputIds = inputIds;
78+
processAttributes->tpcSectors = tpcSectors;
7979
auto& verify = processAttributes->verify;
8080
auto& sizes = processAttributes->sizes;
8181
auto& verbosity = processAttributes->verbosity;
82+
unsigned long tpcSectorMask = 0;
83+
for (auto s : tpcSectors) {
84+
tpcSectorMask |= (1ul << s);
85+
}
8286

83-
auto processingFct = [processAttributes, zs10bit, threshold, outRaw](
87+
auto processingFct = [processAttributes, zs10bit, threshold, outRaw, tpcSectorMask](
8488
ProcessingContext& pc) {
8589
if (processAttributes->finished) {
8690
return;
@@ -99,7 +103,7 @@ DataProcessorSpec getZSEncoderSpec(std::vector<int> const& inputIds, bool zs10bi
99103
config.ReadConfigurableParam();
100104
_GPUParam.SetDefaults(&config.configGRP, &config.configReconstruction, &config.configProcessing, nullptr);
101105

102-
const auto& inputs = getWorkflowTPCInput(pc, 0, false, false, 0xFFFFFFFFF, true);
106+
const auto& inputs = getWorkflowTPCInput(pc, 0, false, false, tpcSectorMask, true);
103107
sizes.resize(NSectors * NEndpoints);
104108
bool zs12bit = !zs10bit;
105109
o2::InteractionRecord ir = o2::raw::HBFUtils::Instance().getFirstIR();
@@ -164,16 +168,16 @@ DataProcessorSpec getZSEncoderSpec(std::vector<int> const& inputIds, bool zs10bi
164168
return processingFct;
165169
};
166170

167-
auto createInputSpecs = [inputIds]() {
171+
auto createInputSpecs = [tpcSectors]() {
168172
Inputs inputs;
169173
// inputs.emplace_back(InputSpec{"input", ConcreteDataTypeMatcher{gDataOriginTPC, "DIGITS"}, Lifetime::Timeframe});
170174
inputs.emplace_back(InputSpec{"input", gDataOriginTPC, "DIGITS", 0, Lifetime::Timeframe});
171-
return std::move(mergeInputs(inputs, inputIds.size(),
172-
[inputIds](InputSpec& input, size_t index) {
175+
return std::move(mergeInputs(inputs, tpcSectors.size(),
176+
[tpcSectors](InputSpec& input, size_t index) {
173177
// using unique input names for the moment but want to find
174178
// an input-multiplicity-agnostic way of processing
175-
input.binding += std::to_string(inputIds[index]);
176-
DataSpecUtils::updateMatchingSubspec(input, inputIds[index]);
179+
input.binding += std::to_string(tpcSectors[index]);
180+
DataSpecUtils::updateMatchingSubspec(input, tpcSectors[index]);
177181
}));
178182
return inputs;
179183
};
@@ -195,7 +199,7 @@ DataProcessorSpec getZSEncoderSpec(std::vector<int> const& inputIds, bool zs10bi
195199
AlgorithmSpec(initFunction)};
196200
} //spec end
197201

198-
DataProcessorSpec getZStoDigitsSpec(std::vector<int> const& inputIds)
202+
DataProcessorSpec getZStoDigitsSpec(std::vector<int> const& tpcSectors)
199203
{
200204
std::string processorName = "tpc-zs-to-Digits";
201205
constexpr static size_t NSectors = o2::tpc::Sector::MAXSECTOR;
@@ -206,7 +210,7 @@ DataProcessorSpec getZStoDigitsSpec(std::vector<int> const& inputIds)
206210
std::unique_ptr<unsigned long long int[]> zsinput;
207211
std::vector<unsigned int> sizes;
208212
std::unique_ptr<o2::tpc::ZeroSuppress> decoder;
209-
std::vector<int> inputIds;
213+
std::vector<int> tpcSectors;
210214
bool verify = false;
211215
int verbosity = 1;
212216
bool finished = false;
@@ -229,9 +233,9 @@ DataProcessorSpec getZStoDigitsSpec(std::vector<int> const& inputIds)
229233
}
230234
};
231235

232-
auto initFunction = [inputIds](InitContext& ic) {
236+
auto initFunction = [tpcSectors](InitContext& ic) {
233237
auto processAttributes = std::make_shared<ProcessAttributes>();
234-
processAttributes->inputIds = inputIds;
238+
processAttributes->tpcSectors = tpcSectors;
235239
auto& outDigits = processAttributes->outDigits;
236240
auto& decoder = processAttributes->decoder;
237241
decoder = std::make_unique<o2::tpc::ZeroSuppress>();

Detectors/TPC/workflow/src/tpc-reco-workflow.cxx

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ void customize(std::vector<o2::framework::ConfigParamSpec>& workflowOptions)
4949
{"output-type", VariantType::String, "tracks", {"digits, zsraw, clustershw, clustersnative, tracks, compressed-clusters, encoded-clusters, disable-writer, send-clusters-per-sector, qa, no-shared-cluster-map"}},
5050
{"no-ca-clusterer", VariantType::Bool, false, {"Use HardwareClusterer instead of clusterer of GPUCATracking"}},
5151
{"disable-mc", VariantType::Bool, false, {"disable sending of MC information"}},
52-
//{"tpc-sectors", VariantType::String, "0-35", {"TPC sector range, e.g. 5-7,8,9"}},
52+
{"tpc-sectors", VariantType::String, "0-35", {"TPC sector range, e.g. 5-7,8,9"}},
5353
{"tpc-lanes", VariantType::Int, 1, {"number of parallel lanes up to the tracker"}},
5454
{"dispatching-mode", VariantType::String, "prompt", {"determines when to dispatch: prompt, complete"}},
5555
{"no-tpc-zs-on-the-fly", VariantType::Bool, false, {"Do not use TPC zero suppression on the fly"}},
@@ -111,9 +111,7 @@ using namespace o2::framework;
111111
/// This function hooks up the the workflow specifications into the DPL driver.
112112
WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
113113
{
114-
//auto tpcSectors = o2::RangeTokenizer::tokenize<int>(cfgc.options().get<std::string>("tpc-sectors"));
115-
std::vector<int> tpcSectors(36);
116-
std::iota(tpcSectors.begin(), tpcSectors.end(), 0);
114+
std::vector<int> tpcSectors = o2::RangeTokenizer::tokenize<int>(cfgc.options().get<std::string>("tpc-sectors"));
117115
// the lane configuration defines the subspecification ids to be distributed among the lanes.
118116
std::vector<int> laneConfiguration = tpcSectors; // Currently just a copy of the tpcSectors, why?
119117
auto nLanes = cfgc.options().get<int>("tpc-lanes");

0 commit comments

Comments
 (0)