Skip to content

Commit 07a9e76

Browse files
knopers8chiarazampolli
knopers8
authored and
chiarazampolli
committed
[QC-779] Allow to run QC postprocessing at the end of MC, show how
1 parent 04d2155 commit 07a9e76

File tree

4 files changed

+125
-6
lines changed

4 files changed

+125
-6
lines changed

‎MC/README.md‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ Below are the steps to integrate a new QC Task to the main simulation, reco and
1111
aliBuild build O2 QualityControl O2Physics O2DPG --defaults o2 -j <jobs>
1212
```
1313

14-
2. Make sure that the setup works by loading the environment and running the example script.
14+
2. (Optional) Make sure that the setup works by loading the environment and running the example script.
1515
It runs a series of tasks, which are usually DPL workflows that depend on each other and write/read processing results in form of ROOT files.
1616
It will simulate 3 TimeFrames, reconstruct them and run any QC.
1717
Corresponding files will be created in the current directory, QC objects will be also uploaded to QCDB.
@@ -69,6 +69,29 @@ def include_all_QC_finalization(ntimeframes, standalone):
6969

7070
6. Delete the files generated by the workflow during step 2 and run the `O2DPG_pp_minbias_multiple_tf_qc.sh` script again.
7171
Verify that the QC Task succeeds.
72-
Log are available under task names in their working directories: tf<n> when processing TFs and QC during finalization.
72+
You can run only the parts of the workflow which are required to reach your QC task by adding `-tt <task_name>_finalize` to `o2_dpg_workflow_runner.py`.
73+
Logs are available under task names in their working directories: tf<n> when processing TFs and QC during finalization.
7374

7475
7. Ask Catalin to add the file with QC results to the list of merged files on Grid productions. The file has the same name as `taskName`, but with the `.root` suffix. If you update the task name, also please let Catalin know.
76+
77+
## Adding QC post-processing tasks to the simulation script
78+
79+
One can execute a QC post-processing workflow as the last stage of QC.
80+
To do that, please follow the points below.
81+
82+
1. Similarly to the points 1-3 in the previous chapter, verify that you can run the workflow before making changes.
83+
Prepare the QC config file, putting correct values in the `Activity` section.
84+
Pay attention to set applicable triggers in the post-processing task, see the [QC doc](https://github.com/AliceO2Group/QualityControl/blob/master/doc/PostProcessing.md#more-examples) for relevant examples.
85+
Most likely, you will need to use either `ForEachObject` or `ForEachLatest` triggers.
86+
87+
2. In `o2dpg_qc_finalization_workflow.py`, find the `include_all_QC_finalization` function.
88+
89+
Add your QC post-processing workflow there using the `add_QC_postprocessing` function.
90+
See its in-code documentation for argument explanation and follow the existing examples.
91+
Please make sure to set the correct `needs`, so the post-processing workflow is executed only when the required QC objects are already in the QCDB.
92+
93+
3. Delete the files generated by the workflow during the previous steps and run the `O2DPG_pp_minbias_multiple_tf_qc.sh` script again.
94+
Verify that the QC post-processing workflow succeeds.
95+
You can run only the parts of the workflow which are required to reach your QC task by adding `-tt <task_name>_finalize` to `o2_dpg_workflow_runner.py`.
96+
Relevant logs are available under task names in the `QC` directory.
97+

‎MC/bin/o2dpg_qc_finalization_workflow.py‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,27 +28,53 @@ def getDPL_global_options(bigshm=False, noIPC=None):
2828
else:
2929
return common
3030

31+
32+
def QC_finalize_name(name):
33+
return name + "_finalize"
34+
3135
qcdir = "QC"
3236
def include_all_QC_finalization(ntimeframes, standalone, run, productionTag):
3337

3438
stages = []
39+
40+
## Adds a 'remote-batch' part of standard QC workflows
41+
# taskName - name of the QC workflow, it should be the same as in the main workflow
42+
# qcConfigPath - path to the QC config file
43+
# needs - a list of tasks to be finished first. By default, the function puts the 'local-batch' part of the QC workflow
3544
def add_QC_finalization(taskName, qcConfigPath, needs=None):
3645
if standalone == True:
3746
needs = []
3847
elif needs == None:
3948
needs = [taskName + '_local' + str(tf) for tf in range(1, ntimeframes + 1)]
4049

41-
task = createTask(name=taskName + '_finalize', needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000')
50+
task = createTask(name=QC_finalize_name(taskName), needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000')
4251
task['cmd'] = f'o2-qc --config {qcConfigPath} --remote-batch {taskName}.root' + \
4352
f' --override-values "qc.config.Activity.number={run};qc.config.Activity.periodName={productionTag}"' + \
4453
' ' + getDPL_global_options()
4554
stages.append(task)
4655

47-
# to be enabled once MFT Digits should be ran 5 times with different settings
56+
## Adds a postprocessing QC workflow
57+
# taskName - name of the QC workflow
58+
# qcConfigPath - path to the QC config file
59+
# needs - a list of tasks to be finished first. Usually it should include QC finalization tasks
60+
# which produce objects needed for given post-processing
61+
# runSpecific - if set as true, a concrete run number is put to the QC config,
62+
# thus the post-processing should cover objects only for that run
63+
# prodSpecific - if set as true, a concrete production name is put to the config,
64+
# thus the post-processing should cover objects only for that production
65+
def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecific):
66+
task = createTask(name=taskName, needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000')
67+
overrideValues = '--override-values "'
68+
overrideValues += f'qc.config.Activity.number={run};' if runSpecific else 'qc.config.Activity.number=0;'
69+
overrideValues += f'qc.config.Activity.periodName={productionTag}"' if prodSpecific else 'qc.config.Activity.periodName="'
70+
task['cmd'] = f'o2-qc --config {qcConfigPath} ' + \
71+
overrideValues + ' ' + getDPL_global_options()
72+
stages.append(task)
73+
74+
## The list of remote-batch workflows (reading the merged QC tasks results, applying Checks, uploading them to QCDB)
4875
MFTDigitsQCneeds = []
4976
for flp in range(5):
5077
MFTDigitsQCneeds.extend(['mftDigitsQC'+str(flp)+'_local'+str(tf) for tf in range(1, ntimeframes + 1)])
51-
#
5278
add_QC_finalization('mftDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/qc-mft-digit-0.json', MFTDigitsQCneeds)
5379
add_QC_finalization('mftClustersQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/qc-mft-cluster.json')
5480
add_QC_finalization('mftAsyncQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/qc-mft-async.json')
@@ -65,6 +91,9 @@ def add_QC_finalization(taskName, qcConfigPath, needs=None):
6591
add_QC_finalization('tofft0PIDQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/pidft0tof.json')
6692
add_QC_finalization('tofPIDQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/pidtof.json')
6793
add_QC_finalization('RecPointsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/ft0-reconstruction-config.json')
94+
95+
# The list of QC Post-processing workflows
96+
add_QC_postprocessing('tofTrendingHits', 'json://${O2DPG_ROOT}/MC/config/QC/json/tof-trending-hits.json', [QC_finalize_name('tofDigitsQC')], runSpecific=False, prodSpecific=True)
6897

6998
return stages
7099

‎MC/config/QC/json/tof-trending-hits.json‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
"qc": {
3+
"config": {
4+
"database": {
5+
"implementation": "CCDB",
6+
"host": "ccdb-test.cern.ch:8080",
7+
"username": "not_applicable",
8+
"password": "not_applicable",
9+
"name": "not_applicable"
10+
},
11+
"Activity": {
12+
"number": "",
13+
"passName": "passMC",
14+
"periodName": "",
15+
"provenance" : "qc_mc"
16+
},
17+
"monitoring": {
18+
"url": "infologger:///debug?qc"
19+
},
20+
"consul": {
21+
"url": ""
22+
},
23+
"conditionDB": {
24+
"url": "ccdb-test.cern.ch:8080"
25+
}
26+
},
27+
"postprocessing": {
28+
"TOFTrendingHits": {
29+
"active": "true",
30+
"className": "o2::quality_control_modules::tof::TrendingHits",
31+
"moduleName": "QcTOF",
32+
"detectorName": "TOF",
33+
"dataSources": [
34+
{
35+
"type": "repository",
36+
"path": "TOF/MO/TaskDigits",
37+
"names": [
38+
"TOFRawsMulti"
39+
],
40+
"reductorName": "o2::quality_control_modules::common::TH1Reductor",
41+
"moduleName": "QcCommon"
42+
}
43+
],
44+
"plots": [
45+
{
46+
"name": "mean_of_hits",
47+
"title": "Mean trend of TOF hits",
48+
"varexp": "TOFRawsMulti.mean:time",
49+
"selection": "",
50+
"option": "*L"
51+
}
52+
],
53+
"initTrigger": [
54+
"userorcontrol"
55+
],
56+
"updateTrigger": [
57+
"foreachlatest:qcdb:TOF/MO/TaskDigits/TOFRawsMulti"
58+
],
59+
"stopTrigger": [
60+
"userorcontrol"
61+
]
62+
}
63+
}
64+
}
65+
}
66+

‎MC/config/QC/json/tofdigits.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010
},
1111
"Activity": {
1212
"number": "42",
13-
"type": "2"
13+
"type": "2",
14+
"provenance" : "qc_mc"
1415
},
1516
"monitoring": {
1617
"url": "infologger:///debug?qc"

0 commit comments

Comments
 (0)