Skip to content

Commit 89680ba

Browse files
knopers8sawenzel
knopers8
and
sawenzel
authored
Initial integration with QC Tasks (#79)
* First dirty try with vertex QC Task * Initial integration with QC Tasks This adds 5 QC Tasks, which can be enabled with --include-qc flag passed to o2dpg_sim_workflow.py. In this iteration they publish results directly to QCDB (at ccdb-test) and results from many TFs and sub-jobs are not merged. More Tasks to come, also merging will be done in future PRs. Co-authored-by: Sandro Wenzel <sandro.wenzel@cern.ch>
1 parent f07de84 commit 89680ba

File tree

10 files changed

+567
-0
lines changed

10 files changed

+567
-0
lines changed

‎MC/bin/o2dpg_sim_workflow.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@
7575
parser.add_argument('--combine-smaller-digi', action='store_true', help=argparse.SUPPRESS)
7676
parser.add_argument('--combine-tpc-clusterization', action='store_true', help=argparse.SUPPRESS) #<--- useful for small productions (pp, low interaction rate, small number of events)
7777

78+
# QC related arguments
79+
parser.add_argument('--include-qc', action='store_true', help='a flag to include QC in the workflow')
80+
7881
args = parser.parse_args()
7982
print (args)
8083

@@ -115,6 +118,7 @@ def getDPL_global_options(bigshm=False,nosmallrate=False):
115118

116119
doembedding=True if args.embedding=='True' or args.embedding==True else False
117120
usebkgcache=args.use_bkg_from!=None
121+
includeQC=True if args.include_qc=='True' or args.include_qc==True else False
118122

119123
if doembedding:
120124
if not usebkgcache:
@@ -500,6 +504,52 @@ def createRestDigiTask(name, det='ALLSMALLER'):
500504
PVFINDERtask['cmd'] = 'o2-primary-vertexing-workflow ' + getDPL_global_options(nosmallrate=False)
501505
# PVFINDERtask['cmd'] += ' --vertexing-sources "ITS,ITS-TPC,ITS-TPC-TOF" --vetex-track-matching-sources "ITS,ITS-TPC,ITS-TPC-TOF"'
502506
workflow['stages'].append(PVFINDERtask)
507+
508+
if includeQC:
509+
510+
### ITS
511+
# fixme: not working yet, ITS will prepare a way to read clusters and tracks. Also ITSDictionary will be needed.
512+
# ITSClustersTracksQCneeds = [ITSRECOtask['name']]
513+
# ITSClustersTracksQCtask = createTask(name='itsClustersTracksQC_'+str(tf), needs=ITSClustersTracksQCneeds, tf=tf, cwd=timeframeworkdir, lab=["QC"], cpu=1, mem='2000')
514+
# ITSClustersTracksQCtask['cmd'] = 'o2-missing-reader | o2-qc --config json://${O2DPG_ROOT}/MC/config/QC/json/its-clusters-tracks-qc.json ' + getDPL_global_options(nosmallrate=False)
515+
# workflow['stages'].append(ITSClustersTracksQCtask)
516+
517+
### MFT
518+
# fixme: there is a bug in Check which causes a segfault, uncomment when the fix is merged
519+
# MFTDigitsQCneeds = [det_to_digitask["MFT"]['name']]
520+
# MFTDigitsQCtask = createTask(name='mftDigitsQC_'+str(tf), needs=MFTDigitsQCneeds, tf=tf, cwd=timeframeworkdir, lab=["QC"], cpu=1, mem='2000')
521+
# MFTDigitsQCtask['cmd'] = 'o2-qc-mft-digits-root-file-reader --mft-digit-infile=mftdigits.root | o2-qc --config json://${O2DPG_ROOT}/MC/config/QC/json/qc-mft-digit.json ' + getDPL_global_options(nosmallrate=False)
522+
# workflow['stages'].append(MFTDigitsQCtask)
523+
524+
MFTClustersQCneeds = [MFTRECOtask['name']]
525+
MFTClustersQCtask = createTask(name='mftClustersQC_'+str(tf), needs=MFTClustersQCneeds, tf=tf, cwd=timeframeworkdir, lab=["QC"], cpu=1, mem='2000')
526+
MFTClustersQCtask['cmd'] = 'o2-qc-mft-clusters-root-file-reader --mft-cluster-infile=mftclusters.root | o2-qc --config json://${O2DPG_ROOT}/MC/config/QC/json/qc-mft-cluster.json ' + getDPL_global_options(nosmallrate=False)
527+
workflow['stages'].append(MFTClustersQCtask)
528+
529+
MFTTracksQCneeds = [MFTRECOtask['name']]
530+
MFTTracksQCtask = createTask(name='mftTracksQC_'+str(tf), needs=MFTTracksQCneeds, tf=tf, cwd=timeframeworkdir, lab=["QC"], cpu=1, mem='2000')
531+
MFTTracksQCtask['cmd'] = 'o2-qc-mft-tracks-root-file-reader --mft-track-infile=mfttracks.root | o2-qc --config json://${O2DPG_ROOT}/MC/config/QC/json/qc-mft-track.json ' + getDPL_global_options(nosmallrate=False)
532+
workflow['stages'].append(MFTTracksQCtask)
533+
534+
### TPC
535+
TPCTrackingQCneeds = [TPCRECOtask['name']]
536+
TPCTrackingQCtask = createTask(name='tpcTrackingQC_'+str(tf), needs=TPCTrackingQCneeds, tf=tf, cwd=timeframeworkdir, lab=["QC"], cpu=2, mem='2000')
537+
TPCTrackingQCtask['cmd'] = 'o2-tpc-track-reader | o2-tpc-reco-workflow --input-type clusters --infile tpc-native-clusters.root --output-type disable-writer | o2-qc --config json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-tracking-direct.json ' + getDPL_global_options(nosmallrate=False)
538+
workflow['stages'].append(TPCTrackingQCtask)
539+
540+
### TRD
541+
TRDDigitsQCneeds = [TRDDigitask['name']]
542+
TRDDigitsQCtask = createTask(name='trdDigitsQC_'+str(tf), needs=TRDDigitsQCneeds, tf=tf, cwd=timeframeworkdir, lab=["QC"], cpu=1, mem='2000')
543+
TRDDigitsQCtask['cmd'] = 'o2-trd-trap-sim | o2-qc --config json://${O2DPG_ROOT}/MC/config/QC/json/trd-digits-task.json ' + getDPL_global_options(nosmallrate=False)
544+
workflow['stages'].append(TRDDigitsQCtask)
545+
546+
### RECO
547+
vertexQCneeds = [PVFINDERtask['name']]
548+
vertexQCtask = createTask(name='vertexQC_'+str(tf), needs=vertexQCneeds, tf=tf, cwd=timeframeworkdir, lab=["QC"], cpu=1, mem='2000')
549+
vertexQCtask['cmd'] = 'o2-primary-vertex-reader-workflow | o2-qc --config json://${O2DPG_ROOT}/MC/config/QC/json/vertexing-qc-direct-mc.json ' + getDPL_global_options(nosmallrate=False)
550+
workflow['stages'].append(vertexQCtask)
551+
552+
503553

504554
#secondary vertexer
505555
SVFINDERtask = createTask(name='svfinder_'+str(tf), needs=[PVFINDERtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu=1, mem='2000')

‎MC/config/QC/json/dpl-config.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"internal-dpl-clock": "",
3+
"digits-root-file-reader-mft": {
4+
"mft-digit-infile": "mftdigits2.root",
5+
"orbit-offset-enumeration": "0",
6+
"orbit-multiplier-enumeration": "0",
7+
"start-value-enumeration": "0",
8+
"end-value-enumeration": "-1",
9+
"step-value-enumeration": "1"
10+
},
11+
"Dispatcher": {
12+
"period-timer-stats": "10000000"
13+
},
14+
"QC-TASK-RUNNER-QcMFTDigitTask": {
15+
"period-timer-cycle": "60000000"
16+
},
17+
"QC-CHECK-RUNNER-QcMFTDigitCheck": "",
18+
"internal-dpl-injected-dummy-sink": ""
19+
}

‎MC/config/QC/json/its-clusters-tracks-qc.json‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
{
2+
"qc": {
3+
"config": {
4+
"database": {
5+
"implementation": "CCDB",
6+
"host": "ccdb-test.cern.ch:8080",
7+
"username": "not_applicable",
8+
"password": "not_applicable",
9+
"name": "not_applicable"
10+
},
11+
"Activity": {
12+
"number": "42",
13+
"type": "2"
14+
},
15+
"monitoring": {
16+
"url": "infologger:///debug?qc"
17+
},
18+
"consul": {
19+
"url": "http://consul-test.cern.ch:8500"
20+
},
21+
"conditionDB": {
22+
"url": "ccdb-test.cern.ch:8080"
23+
}
24+
},
25+
"tasks": {
26+
"ITSClusterTask": {
27+
"active": "true",
28+
"className": "o2::quality_control_modules::its::ITSClusterTask",
29+
"moduleName": "QcITS",
30+
"detectorName": "ITS",
31+
"cycleDurationSeconds": "180",
32+
"maxNumberCycles": "-1",
33+
"dataSource_comment": "The other type of dataSource is \"direct\", see basic-no-sampling.json.",
34+
"dataSource": {
35+
"type": "direct",
36+
"query": "compclus:ITS/COMPCLUSTERS/0;clustersrof:ITS/CLUSTERSROF/0"
37+
},
38+
"location": "remote",
39+
"taskParameters": {
40+
"layer": "1111111",
41+
"clusterDictionaryPath": "/home/epn/odc/files/ITSdictionary.bin",
42+
"runNumberPath": "/home/its/QC/workdir/infiles/RunNumber.dat",
43+
"geomPath": "./o2sim_geometry.root",
44+
"nThreads": "4"
45+
}
46+
47+
},
48+
"ITSTrackTask": {
49+
"active": "true",
50+
"className": "o2::quality_control_modules::its::ITSTrackTask",
51+
"moduleName": "QcITS",
52+
"detectorName": "ITS",
53+
"cycleDurationSeconds": "30",
54+
"maxNumberCycles": "-1",
55+
"dataSource_comment": "The other type of dataSource is \"direct\", see basic-no-sampling.json.",
56+
"dataSource": {
57+
"type": "direct",
58+
"query": "tracks:ITS/TRACKS/0;rofs:ITS/ITSTrackROF/0;compclus:ITS/COMPCLUSTERS/0"
59+
},
60+
"location": "remote"
61+
}
62+
63+
},
64+
"checks": {
65+
"ITSClusterCheck": {
66+
"active": "true",
67+
"className": "o2::quality_control_modules::its::ITSClusterCheck",
68+
"moduleName": "QcITS",
69+
"policy": "OnAny",
70+
"detectorName": "ITS",
71+
"dataSource": [{
72+
"type": "Task",
73+
"name": "ITSClusterTask",
74+
"MOs": ["Layer0/AverageClusterSize"]
75+
}]
76+
},
77+
"ITSTrackCheck": {
78+
"active": "true",
79+
"className": "o2::quality_control_modules::its::ITSTrackCheck",
80+
"moduleName": "QcITS",
81+
"policy": "OnAny",
82+
"detectorName": "ITS",
83+
"dataSource": [{
84+
"type": "Task",
85+
"name": "ITSTrackTask",
86+
"MOs": ["NClusters"]
87+
}]
88+
}
89+
}
90+
},
91+
"dataSamplingPolicies": [
92+
]
93+
}

‎MC/config/QC/json/qc-mft-cluster.json‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
{
2+
"qc": {
3+
"config": {
4+
"database": {
5+
"implementation": "CCDB",
6+
"host": "ccdb-test.cern.ch:8080",
7+
"username": "not_applicable",
8+
"password": "not_applicable",
9+
"name": "not_applicable"
10+
},
11+
"Activity": {
12+
"number": "42",
13+
"type": "2"
14+
},
15+
"monitoring": {
16+
"url": "infologger:///debug?qc"
17+
},
18+
"consul": {
19+
"url": "http://consul-test.cern.ch:8500"
20+
},
21+
"conditionDB": {
22+
"url": "ccdb-test.cern.ch:8080"
23+
}
24+
},
25+
"tasks": {
26+
"QcMFTClusterTask": {
27+
"active": "true",
28+
"className": "o2::quality_control_modules::mft::QcMFTClusterTask",
29+
"moduleName": "QcMFT",
30+
"detectorName": "MFT",
31+
"cycleDurationSeconds": "60",
32+
"maxNumberCycles": "-1",
33+
"dataSource_comment": "The other type of dataSource is \"direct\", see basic-no-sampling.json.",
34+
"dataSource": {
35+
"type": "direct",
36+
"query": "randomcluster:MFT/COMPCLUSTERS/0;clustersrof:MFT/CLUSTERSROF/0"
37+
},
38+
"taskParameters": {
39+
"myOwnKey": "myOwnValue"
40+
},
41+
"location": "remote"
42+
}
43+
},
44+
"checks": {
45+
"QcMFTClusterCheck": {
46+
"active": "true",
47+
"dataSource": [{
48+
"type": "Task",
49+
"name": "QcMFTClusterTask",
50+
"MOs": ["mMFTClusterSensorIndex"]
51+
}],
52+
"className": "o2::quality_control_modules::mft::QcMFTClusterCheck",
53+
"moduleName": "QcMFT",
54+
"detectorName": "MFT",
55+
"policy": "OnAny"
56+
}
57+
}
58+
},
59+
"dataSamplingPolicies": [
60+
]
61+
}

‎MC/config/QC/json/qc-mft-digit.json‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
{
2+
"qc": {
3+
"config": {
4+
"database": {
5+
"implementation": "CCDB",
6+
"host": "ccdb-test.cern.ch:8080",
7+
"username": "not_applicable",
8+
"password": "not_applicable",
9+
"name": "not_applicable"
10+
},
11+
"Activity": {
12+
"number": "42",
13+
"type": "2"
14+
},
15+
"monitoring": {
16+
"url": "infologger:///debug?qc"
17+
},
18+
"consul": {
19+
"url": "http://consul-test.cern.ch:8500"
20+
},
21+
"conditionDB": {
22+
"url": "ccdb-test.cern.ch:8080"
23+
}
24+
},
25+
"tasks": {
26+
"QcMFTDigitTask": {
27+
"active": "true",
28+
"className": "o2::quality_control_modules::mft::QcMFTDigitTask",
29+
"moduleName": "QcMFT",
30+
"detectorName": "MFT",
31+
"cycleDurationSeconds": "60",
32+
"maxNumberCycles": "-1",
33+
"dataSource_comment": "The other type of dataSource is \"direct\", see basic-no-sampling.json.",
34+
"dataSource": {
35+
"type": "direct",
36+
"query": "randomdigit:MFT/DIGITS/0;digitsrof:MFT/DIGITSROF/0"
37+
},
38+
"taskParameters": {
39+
"FLP": "0",
40+
"TaskLevel": "4"
41+
},
42+
"location": "remote"
43+
}
44+
},
45+
"checks": {
46+
"QcMFTDigitCheck": {
47+
"active": "true",
48+
"className": "o2::quality_control_modules::mft::QcMFTDigitCheck",
49+
"moduleName": "QcMFT",
50+
"detectorName": "MFT",
51+
"policy": "OnEachSeparately",
52+
"dataSource": [{
53+
"type": "Task",
54+
"name": "QcMFTDigitTask"
55+
}]
56+
}
57+
}
58+
},
59+
"dataSamplingPolicies": [
60+
]
61+
}

‎MC/config/QC/json/qc-mft-track.json‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
{
2+
"qc": {
3+
"config": {
4+
"database": {
5+
"implementation": "CCDB",
6+
"host": "ccdb-test.cern.ch:8080",
7+
"username": "not_applicable",
8+
"password": "not_applicable",
9+
"name": "not_applicable"
10+
},
11+
"Activity": {
12+
"number": "42",
13+
"type": "2"
14+
},
15+
"monitoring": {
16+
"url": "infologger:///debug?qc"
17+
},
18+
"consul": {
19+
"url": "http://consul-test.cern.ch:8500"
20+
},
21+
"conditionDB": {
22+
"url": "ccdb-test.cern.ch:8080"
23+
}
24+
},
25+
"tasks": {
26+
"QcMFTTrackTask": {
27+
"active": "true",
28+
"className": "o2::quality_control_modules::mft::QcMFTTrackTask",
29+
"moduleName": "QcMFT",
30+
"detectorName": "MFT",
31+
"cycleDurationSeconds": "60",
32+
"maxNumberCycles": "-1",
33+
"dataSource_comment": "The other type of dataSource is \"direct\", see basic-no-sampling.json.",
34+
"dataSource": {
35+
"type": "direct",
36+
"query": "randomtrack:MFT/TRACKS/0;tracksrof:MFT/MFTTrackROF/0"
37+
},
38+
"taskParameters": {
39+
"myOwnKey": "myOwnValue"
40+
},
41+
"location": "remote"
42+
}
43+
},
44+
"checks": {
45+
"QcMFTTrackCheck": {
46+
"active": "false",
47+
"dataSource": [{
48+
"type": "Task",
49+
"name": "QcMFTTrackTask",
50+
"MOs": ["mMFTTrackCharge"]
51+
}],
52+
"className": "o2::quality_control_modules::mft::QcMFTTrackCheck",
53+
"moduleName": "QcMFT",
54+
"detectorName": "MFT",
55+
"policy": "OnAny"
56+
}
57+
}
58+
},
59+
"dataSamplingPolicies": [
60+
]
61+
}

0 commit comments

Comments
 (0)