Skip to content

Commit 33bf20a

Browse files
knopers8knopers8
authored
Modernize the advanced QC workflow example (#2585)
Now it includes also Aggregators, PostProcessings and Checks on PostProcessing results. It represents a typical recent P2 setup of a detector. It can be used by developers to manually try out changes in the framework on a more complex example.
1 parent 9d7f6ec commit 33bf20a

File tree

3 files changed

+228
-34
lines changed

3 files changed

+228
-34
lines changed

‎Framework/advanced.json‎

Lines changed: 221 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -27,84 +27,273 @@
2727
}
2828
},
2929
"tasks": {
30-
"dataSizeTask": {
30+
"AdvTaskA": {
3131
"active": "true",
3232
"className": "o2::quality_control_modules::skeleton::SkeletonTask",
3333
"moduleName": "QcSkeleton",
3434
"detectorName": "TST",
3535
"cycleDurationSeconds": "10",
3636
"dataSource": {
3737
"type": "dataSamplingPolicy",
38-
"name": "tst2"
38+
"name": "tst"
3939
}
4040
},
41-
"someNumbersTask": {
41+
"AdvTaskB": {
4242
"active": "true",
4343
"className": "o2::quality_control_modules::skeleton::SkeletonTask",
4444
"moduleName": "QcSkeleton",
4545
"detectorName": "TST",
46-
"cycleDurationSeconds": "10",
46+
"cycleDurationSeconds": "25",
4747
"dataSource": {
48-
"type": "dataSamplingPolicy",
49-
"name": "tst1"
48+
"type": "direct",
49+
"query": "data:TST/SUM"
5050
}
5151
}
5252
},
5353
"checks": {
54-
"dataSizeCheck": {
54+
"AdvCheckA1": {
5555
"active": "true",
5656
"className": "o2::quality_control_modules::skeleton::SkeletonCheck",
5757
"moduleName": "QcSkeleton",
5858
"policy": "OnAny",
5959
"detectorName": "TST",
6060
"dataSource": [{
6161
"type": "Task",
62-
"name": "dataSizeTask",
62+
"name": "AdvTaskA",
6363
"MOs": ["example"]
6464
}]
6565
},
66-
"someNumbersCheck": {
66+
"AdvCheckA2": {
6767
"active": "true",
6868
"className": "o2::quality_control_modules::skeleton::SkeletonCheck",
6969
"moduleName": "QcSkeleton",
70-
"policy": "OnAny",
70+
"policy": "OnAll",
7171
"detectorName": "TST",
7272
"dataSource": [{
7373
"type": "Task",
74-
"name": "someNumbersTask",
75-
"MOs": ["example"]
74+
"name": "AdvTaskA",
75+
"MOs": ["example", "example2"]
76+
}]
77+
},
78+
"AdvCheckB": {
79+
"active": "true",
80+
"className": "o2::quality_control_modules::common::TrendCheck",
81+
"moduleName": "QualityControl",
82+
"detectorName": "TST",
83+
"policy": "OnAll",
84+
"extendedCheckParameters": {
85+
"default": {
86+
"default": {
87+
"trendCheckMode": "DeviationFromMean",
88+
"nPointsForAverage": "3",
89+
"thresholdsBad": "-0.02,0.02",
90+
"thresholdsMedium": "-0.005,0.005"
91+
}
92+
}
93+
},
94+
"dataSource": [
95+
{
96+
"type": "PostProcessing",
97+
"name": "AdvTrendB",
98+
"MOs" : [
99+
"mean_of_histogram"
100+
]
101+
}
102+
]
103+
}
104+
},
105+
"aggregators": {
106+
"AdvAggregatorA": {
107+
"active": "true",
108+
"className": "o2::quality_control_modules::common::WorstOfAllAggregator",
109+
"moduleName": "QcCommon",
110+
"policy": "OnAny",
111+
"detectorName": "TST",
112+
"dataSource": [{
113+
"type": "Check",
114+
"name": "AdvCheckA1"
115+
}, {
116+
"type": "Check",
117+
"name": "AdvCheckA2"
118+
}]
119+
},
120+
"AdvAggregatorB": {
121+
"active": "true",
122+
"className": "o2::quality_control_modules::common::WorstOfAllAggregator",
123+
"moduleName": "QcCommon",
124+
"policy": "OnAll",
125+
"detectorName": "TST",
126+
"dataSource": [{
127+
"type": "Check",
128+
"name": "AdvCheckB"
129+
}]
130+
},
131+
"AdvAggregatorC": {
132+
"active": "true",
133+
"className": "o2::quality_control_modules::common::WorstOfAllAggregator",
134+
"moduleName": "QcCommon",
135+
"policy": "OnAll",
136+
"detectorName": "TST",
137+
"dataSource": [{
138+
"type": "Aggregator",
139+
"name": "AdvAggregatorA",
140+
"QOs": []
141+
}, {
142+
"type": "Aggregator",
143+
"name": "AdvAggregatorB",
144+
"QOs": []
76145
}]
77146
}
147+
148+
},
149+
"postprocessing": {
150+
"AdvTrendB": {
151+
"active": "true",
152+
"className": "o2::quality_control::postprocessing::TrendingTask",
153+
"moduleName": "QualityControl",
154+
"detectorName": "TST",
155+
"dataSources": [
156+
{
157+
"type": "repository",
158+
"path": "TST/MO/AdvTaskB",
159+
"names": [ "example" ],
160+
"reductorName": "o2::quality_control_modules::common::TH1Reductor",
161+
"moduleName": "QcCommon"
162+
}
163+
],
164+
"plots": [
165+
{
166+
"name": "mean_of_histogram",
167+
"title": "Mean trend of the example histogram",
168+
"graphAxisLabel": "Mean X:time",
169+
"graphYRange": "0:10000",
170+
"graphs" : [
171+
{
172+
"name": "mean_trend",
173+
"title": "mean trend",
174+
"varexp": "example.mean:time",
175+
"selection": "",
176+
"option": "*L PLC PMC"
177+
}
178+
]
179+
}
180+
],
181+
"initTrigger": [
182+
"userorcontrol"
183+
],
184+
"updateTrigger": [
185+
"newobject:qcdb:TST/MO/AdvTaskB/example"
186+
],
187+
"stopTrigger": [
188+
"userorcontrol"
189+
]
190+
},
191+
"AdvTrendA": {
192+
"active": "true",
193+
"className": "o2::quality_control::postprocessing::TrendingTask",
194+
"moduleName": "QualityControl",
195+
"detectorName": "TST",
196+
"dataSources": [
197+
{
198+
"type": "repository-quality",
199+
"path": "TST/QO",
200+
"names": [ "AdvCheckA1" ],
201+
"reductorName": "o2::quality_control_modules::common::QualityReductor",
202+
"moduleName": "QcCommon"
203+
}
204+
],
205+
"plots": [
206+
{
207+
"name": "example_quality",
208+
"title": "Trend of the example histogram's quality",
209+
"graphs" : [{
210+
"varexp": "AdvCheckA1.name:time",
211+
"selection": "",
212+
"option": "*"
213+
}]
214+
}
215+
],
216+
"initTrigger": [
217+
"userorcontrol"
218+
],
219+
"updateTrigger": [
220+
"newobject:qcdb:TST/QO/AdvCheckA1"
221+
],
222+
"stopTrigger": [
223+
"userorcontrol"
224+
]
225+
},
226+
"AdvQualityTask": {
227+
"active": "true",
228+
"className": "o2::quality_control_modules::common::QualityTask",
229+
"moduleName": "QualityControl",
230+
"detectorName": "TST",
231+
"qualityGroups": [
232+
{
233+
"name" : "global",
234+
"title" : "Advanced example: Aggregators",
235+
"path": "TST/QO",
236+
"ignoreQualitiesDetails" : [],
237+
"inputObjects": [
238+
{
239+
"name" : "AdvAggregatorC/AdvAggregatorC",
240+
"title" : "Adv. Aggregator C (total)"
241+
},
242+
{
243+
"name" : "AdvAggregatorA/AdvAggregatorA",
244+
"title" : "Adv. Aggregator A"
245+
},
246+
{
247+
"name" : "AdvAggregatorB/AdvAggregatorB",
248+
"title" : "Adv. Aggregator B"
249+
}
250+
]
251+
},
252+
{
253+
"name" : "details",
254+
"title" : "Advanced example: Checks",
255+
"path": "TST/QO",
256+
"ignoreQualitiesDetails" : [],
257+
"inputObjects": [
258+
{
259+
"name" : "AdvCheckA1",
260+
"title" : ""
261+
},
262+
{
263+
"name" : "AdvCheckA2",
264+
"title" : ""
265+
},
266+
{
267+
"name" : "AdvCheckB",
268+
"title" : ""
269+
}
270+
]
271+
}
272+
],
273+
"initTrigger": [
274+
"userorcontrol"
275+
],
276+
"updateTrigger": [
277+
"newobject:qcdb:TST/QO/AdvAggregatorC/AdvAggregatorC"
278+
],
279+
"stopTrigger": [
280+
"userorcontrol"
281+
]
282+
}
78283
}
79284
},
80285
"dataSamplingPolicies": [
81286
{
82-
"id": "tst1",
287+
"id": "tst",
83288
"active": "true",
84-
"machines": [],
85-
"query" : "data:TST/SUM/2;param:TST/PARAM/2",
289+
"query" : "data:TST/DATA/1",
86290
"samplingConditions": [
87291
{
88292
"condition": "random",
89293
"fraction": "0.1",
90-
"seed": "32112332123"
91-
}
92-
],
93-
"blocking": "false"
94-
},
95-
{
96-
"id": "tst2",
97-
"active": "true",
98-
"machines": [],
99-
"query" : "data:TST/DATA",
100-
"samplingConditions": [
101-
{
102-
"condition": "payloadSize",
103-
"lowerLimit": "8000",
104-
"upperLimit": "10000"
294+
"seed": "0"
105295
}
106-
],
107-
"blocking": "false"
296+
]
108297
}
109298
]
110299
}

‎doc/ModulesDevelopment.md‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ Data Sampling is used by Quality Control to feed the tasks with data. Below we p
8686
}
8787
```
8888

89-
An example of using the data sampling in a DPL workflow is visible in [runAdvanced.cxx](https://github.com/AliceO2Group/QualityControl/blob/master/Framework/src/runAdvanced.cxx).
89+
QC framework takes care of creating the necessary Data Sampling infrastructure when it finds it in the configuration file.
9090

9191
#### Custom Data Sampling Condition
9292

@@ -521,16 +521,21 @@ The way we compute the run number is done in this order:
521521

522522
## A more advanced example
523523

524-
A more complete example is available. The config file is called [advanced.json](../Framework/advanced.json). The workflow is made of 3 sources, intermediate processing steps, 3 sinks and a Dispatcher connecting two QC tasks to a number of these steps and 2 checks. The topology doesn't mean to represent any particular physics processing, it is just an example with multiple data processors.
524+
A more complete example is available. The config file is called [advanced.json](../Framework/advanced.json).
525+
The workflow is made of 3 sources, intermediate processing steps, 3 sinks and a Dispatcher connecting two QC tasks to selected data outputs.
526+
It also includes a few Checks, Aggregators and PostProcessing tasks.
527+
The topology is not intended to represent any particular physics processing, it is just an example with multiple data processors.
525528

526529
![alt text](images/advanced.png)
527530

528531
To run it do either
529532
```
533+
# runs a fake processing workflow and associated QC
530534
o2-qc-run-advanced
531535
```
532536
or
533537
```
538+
# o2-qc-run-advanced creates a fake processing workflow, while the QC is spawned by o2-qc
534539
o2-qc-run-advanced --no-qc | o2-qc --config json://${QUALITYCONTROL_ROOT}/etc/advanced.json
535540
```
536541

‎doc/images/advanced.png‎

-152 KB
Loading

0 commit comments

Comments
 (0)