|
| 1 | +#!/usr/bin/env python3 |
| 2 | + |
| 3 | +# |
| 4 | +# A script producing the QC finalization workflow. |
| 5 | +# If run as main, it will dump the workflow to the specified output file and tasks will not have dependencies. |
| 6 | +# For example: |
| 7 | +# ${O2DPG_ROOT}/DATA/production/o2dpg_qc_pp_workflow.py -o qc_workflow.json |
| 8 | + |
| 9 | + |
| 10 | +# The script can be also imported. |
| 11 | +# In such case, one can use include_all_QC_finalization to get the QC finalization from within other workflow script. |
| 12 | + |
| 13 | +import sys |
| 14 | +import importlib.util |
| 15 | +import argparse |
| 16 | +from os import environ, mkdir |
| 17 | +from os.path import join, dirname, isdir |
| 18 | + |
| 19 | +# make sure O2DPG, O2 and QC are loaded |
| 20 | +O2DPG_ROOT=environ.get('O2DPG_ROOT') |
| 21 | +O2_ROOT=environ.get('O2_ROOT') |
| 22 | +QUALITYCONTROL_ROOT=environ.get('QUALITYCONTROL_ROOT') |
| 23 | + |
| 24 | +if O2DPG_ROOT == None: |
| 25 | + print('Error: This needs O2DPG loaded') |
| 26 | + sys.exit(1) |
| 27 | + |
| 28 | +if O2_ROOT == None: |
| 29 | + print('Error: This needs O2 loaded') |
| 30 | + sys.exit(1) |
| 31 | + |
| 32 | +if QUALITYCONTROL_ROOT is None: |
| 33 | + print('Error: This needs QUALITYCONTROL_ROOT loaded') |
| 34 | + sys.exit(1) |
| 35 | + |
| 36 | +# dynamically import required utilities |
| 37 | +module_name = "o2dpg_workflow_utils" |
| 38 | +spec = importlib.util.spec_from_file_location(module_name, join(O2DPG_ROOT, "MC", "bin", "o2dpg_workflow_utils.py")) |
| 39 | +o2dpg_workflow_utils = importlib.util.module_from_spec(spec) |
| 40 | +sys.modules[module_name] = o2dpg_workflow_utils |
| 41 | +spec.loader.exec_module(o2dpg_workflow_utils) |
| 42 | +from o2dpg_workflow_utils import createTask, dump_workflow |
| 43 | + |
| 44 | +def getDPL_global_options(bigshm=False, noIPC=None): |
| 45 | + common="-b --run --driver-client-backend ws:// " |
| 46 | + if noIPC != None: |
| 47 | + return common + " --no-IPC " |
| 48 | + if bigshm: |
| 49 | + return common + " --shm-segment-size ${SHMSIZE:-50000000000} " |
| 50 | + else: |
| 51 | + return common |
| 52 | + |
| 53 | +qcdir = "QC" |
| 54 | +def QC_postprocessing_workflow(runNumber, periodName, passName, qcdbUrl): |
| 55 | + |
| 56 | + stages = [] |
| 57 | + |
| 58 | + ## Adds a postprocessing QC workflow |
| 59 | + # taskName - name of the QC workflow |
| 60 | + # qcConfigPath - path to the QC config file |
| 61 | + # needs - a list of tasks to be finished before (e.g. if one post-processing workflow needs another to finish first) |
| 62 | + # runSpecific - if set as true, a concrete run number is put to the QC config, |
| 63 | + # thus the post-processing should cover objects only for this run |
| 64 | + # periodSpecific - if set as true, a concrete period name is put to the config, |
| 65 | + # thus the post-processing should cover objects only for this period |
| 66 | + # passSpecific - if set as true, a concrete pass name is put to the config, |
| 67 | + # thus the post-processing should cover objects only for this pass |
| 68 | + def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, periodSpecific, passSpecific): |
| 69 | + task = createTask(name=taskName, needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000') |
| 70 | + overrideValues = '--override-values "' |
| 71 | + overrideValues += f'qc.config.database.host={qcdbUrl};' |
| 72 | + overrideValues += f'qc.config.Activity.number={runNumber};' if runSpecific else 'qc.config.Activity.number=0;' |
| 73 | + overrideValues += f'qc.config.Activity.periodName={periodName};' if periodSpecific else 'qc.config.Activity.periodName=;' |
| 74 | + overrideValues += f'qc.config.Activity.passName={passName};' if passSpecific else 'qc.config.Activity.passName=;' |
| 75 | + overrideValues += '"' |
| 76 | + task['cmd'] = f'o2-qc --config {qcConfigPath} ' + overrideValues + ' ' + getDPL_global_options() |
| 77 | + stages.append(task) |
| 78 | + |
| 79 | + ## The list of QC Post-processing workflows, add the new ones below |
| 80 | + add_QC_postprocessing('example', 'json://${O2DPG_ROOT}/DATA/production/qc-pp-async/example.json', needs=[], runSpecific=False, periodSpecific=False, passSpecific=True) |
| 81 | + |
| 82 | + return stages |
| 83 | + |
| 84 | + |
| 85 | +def main() -> int: |
| 86 | + |
| 87 | + parser = argparse.ArgumentParser(description='Create the ALICE data QC postprocessing workflow') |
| 88 | + |
| 89 | + parser.add_argument('--noIPC',help='disable shared memory in DPL') |
| 90 | + parser.add_argument('-o',help='output workflow file', default='workflow.json') |
| 91 | + parser.add_argument('--run',help="Run number (0 for any", default=0) |
| 92 | + parser.add_argument('--periodName',help="Period name", default='') |
| 93 | + parser.add_argument('--passName',help="Pass name", default='') |
| 94 | + parser.add_argument('--qcdbUrl',help="Quality Control Database URL", default='ccdb-test.cern.ch:8080') |
| 95 | + |
| 96 | + args = parser.parse_args() |
| 97 | + print (args) |
| 98 | + |
| 99 | + if not isdir(qcdir): |
| 100 | + mkdir(qcdir) |
| 101 | + |
| 102 | + workflow={} |
| 103 | + workflow['stages'] = QC_postprocessing_workflow(runNumber=args.run, periodName=args.periodName, passName=args.passName, qcdbUrl=args.qcdbUrl) |
| 104 | + |
| 105 | + dump_workflow(workflow["stages"], args.o) |
| 106 | + |
| 107 | + return 0 |
| 108 | + |
| 109 | + |
| 110 | +if __name__ == '__main__': |
| 111 | + sys.exit(main()) |
0 commit comments