Merge branch 'AliceO2Group:master' into master

Author: FDUEnrich

DATA/common/setenv.sh

@@ -195,6 +195,7 @@ TRD_SOURCES=
 TOF_SOURCES=
 HMP_SOURCES=
 TRACK_SOURCES=
+: ${TRACK_SOURCES_GLO:=}
 has_detectors_reco ITS TPC && has_detector_matching ITSTPC && add_comma_separated TRACK_SOURCES "ITS-TPC"
 has_detectors_reco TPC TRD && has_detector_matching TPCTRD && { add_comma_separated TRD_SOURCES TPC; add_comma_separated TRACK_SOURCES "TPC-TRD"; }
 has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPC && has_detector_matching ITSTPCTRD && { add_comma_separated TRD_SOURCES ITS-TPC; add_comma_separated TRACK_SOURCES "ITS-TPC-TRD"; }
@@ -211,6 +212,8 @@ has_detectors_reco HMP TPC TOF && has_detector_matching TPCTOF && add_comma_sepa
 has_detectors_reco HMP TPC TRD TOF && has_detector_matching TPCTRD && has_detector_matching TPCTRDTOF && add_comma_separated HMP_SOURCES "TPC-TRD-TOF"
 has_detectors_reco MFT MCH && has_detector_matching MFTMCH && add_comma_separated TRACK_SOURCES "MFT-MCH"
 has_detectors_reco MCH MID && has_detector_matching MCHMID && add_comma_separated TRACK_SOURCES "MCH-MID"
+[[ "0$TRACK_SOURCES_GLO" == "0" ]] && TRACK_SOURCES_GLO=$TRACK_SOURCES
+
 for det in `echo $LIST_OF_DETECTORS | sed "s/,/ /g"`; do
   if [[ $LIST_OF_ASYNC_RECO_STEPS =~ (^| )${det}( |$) ]]; then
     has_detector ${det} && has_processing_step ${det}_RECO && add_comma_separated TRACK_SOURCES "$det"
@@ -241,12 +244,6 @@ fi
 [[ ! -z $VERTEXING_SOURCES ]] && PVERTEX_CONFIG+=" --vertexing-sources $VERTEXING_SOURCES"
 [[ ! -z $VERTEX_TRACK_MATCHING_SOURCES ]] && PVERTEX_CONFIG+=" --vertex-track-matching-sources $VERTEX_TRACK_MATCHING_SOURCES"
 
-if [[ -z ${SVERTEXING_SOURCES:-} ]]; then
-  SVERTEXING_SOURCES="$VERTEXING_SOURCES"
-elif [[ "${SVERTEXING_SOURCES^^}" == "NONE" ]]; then
-  SVERTEXING_SOURCES=
-fi
-
 # this option requires well calibrated timing beween different detectors, at the moment suppress it
 #has_detector_reco FT0 && PVERTEX_CONFIG+=" --validate-with-ft0"
 

GRID/utils/grid_submit.sh

@@ -208,7 +208,7 @@ while [ $# -gt 0 ] ; do
         --cores) CPUCORES=$2; shift 2 ;; # allow to specify the CPU cores (check compatibility with partition !)
         --dry) DRYRUN="ON"; shift 1 ;; # do a try run and not actually interact with the GRID (just produce local jdl file)
         --o2tag) O2TAG=$2; shift 2 ;; #
-	--packagespec) PACKAGESPEC=$2; shift 2 ;; # the alisw, cvmfs package list (command separated - example: '"VO_ALICE@FLUKA_VMC::4-1.1-vmc3-1","VO_ALICE@O2::daily-20230628-0200-1"')
+        --packagespec) PACKAGESPEC=$2; shift 2 ;; # the alisw, cvmfs package list (command separated - example: '"VO_ALICE@FLUKA_VMC::4-1.1-vmc3-1","VO_ALICE@O2::daily-20230628-0200-1"')
         --asuser) ASUSER=$2; shift 2 ;; #
         --label) JOBLABEL=$2; shift 2 ;; # label identifying the production (e.g. as a production identifier)
         --mattermost) MATTERMOSTHOOK=$2; shift 2 ;; # if given, status and metric information about the job will be sent to this hook
@@ -218,7 +218,8 @@ while [ $# -gt 0 ] ; do
         --wait) WAITFORALIEN=ON; shift 1 ;; #wait for alien jobs to finish
         --wait-any) WAITFORALIENANY=ON; WAITFORALIEN=ON; shift 1 ;; #wait for any good==done alien jobs to return
         --outputspec) OUTPUTSPEC=$2; shift 2 ;; #provide comma separate list of JDL file specs to be put as part of JDL Output field (example '"*.log@disk=1","*.root@disk=2"')
-	-h) Usage ; exit ;;
+        --split-on-collection) DATACOLLECTION=$2; shift 2 ;; # this will split the jobs on InputDataCollection and "file" mode
+        -h) Usage ; exit ;;
         --help) Usage ; exit ;;
         --fetch-output) FETCHOUTPUT=ON; shift 1 ;; # if to fetch all JOB output locally (to make this job as if it ran locally); only works when we block until all JOBS EXIT
         *) break ;;
@@ -355,13 +356,21 @@ if [[ "${IS_ALIEN_JOB_SUBMITTER}" ]]; then
   cd "${GRID_SUBMIT_WORKDIR}"
 
   QUOT='"'
+  SPLITMODE="production:1-${PRODSPLIT}"
+  if [ "${DATACOLLECTION}" ]; then
+    SPLITMODE="file"
+  fi
   # ---- Generate JDL ----------------
   # TODO: Make this configurable or read from a preamble section in the jobfile
   cat > "${MY_JOBNAMEDATE}.jdl" <<EOF
 Executable = "${MY_BINDIR}/${MY_JOBNAMEDATE}.sh";
 Arguments = "${CONTINUE_WORKDIR:+"-c ${CONTINUE_WORKDIR}"} --local ${O2TAG:+--o2tag ${O2TAG}} --ttl ${JOBTTL} --label ${JOBLABEL:-label} --prodsplit ${PRODSPLIT} ${MATTERMOSTHOOK:+--mattermost ${MATTERMOSTHOOK}} ${CONTROLSERVER:+--controlserver ${CONTROLSERVER}}";
 InputFile = "LF:${MY_JOBWORKDIR}/alien_jobscript.sh";
-${PRODSPLIT:+Split = ${QUOT}production:1-${PRODSPLIT}${QUOT};}
+${DATACOLLECTION:+InputDataList = ${QUOT}input.list${QUOT};}
+${DATACOLLECTION:+InputDataListFormat = ${QUOT}txt-list${QUOT};}
+${DATACOLLECTION:+InputDataCollection = ${QUOT}LF:${MY_JOBWORKDIR}/collection.xml,nodownload${QUOT};}
+${PRODSPLIT:+Split = ${QUOT}${SPLITMODE}${QUOT};}
+${DATACOLLECTION:+SplitMaxInputFileNumber = 1;}
 OutputDir = "${MY_JOBWORKDIR}/${PRODSPLIT:+#alien_counter_03i#}";
 Requirements = member(other.GridPartitions,"${GRIDPARTITION:-multicore_8}");
 CPUCores = "${CPUCORES}";
@@ -382,6 +391,15 @@ EOF
   if [ ! "${DRYRUN}" ]; then
     command_file="alien_commands.txt"
 
+    if [ "$DATACOLLECTION" ]; then
+      pok "Preparing data collection XML file"
+      alien.py toXml $(cat ${DATACOLLECTION}) > collection.xml
+      if [ "$?" != "0" ]; then
+        per "Problem with data collection preparation"
+        exit 1
+      fi
+    fi
+
     pok "Preparing job \"$MY_JOBNAMEDATE\""
     (
       # assemble all GRID interaction in a single script / transaction
@@ -396,6 +414,9 @@ EOF
       echo "rm ${MY_BINDIR}/${MY_JOBNAMEDATE}.sh" >> ${command_file}    # remove current job script
       echo "cp file:${PWD}/${MY_JOBNAMEDATE}.jdl alien://${MY_JOBWORKDIR}/${MY_JOBNAMEDATE}.jdl@DISK=1" >> ${command_file}  # copy the jdl
       echo "cp file:${THIS_SCRIPT} alien://${MY_BINDIR}/${MY_JOBNAMEDATE}.sh@DISK=1" >> ${command_file}  # copy current job script to AliEn
+      if [ "${DATACOLLECTION}" ]; then
+        echo "cp file:collection.xml alien://${MY_JOBWORKDIR}/collection.xml"  >> ${command_file}
+      fi
       [ ! "${CONTINUE_WORKDIR}" ] && echo "cp file:${MY_JOBSCRIPT} alien://${MY_JOBWORKDIR}/alien_jobscript.sh" >> ${command_file}
     ) > alienlog.txt 2>&1
 

MC/bin/o2dpg_sim_workflow.py

@@ -8,12 +8,12 @@
 #
 # Execution examples:
 #  - pp PYTHIA jets, 2 events, triggered on high pT decay photons on all barrel calorimeters acceptance, eCMS 13 TeV
-#     ./o2dpg_sim_workflow.py -e TGeant3 -ns 2 -j 8 -tf 1 -mod "--skipModules ZDC" -col pp -eCM 13000 \
+#     ./o2dpg_sim_workflow.py -e TGeant3 -ns 2 -j 8 -tf 1 -col pp -eCM 13000 \
 #                             -proc "jets" -ptHatBin 3 \
 #                             -trigger "external" -ini "\$O2DPG_ROOT/MC/config/PWGGAJE/ini/trigger_decay_gamma_allcalo_TrigPt3_5.ini"
 #
 #  - pp PYTHIA ccbar events embedded into heavy-ion environment, 2 PYTHIA events into 1 bkg event, beams energy 2.510
-#     ./o2dpg_sim_workflow.py -e TGeant3 -nb 1 -ns 2 -j 8 -tf 1 -mod "--skipModules ZDC"  \
+#     ./o2dpg_sim_workflow.py -e TGeant3 -nb 1 -ns 2 -j 8 -tf 1  \
 #                             -col pp -eA 2.510 -proc "ccbar"  --embedding
 #
 
@@ -108,7 +108,8 @@
 parser.add_argument('--force-n-workers', dest='force_n_workers', action='store_true', help='by default, number of workers is re-computed '
                                                                                            'for given interaction rate; '
                                                                                            'pass this to avoid that')
-parser.add_argument('-mod',help='Active modules (deprecated)', default='--skipModules ZDC')
+parser.add_argument('--skipModules',nargs="*", help="List of modules to skip in geometry budget (and therefore processing)", default=["ZDC"])
+parser.add_argument('--skipReadout',nargs="*", help="List of modules to take out from readout", default=[""])
 parser.add_argument('--with-ZDC', action='store_true', help='Enable ZDC in workflow')
 parser.add_argument('-seed',help='random seed number', default=None)
 parser.add_argument('-o',help='output workflow file', default='workflow.json')
@@ -124,6 +125,7 @@
 # power features (for playing) --> does not appear in help message
 #  help='Treat smaller sensors in a single digitization')
 parser.add_argument('--pregenCollContext', action='store_true', help=argparse.SUPPRESS) # Now the default, giving this option or not makes not difference. We keep it for backward compatibility
+parser.add_argument('--data-anchoring', type=str, default='', help="Take collision contexts (from data) stored in this path")
 parser.add_argument('--no-combine-smaller-digi', action='store_true', help=argparse.SUPPRESS)
 parser.add_argument('--no-combine-dpl-devices', action='store_true', help=argparse.SUPPRESS)
 parser.add_argument('--no-mc-labels', action='store_true', default=False, help=argparse.SUPPRESS)
@@ -256,6 +258,19 @@ def load_external_config(configfile):
    print(f"INFO: Written additional config key parameters to JSON {config_key_param_path}")
    json.dump(anchorConfig, f, indent=2)
 
+# Processing skipped material budget (modules):
+# - If user did NOT specify --with-ZDC
+# - AND ZDC is not already in the list
+# --> append ZDC automatically
+if args.with_ZDC:
+   # User wants ZDC to *not* be skipped → ensure it's removed
+   args.skipModules = [m for m in args.skipModules if m != "ZDC"]
+else:
+   # If user did not request --with-ZDC,
+   # auto-append ZDC unless already present
+   if "ZDC" not in args.skipModules:
+      args.skipModules.append("ZDC")
+
 # with this we can tailor the workflow to the presence of
 # certain detectors
 # these are all detectors that should be assumed active
@@ -266,14 +281,14 @@ def load_external_config(configfile):
     # if "all" here, there was in fact nothing in the anchored script, set to what is passed to this script (which it either also "all" or a subset)
     activeDetectors = readout_detectors
 elif readout_detectors != 'all' and activeDetectors != 'all':
-    # in this case both are comma-seperated lists. Take intersection
+    # in this case both are comma-separated lists. Take intersection
     r = set(readout_detectors.split(','))
     a = set(activeDetectors.split(','))
     activeDetectors = ','.join(r & a)
 # the last case: simply take what comes from the anchored config
 
 # convert to set/hashmap
-activeDetectors = { det:1 for det in activeDetectors.split(',') }
+activeDetectors = { det:1 for det in activeDetectors.split(',') if det not in args.skipModules and det not in args.skipReadout}
 for det in activeDetectors:
     activate_detector(det)
 
@@ -407,7 +422,9 @@ def extractVertexArgs(configKeyValuesStr, finalDiamondDict):
 
 NTIMEFRAMES=int(args.tf)
 NWORKERS=args.n_workers
-MODULES = "--skipModules ZDC" if not isActive("ZDC") else ""
+
+
+SKIPMODULES = " ".join(["--skipModules"] + args.skipModules) if len(args.skipModules) > 0 else ""
 SIMENGINE=args.e
 BFIELD=args.field
 RNDSEED=args.seed # typically the argument should be the jobid, but if we get None the current time is used for the initialisation
@@ -623,7 +640,8 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True, runcommand=True):
                             + ' --extract-per-timeframe tf:sgn'                                                            \
                             + ' --with-vertices ' + vtxmode_precoll                                                        \
                             + ' --maxCollsPerTF ' + str(args.ns)                                                           \
-                            + ' --orbitsEarly ' + str(args.orbits_early)
+                            + ' --orbitsEarly ' + str(args.orbits_early)                                                   \
+                            + ('',f" --import-external {args.data_anchoring}")[len(args.data_anchoring) > 0]
 
 PreCollContextTask['cmd'] += ' --bcPatternFile ccdb'  # <--- the object should have been set in (local) CCDB
 if includeQED:
@@ -714,7 +732,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True, runcommand=True):
         bkgsimneeds = [BKG_CONFIG_task['name'], GRP_TASK['name'], PreCollContextTask['name']]
         BKGtask=createTask(name='bkgsim', lab=["GEANT"], needs=bkgsimneeds, cpu=NWORKERS)
         BKGtask['cmd']='${O2_ROOT}/bin/o2-sim -e ' + SIMENGINE   + ' -j ' + str(NWORKERS) + ' -n '     + str(NBKGEVENTS) \
-                     + ' -g  '      + str(GENBKG) + ' '    + str(MODULES)  + ' -o bkg ' + str(INIBKG)                    \
+                     + ' -g  '      + str(GENBKG) + ' '    + str(SKIPMODULES)  + ' -o bkg ' + str(INIBKG)                \
                      + ' --field ccdb ' + str(CONFKEYBKG)                                                                \
                      + ('',' --timestamp ' + str(args.timestamp))[args.timestamp!=-1] + ' --run ' + str(args.run)        \
                      + ' --vertexMode ' + vtxmode_sgngen                                                                 \
@@ -945,7 +963,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True, runcommand=True):
    sgnmem = 6000 if COLTYPE == 'PbPb' else 4000
    SGNtask=createTask(name='sgnsim_'+str(tf), needs=signalneeds, tf=tf, cwd='tf'+str(tf), lab=["GEANT"],
                       relative_cpu=7/8, n_workers=NWORKERS_TF, mem=str(sgnmem))
-   sgncmdbase = '${O2_ROOT}/bin/o2-sim -e ' + str(SIMENGINE) + ' '  + str(MODULES) + ' -n ' + str(NSIGEVENTS) + ' --seed ' + str(TFSEED)       \
+   sgncmdbase = '${O2_ROOT}/bin/o2-sim -e ' + str(SIMENGINE) + ' '  + str(SKIPMODULES) + ' -n ' + str(NSIGEVENTS) + ' --seed ' + str(TFSEED)      \
               + ' --field ccdb -j ' + str(NWORKERS_TF) + ' ' + str(CONFKEY) + ' ' + str(INIFILE) + ' -o ' + signalprefix + ' ' + embeddinto       \
               + ' --detectorList ' + args.detectorList                                                                                            \
               + ('', ' --timestamp ' + str(args.timestamp))[args.timestamp!=-1] + ' --run ' + str(args.run)
@@ -1105,6 +1123,7 @@ def putConfigValues(listOfMainKeys=[], localCF = {}, globalTFConfig = True):
                          + ' --onlyDet TPC --TPCuseCCDB --interactionRate ' + str(INTRATE) + '  --tpc-lanes ' + str(NWORKERS_TF)             \
                          + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini --early-forward-policy always --forceSelectedDets ' \
                          + ' --tpc-distortion-type ' + str(tpcDistortionType)                                                             \
+                         + ' --n-threads-distortions 1 '                                                                                  \
                          + putConfigValues(["TPCGasParam","TPCGEMParam","TPCEleParam","TPCITCorr","TPCDetParam"],
                                               localCF=tpcLocalCF)
    TPCDigitask['cmd'] += (' --tpc-chunked-writer','')[args.no_tpc_digitchunking]
@@ -1427,7 +1446,7 @@ def getDigiTaskName(det):
                        TPCRECOtask['name'],
                        ITSTPCMATCHtask['name'],
                        TRDTRACKINGtask2['name'] if isActive("TRD") else None]
-   toftracksrcdefault = dpl_option_from_config(anchorConfig, 'o2-tof-matcher-workflow', '--track-sources', default_value='TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD')
+   toftracksrcdefault = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-tof-matcher-workflow', '--track-sources', default_value='TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD'))
    tofusefit = option_if_available('o2-tof-matcher-workflow', '--use-fit', envfile=async_envfile)
    TOFTPCMATCHERtask = createTask(name='toftpcmatch_'+str(tf), needs=toftpcmatchneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000')
    tofmatcher_cmd_parts = [
@@ -1495,7 +1514,8 @@ def getDigiTaskName(det):
    MIDRECOtask['cmd'] += task_finalizer(['${O2_ROOT}/bin/o2-mid-reco-workflow', 
                                              getDPL_global_options(), 
                                              putConfigValues(),('',' --disable-mc')[args.no_mc_labels]])
-   workflow['stages'].append(MIDRECOtask)
+   if isActive('MID'):
+      workflow['stages'].append(MIDRECOtask)
 
    #<--------- FDD reco workflow
    FDDRECOtask = createTask(name='fddreco_'+str(tf), needs=[getDigiTaskName("FDD")], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1500')
@@ -1583,15 +1603,19 @@ def getDigiTaskName(det):
        getDPL_global_options(ccdbbackend=False),
        putConfigValues(),
        ('',' --disable-mc')[args.no_mc_labels]])
-   workflow['stages'].append(MCHMIDMATCHtask)
+   if isActive("MID") and isActive("MCH"):
+      workflow['stages'].append(MCHMIDMATCHtask)
 
    #<--------- MFT-MCH forward matching
-   MFTMCHMATCHtask = createTask(name='mftmchMatch_'+str(tf), needs=[MCHMIDMATCHtask['name'], MFTRECOtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1500')
+   forwardmatchneeds = [MCHRECOtask['name'],
+                        MFTRECOtask['name'],
+                        MCHMIDMATCHtask['name'] if isActive("MID") else None]
+   MFTMCHMATCHtask = createTask(name='mftmchMatch_'+str(tf), needs=forwardmatchneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1500')
    MFTMCHMATCHtask['cmd'] = task_finalizer(
       ['${O2_ROOT}/bin/o2-globalfwd-matcher-workflow',
         putConfigValues(['ITSAlpideConfig',
                          'MFTAlpideConfig',
-                         'FwdMatching'],{"FwdMatching.useMIDMatch":"true"}),
+                         'FwdMatching'],{"FwdMatching.useMIDMatch": "true" if isActive("MID") else "false"}),
        ('',' --disable-mc')[args.no_mc_labels]])
 
    if args.fwdmatching_assessment_full == True:

MC/config/ALICE3/ini/pythia8_pp_clr-blc0_13tev.ini

@@ -0,0 +1,9 @@
+[Diamond]
+width[2]=6.0
+
+[GeneratorExternal]
+fileName=${O2DPG_MC_CONFIG_ROOT}/MC/config/ALICE3/pythia8/generator_pythia8_ALICE3.C
+funcName=generator_pythia8_ALICE3()
+
+[GeneratorPythia8]
+config=${O2DPG_MC_CONFIG_ROOT}/MC/config/ALICE3/pythia8/generator/pythia8_pp_clr-blc0_13tev.cfg

MC/config/ALICE3/ini/pythia8_pp_clr-blc2_13tev.ini

@@ -0,0 +1,9 @@
+[Diamond]
+width[2]=6.0
+
+[GeneratorExternal]
+fileName=${O2DPG_MC_CONFIG_ROOT}/MC/config/ALICE3/pythia8/generator_pythia8_ALICE3.C
+funcName=generator_pythia8_ALICE3()
+
+[GeneratorPythia8]
+config=${O2DPG_MC_CONFIG_ROOT}/MC/config/ALICE3/pythia8/generator/pythia8_pp_clr-blc2_13tev.cfg

MC/config/ALICE3/ini/pythia8_pp_clr-blc3_13tev.ini

@@ -0,0 +1,9 @@
+[Diamond]
+width[2]=6.0
+
+[GeneratorExternal]
+fileName=${O2DPG_MC_CONFIG_ROOT}/MC/config/ALICE3/pythia8/generator_pythia8_ALICE3.C
+funcName=generator_pythia8_ALICE3()
+
+[GeneratorPythia8]
+config=${O2DPG_MC_CONFIG_ROOT}/MC/config/ALICE3/pythia8/generator/pythia8_pp_clr-blc3_13tev.cfg

MC/config/ALICE3/ini/tests/pythia8_pp_clr-blc0_13tev.C

@@ -0,0 +1,19 @@
+int External()
+{
+  std::string path{"o2sim_Kine.root"};
+
+  TFile file(path.c_str(), "read");
+  if (file.IsZombie()) {
+    std::cerr << "Cannot open ROOT file " << path << std::endl;
+    return 1;
+  }
+
+  TTree* tree = (TTree*)file.Get("o2sim");
+
+  if (!tree) {
+    std::cerr << "Cannot find tree o2sim in file " << path << "\n";
+    return 1;
+  }
+
+  return 0;
+}

MC/config/ALICE3/ini/tests/pythia8_pp_clr-blc2_13tev.C

@@ -0,0 +1 @@
+pythia8_pp_clr-blc0_13tev.C

MC/config/ALICE3/ini/tests/pythia8_pp_clr-blc3_13tev.C

@@ -0,0 +1 @@
+pythia8_pp_clr-blc0_13tev.C