Merge branch 'master' into a

Author: arvindkhuntia

CODEOWNERS

@@ -17,7 +17,7 @@
 /DATA/testing/detectors/TRD
 /DATA/testing/detectors/ZDC
 
-/MC @sawenzel @chiarazampolli @jackal1-66
+/MC @sawenzel @jackal1-66
 
 # PWG experts
 /MC/*/ALICE3 @njacazio

DATA/production/calib/its-threshold-aggregator.sh

@@ -10,7 +10,7 @@ PROXY_INSPEC="tunestring:ITS/TSTR;runtype:ITS/RUNT;fittype:ITS/FITT;scantype:ITS
 
 CCDBPATH1=""
 CCDBPATH2=""
-if [ $RUNTYPE_ITS == "tuning" ] || [ $RUNTYPE_ITS == "digital" ] || [ $RUNTYPE_ITS == "tuningbb" ]; then
+if [[ $RUNTYPE_ITS == "tuning" ]] || [[ $RUNTYPE_ITS == *digital* ]] || [[ $RUNTYPE_ITS == "tuningbb" ]]; then
   CCDBPATH1="$DCSCCDBSERVER_PERS"
   CCDBPATH2="http://o2-ccdb.internal"
 else 
@@ -26,7 +26,7 @@ WORKFLOW=
 add_W o2-dpl-raw-proxy "--exit-transition-timeout 20 --proxy-name its-thr-input-proxy --dataspec \"$PROXY_INSPEC\" --network-interface ib0 --channel-config \"name=its-thr-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq\"" "" 0
 add_W o2-its-threshold-aggregator-workflow "-b" "" 0
 add_W o2-calibration-ccdb-populator-workflow "--ccdb-path=\"$CCDBPATH1\" --sspec-min 0 --sspec-max 0 --name-extention dcs"
-if [ $RUNTYPE_ITS == "digital" ]; then
+if [[ $RUNTYPE_ITS == *digital* ]]; then
   add_W o2-calibration-ccdb-populator-workflow "--ccdb-path=\"$CCDBPATH2\" --sspec-min 1 --sspec-max 1"
 fi
 

DATA/production/calib/its-threshold-processing.sh

@@ -14,15 +14,19 @@ PROXY_OUTSPEC="tunestring:ITS/TSTR;runtype:ITS/RUNT;fittype:ITS/FITT;scantype:IT
 
 CHIPMODBASE=5
 NDECODERS=6
+ADDITIONAL_OPTIONS_DEC=""
+ADDITIONAL_OPTIONS_CAL=""
 if [ $RUNTYPE_ITS == "digital" ]; then
   CHIPMODBASE=10
 fi
+if [ $RUNTYPE_ITS == "digitalnomask" ]; then
+  CHIPMODBASE=10
+  ADDITIONAL_OPTIONS_CAL="--ninj 5"
+fi
 if [ $RUNTYPE_ITS == "thrfull" ]; then
   CHIPMODBASE=20
   NDECODERS=10
 fi
-ADDITIONAL_OPTIONS_DEC=""
-ADDITIONAL_OPTIONS_CAL=""
 if [ $RUNTYPE_ITS == "tuningbb" ]; then
   ADDITIONAL_OPTIONS_CAL="--min-vcasn 30 --max-vcasn 130"
 fi

DATA/production/configurations/asyncReco/async_pass.sh

@@ -412,67 +412,46 @@ fi
 echo "SETTING_ROOT_OUTPUT = $SETTING_ROOT_OUTPUT"
 
 # Enabling GPUs
-if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]] ; then
-  echo "Enabling GPUS"
-  [[ -z ${GPUTYPE:-} ]] && export GPUTYPE="HIP"
-  [[ -z ${GPUMEMSIZE:-} ]] && export GPUMEMSIZE=$((25 << 30))
-  if [[ "0$ASYNC_PASS_NO_OPTIMIZED_DEFAULTS" != "01" ]]; then
+if [[ $ASYNC_PASS_NO_OPTIMIZED_DEFAULTS != 1 ]]; then
+  export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1
+  if [[ $ALIEN_JDL_USEGPUS == 1 ]] ; then
+    echo "Enabling GPUS"
+    if [[ -z $ALIEN_JDL_SITEARCH ]]; then echo "ERROR: Must set ALIEN_JDL_SITEARCH to define GPU architecture!"; exit 1; fi
+    if [[ $ALIEN_JDL_SITEARCH == "NERSC" ]]; then # Disable mlock / ulimit / gpu memory registration - has performance impact, but doesn't work at NERSC for now
+      export SETENV_NO_ULIMIT=1
+      export CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow+="GPU_proc.noGPUMemoryRegistration=1;"
+    fi
+    ALIEN_JDL_SITEARCH_TMP=$ALIEN_JDL_SITEARCH
+    if [[ $ALIEN_JDL_SITEARCH == "EPN_MI100" ]]; then
+      ALIEN_JDL_SITEARCH_TMP=EPN
+      export EPN_NODE_MI100=1
+    elif [[ $ALIEN_JDL_SITEARCH == "EPN_MI50" ]]; then
+      ALIEN_JDL_SITEARCH_TMP=EPN
+    fi
     if [[ "ALIEN_JDL_USEFULLNUMADOMAIN" == 0 ]]; then
       if [[ $keep -eq 0 ]]; then
         if [[ $ALIEN_JDL_UNOPTIMIZEDGPUSETTINGS != 1 ]]; then
-          export OPTIMIZED_PARALLEL_ASYNC=pp_1gpu  # sets the multiplicities to optimized defaults for this configuration (1 job with 1 gpu on EPNs)
-          export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1
+          export OPTIMIZED_PARALLEL_ASYNC=pp_1gpu_${ALIEN_JDL_SITEARCH_TMP} # (16 cores, 1 gpu per job, pp)
         else
-          # forcing multiplicities to be 1
-          export MULTIPLICITY_PROCESS_tof_matcher=1
-          export MULTIPLICITY_PROCESS_mch_cluster_finder=1
-          export MULTIPLICITY_PROCESS_tpc_entropy_decoder=1
-          export MULTIPLICITY_PROCESS_itstpc_track_matcher=1
-          export MULTIPLICITY_PROCESS_its_tracker=1
-          export OMP_NUM_THREADS=4
-          export TIMEFRAME_RATE_LIMIT=8
-          export SHMSIZE=30000000000
+          export OPTIMIZED_PARALLEL_ASYNC=pp_1gpu_${ALIEN_JDL_SITEARCH_TMP}_unoptimized # (16 cores, 1 gpu per job, pp, low CPU multiplicities)
         fi
       else
-        export TIMEFRAME_RATE_LIMIT=4
-        export SHMSIZE=30000000000
+        export OPTIMIZED_PARALLEL_ASYNC=keep_root
       fi
     else
       if [[ $BEAMTYPE == "pp" ]]; then
-        export OPTIMIZED_PARALLEL_ASYNC=pp_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, pp)
-        export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1
+        export OPTIMIZED_PARALLEL_ASYNC=pp_4gpu_${ALIEN_JDL_SITEARCH_TMP} # (64 cores, 1 NUMA, 4 gpu per job, pp)
       else  # PbPb
-        export OPTIMIZED_PARALLEL_ASYNC=PbPb_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, PbPb)
-        export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1
+        export OPTIMIZED_PARALLEL_ASYNC=PbPb_4gpu_${ALIEN_JDL_SITEARCH_TMP} # (64 cores, 1 NUMA 4 gpu per job, PbPb)
       fi
     fi
-  fi
-else
-  # David, Oct 13th
-  # the optimized settings for the 8 core GRID queue without GPU are
-  # (overwriting the values above)
-  #
-  if [[ "0$ASYNC_PASS_NO_OPTIMIZED_DEFAULTS" != "01" ]]; then
-    if [[ "$ALIEN_JDL_EPNFULLNUMACPUONLY" != 1 ]]; then
-      if [[ $BEAMTYPE == "pp" ]]; then
-        if (( $(echo "$RUN_IR > 800000" | bc -l) )); then
-          export TIMEFRAME_RATE_LIMIT=1
-        elif (( $(echo "$RUN_IR < 50000" | bc -l) )); then
-          export TIMEFRAME_RATE_LIMIT=6
-        else
-          export TIMEFRAME_RATE_LIMIT=3
-        fi
-        export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu # sets the multiplicities to optimized defaults for this configuration (grid)
-        export SHMSIZE=16000000000
-      else # PbPb
-        export TIMEFRAME_RATE_LIMIT=2
-        export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu
-        export SHMSIZE=16000000000
-        export SVERTEX_THREADS=5
-      fi
+  else
+    export SETENV_NO_ULIMIT=1
+    export DPL_DEFAULT_PIPELINE_LENGTH=16 # to avoid memory issues - affects performance, so don't do with GPUs
+    if [[ $ALIEN_JDL_EPNFULLNUMACPUONLY != 1 ]]; then
+      export OPTIMIZED_PARALLEL_ASYNC=8cpu # (8 cores per job, grid)
     else
-      export OPTIMIZED_PARALLEL_ASYNC=pp_64cpu # to use EPNs with full NUMA domain but without GPUs
-      export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1
+      export OPTIMIZED_PARALLEL_ASYNC=pp_64cpu # (64 cores per job, 1 NUMA, EPN)
     fi
   fi
 fi

DATA/production/configurations/asyncReco/setenv_extra.sh

@@ -3,15 +3,6 @@
 
 # process flags passed to the script
 
-if [[ -z "$ALIEN_JDL_USEGPUS" || $ALIEN_JDL_USEGPUS != 1 ]]; then
-  export SETENV_NO_ULIMIT=1
-fi
-
-# to avoid memory issues - we don't do this on the EPNs, since it can affect the performance
-if [[ $ALIEN_JDL_USEGPUS != 1 ]]; then
-  export DPL_DEFAULT_PIPELINE_LENGTH=16
-fi
-
 # check if this is a production on skimmed data
 if grep -q /skimmed/ wn.xml ; then
   export ON_SKIMMED_DATA=1;
@@ -574,6 +565,9 @@ fi
 # ad-hoc options for GPU reco workflow
 export CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow+=";GPU_global.dEdxDisableResidualGainMap=1;$TRACKTUNETPC;$VDRIFTPARAMOPTION;"
 
+# if ITS is running on GPU we need to give the workflow most options from ITS reco
+has_detector_gpu ITS && export CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow+=";$CONFIG_EXTRA_PROCESS_o2_its_reco_workflow;"
+
 [[ ! -z $TPCCLUSTERTIMESHIFT ]] && [[ $ALIEN_JDL_LPMPRODUCTIONTYPE != "MC" ]] && export CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow+=";GPU_rec_tpc.clustersShiftTimebins=$TPCCLUSTERTIMESHIFT;"
 
 # ad-hoc settings for TOF reco

DATA/production/qc-async/mch-digits.json

@@ -36,7 +36,12 @@
                     "type": "direct",
                     "query": "digits:MCH/DIGITS"
                 },
+                "movingWindows": [
+                    "RateSignalPerDualSampa"
+                ],
                 "taskParameters": {
+                    "Enable1DRateMaps": "true",
+                    "Enable2DRateMaps": "false",
                     "Diagnostic": "false"
                 },
                 "grpGeomRequest": {

DATA/production/qc-async/mch-reco.json

@@ -49,6 +49,13 @@
                 "dataSource": {
                     "type": "direct",
                     "query": "preclusters:MCH/PRECLUSTERS/0;preclusterdigits:MCH/PRECLUSTERDIGITS/0"
+                },
+                "movingWindows": [
+                    "PseudoeffPerDualSampa"
+                ],
+                "taskParameters": {
+                    "Enable1DPseudoeffMaps": "true",
+                    "Enable2DPseudoeffMaps": "false"
                 }
             },
             "FRofs": {
@@ -68,4 +75,4 @@
             }
         }
     }
-}
+}

DATA/production/standalone-calibration.desc

@@ -18,6 +18,8 @@ ITS-thr-full: "O2PDPSuite" reco,80,80,"RUNTYPE_ITS=thrfull production/calib/its-
 
 ITS-digital: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=digital production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=digital production/calib/its-threshold-aggregator.sh"
 
+ITS-digital-no-mask: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=digitalnomask production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=digitalnomask production/calib/its-threshold-aggregator.sh"
+
 ITS-pulselength: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=pulselength production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=pulselength production/calib/its-threshold-aggregator.sh"
 
 ITS-tot-1row: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=tot1row production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=tot1row production/calib/its-threshold-aggregator.sh"

DATA/production/workflow-multiplicities.sh

@@ -53,11 +53,23 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then
     [[ ! -z ${TIMEFRAME_RATE_LIMIT:-} ]] && unset TIMEFRAME_RATE_LIMIT
     [[ ! -z ${SHMSIZE:-} ]] && unset SHMSIZE
   fi
-  if [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_8cpu" ]]; then
+  if [[ $OPTIMIZED_PARALLEL_ASYNC == "8cpu" ]]; then
     [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=3
     [[ -z ${SHMSIZE:-} ]] && SHMSIZE=16000000000
     NGPURECOTHREADS=5
-  elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_16cpu" ]]; then
+    if [[ $BEAMTYPE == "pp" ]]; then
+      if (( $(echo "$RUN_IR > 800000" | bc -l) )); then
+        TIMEFRAME_RATE_LIMIT=1
+      elif (( $(echo "$RUN_IR < 50000" | bc -l) )); then
+        TIMEFRAME_RATE_LIMIT=6
+      else
+        TIMEFRAME_RATE_LIMIT=3
+      fi
+    else # PbPb
+      TIMEFRAME_RATE_LIMIT=2
+      SVERTEX_THREADS=5
+    fi
+  elif [[ $OPTIMIZED_PARALLEL_ASYNC == "16cpu" ]]; then
     [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=8
     [[ -z ${SHMSIZE:-} ]] && SHMSIZE=22000000000
     NGPURECOTHREADS=9
@@ -78,20 +90,28 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then
     N_MCHCL=3
     N_TOFMATCH=2
     N_TPCENTDEC=3
-  elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_1gpu" ]]; then
+  elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_1gpu_EPN" || $OPTIMIZED_PARALLEL_ASYNC == "pp_1gpu_EPN_unoptimized" ]]; then
     [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=8
     [[ -z ${SHMSIZE:-} ]] && SHMSIZE=30000000000
-    N_TOFMATCH=2
-    N_MCHCL=3
-    N_TPCENTDEC=2
-    N_TPCITS=3
+    NGPUS=1
+    GPUTYPE=HIP
+    GPUMEMSIZE=$((25 << 30))
     N_MCHTRK=2
-    N_ITSTRK=3
-    NGPURECOTHREADS=8
+    N_TPCTRK=$NGPUS
+    if [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_1gpu_EPN" ]]; then
+      N_TOFMATCH=2
+      N_MCHCL=3
+      N_TPCENTDEC=2
+      N_TPCITS=3
+      N_ITSTRK=3
+      NGPURECOTHREADS=8
+    else
+      NGPURECOTHREADS=4
+    fi
     NTRDTRKTHREADS=3
     ITSTRK_THREADS=2
     ITSTPC_THREADS=2
-  elif [[ $OPTIMIZED_PARALLEL_ASYNC =~ ^pp_4gpu(_|$) ]]; then
+  elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_gpu_NERSC" || $OPTIMIZED_PARALLEL_ASYNC == "pp_4gpu_EPN" ]]; then
     if [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]]; then
       if [[ ! -z ${ALIEN_JDL_LPMANCHORYEAR} && ${ALIEN_JDL_LPMANCHORYEAR} -lt 2023 ]]; then
         TIMEFRAME_RATE_LIMIT=45
@@ -100,19 +120,21 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then
       fi
     fi
     [[ -z ${SHMSIZE:-} ]] && SHMSIZE=100000000000
-    if [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_4gpu_NERSC" ]]; then
+    if [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_gpu_NERSC" ]]; then
       NGPUS=1
       GPUTYPE=CUDA
     else
       NGPUS=4
+      GPUTYPE=HIP
     fi
+    GPUMEMSIZE=$((25 << 30))
     NGPURECOTHREADS=8
     NTRDTRKTHREADS=2
     ITSTRK_THREADS=2
     ITSTPC_THREADS=2
     SVERTEX_THREADS=4
     TPCTIMESERIES_THREADS=2
-    N_TPCTRK=4
+    N_TPCTRK=$NGPUS
     N_FWDMATCH=2
     N_PRIMVTXMATCH=1
     N_PRIMVTX=1
@@ -128,16 +150,18 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then
     N_ITSTRK=12
     N_ITSCL=2
     export DPL_SMOOTH_RATE_LIMITING=1
-  elif [[ $OPTIMIZED_PARALLEL_ASYNC =~ ^PbPb_4gpu(_|$) ]]; then
+  elif [[ $OPTIMIZED_PARALLEL_ASYNC == "PbPb_gpu_NERSC" || $OPTIMIZED_PARALLEL_ASYNC == "PbPb_4gpu_EPN" ]]; then
     [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=35
     [[ -z ${SHMSIZE:-} ]] && SHMSIZE=100000000000 # SHM_LIMIT 3/4
     [[ -z ${TIMEFRAME_SHM_LIMIT:-} ]] && TIMEFRAME_SHM_LIMIT=$(($SHMSIZE / 3))
-    if [[ $OPTIMIZED_PARALLEL_ASYNC == "PbPb_4gpu_NERSC" ]]; then
+    if [[ $OPTIMIZED_PARALLEL_ASYNC == "PbPb_gpu_NERSC" ]]; then
       NGPUS=1
       GPUTYPE=CUDA
     else
       NGPUS=4
+      GPUTYPE=HIP
     fi
+    GPUMEMSIZE=$((25 << 30))
     NGPURECOTHREADS=8
     NTRDTRKTHREADS=8
     ITSTRK_THREADS=5
@@ -172,6 +196,9 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then
     N_MCHTRK=1
     N_TOFMATCH=9
     N_TPCTRK=6
+  elif [[ $OPTIMIZED_PARALLEL_ASYNC == "keep_root" ]]; then
+    TIMEFRAME_RATE_LIMIT=4
+    SHMSIZE=30000000000
   else
     echo "Invalid optimized setting '$OPTIMIZED_PARALLEL_ASYNC'" 1>&2
     exit 1
@@ -192,6 +219,7 @@ elif [[ $EPNPIPELINES != 0 ]]; then
     if [[ "${GEN_TOPO_AUTOSCALE_PROCESSES:-}" == "1" && $RUNTYPE == "PHYSICS" ]]; then
       N_MCHCL=$(math_max $((6 * 100 / $RECO_NUM_NODES_WORKFLOW_CMP)) 1)
     fi
+    N_MCHRAWDEC=2
     if [[ "$HIGH_RATE_PP" == "1" ]]; then
       N_TPCITS=$(math_max $((5 * $EPNPIPELINES * $NGPUS / 4)) 1)
       N_TPCENT=$(math_max $((4 * $EPNPIPELINES * $NGPUS / 4)) 1)
@@ -208,6 +236,7 @@ elif [[ $EPNPIPELINES != 0 ]]; then
   else
     if [[ $BEAMTYPE == "PbPb" ]]; then
       N_ITSTRK=$(math_max $((2 * $EPNPIPELINES * $NGPUS / 4)) 1)
+      N_MCHRAWDEC=2
     elif [[ $BEAMTYPE == "cosmic" ]]; then
       N_ITSTRK=$(math_max $((4 * $EPNPIPELINES * $NGPUS / 4)) 1)
     fi

DATA/tools/epn/gen_topo_o2dpg.sh

@@ -94,7 +94,7 @@ while true; do
     done
     git checkout $GEN_TOPO_SOURCE &> /dev/null
     if [[ $? != 0 ]]; then
-      git fetch --tags origin 1>&2 || { echo Repository update failed 1>&2; exit 1; }
+      git fetch -f --tags origin 1>&2 || { echo Repository update failed 1>&2; exit 1; }
       git checkout $GEN_TOPO_SOURCE &> /dev/null || { echo commit does not exist 1>&2; exit 1; }
     fi
     # At a tag, or a detached non-dirty commit, but not on a branch