set parallel world in test script + toggle ideal ITS in anchored simulation (#1908)

* set pw in test script + toggle ideal ITS in anchored simulation

* remove duplicated flag for pw activation

* fix string check

* fix variable declaration

* fix check

Author: maciacco

MC/run/ANCHOR/anchorMC.sh

@@ -1,4 +1,4 @@
-#!/bin/bashMore actions
+#!/bin/bash
 
 # add distortion maps
 # https://alice.its.cern.ch/jira/browse/O2-3346?focusedCommentId=300982&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-300982
@@ -60,7 +60,6 @@ print_help()
   echo "ALIEN_JDL_MC_ORBITS_PER_TF=N, enforce some orbits per timeframe, instead of determining from CCDB"
   echo "ALIEN_JDL_RUN_TIME_SPAN_FILE=FILE, use a run-time-span file to exclude bad data-taking periods"
   echo "ALIEN_JDL_INVERT_IRFRAME_SELECTION, invertes the choice of ALIEN_JDL_RUN_TIME_SPAN_FILE"
-  echo "ALIEN_JDL_CCDB_CONDITION_NOT_AFTER, sets the condition_not_after timestamp for CCDB queries"
   echo "DISABLE_QC, set this to disable QC, e.g. to 1"
 }
 
@@ -125,6 +124,10 @@ ALIEN_JDL_LPMPRODUCTIONTAG_KEEP=$ALIEN_JDL_LPMPRODUCTIONTAG
 echo_info "Substituting ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMPRODUCTIONTAG with ALIEN_JDL_LPMANCHORPRODUCTION=$ALIEN_JDL_LPMANCHORPRODUCTION for simulating reco pass..."
 ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMANCHORPRODUCTION
 
+if [[ $ALIEN_JDL_ANCHOR_SIM_OPTIONS == *"--tpc-distortion-type 2"* ]]; then
+  export O2DPG_ENABLE_TPC_DISTORTIONS=ON
+fi
+
 # check variables that need to be set
 [ -z "${ALIEN_JDL_LPMANCHORPASSNAME}" ] && { echo_error "Set ALIEN_JDL_LPMANCHORPASSNAME or ANCHORPASSNAME" ; exit 1 ; }
 [ -z "${ALIEN_JDL_LPMRUNNUMBER}" ] && { echo_error "Set ALIEN_JDL_LPMRUNNUMBER or RUNNUMBER" ; exit 1 ; }
@@ -146,6 +149,48 @@ NWORKERS=${NWORKERS:-8}
 # set a default seed if not given
 SEED=${ALIEN_PROC_ID:-${SEED:-1}}
 
+ONCVMFS=0
+
+if [ "${ALIEN_JDL_O2DPG_OVERWRITE}" ]; then
+  echo "Setting O2DPG_ROOT to overwritten path"
+  export O2DPG_ROOT=${ALIEN_JDL_O2DPG_OVERWRITE}
+fi
+
+export > env_base.env
+
+if ! declare -F module > /dev/null; then
+  module() {
+    eval "$(/usr/bin/modulecmd bash "$@")";
+  }
+  export -f module
+fi
+
+[[ "${BASEDIR}" == /cvmfs/* ]] && ONCVMFS=1
+if [ ! "${MODULEPATH}" ]; then
+  export MODULEPATH=${BASEDIR}/../Modules/modulefiles
+  if [ "${ONCVMFS}" == "1" ]; then
+    PLATFORM=$(echo "${BASEDIR}" | sed -E 's|.*/([^/]+)/Packages|\1|')
+    export MODULEPATH=${MODULEPATH}:${BASEDIR}/../../etc/toolchain/modulefiles/${PLATFORM}
+  fi
+  echo "Determined Modulepath to be ${MODULEPATH}"
+fi
+
+#<----- START OF part that should run under a clean alternative software environment if this was given ------
+if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then
+  if [ "${LOADEDMODULES}" ]; then
+    echo "Stashing initial modules"
+    module save initial_modules.list # we stash the current modules environment
+    module list --no-pager
+    module purge --no-pager
+    export > env_after_stashing.env
+    echo "Modules after purge"
+    module list --no-pager
+  fi
+  echo_info "Using tag ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG} to setup anchored MC"
+  /cvmfs/alice.cern.ch/bin/alienv printenv "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" &> async_environment.env
+  source async_environment.env
+  export > env_async.env
+fi
 
 # default async_pass.sh script
 DPGRECO=$O2DPG_ROOT/DATA/production/configurations/asyncReco/async_pass.sh
@@ -173,10 +218,6 @@ chmod u+x setenv_extra.sh
 
 echo_info "Setting up DPGRECO to ${DPGRECO}"
 
-# settings that are MC-specific, modify setenv_extra.sh in-place
-sed -i 's/GPU_global.dEdxUseFullGainMap=1;GPU_global.dEdxDisableResidualGainMap=1/GPU_global.dEdxSplineTopologyCorrFile=splines_for_dedx_V1_MC_iter0_PP.root;GPU_global.dEdxDisableTopologyPol=1;GPU_global.dEdxDisableGainMap=1;GPU_global.dEdxDisableResidualGainMap=1;GPU_global.dEdxDisableResidualGain=1/' setenv_extra.sh
-### ???
-
 # take out line running the workflow (if we don't have data input)
 [ ${CTF_TEST_FILE} ] || sed -i '/WORKFLOWMODE=run/d' async_pass.sh
 
@@ -194,6 +235,7 @@ if [[ "${RECO_RC}" != "0" ]] ; then
     exit ${RECO_RC}
 fi
 
+
 ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMPRODUCTIONTAG_KEEP
 echo_info "Setting back ALIEN_JDL_LPMPRODUCTIONTAG to $ALIEN_JDL_LPMPRODUCTIONTAG"
 
@@ -207,16 +249,32 @@ if [[ "${ASYNC_WF_RC}" != "0" || `grep "o2-ctf-reader-workflow-options" config-j
   exit 1
 fi
 
+# get rid of the temporary software environment
+if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then
+  module purge --no-pager
+  # restore the initial software environment
+  echo "Restoring initial environment"
+  module --no-pager restore initial_modules.list
+  module saverm initial_modules.list
+  if [ "${ALIEN_JDL_O2DPG_OVERWRITE}" ]; then
+    echo "Setting back O2DPG_ROOT to overwritten path ${ALIEN_JDL_O2DPG_OVERWRITE}"
+    export O2DPG_ROOT=${ALIEN_JDL_O2DPG_OVERWRITE}
+  fi
+fi
+#<----- END OF part that should run under a clean alternative software environment if this was given ------
+
 # -- CREATE THE MC JOB DESCRIPTION ANCHORED TO RUN --
 
 MODULES="--skipModules ZDC"
 # Since this is used, set it explicitly
 ALICEO2_CCDB_LOCALCACHE=${ALICEO2_CCDB_LOCALCACHE:-$(pwd)/ccdb}
 
 # these arguments will be digested by o2dpg_sim_workflow_anchored.py
-baseargs="-tf ${NTIMEFRAMES} --split-id ${SPLITID} --prod-split ${PRODSPLIT} --cycle ${CYCLE} --run-number ${ALIEN_JDL_LPMRUNNUMBER}"
+baseargs="-tf ${NTIMEFRAMES} --split-id ${SPLITID} --prod-split ${PRODSPLIT} --cycle ${CYCLE} --run-number ${ALIEN_JDL_LPMRUNNUMBER}                                \
+          ${ALIEN_JDL_RUN_TIME_SPAN_FILE:+--run-time-span-file ${ALIEN_JDL_RUN_TIME_SPAN_FILE} ${ALIEN_JDL_INVERT_IRFRAME_SELECTION:+--invert-irframe-selection}}   \
+          ${ALIEN_JDL_MC_ORBITS_PER_TF:+--orbitsPerTF ${ALIEN_JDL_MC_ORBITS_PER_TF}}"
 
-# these arguments will be passed as well but only evetually be digested by o2dpg_sim_workflow.py which is called from o2dpg_sim_workflow_anchored.py
+# these arguments will be passed as well but only eventually be digested by o2dpg_sim_workflow.py which is called from o2dpg_sim_workflow_anchored.py
 remainingargs="-seed ${SEED} -ns ${NSIGEVENTS} --include-local-qc --pregenCollContext"
 remainingargs="${remainingargs} -e ${ALIEN_JDL_SIMENGINE} -j ${NWORKERS}"
 remainingargs="${remainingargs} -productionTag ${ALIEN_JDL_LPMPRODUCTIONTAG:-alibi_anchorTest_tmp}"
@@ -226,24 +284,39 @@ remainingargs="${ALIEN_JDL_ANCHOR_SIM_OPTIONS} ${remainingargs} --anchor-config
 # apply software tagging choice
 # remainingargs="${remainingargs} ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG:+--alternative-reco-software ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}}"
 remainingargs="${remainingargs} ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG:+--alternative-reco-software ${PWD}/env_async.env}"
-# potentially add CCDB timemachine timestamp
-remainingargs="${remainingargs} ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER:+--condition-not-after ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER}}"
+
 
 echo_info "baseargs passed to o2dpg_sim_workflow_anchored.py: ${baseargs}"
 echo_info "remainingargs forwarded to o2dpg_sim_workflow.py: ${remainingargs}"
 
+anchoringLogFile=timestampsampling_${ALIEN_JDL_LPMRUNNUMBER}.log
 # query CCDB has changed, w/o "_"
-${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow_anchored.py ${baseargs} -- ${remainingargs} &> timestampsampling_${ALIEN_JDL_LPMRUNNUMBER}.log
+${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow_anchored.py ${baseargs} -- ${remainingargs} &> ${anchoringLogFile}
 WF_RC="${?}"
 if [ "${WF_RC}" != "0" ] ; then
     echo_error "Problem during anchor timestamp sampling and workflow creation. Exiting."
     exit ${WF_RC}
 fi
 
-TIMESTAMP=`grep "Determined timestamp to be" timestampsampling_${ALIEN_JDL_LPMRUNNUMBER}.log | awk '//{print $6}'`
+TIMESTAMP=`grep "Determined timestamp to be" ${anchoringLogFile} | awk '//{print $6}'`
 echo_info "TIMESTAMP IS ${TIMESTAMP}"
 
+
+# check if this job is exluded because it falls inside a bad data-taking period
+ISEXCLUDED=$(grep "TIMESTAMP IS EXCLUDED IN RUN" ${anchoringLogFile})
+if [ "${ISEXCLUDED}" ]; then
+  # we can quit here; there is nothing to do
+  # (apart from maybe creating a fake empty AO2D.root file or the like)
+  echo "Timestamp is excluded from run. Nothing to do here"
+  exit 0
+fi
+
 # -- Create aligned geometry using ITS ideal alignment to avoid overlaps in geant
+ENABLEPW=0
+if [[ ${remainingargs} == *"GeometryManagerParam.useParallelWorld=1"* ]]; then
+  ENABLEPW=1
+fi
+
 if [ "${ENABLEPW}" == "0" ]; then
   CCDBOBJECTS_IDEAL_MC="ITS/Calib/Align"
   TIMESTAMP_IDEAL_MC=1
@@ -257,13 +330,15 @@ fi
 
 # TODO This can potentially be removed or if needed, should be taken over by o2dpg_sim_workflow_anchored.py and O2_dpg_workflow_runner.py
 if [ "${ENABLEPW}" == "0" ]; then
-  echo "run with echo in pipe" | ${O2_ROOT}/bin/o2-create-aligned-geometry-workflow ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER:+--condition-not-after ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER}} --configKeyValues "HBFUtils.startTime=${TIMESTAMP}" --condition-remap=file://${ALICEO2_CCDB_LOCALCACHE}=ITS/Calib/Align -b --run
+  echo "run with echo in pipe" | ${O2_ROOT}/bin/o2-create-aligned-geometry-workflow --configKeyValues "HBFUtils.startTime=${TIMESTAMP}" --condition-remap=file://${ALICEO2_CCDB_LOCALCACHE}=ITS/Calib/Align -b --run
 else
-  echo "run with echo in pipe" | ${O2_ROOT}/bin/o2-create-aligned-geometry-workflow ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER:+--condition-not-after ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER}} --configKeyValues "HBFUtils.startTime=${TIMESTAMP}" -b --run
+  echo "run with echo in pipe" | ${O2_ROOT}/bin/o2-create-aligned-geometry-workflow --configKeyValues "HBFUtils.startTime=${TIMESTAMP}" -b --run
 fi
 mkdir -p $ALICEO2_CCDB_LOCALCACHE/GLO/Config/GeometryAligned
 ln -s -f $PWD/o2sim_geometry-aligned.root $ALICEO2_CCDB_LOCALCACHE/GLO/Config/GeometryAligned/snapshot.root
-[[ -f $PWD/its_GeometryTGeo.root ]] && mkdir -p $ALICEO2_CCDB_LOCALCACHE/ITS/Config/Geometry && ln -s -f $PWD/its_GeometryTGeo.root $ALICEO2_CCDB_LOCALCACHE/ITS/Config/Geometry/snapshot.root
+if [ "${ENABLEPW}" == "0" ]; then
+  [[ -f $PWD/its_GeometryTGeo.root ]] && mkdir -p $ALICEO2_CCDB_LOCALCACHE/ITS/Config/Geometry && ln -s -f $PWD/its_GeometryTGeo.root $ALICEO2_CCDB_LOCALCACHE/ITS/Config/Geometry/snapshot.root
+fi
 [[ -f $PWD/mft_GeometryTGeo.root ]] && mkdir -p $ALICEO2_CCDB_LOCALCACHE/MFT/Config/Geometry && ln -s -f $PWD/mft_GeometryTGeo.root $ALICEO2_CCDB_LOCALCACHE/MFT/Config/Geometry/snapshot.root
 
 # -- RUN THE MC WORKLOAD TO PRODUCE AOD --
@@ -272,7 +347,7 @@ export FAIRMQ_IPC_PREFIX=./
 
 echo_info "Ready to start main workflow"
 
-${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${ALIEN_JDL_O2DPGWORKFLOWTARGET:-aod} --cpu-limit ${ALIEN_JDL_CPULIMIT:-8}
+${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${ALIEN_JDL_O2DPGWORKFLOWTARGET:-aod} --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} --dynamic-resources
 MCRC=$?  # <--- we'll report back this code
 if [[ "${ALIEN_JDL_ADDTIMESERIESINMC}" != "0" ]]; then
   # Default value is 1 so this is run by default.
@@ -296,7 +371,9 @@ fi
 #
 if [[ -n "$ALIEN_PROC_ID" ]]; then
   find ./ \( -name "*.log*" -o -name "*mergerlog*" -o -name "*serverlog*" -o -name "*workerlog*" -o -name "pythia8.cfg" \) | tar -czvf debug_log_archive.tgz -T -
-  find ./ \( -name "*.log*" -o -name "*mergerlog*" -o -name "*serverlog*" -o -name "*workerlog*" -o -name "*.root" \) | tar -czvf debug_full_archive.tgz -T -
+  if [[ "$ALIEN_JDL_CREATE_TAR_IN_MC" == "1" ]]; then
+    find ./ \( -name "*.log*" -o -name "*mergerlog*" -o -name "*serverlog*" -o -name "*workerlog*" -o -name "*.root" \) | tar -czvf debug_full_archive.tgz -T -
+  fi
 fi
 
 unset FAIRMQ_IPC_PREFIX