Developments for 2-tag software processing

a) make workflow construction more independent from actual
   O2 commit: Allow to detect if options are present in DPL executables
   and react accordingly. This can allow to write more resilit workflow construction
   code.

b) Improvements to swap in and out software environments in the anchoring script

Author: sawenzel

MC/bin/o2dpg_sim_config.py

@@ -1,3 +1,7 @@
+from functools import lru_cache
+import subprocess
+import re
+
 def create_sim_config(args):
     # creates a generic simulation config
     # based on arguments args (run number, energy, ...) originally passed
@@ -138,3 +142,41 @@ def constructConfigKeyArg(config):
           arg = arg + mainkey + '.' + subkey + '=' + config[mainkey][subkey] + ';'
     arg = arg + '"'
     return arg
+
+# some functions to determine dpl option availability on the fly
+def parse_dpl_help_output(executable):
+    """Parses the --help full output of an executable to extract available options."""
+    try:
+        output = subprocess.check_output([executable, "--help", "full"], text=True)
+    except subprocess.CalledProcessError:
+        return {}, {}
+    
+    option_pattern = re.compile(r"(\-\-[\w\-]+)")
+    sections = {}
+    inverse_lookup = {}
+    current_section = "global"
+    
+    for line in output.split("\n"):
+        section_match = re.match(r"^([A-Za-z\s]+):$", line.strip())
+        if section_match:
+            current_section = section_match.group(1).strip()
+            sections[current_section] = []
+            continue
+        
+        option_match = option_pattern.findall(line)
+        if option_match:
+            for option in option_match:
+                sections.setdefault(current_section, []).append(option)
+                inverse_lookup.setdefault(option, []).append(current_section)
+    
+    return sections, inverse_lookup
+
+@lru_cache(maxsize=10)
+def get_dpl_options_for_executable(executable):
+    """Returns available options and inverse lookup for a given executable, caching the result."""
+    return parse_dpl_help_output(executable)
+
+def option_if_available(executable, option):
+    """Checks if an option is available for a given executable and returns it as a string. Otherwise empty string"""
+    _, inverse_lookup = get_dpl_options_for_executable(executable)
+    return ' ' + option if option in inverse_lookup else ''

MC/bin/o2dpg_sim_workflow.py

@@ -27,6 +27,8 @@
 import itertools
 import math
 import requests, re
+from functools import lru_cache
+
 pandas_available = True
 try:
     import pandas as pd
@@ -37,7 +39,7 @@
 
 from o2dpg_workflow_utils import createTask, createGlobalInitTask, dump_workflow, adjust_RECO_environment, isActive, activate_detector, deactivate_detector, compute_n_workers, merge_dicts
 from o2dpg_qc_finalization_workflow import include_all_QC_finalization
-from o2dpg_sim_config import create_sim_config, create_geant_config, constructConfigKeyArg
+from o2dpg_sim_config import create_sim_config, create_geant_config, constructConfigKeyArg, option_if_available
 
 parser = argparse.ArgumentParser(description='Create an ALICE (Run3) MC simulation workflow')
 
@@ -1154,7 +1156,8 @@ def getDigiTaskName(det):
    TPCRECOtask['cmd'] = '${O2_ROOT}/bin/o2-tpc-reco-workflow ' + getDPL_global_options(bigshm=True) + ' --input-type clusters --output-type tracks,send-clusters-per-sector ' \
                         + putConfigValuesNew(["GPU_global","TPCGasParam", "TPCCorrMap", "GPU_rec_tpc", "trackTuneParams"], {"GPU_proc.ompThreads":NWORKERS_TF} | tpcLocalCFreco) + ('',' --disable-mc')[args.no_mc_labels] \
                         + tpc_corr_scaling_options + tpc_corr_options_mc \
-                        + ' --tpc-mc-time-gain'
+                        + option_if_available('o2-tpc-reco-workflow', '--tpc-mc-time-gain')
+   
    workflow['stages'].append(TPCRECOtask)
 
    # END TPC reco

MC/run/ANCHOR/anchorMC.sh

@@ -145,10 +145,23 @@ NWORKERS=${NWORKERS:-8}
 # set a default seed if not given
 SEED=${ALIEN_PROC_ID:-${SEED:-1}}
 
-#<----- START OF part that should run under a clean alternative software environment if this was given ------
-(
+ONCVMFS=0
+[[ "${BASEDIR}" == /cvmfs/* ]] && ONCVMFS=1
+if [ ! "${MODULEPATH}" ]; then
+  export MODULEPATH=${BASEDIR}/../Modules/modulefiles
+  if [ "${ONCVMFS}" == "1" ]; then
+    PLATFORM=$(echo "${BASEDIR}" | sed -E 's|.*/([^/]+)/Packages|\1|')
+    export MODULEPATH=${MODULEPATH}:${BASEDIR}/../../etc/toolchain/modulefiles/${PLATFORM}
+  fi
+  echo "Determined Modulepath to be ${MODULEPATH}"
+fi
 
+#<----- START OF part that should run under a clean alternative software environment if this was given ------
 if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then
+  if [ "${LOADEDMODULES}" ]; then
+    module save initial_modules.list # we stash the current modules environment
+    module purge
+  fi
   echo_info "Using tag ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG} to setup anchored MC"
   /cvmfs/alice.cern.ch/bin/alienv printenv "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" &> async_environment.env
   source async_environment.env
@@ -201,8 +214,6 @@ if [[ "${RECO_RC}" != "0" ]] ; then
     exit ${RECO_RC}
 fi
 
-)
-#<----- END OF part that should run under a clean alternative software environment if this was given ------
 
 ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMPRODUCTIONTAG_KEEP
 echo_info "Setting back ALIEN_JDL_LPMPRODUCTIONTAG to $ALIEN_JDL_LPMPRODUCTIONTAG"
@@ -217,6 +228,16 @@ if [[ "${ASYNC_WF_RC}" != "0" || `grep "o2-ctf-reader-workflow-options" config-j
   exit 1
 fi
 
+# get rid of the temporary software environment
+if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then
+  module purge
+  # restore the initial software environment
+  echo "Restoring initial environment"
+  module --no-pager restore initial_modules.list
+  module saverm initial_modules.list
+fi
+#<----- END OF part that should run under a clean alternative software environment if this was given ------
+
 # -- CREATE THE MC JOB DESCRIPTION ANCHORED TO RUN --
 
 MODULES="--skipModules ZDC"