FST: Add INPUT_FILE_LIST and INPUT_FILE_COPY_CMD options

Author: davidrohr

prodtests/full-system-test/datadistribution.sh

@@ -3,25 +3,30 @@
 MYDIR="$(dirname $(readlink -f $0))"
 source $MYDIR/setenv.sh
 
-if [ `which StfBuilder 2> /dev/null | wc -l` == "0" ]; then
+if [[ `which StfBuilder 2> /dev/null | wc -l` == "0" ]]; then
   eval "`alienv shell-helper`"
   alienv --no-refresh load DataDistribution/latest
 fi
 
 # For benchmark only, do NOT copy&paste!
-export DATADIST_SHM_DELAY=30
+[[ $NUMAGPUIDS == 1 ]] && export DATADIST_SHM_DELAY=30
 
-if [ ! -z $DD_STARTUP_DELAY ]; then
+if [[ ! -z $DD_STARTUP_DELAY ]]; then
   sleep $DD_STARTUP_DELAY
 fi
 
-if [ -z $INPUT_FILE_LIST ]; then
+if [[ -z $INPUT_FILE_LIST ]]; then
   DD_INPUT_CMD="--data-source-dir ./raw/timeframe"
 else
-  DD_INPUT_CMD="--data-source-copy-cmd=\"XrdSecPROTOCOL=sss,unix xrdcp -N root://eosaliceo2.cern.ch/?src ?dst\" --data-source-file-list $INPUT_FILE_LIST"
+  DD_INPUT_CMD="--data-source-file-list $INPUT_FILE_LIST"
+  if [[ -z $INPUT_FILE_COPY_CMD ]]; then
+    DD_INPUT_CMD+=" --data-source-copy-cmd \"XrdSecPROTOCOL=sss,unix xrdcp -N root://eosaliceo2.cern.ch/?src ?dst\""
+  else
+    DD_INPUT_CMD+=" --data-source-copy-cmd \"$INPUT_FILE_COPY_CMD\""
+  fi
 fi
 
-if [ $NTIMEFRAMES != -1 ]; then
+if [[ $NTIMEFRAMES != -1 ]]; then
   export DATADIST_FILE_READ_COUNT=$NTIMEFRAMES
   DD_INPUT_CMD+=" --data-source-repeat"
 fi

prodtests/full-system-test/dpl-workflow.sh

@@ -283,18 +283,16 @@ if [ $CTFINPUT == 1 ]; then
   GPU_INPUT=compressed-clusters-ctf
   TOF_INPUT=digits
   CTFName=`ls -t $FILEWORKDIR/o2_ctf_*.root 2> /dev/null | head -n1`
-  if [[ -z $CTFName && $WORKFLOWMODE == "print" ]]; then
-    CTFName='$CTFName'
-  fi
+  [[ -z $CTFName && $WORKFLOWMODE == "print" ]] && CTFName='$CTFName'
+  [[ ! -z $INPUT_FILE_LIST ]] && CTFName=$INPUT_FILE_LIST
   if [ $NTIMEFRAMES == -1 ]; then NTIMEFRAMES_CMD= ; else NTIMEFRAMES_CMD="--max-tf $NTIMEFRAMES"; fi
-  WORKFLOW="o2-ctf-reader-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --delay $TFDELAY --loop $TFLOOP $NTIMEFRAMES_CMD --ctf-input ${CTFName} --ctf-dict ${CTF_DICT} --onlyDet $WORKFLOW_DETECTORS --pipeline tpc-entropy-decoder:$N_TPCENTDEC | "
+  WORKFLOW="o2-ctf-reader-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --delay $TFDELAY --loop $TFLOOP $NTIMEFRAMES_CMD --ctf-input ${CTFName} ${INPUT_FILE_COPY_CMD+--copy-cmd} ${INPUT_FILE_COPY_CMD} --ctf-dict ${CTF_DICT} --onlyDet $WORKFLOW_DETECTORS --pipeline tpc-entropy-decoder:$N_TPCENTDEC | "
 elif [ $RAWTFINPUT == 1 ]; then
   TFName=`ls -t $FILEWORKDIR/o2_*.tf 2> /dev/null | head -n1`
-  if [[ -z $TFName && $WORKFLOWMODE == "print" ]]; then
-    TFName='$TFName'
-  fi
+  [[ -z $TFName && $WORKFLOWMODE == "print" ]] && TFName='$TFName'
+  [[ ! -z $INPUT_FILE_LIST ]] && TFName=$INPUT_FILE_LIST
   if [ $NTIMEFRAMES == -1 ]; then NTIMEFRAMES_CMD= ; else NTIMEFRAMES_CMD="--max-tf $NTIMEFRAMES"; fi
-  WORKFLOW="o2-raw-tf-reader-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --delay $TFDELAY --loop $TFLOOP $NTIMEFRAMES_CMD --input-data ${TFName} --onlyDet $WORKFLOW_DETECTORS | "
+  WORKFLOW="o2-raw-tf-reader-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --delay $TFDELAY --loop $TFLOOP $NTIMEFRAMES_CMD --input-data ${TFName} ${INPUT_FILE_COPY_CMD+--copy-cmd} ${INPUT_FILE_COPY_CMD} --onlyDet $WORKFLOW_DETECTORS | "
 elif [ $EXTINPUT == 1 ]; then
   PROXY_CHANNEL="name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0"
   PROXY_INSPEC="dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION"