Skip to content

Commit 5b7c0a1

Browse files
davidrohrdavidrohr
committed
FST: Sync changes from O2DataProcessing
1 parent ab669b9 commit 5b7c0a1

File tree

1 file changed

+36
-24
lines changed

1 file changed

+36
-24
lines changed

‎prodtests/full-system-test/dpl-workflow.sh‎

Lines changed: 36 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ if [ -z $CTF_DICT_DIR ]; then CTF_DICT_DIR=$FILEWORKDIR; fi # D
1414
if [ -z $CTF_METAFILES_DIR ]; then CTF_METAFILES_DIR="/dev/null"; fi # Directory where to store CTF files metada, /dev/null : skip their writing
1515
if [ -z $RECO_NUM_NODES_WORKFLOW ]; then RECO_NUM_NODES_WORKFLOW=250; fi # Number of EPNs running this workflow in parallel, to increase multiplicities if necessary, by default assume we are 1 out of 250 servers
1616
if [ -z $CTF_MINSIZE ]; then CTF_MINSIZE="500000000"; fi # accumulate CTFs until file size reached
17-
if [ -z $CTF_MAX_PER_FILE ]; then CTF_MAX_PER_FILE="200"; fi # but no more than given number of CTFs per file
17+
if [ -z $CTF_MAX_PER_FILE ]; then CTF_MAX_PER_FILE="10000"; fi # but no more than given number of CTFs per file
1818
if [ -z $IS_SIMULATED_DATA ]; then IS_SIMULATED_DATA=1; fi # processing simulated data
1919

2020
if [ $SYNCMODE == 1 ]; then
@@ -29,7 +29,9 @@ workflow_has_parameter GPU && { export GPUTYPE=HIP; export NGPUS=4; }
2929
[ -z $ITSCLUSDICT ] && ITSCLUSDICT="${FILEWORKDIR}/ITSdictionary.bin"
3030
[ -z $MFTCLUSDICT ] && MFTCLUSDICT="${FILEWORKDIR}/MFTdictionary.bin"
3131
[ -z $ITS_NOISE ] && ITS_NOISE="${FILEWORKDIR}"
32-
[ -z $MFT_NOISE ] && MFT_NOISE="${FILEWORKDIR}/mft_noise_220721_R3C-520.root"
32+
[ -z $MFT_NOISE ] && MFT_NOISE="${FILEWORKDIR}"
33+
[ -z $ITS_STROBE ] && ITS_STROBE="891"
34+
[ -z $MFT_STROBE ] && MFT_STROBE="198"
3335

3436
MID_FEEID_MAP="$FILEWORKDIR/mid-feeId_mapper.txt"
3537
NITSDECTHREADS=2
@@ -48,7 +50,7 @@ has_processing_step()
4850
[[ $WORKFLOW_EXTRA_PROCESSING_STEPS =~ (^|,)"$1"(,|$) ]]
4951
}
5052

51-
for i in ITSTPC TPCTRD ITSTPCTRD TPCTOF ITSTPCTOF MFTMCH PRIMVTX SECVTX; do
53+
for i in `echo $LIST_OF_GLORECO | sed "s/,/ /g"`; do
5254
has_processing_step MATCH_$i && add_comma_separated WORKFLOW_DETECTORS_MATCHING $i # Enable extra matchings requested via WORKFLOW_EXTRA_PROCESSING_STEPS
5355
done
5456
if [ $SYNCMODE == 1 ]; then # Add default steps for synchronous mode
@@ -140,7 +142,7 @@ has_detector_flp_processing CPV && CPV_INPUT=digits
140142

141143
if [ $EPNMODE == 1 ]; then
142144
EVE_CONFIG+=" --eve-dds-collection-index 0"
143-
ITSMFT_FILES+=";ITSClustererParam.noiseFilePath=$ITS_NOISE;MFTClustererParam.noiseFilePath=$MFT_NOISE"
145+
ITSMFT_FILES+=";ITSClustererParam.noiseFilePath=$ITS_NOISE;MFTClustererParam.noiseFilePath=$MFT_NOISE;ITSAlpideParam.roFrameLengthInBC=$ITS_STROBE;MFTAlpideParam.roFrameLengthInBC=$MFT_STROBE;"
144146
MIDDEC_CONFIG+=" --feeId-config-file \"$MID_FEEID_MAP\""
145147
GPU_CONFIG_KEY+="GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;"
146148
# Options for decoding current TRD real raw data (not needed for data converted from MC)
@@ -219,6 +221,7 @@ get_N() # USAGE: get_N [processor-name] [DETECTOR_NAME] [RAW|CTF|REST] [optional
219221
local NAME_DEFAULT="N_$4"
220222
echo $1:${!NAME_PROC:-$((${!NAME_FACTOR} * ${!NAME_DET:-1} * ${!NAME_DEFAULT:-1}))}
221223
}
224+
222225
math_max()
223226
{
224227
echo $(($1 > $2 ? $1 : $2))
@@ -279,8 +282,17 @@ N_TRDRAWDEC=$(math_max $((3 * 60 / $RECO_NUM_NODES_WORKFLOW_CMP)) ${N_TRDRAWDEC:
279282
if [ $CTFINPUT == 1 ]; then
280283
GPU_INPUT=compressed-clusters-ctf
281284
TOF_INPUT=digits
282-
CTFName=`ls -t $FILEWORKDIR/o2_ctf_*.root | head -n1`
283-
WORKFLOW="o2-ctf-reader-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --delay $TFDELAY --loop $NTIMEFRAMES --ctf-input ${CTFName} --ctf-dict ${CTF_DICT} --onlyDet $WORKFLOW_DETECTORS --pipeline tpc-entropy-decoder:$N_TPCENTDEC | "
285+
CTFName=`ls -t $FILEWORKDIR/o2_ctf_*.root 2> /dev/null | head -n1`
286+
if [ -z $CTFName ] && [ $WORKFLOWMODE == "print" ]; then
287+
CTFName='$CTFName'
288+
fi
289+
WORKFLOW="o2-ctf-reader-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --delay $TFDELAY --loop $TFLOOP --max-tf $NTIMEFRAMES --ctf-input ${CTFName} --ctf-dict ${CTF_DICT} --onlyDet $WORKFLOW_DETECTORS --pipeline tpc-entropy-decoder:$N_TPCENTDEC | "
290+
elif [ $RAWTFINPUT == 1 ]; then
291+
TFName=`ls -t $FILEWORKDIR/o2_*.tf 2> /dev/null | head -n1`
292+
if [ -z $TFName ] && [ $WORKFLOWMODE == "print" ]; then
293+
TFName='$TFName'
294+
fi
295+
WORKFLOW="o2-raw-tf-reader-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --delay $TFDELAY --loop $TFLOOP --max-tf $NTIMEFRAMES --input-data ${TFName} --onlyDet $WORKFLOW_DETECTORS | "
284296
elif [ $EXTINPUT == 1 ]; then
285297
PROXY_CHANNEL="name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0"
286298
PROXY_INSPEC="dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION"
@@ -371,22 +383,22 @@ fi
371383
# Entropy encoding / ctf creation workflows - disabled in async mode
372384
if has_processing_step ENTROPY_ENCODER && [ ! -z "$WORKFLOW_DETECTORS_CTF" ]; then
373385
# Entropy encoder workflows
374-
has_detector_ctf MFT && WORKFLOW+="o2-itsmft-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --runmft true --pipeline $(get_N mft-entropy-encoder MFT CTF) | "
375-
has_detector_ctf FT0 && WORKFLOW+="o2-ft0-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N ft0-entropy-encoder FT0 CTF) | "
376-
has_detector_ctf FV0 && WORKFLOW+="o2-fv0-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N fv0-entropy-encoder FV0 CTF) | "
377-
has_detector_ctf MID && WORKFLOW+="o2-mid-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N mid-entropy-encoder MID CTF) | "
378-
has_detector_ctf MCH && WORKFLOW+="o2-mch-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N mch-entropy-encoder MCH CTF) | "
379-
has_detector_ctf PHS && WORKFLOW+="o2-phos-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N phos-entropy-encoder PHS CTF) | "
380-
has_detector_ctf CPV && WORKFLOW+="o2-cpv-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N cpv-entropy-encoder CPV CTF) | "
381-
has_detector_ctf EMC && WORKFLOW+="o2-emcal-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N emcal-entropy-encoder EMC CTF) | "
382-
has_detector_ctf ZDC && WORKFLOW+="o2-zdc-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N zdc-entropy-encoder ZDC CTF) | "
383-
has_detector_ctf FDD && WORKFLOW+="o2-fdd-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N fdd-entropy-encoder FDD CTF) | "
384-
has_detector_ctf HMP && WORKFLOW+="o2-hmpid-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N hmpid-entropy-encoder HMP CTF) | "
385-
has_detector_ctf TOF && WORKFLOW+="o2-tof-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N tof-entropy-encoder TOF CTF) | "
386-
has_detector_ctf ITS && WORKFLOW+="o2-itsmft-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N its-entropy-encoder ITS CTF) | "
387-
has_detector_ctf TRD && WORKFLOW+="o2-trd-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N trd-entropy-encoder TRD CTF TRDENT) | "
388-
has_detector_ctf TPC && WORKFLOW+="o2-tpc-reco-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --input-type compressed-clusters-flat --output-type encoded-clusters,disable-writer --pipeline $(get_N tpc-entropy-encoder TPC CTF TPCENT) | "
389-
has_detector_ctf CTP && WORKFLOW+="o2-ctp-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --pipeline $(get_N its-entropy-encoder CTP CTF)| "
386+
has_detector_ctf MFT && WORKFLOW+="o2-itsmft-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${MFT_ENC_MEMFACT:-1.5} --runmft true --pipeline $(get_N mft-entropy-encoder MFT CTF) | "
387+
has_detector_ctf FT0 && WORKFLOW+="o2-ft0-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${FT0_ENC_MEMFACT:-1.5} --pipeline $(get_N ft0-entropy-encoder FT0 CTF) | "
388+
has_detector_ctf FV0 && WORKFLOW+="o2-fv0-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${FV0_ENC_MEMFACT:-1.5} --pipeline $(get_N fv0-entropy-encoder FV0 CTF) | "
389+
has_detector_ctf MID && WORKFLOW+="o2-mid-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${MID_ENC_MEMFACT:-1.5} --pipeline $(get_N mid-entropy-encoder MID CTF) | "
390+
has_detector_ctf MCH && WORKFLOW+="o2-mch-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${MCH_ENC_MEMFACT:-1.5} --pipeline $(get_N mch-entropy-encoder MCH CTF) | "
391+
has_detector_ctf PHS && WORKFLOW+="o2-phos-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${PHS_ENC_MEMFACT:-1.5} --pipeline $(get_N phos-entropy-encoder PHS CTF) | "
392+
has_detector_ctf CPV && WORKFLOW+="o2-cpv-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${CPV_ENC_MEMFACT:-1.5} --pipeline $(get_N cpv-entropy-encoder CPV CTF) | "
393+
has_detector_ctf EMC && WORKFLOW+="o2-emcal-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${EMC_ENC_MEMFACT:-1.5} --pipeline $(get_N emcal-entropy-encoder EMC CTF) | "
394+
has_detector_ctf ZDC && WORKFLOW+="o2-zdc-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${ZDC_ENC_MEMFACT:-1.5} --pipeline $(get_N zdc-entropy-encoder ZDC CTF) | "
395+
has_detector_ctf FDD && WORKFLOW+="o2-fdd-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${FDD_ENC_MEMFACT:-1.5} --pipeline $(get_N fdd-entropy-encoder FDD CTF) | "
396+
has_detector_ctf HMP && WORKFLOW+="o2-hmpid-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${HMP_ENC_MEMFACT:-1.5} --pipeline $(get_N hmpid-entropy-encoder HMP CTF) | "
397+
has_detector_ctf TOF && WORKFLOW+="o2-tof-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${TOF_ENC_MEMFACT:-1.5} --pipeline $(get_N tof-entropy-encoder TOF CTF) | "
398+
has_detector_ctf ITS && WORKFLOW+="o2-itsmft-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${ITS_ENC_MEMFACT:-1.5} --pipeline $(get_N its-entropy-encoder ITS CTF) | "
399+
has_detector_ctf TRD && WORKFLOW+="o2-trd-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${TRD_ENC_MEMFACT:-1.5} --pipeline $(get_N trd-entropy-encoder TRD CTF TRDENT) | "
400+
has_detector_ctf TPC && WORKFLOW+="o2-tpc-reco-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --input-type compressed-clusters-flat --output-type encoded-clusters,disable-writer --mem-factor ${TPC_ENC_MEMFACT:-1.5} --pipeline $(get_N tpc-entropy-encoder TPC CTF TPCENT) | "
401+
has_detector_ctf CTP && WORKFLOW+="o2-ctp-entropy-encoder-workflow $ARGS_ALL --ctf-dict \"${CTF_DICT}\" --configKeyValues \"$ARGS_ALL_CONFIG\" --mem-factor ${CTP_ENC_MEMFACT:-1.5} --pipeline $(get_N its-entropy-encoder CTP CTF)| "
390402

391403
# CTF / dictionary writer workflow
392404
if [ $SAVECTF == 1 ]; then
@@ -400,8 +412,8 @@ if has_processing_step ENTROPY_ENCODER && [ ! -z "$WORKFLOW_DETECTORS_CTF" ]; th
400412
if [ $CREATECTFDICT == 1 ] && [ $SAVECTF == 1 ]; then CTF_OUTPUT_TYPE="both"; fi
401413
if [ $CREATECTFDICT == 1 ] && [ $SAVECTF == 0 ]; then CTF_OUTPUT_TYPE="dict"; fi
402414
if [ $CREATECTFDICT == 0 ] && [ $SAVECTF == 1 ]; then CTF_OUTPUT_TYPE="ctf"; fi
403-
CMD_CTF="o2-ctf-writer-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --output-dir \"$CTF_DIR\" --ctf-dict-dir \"$CTF_DICT_DIR\" --output-type $CTF_OUTPUT_TYPE --min-file-size ${CTF_MINSIZE} --max-ctf-per-file ${CTF_MAX_PER_FILE} --onlyDet $WORKFLOW_DETECTORS --meta-output-dir $CTF_METAFILES_DIR "
404-
if [ $CREATECTFDICT == 1 ] && [ $EXTINPUT == 1 ]; then CMD_CTF+=" --save-dict-after $NTIMEFRAMES"; fi
415+
CMD_CTF="o2-ctf-writer-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --output-dir \"$CTF_DIR\" --ctf-dict-dir \"$CTF_DICT_DIR\" --output-type $CTF_OUTPUT_TYPE --min-file-size ${CTF_MINSIZE} --max-ctf-per-file ${CTF_MAX_PER_FILE} --onlyDet $WORKFLOW_DETECTORS --append-det-to-period $CTF_MAXDETEXT --meta-output-dir $CTF_METAFILES_DIR "
416+
if [ $CREATECTFDICT == 1 ] && [ $EXTINPUT == 1 ]; then CMD_CTF+=" --save-dict-after $SAVE_CTFDICT_NTIMEFRAMES"; fi
405417
WORKFLOW+="$CMD_CTF | "
406418
fi
407419

0 commit comments

Comments
 (0)