Skip to content

Commit c48da4d

Browse files
authored
Merge branch 'AliceO2Group:master' into fix-laser-raw-qc
2 parents 4b20a84 + 3be71b2 commit c48da4d

990 files changed

Lines changed: 50140 additions & 3484 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/clean-ci.yml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
---
2+
name: Clean PR builds
3+
4+
'on':
5+
workflow_dispatch:
6+
inputs:
7+
pr:
8+
description: PR number in this repo to be cleaned
9+
type: string # can't use number here
10+
required: true
11+
12+
# Warning: GitHub limits the total number of inputs to 10, so a maximum of
13+
# 9 checks is allowed here!
14+
# Warning: the check_* keys are magic and must consist of the string
15+
# "check_" followed by the applicable check name exactly. The
16+
# "description" field is only the human-readable label for the input.
17+
'check_build/O2DPG/sim/o2':
18+
description: build/O2DPG/sim/o2
19+
type: boolean
20+
default: true
21+
'check_build/O2DPG/O2fst/o2':
22+
description: build/O2DPG/O2fst/o2
23+
type: boolean
24+
default: true
25+
26+
permissions: {}
27+
28+
jobs:
29+
clean:
30+
name: Clean PR checks
31+
uses: alisw/ali-bot/.github/workflows/clean-pr-checks.yml@master
32+
with:
33+
owner: ${{ github.event.repository.owner.login }}
34+
repo: ${{ github.event.repository.name }}
35+
pr: ${{ github.event.inputs.pr }}
36+
checks: ${{ toJSON(github.event.inputs) }}
37+
permissions:
38+
pull-requests: read # to get last commit for pr (octokit/graphql-action)
39+
statuses: write # for set-github-status

.github/workflows/syntax-checks.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,19 @@ jobs:
9292
done
9393
exit "$error"
9494
95+
- name: Run .C configs check
96+
run: |
97+
error=0
98+
readarray -d '' files < \
99+
<(find . -path ./.git -prune -false -or -type f -name '*.C' -print0)
100+
for cfg in "${files[@]}"; do
101+
if grep -q "\$O2DPG_ROOT" "$cfg" || grep -q "\${O2DPG_ROOT}" "$cfg"; then
102+
error=1
103+
echo "Deprecated O2DPG_ROOT detected in $cfg, replace with O2DPG_MC_CONFIG_ROOT" >&2
104+
fi
105+
done
106+
exit "$error"
107+
95108
pylint:
96109
name: Pylint
97110
runs-on: ubuntu-latest

CODEOWNERS

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
* @davidrohr @martenole @shahor02 @chiarazampolli @sawenzel @benedikt-voelkel
2-
/DATA/production @davidrohr @martenole @shahor02 @chiarazampolli
1+
* @davidrohr @shahor02 @chiarazampolli @sawenzel
2+
/DATA/production @davidrohr @shahor02 @chiarazampolli
33

44
/DATA/testing/detectors/CPV
55
/DATA/testing/detectors/EMC
@@ -8,24 +8,24 @@
88
/DATA/testing/detectors/FV0
99
/DATA/testing/detectors/HMP
1010
/DATA/testing/detectors/ITS
11-
/DATA/testing/detectors/MCH @aphecetche
11+
/DATA/testing/detectors/MCH
1212
/DATA/testing/detectors/MFT
13-
/DATA/testing/detectors/MID @aphecetche
13+
/DATA/testing/detectors/MID
1414
/DATA/testing/detectors/PHS
1515
/DATA/testing/detectors/TOF @noferini @chiarazampolli
1616
/DATA/testing/detectors/TPC @wiechula
17-
/DATA/testing/detectors/TRD @martenole
17+
/DATA/testing/detectors/TRD
1818
/DATA/testing/detectors/ZDC
1919

20-
/MC @sawenzel @chiarazampolli @benedikt-voelkel
20+
/MC @sawenzel @jackal1-66
2121

2222
# PWG experts
23-
/MC/*/PWGHF @AliceO2Group/reviewers-pwg-hf @sawenzel @chiarazampolli @benedikt-voelkel
24-
/MC/*/PWGLF @AliceO2Group/reviewers-pwg-lf @sawenzel @chiarazampolli @benedikt-voelkel
25-
/MC/*/PWGEM @AliceO2Group/reviewers-pwg-em @sawenzel @chiarazampolli @benedikt-voelkel
26-
/MC/*/PWGGAJE @AliceO2Group/reviewers-pwg-je @sawenzel @chiarazampolli @benedikt-voelkel
27-
/MC/*/PWGCF @AliceO2Group/reviewers-pwg-cf @sawenzel @chiarazampolli @benedikt-voelkel
28-
/MC/*/PWGDQ @AliceO2Group/reviewers-pwg-dq @sawenzel @chiarazampolli @benedikt-voelkel
29-
/MC/*/PWGUD @AliceO2Group/reviewers-pwg-ud @sawenzel @chiarazampolli @benedikt-voelkel
30-
31-
/RelVal @sawenzel @chiarazampolli @benedikt-voelkel
23+
/MC/*/ALICE3 @njacazio
24+
/MC/*/PWGHF @AliceO2Group/reviewers-pwg-hf
25+
/MC/*/PWGLF @AliceO2Group/reviewers-pwg-lf
26+
/MC/*/PWGEM @AliceO2Group/reviewers-pwg-em
27+
/MC/*/PWGGAJE @AliceO2Group/reviewers-pwg-je
28+
/MC/*/PWGCF @AliceO2Group/reviewers-pwg-cf
29+
/MC/*/PWGDQ @AliceO2Group/reviewers-pwg-dq
30+
/MC/*/PWGUD @AliceO2Group/reviewers-pwg-ud
31+
/RelVal @sawenzel @chiarazampolli

DATA/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ export WORKFLOW_PARAMETERS= # Additiona
227227
export RECO_NUM_NODES_OVERRIDE=0 # Override the number of EPN compute nodes to use (default is specified in description library file)
228228
export NHBPERTF=256 # Number of HBF per TF
229229
230-
/opt/alisw/el8/GenTopo/bin/gen_topo.sh > $HOME/gen_topo_output.xml
230+
/opt/alisw/el9/GenTopo/bin/gen_topo.sh > $HOME/gen_topo_output.xml
231231
[drohr@epn245 test]$ ./run.sh
232232
Loading ODC/0.36-1
233233
Loading requirement: BASE/1.0 GCC-Toolchain/v10.2.0-alice2-3 fmt/7.1.0-10 FairLogger/v1.9.1-7 zlib/v1.2.8-8 OpenSSL/v1.0.2o-9 libpng/v1.6.34-9 sqlite/v3.15.0-2 libffi/v3.2.1-2 FreeType/v2.10.1-8 Python/v3.6.10-12 Python-modules/1.0-16 boost/v1.75.0-13 ZeroMQ/v4.3.3-6 ofi/v1.7.1-8 asio/v1.19.1-2 asiofi/v0.5.1-2 DDS/3.5.16-5 FairMQ/v1.4.40-4
@@ -277,7 +277,7 @@ export WORKFLOW_PARAMETERS=EVENT_DISPLAY,CTF,GPU # Additiona
277277
export RECO_NUM_NODES_OVERRIDE=0 # Override the number of EPN compute nodes to use (default is specified in description library file)
278278
export NHBPERTF=256 # Number of HBF per TF
279279
280-
/opt/alisw/el8/GenTopo/bin/gen_topo.sh > $HOME/gen_topo_output.xml
280+
/opt/alisw/el9/GenTopo/bin/gen_topo.sh > $HOME/gen_topo_output.xml
281281
[drohr@epn245 test]$ ./run.sh
282282
Loading ODC/0.36-1
283283
Loading requirement: BASE/1.0 GCC-Toolchain/v10.2.0-alice2-3 fmt/7.1.0-10 FairLogger/v1.9.1-7 zlib/v1.2.8-8 OpenSSL/v1.0.2o-9 libpng/v1.6.34-9 sqlite/v3.15.0-2 libffi/v3.2.1-2 FreeType/v2.10.1-8 Python/v3.6.10-12 Python-modules/1.0-16 boost/v1.75.0-13 ZeroMQ/v4.3.3-6 ofi/v1.7.1-8 asio/v1.19.1-2 asiofi/v0.5.1-2 DDS/3.5.16-5 FairMQ/v1.4.40-4

DATA/common/README.md

Lines changed: 1 addition & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1 @@
1-
The `setenv-sh` script sets the following environment options
2-
* `NTIMEFRAMES`: Number of time frames to process.
3-
* `TFDELAY`: Delay in seconds between publishing time frames (1 / rate).
4-
* `NGPUS`: Number of GPUs to use, data distributed round-robin.
5-
* `GPUTYPE`: GPU Tracking backend to use, can be CPU / CUDA / HIP / OCL / OCL2.
6-
* `SHMSIZE`: Size of the global shared memory segment.
7-
* `DDSHMSIZE`: Size of shared memory unmanaged region for DataDistribution Input.
8-
* `GPUMEMSIZE`: Size of allocated GPU memory (if GPUTYPE != CPU)
9-
* `HOSTMEMSIZE`: Size of allocated host memory for GPU reconstruction (0 = default).
10-
* For `GPUTYPE = CPU`: TPC Tracking scratch memory size. (Default 0 -> dynamic allocation.)
11-
* Otherwise : Size of page-locked host memory for GPU processing. (Defauls 0 -> 1 GB.)
12-
* `CREATECTFDICT`: Create CTF dictionary.
13-
* `SAVECTF`: Save the CTF to a root file.
14-
* 0: Read `ctf_dictionary.root` as input.
15-
* 1: Create `ctf_dictionary.root`. Note that this was already done automatically if the raw data was simulated with `full_system_test.sh`.
16-
* `SYNCMODE`: Run only reconstruction steps of the synchronous reconstruction.
17-
* Note that there is no `ASYNCMODE` but instead the `CTFINPUT` option already enforces asynchronous processing.
18-
* `NUMAGPUIDS`: NUMAID-aware GPU id selection. Needed for the full EPN configuration with 8 GPUs, 2 NUMA domains, 4 GPUs per domain.
19-
In this configuration, 2 instances of `dpl-workflow.sh` must run in parallel.
20-
To be used in combination with `NUMAID` to select the id per workflow.
21-
`start_tmux.sh` will set up these variables automatically.
22-
* `NUMAID`: SHM segment id to use for shipping data as well as set of GPUs to use (use `0` / `1` for 2 NUMA domains, 0 = GPUS `0` to `NGPUS - 1`, 1 = GPUS `NGPUS` to `2 * NGPUS - 1`)
23-
* 0: Runs all reconstruction steps, of sync and of async reconstruction, using raw data input.
24-
* 1: Runs only the steps of synchronous reconstruction, using raw data input.
25-
* `EXTINPUT`: Receive input from raw FMQ channel instead of running o2-raw-file-reader.
26-
* 0: `dpl-workflow.sh` can run as standalone benchmark, and will read the input itself.
27-
* 1: To be used in combination with either `datadistribution.sh` or `raw-reader.sh` or with another DataDistribution instance.
28-
* `CTFINPUT`: Read input from CTF ROOT file. This option is incompatible to EXTINPUT=1. The CTF ROOT file can be stored via SAVECTF=1.
29-
* `NHBPERTF`: Time frame length (in HBF)
30-
* `GLOBALDPLOPT`: Global DPL workflow options appended to o2-dpl-run.
31-
* `EPNPIPELINES`: Set default EPN pipeline multiplicities.
32-
Normally the workflow will start 1 dpl device per processor.
33-
For some of the CPU parts, this is insufficient to keep step with the GPU processing rate, e.g. one ITS-TPC matcher on the CPU is slower than the TPC tracking on multiple GPUs.
34-
This option adds some multiplicies for CPU processes using DPL's pipeline feature.
35-
The settings were tuned for EPN processing with 4 GPUs (i.e. the default multiplicities are per NUMA domain).
36-
The multiplicities are scaled with the `NGPUS` setting, i.e. with 1 GPU only 1/4th are applied.
37-
You can pass an option different to 1, and than it will be applied as factor on top of the multiplicities.
38-
It is auto-selected by `start-tmux.sh`.
39-
* `SEVERITY`: Log verbosity (e.g. info or error, default: info)
40-
* `INFOLOGGER_SEVERITY`: Min severity for messages sent to Infologger. (default: `$SEVERITY`)
41-
* `SHMTHROW`: Throw exception when running out of SHM memory.
42-
It is suggested to leave this enabled (default) on tests on the laptop to get an actual error when it runs out of memory.
43-
This is disabled in `start_tmux.sh`, to avoid breaking the processing while there is a chance that another process might free memory and we can continue.
44-
* `NORATELOG`: Disable FairMQ Rate Logging.
45-
* `INRAWCHANNAME`: FairMQ channel name used by the raw proxy, must match the name used by DataDistribution.
46-
* `WORKFLOWMODE`: run (run the workflow (default)), print (print the command to stdout), dds (create partial DDS topology)
47-
* `FILEWORKDIR`: directory for all input / output files. E.g. grp / geometry / dictionaries etc. are read from here, and dictionaries / ctf / etc. are written to there.
48-
Some files have more fine grained control via other environment variables (e.g. to store the CTF to somewhere else). Such variables are initialized to `$FILEWORKDIR` by default but can be overridden.
49-
* `EPNSYNCMODE`: Specify that this is a workflow running on the EPN for synchronous processing, e.g. logging goes to InfoLogger, DPL metrics to to the AliECS monitoring, etc.
50-
* `BEAMTYPE`: Beam type, must be PbPb, pp, pPb, cosmic, technical.
51-
* `IS_SIMULATED_DATA` : 1 for MC data, 0 for RAW data.
1+
For a reference to available env-variables, please check https://github.com/AliceO2Group/AliceO2/blob/dev/prodtests/full-system-test/documentation/env-variables.md

DATA/common/gen_topo_helper_functions.sh

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,21 @@ has_detector_matching()
4949
[[ $WORKFLOW_DETECTORS_MATCHING =~ (^|,)"ALL"(,|$) ]] || [[ $WORKFLOW_DETECTORS_MATCHING =~ (^|,)"$1"(,|$) ]]
5050
}
5151

52+
has_detector_gpu()
53+
{
54+
has_detector $1 && [[ $WORKFLOW_DETECTORS_GPU =~ (^|,)"$1"(,|$) ]]
55+
}
56+
57+
has_secvtx_source()
58+
{
59+
[[ $SVERTEXING_SOURCES =~ (^|,)"ALL"(,|$) ]] || [[ $SVERTEXING_SOURCES =~ (^|,)"$1"(,|$) ]]
60+
}
61+
62+
has_detector_in_secvtx_sources()
63+
{
64+
[[ $SVERTEXING_SOURCES =~ (^|,)"ALL"(,|$) ]] || [[ $SVERTEXING_SOURCES =~ (^|,|-)"$1"(-|,|$) ]]
65+
}
66+
5267
has_detector_qc()
5368
{
5469
has_detector $1 && [[ $WORKFLOW_DETECTORS_QC =~ (^|,)"$1"(,|$) ]]
@@ -132,6 +147,11 @@ has_detectors_flp_processing()
132147
_check_multiple has_detector_flp_processing "$@"
133148
}
134149

150+
has_detectors_gpu()
151+
{
152+
_check_multiple has_detector_gpu "$@"
153+
}
154+
135155
workflow_has_parameters()
136156
{
137157
_check_multiple workflow_has_parameter "$@"
@@ -247,19 +267,22 @@ add_W() # Add binarry to workflow command USAGE: add_W [BINARY] [COMMAND_LINE_OP
247267
WORKFLOW+=$WFADD
248268
}
249269

250-
if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then
251-
GEN_TOPO_QC_CONSUL_SERVER=ali-staging.cern.ch
252-
else
253-
GEN_TOPO_QC_CONSUL_SERVER=alio2-cr1-hv-con01.cern.ch
270+
if [[ ${EPNSYNCMODE:-0} == 1 ]]; then
271+
if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then
272+
GEN_TOPO_QC_CONSUL_SERVER=ali-staging.cern.ch
273+
else
274+
GEN_TOPO_QC_CONSUL_SERVER=alio2-cr1-hv-con01.cern.ch
275+
fi
276+
GEN_TOPO_QC_APRICOT_SERVER=`curl -s "http://${GEN_TOPO_QC_CONSUL_SERVER}:8500/v1/kv/o2/runtime/aliecs/vars/apricot_endpoint?raw"`
254277
fi
255-
GEN_TOPO_QC_APRICOT_SERVER=`curl -s "http://${GEN_TOPO_QC_CONSUL_SERVER}:8500/v1/kv/o2/runtime/aliecs/vars/apricot_endpoint?raw"`
256278

257279
add_QC_from_consul()
258280
{
281+
[[ ${EPNSYNCMODE:-0} == 1 ]] || { echo "Error fetching QC JSON $1: consul server only set for EPNSYNCMODE == 1 " 1>&2 && exit 1; }
259282
if [[ ! -z ${GEN_TOPO_QC_JSON_FILE:-} ]]; then
260283
curl -s -o $GEN_TOPO_QC_JSON_FILE "http://${GEN_TOPO_QC_CONSUL_SERVER}:8500/v1/kv${1}?raw"
261284
if [[ $? != 0 ]]; then
262-
echo "Error fetching QC JSON $1"
285+
echo "Error fetching QC JSON $1 (1)" 1>&2
263286
exit 1
264287
fi
265288
QC_CONFIG_ARG="json://${GEN_TOPO_QC_JSON_FILE}"
@@ -271,14 +294,15 @@ add_QC_from_consul()
271294

272295
add_QC_from_apricot()
273296
{
297+
[[ ${EPNSYNCMODE:-0} == 1 ]] || { echo "Error fetching QC JSON $1: apricot server only set for EPNSYNCMODE == 1 " 1>&2 && exit 1; }
274298
if [[ ! -z ${GEN_TOPO_QC_JSON_FILE:-} ]]; then
275-
if [[ ${1} =~ "?" ]]; then
276-
curl -s -o $GEN_TOPO_QC_JSON_FILE "${GEN_TOPO_QC_APRICOT_SERVER}/${1}\&process=true"
277-
else
278-
curl -s -o $GEN_TOPO_QC_JSON_FILE "${GEN_TOPO_QC_APRICOT_SERVER}/${1}?process=true"
279-
fi
299+
if [[ ${1} =~ "?" ]]; then
300+
curl -s -o $GEN_TOPO_QC_JSON_FILE "${GEN_TOPO_QC_APRICOT_SERVER}/${1}\&process=true"
301+
else
302+
curl -s -o $GEN_TOPO_QC_JSON_FILE "${GEN_TOPO_QC_APRICOT_SERVER}/${1}?process=true"
303+
fi
280304
if [[ $? != 0 ]]; then
281-
echo "Error fetching QC JSON $1"
305+
echo "Error fetching QC JSON $1 (2)" 1>&2
282306
exit 1
283307
fi
284308
QC_CONFIG_ARG="json://${GEN_TOPO_QC_JSON_FILE}"

DATA/common/getCommonArgs.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ ARGS_ALL="--session ${OVERRIDE_SESSION:-default} --severity $SEVERITY --shm-segm
1313
ARGS_ALL_CONFIG="keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null;${ALL_EXTRA_CONFIG:-}"
1414
if [[ $EPNSYNCMODE == 1 ]]; then
1515
ARGS_ALL+=" --infologger-severity $INFOLOGGER_SEVERITY"
16-
ARGS_ALL+=" --monitoring-backend influxdb-unix:///tmp/telegraf.sock --resources-monitoring 15"
16+
ARGS_ALL+=" --monitoring-backend influxdb-unix:///tmp/telegraf.sock --resources-monitoring ${GEN_TOPO_RESOURCE_MONITORING:-15} ${GEN_TOPO_RESOURCE_MONITORING:+--dpl-stats-min-online-publishing-interval $GEN_TOPO_RESOURCE_MONITORING}"
1717
ARGS_ALL_CONFIG+="NameConf.mCCDBServer=$GEN_TOPO_EPN_CCDB_SERVER;"
1818
export DPL_CONDITION_BACKEND=$GEN_TOPO_EPN_CCDB_SERVER
1919
[[ -z ${O2_DPL_DEPLOYMENT_MODE:-} ]] && O2_DPL_DEPLOYMENT_MODE=OnlineECS
@@ -25,6 +25,7 @@ fi
2525
[[ $NORATELOG == 1 ]] && ARGS_ALL+=" --fairmq-rate-logging 0"
2626

2727
[[ ! -z ${O2_DPL_EXIT_TRANSITION_TIMEOUT_DEFAULT:-} ]] && ARGS_ALL+=" --exit-transition-timeout $O2_DPL_EXIT_TRANSITION_TIMEOUT_DEFAULT"
28+
[[ ! -z ${O2_DPL_PROCESSING_TRANSITION_TIMEOUT_DEFAULT:-} ]] && ARGS_ALL+=" --data-processing-timeout $O2_DPL_PROCESSING_TRANSITION_TIMEOUT_DEFAULT"
2829

2930
true
3031

0 commit comments

Comments
 (0)