devito/.github/workflows/pytest-gpu.yaml at main · devitocodes/devito · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
# Workflow information:
#  - OpenACC/OpenMP on NVIDIA runs on runners labeled `nvidiagpu`
#  - OpenMP on AMD runs on runners labeled `amdgpu`
#  - Respect CUDA_VISIBLE_DEVICES for NVIDIA jobs by passing it AND restricting Docker with --gpus "device=…"
#  - Tag images and container names to avoid cross-runner races and maximize cache reuse

name: CI-gpu

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  RESOURCE_GROUP: CI-gpu

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:
    inputs:
      tags:
        description: "Run GPU tests"

jobs:
  test-gpu-docker:
    name: ${{ matrix.name }}
    runs-on:
      - self-hosted
      - ${{ matrix.runner_label }}

    strategy:
      fail-fast: false
      matrix:
        name: [pytest-gpu-acc-nvidia, pytest-gpu-omp-amd]
        test_examples: ["examples/seismic/tti/tti_example.py examples/seismic/acoustic/acoustic_example.py examples/seismic/viscoacoustic/viscoacoustic_example.py examples/seismic/viscoelastic/viscoelastic_example.py examples/seismic/elastic/elastic_example.py"]

        include:
        # -------------------- NVIDIA job --------------------
        - name: pytest-gpu-acc-nvidia
          test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openacc.py tests/test_operator.py::TestEstimateMemory"
          base: "devitocodes/bases:nvidia-nvc12"
          runner_label: nvidiagpu
          test_drive_cmd: "nvidia-smi"
          # Respect CUDA_VISIBLE_DEVICES and also hard-limit Docker to that device.
          # NOTE: CUDA_VISIBLE_DEVICES must be set by the runner (systemd drop-in etc.).
          dockerflags: --gpus "device=${CUDA_VISIBLE_DEVICES:-all}"

        # -------------------- AMD job -----------------------
        - name: pytest-gpu-omp-amd
          test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py tests/test_operator.py::TestEstimateMemory"
          runner_label: amdgpu
          base: "devitocodes/bases:amd"
          test_drive_cmd: "rocm-smi"
          # Passes through required /dev nodes etc.
          dockerflags: >-
            --network=host
            --device=/dev/kfd --device=/dev/dri
            --ipc=host
            --group-add video --group-add "$(getent group render | cut -d: -f3)"
            --cap-add=SYS_PTRACE --security-opt seccomp=unconfined

    steps:
    - name: Checkout devito
      uses: actions/checkout@v6

    - id: build
      name: Build docker image
      uses: ./.github/actions/docker-build
      with:
        file: docker/Dockerfile.devito
        tag: ${{ matrix.name }}
        base: ${{ matrix.base }}

    - name: Probe GPU
      uses: ./.github/actions/docker-run
      with:
        uid: ${{ steps.build.outputs.unique }}
        tag: ${{ matrix.name }}
        args: ${{ matrix.dockerflags }}
        command: ${{ matrix.test_drive_cmd }}

    - name: Test with pytest
      uses: ./.github/actions/docker-run
      with:
        uid: ${{ steps.build.outputs.unique }}
        tag: ${{ matrix.name }}
        args: ${{ matrix.dockerflags }}
        env: |
          CI=true
          CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}
          DEVITO_LOGGING=DEBUG
          PYTHONFAULTHANDLER=1
        command: |
          pytest \
            -vvv \
            --capture=no \
            --showlocals \
            --log-cli-level=DEBUG \
            -o log_cli=true \
            --full-trace \
            --durations=10 \
            --cov \
            --cov-config=.coveragerc \
            --cov-report=xml \
            ${{ matrix.test_files }}

    - name: Test examples
      uses: ./.github/actions/docker-run
      with:
        uid: ${{ steps.build.outputs.unique }}
        tag: ${{ matrix.name }}
        args: ${{ matrix.dockerflags }}
        command: |
          pytest \
            -v \
            ${{ matrix.test_examples }}

    - name: Test examples with MPI
      uses: ./.github/actions/docker-run
      with:
        uid: ${{ steps.build.outputs.unique }}
        tag: ${{ matrix.name }}
        args: ${{ matrix.dockerflags }}
        env: |
          CI=true
          DEVITO_LOGGING=DEBUG
          DEVITO_MPI=1
        command: |
          mpiexec -n 2 \
            pytest \
              -v \
              ${{ matrix.test_examples }}

    - name: Cleanup docker image
      if: always()
      uses: ./.github/actions/docker-clean
      with:
        uid: ${{ steps.build.outputs.unique }}
        tag: ${{ matrix.name }}