Update kaggle comp with main

.github/actions/install/action.yml

@@ -13,28 +13,61 @@ inputs:
   extras:
     description: "Extras flags for pip installation of graphnet"
     required: true
-    default: "[develop,torch]"
+    default: "develop"
+
+  torch_version:
+    description: "Version of PyTorch"
+    required: true
+    default: "2.6.0"
+    options: ["2.7.0", "2.6.0", "2.5.1", "2.4.0"]
+
   hardware:
     description: "The requirements file depends on the hardware, i.e., CPU, GPU, or macOS"
     required: true
     default: "cpu"
-    options: ["cpu", "gpu", "macos"]
+    options: ["cpu", "cu118", "cu121", "cu124", "cu126", "macos"]
+
+  use_vm:
+    description: "Should the installation happen in a virtual environment named venv?"
+    required: true
+    default: false
+    options: [true, false]
 
 runs:
   using: "composite"
   steps:
-    - name: Infer installation flags
+    - name: Prepare pip flags
+      id: flags
       run: |
-        PIP_FLAG_EDITABLE=`[[ ${{ inputs.editable }} =~ (T|t)rue ]] && printf "%s\n" "-e " || echo " "`
-        PIP_FLAG_USER=`[[ ${{ inputs.user }} =~ (T|t)rue ]] && printf "%s\n" "--user " || echo " "`
-        echo "PIP_FLAGS=${PIP_FLAG_USER}${PIP_FLAG_EDITABLE}" >> $GITHUB_ENV
+        [[ "${{ inputs.editable }}" == "true" ]] && echo "editable_flag=-e" >> $GITHUB_OUTPUT || echo "editable_flag=" >> $GITHUB_OUTPUT
+        [[ "${{ inputs.user }}" == "true" ]] && echo "user_flag=--user" >> $GITHUB_OUTPUT || echo "user_flag=" >> $GITHUB_OUTPUT
+
+        version="${{ inputs.torch_version }}"
+        IFS='.' read -r major minor _ <<< "$version"
+        echo "torch_flag=${major}${minor}" >> $GITHUB_OUTPUT
+      shell: bash
+
+    - name: Install dependencies (pip & setuptools)
       shell: bash
-    - name: Install dependencies
       run: |
-        pip install --upgrade pip>=20
-        pip install wheel setuptools==59.5.0
+        if [[ "${{ inputs.use_vm }}" == "true" ]]; then source ~/venv/bin/activate; fi
+        pip install --no-cache-dir --upgrade 'pip>=20'
+        pip install --no-cache-dir --upgrade 'setuptools>=68.2.2'
+        rm -rf ~/.cache/pip
+
+    - name: Print pip and setuptools versions
       shell: bash
-    - name: Install package
       run: |
-        pip install -r requirements/torch_${{ inputs.hardware }}.txt ${{ env.PIP_FLAGS }} .${{ inputs.extras }}
+        if [[ "${{ inputs.use_vm }}" == "true" ]]; then source ~/venv/bin/activate; fi
+        pip --version
+        pip show setuptools
+
+    - name: Install GraphNeT
       shell: bash
+      run: |
+        if [[ "${{ inputs.use_vm }}" == "true" ]]; then source ~/venv/bin/activate; fi
+        echo "Installing graphnet with flags: ${{ steps.flags.outputs.user_flag }} ${{ steps.flags.outputs.editable_flag }}"
+        echo "pip install ${{ steps.flags.outputs.user_flag }} ${{ steps.flags.outputs.editable_flag }} .[torch-${{ steps.flags.outputs.torch_flag }},${{ inputs.extras }}] -f https://data.pyg.org/whl/torch-${{ inputs.torch_version }}+${{ inputs.hardware }}.html"
+        pip install --no-cache-dir${{ steps.flags.outputs.user_flag }} ${{ steps.flags.outputs.editable_flag }} .[torch-${{ steps.flags.outputs.torch_flag }},${{ inputs.extras }}] -f https://data.pyg.org/whl/torch-${{ inputs.torch_version }}+${{ inputs.hardware }}.html
+        rm -rf ~/.cache/pip
+        pip install --no-cache-dir git+https://github.com/thoglu/jammy_flows.git

.github/workflows/build.yml

@@ -14,98 +14,251 @@ on:
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
 
-  check-codeclimate-credentials:
-    name: Check CodeClimate credentials
-    runs-on: ubuntu-latest
+  check-Qlty-credentials:
+    name: Check Qlty credentials
+    runs-on: ubuntu-22.04
     outputs:
       has_credentials: ${{ steps.setvar.outputs.has_credentials }}
     steps:
       - name: Check secrets
         id: setvar
         run: |
-         if [[ "${{ secrets.CODECLIMATE_TEST_REPORTER_ID }}" != "" ]]; \
-         then
-           echo "Credentials to access CodeClimate found"
-           echo has_credentials="true" >> $GITHUB_OUTPUT
-         else
-           echo "Credentials to access CodeClimate not found"
-           echo has_credentials="false" >> $GITHUB_OUTPUT
-         fi
+          if [[ -n "${{ secrets.QLTY_TOKEN }}" ]]; then
+            echo "Credentials to access Qlty found."
+            echo "has_credentials=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "Credentials to access Qlty not found."
+            echo "has_credentials=false" >> "$GITHUB_OUTPUT"
+          fi
 
   build-icetray:
-    name: Unit tests - IceTray
-    needs: [ check-codeclimate-credentials ]
-    runs-on: ubuntu-latest
-    container: icecube/icetray:combo-stable
+    name: Unit tests - IceTray (v1.13.0 - 3.10)
+    needs: [ check-Qlty-credentials ]
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        torch_version: ['2.7.0']
+        hardware: ["cpu"]
+    container:
+      # GitHub Actions overwrite the docker container entrypoint
+      # inspect <image> --format '{{.Config.Entrypoint}}'
+      image: icecube/icetray:icetray-devel-v1.13.0-ubuntu22.04-2025-02-12
+
     steps:
-      - name: Set environment variables
+      - name: Mimmick Docker Entrypoint
+        # The entrypoint of the container sets python paths
+        # Because the entrypoint is skipped by GitHub, we set it here
+        run: |
+          echo "I3_SRC=/opt/icetray/share/icetray" >> $GITHUB_ENV
+          echo "I3_BUILD=/opt/icetray/share/icetray" >> $GITHUB_ENV
+          echo "I3_TESTDATA=/root/icetray/build/test-data" >> $GITHUB_ENV
+          echo "I3_PRODDATA=/root/icetray/build/prod-data" >> $GITHUB_ENV
+          echo "ROOTSYS=/opt/icetray/cernroot" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=/opt/icetray/lib:/opt/icetray/cernroot/lib:/opt/icetray/lib/tools:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+          echo "DYLD_LIBRARY_PATH=/opt/icetray/lib:/opt/icetray/cernroot/lib:/opt/icetray/lib/tools:$DYLD_LIBRARY_PATH" >> $GITHUB_ENV
+          echo "PYTHONPATH=/opt/icetray/lib:/opt/icetray/cernroot/lib:$PYTHONPATH" >> $GITHUB_ENV
+          echo "PATH=/opt/icetray/bin:$PATH" >> $GITHUB_ENV
+          echo "HDF5_USE_FILE_LOCKING=FALSE" >> $GITHUB_ENV
+      - name: Verify IceCube is importable
+        run: python3 -c "import icecube; print('Import Successful')"
+      - name: Show python version
+        run: |
+          python3 --version
+          pip --version
+          pip3 list
+          python3 -c "import icecube; print(icecube.__path__)"
+          pip show setuptools
+
+      - name: Install git
+        run: |
+          apt-get --yes install sudo
+          sudo apt update --fix-missing --yes
+          sudo apt upgrade --yes
+          sudo apt-get install --yes git
+          sudo apt-get clean 
+
+      - uses: actions/checkout@v4
+
+      - name: Create virtual environment
+        shell: bash
         run: |
-          echo "PATH=/usr/local/icetray/bin:$PATH" >> $GITHUB_ENV
-          echo "PYTHONPATH=/usr/local/icetray/lib:$PYTHONPATH" >> $GITHUB_ENV
-          echo "LD_LIBRARY_PATH=/usr/local/icetray/lib:/usr/local/icetray/cernroot/lib:/usr/local/icetray/lib/tools:$LD_LIBRARY_PATH" >> $GITHUB_ENV
-      - uses: actions/checkout@v3
-      - name: Upgrade packages already installed on icecube/icetray
+          python3 -m venv ~/venv --upgrade-deps
+          source ~/venv/bin/activate
+          pip --version
+          python --version
+          pip show setuptools
+          rm -rf ~/.cache/pip
+
+      - name: Print available disk space before graphnet install
+        run: | 
+          df -h
+      - name: Upgrade packages in virtual environment
+        shell: bash
         run: |
-          pip install --upgrade astropy  # Installed version incompatible with numpy 1.23.0 [https://github.com/astropy/astropy/issues/12534]
-          pip install --ignore-installed PyYAML  # Distutils installed [https://github.com/pypa/pip/issues/5247]
+          source ~/venv/bin/activate
+          pip install --no-cache-dir --upgrade astropy
+          pip install --no-cache-dir --upgrade PyYAML
+          pip install --no-cache-dir --upgrade psutil
+          pip install --no-cache-dir --upgrade setuptools
+          pip install --no-cache-dir --upgrade versioneer
+          pip show setuptools
+          pip show versioneer
+          rm -rf ~/.cache/pip
+      - name: Install PyTorch and dependencies
+        shell: bash
+        run: |
+          source ~/venv/bin/activate
+          pip3 install --no-cache-dir torch==${{ matrix.torch_version }}+${{ matrix.hardware }} torchvision torchaudio --index-url https://download.pytorch.org/whl/${{ matrix.hardware }}
+          rm -rf ~/.cache/pip
       - name: Install package
         uses: ./.github/actions/install
         with:
           editable: true
+          use_vm: true
+          torch_version: ${{ matrix.torch_version }}
+          hardware: ${{ matrix.hardware }}
+      - name: Print packages in pip
+        shell: bash
+        run: |
+          source ~/venv/bin/activate
+          pip show torch
+          pip show torch-geometric
+          pip show torch-cluster
+          pip show torch-sparse
+          pip show torch-scatter
+          pip show jammy_flows
+      - name: Print available disk space after graphnet install
+        run: | 
+          df -h
       - name: Run unit tests and generate coverage report
+        shell: bash
         run: |
-          coverage run --source=graphnet -m pytest tests/
+          source ~/venv/bin/activate
+          coverage run --source=graphnet -m pytest tests/ --ignore=tests/examples/04_training --ignore=tests/utilities
+          coverage run -a --source=graphnet -m pytest tests/examples/04_training
+          coverage run -a --source=graphnet -m pytest tests/utilities
           coverage xml -o coverage.xml
+          coverage report -m
       - name: Work around permission issue
         run: |
           git config --global --add safe.directory /__w/graphnet/graphnet
       - name: Publish code coverage
-        uses: paambaati/codeclimate-action@v3.0.0
-        if: needs.check-codeclimate-credentials.outputs.has_credentials == 'true'
-        env:
-          CC_TEST_REPORTER_ID: ${{ secrets.CODECLIMATE_TEST_REPORTER_ID }}
+        uses: qltysh/qlty-action/coverage@v2
+        if: needs.check-Qlty-credentials.outputs.has_credentials == 'true'
         with:
-          coverageCommand: coverage report
-          coverageLocations: coverage.xml:coverage.py
+          token: ${{secrets.QLTY_TOKEN}}
+          files: coverage.xml
 
-  build-matrix:
-    name: Unit tests - Python versions
-    runs-on: ubuntu-latest
+
+  build-matrix-examples:
+    name: Examples - Ubuntu 22.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10']
+        python-version: ['3.11']
+        torch_version: ['2.7.0', '2.6.0', '2.5.0']
+        hardware: ["cpu"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Print available disk space before graphnet install
+        run: df -h
+      - name: Show existing environment
+        run: |
+          echo "PIP_FLAGS=${PIP_FLAGS}"
+          python --version
+          pip --version
+          pip debug --verbose
       - name: Install package
         uses: ./.github/actions/install
         with:
           editable: true
+          torch_version: ${{ matrix.torch_version }}
+          hardware: ${{ matrix.hardware }}
+      - name: Print available disk space after graphnet install
+        run: df -h
+      - name: Print packages in pip
+        run: |
+          pip show torch
+          pip show torch-geometric
+          pip show torch-cluster
+          pip show torch-sparse
+          pip show torch-scatter
+          pip show dill
+          pip show numpy
       - name: Run unit tests and generate coverage report
         run: |
           set -o pipefail  # To propagate exit code from pytest
-          coverage run --source=graphnet -m pytest tests/ --ignore=tests/data/
-          coverage report -m
+          pytest tests/examples --ignore=tests/examples/01_icetray/
+      - name: Print available disk space after unit tests
+        run: df -h
+
+  build-macos-examples:
+    # Runtime on macOS is multiplied with a factor 10 by github. Minimize!
+    name: Examples - macOS 15
+    runs-on: macos-15
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.11']
+        torch_version: ['2.7.0']
+        hardware: ["cpu"]
 
-  build-macos:
-    name: Unit tests - macOS
-    runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
-      - name: Install package
+          python-version: ${{ matrix.python-version }}
+
+      - name: Show Python version
+        run: |
+          python --version
+          which python
+          pip --version
+
+      - name: Install dependencies
         uses: ./.github/actions/install
         with:
-          editable: true
-          hardware: "macos"
+          torch_version: ${{ matrix.torch_version }}
+          hardware: ${{ matrix.hardware }}
+      - name: Print packages in pip
+        run: |
+            pip show torch
+            pip show torch-geometric
+            pip show torch-cluster
+            pip show torch-sparse
+            pip show torch-scatter
       - name: Run unit tests and generate coverage report
         run: |
           set -o pipefail  # To propagate exit code from pytest
-          coverage run --source=graphnet -m pytest tests/ --ignore=tests/data/
-          coverage report -m
+          pytest tests/examples --ignore=tests/examples/01_icetray/
+
+  docs:
+    name: Documentation Compilation
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: Install package
+        uses: ./.github/actions/install
+      - name: Build documentation
+        run: |
+          cd docs
+          make clean
+          sphinx-apidoc \
+            --module-first \
+            --separate \
+            --force \
+            -d 2 \
+            --templatedir=source/_templates/ \
+            -o source/api ../src/
+          sed -i "2s/.*/API/" source/api/graphnet.rst
+          make html

.github/workflows/code-quality.yml

@@ -17,14 +17,20 @@ jobs:
 
   pre-commit:
     name: Pre-commit - ${{ matrix.hook }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         hook: ["black", "flake8", "docformatter", "pydocstyle", "mypy"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
       - name: Install package
         uses: ./.github/actions/install
+
       - name: Run ${{ matrix.hook }}
         run: |
+          pip show ${{ matrix.hook }}
           pre-commit run ${{ matrix.hook }} --all-files