[2298] Fix CI

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
         uses: astral-sh/setup-uv@v5
         with:
           # Install a specific version of uv.
-          version: "0.7.13"
+          version: "0.11.8"
 
       - name: Run ruff (black)
         # Do not attempt to install the default dependencies, this is much faster.
@@ -46,7 +46,7 @@ jobs:
         uses: astral-sh/setup-uv@v5
         with:
           # Install a specific version of uv.
-          version: "0.7.13"
+          version: "0.11.8"
 
       - name: Check PR is linked to an issue
       # Send the PR number to the script, which will check if it is linked to an issue.
@@ -62,7 +62,7 @@ jobs:
         uses: astral-sh/setup-uv@v5
         with:
           # Install a specific version of uv.
-          version: "0.7.13"
+          version: "0.11.8"
 
       - name: Run all unit tests
       # Send the PR number to the script, which will check if it is linked to an issue.

ci/cscs.yaml

@@ -22,9 +22,15 @@ test_job:
       https://oauth2:${PRIVATE_REPO_TOKEN}@gitlab.jsc.fz-juelich.de/esde/WeatherGenerator-private.git
 
     echo "Sync"
-    ./scripts/actions.sh sync
+    # DEBUG
+    ls .*
+    ./scripts/actions.sh sync-safe
     echo "Create links and run tests"
     ./scripts/actions.sh create-links
+    echo "Activate the environment"
+    source .venv/bin/activate
+    which python3
+    which pytest
     echo "Run tests"
     ./scripts/actions.sh integration-test${STAGE_TYPE:-}
   variables:

pyproject.toml

@@ -27,7 +27,8 @@ dependencies = [
  "anemoi-datasets",
  "weathergen-common",
  "weathergen-evaluate",
- "weathergen-readers-extra"
+ "weathergen-readers-extra",
+ "pytest>=9.0.2",
 ]
 
 
@@ -161,6 +162,7 @@ enable = ["W0201", "W0141"]
 [tool.uv]
 # Most work is done a distributed filesystem, where hardlink is not always possible.
 # Also, trying to resolve some permissions issue, see 44.
+# Another option is 'symlink': this can cause broken symlinks to appear but it is faster (no copy)
 link-mode = "symlink"
 # This guarantees that the build is deterministic and will not be impacted 
 # by future releases of dependencies or sub-dependencies. 
@@ -172,7 +174,7 @@ exclude-newer = "2026-02-27T00:00:00Z"
 # It is tightly controlled because the format of uv.lock has changed 
 # over revisions, causing reformats to happen without reason.
 # Also, relatively recent versions are required to support workspaces.
-required-version = ">=0.7.0"
+required-version = ">=0.11.0"
 
 # The supported environments
 # TODO: add macos and windows (CPU only, for running tests)

scripts/actions.sh

@@ -8,6 +8,7 @@ case "$1" in
   sync)
     (
       cd "$SCRIPT_DIR" || exit 1
+      # Creates a virtual environment without checking the integrity of the cache.
       # If we are running on a mac, use the cpu extra
       if [[ "$(uname)" == "Darwin" ]]; then
         uv sync --all-packages --extra cpu
@@ -17,6 +18,20 @@ case "$1" in
       uv sync --all-packages --extra gpu
     )
     ;;
+  sync-safe)
+    (
+      cd "$SCRIPT_DIR" || exit 1
+      # --refresh --reinstall : LUSTRE may clean up some pieces of the cache.
+      # This ensures basic integrity but it is too slow (adds 60 seconds to the sync) to do it on every sync. So we only do it on mac, where the cache is more likely to get corrupted.
+      # If we are running on a mac, use the cpu extra
+      if [[ "$(uname)" == "Darwin" ]]; then
+        uv sync --all-packages --extra cpu --refresh --reinstall
+        exit 0
+      fi
+      # Otherwise, use the gpu extra
+      uv sync --all-packages --extra gpu --refresh --reinstall
+    )
+    ;;
   lint)
     (
       cd "$SCRIPT_DIR" || exit 1
@@ -128,7 +143,11 @@ case "$1" in
       # 1. Get the path of the private config of the cluster
       # 2. Read the yaml and extract the path of the shared conf
       # This uses the yq command. It is a python package so uvx (bundled with uv) will donwload and create the right venv
-      export working_dir=$(cat $("$PRIVATE_REPO_PATH"/hpc/platform-env.py hpc-config) | uvx yq .path_shared_working_dir)
+      # The 'yq' command is used in a separate virtual environment, because the cache 
+      # around that tool can get corrupted. 
+      # See https://github.com/ecmwf/WeatherGenerator/issues/2298
+      export working_dir=$(cat "$("$PRIVATE_REPO_PATH"/hpc/platform-env.py hpc-config)" |
+        UV_CACHE_DIR="$(mktemp -d)" VIRTUAL_ENV=""  uvx yq .path_shared_working_dir)      
       # Remove quotes
       export working_dir=$(echo "$working_dir" | sed 's/[\"\x27]//g')
       # If the working directory does not exist, exit with an error