Better error message when entering the debugger while parallelizing. (#68)

Author: tobiasraabe

.github/workflows/main.yml

@@ -27,26 +27,16 @@ jobs:
       fail-fast: false
       matrix:
         os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
 
     steps:
       - uses: actions/checkout@v4
-      - uses: mamba-org/setup-micromamba@v1
+      - uses: actions/setup-python@v4
         with:
-          environment-name: gha-testing
-          condarc: |
-            channels:
-              - nodefaults
-              - conda-forge
-          create-args: >-
-            python=${{ matrix.python-version }}
-            mamba
-            tox-conda
-          cache-environment: true
-
-      - name: Install core dependencies.
-        shell: bash -l {0}
-        run: conda install -c conda-forge tox-conda coverage
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          allow-prereleases: true
+      - run: pip install tox
 
       # Unit, integration, and end-to-end tests.
 
@@ -55,24 +45,15 @@ jobs:
         run: tox -e pytest -- tests -m "unit or (not integration and not end_to_end)" --cov=./ --cov-report=xml -n auto
 
       - name: Upload coverage report for unit tests and doctests.
-        if: runner.os == 'Linux' && matrix.python-version == '3.9'
+        if: runner.os == 'Linux' && matrix.python-version == '3.10'
         shell: bash -l {0}
         run: bash <(curl -s https://codecov.io/bash) -F unit -c
 
-      # - name: Run integration tests.
-      #   shell: bash -l {0}
-      #   run: tox -e pytest -- tests -m integration --cov=./ --cov-report=xml -n auto
-
-      # - name: Upload coverage reports of integration tests.
-      #   if: runner.os == 'Linux' && matrix.python-version == '3.9'
-      #   shell: bash -l {0}
-      #   run: bash <(curl -s https://codecov.io/bash) -F integration -c
-
       - name: Run end-to-end tests.
         shell: bash -l {0}
         run: tox -e pytest -- tests -m end_to_end --cov=./ --cov-report=xml -n auto
 
       - name: Upload coverage reports of end-to-end tests.
-        if: runner.os == 'Linux' && matrix.python-version == '3.9'
+        if: runner.os == 'Linux' && matrix.python-version == '3.10'
         shell: bash -l {0}
         run: bash <(curl -s https://codecov.io/bash) -F end_to_end -c

.pre-commit-config.yaml

@@ -8,7 +8,6 @@ repos:
     -   id: check-merge-conflict
     -   id: check-vcs-permalinks
     -   id: check-yaml
-    -   id: debug-statements
     -   id: end-of-file-fixer
     -   id: fix-byte-order-marker
     -   id: mixed-line-ending
@@ -72,9 +71,10 @@ repos:
             --ignore-missing-imports,
         ]
         additional_dependencies: [
+            attrs,
             cloudpickle,
-            optree,
-            pytask==0.4.0rc2,
+            loky,
+            pytask==0.4.0,
             rich,
             types-attrs,
             types-click,

CHANGES.md

@@ -5,13 +5,14 @@ chronological order. Releases follow [semantic versioning](https://semver.org/)
 releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and
 [Anaconda.org](https://anaconda.org/conda-forge/pytask-parallel).
 
-## 0.4.0 - 2023-xx-xx
+## 0.4.0 - 2023-10-07
 
 - {pull}`62` deprecates Python 3.7.
 - {pull}`64` aligns pytask-parallel with pytask v0.4.0rc2.
 - {pull}`66` deactivates parallelization for dry-runs.
 - {pull}`67` fixes parallelization with partialed task functions.
-- {pull}`68` updates dependencies and syntaxes.
+- {pull}`68` raises more informative error message when `breakpoint()` was uses when
+  parallelizing with processes or loky.
 
 ## 0.3.1 - 2023-05-27
 

environment.yml

@@ -12,7 +12,7 @@ dependencies:
   - toml
 
   # Package dependencies
-  - pytask>=0.4.0rc4
+  - pytask>=0.4.0
   - cloudpickle
   - loky
   - optree

setup.cfg

@@ -28,9 +28,8 @@ install_requires =
     click
     cloudpickle
     loky
-    optree>=0.9.0
     pluggy>=1.0.0
-    pytask>=0.4.0rc2
+    pytask>=0.4.0
     rich
 python_requires = >=3.8
 include_package_data = True

src/pytask_parallel/execute.py

@@ -210,6 +210,28 @@ def pytask_execute_task(session: Session, task: PTask) -> Future[Any] | None:
         return None
 
 
+def _raise_exception_on_breakpoint(*args: Any, **kwargs: Any) -> None:  # noqa: ARG001
+    raise RuntimeError(
+        "You cannot use 'breakpoint()' or 'pdb.set_trace()' while parallelizing the "
+        "execution of tasks with pytask-parallel. Please, remove the breakpoint or run "
+        "the task without parallelization to debug it."
+    )
+
+
+def _patch_set_trace_and_breakpoint() -> None:
+    """Patch :func:`pdb.set_trace` and :func:`breakpoint`.
+
+    Patch sys.breakpointhook to intercept any call of breakpoint() and pdb.set_trace in
+    a subprocess and print a better exception message.
+
+    """
+    import pdb  # noqa: T100
+    import sys
+
+    pdb.set_trace = _raise_exception_on_breakpoint
+    sys.breakpointhook = _raise_exception_on_breakpoint
+
+
 def _execute_task(  # noqa: PLR0913
     task: PTask,
     kwargs: dict[str, Any],
@@ -225,6 +247,7 @@ def _execute_task(  # noqa: PLR0913
 
     """
     __tracebackhide__ = True
+    _patch_set_trace_and_breakpoint()
 
     with warnings.catch_warnings(record=True) as log:
         # mypy can't infer that record=True means log is not None; help it.

tests/test_execute.py

@@ -268,3 +268,17 @@ def test_parallel_execution_is_deactivated(runner, tmp_path, flag, parallel_back
     )
     assert result.exit_code == ExitCode.OK
     assert "Started 2 workers" not in result.output
+
+
+@pytest.mark.end_to_end()
+@pytest.mark.parametrize("code", ["breakpoint()", "import pdb; pdb.set_trace()"])
+@pytest.mark.parametrize(
+    "parallel_backend", [i for i in PARALLEL_BACKENDS if i != ParallelBackend.THREADS]
+)
+def test_raise_error_on_breakpoint(runner, tmp_path, code, parallel_backend):
+    tmp_path.joinpath("task_example.py").write_text(f"def task_example(): {code}")
+    result = runner.invoke(
+        cli, [tmp_path.as_posix(), "-n 2", "--parallel-backend", parallel_backend]
+    )
+    assert result.exit_code == ExitCode.FAILED
+    assert "You cannot use 'breakpoint()'" in result.output

tox.ini

@@ -5,20 +5,17 @@ envlist = pytest
 usedevelop = true
 
 [testenv:pytest]
-conda_channels =
-    conda-forge/label/pytask_rc
-    conda-forge
-    nodefaults
-conda_deps =
+deps =
+    # pytest
+    pytest
+    pytest-cov
+    pytest-xdist
     setuptools_scm
     toml
 
+    # Package
+    pytask >=0.4.0
     cloudpickle
     loky
-    pytask >=0.4.0rc2
-    pytest
-    pytest-cov
-    pytest-xdist
 commands =
-    pip install --no-deps -e .
     pytest {posargs}