Release v0.0.2 and make it work with pytask v0.0.9. (#1)

tobiasraabe · web-flow · commit bd244d719a8d · 2020-10-30T15:57:57.000+01:00
diff --git a/.conda/meta.yaml b/.conda/meta.yaml
@@ -20,11 +20,12 @@ requirements:
 
   run:
     - python >=3.6
-    - pytask >=0.0.8
+    - pytask >=0.0.9
 
 test:
   requires:
     - pytest
+    - pytask-parallel >=0.0.4
   source_files:
     - tox.ini
     - tests
@@ -33,6 +34,7 @@ test:
     - pytask --help
     - pytask markers
     - pytask clean
+    - pytask collect
 
     - pytest tests
 
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -6,7 +6,13 @@ chronological order. Releases follow `semantic versioning <https://semver.org/>`
 all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask-stata>`_.
 
 
-0.0.1 - 2020-xx-xx
+0.0.2 - 2020-10-30
+------------------
+
+- :gh:`1` makes pytask-stata work with pytask v0.0.9.
+
+
+0.0.1 - 2020-10-04
 ------------------
 
 - Release v0.0.1.
diff --git a/README.rst b/README.rst
@@ -57,20 +57,47 @@ Here is an example where you want to run ``script.do``.
     def task_run_do_file():
         pass
 
-Note that, you need to apply the ``@pytask.mark.stata`` marker so that pytask-stata
-handles the task. The do-file must be the first dependency. Other dependencies can be
-added after that.
+
+Multiple dependencies and products
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+What happens if a task has more dependencies? Using a list, the do-file which should be
+executed must be found in the first position of the list.
 
 .. code-block:: python
 
     @pytask.mark.stata
     @pytask.mark.depends_on(["script.do", "input.dta"])
-    @pytask.mark.produces("out.dta")
+    @pytask.mark.produces("output.dta")
+    def task_run_do_file():
+        pass
+
+If you use a dictionary to pass dependencies to the task, pytask-stata will, first, look
+for a ``"source"`` key in the dictionary and, secondly, under the key ``0``.
+
+.. code-block:: python
+
+    @pytask.mark.depends_on({"source": "script.do", "input": "input.dta"})
+    def task_run_do_file():
+        pass
+
+
+    # or
+
+
+    @pytask.mark.depends_on({0: "script.do", "input": "input.dta"})
+    def task_run_do_file():
+        pass
+
+
+    # or two decorators for the function, if you do not assign a name to the input.
+
+
+    @pytask.mark.depends_on({"source": "script.do"})
+    @pytask.mark.depends_on("input.dta")
     def task_run_do_file():
         pass
 
-If you are wondering why the function body is empty, know that pytask-stata replaces the
-body with a predefined internal function which will execute the do-file.
 
 
 Command Line Arguments
@@ -142,7 +169,7 @@ include the ``@pytask.mark.stata`` decorator in the parametrization just like wi
     @pytask.mark.depends_on("script.do")
     @pytask.mark.parametrize(
         "produces, stata",
-        [("output_1.dta", 1), ("output_2.dta", 2)],
+        [("output_1.dta", ("1",)), ("output_2.dta", ("2",))],
     )
     def task_execute_do_file():
         pass
@@ -180,6 +207,14 @@ stata_check_log_lines
 
         $ pytask build --stata-check-log-lines 10
 
+stata_source_key
+    If you want to change the name of the key which identifies the do file, change the
+    following default configuration in your pytask configuration file.
+
+    .. code-block:: ini
+
+        stata_source_key = source
+
 
 Changes
 -------
diff --git a/environment.yml b/environment.yml
@@ -12,7 +12,8 @@ dependencies:
   - conda-verify
 
   # Package dependencies
-  - pytask >= 0.0.8
+  - pytask >=0.0.9
+  - pytask-parallel >=0.0.4
 
   # Misc
   - black
diff --git a/setup.cfg b/setup.cfg
@@ -1,7 +1,7 @@
 [bumpversion]
-current_version = 0.0.1
+current_version = 0.0.2
 parse = (?P<major>\d+)\.(?P<minor>\d+)(\.(?P<patch>\d+))(\-?((dev)?(?P<dev>\d+))?)
-serialize =
+serialize = 
 	{major}.{minor}.{patch}dev{dev}
 	{major}.{minor}.{patch}
 
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name="pytask-stata",
-    version="0.0.1",
+    version="0.0.2",
     packages=find_packages(where="src"),
     package_dir={"": "src"},
     entry_points={"pytask": ["pytask_stata = pytask_stata.plugin"]},
diff --git a/src/pytask_stata/__init__.py b/src/pytask_stata/__init__.py
@@ -1 +1 @@
-__version__ = "0.0.1"
+__version__ = "0.0.2"
diff --git a/src/pytask_stata/collect.py b/src/pytask_stata/collect.py
@@ -2,6 +2,7 @@
 import copy
 import functools
 import subprocess
+from pathlib import Path
 from typing import Iterable
 from typing import Optional
 from typing import Union
@@ -56,13 +57,11 @@ def pytask_collect_task(session, path, name, obj):
 @hookimpl
 def pytask_collect_task_teardown(session, task):
     """Perform some checks and prepare the task function."""
-    if task is not None and get_specific_markers_from_task(task, "stata"):
-        if (len(task.depends_on) == 0) or (
-            not (
-                isinstance(task.depends_on[0], FilePathNode)
-                and task.depends_on[0].value.suffix == ".do"
-            )
-        ):
+    if get_specific_markers_from_task(task, "stata"):
+        source = _get_node_from_dictionary(
+            task.depends_on, session.config["stata_source_key"]
+        )
+        if not (isinstance(source, FilePathNode) and source.value.suffix == ".do"):
             raise ValueError(
                 "The first dependency of a Stata task must be the do-file."
             )
@@ -78,6 +77,14 @@ def pytask_collect_task_teardown(session, task):
         task.function = stata_function
 
 
+def _get_node_from_dictionary(obj, key, fallback=0):
+    if isinstance(obj, Path):
+        pass
+    elif isinstance(obj, dict):
+        obj = obj.get(key) or obj.get(fallback)
+    return obj
+
+
 def _merge_all_markers(task):
     """Combine all information from markers for the Stata function."""
     stata_marks = get_specific_markers_from_task(task, "stata")
@@ -94,12 +101,15 @@ def _prepare_cmd_options(session, task, args):
     is unique and does not cause any errors when parallelizing the execution.
 
     """
+    source = _get_node_from_dictionary(
+        task.depends_on, session.config["stata_source_key"]
+    )
     log_name = convert_task_id_to_name_of_log_file(task.name)
     return [
         session.config["stata"],
         "-e",
         "do",
-        task.depends_on[0].value.as_posix(),
+        source.value.as_posix(),
         *args,
         f"-{log_name}",
     ]
diff --git a/src/pytask_stata/config.py b/src/pytask_stata/config.py
@@ -36,6 +36,8 @@ def pytask_parse_config(config, config_from_cli, config_from_file):
         default=10,
     )
 
+    config["stata_source_key"] = config_from_file.get("stata_source_key", "source")
+
 
 def _nonnegative_nonzero_integer(x):
     """Check for nonnegative and nonzero integer."""
diff --git a/src/pytask_stata/execute.py b/src/pytask_stata/execute.py
@@ -33,21 +33,22 @@ def pytask_execute_task_teardown(session, task):
     or ``r(601)``.
 
     """
-    log_name = convert_task_id_to_name_of_log_file(task.name)
-    path_to_log = task.path.with_name(log_name).with_suffix(".log")
+    if get_specific_markers_from_task(task, "stata"):
+        log_name = convert_task_id_to_name_of_log_file(task.name)
+        path_to_log = task.path.with_name(log_name).with_suffix(".log")
 
-    n_lines = session.config["stata_check_log_lines"]
+        n_lines = session.config["stata_check_log_lines"]
 
-    log = path_to_log.read_text()
-    log_tail = log.split("\n")[-n_lines:]
-    for line in log_tail:
-        error_found = re.match(r"r\(([0-9]+)\)", line)
+        log = path_to_log.read_text()
+        log_tail = log.split("\n")[-n_lines:]
+        for line in log_tail:
+            error_found = re.match(r"r\(([0-9]+)\)", line)
 
-        if error_found:
-            if not session.config["stata_keep_log"]:
-                path_to_log.unlink()
+            if error_found:
+                if not session.config["stata_keep_log"]:
+                    path_to_log.unlink()
 
-            raise RuntimeError(
-                f"An error occurred. Here are the last {n_lines} lines of the log:\n\n"
-                + "\n".join(log_tail)
-            )
+                raise RuntimeError(
+                    f"An error occurred. Here are the last {n_lines} lines of the log:"
+                    "\n\n" + "\n".join(log_tail)
+                )
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,6 +1,7 @@
 import shutil
 
 import pytest
+from click.testing import CliRunner
 from pytask_stata.config import STATA_COMMANDS
 
 
@@ -11,3 +12,8 @@
     is None,
     reason="Stata needs to be installed.",
 )
+
+
+@pytest.fixture()
+def runner():
+    return CliRunner()
diff --git a/tests/test_collect.py b/tests/test_collect.py
@@ -4,6 +4,7 @@
 import pytest
 from _pytask.mark import Mark
 from _pytask.nodes import FilePathNode
+from pytask_stata.collect import _get_node_from_dictionary
 from pytask_stata.collect import _merge_all_markers
 from pytask_stata.collect import _prepare_cmd_options
 from pytask_stata.collect import pytask_collect_task
@@ -63,14 +64,15 @@ def test_merge_all_markers(marks, expected):
         ["a", "b"],
     ],
 )
-def test_prepare_cmd_options(args):
+@pytest.mark.parametrize("stata_source_key", ["source", "do"])
+def test_prepare_cmd_options(args, stata_source_key):
     session = DummyClass()
-    session.config = {"stata": "stata"}
+    session.config = {"stata": "stata", "stata_source_key": stata_source_key}
 
     node = DummyClass()
     node.value = Path("script.do")
     task = DummyClass()
-    task.depends_on = [node]
+    task.depends_on = {stata_source_key: node}
     task.name = "task"
 
     result = _prepare_cmd_options(session, task, args)
@@ -115,11 +117,15 @@ def test_pytask_collect_task(name, expected):
 )
 def test_pytask_collect_task_teardown(depends_on, produces, expectation):
     session = DummyClass()
-    session.config = {"stata": "stata"}
+    session.config = {"stata": "stata", "stata_source_key": "source"}
 
     task = DummyClass()
-    task.depends_on = [FilePathNode(n.split(".")[0], Path(n)) for n in depends_on]
-    task.produces = [FilePathNode(n.split(".")[0], Path(n)) for n in produces]
+    task.depends_on = {
+        i: FilePathNode(n.split(".")[0], Path(n)) for i, n in enumerate(depends_on)
+    }
+    task.produces = {
+        i: FilePathNode(n.split(".")[0], Path(n)) for i, n in enumerate(produces)
+    }
     task.function = task_dummy
     task.name = "task_dummy"
 
@@ -129,3 +135,18 @@ def test_pytask_collect_task_teardown(depends_on, produces, expectation):
 
     with expectation:
         pytask_collect_task_teardown(session, task)
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    "obj, key, expected",
+    [
+        (1, "asds", 1),
+        (1, None, 1),
+        ({"a": 1}, "a", 1),
+        ({0: 1}, "a", 1),
+    ],
+)
+def test_get_node_from_dictionary(obj, key, expected):
+    result = _get_node_from_dictionary(obj, key)
+    assert result == expected
diff --git a/tests/test_normal_execution_w_plugin.py b/tests/test_normal_execution_w_plugin.py
@@ -0,0 +1,36 @@
+"""Contains tests which do not require the plugin and ensure normal execution."""
+import textwrap
+
+import pytest
+from pytask import cli
+
+
+@pytest.mark.end_to_end
+@pytest.mark.parametrize(
+    "dependencies",
+    [[], ["in.txt"], ["in_1.txt", "in_2.txt"]],
+)
+@pytest.mark.parametrize("products", [["out.txt"], ["out_1.txt", "out_2.txt"]])
+def test_execution_w_varying_dependencies_products(
+    runner, tmp_path, dependencies, products
+):
+    source = f"""
+    import pytask
+    from pathlib import Path
+
+    @pytask.mark.depends_on({dependencies})
+    @pytask.mark.produces({products})
+    def task_dummy(depends_on, produces):
+        if isinstance(produces, dict):
+            produces = produces.values()
+        elif isinstance(produces, Path):
+            produces = [produces]
+        for product in produces:
+            product.touch()
+    """
+    tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source))
+    for dependency in dependencies:
+        tmp_path.joinpath(dependency).touch()
+
+    result = runner.invoke(cli, [tmp_path.as_posix()])
+    assert result.exit_code == 0
diff --git a/tests/test_parallel.py b/tests/test_parallel.py
diff --git a/tox.ini b/tox.ini

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "0.0.1"`
	`1`	`+__version__ = "0.0.2"`
Original file line number	Diff line number	Diff line change
`@@ -36,6 +36,8 @@ def pytask_parse_config(config, config_from_cli, config_from_file):`
`36`	`36`	`default=10,`
`37`	`37`	`)`
`38`	`38`
	`39`	`+ config["stata_source_key"] = config_from_file.get("stata_source_key", "source")`
	`40`	`+`
`39`	`41`
`40`	`42`	`def _nonnegative_nonzero_integer(x):`
`41`	`43`	`"""Check for nonnegative and nonzero integer."""`