Add depency resolving on advanced copy endpoint

CHANGES/386.feature

@@ -0,0 +1,6 @@
+Implemented ``dependency_solving`` on the advanced copy endpoint. When enabled, the
+copy task BFS-walks the Depends/Pre-Depends closure of every initial package, picks
+the newest version satisfying each relation from the source repository, honours
+Provides aliases, and skips relations already satisfied by the destination base
+version. Unsatisfiable relations raise an explicit error. Resolves
+:redmine:`386`.

pulp_deb/app/serializers/repository_serializers.py

@@ -179,8 +179,9 @@ class CopySerializer(ValidateFieldsMixin, serializers.Serializer):
 
     dependency_solving = serializers.BooleanField(
         help_text=_(
-            "Also copy dependencies of any packages being copied. NOT YET"
-            'IMPLEMENTED! You must keep this at "False"!'
+            "Also copy the transitive Depends/Pre-Depends closure of every package being copied. "
+            "Already-satisfied relations (present in dest_base_version) are skipped. "
+            "Raises an error if any relation cannot be satisfied from the source repository."
         ),
         default=False,
     )

pulp_deb/app/tasks/copy.py

@@ -14,6 +14,7 @@
     Release,
     ReleaseArchitecture,
 )
+from pulp_deb.app.tasks.dependency_solving import solve_dependencies
 
 log = logging.getLogger(__name__)
 
@@ -104,9 +105,6 @@ def process_entry(entry):
             content_filter,
         )
 
-    if dependency_solving:
-        raise NotImplementedError("Advanced copy with dependency solving is not yet implemented.")
-
     for entry in config:
         (
             source_repo_version,
@@ -116,6 +114,16 @@ def process_entry(entry):
         ) = process_entry(entry)
 
         content_to_copy = source_repo_version.content.filter(content_filter)
+
+        if dependency_solving:
+            initial_packages = Package.objects.filter(
+                pk__in=content_to_copy.filter(pulp_type=Package.get_pulp_type()).only("pk")
+            )
+            extra_pks = solve_dependencies(initial_packages, source_repo_version, dest_base_version)
+            content_to_copy = source_repo_version.content.filter(
+                Q(pk__in=content_to_copy.values("pk")) | Q(pk__in=extra_pks)
+            )
+
         if structured:
             content_to_copy = find_structured_publish_content(content_to_copy, source_repo_version)
 

pulp_deb/app/tasks/dependency_solving.py

@@ -0,0 +1,160 @@
+from collections import deque
+from gettext import gettext as _
+
+from debian.deb822 import PkgRelation
+from debian.debian_support import Version
+from django.db.models import Q
+
+from pulp_deb.app.models import Package
+
+
+class DependencySolveError(Exception):
+    """Raised when a transitive dependency cannot be satisfied from the source repository."""
+
+
+_OP_FUNCS = {
+    ">=": lambda a, b: a >= b,
+    ">>": lambda a, b: a > b,
+    "<=": lambda a, b: a <= b,
+    "<<": lambda a, b: a < b,
+    "=": lambda a, b: a == b,
+}
+
+
+def _parse_relations(*depends_strings):
+    combined = ", ".join(s for s in depends_strings if s)
+    if not combined.strip():
+        return []
+    try:
+        return PkgRelation.parse_relations(combined)
+    except Exception:
+        return []
+
+
+def _version_satisfies(pkg_version, constraint):
+    if not constraint:
+        return True
+    op, target = constraint
+    return _OP_FUNCS.get(op, lambda a, b: False)(Version(pkg_version), Version(target))
+
+
+def _candidates_for(alt, package_qs, pkg_arch=None):
+    name = alt["name"]
+    qs = package_qs.filter(package=name)
+    alt_arch = alt.get("arch")
+    if alt_arch:
+        qs = qs.filter(architecture=alt_arch)
+    elif pkg_arch:
+        # Restrict deps to the consumer's arch or arch-independent (`all`).
+        qs = qs.filter(Q(architecture=pkg_arch) | Q(architecture="all"))
+    return qs
+
+
+def _find_satisfier(alternative, package_qs, pkg_arch=None):
+    """Best Package satisfying a single OR-alternative (direct hit or via Provides)."""
+    constraint = alternative.get("version")
+
+    direct = list(_candidates_for(alternative, package_qs, pkg_arch))
+    direct.sort(key=lambda p: Version(p.version), reverse=True)
+    for pkg in direct:
+        if _version_satisfies(pkg.version, constraint):
+            return pkg
+
+    # Provides: a Package P providing the name we need.  Versioned Provides
+    # are rare; we honour them when present, otherwise treat as unversioned.
+    name = alternative["name"]
+    provides_qs = package_qs.exclude(provides__isnull=True).exclude(provides="")
+    if pkg_arch:
+        provides_qs = provides_qs.filter(Q(architecture=pkg_arch) | Q(architecture="all"))
+    for pkg in provides_qs:
+        for or_group in _parse_relations(pkg.provides):
+            for prov in or_group:
+                if prov["name"] != name:
+                    continue
+                prov_version = prov.get("version")
+                if not constraint:
+                    return pkg
+                if (
+                    prov_version
+                    and prov_version[0] == "="
+                    and _version_satisfies(prov_version[1], constraint)
+                ):
+                    return pkg
+                # Unversioned Provides cannot satisfy a versioned dependency per policy.
+    return None
+
+
+def _relation_satisfied_by_set(relation, package_qs, pkg_arch=None):
+    return any(_find_satisfier(alt, package_qs, pkg_arch) is not None for alt in relation)
+
+
+def _packages_in(repo_version):
+    if repo_version is None:
+        return Package.objects.none()
+    return Package.objects.filter(
+        pk__in=repo_version.content.filter(pulp_type=Package.get_pulp_type()).only("pk")
+    )
+
+
+def _format_relation(relation):
+    parts = []
+    for alt in relation:
+        s = alt["name"]
+        if alt.get("arch"):
+            s += ":{}".format(alt["arch"])
+        if alt.get("version"):
+            s += " ({} {})".format(*alt["version"])
+        parts.append(s)
+    return " | ".join(parts)
+
+
+def solve_dependencies(initial_packages, source_repo_version, dest_base_version=None):
+    """BFS over Depends + Pre-Depends. Returns the set of Package pks to copy.
+
+    A relation already satisfied by ``dest_base_version`` is skipped; otherwise
+    the newest satisfying Package in ``source_repo_version`` is chosen, then
+    walked transitively.  Raises :class:`DependencySolveError` on unsatisfiable
+    relations.
+    """
+    source_pkgs = _packages_in(source_repo_version)
+    dest_pkgs = _packages_in(dest_base_version)
+
+    seen = set()
+    queue = deque(initial_packages)
+    final = set()
+
+    while queue:
+        pkg = queue.popleft()
+        if pkg.pk in seen:
+            continue
+        seen.add(pkg.pk)
+        final.add(pkg.pk)
+
+        for relation in _parse_relations(pkg.depends, pkg.pre_depends):
+            if dest_base_version is not None and _relation_satisfied_by_set(
+                relation, dest_pkgs, pkg.architecture
+            ):
+                continue
+
+            chosen = None
+            for alt in relation:
+                chosen = _find_satisfier(alt, source_pkgs, pkg.architecture)
+                if chosen:
+                    break
+
+            if chosen is None:
+                raise DependencySolveError(
+                    _(
+                        "Cannot solve dependency for {pkg}: "
+                        "'{rel}' is not satisfiable in source repository version {srv}"
+                    ).format(
+                        pkg=pkg.name,
+                        rel=_format_relation(relation),
+                        srv=source_repo_version.pk,
+                    )
+                )
+
+            if chosen.pk not in seen:
+                queue.append(chosen)
+
+    return final

pulp_deb/tests/functional/api/test_copy.py

@@ -76,3 +76,32 @@ def test_copy_empty_content(
 
     target_repo = deb_get_repository_by_href(target_repo.pulp_href)
     assert target_repo.latest_version_href.endswith("/versions/0/")
+
+
+@pytest.mark.parallel
+def test_copy_with_dependency_solving(
+    deb_init_and_sync,
+    deb_repository_factory,
+    apt_package_api,
+    deb_copy_content,
+    deb_get_repository_by_href,
+    deb_get_content_summary,
+):
+    """Smoke test that ``dependency_solving=True`` reaches the new code path."""
+    source_repo, _ = deb_init_and_sync()
+    target_repo = deb_repository_factory()
+    package = apt_package_api.list(
+        package="frigg",
+        repository_version=source_repo.latest_version_href,
+    ).results[0]
+
+    deb_copy_content(
+        source_repo_version=source_repo.latest_version_href,
+        dest_repo=target_repo.pulp_href,
+        content=[package.pulp_href],
+        dependency_solving=True,
+    )
+
+    target_repo = deb_get_repository_by_href(target_repo.pulp_href)
+    added_summary = deb_get_content_summary(target_repo).added
+    assert DEB_ADVANCED_COPY_FIXTURE_SUMMARY == get_counts_from_content_summary(added_summary)

pulp_deb/tests/functional/conftest.py

@@ -467,19 +467,26 @@ def _deb_acs_factory(**kwargs):
 def deb_copy_content(apt_copy_api, monitor_task):
     """Fixture that copies deb content from a source repository version to a target repository."""
 
-    def _deb_copy_content(source_repo_version, dest_repo, content=None, structured=True):
+    def _deb_copy_content(
+        source_repo_version,
+        dest_repo,
+        content=None,
+        structured=True,
+        dependency_solving=False,
+    ):
         """Copy deb content from a source repository version to a target repository.
 
         :param source_repo_version: The repository version href from where the content is copied.
         :dest_repo: The repository href where the content should be copied to.
         :content: List of package hrefs that should be copied from the source. Default: None
         :structured: Whether or not the content should be structured copied. Default: True
+        :dependency_solving: Also copy the Depends/Pre-Depends closure. Default: False
         :returns: The task of the copy operation.
         """
         config = {"source_repo_version": source_repo_version, "dest_repo": dest_repo}
         if content is not None:
             config["content"] = content
-        data = Copy(config=[config], structured=structured)
+        data = Copy(config=[config], structured=structured, dependency_solving=dependency_solving)
         response = apt_copy_api.copy_content(data)
         return monitor_task(response.task)
 

pulp_deb/tests/unit/test_dependency_solving.py

@@ -0,0 +1,118 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from pulp_deb.app.tasks.dependency_solving import (
+    DependencySolveError,
+    _find_satisfier,
+    _parse_relations,
+    _version_satisfies,
+    solve_dependencies,
+)
+
+
+@pytest.fixture
+def pkg():
+    """Build a Package mock with the fields the solver reads."""
+
+    def _make(name, version, depends=None, provides=None, pk=None, arch="amd64"):
+        p = MagicMock()
+        p.package = name
+        p.version = version
+        p.architecture = arch
+        p.depends = depends
+        p.pre_depends = None
+        p.provides = provides
+        p.pk = pk if pk is not None else id(p)
+        p.name = "{}_{}_{}".format(name, version, arch)
+        return p
+
+    return _make
+
+
+@pytest.fixture
+def qs():
+    """Build a Django QuerySet stub supporting filter()/exclude() and iteration."""
+
+    def _make(items):
+        q = MagicMock()
+        q._items = list(items)
+
+        def _filter(**kwargs):
+            out = list(q._items)
+            for key, val in kwargs.items():
+                if hasattr(val, "children"):  # Q objects -> no-op for unit tests
+                    continue
+                field = key.split("__")[0]
+                out = [p for p in out if getattr(p, field, None) == val]
+            return _make(out)
+
+        q.filter.side_effect = _filter
+        q.exclude.side_effect = lambda **kw: _make(
+            [p for p in q._items if not all(getattr(p, k, None) == v for k, v in kw.items())]
+        )
+        q.__iter__.side_effect = lambda: iter(q._items)
+        q.__bool__.side_effect = lambda: bool(q._items)
+        q.none.return_value = _make([]) if items else q
+        return q
+
+    return _make
+
+
+def test_parses_or_group_with_version():
+    rels = _parse_relations("libfoo (>= 1.2.3) | libbar")
+    assert len(rels) == 1 and len(rels[0]) == 2
+    assert rels[0][0]["version"] == (">=", "1.2.3")
+    assert rels[0][1]["name"] == "libbar"
+
+
+def test_version_satisfies_respects_epoch():
+    # Epoch wins over upstream version per Debian policy.
+    assert _version_satisfies("2:1.0", (">=", "1:9.9"))
+    assert not _version_satisfies("1.0", (">=", "2.0"))
+
+
+def test_find_satisfier_picks_newest(pkg, qs):
+    pkgs = [pkg("libssl3", v) for v in ("3.0.7-25", "3.0.7-27", "3.0.7-26")]
+    assert _find_satisfier({"name": "libssl3"}, qs(pkgs)).version == "3.0.7-27"
+
+
+def test_find_satisfier_via_provides(pkg, qs):
+    pkgs = [pkg("libssl3", "3.0", provides="libssl1.1")]
+    assert _find_satisfier({"name": "libssl1.1"}, qs(pkgs)).package == "libssl3"
+    # Unversioned Provides cannot satisfy a versioned dep.
+    assert _find_satisfier({"name": "libssl1.1", "version": (">=", "1.1.1")}, qs(pkgs)) is None
+
+
+def test_solver_walks_chain(pkg, qs):
+    a = pkg("a", "1.0", depends="b", pk=1)
+    b = pkg("b", "1.0", depends="c", pk=2)
+    c = pkg("c", "1.0", pk=3)
+    with patch("pulp_deb.app.tasks.dependency_solving.Package") as Pkg:
+        Pkg.objects.filter.return_value = qs([a, b, c])
+        Pkg.get_pulp_type.return_value = "deb.package"
+        srv = MagicMock()
+        srv.content.filter.return_value.only.return_value = qs([a, b, c])
+        assert solve_dependencies([a], srv, dest_base_version=None) == {1, 2, 3}
+
+
+def test_solver_skips_deps_already_in_dest(pkg, qs):
+    a = pkg("a", "1.0", depends="b (>= 1.0)", pk=10)
+    b = pkg("b", "1.0", pk=11)
+    with patch("pulp_deb.app.tasks.dependency_solving.Package") as Pkg:
+        Pkg.objects.filter.side_effect = [qs([a, b]), qs([b])]
+        srv, dest = MagicMock(), MagicMock()
+        srv.content.filter.return_value.only.return_value = qs([a, b])
+        dest.content.filter.return_value.only.return_value = qs([b])
+        assert solve_dependencies([a], srv, dest_base_version=dest) == {10}
+
+
+def test_solver_raises_on_unsatisfiable(pkg, qs):
+    a = pkg("a", "1.0", depends="missing-pkg (>= 1.0)", pk=20)
+    with patch("pulp_deb.app.tasks.dependency_solving.Package") as Pkg:
+        Pkg.objects.filter.return_value = qs([a])
+        srv = MagicMock()
+        srv.pk = 99
+        srv.content.filter.return_value.only.return_value = qs([a])
+        with pytest.raises(DependencySolveError):
+            solve_dependencies([a], srv)