Implement a cache-friendly-url repo layout option

pulp_deb/app/constants.py

@@ -1,3 +1,5 @@
+from types import SimpleNamespace
+
 NO_MD5_WARNING_MESSAGE = (
     "Your pulp instance is configured to prohibit use of the MD5 checksum algorithm!\n"
     'Processing MD5 IN ADDITION to a secure hash like SHA-256 is "highly recommended".\n'
@@ -17,3 +19,15 @@
 
 # Represents null values since nulls can't be used in unique indexes in postgres < 15
 NULL_VALUE = "__!!!NULL VALUE!!!__"
+
+LAYOUT_TYPES = SimpleNamespace(
+    NESTED_ALPHABETICALLY="nested_alphabetically",  # default
+    NESTED_BY_DIGEST="nested_by_digest",
+    NESTED_BY_BOTH="nested_by_both",
+)
+
+LAYOUT_CHOICES = (
+    (LAYOUT_TYPES.NESTED_ALPHABETICALLY, LAYOUT_TYPES.NESTED_ALPHABETICALLY),
+    (LAYOUT_TYPES.NESTED_BY_DIGEST, LAYOUT_TYPES.NESTED_BY_DIGEST),
+    (LAYOUT_TYPES.NESTED_BY_BOTH, LAYOUT_TYPES.NESTED_BY_BOTH),
+)

pulp_deb/app/migrations/0034_aptpublication_layout.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.27 on 2025-12-22 19:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('deb', '0033_aptalternatecontentsource'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='aptpublication',
+            name='layout',
+            field=models.TextField(choices=[('nested_alphabetically', 'nested_alphabetically'), ('nested_by_digest', 'nested_by_digest'), ('nested_by_both', 'nested_by_both')], default='nested_alphabetically'),
+        ),
+    ]

pulp_deb/app/models/content/content.py

@@ -16,6 +16,8 @@
 from pulpcore.plugin.models import Content
 from pulpcore.plugin.util import get_domain_pk
 
+from pulp_deb.app.constants import LAYOUT_TYPES
+
 BOOL_CHOICES = [(True, "yes"), (False, "no")]
 
 
@@ -76,21 +78,30 @@ def name(self):
         """Print a nice name for Packages."""
         return "{}_{}_{}".format(self.package, self.version, self.architecture)
 
-    def filename(self, component=""):
+    def filename(self, component="", layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY):
         """Assemble filename in pool directory."""
-        sourcename = self.source or self.package
-        sourcename = sourcename.split("(", 1)[0].rstrip()
-        if sourcename.startswith("lib"):
-            prefix = sourcename[0:4]
-        else:
-            prefix = sourcename[0]
-        return os.path.join(
-            "pool",
-            component,
-            prefix,
-            sourcename,
-            "{}.{}".format(self.name, self.SUFFIX),
-        )
+        if layout == LAYOUT_TYPES.NESTED_ALPHABETICALLY:
+            sourcename = self.source or self.package
+            sourcename = sourcename.split("(", 1)[0].rstrip()
+            if sourcename.startswith("lib"):
+                prefix = sourcename[0:4]
+            else:
+                prefix = sourcename[0]
+            return os.path.join(
+                "pool",
+                component,
+                prefix,
+                sourcename,
+                "{}.{}".format(self.name, self.SUFFIX),
+            )
+        else:  # NESTED_BY_DIGEST or NESTED_BY_BOTH. The primary URL in BOTH is by digest.
+            return os.path.join(
+                "pool",
+                component,
+                self.sha256[0:2],
+                self.sha256[2:6],
+                "{}.{}".format(self.name, self.SUFFIX),
+            )
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
@@ -209,20 +220,18 @@ def derived_dsc_filename(self):
         """Print a nice name for the Dsc file."""
         return "{}_{}.{}".format(self.source, self.version, self.SUFFIX)
 
-    def derived_dir(self, component=""):
+    def derived_dir(self, component="", layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY):
         """Assemble full dir in pool directory."""
         sourcename = self.source
-        prefix = sourcename[0]
-        return os.path.join(
-            "pool",
-            component,
-            prefix,
-            sourcename,
-        )
-
-    def derived_path(self, name, component=""):
+        if layout == LAYOUT_TYPES.NESTED_ALPHABETICALLY:
+            return os.path.join("pool", component, sourcename[0], sourcename)
+        else:  # NESTED_BY_DIGEST or NESTED_BY_BOTH
+            sha256 = self.sha256
+            return os.path.join("pool", component, sha256[0:2], sha256[2:6], sourcename)
+
+    def derived_path(self, name, component="", layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY):
         """Assemble filename in pool directory."""
-        return os.path.join(self.derived_dir(component), name)
+        return os.path.join(self.derived_dir(component, layout=layout), name)
 
     @property
     def checksums_sha1(self):

pulp_deb/app/models/publication.py

@@ -14,6 +14,7 @@
     RepositoryVersion,
 )
 
+from pulp_deb.app.constants import LAYOUT_CHOICES, LAYOUT_TYPES
 from pulp_deb.app.models.signing_service import AptReleaseSigningService
 
 
@@ -71,6 +72,7 @@ class AptPublication(Publication, AutoAddObjPermsMixin):
 
     simple = models.BooleanField(default=False)
     structured = models.BooleanField(default=True)
+    layout = models.TextField(choices=LAYOUT_CHOICES, default=LAYOUT_TYPES.NESTED_ALPHABETICALLY)
     signing_service = models.ForeignKey(
         AptReleaseSigningService, on_delete=models.PROTECT, null=True
     )

pulp_deb/app/serializers/content_serializers.py

@@ -27,6 +27,7 @@
 from pulp_deb.app.constants import (
     PACKAGE_UPLOAD_DEFAULT_COMPONENT,
     PACKAGE_UPLOAD_DEFAULT_DISTRIBUTION,
+    LAYOUT_TYPES,
 )
 
 from pulp_deb.app.constants import NULL_VALUE
@@ -478,7 +479,13 @@ def from822(cls, data, **kwargs):
         package_fields["custom_fields"] = custom_fields
         return cls(data=package_fields, **kwargs)
 
-    def to822(self, component="", artifact_dict=None, remote_artifact_dict=None):
+    def to822(
+        self,
+        component="",
+        artifact_dict=None,
+        remote_artifact_dict=None,
+        layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY,
+    ):
         """Create deb822.Package object from model."""
         ret = deb822.Packages()
 
@@ -502,7 +509,7 @@ def to822(self, component="", artifact_dict=None, remote_artifact_dict=None):
             ret.update({"SHA1": artifact.sha1} if artifact.sha1 else {})
             ret.update({"SHA256": artifact.sha256})
             ret.update({"Size": str(artifact.size)})
-        ret.update({"Filename": self.instance.filename(component)})
+        ret.update({"Filename": self.instance.filename(component, layout=layout)})
 
         return ret
 
@@ -1066,7 +1073,7 @@ def from822(cls, data, **kwargs):
             data={k: data[v] for k, v in cls.TRANSLATION_DICT.items() if v in data}, **kwargs
         )
 
-    def to822(self, component="", paragraph=False):
+    def to822(self, component="", paragraph=False, layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY):
         """
         Create deb822.Dsc object from model. If the 'paragraph' argument is True then the returned
         object will be adjusted to be a valid paragraph in a source index file.
@@ -1103,8 +1110,7 @@ def to822(self, component="", paragraph=False):
             and include 'Directory'. Currently we skip the optional 'Priority' and 'Section'.
             """
             ret["Package"] = ret.pop("Source")
-            ret["Directory"] = self.instance.derived_dir(component)
-
+            ret["Directory"] = self.instance.derived_dir(component, layout=layout)
         return ret
 
     class Meta:

pulp_deb/app/serializers/publication_serializers.py

@@ -8,6 +8,7 @@
     DetailRelatedField,
 )
 
+from pulp_deb.app.constants import LAYOUT_CHOICES, LAYOUT_TYPES
 from pulp_deb.app.models import (
     AptDistribution,
     AptPublication,
@@ -38,6 +39,13 @@ class AptPublicationSerializer(PublicationSerializer):
     structured = BooleanField(help_text="Activate structured publishing mode.", default=True)
     publish_upstream_release_fields = BooleanField(help_text="", required=False)
     checkpoint = serializers.BooleanField(required=False)
+    layout = serializers.ChoiceField(
+        help_text="How to layout the packages within the published repository.",
+        choices=LAYOUT_CHOICES,
+        required=False,
+        allow_null=False,
+        default=LAYOUT_TYPES.NESTED_ALPHABETICALLY,
+    )
     signing_service = RelatedField(
         help_text="Sign Release files with this signing key",
         many=False,
@@ -62,6 +70,7 @@ class Meta:
             "checkpoint",
             "signing_service",
             "publish_upstream_release_fields",
+            "layout",
         )
         model = AptPublication
 

pulp_deb/app/tasks/publishing.py

@@ -17,6 +17,7 @@
 from django.db.utils import IntegrityError
 from django.forms.models import model_to_dict
 
+from pulp_deb.app.constants import NULL_VALUE
 from pulpcore.plugin.models import (
     Artifact,
     PublishedArtifact,
@@ -26,7 +27,6 @@
 )
 from pulpcore.plugin.util import get_domain
 
-from pulp_deb.app.constants import NULL_VALUE
 from pulp_deb.app.models import (
     AptPublication,
     AptRepository,
@@ -49,6 +49,7 @@
 from pulp_deb.app.constants import (
     NO_MD5_WARNING_MESSAGE,
     CHECKSUM_TYPE_MAP,
+    LAYOUT_TYPES,
 )
 
 import logging
@@ -86,6 +87,7 @@ def publish(
     checkpoint=False,
     signing_service_pk=None,
     publish_upstream_release_fields=None,
+    layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY,
 ):
     """
     Use provided publisher to create a Publication based on a RepositoryVersion.
@@ -96,6 +98,7 @@ def publish(
         structured (bool): Create a structured publication with releases and components.
         checkpoint (bool): Whether to create a checkpoint publication.
         signing_service_pk (str): Use this SigningService to sign the Release files.
+        layout (str): The layout determines the form the package urls take.
 
     """
 
@@ -124,6 +127,7 @@ def publish(
             publication.simple = simple
             publication.structured = structured
             publication.signing_service = signing_service
+            publication.layout = layout
             repository = AptRepository.objects.get(pk=repo_version.repository.pk)
 
             if simple:
@@ -341,19 +345,29 @@ def add_packages(self, packages, artifact_dict, remote_artifact_dict):
         content_artifacts = {
             package.pk: list(package.contentartifact_set.all()) for package in packages
         }
+        layout = self.parent.publication.layout
 
         for package in packages:
             with suppress(IntegrityError):
                 content_artifact = content_artifacts.get(package.pk, [None])[0]
-                relative_path = package.filename(self.component)
-
                 published_artifact = PublishedArtifact(
-                    relative_path=relative_path,
+                    relative_path=package.filename(self.component, layout=layout),
                     publication=self.parent.publication,
                     content_artifact=content_artifact,
                 )
                 published_artifacts.append(published_artifact)
                 package_data.append((package, package.architecture))
+                # In the NESTED_BY_BOTH layout, we want to _also_ publish the package under the
+                # alphabetical path but _not_ reference it in the repo metadata.
+                if layout == LAYOUT_TYPES.NESTED_BY_BOTH:
+                    alt_published_artifact = PublishedArtifact(
+                        relative_path=package.filename(
+                            self.component, layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY
+                        ),
+                        publication=self.parent.publication,
+                        content_artifact=content_artifact,
+                    )
+                    published_artifacts.append(alt_published_artifact)
 
         with transaction.atomic():
             if published_artifacts:
@@ -362,9 +376,9 @@ def add_packages(self, packages, artifact_dict, remote_artifact_dict):
         for package, architecture in package_data:
             package_serializer = Package822Serializer(package, context={"request": None})
             try:
-                package_serializer.to822(self.component, artifact_dict, remote_artifact_dict).dump(
-                    self.package_index_files[architecture][0]
-                )
+                package_serializer.to822(
+                    self.component, artifact_dict, remote_artifact_dict, layout=layout
+                ).dump(self.package_index_files[architecture][0])
             except KeyError:
                 log.warn(
                     f"Published package '{package.relative_path}' with architecture "
@@ -385,12 +399,27 @@ def add_source_packages(self, source_packages):
                 for content_artifact in artifact_set:
                     published_artifact = PublishedArtifact(
                         relative_path=source_package.derived_path(
-                            os.path.basename(content_artifact.relative_path), self.component
+                            os.path.basename(content_artifact.relative_path),
+                            self.component,
+                            layout=self.parent.publication.layout,
                         ),
                         publication=self.parent.publication,
                         content_artifact=content_artifact,
                     )
                     published_artifacts.append(published_artifact)
+                    # In the NESTED_BY_BOTH layout, we want to _also_ publish the source package
+                    # under the alphabetical path but _not_ reference it in the repo metadata.
+                    if self.parent.publication.layout == LAYOUT_TYPES.NESTED_BY_BOTH:
+                        alt_published_artifact = PublishedArtifact(
+                            relative_path=source_package.derived_path(
+                                os.path.basename(content_artifact.relative_path),
+                                self.component,
+                                layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY,
+                            ),
+                            publication=self.parent.publication,
+                            content_artifact=content_artifact,
+                        )
+                        published_artifacts.append(alt_published_artifact)
                 source_package_data.append(source_package)
 
         with transaction.atomic():
@@ -401,9 +430,9 @@ def add_source_packages(self, source_packages):
             dsc_file_822_serializer = DscFile822Serializer(
                 source_package, context={"request": None}
             )
-            dsc_file_822_serializer.to822(self.component, paragraph=True).dump(
-                self.source_index_file_info[0]
-            )
+            dsc_file_822_serializer.to822(
+                self.component, paragraph=True, layout=self.parent.publication.layout
+            ).dump(self.source_index_file_info[0])
             self.source_index_file_info[0].write(b"\n")
 
     def finish(self):

pulp_deb/app/viewsets/publication.py

@@ -216,13 +216,15 @@ def create(self, request):
         publish_upstream_release_fields = serializer.validated_data.get(
             "publish_upstream_release_fields"
         )
+        layout = serializer.validated_data.get("layout")
 
         kwargs = {
             "repository_version_pk": repository_version.pk,
             "simple": simple,
             "structured": structured,
             "signing_service_pk": getattr(signing_service, "pk", None),
             "publish_upstream_release_fields": publish_upstream_release_fields,
+            "layout": layout,
         }
         if checkpoint:
             kwargs["checkpoint"] = True