Allow users to specify distribution and component when modifying content

CHANGES/1491.feature

@@ -0,0 +1,4 @@
+Allow `repositories/deb/apt/{pulp_id}/modify/` requests to assign added packages using the optional
+`distribution` and `component` parameters. Supplying either parameter creates the corresponding
+release component, release architecture, and package-release-component content; omitting both
+preserves the existing package-only behavior.

pulp_deb/app/constants.py

@@ -17,6 +17,9 @@
 PACKAGE_UPLOAD_DEFAULT_DISTRIBUTION = "pulp"
 PACKAGE_UPLOAD_DEFAULT_COMPONENT = "upload"
 
+DEFAULT_DISTRIBUTION = "pulp"
+DEFAULT_COMPONENT = "default"
+
 # Represents null values since nulls can't be used in unique indexes in postgres < 15
 NULL_VALUE = "__!!!NULL VALUE!!!__"
 

pulp_deb/app/serializers/__init__.py

@@ -28,6 +28,7 @@
 from .remote_serializers import AptRemoteSerializer
 
 from .repository_serializers import (
+    AptRepositoryAddRemoveContentSerializer,
     AptRepositorySerializer,
     AptRepositorySyncURLSerializer,
     CopySerializer,

pulp_deb/app/serializers/repository_serializers.py

@@ -10,6 +10,7 @@
 from pulpcore.plugin.serializers import (
     PgpKeyFingerprintField,
     RelatedField,
+    RepositoryAddRemoveContentSerializer,
     RepositorySerializer,
     RepositorySyncURLSerializer,
     ValidateFieldsMixin,
@@ -26,6 +27,28 @@
 from pulp_deb.app.schema import COPY_CONFIG_SCHEMA
 
 
+class AptRepositoryAddRemoveContentSerializer(RepositoryAddRemoveContentSerializer):
+    distribution = serializers.CharField(
+        help_text=_(
+            "Name of the distribution any packages from add_content_units or remove_content_units "
+            "should be added to or removed from. Defaults to DEFAULT_DISTRIBUTION if only a "
+            "component is provided."
+        ),
+        required=False,
+    )
+    component = serializers.CharField(
+        help_text=_(
+            "Name of the component any packages from add_content_units or remove_content_units "
+            "should be added to or removed from. Defaults to DEFAULT_COMPONENT if only a "
+            "distribution is provided.."
+        ),
+        required=False,
+    )
+
+    class Meta(RepositoryAddRemoveContentSerializer.Meta):
+        fields = RepositoryAddRemoveContentSerializer.Meta.fields + ["distribution", "component"]
+
+
 class ServiceOverrideField(serializers.DictField):
     child = RelatedField(
         view_name="signing-services-detail",

pulp_deb/app/tasks/signing.py

@@ -14,13 +14,26 @@
     ContentArtifact,
     CreatedResource,
     PulpTemporaryFile,
+    RepositoryVersion,
     Upload,
     UploadChunk,
 )
 from pulpcore.plugin.tasking import add_and_remove, general_create
 from pulpcore.plugin.util import get_url
 
-from pulp_deb.app.models import AptRepository, Package, PackageReleaseComponent
+from pulp_deb.app.constants import (
+    DEFAULT_COMPONENT,
+    DEFAULT_DISTRIBUTION,
+)
+from pulp_deb.app.models import (
+    AptRepository,
+    Package,
+    PackageReleaseComponent,
+    ReleaseArchitecture,
+    ReleaseComponent,
+    SourcePackage,
+    SourcePackageReleaseComponent,
+)
 from pulp_deb.app.models.signing_service import (
     AptPackageSigningService,
     DebPackageSigningResult,
@@ -29,6 +42,103 @@
 log = logging.getLogger(__name__)
 
 
+def _prepare_package_removals(repo, remove_content_units, base_version_pk, distribution, component):
+    """Expand the removal list to include the release component relationships of each package.
+
+    Removing a (source) package also requires removing its PackageReleaseComponent /
+    SourcePackageReleaseComponent links. When a distribution/component is given, the removal is
+    scoped to that component: a package is only removed from the repository if the scope held its
+    last relationship, so packages linked elsewhere or not linked at all are kept.
+    """
+    # "*" removes all content, so there is nothing to resolve here.
+    if not remove_content_units or "*" in remove_content_units:
+        return
+
+    repository_version = (
+        RepositoryVersion.objects.get(pk=base_version_pk)
+        if base_version_pk
+        else repo.latest_version()
+    )
+    # A distribution/component narrows the removal to a single release component.
+    scoped = distribution is not None or component is not None
+    if scoped:
+        distribution = distribution or DEFAULT_DISTRIBUTION
+        component = component or DEFAULT_COMPONENT
+
+    for model, relationship_model, relationship_field in (
+        (Package, PackageReleaseComponent, "package"),
+        (SourcePackage, SourcePackageReleaseComponent, "source_package"),
+    ):
+        units = model.objects.filter(pk__in=remove_content_units)
+        relationships = relationship_model.objects.filter(
+            **{
+                f"{relationship_field}__in": units,
+                "pk__in": repository_version.content,
+            }
+        )
+        if scoped:
+            scoped_relationships = relationships.filter(
+                release_component__distribution=distribution,
+                release_component__component=component,
+            )
+            # Relationships named in the request are removed alongside the scoped ones.
+            removed_relationship_ids = set(
+                relationship_model.objects.filter(pk__in=remove_content_units).values_list(
+                    "pk", flat=True
+                )
+            )
+            removed_relationship_ids.update(scoped_relationships.values_list("pk", flat=True))
+            still_linked = relationships.exclude(pk__in=removed_relationship_ids)
+            orphaned_unit_ids = {
+                str(pk)
+                for pk in scoped_relationships.values_list(f"{relationship_field}_id", flat=True)
+            } - {str(pk) for pk in still_linked.values_list(f"{relationship_field}_id", flat=True)}
+            kept_unit_ids = {
+                str(pk) for pk in units.values_list("pk", flat=True)
+            } - orphaned_unit_ids
+            remove_content_units[:] = [
+                content_id for content_id in remove_content_units if content_id not in kept_unit_ids
+            ]
+            relationships = scoped_relationships
+        remove_content_units.extend(str(pk) for pk in relationships.values_list("pk", flat=True))
+
+
+def _prepare_package_additions(add_content_units, distribution, component):
+    """Expand the addition list with the metadata needed to publish the packages in a component.
+
+    For each (source) package being added under a distribution/component, ensure the matching
+    ReleaseComponent, ReleaseArchitecture and *ReleaseComponent relationships exist and add them
+    to the content being added.
+    """
+    packages = list(Package.objects.filter(pk__in=add_content_units))
+    source_packages = list(SourcePackage.objects.filter(pk__in=add_content_units))
+    if not (packages or source_packages) or (distribution is None and component is None):
+        return
+
+    distribution = distribution or DEFAULT_DISTRIBUTION
+    component = component or DEFAULT_COMPONENT
+    release_component, _ = ReleaseComponent.objects.get_or_create(
+        distribution=distribution, component=component
+    )
+    add_content_units.append(str(release_component.pk))
+    # Each binary package needs its architecture and a link to the release component.
+    for package in packages:
+        if package.architecture != "all":
+            architecture, _ = ReleaseArchitecture.objects.get_or_create(
+                distribution=distribution, architecture=package.architecture
+            )
+        package_component, _ = PackageReleaseComponent.objects.get_or_create(
+            release_component=release_component, package=package
+        )
+        add_content_units.extend([str(architecture.pk), str(package_component.pk)])
+    # Source packages only need a link to the release component.
+    for source_package in source_packages:
+        source_package_component, _ = SourcePackageReleaseComponent.objects.get_or_create(
+            release_component=release_component, source_package=source_package
+        )
+        add_content_units.append(str(source_package_component.pk))
+
+
 def _save_file(fileobj, final_package):
     with fileobj.file.open() as fd:
         final_package.write(fd.read())
@@ -205,10 +315,19 @@ def _sign_package(package, signing_service, signing_fingerprint, package_release
 
 
 def signed_add_and_remove(
-    repository_pk, add_content_units, remove_content_units, base_version_pk=None, overwrite=True
+    repository_pk,
+    add_content_units,
+    remove_content_units,
+    base_version_pk=None,
+    overwrite=True,
+    distribution=None,
+    component=None,
 ):
     repo = AptRepository.objects.get(pk=repository_pk)
 
+    _prepare_package_removals(repo, remove_content_units, base_version_pk, distribution, component)
+    _prepare_package_additions(add_content_units, distribution, component)
+
     if repo.package_signing_service:
         log.info(
             f"Signing packages for repository {repo.name} with {repo.package_signing_service}."
@@ -260,6 +379,7 @@ async def _bounded_sign(pkg_tuple):
                 new_prc, _ = PackageReleaseComponent.objects.get_or_create(
                     release_component=prc.release_component,
                     package_id=new_id,
+                    index_architecture=prc.index_architecture,
                     _pulp_domain=prc._pulp_domain,
                 )
 

pulp_deb/app/viewsets/repository.py

@@ -5,16 +5,13 @@
 from rest_framework import viewsets
 from rest_framework.serializers import ValidationError as DRFValidationError
 
-from pulp_deb.app.models.content.content import Package
-from pulp_deb.app.models.content.structure_content import PackageReleaseComponent
 from pulp_deb.app.serializers import AptRepositorySyncURLSerializer
 from pulp_deb.app.tasks import signed_add_and_remove
 
-from pulpcore.plugin.util import extract_pk, get_url
+from pulpcore.plugin.util import extract_pk
 from pulpcore.plugin.actions import ModifyRepositoryActionMixin
 from pulpcore.plugin.serializers import (
     AsyncOperationResponseSerializer,
-    RepositoryAddRemoveContentSerializer,
 )
 from pulpcore.plugin.models import ContentArtifact, RepositoryVersion
 from pulpcore.plugin.tasking import dispatch
@@ -37,16 +34,12 @@ class AptModifyRepositoryActionMixin(ModifyRepositoryActionMixin):
         summary="Modify Repository Content",
         responses={202: AsyncOperationResponseSerializer},
     )
-    @action(detail=True, methods=["post"], serializer_class=RepositoryAddRemoveContentSerializer)
+    @action(
+        detail=True,
+        methods=["post"],
+        serializer_class=serializers.AptRepositoryAddRemoveContentSerializer,
+    )
     def modify(self, request, pk, **kwargs):
-        remove_content_units = request.data.get("remove_content_units", [])
-        remove_package_hrefs = [href for href in remove_content_units if "/packages/" in href]
-
-        if remove_package_hrefs:
-            prc_hrefs = self._get_matching_prc_hrefs(remove_package_hrefs)
-            remove_content_units.extend(prc_hrefs)
-            request.data["remove_content_units"] = remove_content_units
-
         add_content_units = request.data.get("add_content_units", [])
         package_ids = [extract_pk(href) for href in add_content_units if "/packages/" in href]
         repository = self.get_object()
@@ -59,14 +52,26 @@ def modify(self, request, pk, **kwargs):
                     _("Cannot add on-demand content to repo with set package signing service.")
                 )
 
-        return super().modify(request, pk)
-
-    def _get_matching_prc_hrefs(self, package_hrefs):
-        package_ids = [extract_pk(href) for href in package_hrefs]
-        matching_packages = Package.objects.filter(pulp_id__in=package_ids)
-        matching_prcs = PackageReleaseComponent.objects.filter(package__in=matching_packages)
-        prc_hrefs = [get_url(component) for component in matching_prcs]
-        return prc_hrefs
+        serializer = self.get_serializer(
+            data=request.data,
+            context={**self.get_serializer_context(), "repository": repository},
+        )
+        serializer.is_valid(raise_exception=True)
+        base_version = serializer.validated_data.get("base_version")
+        task = dispatch(
+            self.modify_task,
+            exclusive_resources=[repository],
+            kwargs={
+                "repository_pk": pk,
+                "base_version_pk": base_version.pk if base_version else None,
+                "add_content_units": serializer.validated_data.get("add_content_units", []),
+                "remove_content_units": serializer.validated_data.get("remove_content_units", []),
+                "overwrite": serializer.validated_data.get("overwrite", True),
+                "distribution": serializer.validated_data.get("distribution"),
+                "component": serializer.validated_data.get("component"),
+            },
+        )
+        return OperationPostponedResponse(task, request)
 
 
 class AptRepositoryViewSet(AptModifyRepositoryActionMixin, RepositoryViewSet, RolesMixin):