Merge pull request #746 from gerrod3/replicate-on-demand

Fix upstream on-demand replicates failing

(cherry picked from commit 4382aa6)

CHANGES/718.bugfix

@@ -0,0 +1 @@
+Fixed replicate failing on upstream on-demand repositories

pulp_python/app/replica.py

@@ -28,6 +28,14 @@ def remote_extra_fields(self, upstream_distribution):
         # strain the upstream Pulp.
         return {"policy": "immediate", "prereleases": True}
 
+    def url(self, upstream_distribution):
+        # Ignore distributions that are only pull-through
+        repo, pub = upstream_distribution["repository"], upstream_distribution["publication"]
+        if repo or pub:
+            return super().url(upstream_distribution)
+
+        return None
+
     def repository_extra_fields(self, remote):
         # Use autopublish since publications result in faster serving times
         return {"autopublish": True}

pulp_python/app/tasks/sync.py

@@ -256,8 +256,9 @@ async def create_content(self, pkg):
             for package in dists:
                 entry = parse_metadata(pkg.info, version, package)
                 url = entry.pop("url")
+                size = package["size"] or None
 
-                artifact = Artifact(sha256=entry["sha256"])
+                artifact = Artifact(sha256=entry["sha256"], size=size)
                 package = PythonPackageContent(**entry)
 
                 da = DeclarativeArtifact(

pulp_python/app/utils.py

@@ -284,19 +284,21 @@ def find_artifact():
     if domain:
         components.insert(2, domain.name)
     url = "/".join(components)
+    md5 = artifact.md5 if artifact and artifact.md5 else ""
+    size = artifact.size if artifact and artifact.size else 0
     return {
         "comment_text": "",
-        "digests": {"md5": artifact.md5, "sha256": artifact.sha256},
+        "digests": {"md5": md5, "sha256": content.sha256},
         "downloads": -1,
         "filename": content.filename,
         "has_sig": False,
-        "md5_digest": artifact.md5,
+        "md5_digest": md5,
         "packagetype": content.packagetype,
         "python_version": content.python_version,
         "requires_python": content.requires_python or None,
-        "size": artifact.size,
-        "upload_time": str(artifact.pulp_created),
-        "upload_time_iso_8601": str(artifact.pulp_created.isoformat()),
+        "size": size,
+        "upload_time": str(content.pulp_created),
+        "upload_time_iso_8601": str(content.pulp_created.isoformat()),
         "url": url,
         "yanked": False,
         "yanked_reason": None

pulp_python/tests/functional/api/test_domains.py

@@ -5,7 +5,12 @@
 
 from pulpcore.app import settings
 
-from pulp_python.tests.functional.constants import PYTHON_URL, PYTHON_EGG_FILENAME
+from pulp_python.tests.functional.constants import (
+    PYTHON_URL,
+    PYTHON_EGG_FILENAME,
+    PYTHON_SM_PROJECT_SPECIFIER,
+    PYTHON_SM_PACKAGE_COUNT,
+)
 from urllib.parse import urlsplit
 
 
@@ -144,7 +149,7 @@ def test_domain_content_replication(
     python_bindings,
     python_file,
     python_repo_factory,
-    python_publication_factory,
+    python_remote_factory,
     python_distribution_factory,
     monitor_task,
     monitor_task_group,
@@ -154,13 +159,12 @@ def test_domain_content_replication(
     """Test replication feature through the usage of domains."""
     # Set up source domain to replicate from
     source_domain = domain_factory()
-    repo = python_repo_factory(pulp_domain=source_domain.name)
+    repo = python_repo_factory(pulp_domain=source_domain.name, autopublish=True)
     body = {"relative_path": PYTHON_EGG_FILENAME, "file": python_file, "repository": repo.pulp_href}
     monitor_task(
         python_bindings.ContentPackagesApi.create(pulp_domain=source_domain.name, **body).task
     )
-    pub = python_publication_factory(repository=repo, pulp_domain=source_domain.name)
-    python_distribution_factory(publication=pub.pulp_href, pulp_domain=source_domain.name)
+    python_distribution_factory(repository=repo, pulp_domain=source_domain.name)
 
     # Create the replica domain
     replica_domain = domain_factory()
@@ -171,6 +175,7 @@ def test_domain_content_replication(
         "domain": source_domain.name,
         "username": bindings_cfg.username,
         "password": bindings_cfg.password,
+        "tls_validation": False,
     }
     upstream_pulp = gen_object_with_cleanup(
         pulpcore_bindings.UpstreamPulpsApi, upstream_pulp_body, pulp_domain=replica_domain.name
@@ -194,6 +199,25 @@ def test_domain_content_replication(
 
     assert all(1 == x for x in counts.values()), f"Replica had more than 1 object {counts}"
 
+    # Test that we can replicate from an Upstream on-demand source (syncs are on-demand by default)
+    remote = python_remote_factory(
+        includes=PYTHON_SM_PROJECT_SPECIFIER, pulp_domain=source_domain.name
+    )
+    body = {"remote": remote.pulp_href}
+    monitor_task(python_bindings.RepositoriesPythonApi.sync(repo.pulp_href, body).task)
+
+    response = pulpcore_bindings.UpstreamPulpsApi.replicate(upstream_pulp.pulp_href)
+    monitor_task_group(response.task_group)
+
+    response = python_bindings.ContentPackagesApi.list(pulp_domain=replica_domain.name)
+    assert PYTHON_SM_PACKAGE_COUNT + 1 == response.count
+    response = python_bindings.PublicationsPypiApi.list(pulp_domain=replica_domain.name)
+    assert 2 == response.count
+    add_to_cleanup(python_bindings.PublicationsPypiApi, response.results[0])
+    assert 1 == python_bindings.RepositoriesPythonApi.list(pulp_domain=replica_domain.name).count
+    assert 1 == python_bindings.DistributionsPypiApi.list(pulp_domain=replica_domain.name).count
+    assert 1 == python_bindings.RemotesPythonApi.list(pulp_domain=replica_domain.name).count
+
 
 @pytest.fixture
 def shelf_reader_cleanup():