From e7ba0a482ec5be807579835927a725b193a4131c Mon Sep 17 00:00:00 2001 From: Grant Gainey Date: Fri, 11 Oct 2024 21:32:38 -0400 Subject: [PATCH] Merge pull request #746 from gerrod3/replicate-on-demand Fix upstream on-demand replicates failing (cherry picked from commit 4382aa63a57673b59d471554e8df053b7eceaa44) --- CHANGES/718.bugfix | 1 + pulp_python/app/replica.py | 8 +++++ pulp_python/app/tasks/sync.py | 3 +- pulp_python/app/utils.py | 12 ++++--- .../tests/functional/api/test_domains.py | 34 ++++++++++++++++--- 5 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 CHANGES/718.bugfix diff --git a/CHANGES/718.bugfix b/CHANGES/718.bugfix new file mode 100644 index 00000000..6f546fee --- /dev/null +++ b/CHANGES/718.bugfix @@ -0,0 +1 @@ +Fixed replicate failing on upstream on-demand repositories diff --git a/pulp_python/app/replica.py b/pulp_python/app/replica.py index e59c8054..0a11bfc3 100644 --- a/pulp_python/app/replica.py +++ b/pulp_python/app/replica.py @@ -28,6 +28,14 @@ def remote_extra_fields(self, upstream_distribution): # strain the upstream Pulp. return {"policy": "immediate", "prereleases": True} + def url(self, upstream_distribution): + # Ignore distributions that are only pull-through + repo, pub = upstream_distribution["repository"], upstream_distribution["publication"] + if repo or pub: + return super().url(upstream_distribution) + + return None + def repository_extra_fields(self, remote): # Use autopublish since publications result in faster serving times return {"autopublish": True} diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index 1293c613..a250b9f7 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -256,8 +256,9 @@ async def create_content(self, pkg): for package in dists: entry = parse_metadata(pkg.info, version, package) url = entry.pop("url") + size = package["size"] or None - artifact = Artifact(sha256=entry["sha256"]) + artifact = Artifact(sha256=entry["sha256"], size=size) package = PythonPackageContent(**entry) da = DeclarativeArtifact( diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index f66dcbd7..126037a8 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -284,19 +284,21 @@ def find_artifact(): if domain: components.insert(2, domain.name) url = "/".join(components) + md5 = artifact.md5 if artifact and artifact.md5 else "" + size = artifact.size if artifact and artifact.size else 0 return { "comment_text": "", - "digests": {"md5": artifact.md5, "sha256": artifact.sha256}, + "digests": {"md5": md5, "sha256": content.sha256}, "downloads": -1, "filename": content.filename, "has_sig": False, - "md5_digest": artifact.md5, + "md5_digest": md5, "packagetype": content.packagetype, "python_version": content.python_version, "requires_python": content.requires_python or None, - "size": artifact.size, - "upload_time": str(artifact.pulp_created), - "upload_time_iso_8601": str(artifact.pulp_created.isoformat()), + "size": size, + "upload_time": str(content.pulp_created), + "upload_time_iso_8601": str(content.pulp_created.isoformat()), "url": url, "yanked": False, "yanked_reason": None diff --git a/pulp_python/tests/functional/api/test_domains.py b/pulp_python/tests/functional/api/test_domains.py index 74316dec..8a37ae14 100644 --- a/pulp_python/tests/functional/api/test_domains.py +++ b/pulp_python/tests/functional/api/test_domains.py @@ -5,7 +5,12 @@ from pulpcore.app import settings -from pulp_python.tests.functional.constants import PYTHON_URL, PYTHON_EGG_FILENAME +from pulp_python.tests.functional.constants import ( + PYTHON_URL, + PYTHON_EGG_FILENAME, + PYTHON_SM_PROJECT_SPECIFIER, + PYTHON_SM_PACKAGE_COUNT, +) from urllib.parse import urlsplit @@ -144,7 +149,7 @@ def test_domain_content_replication( python_bindings, python_file, python_repo_factory, - python_publication_factory, + python_remote_factory, python_distribution_factory, monitor_task, monitor_task_group, @@ -154,13 +159,12 @@ def test_domain_content_replication( """Test replication feature through the usage of domains.""" # Set up source domain to replicate from source_domain = domain_factory() - repo = python_repo_factory(pulp_domain=source_domain.name) + repo = python_repo_factory(pulp_domain=source_domain.name, autopublish=True) body = {"relative_path": PYTHON_EGG_FILENAME, "file": python_file, "repository": repo.pulp_href} monitor_task( python_bindings.ContentPackagesApi.create(pulp_domain=source_domain.name, **body).task ) - pub = python_publication_factory(repository=repo, pulp_domain=source_domain.name) - python_distribution_factory(publication=pub.pulp_href, pulp_domain=source_domain.name) + python_distribution_factory(repository=repo, pulp_domain=source_domain.name) # Create the replica domain replica_domain = domain_factory() @@ -171,6 +175,7 @@ def test_domain_content_replication( "domain": source_domain.name, "username": bindings_cfg.username, "password": bindings_cfg.password, + "tls_validation": False, } upstream_pulp = gen_object_with_cleanup( pulpcore_bindings.UpstreamPulpsApi, upstream_pulp_body, pulp_domain=replica_domain.name @@ -194,6 +199,25 @@ def test_domain_content_replication( assert all(1 == x for x in counts.values()), f"Replica had more than 1 object {counts}" + # Test that we can replicate from an Upstream on-demand source (syncs are on-demand by default) + remote = python_remote_factory( + includes=PYTHON_SM_PROJECT_SPECIFIER, pulp_domain=source_domain.name + ) + body = {"remote": remote.pulp_href} + monitor_task(python_bindings.RepositoriesPythonApi.sync(repo.pulp_href, body).task) + + response = pulpcore_bindings.UpstreamPulpsApi.replicate(upstream_pulp.pulp_href) + monitor_task_group(response.task_group) + + response = python_bindings.ContentPackagesApi.list(pulp_domain=replica_domain.name) + assert PYTHON_SM_PACKAGE_COUNT + 1 == response.count + response = python_bindings.PublicationsPypiApi.list(pulp_domain=replica_domain.name) + assert 2 == response.count + add_to_cleanup(python_bindings.PublicationsPypiApi, response.results[0]) + assert 1 == python_bindings.RepositoriesPythonApi.list(pulp_domain=replica_domain.name).count + assert 1 == python_bindings.DistributionsPypiApi.list(pulp_domain=replica_domain.name).count + assert 1 == python_bindings.RemotesPythonApi.list(pulp_domain=replica_domain.name).count + @pytest.fixture def shelf_reader_cleanup():