From a9ba014959c2cd1d0dc6109cbf317c6bb47d9a2f Mon Sep 17 00:00:00 2001
From: Michael Vogt <michael.vogt@gmail.com>
Date: Thu, 11 Jul 2024 13:38:55 +0200
Subject: [PATCH] test: add test that download mixed https content

When investigating https://github.com/osbuild/osbuild-composer/pull/4247
we found that it would fail when a download required two sets of
`--cacert` keys. This commits adds a test for this that fails on
the centos9 7.76.1 version.
---
 sources/org.osbuild.curl         |  6 ++--
 sources/test/test_curl_source.py | 47 ++++++++++++++++++++++++++++----
 2 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/sources/org.osbuild.curl b/sources/org.osbuild.curl
index 9841e33c9..b350f7d9c 100755
--- a/sources/org.osbuild.curl
+++ b/sources/org.osbuild.curl
@@ -102,6 +102,8 @@ SCHEMA = """
 # own json subset
 CURL_WRITE_OUT = r'\{\"url\": \"%{url}\"\, \"filename_effective\": \"%{filename_effective}\", \"exitcode\": %{exitcode}, \"errormsg\": \"%{errormsg}\" \}\n'
 
+NR_RETRYS = 10
+
 
 def curl_has_parallel_downloads():
     """
@@ -310,7 +312,7 @@ class CurlSource(sources.SourceService):
             # some mirrors are sometimes broken. retry manually, because we could be
             # redirected to a different, working, one on retry.
             return_code = 0
-            for _ in range(10):
+            for _ in range(NR_RETRYS):
                 return_code = fetch_many_new_curl(tmpdir, self.cache, dl_pairs)
                 if return_code == 0:
                     break
@@ -331,7 +333,7 @@ class CurlSource(sources.SourceService):
             # some mirrors are sometimes broken. retry manually, because we could be
             # redirected to a different, working, one on retry.
             return_code = 0
-            for _ in range(10):
+            for _ in range(NR_RETRYS):
                 curl_config_path = f"{tmpdir}/curl-config.txt"
                 gen_curl_download_config(curl_config_path, [(checksum, desc)])
                 curl = subprocess.run(
diff --git a/sources/test/test_curl_source.py b/sources/test/test_curl_source.py
index 5b4942c05..05afff16d 100644
--- a/sources/test/test_curl_source.py
+++ b/sources/test/test_curl_source.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 import hashlib
+import pathlib
 import platform
 import re
 import shutil
@@ -9,7 +10,7 @@
 
 import pytest
 
-from osbuild.testutil.net import http_serve_directory
+from osbuild.testutil.net import http_serve_directory, https_serve_directory
 
 SOURCES_NAME = "org.osbuild.curl"
 
@@ -103,7 +104,7 @@ def test_curl_download_many_fail(curl_parallel):
     assert str(exp.value) == 'curl: error downloading http://localhost:9876/random-not-exists: error code 7'
 
 
-def make_test_sources(fake_httpd_root, port, n_files):
+def make_test_sources(fake_httpd_root, port, n_files, start_n=0, cacert=""):
     """
     Create test sources for n_file. All files have the names
     0,1,2...
@@ -112,13 +113,20 @@ def make_test_sources(fake_httpd_root, port, n_files):
     Returns a sources dict that can be used as input for "fetch_all()" with
     the correct hash/urls.
     """
+    proto = "https" if cacert else "http"
+
     fake_httpd_root.mkdir(exist_ok=True)
     sources = {}
-    for i in range(n_files):
+    for i in range(start_n, start_n + n_files):
         name = f"{i}"
-        sources[f"sha256:{hashlib.sha256(name.encode()).hexdigest()}"] = {
-            "url": f"http://localhost:{port}/{name}",
+        key = f"sha256:{hashlib.sha256(name.encode()).hexdigest()}"
+        val = {
+            "url": f"{proto}://localhost:{port}/{name}",
         }
+        if cacert:
+            val["secrets"] = {}
+            val["secrets"]["ssl_ca_cert"] = cacert
+        sources[key] = val
         (fake_httpd_root / name).write_text(name, encoding="utf8")
 
     return sources
@@ -334,3 +342,32 @@ def test_curl_has_parallel_download(mocked_check_output, monkeypatch, sources_mo
 
     mocked_check_output.return_value = OLD_CURL_OUTPUT
     assert not sources_module.curl_has_parallel_downloads()
+
+
+@pytest.mark.parametrize("curl_parallel", [True, False], indirect=["curl_parallel"])
+def test_curl_download_many_mixed_certs(tmp_path, monkeypatch, sources_module, curl_parallel):
+    monkeypatch.setenv("OSBUILD_SOURCES_CURL_USE_PARALLEL", "1")
+    # ensure this does not accidentaly succeeds because we retry
+    monkeypatch.setattr(sources_module, "NR_RETRYS", 1)
+
+    fake_httpd_root = tmp_path / "fake-httpd-root"
+    git_root = pathlib.Path(__file__).parent.parent.parent
+    cert1_path = git_root / "test/data/certs/cert1.pem"
+    key1_path = git_root / "test/data/certs/key1.pem"
+    cert2_path = git_root / "test/data/certs/cert2.pem"
+    key2_path = git_root / "test/data/certs/key2.pem"
+
+    with https_serve_directory(fake_httpd_root, certfile=cert1_path, keyfile=key1_path) as httpds:
+        with https_serve_directory(fake_httpd_root, certfile=cert2_path, keyfile=key2_path) as httpds2:
+            test_sources = make_test_sources(
+                fake_httpd_root, httpds.server_port, 2, start_n=10, cacert=cert1_path)
+            test_sources.update(make_test_sources(
+                fake_httpd_root, httpds2.server_port, 2, cacert=cert2_path))
+            assert len(test_sources) == 4
+
+            curl_parallel.cache = tmp_path / "curl-download-dir"
+            curl_parallel.cache.mkdir()
+            curl_parallel.fetch_all(test_sources)
+
+            assert httpds.reqs.count == 2
+            assert httpds2.reqs.count == 2