From ada5aed098ac3651594c731b8610a24db5a231e3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 11 Jul 2024 17:23:14 +0200 Subject: [PATCH] sources(curl): use `--next` for each url in curl config curl keeps a global parser state. This means that if there are multiple "cacert =" values they are just overriden and the last one wins. This is why the `test_curl_download_many_mixed_certs` test did not work - the second `cacert = ` overwrites the previous one. To fix this we need to use `--next` when we need to change options on a per url (like `cacert`) basis. With `--next` curl starts a new parser state for the next url (but keeps the options for the previous ones set). This commit does that in a slightly naive way by just repeating our options for each url. Technically we could sort the sources so that we have less repetition but other then slightly smaller auto-generated files it has no advantage. With this commit the `test_curl_download_many_mixed_certs` test works. --- sources/org.osbuild.curl | 24 ++++++++++----------- sources/test/test_curl_source.py | 37 +++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/sources/org.osbuild.curl b/sources/org.osbuild.curl index b350f7d9c..8176105c8 100755 --- a/sources/org.osbuild.curl +++ b/sources/org.osbuild.curl @@ -141,31 +141,30 @@ def _quote_url(url: str) -> str: def gen_curl_download_config(config_path: pathlib.Path, chksum_desc_tuple: List[Tuple[str, Dict]], parallel=False): with open(config_path, "w", encoding="utf8") as fp: - # global options if parallel: - fp.write("parallel\n") - fp.write(textwrap.dedent(f"""\ + fp.write("parallel\n\n") + # Because we use --next which resets the parser state we need to set + # these options for each url. + per_url_opts = textwrap.dedent(f"""\ user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" silent speed-limit = 1000 connect-timeout = 30 fail location - """)) + """) if parallel: - fp.write(textwrap.dedent(f"""\ + per_url_opts += textwrap.dedent(f"""\ write-out = "{CURL_WRITE_OUT}" - """)) - + """) proxy = os.getenv("OSBUILD_SOURCES_CURL_PROXY") if proxy: - fp.write(f'proxy = "{proxy}"\n') - fp.write("\n") - # per url options - for checksum, desc in chksum_desc_tuple: + per_url_opts += f'proxy = "{proxy}"\n' + for i, (checksum, desc) in enumerate(chksum_desc_tuple): url = _quote_url(desc.get("url")) fp.write(f'url = "{url}"\n') fp.write(f'output = "{checksum}"\n') + fp.write(f'{per_url_opts}') secrets = desc.get("secrets") if secrets: ssl_ca_cert = secrets.get('ssl_ca_cert') @@ -182,7 +181,8 @@ def gen_curl_download_config(config_path: pathlib.Path, chksum_desc_tuple: List[ fp.write('insecure\n') else: fp.write('no-insecure\n') - fp.write("\n") + if i + 1 < len(chksum_desc_tuple): + fp.write("next\n\n") def try_parse_curl_line(line): diff --git a/sources/test/test_curl_source.py b/sources/test/test_curl_source.py index 05afff16d..c984cff01 100644 --- a/sources/test/test_curl_source.py +++ b/sources/test/test_curl_source.py @@ -260,17 +260,15 @@ def test_curl_gen_download_config_old_curl(tmp_path, sources_module): assert config_path.exists() assert config_path.read_text(encoding="utf8") == textwrap.dedent(f"""\ + url = "http://example.com/file/0" + output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9" user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" silent speed-limit = 1000 connect-timeout = 30 fail location - - url = "http://example.com/file/0" - output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9" no-insecure - """) @@ -281,6 +279,9 @@ def test_curl_gen_download_config_parallel(tmp_path, sources_module): assert config_path.exists() assert config_path.read_text(encoding="utf8") == textwrap.dedent(f"""\ parallel + + url = "http://example.com/file/0" + output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9" user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" silent speed-limit = 1000 @@ -288,26 +289,46 @@ def test_curl_gen_download_config_parallel(tmp_path, sources_module): fail location write-out = "{sources_module.CURL_WRITE_OUT}" - - url = "http://example.com/file/0" - output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9" no-insecure + next url = "http://example.com/file/1" output = "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" + user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" + silent + speed-limit = 1000 + connect-timeout = 30 + fail + location + write-out = "{sources_module.CURL_WRITE_OUT}" insecure + next url = "http://example.com/file/2" output = "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" + user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" + silent + speed-limit = 1000 + connect-timeout = 30 + fail + location + write-out = "{sources_module.CURL_WRITE_OUT}" cacert = "some-ssl_ca_cert" no-insecure + next url = "http://example.com/file/3" output = "sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce" + user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)" + silent + speed-limit = 1000 + connect-timeout = 30 + fail + location + write-out = "{sources_module.CURL_WRITE_OUT}" cert = "some-ssl_client_cert" key = "some-ssl_client_key" no-insecure - """)