sources(curl): use --next for each url in curl config

curl keeps a global parser state. This means that if there are
multiple "cacert =" values they are just overriden and the last
one wins. This is why the `test_curl_download_many_mixed_certs`
test did not work - the second `cacert = ` overwrites the previous
one.

To fix this we need to use `--next` when we need to change options
on a per url (like `cacert`) basis. With `--next` curl starts a
new parser state for the next url (but keeps the options for the
previous ones set). This commit does that in a slightly naive
way by just repeating our options for each url. Technically
we could sort the sources so that we have less repetition but
other then slightly smaller auto-generated files it has no
advantage.

With this commit the `test_curl_download_many_mixed_certs` test
works.

sources/org.osbuild.curl

@@ -141,31 +141,30 @@ def _quote_url(url: str) -> str:
 
 def gen_curl_download_config(config_path: pathlib.Path, chksum_desc_tuple: List[Tuple[str, Dict]], parallel=False):
     with open(config_path, "w", encoding="utf8") as fp:
-        # global options
         if parallel:
-            fp.write("parallel\n")
-        fp.write(textwrap.dedent(f"""\
+            fp.write("parallel\n\n")
+        # Because we use --next which resets the parser state we need to set
+        # these options for each url.
+        per_url_opts = textwrap.dedent(f"""\
         user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
         silent
         speed-limit = 1000
         connect-timeout = 30
         fail
         location
-        """))
+        """)
         if parallel:
-            fp.write(textwrap.dedent(f"""\
+            per_url_opts += textwrap.dedent(f"""\
             write-out = "{CURL_WRITE_OUT}"
-            """))
-
+            """)
         proxy = os.getenv("OSBUILD_SOURCES_CURL_PROXY")
         if proxy:
-            fp.write(f'proxy = "{proxy}"\n')
-        fp.write("\n")
-        # per url options
-        for checksum, desc in chksum_desc_tuple:
+            per_url_opts += f'proxy = "{proxy}"\n'
+        for i, (checksum, desc) in enumerate(chksum_desc_tuple):
             url = _quote_url(desc.get("url"))
             fp.write(f'url = "{url}"\n')
             fp.write(f'output = "{checksum}"\n')
+            fp.write(f'{per_url_opts}')
             secrets = desc.get("secrets")
             if secrets:
                 ssl_ca_cert = secrets.get('ssl_ca_cert')
@@ -182,7 +181,8 @@ def gen_curl_download_config(config_path: pathlib.Path, chksum_desc_tuple: List[
                 fp.write('insecure\n')
             else:
                 fp.write('no-insecure\n')
-            fp.write("\n")
+            if i + 1 < len(chksum_desc_tuple):
+                fp.write("next\n\n")
 
 
 def try_parse_curl_line(line):

sources/test/test_curl_source.py

@@ -260,17 +260,15 @@ def test_curl_gen_download_config_old_curl(tmp_path, sources_module):
 
     assert config_path.exists()
     assert config_path.read_text(encoding="utf8") == textwrap.dedent(f"""\
+    url = "http://example.com/file/0"
+    output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
     user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
     silent
     speed-limit = 1000
     connect-timeout = 30
     fail
     location
-
-    url = "http://example.com/file/0"
-    output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
     no-insecure
-
     """)
 
 
@@ -281,33 +279,56 @@ def test_curl_gen_download_config_parallel(tmp_path, sources_module):
     assert config_path.exists()
     assert config_path.read_text(encoding="utf8") == textwrap.dedent(f"""\
     parallel
+
+    url = "http://example.com/file/0"
+    output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
     user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
     silent
     speed-limit = 1000
     connect-timeout = 30
     fail
     location
     write-out = "{sources_module.CURL_WRITE_OUT}"
-
-    url = "http://example.com/file/0"
-    output = "sha256:5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
     no-insecure
+    next
 
     url = "http://example.com/file/1"
     output = "sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"
+    user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
+    silent
+    speed-limit = 1000
+    connect-timeout = 30
+    fail
+    location
+    write-out = "{sources_module.CURL_WRITE_OUT}"
     insecure
+    next
 
     url = "http://example.com/file/2"
     output = "sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35"
+    user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
+    silent
+    speed-limit = 1000
+    connect-timeout = 30
+    fail
+    location
+    write-out = "{sources_module.CURL_WRITE_OUT}"
     cacert = "some-ssl_ca_cert"
     no-insecure
+    next
 
     url = "http://example.com/file/3"
     output = "sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce"
+    user-agent = "osbuild (Linux.{platform.machine()}; https://osbuild.org/)"
+    silent
+    speed-limit = 1000
+    connect-timeout = 30
+    fail
+    location
+    write-out = "{sources_module.CURL_WRITE_OUT}"
     cert = "some-ssl_client_cert"
     key = "some-ssl_client_key"
     no-insecure
-
     """)