Skip to content

Commit

Permalink
sources: do not use %{json} when generating curl output
Browse files Browse the repository at this point in the history
We cannot use `curl --write-out %{json}` because older curl
(7.76 from RHEL9/Centos9) will write `{"http_connect":000}`
which python cannot parse.
  • Loading branch information
mvo5 authored and supakeen committed Jul 4, 2024
1 parent 018c15a commit 4697a3f
Showing 1 changed file with 17 additions and 12 deletions.
29 changes: 17 additions & 12 deletions sources/org.osbuild.curl
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ up the download.

import concurrent.futures
import contextlib
import json
import os
import pathlib
import platform
Expand Down Expand Up @@ -113,7 +112,6 @@ def curl_has_parallel_downloads():
if int(major) > 7:
return True
# --parallel got added in 7.68
# --write-out "%{json}" was added in 7.70
# --write-out "%{exitcode} is 7.75
if int(major) == 7 and int(minor) >= 75:
return True
Expand Down Expand Up @@ -166,14 +164,19 @@ def gen_curl_download_config(config_path: pathlib.Path, chksum_desc_tuple: List[
fp.write("\n")


def try_parse_curl_json_line(line):
def try_parse_curl_line(line):
line = line.strip()
if line:
try:
return json.loads(line)
except json.decoder.JSONDecodeError:
print(f"WARNING: cannot decode {line}", file=sys.stderr)
return None
print(line)
if not line.startswith("osbuild-dl\x1c"):
print(f"WARNING: unexpected prefix in {line}", file=sys.stderr)
return None
_, url, filename, exitcode, errormsg = line.split("\x1c")
return {
"url": url,
"filename_effective": filename,
"exitcode": int(exitcode),
"errormsg": errormsg,
}


def validate_and_move_to_targetdir(tmpdir, targetdir, checksum, origin):
Expand Down Expand Up @@ -201,8 +204,10 @@ def fetch_many_new_curl(tmpdir, targetdir, dl_pairs):
# this adds a bunch of noise but might be nice for debug?
# "--show-error",
"--parallel",
# this will write out a json record for each finished download
"--write-out", "%{json}\n",
# this will write out a "record" for each finished download
# Not using %{json} here because older curl (7.76) will write
# {"http_connect":000} which python cannot parse
"--write-out", "osbuild-dl\x1c%{url}\x1c%{filename_effective}\x1c%{exitcode}\x1cerror: %{errormsg}\n",
]
with contextlib.ExitStack() as cm:
curl_p = subprocess.Popen(curl_command, encoding="utf-8", cwd=tmpdir, stdout=subprocess.PIPE)
Expand All @@ -213,7 +218,7 @@ def fetch_many_new_curl(tmpdir, targetdir, dl_pairs):
# empty line means eof/process finished
if line == "":
break
dl_details = try_parse_curl_json_line(line)
dl_details = try_parse_curl_line(line)
if not dl_details:
continue
url = dl_details['url']
Expand Down

0 comments on commit 4697a3f

Please sign in to comment.