From ed3386943069e138b346eef75ac64d9874a07810 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 13 Aug 2024 12:05:24 -0400 Subject: [PATCH] tools/osbuild-mpp: hash mpp-embed urls from stream Currently if one uses `mpp-embed` with URLs, osbuild-mpp still wants to download the full file just so it can hash it. Make this more efficient by hashing from the stream instead, which `hashlib` natively supports. This also makes osbuild-mpp work with large artifacts in environments that may not have enough space to temporarily save the data. --- tools/osbuild-mpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/osbuild-mpp b/tools/osbuild-mpp index 0252eac6d..eb610be2a 100755 --- a/tools/osbuild-mpp +++ b/tools/osbuild-mpp @@ -1608,13 +1608,15 @@ class ManifestFileV2(ManifestFile): f, _ = self.find_and_open_file(path, [], mode="rb", encoding=None) with f: data = f.read() + checksum = hashlib.sha256(data).hexdigest() elif url: response = urllib.request.urlopen(url) - data = response.fp.read() + h = hashlib.file_digest(response.fp, 'sha256') + checksum = h.hexdigest() else: data = bytes(text, "utf-8") + checksum = hashlib.sha256(data).hexdigest() - checksum = hashlib.sha256(data).hexdigest() digest = "sha256:" + checksum if url: @@ -1622,6 +1624,7 @@ class ManifestFileV2(ManifestFile): items = element_enter(source, "items", {}) items[digest] = url else: + assert data is not None encoded = base64.b64encode(data).decode("utf-8") source = element_enter(self.sources, "org.osbuild.inline", {}) items = element_enter(source, "items", {})