Skip to content

Commit

Permalink
fix: Retry individual calls to the /missing_objects endpoint
Browse files Browse the repository at this point in the history
Split out the individual API call into its own function so retry
attempts continue from the position of the loop in the calling function,
rather than looping all over again.
  • Loading branch information
benjamb committed Aug 30, 2024
1 parent f284001 commit 11be962
Showing 1 changed file with 14 additions and 10 deletions.
24 changes: 14 additions & 10 deletions flat-manager-client
Original file line number Diff line number Diff line change
Expand Up @@ -241,24 +241,28 @@ def chunks(iterable, n):
retry=TENACITY_RETRY_EXCEPTIONS,
reraise=True,
)
async def _missing_chunk(session, build_url, chunk, headers):
wanted_json = json.dumps({"wanted": chunk}).encode("utf-8")
data = gzip.compress(wanted_json)

resp = await session.get(build_url + "/missing_objects", data=data, headers=headers)
async with resp:
if resp.status != 200:
raise ApiError(resp, await resp.text())

return await resp.json()


async def missing_objects(session, build_url, token, wanted):
missing = []
for chunk in chunks(wanted, 2000):
wanted_json = json.dumps({"wanted": chunk}).encode("utf-8")
data = gzip.compress(wanted_json)
headers = {
"Authorization": "Bearer " + token,
"Content-Encoding": "gzip",
"Content-Type": "application/json",
}
resp = await session.get(
build_url + "/missing_objects", data=data, headers=headers
)
async with resp:
if resp.status != 200:
raise ApiError(resp, await resp.text())
data = await resp.json()
missing.extend(data["missing"])
data = await _missing_chunk(session, build_url, chunk, headers)
missing.extend(data["missing"])
return missing


Expand Down

0 comments on commit 11be962

Please sign in to comment.