diff --git a/poetry.lock b/poetry.lock index 4ab95f8..10cd9bb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -436,6 +436,20 @@ files = [ {file = "diskcache-5.4.0.tar.gz", hash = "sha256:8879eb8c9b4a2509a5e633d2008634fb2b0b35c2b36192d89655dbde02419644"}, ] +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "filelock" version = "3.9.0" @@ -602,24 +616,25 @@ files = [ [[package]] name = "hypercorn" -version = "0.14.4" +version = "0.15.0" description = "A ASGI Server based on Hyper libraries and inspired by Gunicorn" optional = false python-versions = ">=3.7" files = [ - {file = "hypercorn-0.14.4-py3-none-any.whl", hash = "sha256:f956200dbf8677684e6e976219ffa6691d6cf795281184b41dbb0b135ab37b8d"}, - {file = "hypercorn-0.14.4.tar.gz", hash = "sha256:3fa504efc46a271640023c9b88c3184fd64993f47a282e8ae1a13ccb285c2f67"}, + {file = "hypercorn-0.15.0-py3-none-any.whl", hash = "sha256:5008944999612fd188d7a1ca02e89d20065642b89503020ac392dfed11840730"}, + {file = "hypercorn-0.15.0.tar.gz", hash = "sha256:d517f68d5dc7afa9a9d50ecefb0f769f466ebe8c1c18d2c2f447a24e763c9a63"}, ] [package.dependencies] h11 = "*" h2 = ">=3.1.0" priority = "*" +taskgroup = {version = "*", markers = "python_version < \"3.11\""} tomli = {version = "*", markers = "python_version < \"3.11\""} wsproto = ">=0.14.0" [package.extras] -docs = ["pydata_sphinx_theme"] +docs = ["pydata_sphinx_theme", "sphinxcontrib_mermaid"] h3 = ["aioquic (>=0.9.0,<1.0)"] trio = ["exceptiongroup (>=1.1.0)", "trio (>=0.22.0)"] uvloop = ["uvloop"] @@ -1319,6 +1334,20 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "taskgroup" +version = "0.0.0a4" +description = "backport of asyncio.TaskGroup, asyncio.Runner and asyncio.timeout" +optional = false +python-versions = "*" +files = [ + {file = "taskgroup-0.0.0a4-py2.py3-none-any.whl", hash = "sha256:5c1bd0e4c06114e7a4128583ab75c987597d5378a33948a3b74c662b90f61277"}, + {file = "taskgroup-0.0.0a4.tar.gz", hash = "sha256:eb08902d221e27661950f2a0320ddf3f939f579279996f81fe30779bca3a159c"}, +] + +[package.dependencies] +exceptiongroup = "*" + [[package]] name = "tomli" version = "2.0.1" @@ -1638,4 +1667,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "cb7c5c5801a3652677872db622f5ec969fcbcec5d7f463b96864918b88a235d0" +content-hash = "c58c8e6a3472cf95a8986b8cc4c78f88dc2a47e17c6b49f54cd1d291508ce2df" diff --git a/pyproject.toml b/pyproject.toml index 4c0de9f..8412633 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ pydantic = "^2.3.0" black = "^22.8.0" types-Flask = "^1.1.6" +[tool.poetry.group.dev.dependencies] +hypercorn = "^0.15.0" + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/src/herald/metric.py b/src/herald/metric.py index 7da7eb4..c8e574e 100644 --- a/src/herald/metric.py +++ b/src/herald/metric.py @@ -38,7 +38,7 @@ artifact_size = Histogram( "herald_artifact_size_bytes", "Size of artifacts", - buckets=[1e6, 1e7, 1e8] + [n*1e8 for n in range(2, 10)] + [1e9] + [float("inf")], + buckets=[1e6, 1e7, 1e8] + [n * 1e8 for n in range(2, 10)] + [1e9] + [float("inf")], ) artifact_size_rejected = Counter( "herald_artifact_size_rejected_total", diff --git a/src/herald/templates/loading.html b/src/herald/templates/loading.html index 22fa8a7..6d932ed 100644 --- a/src/herald/templates/loading.html +++ b/src/herald/templates/loading.html @@ -5,11 +5,6 @@ {%- endblock %} {% block content %} -
@@ -29,5 +24,10 @@ Loading
+ +
+ {% endblock %} diff --git a/src/herald/web.py b/src/herald/web.py index f88ea49..ce73786 100644 --- a/src/herald/web.py +++ b/src/herald/web.py @@ -225,41 +225,9 @@ async def async_get_file() -> Tuple[IO[bytes], str]: to_png=to_png, ) - @stream_with_context - async def reload_response(): - yield await render_template( - "loading.html", - file=file, - repo=f"{owner}/{repo}", - artifact_id=artifact_id, - ) - try: - await async_get_file() - except Exception as e: - details: str | None = None - if isinstance(e, github.ArtifactExpired): - details = f"Artifact #{artifact_id} has expired on GitHub" - artifact_expired[artifact_id] = True - if isinstance(e, github.ArtifactTooLarge): - details = f"Artifact #{artifact_id} is too large to download" - elif isinstance(e, fs.errors.ResourceNotFound): - details = f"File {file} not found in artifact" - - message = json.dumps( - await render_template( - "error.html", - file=file, - repo=f"{owner}/{repo}", - artifact_id=artifact_id, - details=details, - ) - ) - yield f""" - """ - return - yield "" + is_cached = gh.is_artifact_cached(f"{owner}/{repo}", artifact_id) + + is_htmx = "HX-Request" in request.headers # Assumption: curl etc will `Accept` *anything* is_browser: bool = request.headers.get("Accept", "*/*") != "*/*" @@ -268,11 +236,7 @@ async def reload_response(): "We think this is a browser request based on Accept header %s", request.headers.get("Accept"), ) - if ( - gh.is_artifact_cached(f"{owner}/{repo}", artifact_id) - or not is_browser - or not config.ENABLE_LOADING_PAGE - ): + if is_cached or not is_browser or not config.ENABLE_LOADING_PAGE: logger.debug("File is cached, call and return immediately") buf, mime = await async_get_file() response = await make_response(buf.read()) @@ -280,12 +244,16 @@ async def reload_response(): response.headers["Cache-Control"] = "max-age=31536000" response.headers["Etag"] = exp_etag - if "HX-Request" in request.headers: - response.headers["HX-Redirect"] = request.url + response.headers["HX-Redirect"] = request.url + response.headers["HX-Redirect"] = request.url else: logger.debug("File is not cached") + if is_browser: + logger.debug("Starting artifact download in the background") + app.add_background_task(async_get_file) + return redirect( url_for( "loading", @@ -313,6 +281,39 @@ async def reload_response(): # except fs.errors.ResourceNotFound: # abort(404) + @app.route("/poll///") + @app.route("/view////") + async def view_poll(owner: str, repo: str, artifact_id: int, path: str = ""): + is_cached = gh.is_artifact_cached(f"{owner}/{repo}", artifact_id) + logger.debug( + "Polling for %s/%s #%d => %s, is cached: %s", + owner, + repo, + artifact_id, + path, + is_cached, + ) + + if is_cached: + return ( + "", + 200, + { + "HX-Redirect": url_for( + "view", + owner=owner, + repo=repo, + artifact_id=artifact_id, + file=path, + ) + }, + ) + poll_url = url_for("view_poll", owner=owner, repo=repo, artifact_id=artifact_id) + + return f""" +
+ """ + @app.route("/loading////") @app.route("/loading////") async def loading(owner: str, repo: str, artifact_id: int, file: str = ""):