From abd4492fa597389268ac73d032c4efd53e11ccc6 Mon Sep 17 00:00:00 2001 From: jstzwj <1103870790@qq.com> Date: Tue, 3 Sep 2024 00:40:05 +0800 Subject: [PATCH] add revision invalid id error message --- olah/cache/stat.py | 9 +++- olah/database/models.py | 9 ++++ olah/errors.py | 9 ++++ olah/server.py | 98 ++++++++++++++++------------------------ olah/utils/repo_utils.py | 1 + 5 files changed, 64 insertions(+), 62 deletions(-) diff --git a/olah/cache/stat.py b/olah/cache/stat.py index e8683f7..94f8040 100644 --- a/olah/cache/stat.py +++ b/olah/cache/stat.py @@ -1,6 +1,11 @@ -import argparse - +# coding=utf-8 +# Copyright 2024 XiaHan +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +import argparse import os import sys from olah.cache.olah_cache import OlahCache diff --git a/olah/database/models.py b/olah/database/models.py index 1d4d9fc..866f046 100644 --- a/olah/database/models.py +++ b/olah/database/models.py @@ -24,3 +24,12 @@ class Meta: class User(BaseModel): username = CharField(unique=True) + + +class DownloadLogs(BaseModel): + id = CharField(unique=True) + org = CharField() + repo = CharField() + path = CharField() + datetime = DateTimeField() + user = CharField() \ No newline at end of file diff --git a/olah/errors.py b/olah/errors.py index a020c57..77491f7 100644 --- a/olah/errors.py +++ b/olah/errors.py @@ -38,3 +38,12 @@ def error_entry_not_found(branch: str, path: str) -> Response: status_code=404, ) +def error_revision_not_found(revision: str) -> Response: + return JSONResponse( + content={"error": f"Invalid rev id: {revision}"}, + headers={ + "x-error-code": "RevisionNotFound", + "x-error-message": f"Invalid rev id: {revision}", + }, + status_code=404, + ) diff --git a/olah/server.py b/olah/server.py index 7c85a25..6169ad4 100644 --- a/olah/server.py +++ b/olah/server.py @@ -49,7 +49,7 @@ raise Exception("Cannot import BaseSettings from pydantic or pydantic-settings") from olah.configs import OlahConfig -from olah.errors import error_repo_not_found, error_page_not_found +from olah.errors import error_repo_not_found, error_page_not_found, error_revision_not_found from olah.mirror.repos import LocalMirrorRepo from olah.proxy.files import cdn_file_get_generator, file_get_generator from olah.proxy.lfs import lfs_get_generator, lfs_head_generator @@ -87,19 +87,13 @@ async def check_connection(url: str) -> bool: async def check_hf_connection() -> None: if app.app_settings.config.offline: return + scheme = app.app_settings.config.hf_scheme + netloc = app.app_settings.config.hf_netloc hf_online_status = await check_connection( - "https://huggingface.co/datasets/Salesforce/wikitext/resolve/main/.gitattributes" + f"{scheme}://{netloc}/datasets/Salesforce/wikitext/resolve/main/.gitattributes" ) if not hf_online_status: - logger.info( - "Cannot reach Huggingface Official Site. Trying to connect hf-mirror." - ) - hf_mirror_online_status = await check_connection( - "https://hf-mirror.com/datasets/Salesforce/wikitext/resolve/main/.gitattributes" - ) - if not hf_online_status and not hf_mirror_online_status: - logger.error("Failed to reach Huggingface Official Site.") - logger.error("Failed to reach hf-mirror Site.") + logger.error("Failed to reach Huggingface Site.") @asynccontextmanager @@ -145,21 +139,16 @@ async def meta_proxy_common(repo_type: str, org: str, repo: str, commit: str, re # Proxy the HF File Meta try: - if not app.app_settings.config.offline and not await check_commit_hf( - app, - repo_type, - org, - repo, - commit=commit, - authorization=request.headers.get("authorization", None), - ): - return error_repo_not_found() - commit_sha = await get_commit_hf( - app, - repo_type, - org, - repo, - commit=commit, + if not app.app_settings.config.offline: + if not await check_commit_hf(app, repo_type, org, repo, commit=None, + authorization=request.headers.get("authorization", None), + ): + return error_repo_not_found() + if not await check_commit_hf(app, repo_type, org, repo, commit=commit, + authorization=request.headers.get("authorization", None), + ): + return error_revision_not_found(revision=commit) + commit_sha = await get_commit_hf(app, repo_type, org, repo, commit=commit, authorization=request.headers.get("authorization", None), ) if commit_sha is None: @@ -275,21 +264,16 @@ async def tree_proxy_common( # Proxy the HF File Meta try: - if not app.app_settings.config.offline and not await check_commit_hf( - app, - repo_type, - org, - repo, - commit=commit, - authorization=request.headers.get("authorization", None), - ): - return error_repo_not_found() - commit_sha = await get_commit_hf( - app, - repo_type, - org, - repo, - commit=commit, + if not app.app_settings.config.offline: + if not await check_commit_hf(app, repo_type, org, repo, commit=None, + authorization=request.headers.get("authorization", None), + ): + return error_repo_not_found() + if not await check_commit_hf(app, repo_type, org, repo, commit=commit, + authorization=request.headers.get("authorization", None), + ): + return error_revision_not_found(revision=commit) + commit_sha = await get_commit_hf(app, repo_type, org, repo, commit=commit, authorization=request.headers.get("authorization", None), ) if commit_sha is None: @@ -381,7 +365,6 @@ async def tree_proxy_commit( ) -# TODO: paths-info async def pathsinfo_proxy_common(repo_type: str, org: str, repo: str, commit: str, paths: List[str], request: Request) -> Response: # TODO: the head method of meta apis # FIXME: do not show the private repos to other user besides owner, even though the repo was cached @@ -396,31 +379,26 @@ async def pathsinfo_proxy_common(repo_type: str, org: str, repo: str, commit: st git_path = os.path.join(mirror_path, repo_type, org, repo) if os.path.exists(git_path): local_repo = LocalMirrorRepo(git_path, repo_type, org, repo) - tree_data = local_repo.get_pathinfos(commit, paths) - if tree_data is None: + pathsinfo_data = local_repo.get_pathinfos(commit, paths) + if pathsinfo_data is None: continue - return JSONResponse(content=tree_data) + return JSONResponse(content=pathsinfo_data) except git.exc.InvalidGitRepositoryError: logger.warning(f"Local repository {git_path} is not a valid git reposity.") continue # Proxy the HF File Meta try: - if not app.app_settings.config.offline and not await check_commit_hf( - app, - repo_type, - org, - repo, - commit=commit, - authorization=request.headers.get("authorization", None), - ): - return error_repo_not_found() - commit_sha = await get_commit_hf( - app, - repo_type, - org, - repo, - commit=commit, + if not app.app_settings.config.offline: + if not await check_commit_hf(app, repo_type, org, repo, commit=None, + authorization=request.headers.get("authorization", None), + ): + return error_repo_not_found() + if not await check_commit_hf(app, repo_type, org, repo, commit=commit, + authorization=request.headers.get("authorization", None), + ): + return error_revision_not_found(revision=commit) + commit_sha = await get_commit_hf(app, repo_type, org, repo, commit=commit, authorization=request.headers.get("authorization", None), ) if commit_sha is None: diff --git a/olah/utils/repo_utils.py b/olah/utils/repo_utils.py index ebbcbd1..56fc3e1 100644 --- a/olah/utils/repo_utils.py +++ b/olah/utils/repo_utils.py @@ -276,6 +276,7 @@ async def get_commit_hf( except: return await get_commit_hf_offline(app, repo_type, org, repo, commit) + @tenacity.retry(stop=tenacity.stop_after_attempt(3)) async def check_commit_hf( app,