From aa3d429e502f81533ec403f8e4a990d99f311f07 Mon Sep 17 00:00:00 2001 From: Dmitry Orlov Date: Mon, 17 Jul 2023 20:59:18 +0300 Subject: [PATCH 1/2] Raise exceptions with request context --- aiohttp_s3_client/client.py | 88 +++++++++++++++++++++++---------- poetry.lock | 66 +++---------------------- pyproject.toml | 6 +-- tests/test_get_file_parallel.py | 13 +++-- 4 files changed, 80 insertions(+), 93 deletions(-) diff --git a/aiohttp_s3_client/client.py b/aiohttp_s3_client/client.py index a6249b7..c8cdb6b 100644 --- a/aiohttp_s3_client/client.py +++ b/aiohttp_s3_client/client.py @@ -3,9 +3,9 @@ import io import logging import os +import sys import typing as t from collections import deque -from contextlib import suppress from functools import partial from http import HTTPStatus from itertools import chain @@ -16,8 +16,12 @@ from urllib.parse import quote from aiohttp import ClientSession, hdrs -from aiohttp.client import _RequestContextManager as RequestContextManager -from aiohttp.client_exceptions import ClientError +# noinspection PyProtectedMember +from aiohttp.client import ( + _RequestContextManager as RequestContextManager, + ClientResponse, +) +from aiohttp.client_exceptions import ClientError, ClientResponseError from aiomisc import asyncbackoff, threaded, threaded_iterable from aws_request_signer import UNSIGNED_PAYLOAD from multidict import CIMultiDict, CIMultiDictProxy @@ -31,10 +35,8 @@ parse_create_multipart_upload_id, parse_list_objects, ) - log = logging.getLogger(__name__) - CHUNK_SIZE = 2 ** 16 DONE = object() EMPTY_STR_HASH = hashlib.sha256(b"").hexdigest() @@ -42,12 +44,20 @@ HeadersType = t.Union[t.Dict, CIMultiDict, CIMultiDictProxy] - threaded_iterable_constrained = threaded_iterable(max_size=2) -class AwsError(ClientError): - pass +class AwsError(ClientResponseError): + def __init__( + self, resp: ClientResponse, message: str, *history: ClientResponse + ): + super().__init__( + headers=resp.headers, + history=(resp, *history), + message=message, + request_info=resp.request_info, + status=resp.status, + ) class AwsUploadError(AwsError): @@ -58,6 +68,20 @@ class AwsDownloadError(AwsError): pass +if sys.version_info < (3, 8): + from contextlib import suppress + + + @threaded + def unlink_path(path: Path) -> None: + with suppress(FileNotFoundError): + os.unlink(path.resolve()) +else: + @threaded + def unlink_path(path: Path) -> None: + path.unlink(missing_ok=True) + + @threaded def concat_files( target_file: Path, files: t.List[t.IO[bytes]], buffer_size: int, @@ -117,7 +141,6 @@ def file_sender( async_file_sender = threaded_iterable_constrained(file_sender) - DataType = t.Union[bytes, str, t.AsyncIterable[bytes]] ParamsType = t.Optional[t.Mapping[str, str]] @@ -282,8 +305,11 @@ async def _create_multipart_upload( payload = await resp.read() if resp.status != HTTPStatus.OK: raise AwsUploadError( - f"Wrong status code {resp.status} from s3 with message " - f"{payload.decode()}.", + resp, + ( + f"Wrong status code {resp.status} from s3 " + f"with message {payload.decode()}." + ), ) return parse_create_multipart_upload_id(payload) @@ -308,8 +334,11 @@ async def _complete_multipart_upload( if resp.status != HTTPStatus.OK: payload = await resp.text() raise AwsUploadError( - f"Wrong status code {resp.status} from s3 with message " - f"{payload}.", + resp, + ( + f"Wrong status code {resp.status} from s3 " + f"with message {payload}." + ), ) async def _put_part( @@ -331,8 +360,11 @@ async def _put_part( payload = await resp.text() if resp.status != HTTPStatus.OK: raise AwsUploadError( - f"Wrong status code {resp.status} from s3 with message " - f"{payload}.", + resp, + ( + f"Wrong status code {resp.status} from s3 " + f"with message {payload}." + ), ) return resp.headers["Etag"].strip('"') @@ -519,7 +551,6 @@ async def _download_range( writer: t.Callable[[bytes, int, int], t.Coroutine], *, etag: str, - pos: int, range_start: int, req_range_start: int, req_range_end: int, @@ -546,8 +577,11 @@ async def _download_range( async with self.get(object_name, headers=headers, **kwargs) as resp: if resp.status not in (HTTPStatus.PARTIAL_CONTENT, HTTPStatus.OK): raise AwsDownloadError( - f"Got wrong status code {resp.status} on range download " - f"of {object_name}", + resp, + ( + f"Got wrong status code {resp.status} on " + f"range download of {object_name}" + ), ) while True: chunk = await resp.content.read(buffer_size) @@ -594,7 +628,6 @@ async def _download_worker( object_name, writer, etag=etag, - pos=(req_range_start - range_start), range_start=range_start, req_range_start=req_range_start, req_range_end=req_range_end - 1, @@ -632,8 +665,11 @@ async def get_file_parallel( async with self.head(str(object_name), headers=headers) as resp: if resp.status != HTTPStatus.OK: raise AwsDownloadError( - f"Got response for HEAD request for {object_name}" - f"of a wrong status {resp.status}", + resp, + ( + f"Got response for HEAD request for " + f"{object_name} of a wrong status {resp.status}" + ), ) etag = resp.headers["Etag"] file_size = int(resp.headers["Content-Length"]) @@ -692,8 +728,7 @@ async def get_file_parallel( "Error on file download. Removing possibly incomplete file %s", file_path, ) - with suppress(FileNotFoundError): - os.unlink(file_path) + await unlink_path(file_path) raise async def list_objects_v2( @@ -746,8 +781,11 @@ async def list_objects_v2( async with self.get(str(object_name), params=params) as resp: if resp.status != HTTPStatus.OK: raise AwsDownloadError( - f"Got response with wrong status for GET request for " - f"{object_name} with prefix '{prefix}'", + resp, + ( + "Got response with wrong status for GET request " + f"for {object_name} with prefix '{prefix}'" + ), ) payload = await resp.read() metadata, continuation_token = parse_list_objects(payload) diff --git a/poetry.lock b/poetry.lock index 3e04377..ac55abd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "aiohttp" version = "3.8.4" description = "Async http client/server framework (asyncio)" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -113,14 +112,13 @@ speedups = ["Brotli", "aiodns", "cchardet"] [[package]] name = "aiomisc" -version = "17.3.2" +version = "17.3.4" description = "aiomisc - miscellaneous utils for asyncio" -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "aiomisc-17.3.2-py3-none-any.whl", hash = "sha256:0a588c78b785233ee3ab0239fadfbfae8990c21eada0f76a562a4743e3bbb13c"}, - {file = "aiomisc-17.3.2.tar.gz", hash = "sha256:b49f1dd3629b1b8cfa7147e5be3485fd8ed41c7f14a964fc518ed716a34f3fea"}, + {file = "aiomisc-17.3.4-py3-none-any.whl", hash = "sha256:67f30a246cad1f273eb52863994280bedd10aa655675a901b3fad7d279ee2b0f"}, + {file = "aiomisc-17.3.4.tar.gz", hash = "sha256:3122d9581824cb73f043106cd4c8fc87f1dc7ab96fce916f3446403e889d4697"}, ] [package.dependencies] @@ -141,7 +139,6 @@ uvloop = ["uvloop (>=0.14,<1)"] name = "aiomisc-pytest" version = "1.1.1" description = "pytest integration for aiomisc" -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -157,7 +154,6 @@ pytest = ">=7.2.1,<8.0.0" name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -172,7 +168,6 @@ frozenlist = ">=1.1.0" name = "async-timeout" version = "4.0.2" description = "Timeout context manager for asyncio programs" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -187,7 +182,6 @@ typing-extensions = {version = ">=3.6.5", markers = "python_version < \"3.8\""} name = "asynctest" version = "0.13.0" description = "Enhance the standard unittest package with features for testing asyncio libraries" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -199,7 +193,6 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -221,7 +214,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "aws-request-signer" version = "1.2.0" description = "A python library to sign AWS requests using AWS Signature V4." -category = "main" optional = false python-versions = ">=3.6.1,<4.0.0" files = [ @@ -237,7 +229,6 @@ requests = ["requests (>=2.21,<3.0)"] name = "cached-property" version = "1.5.2" description = "A decorator for caching properties in classes." -category = "main" optional = false python-versions = "*" files = [ @@ -249,7 +240,6 @@ files = [ name = "cachetools" version = "5.3.1" description = "Extensible memoizing collections and decorators" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -261,7 +251,6 @@ files = [ name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -273,7 +262,6 @@ files = [ name = "chardet" version = "5.1.0" description = "Universal encoding detector for Python 3" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -285,7 +273,6 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -370,7 +357,6 @@ files = [ name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -382,7 +368,6 @@ files = [ name = "colorlog" version = "6.7.0" description = "Add colours to the output of Python's logging module." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -400,7 +385,6 @@ development = ["black", "flake8", "mypy", "pytest", "types-colorama"] name = "coverage" version = "6.5.0" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -466,7 +450,6 @@ toml = ["tomli"] name = "coveralls" version = "3.3.1" description = "Show coverage stats online via coveralls.io" -category = "dev" optional = false python-versions = ">= 3.5" files = [ @@ -475,7 +458,7 @@ files = [ ] [package.dependencies] -coverage = ">=4.1,<6.0.0 || >6.1,<6.1.1 || >6.1.1,<7.0" +coverage = ">=4.1,<6.0.dev0 || >6.1,<6.1.1 || >6.1.1,<7.0" docopt = ">=0.6.1" requests = ">=1.0.0" @@ -486,7 +469,6 @@ yaml = ["PyYAML (>=3.10)"] name = "distlib" version = "0.3.6" description = "Distribution utilities" -category = "dev" optional = false python-versions = "*" files = [ @@ -498,7 +480,6 @@ files = [ name = "docopt" version = "0.6.2" description = "Pythonic argument parser, that will make you smile" -category = "dev" optional = false python-versions = "*" files = [ @@ -509,7 +490,6 @@ files = [ name = "exceptiongroup" version = "1.1.1" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -524,7 +504,6 @@ test = ["pytest (>=6)"] name = "filelock" version = "3.12.2" description = "A platform independent file lock." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -540,7 +519,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "p name = "frozenlist" version = "1.3.3" description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -624,7 +602,6 @@ files = [ name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -636,7 +613,6 @@ files = [ name = "importlib-metadata" version = "6.7.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -657,7 +633,6 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -669,7 +644,6 @@ files = [ name = "logging-journald" version = "0.6.6" description = "Pure python logging handler for writing logs to the journald using native protocol" -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -681,7 +655,6 @@ files = [ name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -693,7 +666,6 @@ files = [ name = "multidict" version = "6.0.4" description = "multidict implementation" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -777,7 +749,6 @@ files = [ name = "mypy" version = "1.4.1" description = "Optional static typing for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -825,7 +796,6 @@ reports = ["lxml"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -837,7 +807,6 @@ files = [ name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -849,7 +818,6 @@ files = [ name = "platformdirs" version = "3.8.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -868,7 +836,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest- name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -887,7 +854,6 @@ testing = ["pytest", "pytest-benchmark"] name = "pycodestyle" version = "2.10.0" description = "Python style guide checker" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -899,7 +865,6 @@ files = [ name = "pydocstyle" version = "6.1.1" description = "Python docstring style checker" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -917,7 +882,6 @@ toml = ["toml"] name = "pyflakes" version = "3.0.1" description = "passive checker of Python programs" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -929,7 +893,6 @@ files = [ name = "pylama" version = "8.4.1" description = "Code audit tool for python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -958,7 +921,6 @@ vulture = ["vulture"] name = "pyproject-api" version = "1.5.2" description = "API to interact with the python pyproject.toml based projects" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -978,7 +940,6 @@ testing = ["covdefaults (>=2.3)", "importlib-metadata (>=6.6)", "pytest (>=7.3.1 name = "pytest" version = "7.4.0" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1002,7 +963,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-aiohttp" version = "1.0.4" description = "Pytest plugin for aiohttp support" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1022,7 +982,6 @@ testing = ["coverage (==6.2)", "mypy (==0.931)"] name = "pytest-asyncio" version = "0.21.0" description = "Pytest support for asyncio" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1042,7 +1001,6 @@ testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy name = "pytest-cov" version = "4.1.0" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1061,7 +1019,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-timeout" version = "2.1.0" description = "pytest plugin to abort hanging tests" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1076,7 +1033,6 @@ pytest = ">=5.0.0" name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1098,7 +1054,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "snowballstemmer" version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." -category = "dev" optional = false python-versions = "*" files = [ @@ -1110,7 +1065,6 @@ files = [ name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1122,7 +1076,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1134,7 +1087,6 @@ files = [ name = "tox" version = "4.6.3" description = "tox is a generic virtualenv management and test command line tool" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1164,7 +1116,6 @@ testing = ["build[virtualenv] (>=0.10)", "covdefaults (>=2.3)", "detect-test-pol name = "typed-ast" version = "1.5.4" description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1198,7 +1149,6 @@ files = [ name = "typing-extensions" version = "4.7.0" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1210,7 +1160,6 @@ files = [ name = "urllib3" version = "2.0.3" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1228,7 +1177,6 @@ zstd = ["zstandard (>=0.18.0)"] name = "virtualenv" version = "20.23.1" description = "Virtual Python Environment builder" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1250,7 +1198,6 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess name = "yarl" version = "1.9.2" description = "Yet another URL library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1339,7 +1286,6 @@ typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1354,4 +1300,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "db24156c0c91b811793da15270c6298fdfdb2fddea9f1f7291c1ee0e6e08c6d7" +content-hash = "0b586a78e7d99dae31c539677b62edfb29e9f233539e8cece86f926a236215f1" diff --git a/pyproject.toml b/pyproject.toml index 897f628..f9ece7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ authors = [ license = "Apache Software License" readme = "README.md" homepage = "https://github.com/aiokitchen/aiohttp-s3-client" -packages = [{include = "aiohttp_s3_client"}] +packages = [{ include = "aiohttp_s3_client" }] classifiers = [ "Development Status :: 4 - Beta", "License :: OSI Approved :: Apache Software License", @@ -40,7 +40,7 @@ classifiers = [ [tool.poetry.dependencies] aiohttp = "^3.8" -aiomisc = "^17" +aiomisc = "^17.3.4" aws-request-signer = "1.2.0" cached-property = [{ version = '^1.5.2', python = "< 3.8" }] typing_extensions = [{ version = '*', python = "< 3.10" }] @@ -52,7 +52,7 @@ aiomisc-pytest = "^1.1" coverage = "!=4.3" coveralls = "^3.3.1" mypy = "*" -pylama = {extras = ["toml"], version = "^8.4.1"} +pylama = { extras = ["toml"], version = "^8.4.1" } pytest = "^7.2.1" pytest-cov = "^4.0.0" pytest-timeout = "^2.1.0" diff --git a/tests/test_get_file_parallel.py b/tests/test_get_file_parallel.py index bccb755..11bf80b 100644 --- a/tests/test_get_file_parallel.py +++ b/tests/test_get_file_parallel.py @@ -5,7 +5,7 @@ import pytest from aiohttp_s3_client import S3Client -from aiohttp_s3_client.client import AwsDownloadError +from aiohttp_s3_client.client import AwsDownloadError, AwsError async def test_get_file_parallel(s3_client: S3Client, tmpdir): @@ -21,7 +21,9 @@ async def test_get_file_parallel(s3_client: S3Client, tmpdir): async def test_get_file_parallel_without_pwrite( - s3_client: S3Client, tmpdir, monkeypatch, + s3_client: S3Client, + tmpdir, + monkeypatch, ): monkeypatch.delattr("os.pwrite") data = b"Hello world! " * 100 @@ -36,7 +38,8 @@ async def test_get_file_parallel_without_pwrite( async def test_get_file_that_changed_in_process_error( - s3_client: S3Client, tmpdir, + s3_client: S3Client, + tmpdir, ): object_name = "test/test" @@ -58,7 +61,7 @@ async def upload(): workers_count=4, ) - with pytest.raises(Exception) as err: + with pytest.raises(AwsError) as err: await asyncio.gather( s3_client.get_file_parallel( object_name, @@ -70,7 +73,7 @@ async def upload(): ) assert err.type is AwsDownloadError - assert err.value.args[0].startswith( + assert err.value.message.startswith( "Got wrong status code 412 on range download of test/test", ) assert not os.path.exists(tmpdir / "temp.dat") From 68044d5766c0a25d9695bab061fa5a242280bc27 Mon Sep 17 00:00:00 2001 From: Dmitry Orlov Date: Mon, 17 Jul 2023 22:19:55 +0300 Subject: [PATCH 2/2] reformat code --- aiohttp_s3_client/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aiohttp_s3_client/client.py b/aiohttp_s3_client/client.py index c8cdb6b..6580991 100644 --- a/aiohttp_s3_client/client.py +++ b/aiohttp_s3_client/client.py @@ -71,7 +71,6 @@ class AwsDownloadError(AwsError): if sys.version_info < (3, 8): from contextlib import suppress - @threaded def unlink_path(path: Path) -> None: with suppress(FileNotFoundError):