bump to botocore 1.24.21 (#926)

CHANGES.rst

@@ -1,5 +1,11 @@
 Changes
 -------
+2.2.0 (2022-03-16)
+^^^^^^^^^^^^^^^^^^
+* remove deprecated APIs
+* bump to botocore 1.24.21
+* re-enable retry of aiohttp.ClientPayloadError
+
 2.1.2 (2022-03-03)
 ^^^^^^^^^^^^^^^^^^
 * fix httpsession close call

aiobotocore/__init__.py

@@ -1,15 +1 @@
-# NOTE: These imports are deprecated and will be removed in 2.x
-import os
-
-# Enabling this will enable the old http exception behavior that exposed raw aiohttp
-# exceptions and old session variables available via __init__.  Disabling will swap to
-# botocore exceptions and will not have these imports to match botocore.
-# NOTE: without setting this to 0, retries may not work, see #876
-DEPRECATED_1_4_0_APIS = int(os.getenv('AIOBOTOCORE_DEPRECATED_1_4_0_APIS', '0'))
-
-if DEPRECATED_1_4_0_APIS:
-    from .session import get_session, AioSession
-
-    __all__ = ['get_session', 'AioSession']
-
-__version__ = '2.1.2'
+__version__ = '2.2.0'

aiobotocore/client.py

@@ -1,6 +1,7 @@
 from botocore.awsrequest import prepare_request_dict
 from botocore.client import logger, PaginatorDocstring, ClientCreator, \
-    BaseClient, ClientEndpointBridge, S3ArnParamHandler, S3EndpointSetter
+    BaseClient, ClientEndpointBridge, S3ArnParamHandler, S3EndpointSetter, \
+    resolve_checksum_context, apply_request_checksum
 from botocore.discovery import block_endpoint_discovery_required_operations
 from botocore.exceptions import OperationNotPageableError
 from botocore.history import get_global_history_recorder
@@ -196,6 +197,7 @@ async def _make_api_call(self, operation_name, api_params):
         }
         request_dict = await self._convert_to_request_dict(
             api_params, operation_model, context=request_context)
+        resolve_checksum_context(request_dict, operation_model, api_params)
 
         service_id = self._service_model.service_id.hyphenize()
         handler, event_response = await self.meta.events.emit_until_response(
@@ -208,6 +210,7 @@ async def _make_api_call(self, operation_name, api_params):
         if event_response is not None:
             http, parsed_response = event_response
         else:
+            apply_request_checksum(request_dict)
             http, parsed_response = await self._make_request(
                 operation_model, request_dict, request_context)
 

aiobotocore/config.py

@@ -5,7 +5,6 @@
 
 
 class AioConfig(botocore.client.Config):
-
     def __init__(self, connector_args=None, **kwargs):
         super().__init__(**kwargs)
 

aiobotocore/configprovider.py

@@ -0,0 +1,37 @@
+from botocore.configprovider import os, SmartDefaultsConfigStoreFactory
+
+
+class AioSmartDefaultsConfigStoreFactory(SmartDefaultsConfigStoreFactory):
+    async def merge_smart_defaults(self, config_store, mode, region_name):
+        if mode == 'auto':
+            mode = await self.resolve_auto_mode(region_name)
+        default_configs = self._default_config_resolver.get_default_config_values(
+            mode)
+        for config_var in default_configs:
+            config_value = default_configs[config_var]
+            method = getattr(self, f'_set_{config_var}', None)
+            if method:
+                method(config_store, config_value)
+
+    async def resolve_auto_mode(self, region_name):
+        current_region = None
+        if os.environ.get('AWS_EXECUTION_ENV'):
+            default_region = os.environ.get('AWS_DEFAULT_REGION')
+            current_region = os.environ.get('AWS_REGION', default_region)
+        if not current_region:
+            if self._instance_metadata_region:
+                current_region = self._instance_metadata_region
+            else:
+                try:
+                    current_region = \
+                        await self._imds_region_provider.provide()
+                    self._instance_metadata_region = current_region
+                except Exception:
+                    pass
+
+        if current_region:
+            if region_name == current_region:
+                return 'in-region'
+            else:
+                return 'cross-region'
+        return 'standard'

aiobotocore/endpoint.py

@@ -4,7 +4,7 @@
 import aiohttp.http_exceptions
 from botocore.endpoint import EndpointCreator, Endpoint, DEFAULT_TIMEOUT, \
     MAX_POOL_CONNECTIONS, logger, history_recorder, create_request_object, \
-    is_valid_ipv6_endpoint_url, is_valid_endpoint_url
+    is_valid_ipv6_endpoint_url, is_valid_endpoint_url, handle_checksum_body
 from botocore.exceptions import ConnectionClosedError
 from botocore.hooks import first_non_none_response
 from urllib3.response import HTTPHeaderDict
@@ -74,6 +74,8 @@ async def create_request(self, params, operation_model=None):
 
     async def _send_request(self, request_dict, operation_model):
         attempts = 1
+        context = request_dict['context']
+        self._update_retries_context(context, attempts)
         request = await self.create_request(request_dict, operation_model)
         context = request_dict['context']
         success_response, exception = await self._get_response(
@@ -82,6 +84,9 @@ async def _send_request(self, request_dict, operation_model):
                                       request_dict, success_response,
                                       exception):
             attempts += 1
+            self._update_retries_context(
+                context, attempts, success_response
+            )
             # If there is a stream associated with the request, we need
             # to reset it before attempting to send the request again.
             # This will ensure that we resend the entire contents of the
@@ -109,7 +114,7 @@ async def _get_response(self, request, operation_model, context):
         # If an exception occurs then the success_response is None.
         # If no exception occurs then exception is None.
         success_response, exception = await self._do_get_response(
-            request, operation_model)
+            request, operation_model, context)
         kwargs_to_emit = {
             'response_dict': None,
             'parsed_response': None,
@@ -127,7 +132,7 @@ async def _get_response(self, request, operation_model, context):
                 service_id, operation_model.name), **kwargs_to_emit)
         return success_response, exception
 
-    async def _do_get_response(self, request, operation_model):
+    async def _do_get_response(self, request, operation_model, context):
         try:
             logger.debug("Sending http request: %s", request)
             history_recorder.record('HTTP_REQUEST', {
@@ -160,6 +165,9 @@ async def _do_get_response(self, request, operation_model):
         # This returns the http_response and the parsed_data.
         response_dict = await convert_to_response_dict(http_response,
                                                        operation_model)
+        handle_checksum_body(
+            http_response, response_dict, context, operation_model,
+        )
 
         http_response_record_dict = response_dict.copy()
         http_response_record_dict['streaming'] = \

aiobotocore/httpsession.py

@@ -14,9 +14,8 @@
     MAX_POOL_CONNECTIONS, InvalidProxiesConfigError, SSLError, \
     EndpointConnectionError, ProxyConnectionError, ConnectTimeoutError, \
     ConnectionClosedError, HTTPClientError, ReadTimeoutError, logger, get_cert_path, \
-    ensure_boolean, urlparse
+    ensure_boolean, urlparse, mask_proxy_url
 
-from aiobotocore import DEPRECATED_1_4_0_APIS
 from aiobotocore._endpoint_helpers import _text, _IOBaseWrapper, \
     ClientResponseProxy
 
@@ -182,43 +181,22 @@ async def send(self, request):
 
             return resp
         except ClientSSLError as e:
-            if DEPRECATED_1_4_0_APIS:
-                raise
-
             raise SSLError(endpoint_url=request.url, error=e)
         except (ClientConnectorError, socket.gaierror) as e:
-            if DEPRECATED_1_4_0_APIS:
-                raise
-
             raise EndpointConnectionError(endpoint_url=request.url, error=e)
         except (ClientProxyConnectionError, ClientHttpProxyError) as e:
-            if DEPRECATED_1_4_0_APIS:
-                raise
-
-            raise ProxyConnectionError(proxy_url=proxy_url, error=e)
+            raise ProxyConnectionError(proxy_url=mask_proxy_url(proxy_url), error=e)
         except ServerTimeoutError as e:
-            if DEPRECATED_1_4_0_APIS:
-                raise
-
             raise ConnectTimeoutError(endpoint_url=request.url, error=e)
         except asyncio.TimeoutError as e:
-            if DEPRECATED_1_4_0_APIS:
-                raise
-
             raise ReadTimeoutError(endpoint_url=request.url, error=e)
-        except ServerDisconnectedError as e:
-            if DEPRECATED_1_4_0_APIS:
-                raise
-
+        except (ServerDisconnectedError, aiohttp.ClientPayloadError) as e:
             raise ConnectionClosedError(
                 error=e,
                 request=request,
                 endpoint_url=request.url
             )
         except Exception as e:
-            if DEPRECATED_1_4_0_APIS:
-                raise
-
             message = 'Exception received when sending urllib3 HTTP request'
             logger.debug(message, exc_info=True)
             raise HTTPClientError(error=e)

aiobotocore/parsers.py

@@ -86,6 +86,7 @@ async def parse(self, response, shape):
             headers = response['headers']
             response_metadata['HTTPHeaders'] = lowercase_dict(headers)
             parsed['ResponseMetadata'] = response_metadata
+            self._add_checksum_response_metadata(response, response_metadata)
         return parsed
 
 

aiobotocore/response.py

@@ -1,5 +1,6 @@
 import asyncio
 
+import aiohttp
 import aiohttp.client_exceptions
 import wrapt
 from botocore.response import ResponseStreamingError, IncompleteReadError, \
@@ -25,7 +26,7 @@ class StreamingBody(wrapt.ObjectProxy):
 
     _DEFAULT_CHUNK_SIZE = 1024
 
-    def __init__(self, raw_stream, content_length):
+    def __init__(self, raw_stream: aiohttp.StreamReader, content_length: str):
         super().__init__(raw_stream)
         self._self_content_length = content_length
         self._self_amount_read = 0
@@ -39,9 +40,8 @@ async def __aexit__(self, exc_type, exc_val, exc_tb):
 
     # NOTE: set_socket_timeout was only for when requests didn't support
     #       read timeouts, so not needed
-
-    def tell(self):
-        return self._self_amount_read
+    def readable(self):
+        return not self.at_eof()
 
     async def read(self, amt=None):
         """Read at most amt bytes from the stream.
@@ -65,6 +65,11 @@ async def read(self, amt=None):
             self._verify_content_length()
         return chunk
 
+    async def readlines(self):
+        # assuming this is not an iterator
+        lines = [line async for line in self.iter_lines()]
+        return lines
+
     def __aiter__(self):
         """Return an iterator to yield 1k chunks from the raw stream.
         """
@@ -80,7 +85,7 @@ async def __anext__(self):
 
     anext = __anext__
 
-    async def iter_lines(self, chunk_size=1024, keepends=False):
+    async def iter_lines(self, chunk_size=_DEFAULT_CHUNK_SIZE, keepends=False):
         """Return an iterator to yield lines from the raw stream.
 
         This is achieved by reading chunk of bytes (of size chunk_size) at a
@@ -115,6 +120,9 @@ def _verify_content_length(self):
                 actual_bytes=self._self_amount_read,
                 expected_bytes=int(self._self_content_length))
 
+    def tell(self):
+        return self._self_amount_read
+
 
 async def get_response(operation_model, http_response):
     protocol = operation_model.metadata['protocol']

aiobotocore/session.py

@@ -1,4 +1,5 @@
-from botocore.session import Session, EVENT_ALIASES, ServiceModel, UnknownServiceError
+from botocore.session import Session, EVENT_ALIASES, ServiceModel, \
+    UnknownServiceError, copy
 
 from botocore import UNSIGNED
 from botocore import retryhandler, translate
@@ -16,7 +17,9 @@
     add_generate_presigned_url as boto_add_generate_presigned_url, \
     add_generate_presigned_post as boto_add_generate_presigned_post, \
     add_generate_db_auth_token as boto_add_generate_db_auth_token
+from .configprovider import AioSmartDefaultsConfigStoreFactory
 from .credentials import create_credential_resolver, AioCredentials
+from .utils import AioIMDSRegionProvider
 
 
 _HANDLER_MAPPING = {
@@ -61,6 +64,16 @@ def _register_response_parser_factory(self):
         self._components.register_component('response_parser_factory',
                                             AioResponseParserFactory())
 
+    def _register_smart_defaults_factory(self):
+        def create_smart_defaults_factory():
+            default_config_resolver = self._get_internal_component(
+                'default_config_resolver')
+            imds_region_provider = AioIMDSRegionProvider(session=self)
+            return AioSmartDefaultsConfigStoreFactory(
+                default_config_resolver, imds_region_provider)
+        self._internal_components.lazy_register_component(
+            'smart_defaults_factory', create_smart_defaults_factory)
+
     def create_client(self, *args, **kwargs):
         return ClientCreatorContext(self._create_client(*args, **kwargs))
 
@@ -114,6 +127,13 @@ async def _create_client(self, service_name, region_name=None,
         endpoint_resolver = self._get_internal_component('endpoint_resolver')
         exceptions_factory = self._get_internal_component('exceptions_factory')
         config_store = self.get_component('config_store')
+        defaults_mode = self._resolve_defaults_mode(config, config_store)
+        if defaults_mode != 'legacy':
+            smart_defaults_factory = self._get_internal_component(
+                'smart_defaults_factory')
+            config_store = copy.deepcopy(config_store)
+            await smart_defaults_factory.merge_smart_defaults(
+                config_store, defaults_mode, region_name)
         client_creator = AioClientCreator(
             loader, endpoint_resolver, self.user_agent(), event_emitter,
             retryhandler, translate, response_parser_factory,