From 9a107f283f31419e8416dce0e187b82e81dc14eb Mon Sep 17 00:00:00 2001 From: karosis88 Date: Tue, 4 Jul 2023 16:46:19 +0300 Subject: [PATCH 1/8] Add proxy_ssl_context argument --- httpcore/_async/http_proxy.py | 10 ++++++++++ httpcore/_sync/http_proxy.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/httpcore/_async/http_proxy.py b/httpcore/_async/http_proxy.py index 62f51097..7d5a7404 100644 --- a/httpcore/_async/http_proxy.py +++ b/httpcore/_async/http_proxy.py @@ -64,6 +64,7 @@ def __init__( proxy_auth: Optional[Tuple[Union[bytes, str], Union[bytes, str]]] = None, proxy_headers: Union[HeadersAsMapping, HeadersAsSequence, None] = None, ssl_context: Optional[ssl.SSLContext] = None, + proxy_ssl_context: Optional[ssl.SSLContext] = None, max_connections: Optional[int] = 10, max_keepalive_connections: Optional[int] = None, keepalive_expiry: Optional[float] = None, @@ -88,6 +89,7 @@ def __init__( ssl_context: An SSL context to use for verifying connections. If not specified, the default `httpcore.default_ssl_context()` will be used. + proxy_ssl_context: The same as `ssl_context`, but for a proxy server rather than a remote origin. max_connections: The maximum number of concurrent HTTP connections that the pool should allow. Any attempt to send a request on a pool that would exceed this amount will block until a connection is available. @@ -123,6 +125,7 @@ def __init__( socket_options=socket_options, ) self._ssl_context = ssl_context + self._proxy_ssl_context = proxy_ssl_context self._proxy_url = enforce_url(proxy_url, name="proxy_url") self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") if proxy_auth is not None: @@ -141,12 +144,14 @@ def create_connection(self, origin: Origin) -> AsyncConnectionInterface: remote_origin=origin, keepalive_expiry=self._keepalive_expiry, network_backend=self._network_backend, + proxy_ssl_context=self._proxy_ssl_context, ) return AsyncTunnelHTTPConnection( proxy_origin=self._proxy_url.origin, proxy_headers=self._proxy_headers, remote_origin=origin, ssl_context=self._ssl_context, + proxy_ssl_context=self._proxy_ssl_context, keepalive_expiry=self._keepalive_expiry, http1=self._http1, http2=self._http2, @@ -163,12 +168,14 @@ def __init__( keepalive_expiry: Optional[float] = None, network_backend: Optional[AsyncNetworkBackend] = None, socket_options: Optional[Iterable[SOCKET_OPTION]] = None, + proxy_ssl_context: Optional[ssl.SSLContext] = None, ) -> None: self._connection = AsyncHTTPConnection( origin=proxy_origin, keepalive_expiry=keepalive_expiry, network_backend=network_backend, socket_options=socket_options, + ssl_context=proxy_ssl_context, ) self._proxy_origin = proxy_origin self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") @@ -222,6 +229,7 @@ def __init__( proxy_origin: Origin, remote_origin: Origin, ssl_context: Optional[ssl.SSLContext] = None, + proxy_ssl_context: Optional[ssl.SSLContext] = None, proxy_headers: Optional[Sequence[Tuple[bytes, bytes]]] = None, keepalive_expiry: Optional[float] = None, http1: bool = True, @@ -234,10 +242,12 @@ def __init__( keepalive_expiry=keepalive_expiry, network_backend=network_backend, socket_options=socket_options, + ssl_context=proxy_ssl_context, ) self._proxy_origin = proxy_origin self._remote_origin = remote_origin self._ssl_context = ssl_context + self._proxy_ssl_context = proxy_ssl_context self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") self._keepalive_expiry = keepalive_expiry self._http1 = http1 diff --git a/httpcore/_sync/http_proxy.py b/httpcore/_sync/http_proxy.py index bb368dd4..96fe8d3b 100644 --- a/httpcore/_sync/http_proxy.py +++ b/httpcore/_sync/http_proxy.py @@ -64,6 +64,7 @@ def __init__( proxy_auth: Optional[Tuple[Union[bytes, str], Union[bytes, str]]] = None, proxy_headers: Union[HeadersAsMapping, HeadersAsSequence, None] = None, ssl_context: Optional[ssl.SSLContext] = None, + proxy_ssl_context: Optional[ssl.SSLContext] = None, max_connections: Optional[int] = 10, max_keepalive_connections: Optional[int] = None, keepalive_expiry: Optional[float] = None, @@ -88,6 +89,7 @@ def __init__( ssl_context: An SSL context to use for verifying connections. If not specified, the default `httpcore.default_ssl_context()` will be used. + proxy_ssl_context: The same as `ssl_context`, but for a proxy server rather than a remote origin. max_connections: The maximum number of concurrent HTTP connections that the pool should allow. Any attempt to send a request on a pool that would exceed this amount will block until a connection is available. @@ -123,6 +125,7 @@ def __init__( socket_options=socket_options, ) self._ssl_context = ssl_context + self._proxy_ssl_context = proxy_ssl_context self._proxy_url = enforce_url(proxy_url, name="proxy_url") self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") if proxy_auth is not None: @@ -141,12 +144,14 @@ def create_connection(self, origin: Origin) -> ConnectionInterface: remote_origin=origin, keepalive_expiry=self._keepalive_expiry, network_backend=self._network_backend, + proxy_ssl_context=self._proxy_ssl_context, ) return TunnelHTTPConnection( proxy_origin=self._proxy_url.origin, proxy_headers=self._proxy_headers, remote_origin=origin, ssl_context=self._ssl_context, + proxy_ssl_context=self._proxy_ssl_context, keepalive_expiry=self._keepalive_expiry, http1=self._http1, http2=self._http2, @@ -163,12 +168,14 @@ def __init__( keepalive_expiry: Optional[float] = None, network_backend: Optional[NetworkBackend] = None, socket_options: Optional[Iterable[SOCKET_OPTION]] = None, + proxy_ssl_context: Optional[ssl.SSLContext] = None, ) -> None: self._connection = HTTPConnection( origin=proxy_origin, keepalive_expiry=keepalive_expiry, network_backend=network_backend, socket_options=socket_options, + ssl_context=proxy_ssl_context, ) self._proxy_origin = proxy_origin self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") @@ -222,6 +229,7 @@ def __init__( proxy_origin: Origin, remote_origin: Origin, ssl_context: Optional[ssl.SSLContext] = None, + proxy_ssl_context: Optional[ssl.SSLContext] = None, proxy_headers: Optional[Sequence[Tuple[bytes, bytes]]] = None, keepalive_expiry: Optional[float] = None, http1: bool = True, @@ -234,10 +242,12 @@ def __init__( keepalive_expiry=keepalive_expiry, network_backend=network_backend, socket_options=socket_options, + ssl_context=proxy_ssl_context, ) self._proxy_origin = proxy_origin self._remote_origin = remote_origin self._ssl_context = ssl_context + self._proxy_ssl_context = proxy_ssl_context self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") self._keepalive_expiry = keepalive_expiry self._http1 = http1 From 7333a5844df98c24fad531fa90fd5bf405c502f0 Mon Sep 17 00:00:00 2001 From: karosis88 Date: Fri, 7 Jul 2023 08:15:18 +0300 Subject: [PATCH 2/8] Add changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 366edf8e..2410f386 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## development + +- Add support for HTTPS proxies. (#745) + ## 0.17.3 (5th July 2023) - Support async cancellations, ensuring that the connection pool is left in a clean state when cancellations occur. (#726) From bee1738f131b93d01a1e0dbb7ae6b812f7eb32bb Mon Sep 17 00:00:00 2001 From: karosis88 Date: Fri, 1 Sep 2023 12:44:44 +0300 Subject: [PATCH 3/8] Document HTTPS proxies --- docs/proxies.md | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/proxies.md b/docs/proxies.md index 6c2dc875..3640b216 100644 --- a/docs/proxies.md +++ b/docs/proxies.md @@ -51,10 +51,33 @@ proxy = httpcore.HTTPProxy( ) ``` -## Proxy SSL and HTTP Versions +## Proxy SSL -Proxy support currently only allows for HTTP/1.1 connections to the proxy, -and does not currently support SSL proxy connections, which require HTTPS-in-HTTPS, +The `httpcore` package also supports HTTPS proxies for http and https destinations. + +HTTPS proxies can be used in the same way that HTTP proxies are. + +```python +proxy = httpcore.HTTPProxy(proxy_url="https://127.0.0.1:8080/") +``` + +Also, when using HTTPS proxies, you may need to configure the SSL context, which you can do with the `proxy_ssl_context` argument. + +```python +import ssl +import httpcore + +proxy_ssl_context = ssl.create_default_context() +proxy_ssl_context.check_hostname = False + +proxy = httpcore.HTTPProxy('http://127.0.0.1:8080/', proxy_ssl_context=proxy_ssl_context) +``` + +It is important to note that the `ssl_context` argument is always used for the remote connection, and the `proxy_ssl_context` argument is always used for the proxy connection. + +## HTTP Versions + +If you use proxies, keep in mind that the `httpcore` package only supports proxies to HTTP/1.1 servers. ## SOCKS proxy support From 68d0653b5297b7f866aee2394ab664f056fa66e4 Mon Sep 17 00:00:00 2001 From: karosis88 Date: Fri, 1 Sep 2023 14:08:48 +0300 Subject: [PATCH 4/8] Raise exception when proxy_ssl_context used with the http scheme --- httpcore/_async/http_proxy.py | 10 +++++++++- httpcore/_sync/http_proxy.py | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/httpcore/_async/http_proxy.py b/httpcore/_async/http_proxy.py index 7d5a7404..4aa7d874 100644 --- a/httpcore/_async/http_proxy.py +++ b/httpcore/_async/http_proxy.py @@ -124,9 +124,17 @@ def __init__( uds=uds, socket_options=socket_options, ) + + self._proxy_url = enforce_url(proxy_url, name="proxy_url") + if ( + self._proxy_url.scheme == b"http" and proxy_ssl_context is not None + ): # pragma: no cover + raise RuntimeError( + "The `proxy_ssl_context` argument is not allowed for the http scheme" + ) + self._ssl_context = ssl_context self._proxy_ssl_context = proxy_ssl_context - self._proxy_url = enforce_url(proxy_url, name="proxy_url") self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") if proxy_auth is not None: username = enforce_bytes(proxy_auth[0], name="proxy_auth") diff --git a/httpcore/_sync/http_proxy.py b/httpcore/_sync/http_proxy.py index 96fe8d3b..6acac9a7 100644 --- a/httpcore/_sync/http_proxy.py +++ b/httpcore/_sync/http_proxy.py @@ -124,9 +124,17 @@ def __init__( uds=uds, socket_options=socket_options, ) + + self._proxy_url = enforce_url(proxy_url, name="proxy_url") + if ( + self._proxy_url.scheme == b"http" and proxy_ssl_context is not None + ): # pragma: no cover + raise RuntimeError( + "The `proxy_ssl_context` argument is not allowed for the http scheme" + ) + self._ssl_context = ssl_context self._proxy_ssl_context = proxy_ssl_context - self._proxy_url = enforce_url(proxy_url, name="proxy_url") self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") if proxy_auth is not None: username = enforce_bytes(proxy_auth[0], name="proxy_auth") From 207a2298ca37f164ad584e478baf8f804a2e0323 Mon Sep 17 00:00:00 2001 From: Kar Petrosyan <92274156+karosis88@users.noreply.github.com> Date: Fri, 1 Sep 2023 14:13:58 +0300 Subject: [PATCH 5/8] Update docs/proxies.md Co-authored-by: Tom Christie --- docs/proxies.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proxies.md b/docs/proxies.md index 3640b216..75d9844c 100644 --- a/docs/proxies.md +++ b/docs/proxies.md @@ -70,7 +70,7 @@ import httpcore proxy_ssl_context = ssl.create_default_context() proxy_ssl_context.check_hostname = False -proxy = httpcore.HTTPProxy('http://127.0.0.1:8080/', proxy_ssl_context=proxy_ssl_context) +proxy = httpcore.HTTPProxy('https://127.0.0.1:8080/', proxy_ssl_context=proxy_ssl_context) ``` It is important to note that the `ssl_context` argument is always used for the remote connection, and the `proxy_ssl_context` argument is always used for the proxy connection. From 9c9fe71b71ff0e9695cd5f97dac2bc1360e21ed8 Mon Sep 17 00:00:00 2001 From: karosis88 Date: Fri, 1 Sep 2023 14:17:43 +0300 Subject: [PATCH 6/8] Raise exception when TLS over TLS is used for sync stream --- httpcore/_backends/sync.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/httpcore/_backends/sync.py b/httpcore/_backends/sync.py index a4c85f04..730795f6 100644 --- a/httpcore/_backends/sync.py +++ b/httpcore/_backends/sync.py @@ -47,6 +47,12 @@ def start_tls( server_hostname: typing.Optional[str] = None, timeout: typing.Optional[float] = None, ) -> NetworkStream: + if isinstance(self._sock, ssl.SSLSocket): # pragma: no cover + raise RuntimeError( + "Attempted to add a TLS layer on top of the existing " + "TLS stream, which is not supported by httpcore package" + ) + exc_map: ExceptionMapping = { socket.timeout: ConnectTimeout, OSError: ConnectError, From 5ee48a1ef5e5bb11ba242116b4de6b0dd3a5d6d2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 1 Sep 2023 12:22:17 +0100 Subject: [PATCH 7/8] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e97bc2f..a4569a77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## development -- Add support for HTTPS proxies. (#745) +- Add support for HTTPS proxies. Currently only available for async. (#745) - Change the type of `Extensions` from `Mapping[Str, Any]` to `MutableMapping[Str, Any]`. (#762) - Handle HTTP/1.1 half-closed connections gracefully. (#641) - Drop Python 3.7 support. (#727) From 1adde9e73b2b0b2f876f09d935785318f82f0a16 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 1 Sep 2023 12:23:04 +0100 Subject: [PATCH 8/8] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4569a77..9e5e15cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## development +## Unreleased - Add support for HTTPS proxies. Currently only available for async. (#745) - Change the type of `Extensions` from `Mapping[Str, Any]` to `MutableMapping[Str, Any]`. (#762)