Add support for HTTPS proxies (available to trio/asyncio)

CHANGELOG.md

@@ -4,8 +4,9 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
-## Unreleased
+## development
 
+- Add support for HTTPS proxies. Currently only available for async. (#745)
 - Change the type of `Extensions` from `Mapping[Str, Any]` to `MutableMapping[Str, Any]`. (#762)
 - Handle HTTP/1.1 half-closed connections gracefully. (#641)
 - Drop Python 3.7 support. (#727)

docs/proxies.md

@@ -51,10 +51,33 @@ proxy = httpcore.HTTPProxy(
 )
 ```
 
-## Proxy SSL and HTTP Versions
+## Proxy SSL
 
-Proxy support currently only allows for HTTP/1.1 connections to the proxy,
-and does not currently support SSL proxy connections, which require HTTPS-in-HTTPS,
+The `httpcore` package also supports HTTPS proxies for http and https destinations.
+
+HTTPS proxies can be used in the same way that HTTP proxies are.
+
+```python
+proxy = httpcore.HTTPProxy(proxy_url="https://127.0.0.1:8080/")
+```
+
+Also, when using HTTPS proxies, you may need to configure the SSL context, which you can do with the `proxy_ssl_context` argument.
+
+```python
+import ssl
+import httpcore
+
+proxy_ssl_context = ssl.create_default_context()
+proxy_ssl_context.check_hostname = False
+
+proxy = httpcore.HTTPProxy('https://127.0.0.1:8080/', proxy_ssl_context=proxy_ssl_context)
+```
+
+It is important to note that the `ssl_context` argument is always used for the remote connection, and the `proxy_ssl_context` argument is always used for the proxy connection.
+
+## HTTP Versions
+
+If you use proxies, keep in mind that the `httpcore` package only supports proxies to HTTP/1.1 servers.
 
 ## SOCKS proxy support
 

httpcore/_async/http_proxy.py

@@ -64,6 +64,7 @@ def __init__(
         proxy_auth: Optional[Tuple[Union[bytes, str], Union[bytes, str]]] = None,
         proxy_headers: Union[HeadersAsMapping, HeadersAsSequence, None] = None,
         ssl_context: Optional[ssl.SSLContext] = None,
+        proxy_ssl_context: Optional[ssl.SSLContext] = None,
         max_connections: Optional[int] = 10,
         max_keepalive_connections: Optional[int] = None,
         keepalive_expiry: Optional[float] = None,
@@ -88,6 +89,7 @@ def __init__(
             ssl_context: An SSL context to use for verifying connections.
                 If not specified, the default `httpcore.default_ssl_context()`
                 will be used.
+            proxy_ssl_context: The same as `ssl_context`, but for a proxy server rather than a remote origin.
             max_connections: The maximum number of concurrent HTTP connections that
                 the pool should allow. Any attempt to send a request on a pool that
                 would exceed this amount will block until a connection is available.
@@ -122,8 +124,17 @@ def __init__(
             uds=uds,
             socket_options=socket_options,
         )
-        self._ssl_context = ssl_context
+
         self._proxy_url = enforce_url(proxy_url, name="proxy_url")
+        if (
+            self._proxy_url.scheme == b"http" and proxy_ssl_context is not None
+        ):  # pragma: no cover
+            raise RuntimeError(
+                "The `proxy_ssl_context` argument is not allowed for the http scheme"
+            )
+
+        self._ssl_context = ssl_context
+        self._proxy_ssl_context = proxy_ssl_context
         self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers")
         if proxy_auth is not None:
             username = enforce_bytes(proxy_auth[0], name="proxy_auth")
@@ -141,12 +152,14 @@ def create_connection(self, origin: Origin) -> AsyncConnectionInterface:
                 remote_origin=origin,
                 keepalive_expiry=self._keepalive_expiry,
                 network_backend=self._network_backend,
+                proxy_ssl_context=self._proxy_ssl_context,
             )
         return AsyncTunnelHTTPConnection(
             proxy_origin=self._proxy_url.origin,
             proxy_headers=self._proxy_headers,
             remote_origin=origin,
             ssl_context=self._ssl_context,
+            proxy_ssl_context=self._proxy_ssl_context,
             keepalive_expiry=self._keepalive_expiry,
             http1=self._http1,
             http2=self._http2,
@@ -163,12 +176,14 @@ def __init__(
         keepalive_expiry: Optional[float] = None,
         network_backend: Optional[AsyncNetworkBackend] = None,
         socket_options: Optional[Iterable[SOCKET_OPTION]] = None,
+        proxy_ssl_context: Optional[ssl.SSLContext] = None,
     ) -> None:
         self._connection = AsyncHTTPConnection(
             origin=proxy_origin,
             keepalive_expiry=keepalive_expiry,
             network_backend=network_backend,
             socket_options=socket_options,
+            ssl_context=proxy_ssl_context,
         )
         self._proxy_origin = proxy_origin
         self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers")
@@ -222,6 +237,7 @@ def __init__(
         proxy_origin: Origin,
         remote_origin: Origin,
         ssl_context: Optional[ssl.SSLContext] = None,
+        proxy_ssl_context: Optional[ssl.SSLContext] = None,
         proxy_headers: Optional[Sequence[Tuple[bytes, bytes]]] = None,
         keepalive_expiry: Optional[float] = None,
         http1: bool = True,
@@ -234,10 +250,12 @@ def __init__(
             keepalive_expiry=keepalive_expiry,
             network_backend=network_backend,
             socket_options=socket_options,
+            ssl_context=proxy_ssl_context,
         )
         self._proxy_origin = proxy_origin
         self._remote_origin = remote_origin
         self._ssl_context = ssl_context
+        self._proxy_ssl_context = proxy_ssl_context
         self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers")
         self._keepalive_expiry = keepalive_expiry
         self._http1 = http1

httpcore/_backends/sync.py

@@ -47,6 +47,12 @@ def start_tls(
         server_hostname: typing.Optional[str] = None,
         timeout: typing.Optional[float] = None,
     ) -> NetworkStream:
+        if isinstance(self._sock, ssl.SSLSocket):  # pragma: no cover
+            raise RuntimeError(
+                "Attempted to add a TLS layer on top of the existing "
+                "TLS stream, which is not supported by httpcore package"
+            )
+
         exc_map: ExceptionMapping = {
             socket.timeout: ConnectTimeout,
             OSError: ConnectError,

httpcore/_sync/http_proxy.py

@@ -64,6 +64,7 @@ def __init__(
         proxy_auth: Optional[Tuple[Union[bytes, str], Union[bytes, str]]] = None,
         proxy_headers: Union[HeadersAsMapping, HeadersAsSequence, None] = None,
         ssl_context: Optional[ssl.SSLContext] = None,
+        proxy_ssl_context: Optional[ssl.SSLContext] = None,
         max_connections: Optional[int] = 10,
         max_keepalive_connections: Optional[int] = None,
         keepalive_expiry: Optional[float] = None,
@@ -88,6 +89,7 @@ def __init__(
             ssl_context: An SSL context to use for verifying connections.
                 If not specified, the default `httpcore.default_ssl_context()`
                 will be used.
+            proxy_ssl_context: The same as `ssl_context`, but for a proxy server rather than a remote origin.
             max_connections: The maximum number of concurrent HTTP connections that
                 the pool should allow. Any attempt to send a request on a pool that
                 would exceed this amount will block until a connection is available.
@@ -122,8 +124,17 @@ def __init__(
             uds=uds,
             socket_options=socket_options,
         )
-        self._ssl_context = ssl_context
+
         self._proxy_url = enforce_url(proxy_url, name="proxy_url")
+        if (
+            self._proxy_url.scheme == b"http" and proxy_ssl_context is not None
+        ):  # pragma: no cover
+            raise RuntimeError(
+                "The `proxy_ssl_context` argument is not allowed for the http scheme"
+            )
+
+        self._ssl_context = ssl_context
+        self._proxy_ssl_context = proxy_ssl_context
         self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers")
         if proxy_auth is not None:
             username = enforce_bytes(proxy_auth[0], name="proxy_auth")
@@ -141,12 +152,14 @@ def create_connection(self, origin: Origin) -> ConnectionInterface:
                 remote_origin=origin,
                 keepalive_expiry=self._keepalive_expiry,
                 network_backend=self._network_backend,
+                proxy_ssl_context=self._proxy_ssl_context,
             )
         return TunnelHTTPConnection(
             proxy_origin=self._proxy_url.origin,
             proxy_headers=self._proxy_headers,
             remote_origin=origin,
             ssl_context=self._ssl_context,
+            proxy_ssl_context=self._proxy_ssl_context,
             keepalive_expiry=self._keepalive_expiry,
             http1=self._http1,
             http2=self._http2,
@@ -163,12 +176,14 @@ def __init__(
         keepalive_expiry: Optional[float] = None,
         network_backend: Optional[NetworkBackend] = None,
         socket_options: Optional[Iterable[SOCKET_OPTION]] = None,
+        proxy_ssl_context: Optional[ssl.SSLContext] = None,
     ) -> None:
         self._connection = HTTPConnection(
             origin=proxy_origin,
             keepalive_expiry=keepalive_expiry,
             network_backend=network_backend,
             socket_options=socket_options,
+            ssl_context=proxy_ssl_context,
         )
         self._proxy_origin = proxy_origin
         self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers")
@@ -222,6 +237,7 @@ def __init__(
         proxy_origin: Origin,
         remote_origin: Origin,
         ssl_context: Optional[ssl.SSLContext] = None,
+        proxy_ssl_context: Optional[ssl.SSLContext] = None,
         proxy_headers: Optional[Sequence[Tuple[bytes, bytes]]] = None,
         keepalive_expiry: Optional[float] = None,
         http1: bool = True,
@@ -234,10 +250,12 @@ def __init__(
             keepalive_expiry=keepalive_expiry,
             network_backend=network_backend,
             socket_options=socket_options,
+            ssl_context=proxy_ssl_context,
         )
         self._proxy_origin = proxy_origin
         self._remote_origin = remote_origin
         self._ssl_context = ssl_context
+        self._proxy_ssl_context = proxy_ssl_context
         self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers")
         self._keepalive_expiry = keepalive_expiry
         self._http1 = http1