diff --git a/docs/proxies.md b/docs/proxies.md index 72eaeb64..e22978b1 100644 --- a/docs/proxies.md +++ b/docs/proxies.md @@ -51,6 +51,30 @@ proxy = httpcore.HTTPProxy( ) ``` +## Proxy Connection Mode + +There are two types of HTTP proxy: + +1. Forwarding (HTTP [absolute-url](https://tools.ietf.org/html/rfc7230#section-5.3.2)) +2. Tunneling (HTTP [`CONNECT` method](https://tools.ietf.org/html/rfc7231#section-4.3.6)) + +By default `httpcore` will use forwarding for http requests and tunneling for https requests. + +You can change this behavior with `proxy_mode: httpcore.ProxyMode` parameter: + +```py +import httpcore +import base64 + +proxy = httpcore.HTTPProxy( + proxy_url="http://127.0.0.1:8080/", + proxy_mode=httpcore.ProxyMode.TUNNEL, +) +``` + +Note that `ProxyMode.FORWARD` will enable forwarding https requests, so they will be visible for proxy server. +It means handling TLS stuffs like certificate validation would on proxy side. + ## Proxy SSL The `httpcore` package also supports HTTPS proxies for http and https destinations. diff --git a/httpcore/__init__.py b/httpcore/__init__.py index 014213ba..2c39ad6a 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -34,7 +34,7 @@ WriteError, WriteTimeout, ) -from ._models import URL, Origin, Request, Response +from ._models import URL, Origin, ProxyMode, Request, Response from ._ssl import default_ssl_context from ._sync import ( ConnectionInterface, @@ -75,8 +75,9 @@ def __init__(self, *args, **kwargs): # type: ignore "request", "stream", # models - "Origin", "URL", + "Origin", + "ProxyMode", "Request", "Response", # async diff --git a/httpcore/_async/http_proxy.py b/httpcore/_async/http_proxy.py index 4aa7d874..1f0110ee 100644 --- a/httpcore/_async/http_proxy.py +++ b/httpcore/_async/http_proxy.py @@ -8,6 +8,7 @@ from .._models import ( URL, Origin, + ProxyMode, Request, Response, enforce_bytes, @@ -25,7 +26,6 @@ HeadersAsSequence = Sequence[Tuple[Union[bytes, str], Union[bytes, str]]] HeadersAsMapping = Mapping[Union[bytes, str], Union[bytes, str]] - logger = logging.getLogger("httpcore.proxy") @@ -75,6 +75,7 @@ def __init__( uds: Optional[str] = None, network_backend: Optional[AsyncNetworkBackend] = None, socket_options: Optional[Iterable[SOCKET_OPTION]] = None, + proxy_mode: Optional[ProxyMode] = None, ) -> None: """ A connection pool for making HTTP requests. @@ -110,6 +111,7 @@ def __init__( `AF_INET6` address (IPv6). uds: Path to a Unix Domain Socket to use instead of TCP sockets. network_backend: A backend instance to use for handling network I/O. + proxy_mode: Allow HTTP connection be tunnelable and HTTPS be forwardable. """ super().__init__( ssl_context=ssl_context, @@ -136,6 +138,7 @@ def __init__( self._ssl_context = ssl_context self._proxy_ssl_context = proxy_ssl_context self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") + self._proxy_mode = proxy_mode if proxy_auth is not None: username = enforce_bytes(proxy_auth[0], name="proxy_auth") password = enforce_bytes(proxy_auth[1], name="proxy_auth") @@ -145,7 +148,9 @@ def __init__( ] + self._proxy_headers def create_connection(self, origin: Origin) -> AsyncConnectionInterface: - if origin.scheme == b"http": + if (self._proxy_mode is ProxyMode.FORWARD) or ( + self._proxy_mode is None and origin.scheme == b"http" + ): return AsyncForwardHTTPConnection( proxy_origin=self._proxy_url.origin, proxy_headers=self._proxy_headers, @@ -298,31 +303,33 @@ async def handle_async_request(self, request: Request) -> Response: raise ProxyError(msg) stream = connect_response.extensions["network_stream"] - - # Upgrade the stream to SSL - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": self._remote_origin.host.decode("ascii"), - "timeout": timeout, - } - async with Trace("start_tls", logger, request, kwargs) as trace: - stream = await stream.start_tls(**kwargs) - trace.return_value = stream - - # Determine if we should be using HTTP/1.1 or HTTP/2 - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) + http2_negotiated = False + + if self._remote_origin.scheme == b"https": + # Upgrade the stream to SSL + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + async with Trace("start_tls", logger, request, kwargs) as trace: + stream = await stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) # Create the HTTP/1.1 or HTTP/2 connection if http2_negotiated or (self._http2 and not self._http1): diff --git a/httpcore/_models.py b/httpcore/_models.py index dadee79f..b2e1941a 100644 --- a/httpcore/_models.py +++ b/httpcore/_models.py @@ -1,3 +1,4 @@ +import enum from typing import ( Any, AsyncIterable, @@ -490,3 +491,8 @@ async def aclose(self) -> None: ) if hasattr(self.stream, "aclose"): await self.stream.aclose() + + +class ProxyMode(enum.IntEnum): + FORWARD = 1 + TUNNEL = 2 diff --git a/httpcore/_sync/http_proxy.py b/httpcore/_sync/http_proxy.py index 6acac9a7..020dd963 100644 --- a/httpcore/_sync/http_proxy.py +++ b/httpcore/_sync/http_proxy.py @@ -8,6 +8,7 @@ from .._models import ( URL, Origin, + ProxyMode, Request, Response, enforce_bytes, @@ -25,7 +26,6 @@ HeadersAsSequence = Sequence[Tuple[Union[bytes, str], Union[bytes, str]]] HeadersAsMapping = Mapping[Union[bytes, str], Union[bytes, str]] - logger = logging.getLogger("httpcore.proxy") @@ -75,6 +75,7 @@ def __init__( uds: Optional[str] = None, network_backend: Optional[NetworkBackend] = None, socket_options: Optional[Iterable[SOCKET_OPTION]] = None, + proxy_mode: Optional[ProxyMode] = None, ) -> None: """ A connection pool for making HTTP requests. @@ -110,6 +111,7 @@ def __init__( `AF_INET6` address (IPv6). uds: Path to a Unix Domain Socket to use instead of TCP sockets. network_backend: A backend instance to use for handling network I/O. + proxy_mode: Allow HTTP connection be tunnelable and HTTPS be forwardable. """ super().__init__( ssl_context=ssl_context, @@ -136,6 +138,7 @@ def __init__( self._ssl_context = ssl_context self._proxy_ssl_context = proxy_ssl_context self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") + self._proxy_mode = proxy_mode if proxy_auth is not None: username = enforce_bytes(proxy_auth[0], name="proxy_auth") password = enforce_bytes(proxy_auth[1], name="proxy_auth") @@ -145,7 +148,9 @@ def __init__( ] + self._proxy_headers def create_connection(self, origin: Origin) -> ConnectionInterface: - if origin.scheme == b"http": + if (self._proxy_mode is ProxyMode.FORWARD) or ( + self._proxy_mode is None and origin.scheme == b"http" + ): return ForwardHTTPConnection( proxy_origin=self._proxy_url.origin, proxy_headers=self._proxy_headers, @@ -298,31 +303,33 @@ def handle_request(self, request: Request) -> Response: raise ProxyError(msg) stream = connect_response.extensions["network_stream"] - - # Upgrade the stream to SSL - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": self._remote_origin.host.decode("ascii"), - "timeout": timeout, - } - with Trace("start_tls", logger, request, kwargs) as trace: - stream = stream.start_tls(**kwargs) - trace.return_value = stream - - # Determine if we should be using HTTP/1.1 or HTTP/2 - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) + http2_negotiated = False + + if self._remote_origin.scheme == b"https": + # Upgrade the stream to SSL + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + with Trace("start_tls", logger, request, kwargs) as trace: + stream = stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) # Create the HTTP/1.1 or HTTP/2 connection if http2_negotiated or (self._http2 and not self._http1):