From 8b9fc531ef4923a31a335c78b921df4c7589af5d Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Fri, 15 Mar 2024 14:59:06 +0100 Subject: [PATCH 1/5] Add `Req.TransportError`, support returning exceptions in `put_plug` --- lib/req/checksum_mismatch_error.ex | 5 +++ lib/req/steps.ex | 66 +++++++++++++++++++++++++++-- lib/req/too_many_redirects_error.ex | 5 +++ lib/req/transport_error.ex | 17 ++++++++ test/req/steps_test.exs | 33 +++++++++++++-- 5 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 lib/req/transport_error.ex diff --git a/lib/req/checksum_mismatch_error.ex b/lib/req/checksum_mismatch_error.ex index 178950c7..984c1ec8 100644 --- a/lib/req/checksum_mismatch_error.ex +++ b/lib/req/checksum_mismatch_error.ex @@ -1,6 +1,11 @@ defmodule Req.ChecksumMismatchError do + @moduledoc """ + Represents a checksum mismatch error returned by `Req.Steps.checksum/1`. + """ + defexception [:expected, :actual] + @impl true def message(%{expected: expected, actual: actual}) do """ checksum mismatch diff --git a/lib/req/steps.ex b/lib/req/steps.ex index e0c890c5..51cbf16c 100644 --- a/lib/req/steps.ex +++ b/lib/req/steps.ex @@ -666,6 +666,9 @@ defmodule Req.Steps do ["Adapter" section in the `Req.Request`](Req.Request.html#module-adapter) module documentation for more information on adapters. + Finch returns `Mint.TransportError` exceptions on HTTP connection problems. These are automatically + converted to `Req.TransportError` exceptions. + ## HTTP/1 Pools On HTTP/1 connections, Finch creates a pool per `{scheme, host, port}` tuple. These pools @@ -757,6 +760,11 @@ defmodule Req.Steps do iex> Req.get!(url, connect_options: [transport_opts: [cacerts: :public_key.cacerts_get()]]) + Transport errors are represented as `Req.TransportError` exceptions: + + iex> Req.get("https://httpbin.org/delay/1", receive_timeout: 0, retry: false) + {:error, %Req.TransportError{reason: :timeout}} + Stream response body using `Finch.stream/5`: fun = fn request, finch_request, finch_name, finch_options -> @@ -876,6 +884,9 @@ defmodule Req.Steps do {:ok, acc} -> acc + {:error, %Mint.TransportError{reason: reason}} -> + {req, %Req.TransportError{reason: reason}} + {:error, exception} -> {req, exception} end @@ -909,6 +920,9 @@ defmodule Req.Steps do acc = collector.(acc, :done) {req, %{resp | body: acc}} + {:error, %Mint.TransportError{reason: reason}} -> + {req, %Req.TransportError{reason: reason}} + {:error, exception} -> {req, exception} end @@ -977,8 +991,14 @@ defmodule Req.Steps do defp run_finch_request(finch_request, finch_name, finch_options) do case Finch.request(finch_request, finch_name, finch_options) do - {:ok, response} -> Req.Response.new(response) - {:error, exception} -> exception + {:ok, response} -> + Req.Response.new(response) + + {:error, %Mint.TransportError{reason: reason}} -> + %Req.TransportError{reason: reason} + + {:error, exception} -> + exception end end @@ -1156,6 +1176,18 @@ defmodule Req.Steps do assert Req.get!(plug: plug, into: []).body == ["echoecho"] end + + You can simulate failure conditions by returning exception structs from your plugs. + For network related issues, use `Req.TransportError` exception: + + test "timeout" do + plug = fn _conn -> + %Req.TransportError{reason: :econnrefused} + end + + assert Req.get(plug: plug, retry: false) == + {:error, %Req.TransportError{reason: :econnrefused}} + end """ @doc step: :request def put_plug(request) do @@ -1195,8 +1227,36 @@ defmodule Req.Steps do conn = Plug.Test.conn(request.method, request.url, req_body) conn = put_in(conn.req_headers, req_headers) - conn = call_plug(conn, plug) + case call_plug(conn, plug) do + %Plug.Conn{} = conn -> + handle_plug_result(conn, request) + + %Req.TransportError{} = exception -> + validate_transport_error!(exception.reason) + {request, exception} + + %_{__exception__: true} = exception -> + {request, exception} + end + end + + defp validate_transport_error!(:protocol_not_negotiated), do: :ok + defp validate_transport_error!({:bad_alpn_protocol, _}), do: :ok + defp validate_transport_error!(:closed), do: :ok + defp validate_transport_error!(:timeout), do: :ok + + defp validate_transport_error!(reason) do + case :ssl.format_error(reason) do + ~c"Unexpected error:" ++ _ -> + raise ArgumentError, "unexpected Req.Transport reason: #{inspect(reason)}" + + _ -> + :ok + end + end + + defp handle_plug_result(conn, request) do # consume messages sent by Plug.Test adapter {_, %{ref: ref}} = conn.adapter diff --git a/lib/req/too_many_redirects_error.ex b/lib/req/too_many_redirects_error.ex index 1a972ec9..e92f218b 100644 --- a/lib/req/too_many_redirects_error.ex +++ b/lib/req/too_many_redirects_error.ex @@ -1,6 +1,11 @@ defmodule Req.TooManyRedirectsError do + @moduledoc """ + Represents an error when too many redirects occured, returned by `Req.Steps.redirect/1`. + """ + defexception [:max_redirects] + @impl true def message(%{max_redirects: max_redirects}) do "too many redirects (#{max_redirects})" end diff --git a/lib/req/transport_error.ex b/lib/req/transport_error.ex new file mode 100644 index 00000000..24e4d782 --- /dev/null +++ b/lib/req/transport_error.ex @@ -0,0 +1,17 @@ +defmodule Req.TransportError do + @moduledoc """ + Represents an error with the transport used by an HTTP connection. + + This is a standardised exception that all Req adapters should use for transport layer related + errors. + + This exception is based on `Mint.TransportError`. + """ + + defexception [:reason] + + @impl true + def message(%__MODULE__{reason: reason}) do + Mint.TransportError.message(%Mint.TransportError{reason: reason}) + end +end diff --git a/test/req/steps_test.exs b/test/req/steps_test.exs index 06451653..1cab65f0 100644 --- a/test/req/steps_test.exs +++ b/test/req/steps_test.exs @@ -1777,6 +1777,33 @@ defmodule Req.StepsTest do assert ["defmodule Req.MixProject do" <> _] = resp.body refute_receive _ end + + test "errors" do + req = + Req.new( + plug: fn _conn -> + %Req.TransportError{reason: :timeout} + end, + retry: false + ) + + assert Req.request(req) == + {:error, %Req.TransportError{reason: :timeout}} + end + + test "validate Req.TransportError reason" do + req = + Req.new( + plug: fn _conn -> + %Req.TransportError{reason: :bad} + end, + retry: false + ) + + assert_raise ArgumentError, "unexpected Req.Transport reason: :bad", fn -> + Req.request(req) + end + end end describe "run_finch" do @@ -1849,7 +1876,7 @@ defmodule Req.StepsTest do end) req = Req.new(url: url, receive_timeout: 50, retry: false) - assert {:error, %Mint.TransportError{reason: :timeout}} = Req.request(req) + assert {:error, %Req.TransportError{reason: :timeout}} = Req.request(req) assert_received :ping end @@ -2044,7 +2071,7 @@ defmodule Req.StepsTest do test "into: fun handle error", %{bypass: bypass, url: url} do Bypass.down(bypass) - assert {:error, %Mint.TransportError{reason: :econnrefused}} = + assert {:error, %Req.TransportError{reason: :econnrefused}} = Req.get( url: url, retry: false, @@ -2097,7 +2124,7 @@ defmodule Req.StepsTest do test "into: collectable handle error", %{bypass: bypass, url: url} do Bypass.down(bypass) - assert {:error, %Mint.TransportError{reason: :econnrefused}} = + assert {:error, %Req.TransportError{reason: :econnrefused}} = Req.get( url: url, retry: false, From ce3f4fd44f44a65e6565ebb791f1a916a4a3e49a Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Fri, 15 Mar 2024 15:01:24 +0100 Subject: [PATCH 2/5] up --- lib/req/steps.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/req/steps.ex b/lib/req/steps.ex index 51cbf16c..19926a3b 100644 --- a/lib/req/steps.ex +++ b/lib/req/steps.ex @@ -1180,7 +1180,7 @@ defmodule Req.Steps do You can simulate failure conditions by returning exception structs from your plugs. For network related issues, use `Req.TransportError` exception: - test "timeout" do + test "network issues" do plug = fn _conn -> %Req.TransportError{reason: :econnrefused} end From e7dd7ebd585de3d617ce73eff39dfcc96af1ca3b Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Sat, 16 Mar 2024 19:53:33 +0100 Subject: [PATCH 3/5] Update lib/req/transport_error.ex Co-authored-by: Andrea Leopardi --- lib/req/transport_error.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/req/transport_error.ex b/lib/req/transport_error.ex index 24e4d782..689d1573 100644 --- a/lib/req/transport_error.ex +++ b/lib/req/transport_error.ex @@ -2,7 +2,7 @@ defmodule Req.TransportError do @moduledoc """ Represents an error with the transport used by an HTTP connection. - This is a standardised exception that all Req adapters should use for transport layer related + This is a standardised exception that all Req adapters should use for transport-layer-related errors. This exception is based on `Mint.TransportError`. From 5e498df75706dcec8aff0fa35543c3cfc27b324c Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Sat, 16 Mar 2024 20:27:38 +0100 Subject: [PATCH 4/5] Add Req.Test.transport_error/2 --- lib/req/steps.ex | 46 ++++++++++++----------------------------- lib/req/test.ex | 40 +++++++++++++++++++++++++++++++++++ test/req/steps_test.exs | 18 ++-------------- test/req/test_test.exs | 8 +++++++ 4 files changed, 63 insertions(+), 49 deletions(-) diff --git a/lib/req/steps.ex b/lib/req/steps.ex index 19926a3b..dc8c59b5 100644 --- a/lib/req/steps.ex +++ b/lib/req/steps.ex @@ -1177,16 +1177,16 @@ defmodule Req.Steps do assert Req.get!(plug: plug, into: []).body == ["echoecho"] end - You can simulate failure conditions by returning exception structs from your plugs. - For network related issues, use `Req.TransportError` exception: + You can simulate network errors by calling `Req.Test.transport_error/2` + in your plugs: test "network issues" do - plug = fn _conn -> - %Req.TransportError{reason: :econnrefused} + plug = fn conn -> + Req.Test.transport_error(conn, :timeout) end assert Req.get(plug: plug, retry: false) == - {:error, %Req.TransportError{reason: :econnrefused}} + {:error, %Req.TransportError{reason: :timeout}} end """ @doc step: :request @@ -1224,35 +1224,15 @@ defmodule Req.Steps do end end - conn = Plug.Test.conn(request.method, request.url, req_body) - - conn = put_in(conn.req_headers, req_headers) - - case call_plug(conn, plug) do - %Plug.Conn{} = conn -> - handle_plug_result(conn, request) + conn = + Plug.Test.conn(request.method, request.url, req_body) + |> Map.replace!(:req_headers, req_headers) + |> call_plug(plug) - %Req.TransportError{} = exception -> - validate_transport_error!(exception.reason) - {request, exception} - - %_{__exception__: true} = exception -> - {request, exception} - end - end - - defp validate_transport_error!(:protocol_not_negotiated), do: :ok - defp validate_transport_error!({:bad_alpn_protocol, _}), do: :ok - defp validate_transport_error!(:closed), do: :ok - defp validate_transport_error!(:timeout), do: :ok - - defp validate_transport_error!(reason) do - case :ssl.format_error(reason) do - ~c"Unexpected error:" ++ _ -> - raise ArgumentError, "unexpected Req.Transport reason: #{inspect(reason)}" - - _ -> - :ok + if exception = conn.private[:req_test_exception] do + {request, exception} + else + handle_plug_result(conn, request) end end diff --git a/lib/req/test.ex b/lib/req/test.ex index 0d710836..3f5d67a6 100644 --- a/lib/req/test.ex +++ b/lib/req/test.ex @@ -206,6 +206,46 @@ defmodule Req.Test do end end + @doc """ + """ + def transport_error(conn, reason) + + if Code.ensure_loaded?(Plug.Conn) do + @spec transport_error(Plug.Conn.t(), reason :: atom()) :: Plug.Conn.t() + def transport_error(%Plug.Conn{} = conn, reason) do + validate_transport_error!(reason) + exception = Req.TransportError.exception(reason: reason) + put_in(conn.private[:req_test_exception], exception) + end + + defp validate_transport_error!(:protocol_not_negotiated), do: :ok + defp validate_transport_error!({:bad_alpn_protocol, _}), do: :ok + defp validate_transport_error!(:closed), do: :ok + defp validate_transport_error!(:timeout), do: :ok + + defp validate_transport_error!(reason) do + case :ssl.format_error(reason) do + ~c"Unexpected error:" ++ _ -> + raise ArgumentError, "unexpected Req.TransportError reason: #{inspect(reason)}" + + _ -> + :ok + end + end + else + def transport_error(_conn, _reason) do + Logger.error(""" + Could not find plug dependency. + + Please add :plug to your dependencies: + + {:plug, "~> 1.0"} + """) + + raise "missing plug dependency" + end + end + @doc """ Returns the stub created by `stub/2`. """ diff --git a/test/req/steps_test.exs b/test/req/steps_test.exs index 1cab65f0..596883ea 100644 --- a/test/req/steps_test.exs +++ b/test/req/steps_test.exs @@ -1781,8 +1781,8 @@ defmodule Req.StepsTest do test "errors" do req = Req.new( - plug: fn _conn -> - %Req.TransportError{reason: :timeout} + plug: fn conn -> + Req.Test.transport_error(conn, :timeout) end, retry: false ) @@ -1790,20 +1790,6 @@ defmodule Req.StepsTest do assert Req.request(req) == {:error, %Req.TransportError{reason: :timeout}} end - - test "validate Req.TransportError reason" do - req = - Req.new( - plug: fn _conn -> - %Req.TransportError{reason: :bad} - end, - retry: false - ) - - assert_raise ArgumentError, "unexpected Req.Transport reason: :bad", fn -> - Req.request(req) - end - end end describe "run_finch" do diff --git a/test/req/test_test.exs b/test/req/test_test.exs index ddf9a14f..520e2026 100644 --- a/test/req/test_test.exs +++ b/test/req/test_test.exs @@ -67,4 +67,12 @@ defmodule Req.TestTest do assert_receive {^ref, 1} end end + + describe "transport_error/2" do + test "validate reason" do + assert_raise ArgumentError, "unexpected Req.TransportError reason: :bad", fn -> + Req.Test.transport_error(%Plug.Conn{}, :bad) + end + end + end end From 44412f5e1b90715b5b9f50c7227da538befb73f8 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Sat, 16 Mar 2024 20:31:19 +0100 Subject: [PATCH 5/5] update docs --- lib/req/steps.ex | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/req/steps.ex b/lib/req/steps.ex index dc8c59b5..63f81901 100644 --- a/lib/req/steps.ex +++ b/lib/req/steps.ex @@ -1160,7 +1160,7 @@ defmodule Req.Steps do assert Req.get!("http:///hello", plug: echo).body == "hello" end - which is particularly useful to create HTTP service mocks, similar to tools like + which is particularly useful to create HTTP service stubs, similar to tools like [Bypass](https://github.com/PSPDFKit-labs/bypass). Response streaming is also supported however at the moment the entire response @@ -1177,6 +1177,19 @@ defmodule Req.Steps do assert Req.get!(plug: plug, into: []).body == ["echoecho"] end + When testing JSON APIs, it's common to use the `Req.Test.json/2` helper: + + test "JSON" do + plug = fn conn -> + Req.Test.json(conn, %{message: "Hello, World!"}) + end + + resp = Req.get!(plug: plug) + assert resp.status == 200 + assert resp.headers["content-type"] == ["application/json; charset=utf-8"] + assert resp.body == %{"message" => "Hello, World!"} + end + You can simulate network errors by calling `Req.Test.transport_error/2` in your plugs: