diff --git a/.github/workflows/hex_pub.yml b/.github/workflows/hex_pub.yml index d78b03a1..6865ca6f 100644 --- a/.github/workflows/hex_pub.yml +++ b/.github/workflows/hex_pub.yml @@ -4,8 +4,8 @@ on: - '*' jobs: - if: false publish: + if: false runs-on: ubuntu-latest steps: - name: Check out diff --git a/include/quicer.hrl b/include/quicer.hrl index 9a7f969a..ef13ebc6 100644 --- a/include/quicer.hrl +++ b/include/quicer.hrl @@ -149,4 +149,11 @@ -define(QUIC_CONGESTION_CONTROL_ALGORITHM_CUBIC, 0). -define(QUIC_CONGESTION_CONTROL_ALGORITHM_BBR, 1). +-record(probe_state, { + final :: term() | undefined, + sent_at :: integer() | undefined, + suspect_lost_at :: integer() | undefined, + final_at :: integer() | undefined +}). + -endif. %% QUICER_HRL diff --git a/include/quicer_types.hrl b/include/quicer_types.hrl index 8a46ee82..6230522e 100644 --- a/include/quicer_types.hrl +++ b/include/quicer_types.hrl @@ -506,5 +506,11 @@ dgram_max_len := uint64() }. +-type probe_state() :: #probe_state{}. +-type probe_res() :: + #probe_state{} + | {error, dgram_send_error, atom()} + | {error, atom()}. + %% QUICER_TYPES_HRL -endif. diff --git a/src/quicer.erl b/src/quicer.erl index ef126f81..d8311b30 100644 --- a/src/quicer.erl +++ b/src/quicer.erl @@ -58,6 +58,7 @@ close_connection/4, async_close_connection/1, async_close_connection/3, + probe/2, accept_stream/2, accept_stream/3, async_accept_stream/2, @@ -68,6 +69,7 @@ async_send/2, async_send/3, recv/2, + async_send_dgram/2, send_dgram/2, shutdown_stream/1, shutdown_stream/2, @@ -171,7 +173,10 @@ quicer_addr/0, %% Registraion Profiles - registration_profile/0 + registration_profile/0, + + %% probes + probe_res/0 ]). -type connection_opts() :: proplists:proplist() | conn_opts(). @@ -809,35 +814,49 @@ do_recv(Stream, Count, Buff) -> E end. +%% @doc Sending Unreliable Datagram. +%% Caller should handle the async signals for the send results +%% +%% ref: [https://datatracker.ietf.org/doc/html/rfc9221] +%% @see send/2 send_dgram/2 +-spec async_send_dgram(connection_handle(), binary()) -> + {ok, non_neg_integer()} + | {error, badarg | not_enough_mem | invalid_parameter | closed} + | {error, dgram_send_error, atom_reason()}. +async_send_dgram(Conn, Data) -> + quicer_nif:send_dgram(Conn, Data, _IsSyncRel = 1). + %% @doc Sending Unreliable Datagram +%% return error only if sending could not be scheduled such as +%% not_enough_mem, connection is already closed or wrong args. +%% otherwise, it is fire and forget. %% -%% ref: [https://datatracker.ietf.org/doc/html/draft-ietf-quic-datagram] -%% @see send/2 +%% %% ref: [https://datatracker.ietf.org/doc/html/rfc9221] +%% @see send/2, async_send_dgram/2 -spec send_dgram(connection_handle(), binary()) -> - {ok, BytesSent :: pos_integer()} - | {error, badarg | not_enough_mem | closed} + {ok, BytesSent :: non_neg_integer()} + | {error, badarg | not_enough_mem | invalid_parameter | closed} | {error, dgram_send_error, atom_reason()}. send_dgram(Conn, Data) -> case quicer_nif:send_dgram(Conn, Data, _IsSync = 1) of - %% @todo we need find tuned event mask {ok, _Len} = OK -> - receive - {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_SENT}} -> - receive - {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} -> - OK; - {quic, dgram_send_state, Conn, #{state := Other}} -> - {error, dgram_send_error, Other} - end; - {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} -> + case quicer_lib:handle_dgram_send_states(Conn) of + ok -> OK; - {quic, dgram_send_state, Conn, #{state := Other}} -> - {error, dgram_send_error, Other} + {error, E} -> + {error, dgram_send_error, E} end; + {error, E} -> + {error, E}; E -> E end. +%% @doc Probe conn state with 0 len dgram. +-spec probe(connection_handle(), timeout()) -> probe_res(). +probe(Conn, Timeout) -> + quicer_lib:probe(Conn, Timeout). + %% @doc Shutdown stream gracefully, with infinity timeout %% %% @see shutdown_stream/1 diff --git a/src/quicer_lib.erl b/src/quicer_lib.erl index 90cdbade..a66632c9 100644 --- a/src/quicer_lib.erl +++ b/src/quicer_lib.erl @@ -20,6 +20,7 @@ cb_ret/0, cb_state/0 ]). + -type cb_ret() :: cb_ret_noreply() | cb_ret_reply(). -type cb_state() :: term(). @@ -41,7 +42,12 @@ -type action() :: hibernate | timeout() | {continue, Continue :: term()}. --export([default_cb_ret/2]). +-export([ + default_cb_ret/2, + handle_dgram_send_states/1, + handle_dgram_send_states/3, + probe/2 +]). -spec default_cb_ret(cb_ret(), State :: term()) -> {reply, NewState :: term()} @@ -69,3 +75,86 @@ default_cb_ret({reply, Reply, NewCBState, Action}, State) -> {reply, Reply, State#{callback_state := NewCBState}, Action}; default_cb_ret({reply, Reply, NewCBState}, State) -> {reply, Reply, State#{callback_state := NewCBState}}. + +-spec probe(connection_handle(), timeout()) -> probe_res(). +probe(Conn, Timeout) -> + case quicer_nif:send_dgram(Conn, <<>>, _IsSync = 1) of + {ok, _Len} -> + handle_dgram_send_states(Conn, probe_dgram_send_cb(), Timeout); + {error, E} -> + {error, dgram_send_error, E}; + E -> + E + end. + +-spec handle_dgram_send_states(connection_handle()) -> + ok + | {error, + dgram_send_canceled + | dgram_send_unknown + | dgram_send_lost_discarded}. +handle_dgram_send_states(Conn) -> + handle_dgram_send_states(init, Conn, default_dgram_suspect_lost_cb(), 5000). + +-type lost_suspect_callback() :: + {fun((connection_handle(), term(), term()) -> term()), term()} + | {atom(), term()}. +-spec handle_dgram_send_states(connection_handle(), lost_suspect_callback(), timeout()) -> any(). +handle_dgram_send_states(Conn, {_CBFun, _CBState} = CB, Timeout) -> + handle_dgram_send_states(init, Conn, CB, Timeout). + +handle_dgram_send_states(init, Conn, {Fun, CallbackState}, Timeout) -> + receive + {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_SENT}} -> + NewCBState = Fun(Conn, ?QUIC_DATAGRAM_SEND_SENT, CallbackState), + handle_dgram_send_states(sent, Conn, {Fun, NewCBState}, Timeout); + {quic, dgram_send_state, Conn, #{state := Final}} -> + Fun(Conn, Final, CallbackState) + after 5000 -> + %% @TODO proper test caught this, may fire a bug report to msquic + Fun(Conn, timeout, CallbackState) + end; +handle_dgram_send_states(sent, Conn, {Fun, CallbackState}, Timeout) -> + receive + {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_LOST_SUSPECT}} -> + %% Lost suspected, call the callback for the return hits. + %% however, we still need to wait for the final state. + NewCBState = Fun(Conn, ?QUIC_DATAGRAM_SEND_LOST_SUSPECT, CallbackState), + receive + {quic, dgram_send_state, Conn, #{state := EState}} -> + Fun(Conn, EState, NewCBState) + after Timeout -> + %% @TODO proper test caught this, may fire a bug report to msquic + Fun(Conn, timeout, CallbackState) + end; + {quic, dgram_send_state, Conn, #{state := Final}} -> + Fun(Conn, Final, CallbackState) + after Timeout -> + %% @TODO proper test caught this, may fire a bug report to msquic + Fun(Conn, timeout, CallbackState) + end. + +%% Default Callback for Datagram Send lost suspected +default_dgram_suspect_lost_cb() -> + Fun = fun(_Conn, _, _CallbackState) -> + %% just return ok, even it is lost, we don't care. + ok + end, + {Fun, undefined}. + +probe_dgram_send_cb() -> + Fun = fun + (_Conn, ?QUIC_DATAGRAM_SEND_SENT, CallbackState) -> + CallbackState#probe_state{sent_at = ts_ms()}; + (_Conn, ?QUIC_DATAGRAM_SEND_LOST_SUSPECT, CallbackState) -> + CallbackState#probe_state{suspect_lost_at = ts_ms()}; + (_Conn, State, CallbackState) -> + CallbackState#probe_state{ + final_at = ts_ms(), + final = State + } + end, + {Fun, #probe_state{}}. + +ts_ms() -> + erlang:monotonic_time(millisecond). diff --git a/src/quicer_nif.erl b/src/quicer_nif.erl index 04d74928..1d8d8d15 100644 --- a/src/quicer_nif.erl +++ b/src/quicer_nif.erl @@ -297,7 +297,7 @@ recv(_Stream, _Len) -> -spec send_dgram(connection_handle(), iodata(), send_flags()) -> {ok, BytesSent :: pos_integer()} - | {error, badarg | not_enough_memory | closed} + | {error, badarg | not_enough_memory | invalid_parameter | closed} | {error, dgram_send_error, atom_reason()}. send_dgram(_Conn, _Data, _Flags) -> erlang:nif_error(nif_library_not_loaded). diff --git a/test/prop_stateful_client_conn.erl b/test/prop_stateful_client_conn.erl index ce28d503..5aed6a80 100644 --- a/test/prop_stateful_client_conn.erl +++ b/test/prop_stateful_client_conn.erl @@ -55,6 +55,7 @@ prop_client_state_test() -> %%%%%%%%%%%%% %% @doc Initial model value at system start. Should be deterministic. initial_state() -> + net_kernel:start([?MODULE, shortnames]), {ok, H} = quicer:connect("localhost", 14568, default_conn_opts(), 10000), #{ state => connected, @@ -74,6 +75,8 @@ command(#{handle := Handle}) -> {call, quicer, async_accept_stream, [Handle, ?LET(Opts, quicer_acceptor_opts(), Opts)]}}, {100, {call, quicer, peername, [Handle]}}, {50, {call, quicer, peercert, [Handle]}}, + {50, {call, quicer, probe, [Handle, 5000]}}, + {50, {call, quicer, send_dgram, [Handle, binary()]}}, {10, {call, quicer, negotiated_protocol, [Handle]}}, {10, {call, quicer, get_connections, []}}, {10, {call, quicer, get_conn_owner, [Handle]}}, @@ -190,6 +193,36 @@ postcondition( {error, not_owner} ) -> Owner =/= self(); +postcondition( + #{state := ConnState}, + {call, quicer, probe, [_, _]}, + {error, dgram_send_error, _} +) -> + ConnState =/= connected; +postcondition( + #{state := _ConnState}, + {call, quicer, probe, [_, _]}, + #probe_state{final = FinalState, final_at = FinalTs} +) -> + FinalState =/= undefined andalso FinalTs =/= undefined; +postcondition( + #{state := _ConnState}, + {call, quicer, send_dgram, [_, _]}, + {ok, _} +) -> + true; +postcondition( + #{state := ConnState}, + {call, quicer, send_dgram, [_, _]}, + {error, _, _} +) -> + ConnState =/= connected; +postcondition( + #{state := ConnState}, + {call, quicer, send_dgram, [_, _]}, + {error, _} +) -> + ConnState =/= connected; postcondition( #{owner := _, state := connected}, {call, quicer, controlling_process, [_, NewOwner]}, @@ -265,7 +298,8 @@ default_listen_opts() -> {handshake_idle_timeout_ms, 10000}, % QUIC_SERVER_RESUME_AND_ZERORTT {server_resumption_level, 2}, - {peer_bidi_stream_count, 10} + {peer_bidi_stream_count, 10}, + {datagram_receive_enabled, 1} ]. default_conn_opts() -> @@ -276,5 +310,6 @@ default_conn_opts() -> {idle_timeout_ms, 0}, {cacertfile, "./msquic/submodules/openssl/test/certs/rootCA.pem"}, {certfile, "./msquic/submodules/openssl/test/certs/servercert.pem"}, - {keyfile, "./msquic/submodules/openssl/test/certs/serverkey.pem"} + {keyfile, "./msquic/submodules/openssl/test/certs/serverkey.pem"}, + {datagram_receive_enabled, 1} ]. diff --git a/test/prop_stateful_server_conn.erl b/test/prop_stateful_server_conn.erl index 8b52ee49..8df60568 100644 --- a/test/prop_stateful_server_conn.erl +++ b/test/prop_stateful_server_conn.erl @@ -100,6 +100,8 @@ command(#{handle := Handle}) -> {call, quicer, async_accept_stream, [Handle, ?LET(Opts, quicer_acceptor_opts(), Opts)]}}, {100, {call, quicer, peername, [Handle]}}, {50, {call, quicer, peercert, [Handle]}}, + {50, {call, quicer, probe, [Handle, 5000]}}, + {50, {call, quicer, send_dgram, [Handle, binary()]}}, {10, {call, quicer, negotiated_protocol, [Handle]}}, {10, {call, quicer, get_connections, []}}, {10, {call, quicer, get_conn_owner, [Handle]}}, @@ -155,6 +157,10 @@ postcondition(#{state := S}, {call, quicer, handshake, _Args}, {error, invalid_s S =/= accepted -> true; +postcondition(#{state := S}, {call, quicer, handshake, _Args}, {error, timeout}) when + S =/= accepted +-> + true; postcondition(_State, {call, quicer, getopt, _Args}, {ok, _}) -> true; postcondition(_State, {call, quicer, getopt, [_, password]}, {error, badarg}) -> @@ -278,6 +284,36 @@ postcondition(#{state := closed}, {call, _Mod, _Fun, _Args}, {error, closed}) -> postcondition(#{state := accepted}, {call, _Mod, _Fun, _Args}, {error, closed}) -> %% handshake didnt take place on time true; +postcondition( + #{state := ConnState}, + {call, quicer, probe, [_, _]}, + {error, dgram_send_error, _} +) -> + ConnState =/= connected; +postcondition( + #{state := _ConnState}, + {call, quicer, probe, [_, _]}, + #probe_state{final = FinalState, final_at = FinalTs} +) -> + FinalState =/= undefined andalso FinalTs =/= undefined; +postcondition( + #{state := _ConnState}, + {call, quicer, send_dgram, [_, _]}, + {ok, _} +) -> + true; +postcondition( + #{state := ConnState}, + {call, quicer, send_dgram, [_, _]}, + {error, _, _} +) -> + ConnState =/= connected; +postcondition( + #{state := ConnState}, + {call, quicer, send_dgram, [_, _]}, + {error, _} +) -> + ConnState =/= connected; postcondition(_State, {call, _Mod, _Fun, _Args} = _Call, _Res) -> false. @@ -286,6 +322,10 @@ postcondition(_State, {call, _Mod, _Fun, _Args} = _Call, _Res) -> next_state(State, Res, Call) -> step_calls(do_next_state(State, Res, Call)). +do_next_state( + #{state := _} = State, {error, closed}, {call, _M, _F, _A} +) -> + State#{state := closed}; do_next_state(#{state := accepted} = State, {error, _}, {call, quicer, handshake, _Args}) -> State; do_next_state(#{state := accepted} = State, _Res, {call, quicer, handshake, _Args}) -> @@ -347,7 +387,8 @@ default_listen_opts() -> {handshake_idle_timeout_ms, 100}, % QUIC_SERVER_RESUME_AND_ZERORTT {server_resumption_level, 2}, - {peer_bidi_stream_count, 10} + {peer_bidi_stream_count, 10}, + {datagram_receive_enabled, true} ]. default_conn_opts() -> @@ -358,5 +399,6 @@ default_conn_opts() -> {idle_timeout_ms, 5000}, {cacertfile, "./msquic/submodules/openssl/test/certs/rootCA.pem"}, {certfile, "./msquic/submodules/openssl/test/certs/servercert.pem"}, - {keyfile, "./msquic/submodules/openssl/test/certs/serverkey.pem"} + {keyfile, "./msquic/submodules/openssl/test/certs/serverkey.pem"}, + {datagram_receive_enabled, true} ]. diff --git a/test/quicer_SUITE.erl b/test/quicer_SUITE.erl index 14c65954..08cb20a6 100644 --- a/test/quicer_SUITE.erl +++ b/test/quicer_SUITE.erl @@ -74,6 +74,8 @@ tc_stream_get_owner_remote/1, tc_dgram_client_send/1, + tc_dgram_client_send_fail/1, + tc_dgram_client_send_exceed_mtu/1, % , tc_getopt_raw/1 tc_getopt/1, @@ -845,7 +847,7 @@ tc_stream_controlling_process_demon(Config) -> ok = quicer:setopt(Stm, active, true), {ok, _Len} = quicer:send(Stm, <<"owner_changed">>), receive - {quic, <<"owner_changed">>, Stm, _} -> + {quic, <<"owner_changed">>, _Stm, _} -> ok end, %% Set controlling_process again @@ -867,6 +869,48 @@ tc_stream_controlling_process_demon(Config) -> ct:fail("timeout") end. +tc_dgram_client_send_fail(_) -> + Opts = default_conn_opts() ++ [{datagram_receive_enabled, 1}], + {ok, Conn} = quicer:async_connect("localhost", 65535, Opts), + ?assertEqual( + %% fire and forget + {ok, 4}, + quicer:send_dgram(Conn, <<"ping">>) + ), + ok. + +tc_dgram_client_send_exceed_mtu(Config) -> + Port = select_port(), + Owner = self(), + {SPid, Ref} = spawn_monitor(fun() -> ping_pong_server_dgram(Owner, Config, Port) end), + receive + listener_ready -> + ok + after 1000 -> + ct:fail("timeout here") + end, + %% GIVEN: datagram is enabled + Opts = default_conn_opts() ++ [{datagram_receive_enabled, 1}], + {ok, Conn} = quicer:connect("localhost", Port, Opts, 5000), + {ok, Stm} = quicer:start_stream(Conn, []), + {ok, 4} = quicer:send(Stm, <<"ping">>), + {ok, V2stats} = quicer:getopt(Conn, statistics_v2), + MtuMax = proplists:get_value(send_path_mtu, V2stats), + %% WHEN: send a datagram that is less than MTU-100 + Length = MtuMax - 100, + %% THEN: send should success + {ok, Length} = quicer:send_dgram(Conn, crypto:strong_rand_bytes(Length)), + ?assertEqual( + {error, dgram_send_error, invalid_parameter}, + quicer:send_dgram(Conn, crypto:strong_rand_bytes(MtuMax * 2)) + ), + flush_streams_available(Conn), + flush_datagram_state_changed(Conn), + %% WHEN: send a datagram that is 2xMTU size + %% THEN: send should fail + SPid ! done, + ok = ensure_server_exit_normal(Ref). + tc_dgram_client_send(Config) -> Port = select_port(), Owner = self(), diff --git a/test/quicer_connection_SUITE.erl b/test/quicer_connection_SUITE.erl index 9c8b65d4..ab24a5ec 100644 --- a/test/quicer_connection_SUITE.erl +++ b/test/quicer_connection_SUITE.erl @@ -897,6 +897,16 @@ tc_get_conn_owner_server(Config) -> ct:fail({client_fail, Reason}) end. +tc_conn_probe(_) -> + Opts = default_conn_opts() ++ [{datagram_receive_enabled, 1}], + {ok, Conn} = quicer:async_connect("localhost", 65535, Opts), + ?assertMatch( + #probe_state{final_at = TS, final = ?QUIC_DATAGRAM_SEND_CANCELED} when + TS =/= undefined, + quicer:probe(Conn, 5000) + ), + ok. + %%% %%% Helpers %%%