Probe conn state with dgram

.github/workflows/hex_pub.yml

@@ -4,8 +4,8 @@ on:
       - '*'
 
 jobs:
-  if: false
   publish:
+    if: false
     runs-on: ubuntu-latest
     steps:
       - name: Check out

include/quicer.hrl

@@ -149,4 +149,11 @@
 -define(QUIC_CONGESTION_CONTROL_ALGORITHM_CUBIC, 0).
 -define(QUIC_CONGESTION_CONTROL_ALGORITHM_BBR, 1).
 
+-record(probe_state, {
+    final :: term() | undefined,
+    sent_at :: integer() | undefined,
+    suspect_lost_at :: integer() | undefined,
+    final_at :: integer() | undefined
+}).
+
 -endif. %% QUICER_HRL

include/quicer_types.hrl

@@ -506,5 +506,11 @@
     dgram_max_len := uint64()
 }.
 
+-type probe_state() :: #probe_state{}.
+-type probe_res() ::
+    #probe_state{}
+    | {error, dgram_send_error, atom()}
+    | {error, atom()}.
+
 %% QUICER_TYPES_HRL
 -endif.

src/quicer.erl

@@ -58,6 +58,7 @@
     close_connection/4,
     async_close_connection/1,
     async_close_connection/3,
+    probe/2,
     accept_stream/2,
     accept_stream/3,
     async_accept_stream/2,
@@ -68,6 +69,7 @@
     async_send/2,
     async_send/3,
     recv/2,
+    async_send_dgram/2,
     send_dgram/2,
     shutdown_stream/1,
     shutdown_stream/2,
@@ -171,7 +173,10 @@
     quicer_addr/0,
 
     %% Registraion Profiles
-    registration_profile/0
+    registration_profile/0,
+
+    %% probes
+    probe_res/0
 ]).
 
 -type connection_opts() :: proplists:proplist() | conn_opts().
@@ -809,35 +814,49 @@ do_recv(Stream, Count, Buff) ->
             E
     end.
 
+%% @doc Sending Unreliable Datagram.
+%% Caller should handle the async signals for the send results
+%%
+%% ref: [https://datatracker.ietf.org/doc/html/rfc9221]
+%% @see send/2 send_dgram/2
+-spec async_send_dgram(connection_handle(), binary()) ->
+    {ok, non_neg_integer()}
+    | {error, badarg | not_enough_mem | invalid_parameter | closed}
+    | {error, dgram_send_error, atom_reason()}.
+async_send_dgram(Conn, Data) ->
+    quicer_nif:send_dgram(Conn, Data, _IsSyncRel = 1).
+
 %% @doc Sending Unreliable Datagram
+%%  return error only if sending could not be scheduled such as
+%%  not_enough_mem, connection is already closed or wrong args.
+%%  otherwise, it is fire and forget.
 %%
-%% ref: [https://datatracker.ietf.org/doc/html/draft-ietf-quic-datagram]
-%% @see send/2
+%% %% ref: [https://datatracker.ietf.org/doc/html/rfc9221]
+%% @see send/2, async_send_dgram/2
 -spec send_dgram(connection_handle(), binary()) ->
-    {ok, BytesSent :: pos_integer()}
-    | {error, badarg | not_enough_mem | closed}
+    {ok, BytesSent :: non_neg_integer()}
+    | {error, badarg | not_enough_mem | invalid_parameter | closed}
     | {error, dgram_send_error, atom_reason()}.
 send_dgram(Conn, Data) ->
     case quicer_nif:send_dgram(Conn, Data, _IsSync = 1) of
-        %% @todo we need find tuned event mask
         {ok, _Len} = OK ->
-            receive
-                {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_SENT}} ->
-                    receive
-                        {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} ->
-                            OK;
-                        {quic, dgram_send_state, Conn, #{state := Other}} ->
-                            {error, dgram_send_error, Other}
-                    end;
-                {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} ->
+            case quicer_lib:handle_dgram_send_states(Conn) of
+                ok ->
                     OK;
-                {quic, dgram_send_state, Conn, #{state := Other}} ->
-                    {error, dgram_send_error, Other}
+                {error, E} ->
+                    {error, dgram_send_error, E}
             end;
+        {error, E} ->
+            {error, E};
         E ->
             E
     end.
 
+%% @doc Probe conn state with 0 len dgram.
+-spec probe(connection_handle(), timeout()) -> probe_res().
+probe(Conn, Timeout) ->
+    quicer_lib:probe(Conn, Timeout).
+
 %% @doc Shutdown stream gracefully, with infinity timeout
 %%
 %% @see shutdown_stream/1

src/quicer_lib.erl

@@ -20,6 +20,7 @@
     cb_ret/0,
     cb_state/0
 ]).
+
 -type cb_ret() :: cb_ret_noreply() | cb_ret_reply().
 -type cb_state() :: term().
 
@@ -41,7 +42,12 @@
 
 -type action() :: hibernate | timeout() | {continue, Continue :: term()}.
 
--export([default_cb_ret/2]).
+-export([
+    default_cb_ret/2,
+    handle_dgram_send_states/1,
+    handle_dgram_send_states/3,
+    probe/2
+]).
 
 -spec default_cb_ret(cb_ret(), State :: term()) ->
     {reply, NewState :: term()}
@@ -69,3 +75,86 @@ default_cb_ret({reply, Reply, NewCBState, Action}, State) ->
     {reply, Reply, State#{callback_state := NewCBState}, Action};
 default_cb_ret({reply, Reply, NewCBState}, State) ->
     {reply, Reply, State#{callback_state := NewCBState}}.
+
+-spec probe(connection_handle(), timeout()) -> probe_res().
+probe(Conn, Timeout) ->
+    case quicer_nif:send_dgram(Conn, <<>>, _IsSync = 1) of
+        {ok, _Len} ->
+            handle_dgram_send_states(Conn, probe_dgram_send_cb(), Timeout);
+        {error, E} ->
+            {error, dgram_send_error, E};
+        E ->
+            E
+    end.
+
+-spec handle_dgram_send_states(connection_handle()) ->
+    ok
+    | {error,
+        dgram_send_canceled
+        | dgram_send_unknown
+        | dgram_send_lost_discarded}.
+handle_dgram_send_states(Conn) ->
+    handle_dgram_send_states(init, Conn, default_dgram_suspect_lost_cb(), 5000).
+
+-type lost_suspect_callback() ::
+    {fun((connection_handle(), term(), term()) -> term()), term()}
+    | {atom(), term()}.
+-spec handle_dgram_send_states(connection_handle(), lost_suspect_callback(), timeout()) -> any().
+handle_dgram_send_states(Conn, {_CBFun, _CBState} = CB, Timeout) ->
+    handle_dgram_send_states(init, Conn, CB, Timeout).
+
+handle_dgram_send_states(init, Conn, {Fun, CallbackState}, Timeout) ->
+    receive
+        {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_SENT}} ->
+            NewCBState = Fun(Conn, ?QUIC_DATAGRAM_SEND_SENT, CallbackState),
+            handle_dgram_send_states(sent, Conn, {Fun, NewCBState}, Timeout);
+        {quic, dgram_send_state, Conn, #{state := Final}} ->
+            Fun(Conn, Final, CallbackState)
+    after 5000 ->
+        %% @TODO proper test caught this, may fire a bug report to msquic
+        Fun(Conn, timeout, CallbackState)
+    end;
+handle_dgram_send_states(sent, Conn, {Fun, CallbackState}, Timeout) ->
+    receive
+        {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_LOST_SUSPECT}} ->
+            %% Lost suspected, call the callback for the return hits.
+            %% however, we still need to wait for the final state.
+            NewCBState = Fun(Conn, ?QUIC_DATAGRAM_SEND_LOST_SUSPECT, CallbackState),
+            receive
+                {quic, dgram_send_state, Conn, #{state := EState}} ->
+                    Fun(Conn, EState, NewCBState)
+            after Timeout ->
+                %% @TODO proper test caught this, may fire a bug report to msquic
+                Fun(Conn, timeout, CallbackState)
+            end;
+        {quic, dgram_send_state, Conn, #{state := Final}} ->
+            Fun(Conn, Final, CallbackState)
+    after Timeout ->
+        %% @TODO proper test caught this, may fire a bug report to msquic
+        Fun(Conn, timeout, CallbackState)
+    end.
+
+%% Default Callback for Datagram Send lost suspected
+default_dgram_suspect_lost_cb() ->
+    Fun = fun(_Conn, _, _CallbackState) ->
+        %% just return ok, even it is lost, we don't care.
+        ok
+    end,
+    {Fun, undefined}.
+
+probe_dgram_send_cb() ->
+    Fun = fun
+        (_Conn, ?QUIC_DATAGRAM_SEND_SENT, CallbackState) ->
+            CallbackState#probe_state{sent_at = ts_ms()};
+        (_Conn, ?QUIC_DATAGRAM_SEND_LOST_SUSPECT, CallbackState) ->
+            CallbackState#probe_state{suspect_lost_at = ts_ms()};
+        (_Conn, State, CallbackState) ->
+            CallbackState#probe_state{
+                final_at = ts_ms(),
+                final = State
+            }
+    end,
+    {Fun, #probe_state{}}.
+
+ts_ms() ->
+    erlang:monotonic_time(millisecond).

src/quicer_nif.erl

@@ -297,7 +297,7 @@ recv(_Stream, _Len) ->
 
 -spec send_dgram(connection_handle(), iodata(), send_flags()) ->
     {ok, BytesSent :: pos_integer()}
-    | {error, badarg | not_enough_memory | closed}
+    | {error, badarg | not_enough_memory | invalid_parameter | closed}
     | {error, dgram_send_error, atom_reason()}.
 send_dgram(_Conn, _Data, _Flags) ->
     erlang:nif_error(nif_library_not_loaded).

test/prop_stateful_client_conn.erl

@@ -55,6 +55,7 @@ prop_client_state_test() ->
 %%%%%%%%%%%%%
 %% @doc Initial model value at system start. Should be deterministic.
 initial_state() ->
+    net_kernel:start([?MODULE, shortnames]),
     {ok, H} = quicer:connect("localhost", 14568, default_conn_opts(), 10000),
     #{
         state => connected,
@@ -74,6 +75,8 @@ command(#{handle := Handle}) ->
             {call, quicer, async_accept_stream, [Handle, ?LET(Opts, quicer_acceptor_opts(), Opts)]}},
         {100, {call, quicer, peername, [Handle]}},
         {50, {call, quicer, peercert, [Handle]}},
+        {50, {call, quicer, probe, [Handle, 5000]}},
+        {50, {call, quicer, send_dgram, [Handle, binary()]}},
         {10, {call, quicer, negotiated_protocol, [Handle]}},
         {10, {call, quicer, get_connections, []}},
         {10, {call, quicer, get_conn_owner, [Handle]}},
@@ -190,6 +193,36 @@ postcondition(
     {error, not_owner}
 ) ->
     Owner =/= self();
+postcondition(
+    #{state := ConnState},
+    {call, quicer, probe, [_, _]},
+    {error, dgram_send_error, _}
+) ->
+    ConnState =/= connected;
+postcondition(
+    #{state := _ConnState},
+    {call, quicer, probe, [_, _]},
+    #probe_state{final = FinalState, final_at = FinalTs}
+) ->
+    FinalState =/= undefined andalso FinalTs =/= undefined;
+postcondition(
+    #{state := _ConnState},
+    {call, quicer, send_dgram, [_, _]},
+    {ok, _}
+) ->
+    true;
+postcondition(
+    #{state := ConnState},
+    {call, quicer, send_dgram, [_, _]},
+    {error, _, _}
+) ->
+    ConnState =/= connected;
+postcondition(
+    #{state := ConnState},
+    {call, quicer, send_dgram, [_, _]},
+    {error, _}
+) ->
+    ConnState =/= connected;
 postcondition(
     #{owner := _, state := connected},
     {call, quicer, controlling_process, [_, NewOwner]},
@@ -265,7 +298,8 @@ default_listen_opts() ->
         {handshake_idle_timeout_ms, 10000},
         % QUIC_SERVER_RESUME_AND_ZERORTT
         {server_resumption_level, 2},
-        {peer_bidi_stream_count, 10}
+        {peer_bidi_stream_count, 10},
+        {datagram_receive_enabled, 1}
     ].
 
 default_conn_opts() ->
@@ -276,5 +310,6 @@ default_conn_opts() ->
         {idle_timeout_ms, 0},
         {cacertfile, "./msquic/submodules/openssl/test/certs/rootCA.pem"},
         {certfile, "./msquic/submodules/openssl/test/certs/servercert.pem"},
-        {keyfile, "./msquic/submodules/openssl/test/certs/serverkey.pem"}
+        {keyfile, "./msquic/submodules/openssl/test/certs/serverkey.pem"},
+        {datagram_receive_enabled, 1}
     ].