Use variant types in many places

Jane Street have requested that Eio not use objects. This commit
switches to an alternative scheme for representing OS resources using
variants instead. The changes for users of the library are minimal -
only the types change. The exception to this is if you want to provide
your own implementations of resources, in which case you now provide a
module rather than a class. The (small) changes to the README give a
good idea of the user-facing effect.

README.md

@@ -1524,19 +1524,26 @@ See Eio's own tests for examples, e.g., [tests/switch.md](tests/switch.md).
 ## Provider Interfaces
 
 Eio applications use resources by calling functions (such as `Eio.Flow.write`).
-These functions are actually wrappers that call methods on the resources.
+These functions are actually wrappers that look up the implementing module and call
+the appropriate function on that.
 This allows you to define your own resources.
 
 Here's a flow that produces an endless stream of zeros (like "/dev/zero"):
 
 ```ocaml
-let zero = object
-  inherit Eio.Flow.source
+module Zero = struct
+  type t = unit
 
-  method read_into buf =
+  let single_read () buf = 
     Cstruct.memset buf 0;
     Cstruct.length buf
+
+  let read_methods = []         (* Optional optimisations *)
 end
+
+let ops = Eio.Flow.Pi.source (module Zero)
+
+let zero = Eio.Resource.T ((), ops)
 ```
 
 It can then be used like any other Eio flow:
@@ -1549,34 +1556,6 @@ It can then be used like any other Eio flow:
 - : unit = ()
 ```
 
-The `Flow.source` interface has some extra methods that can be used for optimisations
-(for example, instead of filling a buffer with zeros it could be more efficient to share
-a pre-allocated block of zeros).
-Using `inherit` provides default implementations of these methods that say no optimisations are available.
-It also protects you somewhat from API changes in future, as defaults can be provided for any new methods that get added.
-
-Although it is possible to *use* an object by calling its methods directly,
-it is recommended that you use the functions instead.
-The functions provide type information to the compiler, leading to clearer error messages,
-and may provide extra features or sanity checks.
-
-For example `Eio.Flow.single_read` is defined as:
-
-```ocaml
-let single_read (t : #Eio.Flow.source) buf =
-  let got = t#read_into buf in
-  assert (got > 0 && got <= Cstruct.length buf);
-  got
-```
-
-As an exception to this rule, it is fine to use the methods of `env` directly
-(e.g. using `main env#stdin` instead of `main (Eio.Stdenv.stdin env)`.
-Here, the compiler already has the type from the `Eio_main.run` call immediately above it,
-and `env` is acting as a simple record.
-We avoid doing that in this guide only to avoid alarming OCaml users unfamiliar with object syntax.
-
-See [Dynamic Dispatch](doc/rationale.md#dynamic-dispatch) for more discussion about the use of objects here.
-
 ## Example Applications
 
 - [gemini-eio][] is a simple Gemini browser. It shows how to integrate Eio with `ocaml-tls` and `notty`.
@@ -1729,9 +1708,8 @@ Of course, you could use `with_open_in` in this case to simplify it further.
 
 ### Casting
 
-Unlike many languages, OCaml does not automatically cast objects (polymorphic records) to super-types as needed.
+Unlike many languages, OCaml does not automatically cast to super-types as needed.
 Remember to keep the type polymorphic in your interface so users don't need to do this manually.
-This is similar to the case with polymorphic variants (where APIs should use `[< ...]` or `[> ...]`).
 
 For example, if you need an `Eio.Flow.source` then users should be able to use a `Flow.two_way`
 without having to cast it first:
@@ -1741,13 +1719,13 @@ without having to cast it first:
 (* BAD - user must cast to use function: *)
 module Message : sig
   type t
-  val read : Eio.Flow.source -> t
+  val read : Eio.Flow.source_ty r -> t
 end
 
 (* GOOD - a Flow.two_way can be used without casting: *)
 module Message : sig
   type t
-  val read : #Eio.Flow.source -> t
+  val read : _ Eio.Flow.source -> t
 end
 ```
 
@@ -1756,20 +1734,18 @@ If you want to store the argument, this may require you to cast internally:
 ```ocaml
 module Foo : sig
   type t
-  val of_source : #Eio.Flow.source -> t
+  val of_source : _ Eio.Flow.source -> t
 end = struct
   type t = {
-    src : Eio.Flow.source;
+    src : Eio.Flow.source_ty r;
   }
 
   let of_source x = {
-    src = (x :> Eio.Flow.source);
+    src = (x :> Eio.Flow.source_ty r);
   }
 end
 ```
 
-Note: the `#type` syntax only works on types defined by classes, whereas the slightly more verbose `<type; ..>` works on all object types.
-
 ### Passing env
 
 The `env` value you get from `Eio_main.run` is a powerful capability,

doc/prelude.ml

@@ -43,4 +43,4 @@ module Eio_main = struct
     end
 end
 
-let parse_config (flow : #Eio.Flow.source) = ignore
+let parse_config (flow : _ Eio.Flow.source) = ignore

doc/rationale.md

@@ -125,32 +125,27 @@ For dynamic dispatch with subtyping, objects seem to be the best choice:
   An object uses a single block to store the object's fields and a pointer to the shared method table.
 
 - First-class modules and GADTs are an advanced feature of the language.
-  The new users we hope to attract to OCaml 5.00 are likely to be familiar with objects already.
+  The new users we hope to attract to OCaml 5.0 are likely to be familiar with objects already.
 
 - It is possible to provide base classes with default implementations of some methods.
   This can allow adding new operations to the API in future without breaking existing providers.
 
 In general, simulating objects using other features of the language leads to worse performance
 and worse ergonomics than using the language's built-in support.
 
-In Eio, we split the provider and consumer APIs:
-
-- To *provide* a flow, you implement an object type.
-- To *use* a flow, you call a function (e.g. `Flow.close`).
-
-The functions mostly just call the corresponding method on the object.
-If you call object methods directly in OCaml then you tend to get poor compiler error messages.
-This is because OCaml can only refer to the object types by listing the methods you seem to want to use.
-Using functions avoids this, because the function signature specifies the type of its argument,
-allowing type inference to work as for non-object code.
-In this way, users of Eio can be largely unaware that objects are being used at all.
-
-The function wrappers can also provide extra checks that the API is being followed correctly,
-such as asserting that a read does not return 0 bytes,
-or add extra convenience functions without forcing every implementor to add them too.
-
-Note that the use of objects in Eio is not motivated by the use of the "Object Capabilities" security model.
-Despite the name, that is not specific to objects at all.
+However, in order for Eio to be widely accepted in the OCaml community,
+we no longer use of objects and instead use a pair of a value and a function for looking up interfaces.
+There is a problem here, because each interface has a different type,
+so the function's return type depends on its input (the interface ID).
+This requires using a GADT. However, GADT's don't support sub-typing.
+To get around this, we use an extensible GADT to get the correct typing
+(but which will raise an exception if the interface isn't supported),
+and then wrap this with a polymorphic variant phantom type to help ensure
+it is used correctly.
+
+This system gives the same performance as using objects and without requiring allocation.
+However, care is needed when defining new interfaces,
+since the compiler can't check that the resource really implements all the interfaces its phantom type suggests.
 
 ## Results vs Exceptions
 

fuzz/fuzz_buf_read.ml

@@ -26,26 +26,30 @@ exception Buffer_limit_exceeded = Buf_read.Buffer_limit_exceeded
 let initial_size = 10
 let max_size = 100
 
-let mock_flow next = object (self)
-  inherit Eio.Flow.source
+module Mock_flow = struct
+  type t = string list ref
 
-  val mutable next = next
-
-  method read_into buf =
-    match next with
+  let rec single_read t buf =
+    match !t with
     | [] ->
       raise End_of_file
     | "" :: xs ->
-      next <- xs;
-      self#read_into buf
+      t := xs;
+      single_read t buf
     | x :: xs ->
       let len = min (Cstruct.length buf) (String.length x) in
       Cstruct.blit_from_string x 0 buf 0 len;
       let x' = String.drop x len in
-      next <- (if x' = "" then xs else x' :: xs);
+      t := (if x' = "" then xs else x' :: xs);
       len
+
+  let read_methods = []
 end
 
+let mock_flow =
+  let ops = Eio.Flow.Pi.source (module Mock_flow) in
+  fun chunks -> Eio.Resource.T (ref chunks, ops)
+
 module Model = struct
   type t = string ref
 

lib_eio/buf_read.ml

@@ -1,11 +1,13 @@
 exception Buffer_limit_exceeded
 
+open Std
+
 type t = {
   mutable buf : Cstruct.buffer;
   mutable pos : int;
   mutable len : int;
-  mutable flow : Flow.source option;    (* None if we've seen eof *)
-  mutable consumed : int;               (* Total bytes consumed so far *)
+  mutable flow : Flow.source_ty r option;    (* None if we've seen eof *)
+  mutable consumed : int;                    (* Total bytes consumed so far *)
   max_size : int;
 }
 
@@ -45,7 +47,7 @@ open Syntax
 let capacity t = Bigarray.Array1.dim t.buf
 
 let of_flow ?initial_size ~max_size flow =
-  let flow = (flow :> Flow.source) in
+  let flow = (flow :> Flow.source_ty r) in
   if max_size <= 0 then Fmt.invalid_arg "Max size %d should be positive!" max_size;
   let initial_size = Option.value initial_size ~default:(min 4096 max_size) in
   let buf = Bigarray.(Array1.create char c_layout initial_size) in
@@ -128,17 +130,22 @@ let ensure_slow_path t n =
 let ensure t n =
   if t.len < n then ensure_slow_path t n
 
-let as_flow t =
-  object
-    inherit Flow.source
+module F = struct
+  type nonrec t = t
 
-    method read_into dst =
-      ensure t 1;
-      let len = min (buffered_bytes t) (Cstruct.length dst) in
-      Cstruct.blit (peek t) 0 dst 0 len;
-      consume t len;
-      len
-  end
+  let single_read t dst =
+    ensure t 1;
+    let len = min (buffered_bytes t) (Cstruct.length dst) in
+    Cstruct.blit (peek t) 0 dst 0 len;
+    consume t len;
+    len
+
+  let read_methods = []
+end
+
+let as_flow =
+  let ops = Flow.Pi.source (module F) in
+  fun t -> Resource.T (t, ops)
 
 let get t i =
   Bigarray.Array1.get t.buf (t.pos + i)

lib_eio/buf_read.mli

@@ -9,6 +9,8 @@
     ]}
 *)
 
+open Std
+
 type t
 (** An input buffer. *)
 
@@ -21,7 +23,7 @@ type 'a parser = t -> 'a
     @raise End_of_file The flow ended without enough data to parse an ['a].
     @raise Buffer_limit_exceeded Parsing the value would exceed the configured size limit. *)
 
-val parse : ?initial_size:int -> max_size:int -> 'a parser -> #Flow.source -> ('a, [> `Msg of string]) result
+val parse : ?initial_size:int -> max_size:int -> 'a parser -> _ Flow.source -> ('a, [> `Msg of string]) result
 (** [parse p flow ~max_size] uses [p] to parse everything in [flow].
 
     It is a convenience function that does
@@ -32,7 +34,7 @@ val parse : ?initial_size:int -> max_size:int -> 'a parser -> #Flow.source -> ('
 
     @param initial_size see {!of_flow}. *)
 
-val parse_exn : ?initial_size:int -> max_size:int -> 'a parser -> #Flow.source -> 'a
+val parse_exn : ?initial_size:int -> max_size:int -> 'a parser -> _ Flow.source -> 'a
 (** [parse_exn] wraps {!parse}, but raises [Failure msg] if that returns [Error (`Msg msg)].
 
     Catching exceptions with [parse] and then raising them might seem pointless,
@@ -46,7 +48,7 @@ val parse_string : 'a parser -> string -> ('a, [> `Msg of string]) result
 val parse_string_exn : 'a parser -> string -> 'a
 (** [parse_string_exn] is like {!parse_string}, but handles errors like {!parse_exn}. *)
 
-val of_flow : ?initial_size:int -> max_size:int -> #Flow.source -> t
+val of_flow : ?initial_size:int -> max_size:int -> _ Flow.source -> t
 (** [of_flow ~max_size flow] is a buffered reader backed by [flow].
 
     @param initial_size The initial amount of memory to allocate for the buffer.
@@ -68,7 +70,7 @@ val of_buffer : Cstruct.buffer -> t
 val of_string : string -> t
 (** [of_string s] is a reader that reads from [s]. *)
 
-val as_flow : t -> Flow.source
+val as_flow : t -> Flow.source_ty r
 (** [as_flow t] is a buffered flow.
 
     Reading from it will return data from the buffer,

lib_eio/buf_write.mli

@@ -85,7 +85,7 @@ exception Flush_aborted
 
 (** {2 Running} *)
 
-val with_flow : ?initial_size:int -> #Flow.sink -> (t -> 'a) -> 'a
+val with_flow : ?initial_size:int -> _ Flow.sink -> (t -> 'a) -> 'a
 (** [with_flow flow fn] runs [fn writer], where [writer] is a buffer that flushes to [flow].
 
     Concurrently with [fn], it also runs a fiber that copies from [writer] to [flow].

lib_eio/eio.ml

@@ -3,19 +3,13 @@ include Eio__core
 module Debug = Private.Debug
 let traceln = Debug.traceln
 
-module Std = struct
-  module Promise = Promise
-  module Fiber = Fiber
-  module Switch = Switch
-  let traceln = Debug.traceln
-end
-
+module Std = Std
 module Semaphore = Semaphore
 module Mutex = Eio_mutex
 module Condition = Condition
 module Stream = Stream
 module Exn = Exn
-module Generic = Generic
+module Resource = Resource
 module Flow = Flow
 module Buf_read = Buf_read
 module Buf_write = Buf_write
@@ -28,17 +22,17 @@ module Fs = Fs
 module Path = Path
 
 module Stdenv = struct
-  let stdin  (t : <stdin  : #Flow.source; ..>) = t#stdin
-  let stdout (t : <stdout : #Flow.sink;   ..>) = t#stdout
-  let stderr (t : <stderr : #Flow.sink;   ..>) = t#stderr
-  let net (t : <net : #Net.t; ..>) = t#net
+  let stdin  (t : <stdin  : _ Flow.source; ..>) = t#stdin
+  let stdout (t : <stdout : _ Flow.sink;   ..>) = t#stdout
+  let stderr (t : <stderr : _ Flow.sink;   ..>) = t#stderr
+  let net (t : <net : _ Net.t; ..>) = t#net
   let process_mgr (t : <process_mgr : #Process.mgr; ..>) = t#process_mgr
   let domain_mgr (t : <domain_mgr : #Domain_manager.t; ..>) = t#domain_mgr
   let clock (t : <clock : #Time.clock; ..>) = t#clock
   let mono_clock (t : <mono_clock : #Time.Mono.t; ..>) = t#mono_clock
-  let secure_random (t: <secure_random : #Flow.source; ..>) = t#secure_random
-  let fs (t : <fs : #Fs.dir Path.t; ..>) = t#fs
-  let cwd (t : <cwd : #Fs.dir Path.t; ..>) = t#cwd
+  let secure_random (t: <secure_random : _ Flow.source; ..>) = t#secure_random
+  let fs (t : <fs : _ Path.t; ..>) = t#fs
+  let cwd (t : <cwd : _ Path.t; ..>) = t#cwd
   let debug (t : <debug : 'a; ..>) = t#debug
   let backend_id (t: <backend_id : string; ..>) = t#backend_id
 end