Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[RFC]: Convert to direct-style with Eio #2149

Open
wants to merge 99 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
99 commits
Select commit Hold shift + click to select a range
00ee488
Convert Irmin from Lwt to direct-style with Eio
patricoferris Jun 5, 2022
6379145
Restore Test_tree.clear tree size
patricoferris Jun 5, 2022
4bc33c4
Fix map_p, filter_map_p and convert irmin-fs
patricoferris Jun 5, 2022
9bc6ab8
Add eio to opam file
patricoferris Jun 5, 2022
cfc7faf
Convert irmin-pack to direct-style
patricoferris Jun 5, 2022
4eb665f
Update to latest eio
patricoferris Sep 2, 2022
e9088a5
Working FS unix backend
patricoferris Oct 8, 2022
94233d2
Convert irmin-containers to direct-style
patricoferris Oct 8, 2022
99142a8
Remove pin-depends for eio
patricoferris Nov 29, 2022
169de7c
Fix irmin-fs file info
patricoferris Nov 29, 2022
ec6f097
Fix ignoring commit in store
patricoferris Nov 29, 2022
1a2a318
Fix generic-key tests
patricoferris Nov 29, 2022
adfa85e
Remove traceln
patricoferris Dec 9, 2022
0d484c1
Update to eio.0.7 and catch IO errors
patricoferris Jan 10, 2023
a077d36
Fix after rebase
art-w May 31, 2023
8fac611
Fix irmin-pack bench
art-w May 31, 2023
96af869
Fix irmin-pack examples
art-w May 31, 2023
536e7eb
Fix irmin-http using lwt_eio
art-w Jun 1, 2023
ed2cbd6
Fix irmin-graphql using lwt_eio
art-w Jun 2, 2023
c93a6e1
Fix irmin-git using lwt_eio
art-w Jun 2, 2023
7f4d26c
Fix irmin-cli using lwt_eio
art-w Jun 2, 2023
331095b
Fix irmin-mirage using lwt_eio
art-w Jun 2, 2023
1a5f58a
Fix examples using lwt_eio
art-w Jun 2, 2023
cee443f
Fix for libirmin
art-w Jun 5, 2023
e6da5e2
Fix opam dependencies to lwt_eio
art-w Jun 29, 2023
b30c33a
Add multicore tests
ElectreAAS Jun 5, 2023
426b76a
Add lock on branch
ElectreAAS May 30, 2023
151c25c
Fix LRU hashtable with a mutex
ElectreAAS Jun 5, 2023
2204a24
Fixed data race in stat counting
ElectreAAS Jun 5, 2023
791695a
Fixed data race in tree counters
ElectreAAS Jun 5, 2023
6afc95c
Fixed again stats, this time not in unix
ElectreAAS Jun 5, 2023
5b055c7
Update multicore testing
ElectreAAS Jun 12, 2023
2c2dffe
Added locks on unix/pack-store, in Tbl
ElectreAAS Jun 19, 2023
878983c
Added locks in irmin-pack atomic_write
ElectreAAS Jun 19, 2023
cff02a6
Added locks in control-file
ElectreAAS Jun 19, 2023
0043f79
Remove unsafe index stats in irmin-pack io
ElectreAAS Jun 19, 2023
df498e4
Added locks in store
ElectreAAS Jun 19, 2023
1a3678d
Safe pack_key promotion
art-w Jun 21, 2023
7972f9d
Optimize control file payload
art-w Jun 22, 2023
ea06897
Remove unsafe usage of Index.Stats
art-w Jun 22, 2023
c90c059
Fix Tree findv_cache
art-w Jun 23, 2023
37086e6
Fix Tree value cache
art-w Jun 23, 2023
8aeb8b8
Remove spinlock on Tree stats
art-w Jun 23, 2023
8bcd565
Fix irmin-pack inode partial_ptr
art-w Jun 23, 2023
37b211f
irmin-pack: add multicore unit tests
art-w Jul 3, 2023
ff6e5cd
Fix alcotest dependency
art-w Jul 3, 2023
d980b06
Fix irmin Tree lazy length
art-w Jul 3, 2023
55dcad5
Fix irmin Tree Contents mutables
art-w Jul 3, 2023
3607a58
Fix irmin proof
art-w Jul 3, 2023
bbaf30b
Fix irmin Tree add and remove
art-w Jul 3, 2023
c3cf319
Fix irmin-pack Append_only_file buffer length
art-w Jul 3, 2023
427a919
Fix irmin-pack Gc cancellation on close
art-w Jul 3, 2023
1a3249e
Fix irmin-pack Async Gc status handling
art-w Jul 3, 2023
f3a2c5c
Fix for append_only_file buffer length
art-w Jul 4, 2023
52969d9
Fix index-unix flushing from another thread
art-w Jul 4, 2023
c889be6
Fix irmin-pack Gc locks
art-w Jul 4, 2023
4f1b3da
Add commit unit test
art-w Jul 4, 2023
a888e7d
Fix irmin-pack batch in batch test
art-w Jul 4, 2023
1634e90
Fix irmin store mutable tree
art-w Jul 4, 2023
7b12d9f
Fix commit write lock
art-w Jul 5, 2023
c26e8eb
Fix parallel tree hash computation
art-w Jul 5, 2023
d086dad
irmin-pack: Add Store.Tree.list test
clecat Jul 5, 2023
24c3edd
irmin-pack: Add tests on commits for of_hash & parents
clecat Jul 5, 2023
dd1c41f
irmin-pack: Add Commit.v test & fix it with a lock
clecat Jul 5, 2023
17da4c8
Fix libirmin: at_exit executes outside of eio
art-w Jul 5, 2023
3d6dea1
Add half and full diamond multicore benchmarks
art-w Jul 10, 2023
5b9c002
Optimize Tree findv_cache for add/remove
art-w Jul 10, 2023
a1150e8
Fix formatting and remove old tests
art-w Jul 10, 2023
5404045
Fix irmin_fsck
art-w Jul 12, 2023
734d6e2
Fix opam dependencies
art-w Jul 12, 2023
1b959e9
eio: irmin-server using lwt_eio
art-w Aug 24, 2023
4f2d16a
irmin-pack: remove Io.Unix constraint
art-w Aug 21, 2023
dabb1fd
upgrade to eio.0.12
art-w Sep 4, 2023
5f550c7
irmin-pack: generic io lib without unix dependency
art-w Sep 6, 2023
b2f32d8
irmin-pack: extract unix Async
art-w Sep 6, 2023
a664ac8
irmin-pack: extract remaining Unix functions
art-w Sep 6, 2023
161310b
irmin-pack: fix Gc.on_finalise
art-w Sep 6, 2023
e42fb19
mirage: irmin-pack.unix is optional
art-w Sep 8, 2023
26a1497
irmin-pack.io: remove rusage dependency
art-w Sep 8, 2023
6e17751
mirage: remove dependency to fmt_tty
art-w Sep 8, 2023
ce0b76f
mirage: remove dependency to mtime.clock.os
art-w Sep 11, 2023
59deb7c
mirage: remove dependency to progress
art-w Sep 11, 2023
2fdb257
fix progress dependency injection
art-w Sep 20, 2023
62051a7
irmin-pack: use checkseum.c
art-w Oct 7, 2023
18af5b0
Update opam dependencies
art-w Sep 20, 2023
855be85
Fix irmin-client tests
art-w Feb 21, 2024
1e87714
Simplify concurrent list operations
art-w Feb 28, 2024
c7b22e3
Fix CI opam dependencies
art-w Mar 5, 2024
05a4af3
Upgrade dune and fix new warnings
art-w Mar 5, 2024
95b8a53
macos: reduce number of open files in unix-client test to avoid EMFIL…
art-w Mar 6, 2024
5905ef5
libirmin: fix compilation on ppc64
art-w Mar 6, 2024
883c706
Upgrade CI coverage to OCaml 5.1.x
clecat Feb 1, 2024
8601dce
Upgrade to Eio 1.0
art-w Mar 26, 2024
3fb0a99
Update CHANGES
art-w Apr 9, 2024
a8263db
Temporarily disable CI for specific platforms issues
art-w Apr 12, 2024
66c6144
Update opam for OCaml 5.1.0
art-w May 6, 2024
49e21c1
Update CI for OCaml 5.2
art-w Jun 25, 2024
7186af2
Fix multicore race in unit test
art-w Jun 25, 2024
aed087e
Fix Codecov CI: add secret token, disable on PRs
art-w Jul 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix irmin-git using lwt_eio
  • Loading branch information
art-w committed Feb 13, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit c93a6e18af5d1f9f73ab669d0b541f1c6d39aeb3
107 changes: 54 additions & 53 deletions src/irmin-git/atomic_write.ml
Original file line number Diff line number Diff line change
@@ -15,6 +15,8 @@
*)

open Import
open Lwt.Infix
open Lwt.Syntax
include Atomic_write_intf

module Check_closed (S : Irmin.Atomic_write.S) = struct
@@ -81,23 +83,23 @@ module Make (K : Key) (G : Git.S) = struct
module W = Irmin.Backend.Watch.Make (Key) (Val)

let handle_git_err = function
| Ok x -> Lwt.return x
| Error e -> Fmt.kstr Lwt.fail_with "%a" G.pp_error e
| Ok x -> x
| Error e -> Fmt.kstr failwith "%a" G.pp_error e

type t = {
bare : bool;
dot_git : Fpath.t;
git_head : G.Hash.t Git.Reference.contents;
t : G.t;
w : W.t;
m : Lwt_mutex.t;
m : Eio.Mutex.t;
}

let watches = Hashtbl.create 10

type key = Key.t
type value = Val.t
type watch = W.watch * (unit -> unit Lwt.t)
type watch = W.watch * (unit -> unit)

let branch_of_git r =
let str = String.trim @@ Git.Reference.to_string r in
@@ -107,6 +109,7 @@ module Make (K : Key) (G : Git.S) = struct
let pp_key = Irmin.Type.pp Key.t

let ref_read_opt t head =
Lwt_eio.run_lwt @@ fun () ->
(* Make a best-effort attempt to check that the reference actually
exists before [read]-ing it, since the [Error `Reference_not_found]
case causes a spurious warning to be logged inside [ocaml-git]. *)
@@ -122,10 +125,12 @@ module Make (K : Key) (G : Git.S) = struct
| Error e -> Fmt.kstr Lwt.fail_with "%a" G.pp_error e)

let mem { t; _ } r =
Lwt_eio.run_lwt @@ fun () ->
[%log.debug "mem %a" pp_key r];
G.Ref.mem t (git_of_branch r)

let find { t; _ } r =
Lwt_eio.run_lwt @@ fun () ->
[%log.debug "find %a" pp_key r];
let b = git_of_branch r in
let* exists = G.Ref.mem t b in
@@ -149,49 +154,50 @@ module Make (K : Key) (G : Git.S) = struct
None
in
W.listen_dir t.w dir ~key ~value:(find t)
else Lwt.return (fun () -> Lwt.return_unit)
else fun () -> ()

let watch_key t key ?init f =
[%log.debug "watch_key %a" pp_key key];
let* stop = listen_dir t in
let+ w = W.watch_key t.w key ?init f in
let stop = listen_dir t in
let w = W.watch_key t.w key ?init f in
(w, stop)

let watch t ?init f =
[%log.debug "watch"];
let* stop = listen_dir t in
let+ w = W.watch t.w ?init f in
let stop = listen_dir t in
let w = W.watch t.w ?init f in
(w, stop)

let unwatch t (w, stop) =
let* () = stop () in
stop ();
W.unwatch t.w w

let v ?lock ~head ~bare t =
let m = match lock with None -> Lwt_mutex.create () | Some l -> l in
let m = match lock with None -> Eio.Mutex.create () | Some l -> l in
let dot_git = G.dotgit t in
let write_head head =
let head = Git.Reference.Ref head in
let+ () =
let+ r =
let () =
let r =
if G.has_global_checkout then
Lwt_mutex.with_lock m (fun () ->
Eio.Mutex.use_rw ~protect:true m (fun () ->
Lwt_eio.run_lwt @@ fun () ->
G.Ref.write t Git.Reference.head head)
else Lwt.return (Ok ())
else Ok ()
in
match r with
| Error e -> [%log.err "Cannot create HEAD: %a" G.pp_error e]
| Ok () -> ()
in
head
in
let+ git_head =
let git_head =
match head with
| Some h -> write_head h
| None -> (
ref_read_opt t Git.Reference.head >>= function
match ref_read_opt t Git.Reference.head with
| None -> write_head (git_of_branch K.main)
| Some head -> Lwt.return head)
| Some head -> head)
in
let w =
try Hashtbl.find watches (G.dotgit t)
@@ -203,6 +209,7 @@ module Make (K : Key) (G : Git.S) = struct
{ git_head; bare; t; w; dot_git; m }

let list { t; _ } =
Lwt_eio.run_lwt @@ fun () ->
[%log.debug "list"];
let+ refs = G.Ref.list t in
List.fold_left
@@ -220,25 +227,26 @@ module Make (K : Key) (G : Git.S) = struct

(* FIXME G.write_index t.t gk *)
let _ = gk in
Lwt.return_unit)
else Lwt.return_unit
())

let pp_branch = Irmin.Type.pp K.t

let set t r k =
[%log.debug "set %a" pp_branch r];
let gr = git_of_branch r in
Lwt_mutex.with_lock t.m @@ fun () ->
let* e = G.Ref.write t.t gr (Git.Reference.Uid k) in
let* () = handle_git_err e in
let* () = W.notify t.w r (Some k) in
Eio.Mutex.use_rw ~protect:true t.m @@ fun () ->
let e =
Lwt_eio.run_lwt @@ fun () -> G.Ref.write t.t gr (Git.Reference.Uid k)
in
handle_git_err e;
W.notify t.w r (Some k);
write_index t gr k

let remove t r =
[%log.debug "remove %a" pp_branch r];
Lwt_mutex.with_lock t.m @@ fun () ->
let* e = G.Ref.remove t.t (git_of_branch r) in
let* () = handle_git_err e in
Eio.Mutex.use_rw ~protect:true t.m @@ fun () ->
let e = Lwt_eio.run_lwt @@ fun () -> G.Ref.remove t.t (git_of_branch r) in
let () = handle_git_err e in
W.notify t.w r None

let eq_head_contents_opt x y =
@@ -255,23 +263,23 @@ module Make (K : Key) (G : Git.S) = struct
let gr = git_of_branch r in
let c = function None -> None | Some h -> Some (Git.Reference.Uid h) in
let ok r =
let+ () = handle_git_err r in
handle_git_err r;
true
in
Lwt_mutex.with_lock t.m (fun () ->
let* x = ref_read_opt t.t gr in
let* b =
if not (eq_head_contents_opt x (c test)) then Lwt.return_false
Eio.Mutex.use_rw ~protect:true t.m (fun () ->
let x = ref_read_opt t.t gr in
let b =
if not (eq_head_contents_opt x (c test)) then false
else
match c set with
| None ->
let* r = G.Ref.remove t.t gr in
let r = Lwt_eio.run_lwt @@ fun () -> G.Ref.remove t.t gr in
ok r
| Some h ->
let* r = G.Ref.write t.t gr h in
let r = Lwt_eio.run_lwt @@ fun () -> G.Ref.write t.t gr h in
ok r
in
let* () =
let () =
if
(* We do not protect [write_index] because it can take a long
time and we don't want to hold the lock for too long. Would
@@ -280,29 +288,22 @@ module Make (K : Key) (G : Git.S) = struct
convenience for the user). *)
b
then W.notify t.w r set
else Lwt.return_unit
in
let+ () =
if b then
match set with
| None -> Lwt.return_unit
| Some v -> write_index t gr v
else Lwt.return_unit
let () =
if b then match set with None -> () | Some v -> write_index t gr v
in
b)

let close _ = Lwt.return_unit
let close _ = ()

let clear t =
[%log.debug "clear"];
Lwt_mutex.with_lock t.m (fun () ->
let* refs = G.Ref.list t.t in
Lwt_list.iter_p
(fun (r, _) ->
let* e = G.Ref.remove t.t r in
let* () = handle_git_err e in
match branch_of_git r with
| Some k -> W.notify t.w k None
| None -> Lwt.return_unit)
refs)
Eio.Mutex.use_rw ~protect:true t.m @@ fun () ->
let refs = Lwt_eio.run_lwt @@ fun () -> G.Ref.list t.t in
List.iter
(fun (r, _) ->
let e = Lwt_eio.run_lwt @@ fun () -> G.Ref.remove t.t r in
handle_git_err e;
match branch_of_git r with Some k -> W.notify t.w k None | None -> ())
refs
end
6 changes: 1 addition & 5 deletions src/irmin-git/atomic_write_intf.ml
Original file line number Diff line number Diff line change
@@ -28,11 +28,7 @@ module type Sigs = sig
include Irmin.Atomic_write.S with type key = K.t and type value = G.Hash.t

val v :
?lock:Lwt_mutex.t ->
head:G.Reference.t option ->
bare:bool ->
G.t ->
t Lwt.t
?lock:Eio.Mutex.t -> head:G.Reference.t option -> bare:bool -> G.t -> t
end

module Check_closed (S : Irmin.Atomic_write.S) : sig
20 changes: 12 additions & 8 deletions src/irmin-git/backend.ml
Original file line number Diff line number Diff line change
@@ -63,15 +63,15 @@ struct
module S = Atomic_write.Make (Schema.Branch) (G)
include Atomic_write.Check_closed (S)

let v ?lock ~head ~bare t = S.v ?lock ~head ~bare t >|= v
let v ?lock ~head ~bare t = S.v ?lock ~head ~bare t |> v
end

module Slice = Irmin.Backend.Slice.Make (Contents) (Node) (Commit)

module Repo = struct
let handle_git_err = function
| Ok x -> Lwt.return x
| Error e -> Fmt.kstr Lwt.fail_with "%a" G.pp_error e
| Ok x -> x
| Error e -> Fmt.kstr failwith "%a" G.pp_error e

type t = { config : Irmin.config; closed : bool ref; g : G.t; b : Branch.t }

@@ -106,24 +106,28 @@ struct
let { root; dot_git; head; bare; _ } = config conf in
let dotgit = fopt Fpath.v dot_git in
let root = Fpath.v root in
let* g = G.v ?dotgit root >>= handle_git_err in
let+ b = Branch.v ~head ~bare g in
let g = Lwt_eio.run_lwt @@ fun () -> G.v ?dotgit root in
let g = handle_git_err g in
let b = Branch.v ~head ~bare g in
{ g; b; closed = ref false; config = (conf :> Irmin.config) }

let config t = t.config
let close t = Branch.close t.b >|= fun () -> t.closed := true

let close t =
Branch.close t.b;
t.closed := true
end

module Remote = struct
include Remote.Make (G) (S) (Schema.Branch)

let v repo = Lwt.return repo.Repo.g
let v repo = repo.Repo.g
end

let git_of_repo r = r.Repo.g

let repo_of_git ?head ?(bare = true) ?lock g =
let+ b = Branch.v ?lock ~head ~bare g in
let b = Branch.v ?lock ~head ~bare g in
{
Repo.config = Irmin.Backend.Conf.empty Conf.spec;
closed = ref false;
6 changes: 1 addition & 5 deletions src/irmin-git/backend.mli
Original file line number Diff line number Diff line change
@@ -43,9 +43,5 @@ module Make
val git_of_repo : Repo.t -> G.t

val repo_of_git :
?head:Git.Reference.t ->
?bare:bool ->
?lock:Lwt_mutex.t ->
G.t ->
Repo.t Lwt.t
?head:Git.Reference.t -> ?bare:bool -> ?lock:Eio.Mutex.t -> G.t -> Repo.t
end
13 changes: 8 additions & 5 deletions src/irmin-git/content_addressable.ml
Original file line number Diff line number Diff line change
@@ -15,6 +15,8 @@
*)

open Import
open Lwt.Infix
open Lwt.Syntax
include Content_addressable_intf

module Make (G : Git.S) (V : Value.S with type value := G.Value.t) = struct
@@ -29,6 +31,7 @@ module Make (G : Git.S) (V : Value.S with type value := G.Value.t) = struct
type value = V.t

let mem t key =
Lwt_eio.run_lwt @@ fun () ->
[%log.debug "mem %a" pp_key key];
G.mem t key >>= function
| false -> Lwt.return_false
@@ -39,28 +42,30 @@ module Make (G : Git.S) (V : Value.S with type value := G.Value.t) = struct
| Ok v -> Lwt.return (V.type_eq (G.Value.kind v)))

let find t key =
Lwt_eio.run_lwt @@ fun () ->
[%log.debug "find %a" pp_key key];
G.read t key >>= function
| Error (`Reference_not_found _ | `Not_found _) -> Lwt.return_none
| Error e -> Fmt.kstr Lwt.fail_with "%a" G.pp_error e
| Ok v -> Lwt.return (V.of_git v)

let add t v =
Lwt_eio.run_lwt @@ fun () ->
let v = V.to_git v in
let* k, _ = G.write t v >>= handle_git_err in
[%log.debug "add %a" pp_key k];
Lwt.return k

let unsafe_add t k v =
let+ k' = add t v in
let k' = add t v in
if equal_key k k' then ()
else
Fmt.failwith
"[Git.unsafe_append] %a is not a valid key. Expecting %a instead.\n"
pp_key k pp_key k'

let batch t f = f t
let close _ = Lwt.return ()
let close _ = ()
end

module Check_closed (S : Irmin.Content_addressable.S) = struct
@@ -90,7 +95,5 @@ module Check_closed (S : Irmin.Content_addressable.S) = struct
check_not_closed t;
S.batch (snd t) (fun x -> f (fst t, x))

let close (c, _) =
c := true;
Lwt.return ()
let close (c, _) = c := true
end
14 changes: 13 additions & 1 deletion src/irmin-git/dune
Original file line number Diff line number Diff line change
@@ -1,7 +1,19 @@
(library
(name irmin_git)
(public_name irmin-git)
(libraries astring cstruct fmt fpath git irmin logs lwt uri irmin.mem mimic)
(libraries
astring
cstruct
fmt
fpath
git
irmin
logs
lwt
lwt_eio
uri
irmin.mem
mimic)
(preprocess
(pps ppx_irmin.internal))
(instrumentation
Loading