diff --git a/irmin.opam b/irmin.opam index 5619596f81..ba3544d83e 100644 --- a/irmin.opam +++ b/irmin.opam @@ -22,6 +22,8 @@ depends: [ "uutf" "jsonm" {>= "1.0.0"} "eio" {>= "1.0"} + "kcas" {= "dev"} + "kcas_data" {= "dev"} "lwt" {>= "5.6.1"} "digestif" {>= "0.9.0"} "ocamlgraph" @@ -41,6 +43,12 @@ depends: [ ] pin-depends: [ + # Fix segv in kcas + [ "kcas.dev" "git+https://git@github.com/ocaml-multicore/kcas#5f3a39dfc72189e2b83f96c3754d402d5e7d6bc5"] + [ "kcas_data.dev" "git+https://git@github.com/ocaml-multicore/kcas#5f3a39dfc72189e2b83f96c3754d402d5e7d6bc5"] + # Metrics may have been unnecessarily constrained in opam-repository + [ "metrics.dev" "git+https://github.com/mirage/metrics#995eb18d2837df02c8ead719c00fb156cf475ab5"] + [ "metrics-unix.dev" "git+https://github.com/mirage/metrics#995eb18d2837df02c8ead719c00fb156cf475ab5"] # Fix race in formatters [ "alcotest.dev" "git+https://github.com/haesbaert/alcotest#99030be1df12a17fc2a0f3d00b181cd31cf6b0d4" ] ] diff --git a/src/irmin-pack/io/atomic_write.ml b/src/irmin-pack/io/atomic_write.ml index fd613893f4..b309d1570e 100644 --- a/src/irmin-pack/io/atomic_write.ml +++ b/src/irmin-pack/io/atomic_write.ml @@ -1,45 +1,19 @@ open Import include Irmin_pack.Atomic_write -module UnsafeTbl (K : Irmin.Type.S) = Hashtbl.Make (struct - type t = K.t [@@deriving irmin ~short_hash ~equal] - - let hash = short_hash ?seed:None -end) - -(** Safe but might be incredibly slow. *) module Table (K : Irmin.Type.S) = struct - module Unsafe = UnsafeTbl (K) - - type 'a t = { lock : Eio.Mutex.t; data : 'a Unsafe.t } - - let create n = - let lock = Eio.Mutex.create () in - let data = Unsafe.create n in - { lock; data } - - let add { lock; data } k v = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.add data k v - - let mem { lock; data } k = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.mem data k - - let find_opt { lock; data } k = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.find_opt data k - - let find t k = match find_opt t k with Some v -> v | None -> raise Not_found + module K = struct + include K - let replace { lock; data } k v = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.replace data k v + type t = K.t [@@deriving irmin ~short_hash ~equal] - let remove { lock; data } k = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.remove data k + let hash = short_hash ?seed:None + let equal = Irmin.Type.(unstage (equal K.t)) + end - let reset { lock; data } = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.reset data + include Kcas_data.Hashtbl - let fold f { lock; data } init = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.fold f data init + let create min_buckets = create ~hashed_type:(module K) ~min_buckets () end module Make_persistent (Io : Io_intf.S) (K : Irmin.Type.S) (V : Value.S) = @@ -53,8 +27,8 @@ struct type watch = W.watch type t = { - index : int63 Tbl.t; - cache : V.t Tbl.t; + index : (K.t, int63) Tbl.t; + cache : (K.t, V.t) Tbl.t; block : Io.t; mutable block_size : int63; w : W.t; diff --git a/src/irmin-pack/io/pack_store.ml b/src/irmin-pack/io/pack_store.ml index 8563a87ae4..bb80bbc2c0 100644 --- a/src/irmin-pack/io/pack_store.ml +++ b/src/irmin-pack/io/pack_store.ml @@ -24,37 +24,17 @@ exception Dangling_hash let invalid_read fmt = Fmt.kstr (fun s -> raise (Invalid_read s)) fmt let corrupted_store fmt = Fmt.kstr (fun s -> raise (Corrupted_store s)) fmt -module UnsafeTbl (K : Irmin.Hash.S) = Hashtbl.Make (struct - type t = K.t - - let hash = K.short_hash - let equal = Irmin.Type.(unstage (equal K.t)) -end) - -(** Safe but might be incredibly slow. *) module Table (K : Irmin.Hash.S) = struct - module Unsafe = UnsafeTbl (K) - - type 'a t = { lock : Eio.Mutex.t; data : 'a Unsafe.t } - - let create n = - let lock = Eio.Mutex.create () in - let data = Unsafe.create n in - { lock; data } + module K = struct + include K - let add { lock; data } k v = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.add data k v - - let mem { lock; data } k = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.mem data k - - let find_opt { lock; data } k = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.find_opt data k + let hash = short_hash + let equal = Irmin.Type.(unstage (equal K.t)) + end - let find t k = match find_opt t k with Some v -> v | None -> raise Not_found + include Kcas_data.Hashtbl - let clear { lock; data } = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.clear data + let create min_buckets = create ~hashed_type:(module K) ~min_buckets () end module Make_without_close_checks @@ -86,7 +66,7 @@ struct type 'a t = { lru : Lru.t; - staging : Val.t Tbl.t; + staging : (Hash.t, Val.t) Tbl.t; indexing_strategy : Irmin_pack.Indexing_strategy.t; fm : Fm.t; dict : Dict.t; diff --git a/src/irmin/dune b/src/irmin/dune index 2e031731df..ed84dccfcc 100644 --- a/src/irmin/dune +++ b/src/irmin/dune @@ -11,6 +11,8 @@ logs logs.fmt eio + kcas + kcas_data mtime ocamlgraph uri diff --git a/src/irmin/lru.ml b/src/irmin/lru.ml index 4ae221f251..4938c6f2f0 100644 --- a/src/irmin/lru.ml +++ b/src/irmin/lru.ml @@ -16,174 +16,148 @@ (* Extracted from https://github.com/pqwy/lru *) -module MakeUnsafe (H : Hashtbl.HashedType) = struct - module HT = Hashtbl.Make (H) +open Kcas + +module Make (H : Hashtbl.HashedType) = struct + module HT = Kcas_data.Hashtbl module Q = struct type 'a node = { value : 'a; - mutable next : 'a node option; - mutable prev : 'a node option; + next : 'a node option Loc.t; + prev : 'a node option Loc.t; } - type 'a t = { - mutable first : 'a node option; - mutable last : 'a node option; - } + type 'a t = { tail : 'a node option Loc.t; head : 'a node option Loc.t } - let detach t n = - let np = n.prev and nn = n.next in + let detach ~xt t n = + let np = Xt.get ~xt n.prev and nn = Xt.get ~xt n.next in (match np with - | None -> t.first <- nn + | None -> Xt.set ~xt t.tail nn | Some x -> - x.next <- nn; - n.prev <- None); + Xt.set ~xt x.next nn; + Xt.set ~xt n.prev None); match nn with - | None -> t.last <- np + | None -> Xt.set ~xt t.head np | Some x -> - x.prev <- np; - n.next <- None + Xt.set ~xt x.prev np; + Xt.set ~xt n.next None - let append t n = + let append ~xt t n = let on = Some n in - match t.last with + let hd = Xt.get ~xt t.head in + match hd with | Some x as l -> - x.next <- on; - t.last <- on; - n.prev <- l + Xt.set ~xt x.next on; + Xt.set ~xt t.head on; + Xt.set ~xt n.prev l | None -> - t.first <- on; - t.last <- on - - let node x = { value = x; prev = None; next = None } - let create () = { first = None; last = None } - - let iter t f = - let rec aux f = function - | Some n -> - let next = n.next in - f n.value; - aux f next - | _ -> () - in - aux f t.first + Xt.set ~xt t.tail on; + Xt.set ~xt t.head on - let clear t = - t.first <- None; - t.last <- None + let node x = { value = x; prev = Loc.make None; next = Loc.make None } + let create () = { tail = Loc.make None; head = Loc.make None } + + let clear ~xt t = + Xt.set ~xt t.tail None; + Xt.set ~xt t.head None end - type key = HT.key + type key = H.t type 'a t = { - ht : (key * 'a) Q.node HT.t; + ht : (key, (key * 'a) Q.node) HT.t; q : (key * 'a) Q.t; cap : cap; - mutable w : int; + w : int Loc.t; } and cap = Uncapped | Capped of int - let weight t = t.w + let weight ~xt t = Xt.get ~xt t.w let create cap = let cap, ht_cap = if cap < 0 then (Uncapped, 65536) else (Capped cap, cap) in - { cap; w = 0; ht = HT.create ht_cap; q = Q.create () } + { + cap; + w = Loc.make 0; + ht = HT.create ~hashed_type:(module H) ~min_buckets:ht_cap (); + q = Q.create (); + } - let drop t = - match t.q.first with + let drop ~xt t = + let tl = Xt.get ~xt t.q.tail in + match tl with | None -> None | Some ({ Q.value = k, v; _ } as n) -> - t.w <- t.w - 1; - HT.remove t.ht k; - Q.detach t.q n; + Xt.modify ~xt t.w (fun tw -> tw - 1); + HT.Xt.remove ~xt t.ht k; + Q.detach ~xt t.q n; Some v - let remove t k = - try - let n = HT.find t.ht k in - t.w <- t.w - 1; - HT.remove t.ht k; - Q.detach t.q n - with Not_found -> () + let remove ~xt t k = + match HT.Xt.find_opt ~xt t.ht k with + | None -> () + | Some n -> + Xt.modify ~xt t.w (fun tw -> tw - 1); + HT.Xt.remove ~xt t.ht k; + Q.detach ~xt t.q n let add t k v = - let add t k v = - remove t k; - let n = Q.node (k, v) in - t.w <- t.w + 1; - HT.add t.ht k n; - Q.append t.q n + let tx ~xt = + let add t k v = + remove ~xt t k; + let n = Q.node (k, v) in + Xt.modify ~xt t.w (fun tw -> tw + 1); + HT.Xt.replace ~xt t.ht k n; + Q.append ~xt t.q n + in + match t.cap with + | Capped c when c = 0 -> () + | Uncapped -> add t k v + | Capped c -> + add t k v; + if weight ~xt t > c then + let _ = drop ~xt t in + () in - match t.cap with - | Capped c when c = 0 -> () - | Uncapped -> add t k v - | Capped c -> - add t k v; - if weight t > c then - let _ = drop t in - () - - let promote t k = - try - let n = HT.find t.ht k in - Q.( - detach t.q n; - append t.q n) - with Not_found -> () - - let find_opt t k = - match HT.find_opt t.ht k with - | Some v -> - promote t k; - Some (snd v.value) - | None -> None - - let mem t k = - match HT.mem t.ht k with - | false -> false - | true -> - promote t k; - true + Xt.commit { tx } - let iter t f = Q.iter t.q (fun (k, v) -> f k v) + let drop t = Xt.commit { tx = drop t } - let clear t = - t.w <- 0; - HT.clear t.ht; - Q.clear t.q -end + let promote ~xt t n = + Q.detach ~xt t.q n; + Q.append ~xt t.q n -(** Safe but might be incredibly slow. *) -module Make (H : Hashtbl.HashedType) = struct - module Unsafe = MakeUnsafe (H) - - type 'a t = { lock : Eio.Mutex.t; data : 'a Unsafe.t } - - let create cap = - let lock = Eio.Mutex.create () in - let data = Unsafe.create cap in - { lock; data } - - let add { lock; data } k v = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.add data k v - - let find_opt { lock; data } k = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.find_opt data k - - let find t k = match find_opt t k with Some v -> v | None -> raise Not_found - - let mem { lock; data } k = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.mem data k + let find t k = + let tx ~xt = + match HT.Xt.find_opt ~xt t.ht k with + | Some v -> + promote ~xt t v; + snd v.value + | None -> raise Not_found + in + Xt.commit { tx } - let iter { lock; data } f = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.iter data f + let mem t k = + let tx ~xt = + match HT.Xt.find_opt ~xt t.ht k with + | None -> false + | Some v -> + promote ~xt t v; + true + in + Xt.commit { tx } - let clear { lock; data } = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.clear data + let clear t = + let tx ~xt = + Xt.set ~xt t.w 0; + HT.Xt.clear ~xt t.ht; + Q.clear ~xt t.q + in + Xt.commit { tx } - let drop { lock; data } = - Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.drop data + let iter t f = HT.iter (fun k q -> f k (snd q.Q.value)) t.ht end