diff --git a/README.md b/README.md index 357f9f7..8e88f21 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,12 @@ If you'd like to see the options for the benchmarking tool, just run `lein bench See the `doc/` folder for technical details of the hitchhiker tree and Redis garbage collection system. +### Async support + +We have preliminary async support that has to be selected before macro expansion +time by setting `hitchhiker.tree.async/*async-backend*` either to `none` or +`core.async`. + ## Gratitude Thanks to the early reviewers, Kovas Boguta & Leif Walsh. diff --git a/project.clj b/project.clj index 3203edc..5307388 100644 --- a/project.clj +++ b/project.clj @@ -1,17 +1,17 @@ -(defproject hitchhiker-tree "0.1.0-SNAPSHOT" +(defproject io.replikativ/hitchhiker-tree "0.1.2" :description "A Hitchhiker Tree Library" :url "https://github.com/dgrnbrg/hitchhiker-tree" :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :dependencies [[org.clojure/clojure "1.8.0"] + :dependencies [[org.clojure/clojure "1.9.0"] [org.clojure/clojurescript "1.8.51" :scope "provided"] [org.clojure/core.memoize "0.5.8"] [com.taoensso/carmine "2.12.2"] [org.clojure/core.rrb-vector "0.0.11"] [org.clojure/core.cache "0.6.5"] - [io.replikativ/incognito "0.2.2-SNAPSHOT"] - [io.replikativ/konserve "0.4.9"]] + [io.replikativ/konserve "0.5.0-beta3"] + ] :aliases {"bench" ["with-profile" "profiling" "run" "-m" "hitchhiker.bench"]} :jvm-opts ["-server" "-Xmx3700m" "-Xms3700m"] :profiles {:test @@ -23,15 +23,18 @@ [org.clojure/tools.cli "0.3.3"] [org.clojure/test.check "0.9.0"] [com.infolace/excel-templates "0.3.3"]]} - :dev {:dependencies [[binaryage/devtools "0.8.2"] - [figwheel-sidecar "0.5.8"] - [com.cemerick/piggieback "0.2.1"] - [org.clojure/test.check "0.9.0"]] + :dev {:dependencies [#_[binaryage/devtools "0.8.2"] + #_[figwheel-sidecar "0.5.8"] + #_[com.cemerick/piggieback "0.2.1"] + [org.clojure/test.check "0.9.0"] + ;; plotting + [aysylu/loom "1.0.1"] + [cheshire "5.8.0"]] :source-paths ["src" "dev"] - :plugins [[lein-figwheel "0.5.8"]] + ;:plugins [[lein-figwheel "0.5.8"]] :repl-options {; for nREPL dev you really need to limit output :init (set! *print-length* 50) - :nrepl-middleware [cemerick.piggieback/wrap-cljs-repl]}}} + #_:nrepl-middleware #_[cemerick.piggieback/wrap-cljs-repl]}}} :clean-targets ^{:protect false} ["resources/public/js/compiled" "target"] diff --git a/src/hitchhiker/konserve.cljc b/src/hitchhiker/konserve.cljc index eeed82e..2a05638 100644 --- a/src/hitchhiker/konserve.cljc +++ b/src/hitchhiker/konserve.cljc @@ -3,13 +3,16 @@ (:require [clojure.core.rrb-vector :refer [catvec subvec]] #?(:clj [clojure.core.async :refer [chan promise-chan put!] :as async] :cljs [cljs.core.async :refer [chan promise-chan put!] :as async]) - [konserve.core :as k] + [konserve.cache :as k] + #?(:clj [clojure.core.cache :as cache] + :cljs [cljs.cache :as cache]) [konserve.memory :refer [new-mem-store]] [hasch.core :refer [uuid]] [clojure.set :as set] - #?(:clj [hitchhiker.tree.core :refer [go-try ( (case *async-backend* + :none (async/KonserveAddr store (core/last-key node) id (synthesize-storage-addr id)))))) (delete-addr [_ addr session] - (swap! session update-in :deletes inc))) + (swap! session update :deletes inc))) (defn get-root-key [tree] - (-> tree :storage-addr (async/poll!))) - -(defn create-tree-from-root-key - [store root-key] - (go-try - (let [val (KonserveAddr store last-key root-key (synthesize-storage-addr root-key))))))) + ;; TODO find out why this is inconsistent + (or + (-> tree :storage-addr (async/poll!) :konserve-key) + (-> tree :storage-addr (async/poll!)))) - -(defn add-hitchhiker-tree-handlers [store] - (swap! (:read-handlers store) merge - {'hitchhiker.konserve.KonserveAddr - #(-> % map->KonserveAddr - (assoc :store store - :storage-addr (synthesize-storage-addr (:konserve-key %)))) - 'hitchhiker.tree.core.DataNode - (fn [{:keys [children cfg]}] - (core/->DataNode (into (sorted-map-by - compare) children) - (promise-chan) - cfg)) - 'hitchhiker.tree.core.IndexNode - (fn [{:keys [children cfg op-buf]}] - (core/->IndexNode (->> children - vec) + (defn create-tree-from-root-key + [store root-key] + (go-try + (let [val (let [ch (k/get-in store [root-key])] + (case *async-backend* + :none (async/KonserveAddr store last-key root-key (synthesize-storage-addr root-key))))))) + + + (defn add-hitchhiker-tree-handlers [store] + (swap! (:read-handlers store) merge + {'hitchhiker.konserve.KonserveAddr + #(-> % map->KonserveAddr + (assoc :store store + :storage-addr (synthesize-storage-addr (:konserve-key %)))) + 'hitchhiker.tree.core.DataNode + (fn [{:keys [children cfg]}] + (core/->DataNode (into (sorted-map-by + compare) children) (promise-chan) + cfg)) + 'hitchhiker.tree.core.IndexNode + (fn [{:keys [children cfg op-buf]}] + (core/->IndexNode (->> children + vec) + (promise-chan) (vec op-buf) cfg)) 'hitchhiker.tree.messaging.InsertOp diff --git a/src/hitchhiker/plot.clj b/src/hitchhiker/plot.clj new file mode 100644 index 0000000..485d42d --- /dev/null +++ b/src/hitchhiker/plot.clj @@ -0,0 +1,211 @@ +(ns hitchhiker.plot + "This namespace provides functions to help visualizing hh-trees. + + It provides a visualization similar to those in https://youtu.be/jdn617M3-P4?t=1583 + " + (:require [konserve.memory :refer [new-mem-store]] + [hitchhiker.konserve :as kons] + [konserve.cache :as kc] + [hitchhiker.tree.core :refer [ \"") + "\" -- \"")) + (.append (dot-esc n2l)) + (.append \")) + (let [eattrs (dissoc eattrs :compass)] + (when (or (:label eattrs) (< 1 (count eattrs))) + (.append sb \space) + (.append sb (dot-attrs eattrs)))) + (.append sb "\n"))) + (doseq [n (nodes g)] + (doto sb + (.append " \"") + (.append (dot-esc (str (or (node-label n) n)))) + (.append \")) + (when-let [nattrs (when a? + (dot-attrs (attrs g n)))] + (.append sb \space) + (.append sb nattrs)) + (.append sb "\n")) + (str (doto sb (.append "}"))))) + + +(ns hitchhiker.plot) + +(def store (kons/add-hitchhiker-tree-handlers + (kc/ensure-cache (async/Config 3 3 3))) + (shuffle (range 1 25)))) + (kons/->KonserveBackend store)))) + + +(def flushed (KonserveBackend store)))) + + + +(defn init-graph [store] + (apply lg/digraph + (->> @(:state store) + (filter (fn [[id {:keys [children]}]] children)) + (map (fn [[id {:keys [children] :as node}]] + (if (:op-buf node) + {id (mapv (fn [c] (:konserve-key c)) children)} + {id []})))))) + + +(defn use-record-nodes [g] + (attr/add-attr-to-nodes g :shape "record" (lg/nodes g))) + + +(defn node-layout [g [id {:keys [children] :as node}]] + (if (core/index-node? node) + (attr/add-attr + g id + :label (str + ;; key space separators + (str/join " | " + (map #(str "<" % "> " ;; compass point (invisible) + "\\<" % "\\>") + (map core/last-key (:children node)))) + ;; render op-log + " | {" + (str/join " | " (map (fn [{:keys [key value]}] + key) + (:op-buf node))) + "}")) + (attr/add-attr + g id + :label (str (str/join "|" (map key (:children node))))))) + + +(defn set-node-layouts [g store] + (->> @(:state store) + (filter (fn [[id {:keys [children]}]] children)) + (reduce node-layout g))) + +(defn edge-hash [id] + (-> id str (subs 0 4))) + +(defn set-edge-layouts [g store] + (let [node-map @(:state store) + edges (lg/edges g)] + (reduce + (fn [g [n1 n2]] + (let [h (:konserve-key (clojure.core.async/ g + (attr/add-attr-to-edges :compass (core/last-key (node-map n2)) + [[n1 n2]]) + (attr/add-attr-to-edges :label (edge-hash h) [[n1 n2]])))) + g + edges))) + + +(defn create-graph [store] + (-> (init-graph store) + use-record-nodes + (set-node-layouts store) + (set-edge-layouts store))) + + + +(comment + (view (create-graph store)) + + (println (lio/dot-str g))) + + + +(defn remove-storage-addrs [[k v]] + (if (core/index-node? v) + [k (-> v + (dissoc :storage-addr :cfg) + (update :children (fn [cs] (mapv #(dissoc % :storage-addr :store) cs))) + (update :op-buf (fn [cs] (mapv #(into {} %) cs))) + )] + [k (dissoc v :storage-addr :cfg)])) + + +(comment + (spit "/tmp/sample-tree.json" + (json/generate-string + (into {} (map remove-storage-addrs @(:state store))))) + + + (prn (map remove-storage-addrs @(:state store)))) + + + + + + + + diff --git a/src/hitchhiker/tree/async.cljc b/src/hitchhiker/tree/async.cljc new file mode 100644 index 0000000..7f75fed --- /dev/null +++ b/src/hitchhiker/tree/async.cljc @@ -0,0 +1,6 @@ +(ns hitchhiker.tree.async) + +;; rebind this *before* loading any other +;; hh-tree namespace, so it has effect at +;; macro-expansion time +(def ^:dynamic *async-backend* :none) diff --git a/src/hitchhiker/tree/core.cljc b/src/hitchhiker/tree/core.cljc index da9b048..93f8b39 100644 --- a/src/hitchhiker/tree/core.cljc +++ b/src/hitchhiker/tree/core.cljc @@ -2,16 +2,19 @@ (:refer-clojure :exclude [compare resolve subvec]) (:require [clojure.core.rrb-vector :refer [catvec subvec]] #?(:clj [clojure.pprint :as pp]) + #?(:clj [clojure.core.async :refer [promise-chan poll! put!]]) #?(:clj [clojure.core.async :refer [go chan put! Split (->IndexNode (subvec children 0 b) (promise-chan) (vec left-buf) @@ -295,7 +367,7 @@ throwable error." [node] (let [out ^Writer *out*] (.write out "IndexNode") - (.write out (node-status-bits node)) + (.write out ^String (node-status-bits node)) (pp/pprint-logical-block :prefix "{" :suffix "}" (pp/pprint-logical-block @@ -319,8 +391,8 @@ throwable error." (defrecord DataNode [children storage-addr cfg] IResolve (index? [this] false) - (resolve [this] (go this)) - (dirty? [this] (not (async/poll! storage-addr))) + (resolve [this] this #_(go this)) + (dirty? [this] (not (poll! storage-addr))) (last-key [this] (when (seq children) (-> children @@ -421,6 +493,7 @@ throwable error." (recur (pop tmp))))))) + (defn right-successor "Given a node on a path, find's that node's right successor node" [path] @@ -441,9 +514,20 @@ throwable error." sibling-lineage (loop [res [new-sibling] s new-sibling] (let [c (-> s :children first) - c (if (tree-node? c) + ;_ (prn (type c) (= (class c) clojure.lang.PersistentTreeMap$BlackVal)) + c (cond + ;; TODO cleanup path + ;; fast path + (or (index-node? c) + (data-node? c) + #?(:clj (= (class c) clojure.lang.PersistentTreeMap$Black)) + #?(:clj (= (class c) clojure.lang.PersistentTreeMap$BlackVal))) + c + + (tree-node? c) ( start-node - :children ; Get the indices of it - (subseq >= start-key))] - ( start-node + :children ; Get the indices of it + (subseq >= start-key)) ; skip to the start-index + next-elements (lazy-seq + (when-let [succ (right-successor (pop path))] + (forward-iterator succ start-key)))] + (concat first-elements next-elements)))) + + + (defn lookup-fwd-iter + [tree key] + (let [path (lookup-path tree key)] + (when path + (forward-iterator path key))))) + + :core.async + (do + #?(:clj + (defn chan-seq [ch] + (when-some [v ( start-node + :children ; Get the indices of it + (subseq >= start-key))] + (> (flush-children-without-root (:children tree) backend stats) + = 0 (core/compare @@ -168,7 +170,7 @@ (map :op-buf))) (rseq) ; highest node should be last in seq (apply catvec) - (sort-by affects-key)) ;must be a stable sort + (sort-by affects-key core/compare)) ;must be a stable sort this-node-index (-> path pop peek) parent (-> path pop pop peek) is-first? (zero? this-node-index) @@ -238,29 +240,76 @@ :tag (uuid) )])) -(defn forward-iterator - "Takes the result of a search and puts the iterated elements onto iter-ch + +(case *async-backend* + :none + (do + (defn forward-iterator + "Takes the result of a search and returns an iterator going + forward over the tree. Does lg(n) backtracking sometimes." + [path] + (assert (core/data-node? (peek path))) + (let [first-elements (apply-ops-in-path path) + next-elements (lazy-seq + (when-let [succ (core/right-successor (pop path))] + (forward-iterator succ)))] + (concat first-elements next-elements))) + + + (defn lookup-fwd-iter + [tree key] + (let [path (core/lookup-path tree key)] + (when path + (drop-while (fn [[k v]] + (neg? (core/compare k key))) + (forward-iterator path)))))) + + + :core.async + (do + (defn forward-iterator + "Takes the result of a search and puts the iterated elements onto iter-ch going forward over the tree as needed. Does lg(n) backtracking sometimes." - [iter-ch path start-key] - (go-try - (loop [path path] - (if path - (let [_ (assert (core/data-node? (peek path))) - elements (drop-while (fn [[k v]] - (neg? (core/compare k start-key))) - (apply-ops-in-path path))] - (KonserveBackend store) init-tree (Config 1 3 (- 3 1)))) @@ -61,7 +67,7 @@ #?(:clj (let [folder "/tmp/async-hitchhiker-tree-test" _ (delete-store folder) - store (kons/add-hitchhiker-tree-handlers (KonserveBackend store) flushed (