Skip to content

Commit

Permalink
Milan - restructured the warmup to resolve circular dependency issues…
Browse files Browse the repository at this point in the history
… and added support to walker to read and un-thin the precompiled/warmed walk maps
  • Loading branch information
green726 committed Nov 19, 2024
1 parent 754f0c1 commit 9e4675e
Show file tree
Hide file tree
Showing 11 changed files with 136 additions and 79 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
_build/
node_modules/
.DS_Store
src/core/material/precompiled/
17 changes: 9 additions & 8 deletions src/core/dune
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,20 @@
(preprocess
(pps
ppx_let
ppx_blob
ppx_sexp_conv
ppx_yojson_conv
ppx_deriving.show
ppx_deriving.ord
ppx_deriving_hash
sedlex.ppx)))

; (executable
; (name warmup)
; (modules Warmup)
; (libraries tylr_core core)
; (preprocess
; (pps ppx_yojson_conv ppx_let ppx_sexp_conv ppx_deriving.show)))
sedlex.ppx))
(preprocessor_deps
(file material/precompiled/stances.txt)
(file material/precompiled/nts.txt)
(file material/precompiled/walk_r_map.txt)
(file material/precompiled/walk_l_map.txt)
(file material/precompiled/enter_r_map.txt)
(file material/precompiled/enter_l_map.txt)))

(env
(dev
Expand Down
4 changes: 2 additions & 2 deletions src/core/grammar/Label.re
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ open Ppx_yojson_conv_lib.Yojson_conv.Primitives;
// todo: add operator class
[@deriving (sexp, yojson, ord)]
type t =
| Const([@sexp.opaque] Padding.t, string)
| Const(/* [@sexp.opaque] */ Padding.t, string)
| Id_lower
| Id_upper
| Int_lit
Expand All @@ -17,7 +17,7 @@ let pp = out =>
| Id_upper => Fmt.pf(out, "Id_upper")
| Int_lit => Fmt.pf(out, "Int_lit")
| Float_lit => Fmt.pf(out, "Float_lit")
| Const(_, s) => Fmt.pf(out, "\'%s\'", s);
| Const(_, s) => Fmt.pf(out, "\"%s\"", s);
let show = Fmt.to_to_string(pp);

module Ord = {
Expand Down
7 changes: 7 additions & 0 deletions src/core/material/PrecompiledFiles.re
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
let warmed_nts = () => [%blob "precompiled/nts.txt"];
let warmed_stances = () => [%blob "precompiled/stances.txt"];

let _warmed_walked_r = () => [%blob "precompiled/walk_r_map.txt"];
let _warmed_walked_l = () => [%blob "precompiled/walk_l_map.txt"];
let _enter_r_map = () => [%blob "precompiled/enter_r_map.txt"];
let _enter_l_map = () => [%blob "precompiled/enter_l_map.txt"];
1 change: 1 addition & 0 deletions src/core/material/Thin.re
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ module NTMap = {

//TODO: rename to t(terminal) map
module StanceMap = {
[@deriving (show({with_path: false}), sexp, yojson)]
include Maps.Make({
[@deriving (show({with_path: false}), sexp, yojson)]
type t = Stance.t;
Expand Down
4 changes: 0 additions & 4 deletions src/core/material/Walk.re
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,6 @@ module T = {
[@deriving (show({with_path: false}), sexp, yojson)]
type t = Chain.t(Swing.t, Stance.t);

//NOTE:milan
//thin walk = Chain.t(int, int);
//

let unit: _ => t = Chain.unit;
let empty = unit(Swing.empty);
let is_empty = (==)(empty);
Expand Down
92 changes: 58 additions & 34 deletions src/core/material/Walker.re
Original file line number Diff line number Diff line change
Expand Up @@ -262,15 +262,19 @@ let stances_flipped = ref(Thin.FlippedStanceMap.empty);
let nts_flipped = ref(Thin.FlippedNTMap.empty);

let read_warmed_stances_nts = () => {
print_endline("got warmed root l");

print_endline("Reading nts");
let nts_map =
Thin.NTMap.t_of_sexp(
Sexplib.Std.int_of_sexp,
Sexplib.Sexp.load_sexp("nts.txt"),
Sexplib.Conv.int_of_sexp,
Sexplib.Sexp.of_string(PrecompiledFiles.warmed_nts()),
);
print_endline("reading stances");
let stances_map =
Thin.StanceMap.t_of_sexp(
Sexplib.Std.int_of_sexp,
Sexplib.Sexp.load_sexp("stances.txt"),
Sexplib.Conv.int_of_sexp,
Sexplib.Sexp.of_string(PrecompiledFiles.warmed_stances()),
);

stances_flipped :=
Expand All @@ -287,49 +291,69 @@ let read_warmed_stances_nts = () => {
};

let read_warmed_walked = () => {
let thin_walk_r =
Thin.ThinEnd.Map.t_of_sexp(
Thin.ThinIndex.t_of_sexp,
Sexplib.Sexp.load_sexp("walk_r_map.txt"),
);
let thin_walk_l =
Thin.ThinEnd.Map.t_of_sexp(
Thin.ThinIndex.t_of_sexp,
Sexplib.Sexp.load_sexp("walk_l_map.txt"),
);

walk_r_map :=
Thin.walk_map_of_thin(thin_walk_r, stances_flipped^, nts_flipped^);
walk_l_map :=
Thin.walk_map_of_thin(thin_walk_l, stances_flipped^, nts_flipped^);
print_endline("reading warmed walked");
print_endline("reading walk r");
let _thin_walk_r_sexp =
Sexplib.Sexp.of_string(PrecompiledFiles._warmed_walked_r());
();
// let _thin_walk_r =
// Thin.ThinEnd.Map.t_of_sexp(
// Thin.ThinIndex.t_of_sexp,
// Sexplib.Sexp.of_string(_warmed_walked_r),
// );
// print_endline("doing a gc full major");
// Gc.full_major();
// print_endline("reading walk l");
// let thin_walk_l =
// Thin.ThinEnd.Map.t_of_sexp(
// Thin.ThinIndex.t_of_sexp,
// Sexplib.Sexp.of_string([%blob "precompiled/walk_l_map.txt"]),
// // Sexplib.Sexp.of_string(""),
// );
//
// walk_r_map :=
// Thin.walk_map_of_thin(thin_walk_r, stances_flipped^, nts_flipped^);
// walk_l_map :=
// Thin.walk_map_of_thin(thin_walk_l, stances_flipped^, nts_flipped^);
};

let read_warmed_enter = () => {
let thin_enter_r =
Thin.ThinNT.Map.t_of_sexp(
Thin.ThinIndex.t_of_sexp,
Sexplib.Sexp.load_sexp("enter_r_map.txt"),
);
let thin_enter_l =
Thin.ThinNT.Map.t_of_sexp(
Thin.ThinIndex.t_of_sexp,
Sexplib.Sexp.load_sexp("enter_l_map.txt"),
);

enter_r_map :=
Thin.enter_map_of_thin(thin_enter_r, stances_flipped^, nts_flipped^);
enter_l_map :=
Thin.enter_map_of_thin(thin_enter_l, stances_flipped^, nts_flipped^);
print_endline("hahahaha");
print_endline(String.sub(PrecompiledFiles._enter_r_map(), 0, 10));
print_endline(String.sub(PrecompiledFiles._enter_l_map(), 0, 10));

();
// let _warmed_entered_r = [%blob "precompiled/enter_r_map.txt"];
// let _warmed_entered_l = [%blob "precompiled/enter_l_map.txt"];
// let thin_enter_r =
// Thin.ThinNT.Map.t_of_sexp(
// Thin.ThinIndex.t_of_sexp,
// // Sexplib.Sexp.of_string(warmed_entered_r),
// Sexplib.Sexp.of_string(""),
// );
// let thin_enter_l =
// Thin.ThinNT.Map.t_of_sexp(
// Thin.ThinIndex.t_of_sexp,
// // Sexplib.Sexp.of_string(warmed_entered_l),
// Sexplib.Sexp.of_string(""),
// );
//
// enter_r_map :=
// Thin.enter_map_of_thin(thin_enter_r, stances_flipped^, nts_flipped^);
// enter_l_map :=
// Thin.enter_map_of_thin(thin_enter_l, stances_flipped^, nts_flipped^);
//
};

let read_warmed = () => {
read_warmed_stances_nts();
Gc.full_major();
print_endline("read warmed stances nts");
read_warmed_walked();
print_endline("read warmed walked");
Gc.full_major();
read_warmed_enter();
print_endline("read warmed entered");
Gc.full_major();
};

Expand Down
6 changes: 4 additions & 2 deletions src/core/material/Warmup.re
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,8 @@ let warmup = () => {
let root_l_walk: Index.t = walk_all(~from=L, Root);
let root_r_walk: Index.t = walk_all(~from=R, Root);

print_endline("got root walks");

process_ts_l_walks(ts, root_l_walk);
Gc.full_major();
process_ts_r_walk(ts, root_r_walk);
Expand All @@ -182,8 +184,8 @@ let warmup = () => {
process_nts_r_walk(nts_list);
Gc.full_major();

let stances_sexp = StanceMap.sexp_of_t(Sexplib.Std.sexp_of_int, stances^);
let nts_sexp = NTMap.sexp_of_t(Sexplib.Std.sexp_of_int, nts^);
let stances_sexp = StanceMap.sexp_of_t(Sexplib.Conv.sexp_of_int, stances^);
let nts_sexp = NTMap.sexp_of_t(Sexplib.Conv.sexp_of_int, nts^);

let _ = Sexplib.Sexp.save("stances.txt", stances_sexp);
let _ = Sexplib.Sexp.save("nts.txt", nts_sexp);
Expand Down
2 changes: 1 addition & 1 deletion src/stds/Maps.re
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ module type S = {
module MakeShowFor = (O: OrderedShowType, S: Map.S with type key = O.t) => {
let pp = (pp_v, f, map) =>
S.iter(
(k, v) => Format.fprintf(f, "%a -> %a@\n", O.pp, k, pp_v, v),
(k, v) => Format.fprintf(f, "(%a, %a);@\n", O.pp, k, pp_v, v),
map,
);
};
Expand Down
1 change: 1 addition & 0 deletions src/web/Main.re
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ module App = {
Js.string("MAC"),
)
>= 0;
Tylr_core.Walker.read_warmed();
// Async_kernel.Clock_ns.run_after(
// Core.Time_ns.Span.of_ms(100.0),
// Tylr_core.Warmup.warmup,
Expand Down
80 changes: 52 additions & 28 deletions src/web/misc/JsUtil.re
Original file line number Diff line number Diff line change
@@ -1,28 +1,52 @@
// open Js_of_ocaml;
// let date_now = () => {
// [%js new Js.date_now];
// };
// let timestamp = () => date_now()##valueOf;
// let copy_to_clipboard = (string: string): unit => {
// /* Note: To use (deprecated) execommand would need to introduce
// an invisible textarea and insert the string as you cannot
// directly copy from a variable using it */
// /*let _ =
// Dom_html.document##execCommand(
// Js.string("copy"),
// Js.bool(true),
// Js.Opt.return(Js.string("testtest")),
// );*/
// /* So instead we use the mode modern clipboard API. however
// js_of_ocaml doesn't have bindings for it, so in the interest
// of time I'm just using Unsafe.js_expr. Note the use of backticks
// around the string in order to make this robust to the presence
// of linebreaks in the string. */
// // note: using unsafe as js_of_ocaml doesn't have clipboard bindings
// print_endline(
// "Copying log to keyboard. An exception reading 'fallback to runtime evaluation' is expected.",
// );
// string
// |> Printf.sprintf("window.navigator.clipboard.writeText(`%s`);")
// |> Js.Unsafe.js_expr;
// };
open Js_of_ocaml;
// let date_now = () => {
// [%js new Js.date_now];
// };
// let timestamp = () => date_now()##valueOf;
// let copy_to_clipboard = (string: string): unit => {
// /* Note: To use (deprecated) execommand would need to introduce
// an invisible textarea and insert the string as you cannot
// directly copy from a variable using it */
// /*let _ =
// Dom_html.document##execCommand(
// Js.string("copy"),
// Js.bool(true),
// Js.Opt.return(Js.string("testtest")),
// );*/
// /* So instead we use the mode modern clipboard API. however
// js_of_ocaml doesn't have bindings for it, so in the interest
// of time I'm just using Unsafe.js_expr. Note the use of backticks
// around the string in order to make this robust to the presence
// of linebreaks in the string. */
// // note: using unsafe as js_of_ocaml doesn't have clipboard bindings
// print_endline(
// "Copying log to keyboard. An exception reading 'fallback to runtime evaluation' is expected.",
// );
// string
// |> Printf.sprintf("window.navigator.clipboard.writeText(`%s`);")
// |> Js.Unsafe.js_expr;
// };
// let read_file = (file, k) => {
// let reader = [%js new File.fileReader];
// reader##readAsText(file);
// reader##.onload :=
// Dom.handler(_ => {
// let result = reader##.result;
// let option = Js.Opt.to_option(File.CoerceTo.string(result));
// let data = Option.map(Js.to_string, option);
// k(data);
// Js._true;
// });
// };

let download_string_file =
(~filename: string, ~content_type: string, ~contents: string) => {
let blob = File.blob_from_string(~contentType=content_type, contents);
let url = Dom_html.window##._URL##createObjectURL(blob);

let link = Dom_html.createA(Dom_html.document);
link##.href := url;
link##setAttribute(Js.string("download"), Js.string(filename));
link##.onclick := Dom_html.handler(_ => {Js._true});
link##click;
};

0 comments on commit 9e4675e

Please sign in to comment.