From 01f33a80930a3c896f880607eca2930cb027c200 Mon Sep 17 00:00:00 2001 From: David Moon Date: Thu, 21 Nov 2024 22:11:23 -0500 Subject: [PATCH] use separate sym enumeration for thin dictionaries and restore walk minimality filtering, which cuts down on serialized map size by an order of magnitude and makes build time with blob strings manageable --- src/core/material/Mtrl.re | 3 +- src/core/material/PrecompiledFiles.re | 7 ++ src/core/material/Walker.re | 50 +++++----- src/core/material/Warmup.re | 137 +++++++++++++++----------- 4 files changed, 116 insertions(+), 81 deletions(-) diff --git a/src/core/material/Mtrl.re b/src/core/material/Mtrl.re index 26da95e1..a825d2fe 100644 --- a/src/core/material/Mtrl.re +++ b/src/core/material/Mtrl.re @@ -122,7 +122,8 @@ module Sym = { |> List.map(Sym.map(grout, grout)), Sort.all |> List.concat_map(Tile.Sym.all) - |> List.map(Sym.map(tile, tile)), + |> List.map(Sym.map(tile, tile)) + |> List.cons(Sym.NT(Tile(Tile.NT.root))), ]); }; diff --git a/src/core/material/PrecompiledFiles.re b/src/core/material/PrecompiledFiles.re index 3b342da0..813a31c8 100644 --- a/src/core/material/PrecompiledFiles.re +++ b/src/core/material/PrecompiledFiles.re @@ -5,3 +5,10 @@ let _warmed_walked_r = () => [%blob "precompiled/walk_r_map.txt"]; let _warmed_walked_l = () => [%blob "precompiled/walk_l_map.txt"]; let _enter_r_map = () => [%blob "precompiled/enter_r_map.txt"]; let _enter_l_map = () => [%blob "precompiled/enter_l_map.txt"]; +// let warmed_nts = () => ""; +// let warmed_stances = () => ""; + +// let _warmed_walked_r = () => ""; +// let _warmed_walked_l = () => ""; +// let _enter_r_map = () => ""; +// let _enter_l_map = () => ""; diff --git a/src/core/material/Walker.re b/src/core/material/Walker.re index b9afb06e..2118a961 100644 --- a/src/core/material/Walker.re +++ b/src/core/material/Walker.re @@ -232,7 +232,7 @@ let walk_all = step_all(~from, src) |> Index.iter((dst, w) => Queue.push((dst, w), q)); bfs(~from, q) |> Index.filter(Walk.is_valid) - // |> Index.filter(is_minimal) + |> Index.filter(is_minimal) |> Index.sort; }); let walk_all = (~from: Dir.t, src: End.t): End.Map.t(list(T.t)) => @@ -246,7 +246,7 @@ let enter_all = |> Index.iter((dst, w) => Queue.push((dst, w), q)); bfs(~from, q) |> Index.filter(Walk.is_valid) - // |> Index.filter(is_minimal) + |> Index.filter(is_minimal) |> Index.sort; }); let enter_all = (~from: Dir.t, nt) => enter_all((from, nt)); @@ -317,29 +317,29 @@ let read_warmed_walked = () => { // Thin.walk_map_of_thin(thin_walk_l, stances_flipped^, nts_flipped^); }; -let read_warmed_enter = () => { - (); - // let _warmed_entered_r = [%blob "precompiled/enter_r_map.txt"]; - // let _warmed_entered_l = [%blob "precompiled/enter_l_map.txt"]; - // let thin_enter_r = - // Thin.ThinNT.Map.t_of_sexp( - // Thin.ThinIndex.t_of_sexp, - // // Sexplib.Sexp.of_string(warmed_entered_r), - // Sexplib.Sexp.of_string(""), - // ); - // let thin_enter_l = - // Thin.ThinNT.Map.t_of_sexp( - // Thin.ThinIndex.t_of_sexp, - // // Sexplib.Sexp.of_string(warmed_entered_l), - // Sexplib.Sexp.of_string(""), - // ); - // - // enter_r_map := - // Thin.enter_map_of_thin(thin_enter_r, stances_flipped^, nts_flipped^); - // enter_l_map := - // Thin.enter_map_of_thin(thin_enter_l, stances_flipped^, nts_flipped^); - // -}; +let read_warmed_enter = () => (); +// () => { +// // let _warmed_entered_r = [%blob "precompiled/enter_r_map.txt"]; +// // let _warmed_entered_l = [%blob "precompiled/enter_l_map.txt"]; +// // let thin_enter_r = +// // Thin.ThinNT.Map.t_of_sexp( +// // Thin.ThinIndex.t_of_sexp, +// // // Sexplib.Sexp.of_string(warmed_entered_r), +// // Sexplib.Sexp.of_string(""), +// // ); +// // let thin_enter_l = +// // Thin.ThinNT.Map.t_of_sexp( +// // Thin.ThinIndex.t_of_sexp, +// // // Sexplib.Sexp.of_string(warmed_entered_l), +// // Sexplib.Sexp.of_string(""), +// // ); +// // +// // enter_r_map := +// // Thin.enter_map_of_thin(thin_enter_r, stances_flipped^, nts_flipped^); +// // enter_l_map := +// // Thin.enter_map_of_thin(thin_enter_l, stances_flipped^, nts_flipped^); +// // +// }; let read_warmed = () => { read_warmed_stances_nts(); diff --git a/src/core/material/Warmup.re b/src/core/material/Warmup.re index 65698a39..c8a4112e 100644 --- a/src/core/material/Warmup.re +++ b/src/core/material/Warmup.re @@ -3,67 +3,78 @@ open Walk; open Thin; let init_stances_swings = () => { - let enter_all: Index.t = Walker.walk_all(~from=L, Root); - - //TODO: write the importing logic to read the serialized walks/enters and see how that goes; use the full reconstructed (not thin maps) at runtime - //TODO: serialize the indices - - enter_all - |> Walk.Index.to_list - |> List.rev_map(fst) - |> List.iteri((idx, end_) => { - switch (end_) { - | Bound.Root => () - | Node(stance) => stances := StanceMap.add(stance, idx, stances^) - } - }); - - let idx = ref(0); - enter_all - |> Walk.Index.to_list - |> List.rev_map(snd) - |> List.concat - |> List.iter(walk => { - Walk.swings(walk) - |> List.iter(swing => { - Chain.loops(swing) - |> List.iter(nt => - switch (NTMap.find_opt(nt, nts^)) { - | Some(_) => () - // print_endline("nt already exists: " ++ Mtrl.NT.show(nt)) - | None => - // print_endline("nt added: " ++ Mtrl.NT.show(nt)); - nts := NTMap.add(nt, idx^, nts^); - idx := idx^ + 1; - } - ) - }) - }); - - let walk_count = ref(0); - let total_walk_size = ref(0); - - enter_all - |> Walk.Index.to_list - |> List.rev_map(snd) - |> List.concat - |> List.iteri((idx, walk) => { - walk_count := idx + 1; - total_walk_size := total_walk_size^ + Chain.length(walk); - (); - }); - - print_endline("Walk Count: " ++ string_of_int(walk_count^)); - print_endline( - "Average Walk Size: " ++ string_of_int(total_walk_size^ / walk_count^), - ); - + let (ts, ns) = + Mtrl.Sym.all |> List.partition_map(Sym.get(Either.left, Either.right)); + ts |> List.iteri((idx, t) => stances := StanceMap.add(t, idx, stances^)); + ns |> List.iteri((idx, nt) => nts := NTMap.add(nt, idx, nts^)); print_endline( "Stances len: " ++ string_of_int(StanceMap.cardinal(stances^)), ); print_endline("nts len: " ++ string_of_int(NTMap.cardinal(nts^))); }; +// let init_stances_swings = () => { +// let enter_all: Index.t = Walker.walk_all(~from=L, Root); + +// //TODO: write the importing logic to read the serialized walks/enters and see how that goes; use the full reconstructed (not thin maps) at runtime +// //TODO: serialize the indices + +// enter_all +// |> Walk.Index.to_list +// |> List.rev_map(fst) +// |> List.iteri((idx, end_) => { +// switch (end_) { +// | Bound.Root => () +// | Node(stance) => stances := StanceMap.add(stance, idx, stances^) +// } +// }); + +// let idx = ref(0); +// enter_all +// |> Walk.Index.to_list +// |> List.rev_map(snd) +// |> List.concat +// |> List.iter(walk => { +// Walk.swings(walk) +// |> List.iter(swing => { +// Chain.loops(swing) +// |> List.iter(nt => +// switch (NTMap.find_opt(nt, nts^)) { +// | Some(_) => () +// // print_endline("nt already exists: " ++ Mtrl.NT.show(nt)) +// | None => +// // print_endline("nt added: " ++ Mtrl.NT.show(nt)); +// nts := NTMap.add(nt, idx^, nts^); +// idx := idx^ + 1; +// } +// ) +// }) +// }); + +// let walk_count = ref(0); +// let total_walk_size = ref(0); + +// enter_all +// |> Walk.Index.to_list +// |> List.rev_map(snd) +// |> List.concat +// |> List.iteri((idx, walk) => { +// walk_count := idx + 1; +// total_walk_size := total_walk_size^ + Chain.length(walk); +// (); +// }); + +// print_endline("Walk Count: " ++ string_of_int(walk_count^)); +// print_endline( +// "Average Walk Size: " ++ string_of_int(total_walk_size^ / walk_count^), +// ); + +// print_endline( +// "Stances len: " ++ string_of_int(StanceMap.cardinal(stances^)), +// ); +// print_endline("nts len: " ++ string_of_int(NTMap.cardinal(nts^))); +// }; + let process_ts_l_walks = (ts, root_r_walk) => { print_endline("Getting ts_l_walks"); @@ -175,6 +186,22 @@ let warmup = () => { print_endline("got root walks"); + let walk_count = ref(0); + let total_walk_size = ref(0); + root_l_walk + |> Walk.Index.to_list + |> List.rev_map(snd) + |> List.concat + |> List.iteri((idx, walk) => { + walk_count := idx + 1; + total_walk_size := total_walk_size^ + Chain.length(walk); + (); + }); + print_endline("Walk Count: " ++ string_of_int(walk_count^)); + print_endline( + "Average Walk Size: " ++ string_of_int(total_walk_size^ / walk_count^), + ); + process_ts_l_walks(ts, root_l_walk); Gc.full_major(); process_ts_r_walk(ts, root_r_walk);