Skip to content

Commit

Permalink
use separate sym enumeration for thin dictionaries and restore walk m…
Browse files Browse the repository at this point in the history 
…inimality filtering, which cuts down on serialized map size by an order of magnitude and makes build time with blob strings manageable
  • Loading branch information
@dm0n3y
dm0n3y committed Nov 22, 2024
1 parent 406a936 commit 01f33a8
Show file tree
Hide file tree
Showing 4 changed files with 116 additions and 81 deletions.
3 changes: 2 additions & 1 deletion src/core/material/Mtrl.re
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ module Sym = {
|> List.map(Sym.map(grout, grout)),
Sort.all
|> List.concat_map(Tile.Sym.all)
|> List.map(Sym.map(tile, tile)),
|> List.map(Sym.map(tile, tile))
|> List.cons(Sym.NT(Tile(Tile.NT.root))),
]);
};

Expand Down
7 changes: 7 additions & 0 deletions src/core/material/PrecompiledFiles.re
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,10 @@ let _warmed_walked_r = () => [%blob "precompiled/walk_r_map.txt"];
let _warmed_walked_l = () => [%blob "precompiled/walk_l_map.txt"];
let _enter_r_map = () => [%blob "precompiled/enter_r_map.txt"];
let _enter_l_map = () => [%blob "precompiled/enter_l_map.txt"];
// let warmed_nts = () => "";
// let warmed_stances = () => "";

// let _warmed_walked_r = () => "";
// let _warmed_walked_l = () => "";
// let _enter_r_map = () => "";
// let _enter_l_map = () => "";
50 changes: 25 additions & 25 deletions src/core/material/Walker.re
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ let walk_all =
step_all(~from, src) |> Index.iter((dst, w) => Queue.push((dst, w), q));
bfs(~from, q)
|> Index.filter(Walk.is_valid)
// |> Index.filter(is_minimal)
|> Index.filter(is_minimal)
|> Index.sort;
});
let walk_all = (~from: Dir.t, src: End.t): End.Map.t(list(T.t)) =>
Expand All @@ -246,7 +246,7 @@ let enter_all =
|> Index.iter((dst, w) => Queue.push((dst, w), q));
bfs(~from, q)
|> Index.filter(Walk.is_valid)
// |> Index.filter(is_minimal)
|> Index.filter(is_minimal)
|> Index.sort;
});
let enter_all = (~from: Dir.t, nt) => enter_all((from, nt));
Expand Down Expand Up @@ -317,29 +317,29 @@ let read_warmed_walked = () => {
// Thin.walk_map_of_thin(thin_walk_l, stances_flipped^, nts_flipped^);
};

let read_warmed_enter = () => {
();
// let _warmed_entered_r = [%blob "precompiled/enter_r_map.txt"];
// let _warmed_entered_l = [%blob "precompiled/enter_l_map.txt"];
// let thin_enter_r =
// Thin.ThinNT.Map.t_of_sexp(
// Thin.ThinIndex.t_of_sexp,
// // Sexplib.Sexp.of_string(warmed_entered_r),
// Sexplib.Sexp.of_string(""),
// );
// let thin_enter_l =
// Thin.ThinNT.Map.t_of_sexp(
// Thin.ThinIndex.t_of_sexp,
// // Sexplib.Sexp.of_string(warmed_entered_l),
// Sexplib.Sexp.of_string(""),
// );
//
// enter_r_map :=
// Thin.enter_map_of_thin(thin_enter_r, stances_flipped^, nts_flipped^);
// enter_l_map :=
// Thin.enter_map_of_thin(thin_enter_l, stances_flipped^, nts_flipped^);
//
};
let read_warmed_enter = () => ();
// () => {
// // let _warmed_entered_r = [%blob "precompiled/enter_r_map.txt"];
// // let _warmed_entered_l = [%blob "precompiled/enter_l_map.txt"];
// // let thin_enter_r =
// // Thin.ThinNT.Map.t_of_sexp(
// // Thin.ThinIndex.t_of_sexp,
// // // Sexplib.Sexp.of_string(warmed_entered_r),
// // Sexplib.Sexp.of_string(""),
// // );
// // let thin_enter_l =
// // Thin.ThinNT.Map.t_of_sexp(
// // Thin.ThinIndex.t_of_sexp,
// // // Sexplib.Sexp.of_string(warmed_entered_l),
// // Sexplib.Sexp.of_string(""),
// // );
// //
// // enter_r_map :=
// // Thin.enter_map_of_thin(thin_enter_r, stances_flipped^, nts_flipped^);
// // enter_l_map :=
// // Thin.enter_map_of_thin(thin_enter_l, stances_flipped^, nts_flipped^);
// //
// };

let read_warmed = () => {
read_warmed_stances_nts();
Expand Down
137 changes: 82 additions & 55 deletions src/core/material/Warmup.re
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,67 +3,78 @@ open Walk;
open Thin;

let init_stances_swings = () => {
let enter_all: Index.t = Walker.walk_all(~from=L, Root);

//TODO: write the importing logic to read the serialized walks/enters and see how that goes; use the full reconstructed (not thin maps) at runtime
//TODO: serialize the indices

enter_all
|> Walk.Index.to_list
|> List.rev_map(fst)
|> List.iteri((idx, end_) => {
switch (end_) {
| Bound.Root => ()
| Node(stance) => stances := StanceMap.add(stance, idx, stances^)
}
});

let idx = ref(0);
enter_all
|> Walk.Index.to_list
|> List.rev_map(snd)
|> List.concat
|> List.iter(walk => {
Walk.swings(walk)
|> List.iter(swing => {
Chain.loops(swing)
|> List.iter(nt =>
switch (NTMap.find_opt(nt, nts^)) {
| Some(_) => ()
// print_endline("nt already exists: " ++ Mtrl.NT.show(nt))
| None =>
// print_endline("nt added: " ++ Mtrl.NT.show(nt));
nts := NTMap.add(nt, idx^, nts^);
idx := idx^ + 1;
}
)
})
});

let walk_count = ref(0);
let total_walk_size = ref(0);

enter_all
|> Walk.Index.to_list
|> List.rev_map(snd)
|> List.concat
|> List.iteri((idx, walk) => {
walk_count := idx + 1;
total_walk_size := total_walk_size^ + Chain.length(walk);
();
});

print_endline("Walk Count: " ++ string_of_int(walk_count^));
print_endline(
"Average Walk Size: " ++ string_of_int(total_walk_size^ / walk_count^),
);

let (ts, ns) =
Mtrl.Sym.all |> List.partition_map(Sym.get(Either.left, Either.right));
ts |> List.iteri((idx, t) => stances := StanceMap.add(t, idx, stances^));
ns |> List.iteri((idx, nt) => nts := NTMap.add(nt, idx, nts^));
print_endline(
"Stances len: " ++ string_of_int(StanceMap.cardinal(stances^)),
);
print_endline("nts len: " ++ string_of_int(NTMap.cardinal(nts^)));
};

// let init_stances_swings = () => {
// let enter_all: Index.t = Walker.walk_all(~from=L, Root);

// //TODO: write the importing logic to read the serialized walks/enters and see how that goes; use the full reconstructed (not thin maps) at runtime
// //TODO: serialize the indices

// enter_all
// |> Walk.Index.to_list
// |> List.rev_map(fst)
// |> List.iteri((idx, end_) => {
// switch (end_) {
// | Bound.Root => ()
// | Node(stance) => stances := StanceMap.add(stance, idx, stances^)
// }
// });

// let idx = ref(0);
// enter_all
// |> Walk.Index.to_list
// |> List.rev_map(snd)
// |> List.concat
// |> List.iter(walk => {
// Walk.swings(walk)
// |> List.iter(swing => {
// Chain.loops(swing)
// |> List.iter(nt =>
// switch (NTMap.find_opt(nt, nts^)) {
// | Some(_) => ()
// // print_endline("nt already exists: " ++ Mtrl.NT.show(nt))
// | None =>
// // print_endline("nt added: " ++ Mtrl.NT.show(nt));
// nts := NTMap.add(nt, idx^, nts^);
// idx := idx^ + 1;
// }
// )
// })
// });

// let walk_count = ref(0);
// let total_walk_size = ref(0);

// enter_all
// |> Walk.Index.to_list
// |> List.rev_map(snd)
// |> List.concat
// |> List.iteri((idx, walk) => {
// walk_count := idx + 1;
// total_walk_size := total_walk_size^ + Chain.length(walk);
// ();
// });

// print_endline("Walk Count: " ++ string_of_int(walk_count^));
// print_endline(
// "Average Walk Size: " ++ string_of_int(total_walk_size^ / walk_count^),
// );

// print_endline(
// "Stances len: " ++ string_of_int(StanceMap.cardinal(stances^)),
// );
// print_endline("nts len: " ++ string_of_int(NTMap.cardinal(nts^)));
// };

let process_ts_l_walks = (ts, root_r_walk) => {
print_endline("Getting ts_l_walks");

Expand Down Expand Up @@ -175,6 +186,22 @@ let warmup = () => {

print_endline("got root walks");

let walk_count = ref(0);
let total_walk_size = ref(0);
root_l_walk
|> Walk.Index.to_list
|> List.rev_map(snd)
|> List.concat
|> List.iteri((idx, walk) => {
walk_count := idx + 1;
total_walk_size := total_walk_size^ + Chain.length(walk);
();
});
print_endline("Walk Count: " ++ string_of_int(walk_count^));
print_endline(
"Average Walk Size: " ++ string_of_int(total_walk_size^ / walk_count^),
);

process_ts_l_walks(ts, root_l_walk);
Gc.full_major();
process_ts_r_walk(ts, root_r_walk);
Expand Down

0 comments on commit 01f33a8

Please sign in to comment.