WASI runtime

.github/workflows/build-wasm_of_ocaml.yml

@@ -16,6 +16,8 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
+        os-name:
+          - Ubuntu
         ocaml-compiler:
           - "4.14"
           - "5.0"
@@ -27,30 +29,50 @@ jobs:
           - false
         all_jane_street_tests:
           - false
+        wasi:
+          - false
         include:
           - os: macos-latest
+            os-name: MacOS
             ocaml-compiler: "5.3"
             separate_compilation: true
             jane_street_tests: false
             all_jane_street_tests: false
+            wasi: false
           - os: windows-latest
+            os-name: Windows
             ocaml-compiler: "5.2"
             separate_compilation: true
             jane_street_tests: true
             all_jane_street_tests: true
+            wasi: false
           - os: ubuntu-latest
+            os-name: Ubuntu
             ocaml-compiler: "5.2"
             separate_compilation: true
             jane_street_tests: true
             all_jane_street_tests: true
+            wasi: false
           - os: ubuntu-latest
+            os-name: Ubuntu
             ocaml-compiler: "5.2"
             separate_compilation: false
             jane_street_tests: true
             all_jane_street_tests: false
+            wasi: false
+          - os: ubuntu-latest
+            os-name: Ubuntu
+            ocaml-compiler: "5.3"
+            separate_compilation: true
+            jane_street_tests: false
+            all_jane_street_tests: false
+            wasi: true
 
     runs-on: ${{ matrix.os }}
 
+    name:
+       ${{ matrix.wasi && 'WASI / ' || '' }}${{ (! matrix.separate_compilation) && 'Whole program / ' || ''}}${{ matrix.ocaml-compiler }} / ${{ matrix.os-name }}${{ matrix.all_jane_street_tests && ' / Jane Street tests' || ''}}
+
     steps:
       - name: Set git to use LF
         if: ${{ matrix.os == 'windows-latest' && matrix.ocaml-compiler < 5.2 }}
@@ -77,6 +99,54 @@ jobs:
         with:
           node-version: latest
 
+      - name: Set-up Rust toolchain
+        if: matrix.wasi
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Checkout Wasmtime
+        if: matrix.wasi
+        uses: actions/checkout@v4
+        with:
+          repository: bytecodealliance/wasmtime
+          path: wasmtime
+          submodules: true
+
+      - name: Build Wasmtime
+        if: matrix.wasi
+        working-directory: ./wasmtime
+        run: |
+          cargo build
+          echo `pwd`/target/debug >> "$GITHUB_PATH"
+
+      - name: Checkout Virgil
+        if: matrix.wasi
+        uses: actions/checkout@v4
+        with:
+          repository: titzer/virgil
+          path: virgil
+
+      - name: Build Virgil
+        if: matrix.wasi
+        working-directory: ./virgil
+        run: |
+          export PATH=$PATH:`pwd`/bin
+          echo `pwd`/bin >> "$GITHUB_PATH"
+          make
+
+      - name: Checkout Wizard engine
+        if: matrix.wasi
+        uses: actions/checkout@v4
+        with:
+          repository: titzer/wizard-engine
+          path: wizard-engine
+
+      - name: Build Wizard engine
+        if: matrix.wasi
+        working-directory: ./wizard-engine
+        run: |
+          make -j 4
+          echo `pwd`/bin >> "$GITHUB_PATH"
+
       - name: Set-up OCaml ${{ matrix.ocaml-compiler }}
         uses: ocaml/setup-ocaml@v3
         with:
@@ -127,7 +197,7 @@ jobs:
           opam install num cohttp-lwt-unix ppx_expect cstruct uucp
 
       - name: Run tests
-        if: ${{ matrix.separate_compilation }}
+        if: ${{ matrix.separate_compilation && ! matrix.wasi }}
         working-directory: ./wasm_of_ocaml
         run: opam exec -- dune build @runtest-wasm
 
@@ -136,11 +206,33 @@ jobs:
       # See https://github.com/libuv/libuv/issues/3622
 
       - name: Run tests with CPS effects
-        if: ${{ matrix.ocaml-compiler >= '5.' && matrix.separate_compilation }}
+        if: ${{ matrix.ocaml-compiler >= '5.' && matrix.separate_compilation && ! matrix.wasi }}
         continue-on-error: ${{ matrix.os == 'windows-latest' }}
         working-directory: ./wasm_of_ocaml
         run: opam exec -- dune build @runtest-wasm --profile with-effects
 
+      - name: Run tests (WASI runtime - node)
+        if: ${{ matrix.wasi }}
+        working-directory: ./wasm_of_ocaml
+        run: opam exec -- dune build @runtest-wasm --profile wasi
+
+      - name: Run tests (WASI runtime - Wizard engine)
+        if: ${{ matrix.wasi }}
+        working-directory: ./wasm_of_ocaml
+        env:
+          WASM_ENGINE: wizard-fast
+        run: opam exec -- dune build @runtest-wasm --profile wasi
+
+      - name: Run tests (WASI runtime - wasmtime)
+        if: ${{ matrix.wasi }}
+        working-directory: ./wasm_of_ocaml
+        env:
+          WASM_ENGINE: wasmtime
+          WASI_FLAGS: --enable trap-on-exception
+          RUST_BACKTRACE: 0
+        continue-on-error: true
+        run: opam exec -- dune build @runtest-wasm --profile wasi
+
       - name: Run Base tests
         if: matrix.all_jane_street_tests
         continue-on-error: ${{ matrix.os == 'windows-latest' }}

compiler/bin-wasm_of_ocaml/compile.ml

@@ -81,6 +81,39 @@ let with_runtime_files ~runtime_wasm_files f =
   in
   Wat_preprocess.with_preprocessed_files ~variables:[] ~inputs f
 
+let build_runtime ~runtime_file =
+  (* Keep this variables in sync with gen/gen.ml *)
+  let variables =
+    [ "wasi", Config.Flag.wasi (); "trap-on-exception", Config.Flag.trap_on_exception () ]
+  in
+  match
+    List.find_opt Runtime_files.precompiled_runtimes ~f:(fun (flags, _) ->
+        assert (
+          List.length flags = List.length variables
+          && List.for_all2 ~f:(fun (k, _) (k', _) -> String.equal k k') flags variables);
+        Poly.equal flags variables)
+  with
+  | Some (_, contents) -> Fs.write_file ~name:runtime_file ~contents
+  | None ->
+      let inputs =
+        List.map
+          ~f:(fun (module_name, contents) ->
+            { Wat_preprocess.module_name
+            ; file = module_name ^ ".wat"
+            ; source = Contents contents
+            })
+          (if Config.Flag.wasi ()
+           then ("libc", Runtime_files.wasi_libc) :: Runtime_files.wat_files
+           else Runtime_files.wat_files)
+      in
+      Runtime.build
+        ~link_options:[ "-g" ]
+        ~opt_options:[ "-g"; "-O2" ]
+        ~variables:
+          (List.map ~f:(fun (k, v) : (_ * Wat_preprocess.value) -> k, Bool v) variables)
+        ~inputs
+        ~output_file:runtime_file
+
 let link_and_optimize
     ~profile
     ~sourcemap_root
@@ -99,7 +132,7 @@ let link_and_optimize
   let enable_source_maps = Option.is_some opt_sourcemap_file in
   Fs.with_intermediate_file (Filename.temp_file "runtime" ".wasm")
   @@ fun runtime_file ->
-  Fs.write_file ~name:runtime_file ~contents:Runtime_files.wasm_runtime;
+  build_runtime ~runtime_file;
   Fs.with_intermediate_file (Filename.temp_file "wasm-merged" ".wasm")
   @@ fun temp_file ->
   opt_with
@@ -125,7 +158,10 @@ let link_and_optimize
   @@ fun opt_temp_sourcemap' ->
   let primitives =
     Binaryen.dead_code_elimination
-      ~dependencies:Runtime_files.dependencies
+      ~dependencies:
+        (if Config.Flag.wasi ()
+         then Runtime_files.wasi_dependencies
+         else Runtime_files.dependencies)
       ~opt_input_sourcemap:opt_temp_sourcemap
       ~opt_output_sourcemap:opt_temp_sourcemap'
       ~input_file:temp_file
@@ -145,7 +181,7 @@ let link_and_optimize
 
 let link_runtime ~profile runtime_wasm_files output_file =
   if List.is_empty runtime_wasm_files
-  then Fs.write_file ~name:output_file ~contents:Runtime_files.wasm_runtime
+  then build_runtime ~runtime_file:output_file
   else
     Fs.with_intermediate_file (Filename.temp_file "extra_runtime" ".wasm")
     @@ fun extra_runtime ->
@@ -167,7 +203,7 @@ let link_runtime ~profile runtime_wasm_files output_file =
       ();
     Fs.with_intermediate_file (Filename.temp_file "runtime" ".wasm")
     @@ fun runtime_file ->
-    Fs.write_file ~name:runtime_file ~contents:Runtime_files.wasm_runtime;
+    build_runtime ~runtime_file;
     Binaryen.link
       ~opt_output_sourcemap:None
       ~inputs:
@@ -246,7 +282,13 @@ let build_js_runtime ~primitives ?runtime_arguments () =
   in
   let prelude = Link.output_js always_required_js in
   let init_fun =
-    match Parse_js.parse (Parse_js.Lexer.of_string Runtime_files.js_runtime) with
+    match
+      Parse_js.parse
+        (Parse_js.Lexer.of_string
+           (if Config.Flag.wasi ()
+            then Runtime_files.js_wasi_launcher
+            else Runtime_files.js_launcher))
+    with
     | [ (Expression_statement f, _) ] -> f
     | _ -> assert false
   in
@@ -487,9 +529,12 @@ let run
              tmp_wasm_file
          in
          let wasm_name =
-           Printf.sprintf
-             "code-%s"
-             (String.sub (Digest.to_hex (Digest.file tmp_wasm_file)) ~pos:0 ~len:20)
+           if Config.Flag.wasi ()
+           then "code"
+           else
+             Printf.sprintf
+               "code-%s"
+               (String.sub (Digest.to_hex (Digest.file tmp_wasm_file)) ~pos:0 ~len:20)
          in
          let tmp_wasm_file' = Filename.concat tmp_dir (wasm_name ^ ".wasm") in
          Sys.rename tmp_wasm_file tmp_wasm_file';

compiler/bin-wasm_of_ocaml/dune

@@ -25,9 +25,13 @@
  (target runtime_files.ml)
  (deps
   gen/gen.exe
-  ../../runtime/wasm/runtime.wasm
   ../../runtime/wasm/runtime.js
-  ../../runtime/wasm/deps.json)
+  ../../runtime/wasm/deps.json
+  ../../runtime/wasm/runtime-wasi.js
+  ../../runtime/wasm/deps-wasi.json
+  ../../runtime/wasm/libc.wasm
+  (glob_files ../../runtime/wasm/*.wat)
+  (glob_files ../../runtime/wasm/runtime-*.wasm))
  (action
   (with-stdout-to
    %{target}

compiler/bin-wasm_of_ocaml/gen/gen.ml

@@ -1,13 +1,70 @@
 let read_file ic = really_input_string ic (in_channel_length ic)
 
+(* Keep the two variables below in sync with function build_runtime in
+   ../compile.ml *)
+
+let default_flags = [ "trap-on-exception", false ]
+
+let interesting_runtimes = [ [ "wasi", false ]; [ "wasi", true ] ]
+
+let name_runtime l =
+  let flags = List.filter_map (fun (k, v) -> if v then Some k else None) l in
+  String.concat "-" ("runtime" :: (if flags = [] then [ "standard" ] else flags))
+  ^ ".wasm"
+
+let print_flags f flags =
+  Format.fprintf
+    f
+    "@[<2>[ %a ]@]"
+    (Format.pp_print_list
+       ~pp_sep:(fun f () -> Format.fprintf f ";@ ")
+       (fun f (k, v) ->
+         Format.fprintf f "@[\"%s\",@ %s@]" k (if v then "true" else "false")))
+    flags
+
 let () =
   let () = set_binary_mode_out stdout true in
   Format.printf
-    "let wasm_runtime = \"%s\"@."
+    "let js_launcher = \"%s\"@."
     (String.escaped (read_file (open_in_bin Sys.argv.(1))));
   Format.printf
-    "let js_runtime = \"%s\"@."
+    "let dependencies = \"%s\"@."
     (String.escaped (read_file (open_in_bin Sys.argv.(2))));
   Format.printf
-    "let dependencies = \"%s\"@."
-    (String.escaped (read_file (open_in_bin Sys.argv.(3))))
+    "let js_wasi_launcher = \"%s\"@."
+    (String.escaped (read_file (open_in_bin Sys.argv.(3))));
+  Format.printf
+    "let wasi_dependencies = \"%s\"@."
+    (String.escaped (read_file (open_in_bin Sys.argv.(4))));
+  Format.printf
+    "let wasi_libc = \"%s\"@."
+    (String.escaped (read_file (open_in_bin Sys.argv.(5))));
+  let wat_files, runtimes =
+    List.partition
+      (fun f -> Filename.check_suffix f ".wat")
+      (Array.to_list (Array.sub Sys.argv 6 (Array.length Sys.argv - 6)))
+  in
+  Format.printf
+    "let wat_files = [%a]@."
+    (Format.pp_print_list (fun f file ->
+         Format.fprintf
+           f
+           "\"%s\", \"%s\"; "
+           Filename.(chop_suffix (basename file) ".wat")
+           (String.escaped (read_file (open_in_bin file)))))
+    wat_files;
+  Format.printf
+    "let precompiled_runtimes = [%a]@."
+    (Format.pp_print_list (fun f flags ->
+         let flags = flags @ default_flags in
+         let name = name_runtime flags in
+         match List.find_opt (fun file -> Filename.basename file = name) runtimes with
+         | None -> failwith ("Missing runtime " ^ name)
+         | Some file ->
+             Format.fprintf
+               f
+               "%a, \"%s\"; "
+               print_flags
+               flags
+               (String.escaped (read_file (open_in_bin file)))))
+    interesting_runtimes

compiler/lib-wasm/binaryen.ml

@@ -36,6 +36,7 @@ let common_options () =
     ; "--enable-bulk-memory"
     ; "--enable-nontrapping-float-to-int"
     ; "--enable-strings"
+    ; "--enable-multimemory" (* To keep wasm-merge happy *)
     ]
   in
   if Config.Flag.pretty () then "-g" :: l else l
@@ -111,9 +112,9 @@ let dead_code_elimination
   filter_unused_primitives primitives usage_file
 
 let optimization_options =
-  [| [ "-O2"; "--skip-pass=inlining-optimizing"; "--traps-never-happen" ]
-   ; [ "-O2"; "--skip-pass=inlining-optimizing"; "--traps-never-happen" ]
-   ; [ "-O3"; "--skip-pass=inlining-optimizing"; "--traps-never-happen" ]
+  [| [ "-O2"; "--skip-pass=inlining-optimizing" ]
+   ; [ "-O2"; "--skip-pass=inlining-optimizing" ]
+   ; [ "-O3"; "--skip-pass=inlining-optimizing" ]
   |]
 
 let optimize
@@ -132,6 +133,7 @@ let optimize
   command
     ("wasm-opt"
      :: (common_options ()
+        @ (if Config.Flag.trap_on_exception () then [] else [ "--traps-never-happen" ])
         @ Option.value ~default:optimization_options.(level - 1) options
         @ [ Filename.quote input_file; "-o"; Filename.quote output_file ])
     @ opt_flag "--input-source-map" opt_input_sourcemap