feat: IPFS retrieval client

.github/workflows/ci.yaml

@@ -31,6 +31,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+          cache: false # caching requires a go.sum file, which we don't have in our project
+
       - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1
         with:
           toolchain: stable
@@ -64,6 +69,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+          cache: false # caching requires a go.sum file, which we don't have in our project
+
       - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1
         with:
           toolchain: stable
@@ -100,6 +110,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+          cache: false # caching requires a go.sum file, which we don't have in our project
+
       - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1
         with:
           toolchain: stable
@@ -137,6 +152,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+          cache: false # caching requires a go.sum file, which we don't have in our project
+
       - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1
         with:
           toolchain: stable

.github/workflows/release.yml

@@ -89,6 +89,11 @@ jobs:
       - name: Setup | Checkout
         uses: actions/checkout@v3
 
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+          cache: false # caching requires a go.sum file, which we don't have in our project
+
       - name: Setup | Rust
         uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1
         with:

cli/README.md

@@ -33,6 +33,12 @@ If you have Rust tooling installed on your machine (see
 [Install Rust](https://www.rust-lang.org/tools/install)), you can build & install Zinnia from the
 source code.
 
+In addition to the Rust build toolchain, you also need Go installed. See
+[Go Downloads](https://go.dev/dl/).
+
+On Windows, Go uses `gcc` to create C libraries. Go recommends installing
+[TDM GCC](https://jmeubank.github.io/tdm-gcc/).
+
 ```sh
 $ cargo install zinnia
 ```

cli/main.rs

@@ -1,6 +1,7 @@
 mod args;
 
 use std::rc::Rc;
+use std::sync::Arc;
 use std::time::Duration;
 
 use args::{CliArgs, Commands};
@@ -9,7 +10,9 @@ use clap::Parser;
 use zinnia_runtime::anyhow::{Context, Error, Result};
 use zinnia_runtime::deno_core::error::JsError;
 use zinnia_runtime::fmt_errors::format_js_error;
-use zinnia_runtime::{colors, resolve_path, run_js_module, BootstrapOptions, ConsoleReporter};
+use zinnia_runtime::{
+    colors, lassie, resolve_path, run_js_module, BootstrapOptions, ConsoleReporter,
+};
 
 #[tokio::main(flavor = "current_thread")]
 async fn main() {
@@ -32,9 +35,35 @@ async fn main_impl() -> Result<()> {
                 &file,
                 &std::env::current_dir().context("unable to get current working directory")?,
             )?;
+
+            let lassie_daemon = Arc::new(
+                lassie::Daemon::start(lassie::DaemonConfig {
+                    // This configuration applies to `zinnia` CLI only. The `zinniad` daemon running
+                    // inside Station uses a different temp_dir config based on the env var
+                    // `CACHE_ROOT` provided by the Station.
+                    //
+                    // By default, Lassie stores its temporary files in the system temp directory.
+                    // That's good enough for now. We can improve this later based on user feedback,
+                    // for example:
+                    // - we can honour CACHE_ROOT
+                    // - we can default to something like
+                    //   `~/.cache/zinnia/lassie` on Unix,
+                    //   `%APPLOCALDATA%\zinnia\lassie' on Windows.
+                    //
+                    // Important: if we tell Lassie to use a specific temp dir that's not
+                    // automatically cleaned by the operating system, we will need to clean any
+                    // leftover files ourselves. See the GH issue for deleting leftover files
+                    // when `zinniad` starts: https://github.com/filecoin-station/zinnia/issues/245
+                    temp_dir: None,
+                    port: 0,
+                })
+                .context("cannot initialize the IPFS retrieval client Lassie")?,
+            );
+
             let config = BootstrapOptions::new(
                 format!("zinnia/{}", env!("CARGO_PKG_VERSION")),
                 Rc::new(ConsoleReporter::new(Duration::from_millis(500))),
+                lassie_daemon,
                 None,
             );
             run_js_module(&main_module, &config).await?;

daemon/main.rs

@@ -4,14 +4,14 @@ mod station_reporter;
 
 use std::path::PathBuf;
 use std::rc::Rc;
+use std::sync::Arc;
 use std::time::Duration;
 
 use args::CliArgs;
 use clap::Parser;
 
-use log::{error, info};
 use zinnia_runtime::anyhow::{anyhow, Context, Error, Result};
-use zinnia_runtime::{get_module_root, resolve_path, run_js_module, BootstrapOptions};
+use zinnia_runtime::{get_module_root, lassie, resolve_path, run_js_module, BootstrapOptions};
 
 use crate::station_reporter::{log_info_activity, StationReporter};
 
@@ -27,7 +27,7 @@ async fn main() {
 }
 
 async fn run(config: CliArgs) -> Result<()> {
-    info!("Starting zinniad with config {config:?}");
+    log::info!("Starting zinniad with config {config:?}");
 
     if config.files.is_empty() {
         return Err(anyhow!("You must provide at least one module to run."));
@@ -41,6 +41,15 @@ async fn run(config: CliArgs) -> Result<()> {
     let state_file = PathBuf::from(config.state_root).join("state.json");
     log::debug!("Using state file: {}", state_file.display());
 
+    let lassie_config = lassie::DaemonConfig {
+        temp_dir: Some(PathBuf::from(config.cache_root).join("lassie")),
+        port: 0,
+    };
+    let lassie_daemon = Arc::new(
+        lassie::Daemon::start(lassie_config)
+            .context("cannot initialize the IPFS retrieval client Lassie")?,
+    );
+
     log_info_activity("Module Runtime started.");
 
     let file = &config.files[0];
@@ -63,6 +72,7 @@ async fn run(config: CliArgs) -> Result<()> {
             Duration::from_millis(200),
             module_name.into(),
         )),
+        lassie_daemon,
         module_root: Some(module_root),
         no_color: true,
         is_tty: false,
@@ -88,6 +98,6 @@ fn exit_with_error(error: Error) {
     let error_string = format!("{error:?}");
     let error_code = 1;
 
-    error!("{}", error_string.trim_start_matches("error: "));
+    log::error!("{}", error_string.trim_start_matches("error: "));
     std::process::exit(error_code);
 }

docs/building-modules.md

@@ -87,6 +87,7 @@ import * as code from "../../other/code.js";
 - [Web APIs](#web-apis)
 - [Unsupported Web APIs](#unsupported-web-apis)
 - [libp2p](#libp2p)
+- [IPFS retrieval client](#ipfs-retrieval-client)
 
 ### Standard JavaScript APIs
 
@@ -329,6 +330,32 @@ Report that a single job was completed.
 
 Call this function every time your module completes a job. It's ok to call it frequently.
 
+### IPFS Retrieval Client
+
+Zinnia provides a built-in IPFS retrieval client making it easy to fetch content-addressed data from
+IPFS and Filecoin networks. You can retrieve data for a given CID using the web platform API `fetch`
+together with the URL scheme `ipfs://`.
+
+Example:
+
+```js
+const response = await fetch("ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni");
+assert(response.ok);
+const data = await response.arrayBuffer();
+// data contains binary data in the CAR format
+```
+
+> Note: At the moment, Zinnia does not provide any tools for interpreting the returned CAR data. We
+> are discussing support for reading UnixFS data in
+> [zinnia#245](https://github.com/filecoin-station/zinnia/issues/246).
+
+Under the hood, Zinnia handles `ipfs://bafy...` requests by calling Lassie's HTTP API. You can learn
+more about supported parameters (request headers, query string arguments), response headers and
+possible error status codes in
+[Lassie's HTTP Specification](https://github.com/filecoin-project/lassie/blob/main/docs/HTTP_SPEC.md).
+The format of CAR data returned by the retrieval client is described in
+[Lassie's Returned CAR Specification](https://github.com/filecoin-project/lassie/blob/main/docs/CAR.md).
+
 ## Testing Guide
 
 Zinnia provides lightweight tooling for writing and running automated tests.

runtime/Cargo.toml

@@ -21,6 +21,8 @@ deno_fetch = "0.129.0"
 deno_url = "0.105.0"
 deno_web = "0.136.0"
 deno_webidl = "0.105.0"
+lassie = "0.3.0"
+# lassie = { git = "https://github.com/filecoin-station/rusty-lassie.git" }
 log.workspace = true
 once_cell = "1.18.0"
 serde.workspace = true

runtime/ext.rs

@@ -52,6 +52,7 @@ deno_core::extension!(
       "90_zinnia_apis.js",
       "98_global_scope.js",
       "internals.js",
+      "fetch.js",
       "test.js",
       "vendored/asserts.bundle.js",
       "99_main.js",

runtime/js/98_global_scope.js

@@ -23,7 +23,7 @@ import * as fileReader from "ext:deno_web/10_filereader.js";
 import * as formData from "ext:deno_fetch/21_formdata.js";
 import * as request from "ext:deno_fetch/23_request.js";
 import * as response from "ext:deno_fetch/23_response.js";
-import * as fetch from "ext:deno_fetch/26_fetch.js";
+import * as fetch from "ext:zinnia_runtime/fetch.js";
 import * as messagePort from "ext:deno_web/13_message_port.js";
 import * as webidl from "ext:deno_webidl/00_webidl.js";
 import DOMException from "ext:deno_web/01_dom_exception.js";

runtime/js/99_main.js

@@ -36,6 +36,7 @@ import {
   mainRuntimeGlobalProperties,
   windowOrWorkerGlobalScope,
 } from "ext:zinnia_runtime/98_global_scope.js";
+import { setLassieUrl } from "ext:zinnia_runtime/fetch.js";
 
 function formatException(error) {
   if (ObjectPrototypeIsPrototypeOf(ErrorPrototype, error)) {
@@ -66,6 +67,8 @@ function runtimeStart(runtimeOptions) {
 
   // deno-lint-ignore prefer-primordials
   Error.prepareStackTrace = core.prepareStackTrace;
+
+  setLassieUrl(runtimeOptions.lassieUrl);
 }
 
 let hasBootstrapped = false;

runtime/js/fetch.js

@@ -0,0 +1,69 @@
+import { fetch as fetchImpl } from "ext:deno_fetch/26_fetch.js";
+import { fromInnerResponse, toInnerResponse } from "ext:deno_fetch/23_response.js";
+import { toInnerRequest, fromInnerRequest, Request } from "ext:deno_fetch/23_request.js";
+import { guardFromHeaders } from "ext:deno_fetch/20_headers.js";
+
+const ipfsScheme = "ipfs://";
+let ipfsBaseUrl = undefined;
+
+export function setLassieUrl(/** @type {string} */ value) {
+  ipfsBaseUrl = value + "ipfs/";
+}
+
+export function fetch(resource, options) {
+  let request = new Request(resource, options);
+  // The `resource` arg can be a string or any other object with a stringifier - including a URL
+  // object - that provides the URL of the resource you want to fetch; or a Request object.
+  // See https://developer.mozilla.org/en-US/docs/Web/API/fetch#parameters
+  // Fortunately, Request's constructor handles the conversions, and Request#url is always a string.
+  // See https://developer.mozilla.org/en-US/docs/Web/API/Request/url
+  if (request.url.startsWith(ipfsScheme)) {
+    return fetchFromIpfs(request);
+  } else {
+    return fetchImpl(request);
+  }
+}
+
+async function fetchFromIpfs(request) {
+  // Rewrite request URL to use Lassie
+  request = buildIpfsRequest(request);
+
+  // Call Deno's `fetch` using the rewritten URL to make the actual HTTP request
+  const response = await fetchImpl(request);
+
+  // Patch the response object to hide the fact that we are calling Lassie
+  // We don't want to leak Lassie's URL
+  return patchIpfsResponse(response);
+}
+
+// Deno's Fetch Request is a thin immutable wrapper around InnerRequest. In order to modify the
+// request URL, we must convert Request to InnerRequest first, make changes on the inner object,
+// and finally convert the InnerRequest back to a new Request instance.
+function buildIpfsRequest(request) {
+  const inner = toInnerRequest(request);
+
+  inner.urlList = /** @type {(() => string)[]}*/ (inner.urlList).map((urlFn) => {
+    const url = urlFn();
+    if (!url.startsWith(ipfsScheme)) return urlFn;
+    const newUrl = ipfsBaseUrl + url.slice(ipfsScheme.length);
+    return () => newUrl;
+  });
+  inner.urlListProcessed = /** @type {string[]} */ (inner.urlListProcessed).map((url) =>
+    url.startsWith(ipfsScheme) ? ipfsBaseUrl + url.slice(ipfsScheme.length) : url,
+  );
+
+  return fromInnerRequest(inner, request.signal, guardFromHeaders(request.headers));
+}
+
+// Deno's Fetch Response is a thin immutable wrapper around InnerResponse. In order to modify the
+// response URL, we must convert Response to InnerResponse first, make changes on the inner object,
+// and finally convert the InnerResponse back to a new Response instance.
+function patchIpfsResponse(response) {
+  const inner = toInnerResponse(response);
+
+  inner.urlList = /** @type {string[])} */ (inner.urlList).map((url) =>
+    url.startsWith(ipfsBaseUrl) ? "ipfs://" + url.slice(ipfsBaseUrl.length) : url,
+  );
+
+  return fromInnerResponse(inner, guardFromHeaders(response.headers));
+}

runtime/lib.rs

@@ -18,4 +18,6 @@ mod reporter;
 pub use console_reporter::*;
 pub use reporter::*;
 
+pub use lassie;
+
 mod ext;