From 2bce5d3693c3b370d7921cb47b4b25df8cc02e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Wed, 7 Jun 2023 15:00:53 +0200 Subject: [PATCH 01/23] test: setup env logger in integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- runtime/tests/fetch_api_tests.rs | 2 ++ runtime/tests/runtime_integration_tests.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/runtime/tests/fetch_api_tests.rs b/runtime/tests/fetch_api_tests.rs index 4362fc6a..de5747e8 100644 --- a/runtime/tests/fetch_api_tests.rs +++ b/runtime/tests/fetch_api_tests.rs @@ -10,6 +10,8 @@ use zinnia_runtime::{anyhow, deno_core, run_js_module, BootstrapOptions, Recordi #[tokio::test] async fn fetch_reports_user_agent() -> Result<()> { + let _ = env_logger::builder().is_test(true).try_init(); + let user_agent = "zinnia_fetch_api_tests agent/007"; let server_port = start_echo_server().await?; diff --git a/runtime/tests/runtime_integration_tests.rs b/runtime/tests/runtime_integration_tests.rs index cc42c137..de0e100d 100644 --- a/runtime/tests/runtime_integration_tests.rs +++ b/runtime/tests/runtime_integration_tests.rs @@ -93,6 +93,8 @@ test_runner_tests!(failing_tests expect_failure); // Run all tests in a single JS file async fn run_js_test_file(name: &str) -> Result<(Vec, Option), AnyError> { + let _ = env_logger::builder().is_test(true).try_init(); + let mut full_path = get_base_dir(); full_path.push(name); From 1a30f474ac209f1f6964f8ece006852ce0d936ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Wed, 7 Jun 2023 15:01:30 +0200 Subject: [PATCH 02/23] feat: add Lassie - IPFS retrieval client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- Cargo.lock | 10 ++++++++++ cli/main.rs | 15 ++++++++++++++- daemon/main.rs | 18 ++++++++++++++---- runtime/Cargo.toml | 2 ++ runtime/lib.rs | 2 ++ runtime/runtime.rs | 10 +++++++++- runtime/tests/fetch_api_tests.rs | 9 ++++++++- runtime/tests/helpers/mod.rs | 15 +++++++++++++++ runtime/tests/runtime_integration_tests.rs | 3 +++ 9 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 runtime/tests/helpers/mod.rs diff --git a/Cargo.lock b/Cargo.lock index b38733e6..a711a0d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2178,6 +2178,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lassie" +version = "0.1.1" +source = "git+https://github.com/filecoin-station/rusty-lassie.git#0980b5c08c3dbd92eb658caf34649dc99d554798" +dependencies = [ + "cc", + "log", +] + [[package]] name = "lazy-regex" version = "2.5.0" @@ -5682,6 +5691,7 @@ dependencies = [ "deno_web", "deno_webidl", "env_logger", + "lassie", "log", "once_cell", "pretty_assertions", diff --git a/cli/main.rs b/cli/main.rs index 0c1dae52..1db2b18e 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -1,6 +1,7 @@ mod args; use std::rc::Rc; +use std::sync::Arc; use std::time::Duration; use args::{CliArgs, Commands}; @@ -9,7 +10,9 @@ use clap::Parser; use zinnia_runtime::anyhow::{Context, Error, Result}; use zinnia_runtime::deno_core::error::JsError; use zinnia_runtime::fmt_errors::format_js_error; -use zinnia_runtime::{colors, resolve_path, run_js_module, BootstrapOptions, ConsoleReporter}; +use zinnia_runtime::{ + colors, lassie, resolve_path, run_js_module, BootstrapOptions, ConsoleReporter, +}; #[tokio::main(flavor = "current_thread")] async fn main() { @@ -32,9 +35,19 @@ async fn main_impl() -> Result<()> { &file, &std::env::current_dir().context("unable to get current working directory")?, )?; + + let lassie_daemon = Arc::new( + lassie::Daemon::start(lassie::DaemonConfig { + temp_dir: None, // TODO: Should we use something like ~/.cache/zinnia/lassie? + port: 0, + }) + .context("cannot initialize the IPFS retrieval client Lassie")?, + ); + let config = BootstrapOptions::new( format!("zinnia/{}", env!("CARGO_PKG_VERSION")), Rc::new(ConsoleReporter::new(Duration::from_millis(500))), + lassie_daemon, None, ); run_js_module(&main_module, &config).await?; diff --git a/daemon/main.rs b/daemon/main.rs index 10e122f9..6e8e731b 100644 --- a/daemon/main.rs +++ b/daemon/main.rs @@ -4,14 +4,14 @@ mod station_reporter; use std::path::PathBuf; use std::rc::Rc; +use std::sync::Arc; use std::time::Duration; use args::CliArgs; use clap::Parser; -use log::{error, info}; use zinnia_runtime::anyhow::{anyhow, Context, Error, Result}; -use zinnia_runtime::{get_module_root, resolve_path, run_js_module, BootstrapOptions}; +use zinnia_runtime::{get_module_root, lassie, resolve_path, run_js_module, BootstrapOptions}; use crate::station_reporter::{log_info_activity, StationReporter}; @@ -27,7 +27,7 @@ async fn main() { } async fn run(config: CliArgs) -> Result<()> { - info!("Starting zinniad with config {config:?}"); + log::info!("Starting zinniad with config {config:?}"); if config.files.is_empty() { return Err(anyhow!("You must provide at least one module to run.")); @@ -41,6 +41,15 @@ async fn run(config: CliArgs) -> Result<()> { let state_file = PathBuf::from(config.state_root).join("state.json"); log::debug!("Using state file: {}", state_file.display()); + let lassie_config = lassie::DaemonConfig { + temp_dir: Some(PathBuf::from(config.cache_root).join("lassie")), + port: 0, + }; + let lassie_daemon = Arc::new( + lassie::Daemon::start(lassie_config) + .context("cannot initialize the IPFS retrieval client Lassie")?, + ); + log_info_activity("Module Runtime started."); let file = &config.files[0]; @@ -63,6 +72,7 @@ async fn run(config: CliArgs) -> Result<()> { Duration::from_millis(200), module_name.into(), )), + lassie_daemon, module_root: Some(module_root), no_color: true, is_tty: false, @@ -88,6 +98,6 @@ fn exit_with_error(error: Error) { let error_string = format!("{error:?}"); let error_code = 1; - error!("{}", error_string.trim_start_matches("error: ")); + log::error!("{}", error_string.trim_start_matches("error: ")); std::process::exit(error_code); } diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 62caa012..a143cfa6 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -21,6 +21,8 @@ deno_fetch = "0.129.0" deno_url = "0.105.0" deno_web = "0.136.0" deno_webidl = "0.105.0" +# lassie = "0.1.1" +lassie = { git = "https://github.com/filecoin-station/rusty-lassie.git" } log.workspace = true once_cell = "1.18.0" serde.workspace = true diff --git a/runtime/lib.rs b/runtime/lib.rs index f5d6fbd3..dc04255d 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -18,4 +18,6 @@ mod reporter; pub use console_reporter::*; pub use reporter::*; +pub use lassie; + mod ext; diff --git a/runtime/runtime.rs b/runtime/runtime.rs index 6dc7b43a..59b3cced 100644 --- a/runtime/runtime.rs +++ b/runtime/runtime.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; use std::rc::Rc; +use std::sync::Arc; use deno_core::{located_script_name, serde_json, JsRuntime, ModuleSpecifier, RuntimeOptions}; @@ -35,12 +36,17 @@ pub struct BootstrapOptions { /// Report activities pub reporter: Rc, + + /// Lassie daemon to use as the IPFS retrieval client. We must use Arc here to allow sharing of + /// the singleton Lassie instance between multiple threads spawned by Rust's test runner. + pub lassie_daemon: Arc, } impl BootstrapOptions { pub fn new( agent_version: String, reporter: Rc, + lassie_daemon: Arc, module_root: Option, ) -> Self { Self { @@ -52,6 +58,7 @@ impl BootstrapOptions { // See https://lotus.filecoin.io/lotus/manage/manage-fil/#public-key-address wallet_address: String::from("t1abjxfbp274xpdqcpuaykwkfb43omjotacm2p3za"), reporter, + lassie_daemon, } } @@ -60,12 +67,13 @@ impl BootstrapOptions { "noColor": self.no_color, "isTty": self.is_tty, "walletAddress": self.wallet_address, + "lassieUrl": format!("http://127.0.0.1:{}", self.lassie_daemon.port()), }); serde_json::to_string_pretty(&payload).unwrap() } } -pub async fn run_js_module( +pub async fn run_js_module<'a>( module_specifier: &ModuleSpecifier, bootstrap_options: &BootstrapOptions, ) -> Result<(), AnyError> { diff --git a/runtime/tests/fetch_api_tests.rs b/runtime/tests/fetch_api_tests.rs index de5747e8..9d548b2c 100644 --- a/runtime/tests/fetch_api_tests.rs +++ b/runtime/tests/fetch_api_tests.rs @@ -8,6 +8,8 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use zinnia_runtime::{anyhow, deno_core, run_js_module, BootstrapOptions, RecordingReporter}; +mod helpers; + #[tokio::test] async fn fetch_reports_user_agent() -> Result<()> { let _ = env_logger::builder().is_test(true).try_init(); @@ -31,7 +33,12 @@ assertArrayIncludes(request_lines, ["user-agent: {user_agent}"]); &std::env::current_dir().context("unable to get current working directory")?, )?; let reporter = Rc::new(RecordingReporter::new()); - let config = BootstrapOptions::new(user_agent.into(), reporter.clone(), None); + let config = BootstrapOptions::new( + user_agent.into(), + reporter.clone(), + helpers::lassie_daemon(), + None, + ); run_js_module(&main_module, &config).await?; // the test passes when the JavaScript code does not throw Ok(()) diff --git a/runtime/tests/helpers/mod.rs b/runtime/tests/helpers/mod.rs new file mode 100644 index 00000000..71e0391d --- /dev/null +++ b/runtime/tests/helpers/mod.rs @@ -0,0 +1,15 @@ +use std::sync::{Arc, OnceLock}; + +pub fn lassie_daemon() -> Arc { + static LASSIE_DAEMON: OnceLock, lassie::StartError>> = + OnceLock::new(); + + let result = LASSIE_DAEMON.get_or_init(|| { + lassie::Daemon::start(lassie::DaemonConfig::default()).map(|d| Arc::new(d)) + }); + + match result { + Ok(ptr) => Arc::clone(ptr), + Err(err) => panic!("could not start Lassie daemon: {err}"), + } +} diff --git a/runtime/tests/runtime_integration_tests.rs b/runtime/tests/runtime_integration_tests.rs index de0e100d..48812bfe 100644 --- a/runtime/tests/runtime_integration_tests.rs +++ b/runtime/tests/runtime_integration_tests.rs @@ -14,6 +14,8 @@ use zinnia_runtime::{anyhow, deno_core, run_js_module, AnyError, BootstrapOption use pretty_assertions::assert_eq; +mod helpers; + macro_rules! js_tests( ( $name:ident ) => { #[tokio::test] @@ -106,6 +108,7 @@ async fn run_js_test_file(name: &str) -> Result<(Vec, Option), let config = BootstrapOptions::new( format!("zinnia_runtime_tests/{}", env!("CARGO_PKG_VERSION")), reporter.clone(), + helpers::lassie_daemon(), None, ); let run_result = run_js_module(&main_module, &config).await; From 5be5140a9e1937efc66f4f36de90d62c57e76ded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Wed, 7 Jun 2023 15:57:32 +0200 Subject: [PATCH 03/23] deps: upgrade lassie to v0.2.0 --- Cargo.lock | 5 +++-- runtime/Cargo.toml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a711a0d0..31e1b7f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2180,8 +2180,9 @@ dependencies = [ [[package]] name = "lassie" -version = "0.1.1" -source = "git+https://github.com/filecoin-station/rusty-lassie.git#0980b5c08c3dbd92eb658caf34649dc99d554798" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cab74cc6be90f99fc5b2e79a0f86877b5499e6b1b1e3c75a3e11d635d589a" dependencies = [ "cc", "log", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index a143cfa6..f0897635 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -21,8 +21,8 @@ deno_fetch = "0.129.0" deno_url = "0.105.0" deno_web = "0.136.0" deno_webidl = "0.105.0" -# lassie = "0.1.1" -lassie = { git = "https://github.com/filecoin-station/rusty-lassie.git" } +lassie = "0.2.0" +# lassie = { git = "https://github.com/filecoin-station/rusty-lassie.git" } log.workspace = true once_cell = "1.18.0" serde.workspace = true From 884f05a54450bed6b0a2620ddd404e9b8d4c986b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Wed, 7 Jun 2023 15:58:17 +0200 Subject: [PATCH 04/23] fixup! remove unused lifetime param MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- runtime/runtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/runtime.rs b/runtime/runtime.rs index 59b3cced..187ef18c 100644 --- a/runtime/runtime.rs +++ b/runtime/runtime.rs @@ -73,7 +73,7 @@ impl BootstrapOptions { } } -pub async fn run_js_module<'a>( +pub async fn run_js_module( module_specifier: &ModuleSpecifier, bootstrap_options: &BootstrapOptions, ) -> Result<(), AnyError> { From 93f2c3123b233da046b9cb50fc83b5a5d2927579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Wed, 7 Jun 2023 17:25:59 +0200 Subject: [PATCH 05/23] feat: fetch('ipfs://bafycid') MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- runtime/ext.rs | 1 + runtime/js/98_global_scope.js | 2 +- runtime/js/99_main.js | 3 ++ runtime/js/fetch.js | 42 ++++++++++++++++++++++ runtime/runtime.rs | 2 +- runtime/tests/js/ipfs_retrieval_tests.js | 23 ++++++++++++ runtime/tests/runtime_integration_tests.rs | 1 + 7 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 runtime/js/fetch.js create mode 100644 runtime/tests/js/ipfs_retrieval_tests.js diff --git a/runtime/ext.rs b/runtime/ext.rs index 9bb23f51..146e69e1 100644 --- a/runtime/ext.rs +++ b/runtime/ext.rs @@ -52,6 +52,7 @@ deno_core::extension!( "90_zinnia_apis.js", "98_global_scope.js", "internals.js", + "fetch.js", "test.js", "vendored/asserts.bundle.js", "99_main.js", diff --git a/runtime/js/98_global_scope.js b/runtime/js/98_global_scope.js index 67c2f1a4..2fd432db 100644 --- a/runtime/js/98_global_scope.js +++ b/runtime/js/98_global_scope.js @@ -23,7 +23,7 @@ import * as fileReader from "ext:deno_web/10_filereader.js"; import * as formData from "ext:deno_fetch/21_formdata.js"; import * as request from "ext:deno_fetch/23_request.js"; import * as response from "ext:deno_fetch/23_response.js"; -import * as fetch from "ext:deno_fetch/26_fetch.js"; +import * as fetch from "ext:zinnia_runtime/fetch.js"; import * as messagePort from "ext:deno_web/13_message_port.js"; import * as webidl from "ext:deno_webidl/00_webidl.js"; import DOMException from "ext:deno_web/01_dom_exception.js"; diff --git a/runtime/js/99_main.js b/runtime/js/99_main.js index 81856f6a..14c036e2 100644 --- a/runtime/js/99_main.js +++ b/runtime/js/99_main.js @@ -36,6 +36,7 @@ import { mainRuntimeGlobalProperties, windowOrWorkerGlobalScope, } from "ext:zinnia_runtime/98_global_scope.js"; +import { setLassieUrl } from "ext:zinnia_runtime/fetch.js"; function formatException(error) { if (ObjectPrototypeIsPrototypeOf(ErrorPrototype, error)) { @@ -66,6 +67,8 @@ function runtimeStart(runtimeOptions) { // deno-lint-ignore prefer-primordials Error.prepareStackTrace = core.prepareStackTrace; + + setLassieUrl(runtimeOptions.lassieUrl); } let hasBootstrapped = false; diff --git a/runtime/js/fetch.js b/runtime/js/fetch.js new file mode 100644 index 00000000..b27732c2 --- /dev/null +++ b/runtime/js/fetch.js @@ -0,0 +1,42 @@ +import { fetch as fetchImpl } from "ext:deno_fetch/26_fetch.js"; +import { fromInnerResponse, toInnerResponse } from "ext:deno_fetch/23_response.js"; + +let ipfsPrefix = undefined; + +export function setLassieUrl(/** @type {string} */ value) { + ipfsPrefix = value + "ipfs/"; +} + +function rewriteRetrievalUrl(resource) { + if (!ipfsPrefix) { + throw new Error("Internal Zinnia error: Lassie URL was not configured."); + } + if (typeof resource !== "string") resource = String(resource); + return resource.replace(/^ipfs:\/\//, ipfsPrefix); +} + +export function fetch(resource, options) { + // TODO: support other objects with stringifiers, e.g. URL + // See https://developer.mozilla.org/en-US/docs/Web/API/fetch#parameters + if (typeof resource === "string" && resource.startsWith("ipfs://")) { + return fetchFromLassie(rewriteRetrievalUrl(resource), options, resource); + } + + // TODO: support `resource` configured as fetch.Request + // See https://developer.mozilla.org/en-US/docs/Web/API/Request + + return fetchImpl(...args); +} + +async function fetchFromLassie(resource, options, originalUrl) { + // Call Deno's `fetch` using the rewritten URL to make the actual HTTP request + const response = await fetchImpl(resource, options); + + // Patch the response object to hide the fact that we are calling Lassie + // We don't want to leak Lassie's URL + const inner = toInnerResponse(response); + inner.urlList = inner.urlList.map((url) => + url.startsWith(ipfsPrefix) ? "ipfs://" + url.slice(ipfsPrefix.length) : url, + ); + return fromInnerResponse(inner); +} diff --git a/runtime/runtime.rs b/runtime/runtime.rs index 187ef18c..74eb8233 100644 --- a/runtime/runtime.rs +++ b/runtime/runtime.rs @@ -67,7 +67,7 @@ impl BootstrapOptions { "noColor": self.no_color, "isTty": self.is_tty, "walletAddress": self.wallet_address, - "lassieUrl": format!("http://127.0.0.1:{}", self.lassie_daemon.port()), + "lassieUrl": format!("http://127.0.0.1:{}/", self.lassie_daemon.port()), }); serde_json::to_string_pretty(&payload).unwrap() } diff --git a/runtime/tests/js/ipfs_retrieval_tests.js b/runtime/tests/js/ipfs_retrieval_tests.js new file mode 100644 index 00000000..190094ab --- /dev/null +++ b/runtime/tests/js/ipfs_retrieval_tests.js @@ -0,0 +1,23 @@ +import { test } from "zinnia:test"; +import { assertEquals, AssertionError } from "zinnia:assert"; + +const EXPECTED_CAR_BASE64 = + "OqJlcm9vdHOB2CpYJQABcBIgO/KicpaH2Kj0sXyJNWLdY4kGpEe2mjY5zovBGRJ+6mpndmVyc2lvbgFrAXASIDvyonKWh9io9LF8iTVi3WOJBqRHtpo2Oc6LwRkSfupqCkUIAhI/TXkgbW9zdCBmYW1vdXMgZHJhd2luZywgYW5kIG9uZSBvZiB0aGUgZmlyc3QgSSBkaWQgZm9yIHRoZSBzaXRlGD8="; + +test("can retrieve CID content as a CAR file", async () => { + const requestUrl = "ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni"; + + const response = await fetch(requestUrl); + if (!response.ok) { + throw new AssertionError( + `Fetch request failed with status code ${response.status}: ${await response.body()}`, + ); + } + const payload = await response.arrayBuffer(); + assertEquals(payload.byteLength, 167, "CAR size in bytes"); + + const payload_encoded = btoa(String.fromCharCode(...new Uint8Array(payload))); + assertEquals(payload_encoded, EXPECTED_CAR_BASE64); + + assertEquals(response.url, requestUrl); +}); diff --git a/runtime/tests/runtime_integration_tests.rs b/runtime/tests/runtime_integration_tests.rs index 48812bfe..2b8839b1 100644 --- a/runtime/tests/runtime_integration_tests.rs +++ b/runtime/tests/runtime_integration_tests.rs @@ -89,6 +89,7 @@ js_tests!(libp2p_tests); js_tests!(station_apis_tests); js_tests!(station_reporting_tests check_activity); js_tests!(module_loader_tests); +js_tests!(ipfs_retrieval_tests); test_runner_tests!(passing_tests); test_runner_tests!(failing_tests expect_failure); From f74672bb4f1fbf7586965928816f5cb454c4a4a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Wed, 7 Jun 2023 17:26:38 +0200 Subject: [PATCH 06/23] REVERT ME: temporarily disable aarch64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bac1c4f6..f66073c2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,8 +21,8 @@ jobs: include: - target: x86_64-apple-darwin os: macos-latest - - target: aarch64-apple-darwin - os: macos-latest + # - target: aarch64-apple-darwin + # os: macos-latest - target: x86_64-pc-windows-msvc os: windows-latest - target: x86_64-unknown-linux-gnu From 31a5b8c1599ee50c19d35140f22f48dc7c42b5e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Thu, 8 Jun 2023 11:22:20 +0200 Subject: [PATCH 07/23] Revert "REVERT ME: temporarily disable aarch64" This reverts commit f74672bb4f1fbf7586965928816f5cb454c4a4a1. --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f66073c2..bac1c4f6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,8 +21,8 @@ jobs: include: - target: x86_64-apple-darwin os: macos-latest - # - target: aarch64-apple-darwin - # os: macos-latest + - target: aarch64-apple-darwin + os: macos-latest - target: x86_64-pc-windows-msvc os: windows-latest - target: x86_64-unknown-linux-gnu From 3c5826a1e702ae64f4fd924daa463abe48847895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Thu, 8 Jun 2023 11:22:28 +0200 Subject: [PATCH 08/23] temporarily use Lassie from git main branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- Cargo.lock | 3 +-- runtime/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31e1b7f2..47de3e02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2181,8 +2181,7 @@ dependencies = [ [[package]] name = "lassie" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cab74cc6be90f99fc5b2e79a0f86877b5499e6b1b1e3c75a3e11d635d589a" +source = "git+https://github.com/filecoin-station/rusty-lassie.git#be9f5ca0c9b2c55463b8300a96cbc73c4eea7615" dependencies = [ "cc", "log", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index f0897635..79ce3f16 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -21,8 +21,8 @@ deno_fetch = "0.129.0" deno_url = "0.105.0" deno_web = "0.136.0" deno_webidl = "0.105.0" -lassie = "0.2.0" -# lassie = { git = "https://github.com/filecoin-station/rusty-lassie.git" } +# lassie = "0.2.0" +lassie = { git = "https://github.com/filecoin-station/rusty-lassie.git" } log.workspace = true once_cell = "1.18.0" serde.workspace = true From f810edd6b7fe65aabde51f4261fd2d0107ee0b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Thu, 8 Jun 2023 14:45:04 +0200 Subject: [PATCH 09/23] upgrade lassie to 0.3.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- runtime/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 79ce3f16..c0d15f2c 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -21,8 +21,8 @@ deno_fetch = "0.129.0" deno_url = "0.105.0" deno_web = "0.136.0" deno_webidl = "0.105.0" -# lassie = "0.2.0" -lassie = { git = "https://github.com/filecoin-station/rusty-lassie.git" } +lassie = "0.3.0" +# lassie = { git = "https://github.com/filecoin-station/rusty-lassie.git" } log.workspace = true once_cell = "1.18.0" serde.workspace = true From 7e4d990de01aa52fb97c6717f73c08a04cb11ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Thu, 8 Jun 2023 14:50:01 +0200 Subject: [PATCH 10/23] fix tests + code cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- Cargo.lock | 5 +++-- runtime/js/fetch.js | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 47de3e02..a888e4de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2180,8 +2180,9 @@ dependencies = [ [[package]] name = "lassie" -version = "0.2.0" -source = "git+https://github.com/filecoin-station/rusty-lassie.git#be9f5ca0c9b2c55463b8300a96cbc73c4eea7615" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5726517411b49924096cc729c41092c134b4e82d710f37af0936db83e02657ff" dependencies = [ "cc", "log", diff --git a/runtime/js/fetch.js b/runtime/js/fetch.js index b27732c2..21b35b1c 100644 --- a/runtime/js/fetch.js +++ b/runtime/js/fetch.js @@ -1,34 +1,35 @@ import { fetch as fetchImpl } from "ext:deno_fetch/26_fetch.js"; import { fromInnerResponse, toInnerResponse } from "ext:deno_fetch/23_response.js"; -let ipfsPrefix = undefined; +let ipfsScheme = "ipfs://"; +let ipfsBaseUrl = undefined; export function setLassieUrl(/** @type {string} */ value) { - ipfsPrefix = value + "ipfs/"; + ipfsBaseUrl = value + "ipfs/"; } function rewriteRetrievalUrl(resource) { - if (!ipfsPrefix) { + if (!ipfsBaseUrl) { throw new Error("Internal Zinnia error: Lassie URL was not configured."); } if (typeof resource !== "string") resource = String(resource); - return resource.replace(/^ipfs:\/\//, ipfsPrefix); + return ipfsBaseUrl + resource.slice(ipfsScheme.length); } export function fetch(resource, options) { // TODO: support other objects with stringifiers, e.g. URL // See https://developer.mozilla.org/en-US/docs/Web/API/fetch#parameters - if (typeof resource === "string" && resource.startsWith("ipfs://")) { - return fetchFromLassie(rewriteRetrievalUrl(resource), options, resource); + if (typeof resource === "string" && resource.startsWith(ipfsScheme)) { + return fetchFromIpfs(rewriteRetrievalUrl(resource), options, resource); } // TODO: support `resource` configured as fetch.Request // See https://developer.mozilla.org/en-US/docs/Web/API/Request - return fetchImpl(...args); + return fetchImpl(resource, options); } -async function fetchFromLassie(resource, options, originalUrl) { +async function fetchFromIpfs(resource, options) { // Call Deno's `fetch` using the rewritten URL to make the actual HTTP request const response = await fetchImpl(resource, options); @@ -36,7 +37,7 @@ async function fetchFromLassie(resource, options, originalUrl) { // We don't want to leak Lassie's URL const inner = toInnerResponse(response); inner.urlList = inner.urlList.map((url) => - url.startsWith(ipfsPrefix) ? "ipfs://" + url.slice(ipfsPrefix.length) : url, + url.startsWith(ipfsBaseUrl) ? "ipfs://" + url.slice(ipfsBaseUrl.length) : url, ); return fromInnerResponse(inner); } From d6f4cdf2498c3ca965fe402c917e23b065345a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Thu, 8 Jun 2023 15:37:13 +0200 Subject: [PATCH 11/23] add support for URL and Request inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- runtime/js/fetch.js | 65 ++++++++++++++++-------- runtime/tests/js/ipfs_retrieval_tests.js | 30 +++++++++++ 2 files changed, 74 insertions(+), 21 deletions(-) diff --git a/runtime/js/fetch.js b/runtime/js/fetch.js index 21b35b1c..3f800e6e 100644 --- a/runtime/js/fetch.js +++ b/runtime/js/fetch.js @@ -1,5 +1,7 @@ import { fetch as fetchImpl } from "ext:deno_fetch/26_fetch.js"; import { fromInnerResponse, toInnerResponse } from "ext:deno_fetch/23_response.js"; +import { toInnerRequest, fromInnerRequest, Request } from "ext:deno_fetch/23_request.js"; +import { guardFromHeaders } from "ext:deno_fetch/20_headers.js"; let ipfsScheme = "ipfs://"; let ipfsBaseUrl = undefined; @@ -8,36 +10,57 @@ export function setLassieUrl(/** @type {string} */ value) { ipfsBaseUrl = value + "ipfs/"; } -function rewriteRetrievalUrl(resource) { - if (!ipfsBaseUrl) { - throw new Error("Internal Zinnia error: Lassie URL was not configured."); - } - if (typeof resource !== "string") resource = String(resource); - return ipfsBaseUrl + resource.slice(ipfsScheme.length); -} - export function fetch(resource, options) { - // TODO: support other objects with stringifiers, e.g. URL - // See https://developer.mozilla.org/en-US/docs/Web/API/fetch#parameters - if (typeof resource === "string" && resource.startsWith(ipfsScheme)) { - return fetchFromIpfs(rewriteRetrievalUrl(resource), options, resource); + let request = new Request(resource, options); + // Fortunately, Request#url is a string, not an instance of URL class + // See https://developer.mozilla.org/en-US/docs/Web/API/Request/url + if (request.url.startsWith(ipfsScheme)) { + return fetchFromIpfs(request); + } else { + return fetchImpl(request); } - - // TODO: support `resource` configured as fetch.Request - // See https://developer.mozilla.org/en-US/docs/Web/API/Request - - return fetchImpl(resource, options); } -async function fetchFromIpfs(resource, options) { +async function fetchFromIpfs(request) { + // Rewrite request URL to use Lassie + request = buildIpfsRequest(request); + // Call Deno's `fetch` using the rewritten URL to make the actual HTTP request - const response = await fetchImpl(resource, options); + const response = await fetchImpl(request); // Patch the response object to hide the fact that we are calling Lassie // We don't want to leak Lassie's URL + return patchIpfsResponse(response); +} + +// Deno's Fetch Request is a thin immutable wrapper around InnerRequest. In order to modify the +// request URL, we must convert Request to InnerRequest first, make changes on the inner object, +// and finally convert the InnerRequest back to a new Request instance. +function buildIpfsRequest(request) { + const inner = toInnerRequest(request); + + inner.urlList = /** @type {(() => string)[]}*/ (inner.urlList).map((urlFn) => { + const url = urlFn(); + if (!url.startsWith(ipfsScheme)) return urlFn; + const newUrl = ipfsBaseUrl + url.slice(ipfsScheme.length); + return () => newUrl; + }); + inner.urlListProcessed = /** @type {string[]} */ (inner.urlListProcessed).map((url) => + url.startsWith(ipfsScheme) ? ipfsBaseUrl + url.slice(ipfsScheme.length) : url, + ); + + return fromInnerRequest(inner, request.signal, guardFromHeaders(request.headers)); +} + +// Deno's Fetch Response is a thin immutable wrapper around InnerResponse. In order to modify the +// response URL, we must convert Response to InnerResponse first, make changes on the inner object, +// and finally convert the InnerResponse back to a new Response instance. +function patchIpfsResponse(response) { const inner = toInnerResponse(response); - inner.urlList = inner.urlList.map((url) => + + inner.urlList = /** @type {string[])} */ (inner.urlList).map((url) => url.startsWith(ipfsBaseUrl) ? "ipfs://" + url.slice(ipfsBaseUrl.length) : url, ); - return fromInnerResponse(inner); + + return fromInnerResponse(inner, guardFromHeaders(response.headers)); } diff --git a/runtime/tests/js/ipfs_retrieval_tests.js b/runtime/tests/js/ipfs_retrieval_tests.js index 190094ab..2194f647 100644 --- a/runtime/tests/js/ipfs_retrieval_tests.js +++ b/runtime/tests/js/ipfs_retrieval_tests.js @@ -21,3 +21,33 @@ test("can retrieve CID content as a CAR file", async () => { assertEquals(response.url, requestUrl); }); + +test("can retrieve IPFS content using URL", async () => { + const requestUrl = new URL("ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni"); + const response = await fetch(requestUrl); + if (!response.ok) { + throw new AssertionError( + `Fetch request failed with status code ${response.status}: ${await response.body()}`, + ); + } + + const payload = await response.arrayBuffer(); + assertEquals(payload.byteLength, 167, "CAR size in bytes"); + + assertEquals(response.url, requestUrl.toString()); +}); + +test("can retrieve IPFS content using Fetch Request object", async () => { + const request = new Request("ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni"); + const response = await fetch(request); + if (!response.ok) { + throw new AssertionError( + `Fetch request failed with status code ${response.status}: ${await response.body()}`, + ); + } + + const payload = await response.arrayBuffer(); + assertEquals(payload.byteLength, 167, "CAR size in bytes"); + + assertEquals(response.url, request.url); +}); From 460b94d54135d232fcd2f4106535abefb97a302b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Thu, 8 Jun 2023 15:38:54 +0200 Subject: [PATCH 12/23] fix clippy warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- runtime/tests/helpers/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/runtime/tests/helpers/mod.rs b/runtime/tests/helpers/mod.rs index 71e0391d..ec9edc52 100644 --- a/runtime/tests/helpers/mod.rs +++ b/runtime/tests/helpers/mod.rs @@ -4,9 +4,8 @@ pub fn lassie_daemon() -> Arc { static LASSIE_DAEMON: OnceLock, lassie::StartError>> = OnceLock::new(); - let result = LASSIE_DAEMON.get_or_init(|| { - lassie::Daemon::start(lassie::DaemonConfig::default()).map(|d| Arc::new(d)) - }); + let result = LASSIE_DAEMON + .get_or_init(|| lassie::Daemon::start(lassie::DaemonConfig::default()).map(Arc::new)); match result { Ok(ptr) => Arc::clone(ptr), From 66b5837d488e9c2a3b61789e3f135d0668925e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Thu, 8 Jun 2023 17:14:26 +0200 Subject: [PATCH 13/23] add docs for module builders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- docs/building-modules.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/building-modules.md b/docs/building-modules.md index f5e21cc7..02f7643a 100644 --- a/docs/building-modules.md +++ b/docs/building-modules.md @@ -87,6 +87,7 @@ import * as code from "../../other/code.js"; - [Web APIs](#web-apis) - [Unsupported Web APIs](#unsupported-web-apis) - [libp2p](#libp2p) +- [IPFS retrieval client](#ipfs-retrieval-client) ### Standard JavaScript APIs @@ -329,6 +330,32 @@ Report that a single job was completed. Call this function every time your module completes a job. It's ok to call it frequently. +### IPFS Retrieval Client + +Zinnia provides a built-in IPFS retrieval client making it easy to fetch content-addressed data from +IPFS and Filecoin networks. You can retrieve data for a given CID using the web platform API `fetch` +and using the URL scheme `ipfs://`. + +Example: + +```js +const response = await fetch("ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni"); +assert(response.ok); +const data = await response.arrayBuffer(); +// data contains binary data in the CAR format +``` + +> Note: At the moment, Zinnia does not provide any tools to interpret the returned CAR data. We are +> discussing support for reading UnixFS data in +> [zinnia#245](https://github.com/filecoin-station/zinnia/issues/246). + +Under the hood, Zinnia handles `ipfs://bafy...` requests by calling Lassie HTTP API. You can learn +more about supported parameters (request headers, query string arguments), response headers and +possible error status codes in +[Lassie's HTTP Specification](https://github.com/filecoin-project/lassie/blob/main/docs/HTTP_SPEC.md). +The format of CAR data returned by the retrieval client is described in +[Lassie's Returned CAR Specification](https://github.com/filecoin-project/lassie/blob/main/docs/CAR.md). + ## Testing Guide Zinnia provides lightweight tooling for writing and running automated tests. From ed48be70dd5dc3a5045862d983b2f57616db7630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Mon, 12 Jun 2023 12:03:26 +0200 Subject: [PATCH 14/23] Apply suggestions from code review Co-authored-by: Julian Gruber --- docs/building-modules.md | 6 +++--- runtime/js/fetch.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/building-modules.md b/docs/building-modules.md index 02f7643a..68548e5c 100644 --- a/docs/building-modules.md +++ b/docs/building-modules.md @@ -334,7 +334,7 @@ Call this function every time your module completes a job. It's ok to call it fr Zinnia provides a built-in IPFS retrieval client making it easy to fetch content-addressed data from IPFS and Filecoin networks. You can retrieve data for a given CID using the web platform API `fetch` -and using the URL scheme `ipfs://`. +together with the URL scheme `ipfs://`. Example: @@ -345,11 +345,11 @@ const data = await response.arrayBuffer(); // data contains binary data in the CAR format ``` -> Note: At the moment, Zinnia does not provide any tools to interpret the returned CAR data. We are +> Note: At the moment, Zinnia does not provide any tools for interpreting the returned CAR data. We are > discussing support for reading UnixFS data in > [zinnia#245](https://github.com/filecoin-station/zinnia/issues/246). -Under the hood, Zinnia handles `ipfs://bafy...` requests by calling Lassie HTTP API. You can learn +Under the hood, Zinnia handles `ipfs://bafy...` requests by calling Lassie's HTTP API. You can learn more about supported parameters (request headers, query string arguments), response headers and possible error status codes in [Lassie's HTTP Specification](https://github.com/filecoin-project/lassie/blob/main/docs/HTTP_SPEC.md). diff --git a/runtime/js/fetch.js b/runtime/js/fetch.js index 3f800e6e..f722534e 100644 --- a/runtime/js/fetch.js +++ b/runtime/js/fetch.js @@ -3,7 +3,7 @@ import { fromInnerResponse, toInnerResponse } from "ext:deno_fetch/23_response.j import { toInnerRequest, fromInnerRequest, Request } from "ext:deno_fetch/23_request.js"; import { guardFromHeaders } from "ext:deno_fetch/20_headers.js"; -let ipfsScheme = "ipfs://"; +const ipfsScheme = "ipfs://"; let ipfsBaseUrl = undefined; export function setLassieUrl(/** @type {string} */ value) { From 12d2c7f073dbb6a176e735635b50225d89b14bb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Mon, 12 Jun 2023 16:44:05 +0200 Subject: [PATCH 15/23] fixup! prettier --write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- docs/building-modules.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/building-modules.md b/docs/building-modules.md index 68548e5c..b76d10df 100644 --- a/docs/building-modules.md +++ b/docs/building-modules.md @@ -345,8 +345,8 @@ const data = await response.arrayBuffer(); // data contains binary data in the CAR format ``` -> Note: At the moment, Zinnia does not provide any tools for interpreting the returned CAR data. We are -> discussing support for reading UnixFS data in +> Note: At the moment, Zinnia does not provide any tools for interpreting the returned CAR data. We +> are discussing support for reading UnixFS data in > [zinnia#245](https://github.com/filecoin-station/zinnia/issues/246). Under the hood, Zinnia handles `ipfs://bafy...` requests by calling Lassie's HTTP API. You can learn From 3078d7fe58e2032731c8474251737ac254f2689d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Mon, 12 Jun 2023 16:50:20 +0200 Subject: [PATCH 16/23] add Go to build dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- .github/workflows/ci.yaml | 8 ++++++++ .github/workflows/release.yml | 2 ++ cli/README.md | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bac1c4f6..5d027b1c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,6 +31,8 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: toolchain: stable @@ -64,6 +66,8 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: toolchain: stable @@ -100,6 +104,8 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: toolchain: stable @@ -137,6 +143,8 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: toolchain: stable diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 73985d26..3e599316 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -89,6 +89,8 @@ jobs: - name: Setup | Checkout uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + - name: Setup | Rust uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: diff --git a/cli/README.md b/cli/README.md index 75d4879c..3ce41d84 100644 --- a/cli/README.md +++ b/cli/README.md @@ -33,6 +33,12 @@ If you have Rust tooling installed on your machine (see [Install Rust](https://www.rust-lang.org/tools/install)), you can build & install Zinnia from the source code. +In addition to the Rust build toolchain, you also need Go installed. See +[Go Downloads](https://go.dev/dl/). + +On Windows, Go uses `gcc` to create C libraries. Go recommends installing +[TDM GCC](https://jmeubank.github.io/tdm-gcc/). + ```sh $ cargo install zinnia ``` From 75f8239060a4a22ba5bba0b465f2b80e6b4b0cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Mon, 12 Jun 2023 17:00:23 +0200 Subject: [PATCH 17/23] fix a bug in test assertions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- runtime/tests/js/ipfs_retrieval_tests.js | 33 ++++++++++++------------ 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/runtime/tests/js/ipfs_retrieval_tests.js b/runtime/tests/js/ipfs_retrieval_tests.js index 2194f647..794f0877 100644 --- a/runtime/tests/js/ipfs_retrieval_tests.js +++ b/runtime/tests/js/ipfs_retrieval_tests.js @@ -6,14 +6,10 @@ const EXPECTED_CAR_BASE64 = test("can retrieve CID content as a CAR file", async () => { const requestUrl = "ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni"; - const response = await fetch(requestUrl); - if (!response.ok) { - throw new AssertionError( - `Fetch request failed with status code ${response.status}: ${await response.body()}`, - ); - } - const payload = await response.arrayBuffer(); + assertResponseIsOk(response); + + payload = await response.arrayBuffer(); assertEquals(payload.byteLength, 167, "CAR size in bytes"); const payload_encoded = btoa(String.fromCharCode(...new Uint8Array(payload))); @@ -25,11 +21,7 @@ test("can retrieve CID content as a CAR file", async () => { test("can retrieve IPFS content using URL", async () => { const requestUrl = new URL("ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni"); const response = await fetch(requestUrl); - if (!response.ok) { - throw new AssertionError( - `Fetch request failed with status code ${response.status}: ${await response.body()}`, - ); - } + assertResponseIsOk(response); const payload = await response.arrayBuffer(); assertEquals(payload.byteLength, 167, "CAR size in bytes"); @@ -40,14 +32,21 @@ test("can retrieve IPFS content using URL", async () => { test("can retrieve IPFS content using Fetch Request object", async () => { const request = new Request("ipfs://bafybeib36krhffuh3cupjml4re2wfxldredkir5wti3dttulyemre7xkni"); const response = await fetch(request); - if (!response.ok) { - throw new AssertionError( - `Fetch request failed with status code ${response.status}: ${await response.body()}`, - ); - } + assertResponseIsOk(response); const payload = await response.arrayBuffer(); assertEquals(payload.byteLength, 167, "CAR size in bytes"); assertEquals(response.url, request.url); }); + +/** + * @param {Response} response Fetch API response + */ +async function assertResponseIsOk(response) { + if (!response.ok) { + throw new AssertionError( + `Fetch request failed with status code ${response.status}: ${await response.text()}`, + ); + } +} From 00e396dd547e4b09580187f900896ad2095943e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Mon, 12 Jun 2023 17:21:02 +0200 Subject: [PATCH 18/23] tweak setup-go config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- .github/workflows/ci.yaml | 12 ++++++++++++ .github/workflows/release.yml | 3 +++ 2 files changed, 15 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5d027b1c..2e38c02a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,6 +32,9 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 + with: + go-version: latest + cache: false # caching requires a go.sum file, which we don't have in our project - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: @@ -67,6 +70,9 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 + with: + go-version: latest + cache: false # caching requires a go.sum file, which we don't have in our project - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: @@ -105,6 +111,9 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 + with: + go-version: latest + cache: false # caching requires a go.sum file, which we don't have in our project - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: @@ -144,6 +153,9 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 + with: + go-version: latest + cache: false # caching requires a go.sum file, which we don't have in our project - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3e599316..360a4002 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -90,6 +90,9 @@ jobs: uses: actions/checkout@v3 - uses: actions/setup-go@v4 + with: + go-version: latest + cache: false # caching requires a go.sum file, which we don't have in our project - name: Setup | Rust uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 From 78f629aedfba0225e26032178e8567a69ceeb976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Tue, 13 Jun 2023 10:44:12 +0200 Subject: [PATCH 19/23] fixup!: go-version latest -> stable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- .github/workflows/ci.yaml | 8 ++++---- .github/workflows/release.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2e38c02a..51aa055d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,7 +33,7 @@ jobs: - uses: actions/setup-go@v4 with: - go-version: latest + go-version: stable cache: false # caching requires a go.sum file, which we don't have in our project - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 @@ -71,7 +71,7 @@ jobs: - uses: actions/setup-go@v4 with: - go-version: latest + go-version: stable cache: false # caching requires a go.sum file, which we don't have in our project - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 @@ -112,7 +112,7 @@ jobs: - uses: actions/setup-go@v4 with: - go-version: latest + go-version: stable cache: false # caching requires a go.sum file, which we don't have in our project - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 @@ -154,7 +154,7 @@ jobs: - uses: actions/setup-go@v4 with: - go-version: latest + go-version: stable cache: false # caching requires a go.sum file, which we don't have in our project - uses: dtolnay/rust-toolchain@52e69531e6f69a396bc9d1226284493a5db969ff # v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 360a4002..01feb693 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,7 +91,7 @@ jobs: - uses: actions/setup-go@v4 with: - go-version: latest + go-version: stable cache: false # caching requires a go.sum file, which we don't have in our project - name: Setup | Rust From 7db4ee8c4ba49157d3c16affee5d98553cd8e540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Tue, 13 Jun 2023 10:49:20 +0200 Subject: [PATCH 20/23] fixup! improve code comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- runtime/js/fetch.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runtime/js/fetch.js b/runtime/js/fetch.js index f722534e..498293a5 100644 --- a/runtime/js/fetch.js +++ b/runtime/js/fetch.js @@ -12,7 +12,10 @@ export function setLassieUrl(/** @type {string} */ value) { export function fetch(resource, options) { let request = new Request(resource, options); - // Fortunately, Request#url is a string, not an instance of URL class + // The `resource` arg can be a string or any other object with a stringifier — including a URL + // object — that provides the URL of the resource you want to fetch; or a Request object. + // See https://developer.mozilla.org/en-US/docs/Web/API/fetch#parameters + // Fortunately, Request's constructor handles the conversions, and Request#url is always a string. // See https://developer.mozilla.org/en-US/docs/Web/API/Request/url if (request.url.startsWith(ipfsScheme)) { return fetchFromIpfs(request); From 371b24a05106d37058c8f2c6c54dde18ec8f1306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Tue, 13 Jun 2023 13:29:10 +0200 Subject: [PATCH 21/23] fixup! remove non-ASCII characters from fetch.js MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- runtime/js/fetch.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/js/fetch.js b/runtime/js/fetch.js index 498293a5..2ead4ae0 100644 --- a/runtime/js/fetch.js +++ b/runtime/js/fetch.js @@ -12,8 +12,8 @@ export function setLassieUrl(/** @type {string} */ value) { export function fetch(resource, options) { let request = new Request(resource, options); - // The `resource` arg can be a string or any other object with a stringifier — including a URL - // object — that provides the URL of the resource you want to fetch; or a Request object. + // The `resource` arg can be a string or any other object with a stringifier - including a URL + // object - that provides the URL of the resource you want to fetch; or a Request object. // See https://developer.mozilla.org/en-US/docs/Web/API/fetch#parameters // Fortunately, Request's constructor handles the conversions, and Request#url is always a string. // See https://developer.mozilla.org/en-US/docs/Web/API/Request/url From b2e1ccaf6ffd7a102616b1575223e3f2fd012b33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Tue, 13 Jun 2023 13:40:04 +0200 Subject: [PATCH 22/23] fix syntax error to fix a failing test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- runtime/tests/js/ipfs_retrieval_tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/tests/js/ipfs_retrieval_tests.js b/runtime/tests/js/ipfs_retrieval_tests.js index 794f0877..08f6012b 100644 --- a/runtime/tests/js/ipfs_retrieval_tests.js +++ b/runtime/tests/js/ipfs_retrieval_tests.js @@ -9,7 +9,7 @@ test("can retrieve CID content as a CAR file", async () => { const response = await fetch(requestUrl); assertResponseIsOk(response); - payload = await response.arrayBuffer(); + const payload = await response.arrayBuffer(); assertEquals(payload.byteLength, 167, "CAR size in bytes"); const payload_encoded = btoa(String.fromCharCode(...new Uint8Array(payload))); From 831b954c29d39ee504efe686dd41da3a1c519752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Tue, 13 Jun 2023 13:40:14 +0200 Subject: [PATCH 23/23] document temp_dir setting in zinnia CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- cli/main.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cli/main.rs b/cli/main.rs index 1db2b18e..df03f920 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -38,7 +38,23 @@ async fn main_impl() -> Result<()> { let lassie_daemon = Arc::new( lassie::Daemon::start(lassie::DaemonConfig { - temp_dir: None, // TODO: Should we use something like ~/.cache/zinnia/lassie? + // This configuration applies to `zinnia` CLI only. The `zinniad` daemon running + // inside Station uses a different temp_dir config based on the env var + // `CACHE_ROOT` provided by the Station. + // + // By default, Lassie stores its temporary files in the system temp directory. + // That's good enough for now. We can improve this later based on user feedback, + // for example: + // - we can honour CACHE_ROOT + // - we can default to something like + // `~/.cache/zinnia/lassie` on Unix, + // `%APPLOCALDATA%\zinnia\lassie' on Windows. + // + // Important: if we tell Lassie to use a specific temp dir that's not + // automatically cleaned by the operating system, we will need to clean any + // leftover files ourselves. See the GH issue for deleting leftover files + // when `zinniad` starts: https://github.com/filecoin-station/zinnia/issues/245 + temp_dir: None, port: 0, }) .context("cannot initialize the IPFS retrieval client Lassie")?,