Skip to content

Commit

Permalink
Update deno crates to latest version (#105)
Browse files Browse the repository at this point in the history 
* build: update rust-toolchain.toml to use Rust 1.79 channel

Signed-off-by: jaudiger <[email protected]>

* build: update deno_core and serde_v8 dependencies to finalize the migration to latest version

Signed-off-by: jaudiger <[email protected]>

* build: update deno_core to 0.300.0

Signed-off-by: jaudiger <[email protected]>

* Spawn a single-threaded Tokio runtime for Deno Core

Squashed commit of the following:

commit 960fc46
Author: Kyle Lacy <[email protected]>
Date:   Tue Aug 6 01:03:14 2024 -0700

    Add some comments

commit ee030ea
Author: Kyle Lacy <[email protected]>
Date:   Tue Aug 6 00:46:34 2024 -0700

    Combine the different Deno task types into new `RuntimeBridge` type

commit d1a73b7
Author: Kyle Lacy <[email protected]>
Date:   Tue Aug 6 00:02:47 2024 -0700

    Update task runners to spawn new Tokio tasks to handle messages

commit cefb0ac
Author: Kyle Lacy <[email protected]>
Date:   Mon Aug 5 23:43:38 2024 -0700

    Fix warnings

commit 55d7b86
Author: Kyle Lacy <[email protected]>
Date:   Mon Aug 5 23:07:03 2024 -0700

    Update `lsp` module to use separate Tokio runtime for Deno

commit f31e5c6
Author: Kyle Lacy <[email protected]>
Date:   Mon Aug 5 22:12:58 2024 -0700

    Update `check` module to se separate Tokio runtime for Deno

commit fae9f7a
Author: Kyle Lacy <[email protected]>
Date:   Sun Aug 4 21:59:30 2024 -0700

    Fix deadlock when resolving promises

commit eda87b0
Author: Kyle Lacy <[email protected]>
Date:   Thu Aug 1 02:40:15 2024 -0700

    [WIP] Use second runtime with channels for Deno

* Update `deno_core` to 0.303.0

---------

Signed-off-by: jaudiger <[email protected]>
Co-authored-by: Kyle Lacy <[email protected]>
  • Loading branch information
@jaudiger @kylewlacy
jaudiger and kylewlacy authored Aug 7, 2024
1 parent d09edd5 commit a06c8f4
Show file tree
Hide file tree
Showing 10 changed files with 1,236 additions and 568 deletions.
254 changes: 168 additions & 86 deletions Cargo.lock
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions crates/brioche-core/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ cfg-if = "1.0.0"
console-subscriber = "0.4.0"
debug-ignore = "1.0.5"
deno_ast = { version = "0.41.1", features = ["transpiling"] }
deno_core = "0.237.0"
deno_core = "0.303.0"
directories = "5.0.1"
futures = "0.3.29"
globset = "0.4.14"
Expand All @@ -42,7 +42,7 @@ reqwest-retry = "0.6.0"
rust-embed = { version = "8.1.0", features = ["debug-embed", "interpolate-folder-path", "include-exclude"] }
serde = { version = "1.0.193", features = ["derive"] }
serde_json = "1.0.108"
serde_v8 = "0.146.0"
serde_v8 = "0.212.0"
serde_with = { version = "3.4.0", features = ["hex"] }
sha2 = "0.10.8"
sqlx = { version = "0.7.3", features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate", "json"] }
Expand Down
232 changes: 110 additions & 122 deletions crates/brioche-core/src/script.rs
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,25 @@
use std::{
cell::RefCell,
collections::{hash_map::Entry, HashMap},
path::PathBuf,
rc::Rc,
sync::Arc,
};

use anyhow::Context as _;
use bridge::RuntimeBridge;
use deno_core::OpState;
use joinery::JoinableIterator as _;
use specifier::BriocheModuleSpecifier;

use crate::{
bake::BakeScope,
project::analyze::{StaticInclude, StaticQuery},
};
use crate::{bake::BakeScope, project::analyze::StaticQuery};

use super::{
blob::BlobHash,
project::Projects,
recipe::{Artifact, Recipe, WithMeta},
script::specifier::BriocheImportSpecifier,
Brioche,
};

mod bridge;
pub mod check;
mod compiler_host;
pub mod evaluate;
Expand All @@ -33,19 +30,74 @@ pub mod specifier;

#[derive(Clone)]
struct BriocheModuleLoader {
pub brioche: Brioche,
pub projects: Projects,
pub bridge: RuntimeBridge,
pub sources: Rc<RefCell<HashMap<BriocheModuleSpecifier, ModuleSource>>>,
}

impl BriocheModuleLoader {
fn new(brioche: &Brioche, projects: &Projects) -> Self {
fn new(bridge: RuntimeBridge) -> Self {
Self {
brioche: brioche.clone(),
projects: projects.clone(),
bridge,
sources: Rc::new(RefCell::new(HashMap::new())),
}
}

fn load_module_source(
&self,
module_specifier: &deno_core::ModuleSpecifier,
) -> Result<deno_core::ModuleSource, anyhow::Error> {
let brioche_module_specifier: BriocheModuleSpecifier = module_specifier.try_into()?;
let contents = self
.bridge
.read_specifier_contents(brioche_module_specifier.clone())?;

let code = std::str::from_utf8(&contents)
.context("failed to parse module contents as UTF-8 string")?;

let parsed = deno_ast::parse_module(deno_ast::ParseParams {
specifier: brioche_module_specifier.clone().into(),
text: code.into(),
media_type: deno_ast::MediaType::TypeScript,
capture_tokens: false,
scope_analysis: false,
maybe_syntax: None,
})?;
let transpiled = parsed.transpile(
&deno_ast::TranspileOptions {
imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Preserve,
..Default::default()
},
&deno_ast::EmitOptions {
source_map: deno_ast::SourceMapOption::Separate,
..Default::default()
},
)?;

if let Entry::Vacant(entry) = self
.sources
.borrow_mut()
.entry(brioche_module_specifier.clone())
{
let source_map = transpiled
.clone()
.into_source()
.source_map
.context("source map not generated")?;
entry.insert(ModuleSource {
source_contents: contents.clone(),
source_map,
});
}

Ok(deno_core::ModuleSource::new(
deno_core::ModuleType::JavaScript,
deno_core::ModuleSourceCode::Bytes(
transpiled.into_source().source.into_boxed_slice().into(),
),
module_specifier,
None,
))
}
}

impl deno_core::ModuleLoader for BriocheModuleLoader {
Expand All @@ -63,7 +115,10 @@ impl deno_core::ModuleLoader for BriocheModuleLoader {

let referrer: BriocheModuleSpecifier = referrer.parse()?;
let specifier: BriocheImportSpecifier = specifier.parse()?;
let resolved = specifier::resolve(&self.projects, &specifier, &referrer)?;

let resolved = self
.bridge
.resolve_specifier(specifier.clone(), referrer.clone())?;

tracing::debug!(%specifier, %referrer, %resolved, "resolved module");

Expand All @@ -76,70 +131,19 @@ impl deno_core::ModuleLoader for BriocheModuleLoader {
module_specifier: &deno_core::ModuleSpecifier,
_maybe_referrer: Option<&deno_core::ModuleSpecifier>,
_is_dyn_import: bool,
) -> std::pin::Pin<Box<deno_core::ModuleSourceFuture>> {
let module_specifier: Result<BriocheModuleSpecifier, _> = module_specifier.try_into();
let sources = self.sources.clone();
let vfs = self.brioche.vfs.clone();
let future = async move {
let module_specifier = module_specifier?;
let contents = specifier::read_specifier_contents(&vfs, &module_specifier)?;

let code = std::str::from_utf8(&contents)
.context("failed to parse module contents as UTF-8 string")?;

let parsed = deno_ast::parse_module(deno_ast::ParseParams {
specifier: module_specifier.clone().into(),
text: code.into(),
media_type: deno_ast::MediaType::TypeScript,
capture_tokens: false,
scope_analysis: false,
maybe_syntax: None,
})?;
let transpiled = parsed.transpile(
&deno_ast::TranspileOptions {
imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Preserve,
..Default::default()
},
&deno_ast::EmitOptions {
source_map: deno_ast::SourceMapOption::Separate,
..Default::default()
},
)?;

let mut sources = sources.borrow_mut();
if let Entry::Vacant(entry) = sources.entry(module_specifier.clone()) {
let source_map = transpiled
.clone()
.into_source()
.source_map
.context("source map not generated")?;
entry.insert(ModuleSource {
source_contents: contents.clone(),
source_map,
});
}

let module_specifier: deno_core::ModuleSpecifier = module_specifier.into();
Ok(deno_core::ModuleSource::new(
deno_core::ModuleType::JavaScript,
transpiled.into_source().into_string()?.text.into(),
&module_specifier,
))
};

Box::pin(future)
_requested_module_type: deno_core::RequestedModuleType,
) -> deno_core::ModuleLoadResponse {
deno_core::ModuleLoadResponse::Sync(self.load_module_source(module_specifier))
}
}

impl deno_core::SourceMapGetter for BriocheModuleLoader {
fn get_source_map(&self, file_name: &str) -> Option<Vec<u8>> {
let sources = self.sources.borrow();
let specifier: BriocheModuleSpecifier = file_name.parse().ok()?;
let code = sources.get(&specifier)?;
Some(code.source_map.clone())
}

fn get_source_line(&self, file_name: &str, line_number: usize) -> Option<String> {
fn get_source_mapped_source_line(&self, file_name: &str, line_number: usize) -> Option<String> {
let sources = self.sources.borrow();
let specifier: BriocheModuleSpecifier = file_name.parse().ok()?;
let source = sources.get(&specifier)?;
Expand All @@ -158,6 +162,22 @@ struct ModuleSource {
pub source_map: Vec<u8>,
}

pub enum ModuleLoaderMessage {
ReadSpecifierContents {
specifier: BriocheModuleSpecifier,
contents_tx: std::sync::mpsc::Sender<anyhow::Result<Arc<Vec<u8>>>>,
},
ResolveSpecifier {
specifier: BriocheImportSpecifier,
referrer: BriocheModuleSpecifier,
resolved_tx: std::sync::mpsc::Sender<anyhow::Result<BriocheModuleSpecifier>>,
},
ProjectRootForModulePath {
module_path: PathBuf,
project_path_tx: tokio::sync::oneshot::Sender<anyhow::Result<PathBuf>>,
},
}

deno_core::extension!(brioche_rt,
ops = [
op_brioche_bake_all,
Expand All @@ -166,13 +186,11 @@ deno_core::extension!(brioche_rt,
op_brioche_get_static,
],
options = {
brioche: Brioche,
projects: Projects,
bridge: RuntimeBridge,
bake_scope: BakeScope,
},
state = |state, options| {
state.put(options.brioche);
state.put(options.projects);
state.put(options.bridge);
state.put(options.bake_scope);
},
);
Expand All @@ -183,11 +201,11 @@ pub async fn op_brioche_bake_all(
state: Rc<RefCell<OpState>>,
#[serde] recipes: Vec<WithMeta<Recipe>>,
) -> Result<Vec<Artifact>, deno_core::error::AnyError> {
let brioche = {
let bridge = {
let state = state.try_borrow()?;
state
.try_borrow::<Brioche>()
.context("failed to get brioche instance")?
.try_borrow::<RuntimeBridge>()
.context("failed to get runtime bridge")?
.clone()
};
let bake_scope = {
Expand All @@ -198,11 +216,9 @@ pub async fn op_brioche_bake_all(
.clone()
};

let mut results = vec![];
for recipe in recipes {
let result = super::bake::bake(&brioche, recipe, &bake_scope).await?;
results.push(result.value);
}
let results = bridge.bake_all(recipes.clone(), bake_scope.clone()).await?;
let results = results.into_iter().map(|result| result.value).collect();

Ok(results)
}

Expand All @@ -212,15 +228,15 @@ pub async fn op_brioche_create_proxy(
state: Rc<RefCell<OpState>>,
#[serde] recipe: Recipe,
) -> Result<Recipe, deno_core::error::AnyError> {
let brioche = {
let bridge = {
let state = state.try_borrow()?;
state
.try_borrow::<Brioche>()
.context("failed to get brioche instance")?
.try_borrow::<RuntimeBridge>()
.context("failed to get runtime bridge")?
.clone()
};

let result = super::bake::create_proxy(&brioche, recipe).await?;
let result = bridge.create_proxy(recipe).await?;
Ok(result)
}

Expand All @@ -231,20 +247,15 @@ pub async fn op_brioche_read_blob(
state: Rc<RefCell<OpState>>,
#[serde] blob_hash: BlobHash,
) -> Result<crate::encoding::TickEncode<Vec<u8>>, deno_core::error::AnyError> {
let brioche = {
let bridge = {
let state = state.try_borrow()?;
state
.try_borrow::<Brioche>()
.context("failed to get brioche instance")?
.try_borrow::<RuntimeBridge>()
.context("failed to get runtime bridge")?
.clone()
};

let permit = crate::blob::get_save_blob_permit().await?;
let path = crate::blob::blob_path(&brioche, permit, blob_hash).await?;
let bytes = tokio::fs::read(path)
.await
.with_context(|| format!("failed to read blob {blob_hash}"))?;

let bytes = bridge.read_blob(blob_hash).await?;
Ok(crate::encoding::TickEncode(bytes))
}

Expand All @@ -255,39 +266,16 @@ pub async fn op_brioche_get_static(
#[string] url: String,
#[serde] static_: StaticQuery,
) -> Result<Recipe, deno_core::error::AnyError> {
let (brioche, projects) = {
let bridge = {
let state = state.try_borrow()?;
let brioche = state
.try_borrow::<Brioche>()
.context("failed to get brioche instance")?
.clone();
let projects = state
.try_borrow::<Projects>()
.context("failed to get projects instance")?
.clone();
(brioche, projects)
state
.try_borrow::<RuntimeBridge>()
.context("failed to get runtime bridge")?
.clone()
};

let specifier: BriocheModuleSpecifier = url.parse()?;

let recipe_hash = projects
.get_static(&specifier, &static_)?
.with_context(|| match static_ {
StaticQuery::Include(StaticInclude::File { path }) => {
format!("failed to resolve Brioche.includeFile({path:?}) from {specifier}, was the path passed in as a string literal?")
}
StaticQuery::Include(StaticInclude::Directory { path }) => {
format!("failed to resolve Brioche.includeDirectory({path:?}) from {specifier}, was the path passed in as a string literal?")
}
StaticQuery::Glob { patterns } => {
let patterns = patterns.iter().map(|pattern| lazy_format::lazy_format!("{pattern:?}")).join_with(", ");
format!("failed to resolve Brioche.glob({patterns}) from {specifier}, were the patterns passed in as string literals?")
}
StaticQuery::Download { url } => {
format!("failed to resolve Brioche.download({url:?}) from {specifier}, was the URL passed in as a string literal?")
}

})?;
let recipe = crate::recipe::get_recipe(&brioche, recipe_hash).await?;
let recipe = bridge.get_static(specifier, static_).await?;
Ok(recipe)
}
Loading

0 comments on commit a06c8f4

Please sign in to comment.