Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: infer async modules when incremental enabled #7927

Merged
merged 6 commits into from
Sep 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions crates/rspack_core/src/chunk_graph/chunk_graph_module.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ use tracing::instrument;

use crate::{
get_chunk_group_from_ukey, AsyncDependenciesBlockIdentifier, ChunkByUkey, ChunkGroup,
ChunkGroupByUkey, ChunkGroupUkey, ChunkUkey, Compilation, ModuleIdentifier, RuntimeGlobals,
RuntimeSpec, RuntimeSpecMap, RuntimeSpecSet,
ChunkGroupByUkey, ChunkGroupUkey, ChunkUkey, Compilation, ModuleGraph, ModuleIdentifier,
RuntimeGlobals, RuntimeSpec, RuntimeSpecMap, RuntimeSpecSet,
};
use crate::{ChunkGraph, Module};

Expand Down Expand Up @@ -219,7 +219,7 @@ impl ChunkGraph {
let cgm = self.get_chunk_graph_module(module_identifier);
cgm.id.as_ref().dyn_hash(&mut hasher);
module.source_types().dyn_hash(&mut hasher);
mg.is_async(&module_identifier).dyn_hash(&mut hasher);
ModuleGraph::is_async(compilation, &module_identifier).dyn_hash(&mut hasher);
mg.get_exports_info(&module_identifier)
.update_hash(&mg, &mut hasher, compilation, runtime);
hasher.finish()
Expand Down
100 changes: 39 additions & 61 deletions crates/rspack_core/src/compiler/compilation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,14 @@ use crate::{
get_chunk_from_ukey, get_mut_chunk_from_ukey, is_source_equal,
old_cache::{use_code_splitting_cache, Cache as OldCache, CodeSplittingCache},
to_identifier,
unaffected_cache::UnaffectedModulesCache,
unaffected_cache::{Mutation, Mutations, UnaffectedModulesCache},
BoxDependency, BoxModule, CacheCount, CacheOptions, Chunk, ChunkByUkey, ChunkContentHash,
ChunkGraph, ChunkGroupByUkey, ChunkGroupUkey, ChunkKind, ChunkUkey, CodeGenerationJob,
CodeGenerationResult, CodeGenerationResults, CompilationLogger, CompilationLogging,
CompilerOptions, DependencyId, DependencyType, Entry, EntryData, EntryOptions, EntryRuntime,
Entrypoint, ExecuteModuleId, Filename, ImportVarMap, LocalFilenameFn, Logger, Module,
ModuleFactory, ModuleGraph, ModuleGraphPartial, ModuleIdentifier, PathData, ResolverFactory,
RuntimeGlobals, RuntimeModule, RuntimeSpecMap, SharedPluginDriver, SourceType, Stats,
Entrypoint, ExecuteModuleId, Filename, ImportVarMap, LocalFilenameFn, Logger, ModuleFactory,
ModuleGraph, ModuleGraphPartial, ModuleIdentifier, PathData, ResolverFactory, RuntimeGlobals,
RuntimeModule, RuntimeSpecMap, SharedPluginDriver, SourceType, Stats,
};

pub type BuildDependency = (
Expand Down Expand Up @@ -164,6 +164,7 @@ pub struct Compilation {
pub named_chunks: HashMap<String, ChunkUkey>,
pub named_chunk_groups: HashMap<String, ChunkGroupUkey>,

pub async_modules: IdentifierSet,
pub code_generation_results: CodeGenerationResults,
pub cgm_hash_results: CgmHashResults,
pub cgm_runtime_requirements_results: CgmRuntimeRequirementsResults,
Expand All @@ -173,6 +174,7 @@ pub struct Compilation {
pub old_cache: Arc<OldCache>,
pub code_splitting_cache: CodeSplittingCache,
pub unaffected_modules_cache: Arc<UnaffectedModulesCache>,
pub mutations: Option<Mutations>,

pub hash: Option<RspackHashDigest>,
pub used_chunk_ids: HashSet<String>,
Expand Down Expand Up @@ -227,6 +229,7 @@ impl Compilation {
modified_files: HashSet<PathBuf>,
removed_files: HashSet<PathBuf>,
) -> Self {
let mutations = options.new_incremental_enabled().then(Mutations::default);
Self {
id: CompilationId::new(),
hot_index: 0,
Expand Down Expand Up @@ -254,6 +257,7 @@ impl Compilation {
named_chunks: Default::default(),
named_chunk_groups: Default::default(),

async_modules: Default::default(),
code_generation_results: Default::default(),
cgm_hash_results: Default::default(),
cgm_runtime_requirements_results: Default::default(),
Expand All @@ -262,6 +266,7 @@ impl Compilation {
build_time_executed_modules: Default::default(),
old_cache,
unaffected_modules_cache,
mutations,
code_splitting_cache: Default::default(),
hash: None,
used_chunk_ids: Default::default(),
Expand Down Expand Up @@ -757,57 +762,30 @@ impl Compilation {
.collect::<Vec<_>>()))
}

#[instrument(name = "compilation:code_generation", skip(self))]
fn code_generation(&mut self) -> Result<()> {
#[instrument(name = "compilation:code_generation", skip_all)]
fn code_generation(&mut self, modules: IdentifierSet) -> Result<()> {
let logger = self.get_logger("rspack.Compilation");
let mut codegen_cache_counter = match self.options.cache {
CacheOptions::Disabled => None,
_ => Some(logger.cache("module code generation cache")),
};

fn run_iteration(
compilation: &mut Compilation,
cache_counter: &mut Option<CacheCount>,
filter_op: impl Fn(&(ModuleIdentifier, &Box<dyn Module>)) -> bool + Sync + Send,
) -> Result<()> {
let module_graph = compilation.get_module_graph();
let modules: IdentifierSet = if compilation.options.new_incremental_enabled() {
let affected_modules = compilation
.unaffected_modules_cache
.get_affected_modules_with_chunk_graph()
.lock()
.expect("should lock")
.clone();
affected_modules
.into_iter()
.map(|module_identifier| {
let module = module_graph
.module_by_identifier(&module_identifier)
.expect("should have module");
(module_identifier, module)
})
.filter(filter_op)
.map(|(id, _)| id)
.collect()
let module_graph = self.get_module_graph();
let mut no_codegen_dependencies_modules = IdentifierSet::default();
let mut has_codegen_dependencies_modules = IdentifierSet::default();
for module_identifier in modules {
let module = module_graph
.module_by_identifier(&module_identifier)
.expect("should have module");
if module.get_code_generation_dependencies().is_none() {
no_codegen_dependencies_modules.insert(module_identifier);
} else {
compilation
.get_module_graph()
.modules()
.into_iter()
.filter(filter_op)
.map(|(id, _)| id)
.collect()
};
compilation.code_generation_modules(cache_counter, modules)
has_codegen_dependencies_modules.insert(module_identifier);
}
}

run_iteration(self, &mut codegen_cache_counter, |(_, module)| {
module.get_code_generation_dependencies().is_none()
})?;

run_iteration(self, &mut codegen_cache_counter, |(_, module)| {
module.get_code_generation_dependencies().is_some()
})?;
self.code_generation_modules(&mut codegen_cache_counter, no_codegen_dependencies_modules)?;
self.code_generation_modules(&mut codegen_cache_counter, has_codegen_dependencies_modules)?;

if let Some(counter) = codegen_cache_counter {
logger.cache_end(counter);
Expand Down Expand Up @@ -1216,16 +1194,25 @@ impl Compilation {
.compute_affected_modules_with_chunk_graph(self);
}

self.create_module_hashes(if self.options.new_incremental_enabled() {
self
let modules = if self.options.new_incremental_enabled()
&& let Some(mutations) = &self.mutations
{
let mut modules = self
.unaffected_modules_cache
.get_affected_modules_with_chunk_graph()
.lock()
.expect("should lock")
.clone()
.clone();
modules.extend(mutations.iter().filter_map(|mutation| match mutation {
Mutation::ModuleGraphModuleSetAsync { module } => Some(module),
_ => None,
}));
modules
} else {
self.get_module_graph().modules().keys().copied().collect()
})?;
};

self.create_module_hashes(modules.clone())?;

let start = logger.time("optimize code generation");
plugin_driver
Expand All @@ -1235,22 +1222,13 @@ impl Compilation {
logger.time_end(start);

let start = logger.time("code generation");
self.code_generation()?;
self.code_generation(modules.clone())?;
logger.time_end(start);

let start = logger.time("runtime requirements");
self
.process_runtime_requirements(
if self.options.new_incremental_enabled() {
self
.unaffected_modules_cache
.get_affected_modules_with_chunk_graph()
.lock()
.expect("should lock")
.clone()
} else {
self.get_module_graph().modules().keys().copied().collect()
},
modules,
self
.chunk_by_ukey
.keys()
Expand Down
1 change: 1 addition & 0 deletions crates/rspack_core/src/compiler/hmr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ impl Compiler {
}

if self.options.new_incremental_enabled() {
new_compilation.async_modules = std::mem::take(&mut self.compilation.async_modules);
new_compilation.cgm_hash_results = std::mem::take(&mut self.compilation.cgm_hash_results);
new_compilation.code_generation_results =
std::mem::take(&mut self.compilation.code_generation_results);
Expand Down
14 changes: 6 additions & 8 deletions crates/rspack_core/src/dependency/runtime_template.rs
Original file line number Diff line number Diff line change
Expand Up @@ -472,14 +472,12 @@ pub fn module_namespace_promise(
fake_type |= FakeNamespaceObjectMode::MERGE_PROPERTIES;
}
runtime_requirements.insert(RuntimeGlobals::CREATE_FAKE_NAMESPACE_OBJECT);
if matches!(
compilation.get_module_graph().is_async(
compilation
.get_module_graph()
.module_identifier_by_dependency_id(dep_id)
.expect("should have module")
),
Some(true)
if ModuleGraph::is_async(
compilation,
compilation
.get_module_graph()
.module_identifier_by_dependency_id(dep_id)
.expect("should have module"),
) {
if let Some(header) = header {
appending = format!(
Expand Down
2 changes: 1 addition & 1 deletion crates/rspack_core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ mod cgm_hash_results;
mod cgm_runtime_requirement_results;
mod dependencies_block;
pub mod diagnostics;
mod unaffected_cache;
pub mod unaffected_cache;
pub use dependencies_block::{
AsyncDependenciesBlock, AsyncDependenciesBlockIdentifier, DependenciesBlock,
};
Expand Down
2 changes: 1 addition & 1 deletion crates/rspack_core/src/module.rs
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ pub struct BuildMeta {
}

// webpack build info
#[derive(Debug, Default, Clone)]
#[derive(Debug, Default)]
pub struct BuildResult {
/// Whether the result is cacheable, i.e shared between builds.
pub build_meta: BuildMeta,
Expand Down
38 changes: 24 additions & 14 deletions crates/rspack_core/src/module_graph/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
use swc_core::ecma::atoms::Atom;

use crate::{
AsyncDependenciesBlock, AsyncDependenciesBlockIdentifier, Dependency, ExportProvided,
ProvidedExports, RuntimeSpec, UsedExports,
AsyncDependenciesBlock, AsyncDependenciesBlockIdentifier, Compilation, Dependency,
ExportProvided, ProvidedExports, RuntimeSpec, UsedExports,
};
mod module;
pub use module::*;
Expand Down Expand Up @@ -370,11 +370,12 @@ impl<'a> ModuleGraph<'a> {

/// Make sure both source and target module are exists in module graph
pub fn clone_module_attributes(
&mut self,
compilation: &mut Compilation,
source_module: &ModuleIdentifier,
target_module: &ModuleIdentifier,
) {
let old_mgm = self
let mut module_graph = compilation.get_module_graph_mut();
let old_mgm = module_graph
.module_graph_module_by_identifier(source_module)
.expect("should have mgm");

Expand All @@ -384,16 +385,17 @@ impl<'a> ModuleGraph<'a> {
old_mgm.pre_order_index,
old_mgm.depth,
old_mgm.exports,
old_mgm.is_async,
);
let new_mgm = self
let new_mgm = module_graph
.module_graph_module_by_identifier_mut(target_module)
.expect("should have mgm");
new_mgm.post_order_index = assign_tuple.0;
new_mgm.pre_order_index = assign_tuple.1;
new_mgm.depth = assign_tuple.2;
new_mgm.exports = assign_tuple.3;
new_mgm.is_async = assign_tuple.4;

let is_async = ModuleGraph::is_async(compilation, source_module);
ModuleGraph::set_async(compilation, *target_module, is_async);
}

pub fn move_module_connections(
Expand Down Expand Up @@ -979,15 +981,23 @@ impl<'a> ModuleGraph<'a> {
has_connections
}

pub fn is_async(&self, module_id: &ModuleIdentifier) -> Option<bool> {
self
.module_graph_module_by_identifier(module_id)
.map(|mgm| mgm.is_async)
pub fn is_async(compilation: &Compilation, module_id: &ModuleIdentifier) -> bool {
compilation.async_modules.contains(module_id)
}

pub fn set_async(&mut self, module_id: &ModuleIdentifier) {
if let Some(mgm) = self.module_graph_module_by_identifier_mut(module_id) {
mgm.is_async = true
pub fn set_async(
compilation: &mut Compilation,
module_id: ModuleIdentifier,
is_async: bool,
) -> bool {
let original = Self::is_async(compilation, &module_id);
if original == is_async {
return false;
}
if original {
compilation.async_modules.remove(&module_id)
} else {
compilation.async_modules.insert(module_id)
}
}

Expand Down
2 changes: 0 additions & 2 deletions crates/rspack_core/src/module_graph/module.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ pub struct ModuleGraphModule {
pub post_order_index: Option<u32>,
pub exports: ExportsInfo,
pub profile: Option<Box<ModuleProfile>>,
pub is_async: bool,
pub depth: Option<usize>,
pub optimization_bailout: Vec<String>,
}
Expand All @@ -41,7 +40,6 @@ impl ModuleGraphModule {
post_order_index: None,
exports: exports_info,
profile: None,
is_async: false,
depth: None,
optimization_bailout: vec![],
}
Expand Down
3 changes: 3 additions & 0 deletions crates/rspack_core/src/unaffected_cache/mod.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
mod mutations;

use std::{
hash::{BuildHasherDefault, Hash, Hasher},
sync::Mutex,
};

pub use mutations::{Mutation, Mutations};
use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
use rspack_collections::{IdentifierDashMap, IdentifierHasher, IdentifierMap, IdentifierSet};
use rspack_util::fx_hash::FxIndexSet;
Expand Down
Loading
Loading