From 0a4a3bba3b9a3ae70ca316db78d98ee831ac2141 Mon Sep 17 00:00:00 2001 From: fireairforce <1344492820@qq.com> Date: Tue, 7 Apr 2026 14:40:37 +0800 Subject: [PATCH] feat(pack): support turbopack bundle analyzer --- Cargo.lock | 16 + Cargo.toml | 1 + crates/pack-api/Cargo.toml | 1 + crates/pack-api/src/analyze.rs | 587 +++++++++++++++++++++++ crates/pack-api/src/app.rs | 26 + crates/pack-api/src/lib.rs | 1 + crates/pack-api/src/library.rs | 11 + crates/pack-napi/src/pack_api/analyze.rs | 139 ++++++ crates/pack-napi/src/pack_api/mod.rs | 1 + crates/pack-napi/src/pack_api/project.rs | 33 ++ examples/with-react/package.json | 3 +- packages/pack-cli/src/commands/build.ts | 7 + packages/pack/src/binding.d.ts | 1 + packages/pack/src/binding.js | 3 +- packages/pack/src/commands/build.ts | 116 +++-- packages/pack/src/core/project.ts | 9 + packages/pack/src/core/types.ts | 1 + packages/pack/src/utils/common.ts | 2 +- 18 files changed, 920 insertions(+), 38 deletions(-) create mode 100644 crates/pack-api/src/analyze.rs create mode 100644 crates/pack-napi/src/pack_api/analyze.rs diff --git a/Cargo.lock b/Cargo.lock index 5dcf3ca34..2e9f8b36d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5502,6 +5502,7 @@ dependencies = [ "turbo-tasks-fs", "turbo-unix-path", "turbopack", + "turbopack-analyze", "turbopack-browser", "turbopack-core", "turbopack-css", @@ -10199,6 +10200,21 @@ dependencies = [ "turbopack-wasm", ] +[[package]] +name = "turbopack-analyze" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode 2.0.1", + "flate2", + "rustc-hash 2.1.1", + "serde", + "turbo-rcstr", + "turbo-tasks", + "turbo-tasks-fs", + "turbopack-core", +] + [[package]] name = "turbopack-browser" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 7f0556d08..207513940 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,7 @@ turbo-tasks-malloc = { path = "./next.js/turbopack/crates/turbo-t turbo-unix-path = { path = "./next.js/turbopack/crates/turbo-unix-path" } ## # Dependencies for bundler specially. turbopack = { path = "./next.js/turbopack/crates/turbopack" } +turbopack-analyze = { path = "./next.js/turbopack/crates/turbopack-analyze" } turbopack-browser = { path = "./next.js/turbopack/crates/turbopack-browser" } turbopack-cli-utils = { path = "./next.js/turbopack/crates/turbopack-cli-utils" } turbopack-core = { path = "./next.js/turbopack/crates/turbopack-core" } diff --git a/crates/pack-api/Cargo.toml b/crates/pack-api/Cargo.toml index 6ced1f6bf..9643ce33d 100644 --- a/crates/pack-api/Cargo.toml +++ b/crates/pack-api/Cargo.toml @@ -29,6 +29,7 @@ turbo-tasks-env = { workspace = true } turbo-tasks-fs = { workspace = true } turbo-unix-path = { workspace = true } turbopack = { workspace = true } +turbopack-analyze = { workspace = true } turbopack-browser = { workspace = true } turbopack-core = { workspace = true } turbopack-css = { workspace = true } diff --git a/crates/pack-api/src/analyze.rs b/crates/pack-api/src/analyze.rs new file mode 100644 index 000000000..1e5ca648a --- /dev/null +++ b/crates/pack-api/src/analyze.rs @@ -0,0 +1,587 @@ +use std::{borrow::Cow, io::Write}; + +use anyhow::Result; +use rustc_hash::FxHashMap; +use serde::Serialize; +use turbo_rcstr::RcStr; +use turbo_tasks::{FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc}; +use turbo_tasks_fs::{ + File, FileContent, FileSystemPath, + rope::{Rope, RopeBuilder}, +}; +use turbopack_analyze::split_chunk::split_output_asset_into_parts; +use turbopack_core::{ + SOURCE_URL_PROTOCOL, + asset::{Asset, AssetContent}, + chunk::ChunkingType, + module::Module, + module_graph::ModuleGraph, + output::{OutputAsset, OutputAssets, OutputAssetsReference}, + reference::all_assets_from_entries, +}; + +pub struct EdgesData { + pub offsets: Vec, + pub data: Vec, +} + +impl EdgesData { + fn from_iterator<'a>(iterable: impl IntoIterator> + Clone) -> Self { + let mut current_offset = 0; + let sum: usize = iterable.clone().into_iter().map(|v| v.len()).sum(); + let mut data = Vec::with_capacity(sum); + let offsets = iterable + .into_iter() + .map(|edges| { + current_offset += edges.len() as u32; + data.extend(edges); + current_offset + }) + .collect(); + Self { offsets, data } + } + + fn write(&self, writer: &mut impl Write) -> Result<()> { + writer.write_all(&(self.offsets.len() as u32).to_be_bytes())?; + for &offset in &self.offsets { + writer.write_all(&offset.to_be_bytes())?; + } + for &data in &self.data { + writer.write_all(&data.to_be_bytes())?; + } + Ok(()) + } +} + +#[derive(Serialize)] +pub struct AnalyzeSource { + pub parent_source_index: Option, + pub path: RcStr, +} + +#[derive(Serialize)] +pub struct AnalyzeModule { + pub ident: RcStr, + pub path: RcStr, +} + +#[derive(Serialize)] +pub struct AnalyzeChunkPart { + pub source_index: u32, + pub output_file_index: u32, + pub size: u32, + pub compressed_size: u32, +} + +#[derive(Serialize)] +pub struct AnalyzeOutputFile { + pub filename: RcStr, +} + +#[derive(Serialize)] +struct EdgesDataReference { + pub offset: u32, + pub length: u32, +} + +#[derive(Serialize)] +struct AnalyzeDataHeader { + pub sources: Vec, + pub chunk_parts: Vec, + pub output_files: Vec, + pub output_file_chunk_parts: EdgesDataReference, + pub source_chunk_parts: EdgesDataReference, + pub source_children: EdgesDataReference, + pub source_roots: Vec, +} + +#[derive(Serialize)] +struct ModulesDataHeader { + pub modules: Vec, + pub module_dependents: EdgesDataReference, + pub async_module_dependents: EdgesDataReference, + pub module_dependencies: EdgesDataReference, + pub async_module_dependencies: EdgesDataReference, +} + +struct AnalyzeOutputFileBuilder { + output_file: AnalyzeOutputFile, + chunk_part_indices: Vec, +} + +struct AnalyzeSourceBuilder { + source: AnalyzeSource, + child_source_indices: Vec, + chunk_part_indices: Vec, +} + +struct AnalyzeModuleBuilder { + module: AnalyzeModule, + dependencies: FxIndexSet, + async_dependencies: FxIndexSet, + dependents: FxIndexSet, + async_dependents: FxIndexSet, +} + +struct AnalyzeDataBuilder { + sources: Vec, + source_index_map: FxHashMap, + chunk_parts: Vec, + output_files: Vec, +} + +struct ModulesDataBuilder { + modules: Vec, + module_index_map: FxHashMap, +} + +struct EdgesDataSectionBuilder { + data: Vec, +} + +impl EdgesDataSectionBuilder { + fn new() -> Self { + Self { data: vec![] } + } + + fn add_edges(&mut self, edges: &EdgesData) -> EdgesDataReference { + let offset = self.data.len().try_into().unwrap(); + edges.write(&mut self.data).unwrap(); + let length = (self.data.len() - offset as usize).try_into().unwrap(); + EdgesDataReference { offset, length } + } +} + +impl AnalyzeDataBuilder { + fn new() -> Self { + Self { + sources: vec![], + source_index_map: FxHashMap::default(), + chunk_parts: vec![], + output_files: vec![], + } + } + + fn ensure_source(&mut self, path: &str) -> (&mut AnalyzeSourceBuilder, u32) { + if let Some(&index) = self.source_index_map.get(path) { + return (&mut self.sources[index as usize], index); + } + let index = self.sources.len() as u32; + let path = RcStr::from(path); + self.source_index_map.insert(path.clone(), index); + self.sources.push(AnalyzeSourceBuilder { + source: AnalyzeSource { + parent_source_index: None, + path, + }, + child_source_indices: vec![], + chunk_part_indices: vec![], + }); + (&mut self.sources[index as usize], index) + } + + fn add_chunk_part(&mut self, chunk_part: AnalyzeChunkPart) -> u32 { + let i = self.chunk_parts.len() as u32; + self.chunk_parts.push(chunk_part); + i + } + + fn add_output_file(&mut self, output_file: AnalyzeOutputFile) -> u32 { + let i = self.output_files.len() as u32; + self.output_files.push(AnalyzeOutputFileBuilder { + output_file, + chunk_part_indices: vec![], + }); + i + } + + fn add_chunk_part_to_output_file(&mut self, output_file_index: u32, chunk_part_index: u32) { + self.output_files[output_file_index as usize] + .chunk_part_indices + .push(chunk_part_index); + } + + fn add_chunk_part_to_source(&mut self, source_index: u32, chunk_part_index: u32) { + self.sources[source_index as usize] + .chunk_part_indices + .push(chunk_part_index); + } + + fn build(self) -> Rope { + let source_roots = self + .sources + .iter() + .enumerate() + .filter_map(|(i, s)| { + if s.source.parent_source_index.is_none() { + Some(i as u32) + } else { + None + } + }) + .collect(); + + let source_children = + EdgesData::from_iterator(self.sources.iter().map(|s| &s.child_source_indices)); + + let source_chunk_parts = + EdgesData::from_iterator(self.sources.iter().map(|s| &s.chunk_part_indices)); + + let output_file_chunk_parts = + EdgesData::from_iterator(self.output_files.iter().map(|of| &of.chunk_part_indices)); + + let mut binary_section = EdgesDataSectionBuilder::new(); + + let header = AnalyzeDataHeader { + sources: self.sources.into_iter().map(|s| s.source).collect(), + chunk_parts: self.chunk_parts, + output_files: self + .output_files + .into_iter() + .map(|of| of.output_file) + .collect(), + output_file_chunk_parts: binary_section.add_edges(&output_file_chunk_parts), + source_chunk_parts: binary_section.add_edges(&source_chunk_parts), + source_children: binary_section.add_edges(&source_children), + source_roots, + }; + + let header_json = serde_json::to_vec(&header).unwrap(); + + let mut rope = RopeBuilder::default(); + rope.push_bytes(&(header_json.len() as u32).to_be_bytes()); + rope.reserve_bytes(header_json.len() + binary_section.data.len()); + rope.push_bytes(&header_json); + rope.push_bytes(&binary_section.data); + rope.build() + } +} + +impl ModulesDataBuilder { + fn new() -> Self { + Self { + modules: vec![], + module_index_map: FxHashMap::default(), + } + } + + fn get_module(&mut self, ident: &str) -> (&mut AnalyzeModuleBuilder, u32) { + if let Some(&index) = self.module_index_map.get(ident) { + return (&mut self.modules[index as usize], index); + } + panic!("Module with ident `{}` not found", ident); + } + + fn ensure_module(&mut self, ident: &str, path: &str) -> (&mut AnalyzeModuleBuilder, u32) { + if let Some(&index) = self.module_index_map.get(ident) { + return (&mut self.modules[index as usize], index); + } + let index = self.modules.len() as u32; + let ident = RcStr::from(ident); + let path = RcStr::from(path); + self.module_index_map.insert(ident.clone(), index); + self.modules.push(AnalyzeModuleBuilder { + module: AnalyzeModule { ident, path }, + dependencies: FxIndexSet::default(), + async_dependencies: FxIndexSet::default(), + dependents: FxIndexSet::default(), + async_dependents: FxIndexSet::default(), + }); + (&mut self.modules[index as usize], index) + } + + fn build(self) -> Rope { + let module_dependencies_vecs: Vec> = self + .modules + .iter() + .map(|s| s.dependencies.iter().copied().collect()) + .collect(); + let async_module_dependencies_vecs: Vec> = self + .modules + .iter() + .map(|s| s.async_dependencies.iter().copied().collect()) + .collect(); + let module_dependents_vecs: Vec> = self + .modules + .iter() + .map(|s| s.dependents.iter().copied().collect()) + .collect(); + let async_module_dependents_vecs: Vec> = self + .modules + .iter() + .map(|s| s.async_dependents.iter().copied().collect()) + .collect(); + + let module_dependencies = EdgesData::from_iterator(&module_dependencies_vecs); + let async_module_dependencies = EdgesData::from_iterator(&async_module_dependencies_vecs); + let module_dependents = EdgesData::from_iterator(&module_dependents_vecs); + let async_module_dependents = EdgesData::from_iterator(&async_module_dependents_vecs); + + let mut binary_section = EdgesDataSectionBuilder::new(); + + let header = ModulesDataHeader { + modules: self.modules.into_iter().map(|s| s.module).collect(), + module_dependents: binary_section.add_edges(&module_dependents), + async_module_dependents: binary_section.add_edges(&async_module_dependents), + module_dependencies: binary_section.add_edges(&module_dependencies), + async_module_dependencies: binary_section.add_edges(&async_module_dependencies), + }; + + let header_json = serde_json::to_vec(&header).unwrap(); + + let mut rope = RopeBuilder::default(); + rope.push_bytes(&(header_json.len() as u32).to_be_bytes()); + rope.reserve_bytes(header_json.len() + binary_section.data.len()); + rope.push_bytes(&header_json); + rope.push_bytes(&binary_section.data); + rope.build() + } +} + +#[turbo_tasks::value(shared, serialization = "none")] +#[derive(Clone)] +pub struct AnalyzeTarget { + pub name: RcStr, + pub output_assets: ResolvedVc, +} + +#[turbo_tasks::value(transparent, serialization = "none")] +pub struct AnalyzeTargets(pub Vec); + +#[turbo_tasks::value(transparent)] +pub struct ModuleGraphs(pub Vec>); + +#[turbo_tasks::function] +pub async fn analyze_output_assets(output_assets: Vc) -> Result> { + let output_assets = all_assets_from_entries(output_assets); + + let mut builder = AnalyzeDataBuilder::new(); + + let prefix = format!("{SOURCE_URL_PROTOCOL}///"); + + for &asset in output_assets.await? { + let filename = asset.path().to_string().owned().await?; + if filename.ends_with(".map") || filename.ends_with(".nft.json") { + continue; + } + + let output_file_index = builder.add_output_file(AnalyzeOutputFile { filename }); + let chunk_parts = split_output_asset_into_parts(*asset).await?; + for chunk_part in chunk_parts { + let decoded_source = urlencoding::decode(&chunk_part.source)?; + let source = if let Some(stripped) = decoded_source.strip_prefix(&prefix) { + Cow::Borrowed(stripped) + } else { + Cow::Owned(format!( + "[project]/{}", + decoded_source.trim_start_matches("../") + )) + }; + let source_index = builder.ensure_source(&source).1; + let chunk_part_index = builder.add_chunk_part(AnalyzeChunkPart { + source_index, + output_file_index, + size: chunk_part.real_size + chunk_part.unaccounted_size, + compressed_size: chunk_part.get_compressed_size().await?, + }); + builder.add_chunk_part_to_output_file(output_file_index, chunk_part_index); + builder.add_chunk_part_to_source(source_index, chunk_part_index); + } + } + + let mut i: u32 = 0; + while i < builder.sources.len().try_into().unwrap() { + let source = &builder.sources[i as usize]; + let path = source.source.path.as_str(); + if !path.is_empty() { + let (parent_path, path) = if let Some(pos) = path.trim_end_matches('/').rfind('/') { + (&path[..pos + 1], &path[pos + 1..]) + } else { + ("", path) + }; + let parent_path = parent_path.to_string(); + let path = path.into(); + let (parent_source, parent_index) = builder.ensure_source(&parent_path); + parent_source.child_source_indices.push(i); + builder.sources[i as usize].source.parent_source_index = Some(parent_index); + builder.sources[i as usize].source.path = path; + } + i += 1; + } + + let rope = builder.build(); + Ok(FileContent::Content(File::from(rope)).cell()) +} + +#[turbo_tasks::function] +pub async fn analyze_module_graphs(module_graphs: Vc) -> Result> { + let mut builder = ModulesDataBuilder::new(); + + let mut all_modules = FxIndexSet::default(); + let mut all_edges = FxIndexSet::default(); + let mut all_async_edges = FxIndexSet::default(); + for &module_graph in module_graphs.await? { + let module_graph = module_graph.await?; + module_graph.traverse_edges_unordered(|parent, node| { + if let Some((parent_node, reference)) = parent { + all_modules.insert(parent_node); + all_modules.insert(node); + match reference.chunking_type { + ChunkingType::Async => { + all_async_edges.insert((parent_node, node)); + } + _ => { + all_edges.insert((parent_node, node)); + } + } + } + Ok(()) + })?; + } + + type ModulePair = (ResolvedVc>, ResolvedVc>); + async fn mapper((from, to): ModulePair) -> Result> { + if from == to { + return Ok(None); + } + let from_ident = from.ident().to_string().owned().await?; + let to_ident = to.ident().to_string().owned().await?; + Ok(Some((from_ident, to_ident))) + } + + let all_modules = all_modules + .iter() + .copied() + .map(async |module| { + let ident = module.ident().to_string().owned().await?; + let path = module.ident().path().to_string().owned().await?; + Ok((ident, path)) + }) + .try_join() + .await?; + + for (ident, path) in all_modules { + builder.ensure_module(&ident, &path); + } + + let all_edges = all_edges + .iter() + .copied() + .map(mapper) + .try_flat_join() + .await?; + let all_async_edges = all_async_edges + .iter() + .copied() + .map(mapper) + .try_flat_join() + .await?; + for (from_ident, to_ident) in all_edges { + let from_index = builder.get_module(&from_ident).1; + let to_index = builder.get_module(&to_ident).1; + if from_index == to_index { + continue; + } + builder.modules[from_index as usize] + .dependencies + .insert(to_index); + builder.modules[to_index as usize] + .dependents + .insert(from_index); + } + for (from_ident, to_ident) in all_async_edges { + let from_index = builder.get_module(&from_ident).1; + let to_index = builder.get_module(&to_ident).1; + if from_index == to_index { + continue; + } + builder.modules[from_index as usize] + .async_dependencies + .insert(to_index); + builder.modules[to_index as usize] + .async_dependents + .insert(from_index); + } + + let rope = builder.build(); + Ok(FileContent::Content(File::from(rope)).cell()) +} + +#[turbo_tasks::value] +pub struct AnalyzeDataOutputAsset { + pub path: FileSystemPath, + pub output_assets: ResolvedVc, +} + +#[turbo_tasks::value_impl] +impl AnalyzeDataOutputAsset { + #[turbo_tasks::function] + pub async fn new( + path: FileSystemPath, + output_assets: ResolvedVc, + ) -> Result> { + Ok(Self { + path, + output_assets, + } + .cell()) + } +} + +#[turbo_tasks::value_impl] +impl Asset for AnalyzeDataOutputAsset { + #[turbo_tasks::function] + fn content(&self) -> Vc { + let file_content = analyze_output_assets(*self.output_assets); + AssetContent::file(file_content) + } +} + +#[turbo_tasks::value_impl] +impl OutputAssetsReference for AnalyzeDataOutputAsset {} + +#[turbo_tasks::value_impl] +impl OutputAsset for AnalyzeDataOutputAsset { + #[turbo_tasks::function] + fn path(&self) -> Vc { + self.path.clone().cell() + } +} + +#[turbo_tasks::value] +pub struct ModulesDataOutputAsset { + pub path: FileSystemPath, + pub module_graphs: ResolvedVc, +} + +#[turbo_tasks::value_impl] +impl ModulesDataOutputAsset { + #[turbo_tasks::function] + pub async fn new(path: FileSystemPath, module_graphs: Vc) -> Result> { + Ok(Self { + path, + module_graphs: module_graphs.to_resolved().await?, + } + .cell()) + } +} + +#[turbo_tasks::value_impl] +impl Asset for ModulesDataOutputAsset { + #[turbo_tasks::function] + fn content(&self) -> Vc { + let file_content = analyze_module_graphs(*self.module_graphs); + AssetContent::file(file_content) + } +} + +#[turbo_tasks::value_impl] +impl OutputAssetsReference for ModulesDataOutputAsset {} + +#[turbo_tasks::value_impl] +impl OutputAsset for ModulesDataOutputAsset { + #[turbo_tasks::function] + fn path(&self) -> Vc { + self.path.clone().cell() + } +} diff --git a/crates/pack-api/src/app.rs b/crates/pack-api/src/app.rs index 3d487d4a5..ca36e3a45 100644 --- a/crates/pack-api/src/app.rs +++ b/crates/pack-api/src/app.rs @@ -41,6 +41,7 @@ use turbopack_core::{ }; use crate::{ + analyze::{AnalyzeTarget, AnalyzeTargets}, endpoint::{Endpoint, EndpointOutput, EndpointOutputPaths}, project::Project, webpack_stats::generate_webpack_stats, @@ -399,6 +400,31 @@ impl AppEndpoint { )), } } + + #[turbo_tasks::function] + pub async fn analyze_targets(self: Vc) -> Result> { + let asset_context = self.app_module_context(); + let runtime_entries = self.app_runtime_entries(); + + let this = self.await?; + let targets = this + .entrypoints + .iter() + .map(|entry| async move { + let name = entry.await?.name.clone(); + Ok(AnalyzeTarget { + name, + output_assets: entry + .output_assets_for_entry(Vc::upcast(asset_context), runtime_entries) + .to_resolved() + .await?, + }) + }) + .try_join() + .await?; + + Ok(AnalyzeTargets(targets).cell()) + } } #[turbo_tasks::value_impl] diff --git a/crates/pack-api/src/lib.rs b/crates/pack-api/src/lib.rs index 858587c4b..86dc6c429 100644 --- a/crates/pack-api/src/lib.rs +++ b/crates/pack-api/src/lib.rs @@ -2,6 +2,7 @@ #![feature(arbitrary_self_types_pointers)] #![allow(unexpected_cfgs)] +pub mod analyze; pub mod app; pub mod endpoint; pub mod entrypoint; diff --git a/crates/pack-api/src/library.rs b/crates/pack-api/src/library.rs index 15fafec01..1ef74aa19 100644 --- a/crates/pack-api/src/library.rs +++ b/crates/pack-api/src/library.rs @@ -40,6 +40,7 @@ use turbopack_core::{ use turbopack_resolve::resolve_options_context::ResolveOptionsContext; use crate::{ + analyze::AnalyzeTarget, endpoint::{Endpoint, EndpointOutput, EndpointOutputPaths}, project::Project, webpack_stats::generate_webpack_stats, @@ -341,6 +342,16 @@ impl LibraryEndpoint { let chunk_group_assets = *self.library_chunk().await?.assets; Ok(chunk_group_assets) } + + #[turbo_tasks::function] + pub async fn analyze_target(self: Vc) -> Result> { + let this = self.await?; + Ok(AnalyzeTarget { + name: this.name.clone(), + output_assets: self.output_assets().to_resolved().await?, + } + .cell()) + } } #[turbo_tasks::value_impl] diff --git a/crates/pack-napi/src/pack_api/analyze.rs b/crates/pack-napi/src/pack_api/analyze.rs new file mode 100644 index 000000000..3c1a36c1c --- /dev/null +++ b/crates/pack-napi/src/pack_api/analyze.rs @@ -0,0 +1,139 @@ +use std::sync::Arc; + +use anyhow::Result; +use pack_api::{ + analyze::{AnalyzeDataOutputAsset, ModulesDataOutputAsset}, + project::ProjectContainer, + utils::strongly_consistent_catch_collectables, +}; +use turbo_tasks::{Effects, ReadRef, ResolvedVc, Vc}; +use turbo_tasks_fs::{File, FileContent}; +use turbopack_core::{ + asset::AssetContent, + diagnostics::PlainDiagnostic, + issue::PlainIssue, + output::{OutputAsset, OutputAssets}, + virtual_output::VirtualOutputAsset, +}; + +#[turbo_tasks::value(serialization = "none")] +pub struct WriteAnalyzeResult { + pub issues: Arc>>, + pub diagnostics: Arc>>, + pub effects: Arc, +} + +#[turbo_tasks::function(operation)] +pub async fn write_analyze_data_with_issues_operation( + project: ResolvedVc, +) -> Result> { + let analyze_data_op = write_analyze_data_with_issues_operation_inner(project); + + let (_analyze_data, issues, diagnostics, effects) = + strongly_consistent_catch_collectables(analyze_data_op).await?; + + Ok(WriteAnalyzeResult { + issues, + diagnostics, + effects, + } + .cell()) +} + +#[turbo_tasks::function(operation)] +async fn write_analyze_data_with_issues_operation_inner( + project: ResolvedVc, +) -> Result<()> { + let analyze_data_op = get_analyze_data_operation(project); + + project + .project() + .emit_all_output_assets(analyze_data_op) + .as_side_effect() + .await?; + + Ok(()) +} + +#[turbo_tasks::function(operation)] +async fn get_analyze_data_operation( + container: ResolvedVc, +) -> Result> { + let project = container.project(); + let analyze_output_root = project + .dist_root() + .owned() + .await? + .join("diagnostics/analyze/data")?; + + let whole_app_module_graphs = project.to_resolved().await?.whole_app_module_graphs(); + + let mut route_names = Vec::new(); + let mut analyze_assets = Vec::>>::new(); + + let app_project = project.app_project().to_resolved().await?.await?; + if let Some(app_project) = *app_project { + let app_endpoint = app_project.get_app_endpoint().to_resolved().await?; + let app_targets = app_endpoint.analyze_targets().await?; + for target in app_targets.iter() { + let route = format!("/apps/{}", target.name); + route_names.push(route.clone()); + + let analyze_data = AnalyzeDataOutputAsset::new( + analyze_output_root + .join(route.trim_start_matches('/'))? + .join("analyze.data")?, + *target.output_assets, + ) + .to_resolved() + .await?; + analyze_assets.push(ResolvedVc::upcast(analyze_data)); + } + } + + let library_project = project.library_project().to_resolved().await?.await?; + if let Some(library_project) = *library_project { + let library_endpoints = library_project.get_library_endpoints().await?; + for endpoint in library_endpoints.iter() { + let target = endpoint.analyze_target().await?; + let route = format!("/libraries/{}", target.name); + route_names.push(route.clone()); + + let analyze_data = AnalyzeDataOutputAsset::new( + analyze_output_root + .join(route.trim_start_matches('/'))? + .join("analyze.data")?, + *target.output_assets, + ) + .to_resolved() + .await?; + analyze_assets.push(ResolvedVc::upcast(analyze_data)); + } + } + + whole_app_module_graphs.as_side_effect().await?; + + let modules_data = ResolvedVc::upcast( + ModulesDataOutputAsset::new( + analyze_output_root.join("modules.data")?, + Vc::cell(vec![whole_app_module_graphs.await?.full]), + ) + .to_resolved() + .await?, + ); + + let routes_json = serde_json::to_string_pretty(&route_names)?; + let routes_data = ResolvedVc::upcast( + VirtualOutputAsset::new( + analyze_output_root.join("routes.json")?, + AssetContent::file(FileContent::from(File::from(routes_json)).cell()), + ) + .to_resolved() + .await?, + ); + + analyze_assets.push(modules_data); + analyze_assets.push(routes_data); + + Ok(Vc::cell(analyze_assets)) +} diff --git a/crates/pack-napi/src/pack_api/mod.rs b/crates/pack-napi/src/pack_api/mod.rs index 4897b2d3b..6dcf3d119 100644 --- a/crates/pack-napi/src/pack_api/mod.rs +++ b/crates/pack-napi/src/pack_api/mod.rs @@ -1,3 +1,4 @@ +pub mod analyze; pub mod endpoint; pub mod project; pub mod turbopack_ctx; diff --git a/crates/pack-napi/src/pack_api/project.rs b/crates/pack-napi/src/pack_api/project.rs index 0354d7973..02a8e43d5 100644 --- a/crates/pack-napi/src/pack_api/project.rs +++ b/crates/pack-napi/src/pack_api/project.rs @@ -60,6 +60,7 @@ use turbopack_trace_utils::{ }; use super::{ + analyze::{WriteAnalyzeResult, write_analyze_data_with_issues_operation}, endpoint::ExternalEndpoint, utils::{NapiDiagnostic, NapiIssue, TurbopackResult, create_turbo_tasks, subscribe}, }; @@ -1023,3 +1024,35 @@ pub fn project_get_source_map_sync( tokio::runtime::Handle::current().block_on(project_get_source_map(project, file_path)) }) } + +#[napi] +pub async fn project_write_analyze_data( + #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External, +) -> napi::Result> { + let container = project.container; + let (issues, diagnostics) = project + .turbopack_ctx + .turbo_tasks() + .run_once(async move { + let analyze_data_op = write_analyze_data_with_issues_operation(container); + let WriteAnalyzeResult { + issues, + diagnostics, + effects, + } = &*analyze_data_op.read_strongly_consistent().await?; + + effects.apply().await?; + Ok((issues.clone(), diagnostics.clone())) + }) + .await + .map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?; + + Ok(TurbopackResult { + result: (), + issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(), + diagnostics: diagnostics + .iter() + .map(|d| NapiDiagnostic::from(d)) + .collect(), + }) +} diff --git a/examples/with-react/package.json b/examples/with-react/package.json index 7521c61b7..274b85b2b 100644 --- a/examples/with-react/package.json +++ b/examples/with-react/package.json @@ -3,7 +3,8 @@ "version": "0.0.1", "scripts": { "build": "up build", - "dev": "up dev" + "dev": "up dev", + "analyze": "ANALYZE=1 up build" }, "dependencies": { "react": "^18.2.0", diff --git a/packages/pack-cli/src/commands/build.ts b/packages/pack-cli/src/commands/build.ts index f3e64a9b9..e276e6dd2 100644 --- a/packages/pack-cli/src/commands/build.ts +++ b/packages/pack-cli/src/commands/build.ts @@ -22,8 +22,15 @@ export default defineCommand({ type: "boolean", description: "Enable webpack mode", }, + analyze: { + type: "boolean", + description: "Generate native Turbopack analyze data", + }, }, async run({ args }) { + if (args.analyze && !process.env.ANALYZE) { + process.env.ANALYZE = "native"; + } const { projectOptions, projectPath, rootPath } = await resolveBuildOptions(args); await utooPack.build(projectOptions, projectPath, rootPath); diff --git a/packages/pack/src/binding.d.ts b/packages/pack/src/binding.d.ts index 4e9b9a1c0..3a32b4e47 100644 --- a/packages/pack/src/binding.d.ts +++ b/packages/pack/src/binding.d.ts @@ -183,6 +183,7 @@ export declare function projectTraceSource(project: { __napiType: "Project" }, f export declare function projectGetSourceForAsset(project: { __napiType: "Project" }, filePath: string): Promise export declare function projectGetSourceMap(project: { __napiType: "Project" }, filePath: RcStr): Promise export declare function projectGetSourceMapSync(project: { __napiType: "Project" }, filePath: RcStr): string | null +export declare function projectWriteAnalyzeData(project: { __napiType: "Project" }): Promise /** Arguments for `NapiTurbopackCallbacks::throw_turbopack_internal_error`. */ export interface TurbopackInternalErrorOpts { message: string diff --git a/packages/pack/src/binding.js b/packages/pack/src/binding.js index 1516d2d6c..a1f336db3 100644 --- a/packages/pack/src/binding.js +++ b/packages/pack/src/binding.js @@ -310,7 +310,7 @@ if (!nativeBinding) { throw new Error(`Failed to load native binding`) } -const { registerWorkerScheduler, workerCreated, recvTaskMessageInWorker, sendTaskMessage, endpointWriteToDisk, endpointServerChangedSubscribe, endpointClientChangedSubscribe, projectNew, projectUpdate, projectOnExit, projectShutdown, projectWriteAllEntrypointsToDisk, projectEntrypointsSubscribe, projectHmrEvents, projectHmrIdentifiersSubscribe, projectUpdateInfoSubscribe, projectTraceSource, projectGetSourceForAsset, projectGetSourceMap, projectGetSourceMapSync, rootTaskDispose, initCustomTraceSubscriber, teardownTraceSubscriber } = nativeBinding +const { registerWorkerScheduler, workerCreated, recvTaskMessageInWorker, sendTaskMessage, endpointWriteToDisk, endpointServerChangedSubscribe, endpointClientChangedSubscribe, projectNew, projectUpdate, projectOnExit, projectShutdown, projectWriteAllEntrypointsToDisk, projectEntrypointsSubscribe, projectHmrEvents, projectHmrIdentifiersSubscribe, projectUpdateInfoSubscribe, projectTraceSource, projectGetSourceForAsset, projectGetSourceMap, projectGetSourceMapSync, projectWriteAnalyzeData, rootTaskDispose, initCustomTraceSubscriber, teardownTraceSubscriber } = nativeBinding module.exports.registerWorkerScheduler = registerWorkerScheduler module.exports.workerCreated = workerCreated @@ -332,6 +332,7 @@ module.exports.projectTraceSource = projectTraceSource module.exports.projectGetSourceForAsset = projectGetSourceForAsset module.exports.projectGetSourceMap = projectGetSourceMap module.exports.projectGetSourceMapSync = projectGetSourceMapSync +module.exports.projectWriteAnalyzeData = projectWriteAnalyzeData module.exports.rootTaskDispose = rootTaskDispose module.exports.initCustomTraceSubscriber = initCustomTraceSubscriber module.exports.teardownTraceSubscriber = teardownTraceSubscriber diff --git a/packages/pack/src/commands/build.ts b/packages/pack/src/commands/build.ts index ab5b425d0..ab1318dae 100644 --- a/packages/pack/src/commands/build.ts +++ b/packages/pack/src/commands/build.ts @@ -16,6 +16,8 @@ import { useWorkerThreads } from "../utils/runtimePluginStratety"; import { validateEntryPaths } from "../utils/validateEntry"; import { xcodeProfilingReady } from "../utils/xcodeProfile"; +type AnalyzeMode = "none" | "native" | "webpack"; + export function build( options: BundleOptions | WebpackConfig, projectPath?: string, @@ -35,6 +37,7 @@ async function buildInternal( projectPath?: string, rootPath?: string, ) { + const analyzeMode = getAnalyzeMode(); blockStdout(); if (process.env.XCODE_PROFILE) { @@ -42,6 +45,10 @@ async function buildInternal( } const resolvedProjectPath = projectPath || process.cwd(); + const outputPath = resolveOutputPath( + resolvedProjectPath, + bundleOptions.config.output?.path, + ); processHtmlEntry(bundleOptions.config, resolvedProjectPath); validateEntryPaths(bundleOptions.config, resolvedProjectPath); @@ -57,7 +64,7 @@ async function buildInternal( config: { ...bundleOptions.config, stats: - Boolean(process.env.ANALYZE) || + analyzeMode === "webpack" || bundleOptions.config.stats || bundleOptions.config.entry.some((e: EntryOptions) => !!e.html), pluginRuntimeStrategy: @@ -72,51 +79,90 @@ async function buildInternal( persistentCaching: bundleOptions.config.persistentCaching ?? false, }, ); - - const entrypoints = await project.writeAllEntrypointsToDisk(); - - handleIssues(entrypoints.issues); - - const htmlConfigs = [ - ...(Array.isArray((bundleOptions.config as any).html) - ? (bundleOptions.config as any).html - : (bundleOptions.config as any).html - ? [(bundleOptions.config as any).html] - : []), - ...bundleOptions.config.entry - .filter((e: EntryOptions) => !!e.html) - .map((e: EntryOptions) => e.html!), - ]; - - if (htmlConfigs.length > 0) { - const assets = { js: [] as string[], css: [] as string[] }; - - const outputDir = - bundleOptions.config.output?.path || path.join(process.cwd(), "dist"); - - if (assets.js.length === 0 && assets.css.length === 0) { - const discovered = getInitialAssetsFromStats(outputDir); - assets.js.push(...discovered.js); - assets.css.push(...discovered.css); + let nativeAnalyzeReady = false; + + try { + const entrypoints = await project.writeAllEntrypointsToDisk(); + + handleIssues(entrypoints.issues); + + const htmlConfigs = [ + ...(Array.isArray((bundleOptions.config as any).html) + ? (bundleOptions.config as any).html + : (bundleOptions.config as any).html + ? [(bundleOptions.config as any).html] + : []), + ...bundleOptions.config.entry + .filter((e: EntryOptions) => !!e.html) + .map((e: EntryOptions) => e.html!), + ]; + + if (htmlConfigs.length > 0) { + const assets = { js: [] as string[], css: [] as string[] }; + + if (assets.js.length === 0 && assets.css.length === 0) { + const discovered = getInitialAssetsFromStats(outputPath); + assets.js.push(...discovered.js); + assets.css.push(...discovered.css); + } + + const publicPath = bundleOptions.config.output?.publicPath; + + for (const config of htmlConfigs) { + const plugin = new HtmlPlugin(config); + await plugin.generate(outputPath, assets, publicPath); + } } - const publicPath = bundleOptions.config.output?.publicPath; - - for (const config of htmlConfigs) { - const plugin = new HtmlPlugin(config); - await plugin.generate(outputDir, assets, publicPath); + if (analyzeMode === "native") { + const analyzeResult = await project.writeAnalyzeData(); + handleIssues(analyzeResult.issues); + nativeAnalyzeReady = true; + } else if (analyzeMode === "webpack") { + await analyzeBundle(outputPath); } + } finally { + await project.shutdown(); } - if (process.env.ANALYZE) { - await analyzeBundle(bundleOptions.config.output?.path || "dist"); + if (nativeAnalyzeReady) { + const analyzeDataDir = path.join( + outputPath, + "diagnostics", + "analyze", + "data", + ); + console.error(`Native analyze data written to ${analyzeDataDir}`); } - await project.shutdown(); // TODO: Maybe run tasks in worker is a better way, see // https://github.com/vercel/next.js/blob/512d8283054407ab92b2583ecce3b253c3be7b85/packages/next/src/lib/worker.ts } +function getAnalyzeMode(): AnalyzeMode { + const analyze = process.env.ANALYZE?.trim().toLowerCase(); + if (!analyze) { + return "none"; + } + if (analyze === "webpack" || analyze === "bundle" || analyze === "legacy") { + return "webpack"; + } + return "native"; +} + +function resolveOutputPath( + projectPath: string, + configuredOutputPath?: string, +): string { + if (!configuredOutputPath) { + return path.join(projectPath, "dist"); + } + + return path.isAbsolute(configuredOutputPath) + ? configuredOutputPath + : path.join(projectPath, configuredOutputPath); +} + async function analyzeBundle(outputPath: string): Promise { const statsPath = path.join(outputPath, "stats.json"); diff --git a/packages/pack/src/core/project.ts b/packages/pack/src/core/project.ts index 8d10a1311..1c0e81352 100644 --- a/packages/pack/src/core/project.ts +++ b/packages/pack/src/core/project.ts @@ -447,6 +447,15 @@ export function projectFactory() { return binding.projectGetSourceMapSync(this._nativeProject, filePath); } + writeAnalyzeData(): Promise> { + return withErrorCause( + () => + binding.projectWriteAnalyzeData(this._nativeProject) as Promise< + TurbopackResult + >, + ); + } + updateInfoSubscribe(aggregationMs: number) { return subscribe>( true, diff --git a/packages/pack/src/core/types.ts b/packages/pack/src/core/types.ts index f7f8a1de6..552c736e1 100644 --- a/packages/pack/src/core/types.ts +++ b/packages/pack/src/core/types.ts @@ -77,6 +77,7 @@ export interface Project { getSourceMap(filePath: string): Promise; getSourceMapSync(filePath: string): string | null; + writeAnalyzeData(): Promise>; traceSource( stackFrame: StackFrame, diff --git a/packages/pack/src/utils/common.ts b/packages/pack/src/utils/common.ts index d9c5bae94..78af641ca 100644 --- a/packages/pack/src/utils/common.ts +++ b/packages/pack/src/utils/common.ts @@ -28,5 +28,5 @@ export function blockStdout() { * - ESM：在构建后由脚本注入 __dirname（基于 import.meta.url），此处仅用 __dirname 以保持 CJS 产物不含 import.meta，避免 Node 将 .js 判为 ESM。 */ export function getPackPath() { - return path.resolve(__dirname, ".."); + return path.resolve(__dirname, "../.."); }