diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e2f410..3ebd937 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Install dependencies of packages in `x` command +### Performance +- Asyncify dependency linking by @daimond113 + ## [0.5.0-rc.12] - 2024-11-22 ### Added - Support fallback Wally registries by @daimond113 diff --git a/src/linking/mod.rs b/src/linking/mod.rs index 3e330c3..0be67cf 100644 --- a/src/linking/mod.rs +++ b/src/linking/mod.rs @@ -1,21 +1,22 @@ use crate::{ linking::generator::get_file_types, lockfile::DownloadedGraph, - names::PackageNames, scripts::{execute_script, ScriptName}, source::{ fs::{cas_path, store_in_cas}, traits::PackageRef, - version_id::VersionId, }, Project, LINK_LIB_NO_FILE_FOUND, PACKAGES_CONTAINER_NAME, }; use fs_err::tokio as fs; +use futures::future::try_join_all; use std::{ - collections::BTreeMap, + collections::HashMap, ffi::OsStr, path::{Path, PathBuf}, + sync::Arc, }; +use tokio::task::spawn_blocking; /// Generates linking modules for a project pub mod generator; @@ -39,220 +40,236 @@ impl Project { graph: &DownloadedGraph, ) -> Result<(), errors::LinkingError> { let manifest = self.deser_manifest().await?; + let manifest_target_kind = manifest.target.kind(); + let roblox_sync_config_gen_script = manifest + .scripts + .get(&ScriptName::RobloxSyncConfigGenerator.to_string()); - let mut package_types = BTreeMap::<&PackageNames, BTreeMap<&VersionId, Vec>>::new(); - - for (name, versions) in graph { - for (version_id, node) in versions { - let Some(lib_file) = node.target.lib_path() else { - continue; - }; - - let container_folder = node.node.container_folder( - &self - .package_dir() - .join(manifest.target.kind().packages_folder(version_id.target())) - .join(PACKAGES_CONTAINER_NAME), - name, - version_id.version(), - ); - - let types = if lib_file.as_str() != LINK_LIB_NO_FILE_FOUND { - let lib_file = lib_file.to_path(&container_folder); - - let contents = match fs::read_to_string(&lib_file).await { - Ok(contents) => contents, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - return Err(errors::LinkingError::LibFileNotFound( - lib_file.display().to_string(), - )); - } - Err(e) => return Err(e.into()), - }; - - let types = match get_file_types(&contents) { - Ok(types) => types, - Err(e) => { - return Err(errors::LinkingError::FullMoon( - lib_file.display().to_string(), - e, - )) - } - }; - - log::debug!("{name}@{version_id} has {} exported types", types.len()); - - types - } else { - vec![] - }; - - package_types - .entry(name) - .or_default() - .insert(version_id, types); - - if let Some(build_files) = Some(&node.target) - .filter(|_| !node.node.pkg_ref.like_wally()) - .and_then(|t| t.build_files()) - { - let script_name = ScriptName::RobloxSyncConfigGenerator.to_string(); - - let Some(script_path) = manifest.scripts.get(&script_name) else { - log::warn!("not having a `{script_name}` script in the manifest might cause issues with Roblox linking"); - continue; - }; - - execute_script( - ScriptName::RobloxSyncConfigGenerator, - &script_path.to_path(self.package_dir()), - std::iter::once(container_folder.as_os_str()) - .chain(build_files.iter().map(OsStr::new)), - self, - false, - ) - .map_err(|e| { - errors::LinkingError::GenerateRobloxSyncConfig( - container_folder.display().to_string(), - e, - ) - })?; - } - } - } - - for (name, versions) in graph { - for (version_id, node) in versions { - let (node_container_folder, node_packages_folder) = { - let base_folder = create_and_canonicalize( - self.package_dir() - .join(manifest.target.kind().packages_folder(version_id.target())), - ) - .await?; - let packages_container_folder = base_folder.join(PACKAGES_CONTAINER_NAME); - - let container_folder = node.node.container_folder( - &packages_container_folder, - name, - version_id.version(), - ); - - if let Some((alias, _, _)) = &node.node.direct.as_ref() { - if let Some((lib_file, types)) = - node.target.lib_path().and_then(|lib_file| { - package_types - .get(name) - .and_then(|v| v.get(version_id)) - .map(|types| (lib_file, types)) - }) - { - write_cas( - base_folder.join(format!("{alias}.luau")), - self.cas_dir(), - &generator::generate_lib_linking_module( - &generator::get_lib_require_path( - &node.target.kind(), - &base_folder, - lib_file, - &container_folder, - node.node.pkg_ref.use_new_structure(), - &base_folder, - container_folder.strip_prefix(&base_folder).unwrap(), - &manifest, - )?, - types, - ), - ) - .await?; + let package_types = try_join_all( + graph + .iter() + .map(|(name, versions)| async move { + Ok::<_, errors::LinkingError>((name, try_join_all(versions.iter().map(|(version_id, node)| async move { + let Some(lib_file) = node.target.lib_path() else { + return Ok((version_id, vec![])); }; - if let Some(bin_file) = node.target.bin_path() { - write_cas( - base_folder.join(format!("{alias}.bin.luau")), - self.cas_dir(), - &generator::generate_bin_linking_module( - &container_folder, - &generator::get_bin_require_path( - &base_folder, - bin_file, - &container_folder, - ), - ), + let container_folder = node.node.container_folder( + &self + .package_dir() + .join(manifest_target_kind.packages_folder(version_id.target())) + .join(PACKAGES_CONTAINER_NAME), + name, + version_id.version(), + ); + + let types = if lib_file.as_str() != LINK_LIB_NO_FILE_FOUND { + let lib_file = lib_file.to_path(&container_folder); + + let contents = match fs::read_to_string(&lib_file).await { + Ok(contents) => contents, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Err(errors::LinkingError::LibFileNotFound( + lib_file.display().to_string(), + )); + } + Err(e) => return Err(e.into()), + }; + + let types = match spawn_blocking(move || get_file_types(&contents)).await.unwrap() { + Ok(types) => types, + Err(e) => { + return Err(errors::LinkingError::FullMoon( + lib_file.display().to_string(), + e, + )) + } + }; + + log::debug!("{name}@{version_id} has {} exported types", types.len()); + + types + } else { + vec![] + }; + + if let Some(build_files) = Some(&node.target) + .filter(|_| !node.node.pkg_ref.like_wally()) + .and_then(|t| t.build_files()) + { + let Some(script_path) = roblox_sync_config_gen_script else { + log::warn!("not having a `{}` script in the manifest might cause issues with Roblox linking", ScriptName::RobloxSyncConfigGenerator); + return Ok((version_id, vec![])); + }; + + execute_script( + ScriptName::RobloxSyncConfigGenerator, + &script_path.to_path(self.package_dir()), + std::iter::once(container_folder.as_os_str()) + .chain(build_files.iter().map(OsStr::new)), + self, + false, ) - .await?; + .map_err(|e| { + errors::LinkingError::GenerateRobloxSyncConfig( + container_folder.display().to_string(), + e, + ) + })?; } + + Ok((version_id, types)) + })).await?.into_iter().collect::>())) + } + ) + ) + .await? + .into_iter() + .collect::>(); + + let manifest = Arc::new(manifest); + let package_types = Arc::new(package_types); + + try_join_all(graph.iter().flat_map(|(name, versions)| { + versions.iter().map(|(version_id, node)| { + let name = name.clone(); + let manifest = manifest.clone(); + let package_types = package_types.clone(); + + async move { + let (node_container_folder, node_packages_folder) = { + let base_folder = create_and_canonicalize( + self.package_dir() + .join(manifest_target_kind.packages_folder(version_id.target())), + ) + .await?; + let packages_container_folder = base_folder.join(PACKAGES_CONTAINER_NAME); + + let container_folder = node.node.container_folder( + &packages_container_folder, + &name, + version_id.version(), + ); + + if let Some((alias, _, _)) = &node.node.direct.as_ref() { + if let Some((lib_file, types)) = + node.target.lib_path().and_then(|lib_file| { + package_types + .get(&name) + .and_then(|v| v.get(version_id)) + .map(|types| (lib_file, types)) + }) + { + write_cas( + base_folder.join(format!("{alias}.luau")), + self.cas_dir(), + &generator::generate_lib_linking_module( + &generator::get_lib_require_path( + &node.target.kind(), + &base_folder, + lib_file, + &container_folder, + node.node.pkg_ref.use_new_structure(), + &base_folder, + container_folder.strip_prefix(&base_folder).unwrap(), + &manifest, + )?, + types, + ), + ) + .await?; + }; + + if let Some(bin_file) = node.target.bin_path() { + write_cas( + base_folder.join(format!("{alias}.bin.luau")), + self.cas_dir(), + &generator::generate_bin_linking_module( + &container_folder, + &generator::get_bin_require_path( + &base_folder, + bin_file, + &container_folder, + ), + ), + ) + .await?; + } + } + + (container_folder, base_folder) + }; + + for (dependency_name, (dependency_version_id, dependency_alias)) in + &node.node.dependencies + { + let Some(dependency_node) = graph + .get(dependency_name) + .and_then(|v| v.get(dependency_version_id)) + else { + return Err(errors::LinkingError::DependencyNotFound( + dependency_name.to_string(), + dependency_version_id.to_string(), + )); + }; + + let Some(lib_file) = dependency_node.target.lib_path() else { + continue; + }; + + let base_folder = create_and_canonicalize( + self.package_dir().join( + version_id + .target() + .packages_folder(dependency_version_id.target()), + ), + ) + .await?; + let packages_container_folder = base_folder.join(PACKAGES_CONTAINER_NAME); + + let container_folder = dependency_node.node.container_folder( + &packages_container_folder, + dependency_name, + dependency_version_id.version(), + ); + + let linker_folder = create_and_canonicalize( + node_container_folder.join( + node.node + .base_folder(version_id, dependency_node.target.kind()), + ), + ) + .await?; + + write_cas( + linker_folder.join(format!("{dependency_alias}.luau")), + self.cas_dir(), + &generator::generate_lib_linking_module( + &generator::get_lib_require_path( + &dependency_node.target.kind(), + &linker_folder, + lib_file, + &container_folder, + dependency_node.node.pkg_ref.use_new_structure(), + &node_packages_folder, + container_folder.strip_prefix(&base_folder).unwrap(), + &manifest, + )?, + package_types + .get(dependency_name) + .and_then(|v| v.get(dependency_version_id)) + .unwrap(), + ), + ) + .await?; } - (container_folder, base_folder) - }; - - for (dependency_name, (dependency_version_id, dependency_alias)) in - &node.node.dependencies - { - let Some(dependency_node) = graph - .get(dependency_name) - .and_then(|v| v.get(dependency_version_id)) - else { - return Err(errors::LinkingError::DependencyNotFound( - dependency_name.to_string(), - dependency_version_id.to_string(), - )); - }; - - let Some(lib_file) = dependency_node.target.lib_path() else { - continue; - }; - - let base_folder = create_and_canonicalize( - self.package_dir().join( - version_id - .target() - .packages_folder(dependency_version_id.target()), - ), - ) - .await?; - let packages_container_folder = base_folder.join(PACKAGES_CONTAINER_NAME); - - let container_folder = dependency_node.node.container_folder( - &packages_container_folder, - dependency_name, - dependency_version_id.version(), - ); - - let linker_folder = create_and_canonicalize( - node_container_folder.join( - node.node - .base_folder(version_id, dependency_node.target.kind()), - ), - ) - .await?; - - write_cas( - linker_folder.join(format!("{dependency_alias}.luau")), - self.cas_dir(), - &generator::generate_lib_linking_module( - &generator::get_lib_require_path( - &dependency_node.target.kind(), - &linker_folder, - lib_file, - &container_folder, - dependency_node.node.pkg_ref.use_new_structure(), - &node_packages_folder, - container_folder.strip_prefix(&base_folder).unwrap(), - &manifest, - )?, - package_types - .get(dependency_name) - .and_then(|v| v.get(dependency_version_id)) - .unwrap(), - ), - ) - .await?; + Ok(()) } - } - } - - Ok(()) + }) + })) + .await + .map(|_| ()) } }