diff --git a/CHANGELOG.md b/CHANGELOG.md index 9925f3a..30ba44b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Add improved CLI styling by @daimond113 +- Install pesde dependencies before Wally to support scripts packages by @daimond113 ### Fixed -- Link dependencies before type extraction to support more use cases +- Link dependencies before type extraction to support more use cases by @daimond113 ## [0.5.0-rc.14] - 2024-11-30 ### Fixed diff --git a/src/cli/commands/execute.rs b/src/cli/commands/execute.rs index 6e88919..d3b77c2 100644 --- a/src/cli/commands/execute.rs +++ b/src/cli/commands/execute.rs @@ -5,7 +5,7 @@ use fs_err::tokio as fs; use indicatif::MultiProgress; use pesde::{ linking::generator::generate_bin_linking_module, - manifest::{target::TargetKind, DependencyType}, + manifest::target::TargetKind, names::PackageName, source::{ pesde::{specifier::PesdeDependencySpecifier, PesdePackageSource}, @@ -17,6 +17,7 @@ use semver::VersionReq; use std::{ collections::HashSet, env::current_dir, ffi::OsString, io::Write, process::Command, sync::Arc, }; +use tokio::sync::Mutex; #[derive(Debug, Args)] pub struct ExecuteCommand { @@ -116,9 +117,17 @@ impl ExecuteCommand { .dependency_graph(None, &mut refreshed_sources, true) .await .context("failed to build dependency graph")?; + let graph = Arc::new(graph); let (rx, downloaded_graph) = project - .download_graph(&graph, &mut refreshed_sources, &reqwest, true, true) + .download_and_link( + &graph, + &Arc::new(Mutex::new(refreshed_sources)), + &reqwest, + true, + true, + |_| async { Ok::<_, std::io::Error>(()) }, + ) .await .context("failed to download dependencies")?; @@ -132,27 +141,9 @@ impl ExecuteCommand { ) .await?; - let downloaded_graph = Arc::into_inner(downloaded_graph) - .unwrap() - .into_inner() - .unwrap(); - - project - .link_dependencies( - &downloaded_graph - .into_iter() - .map(|(n, v)| { - ( - n, - v.into_iter() - .filter(|(_, n)| n.node.resolved_ty != DependencyType::Dev) - .collect(), - ) - }) - .collect(), - ) + downloaded_graph .await - .context("failed to link dependencies")?; + .context("failed to download & link dependencies")?; let mut caller = tempfile::NamedTempFile::new_in(tempdir.path()).context("failed to create tempfile")?; diff --git a/src/cli/commands/install.rs b/src/cli/commands/install.rs index b89a08f..0f0568e 100644 --- a/src/cli/commands/install.rs +++ b/src/cli/commands/install.rs @@ -9,14 +9,14 @@ use fs_err::tokio as fs; use futures::future::try_join_all; use indicatif::MultiProgress; use pesde::{ - lockfile::Lockfile, - manifest::{target::TargetKind, DependencyType}, - Project, MANIFEST_FILE_NAME, + download_and_link::filter_graph, lockfile::Lockfile, manifest::target::TargetKind, Project, + MANIFEST_FILE_NAME, }; use std::{ collections::{BTreeSet, HashMap, HashSet}, sync::Arc, }; +use tokio::sync::Mutex; #[derive(Debug, Args, Copy, Clone)] pub struct InstallCommand { @@ -81,14 +81,18 @@ stdio.ewrite(stdio.color("red") .. "binary `{alias}` not found. are you in the r } #[cfg(feature = "patches")] -const JOBS: u8 = 6; -#[cfg(not(feature = "patches"))] const JOBS: u8 = 5; +#[cfg(not(feature = "patches"))] +const JOBS: u8 = 4; fn job(n: u8) -> ColoredString { format!("[{n}/{JOBS}]").dimmed().bold() } +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +struct CallbackError(#[from] anyhow::Error); + impl InstallCommand { pub async fn run( self, @@ -198,12 +202,74 @@ impl InstallCommand { .dependency_graph(old_graph.as_ref(), &mut refreshed_sources, false) .await .context("failed to build dependency graph")?; + let graph = Arc::new(graph); update_scripts_handle.await??; + let bin_folder = bin_dir().await?; + let downloaded_graph = { let (rx, downloaded_graph) = project - .download_graph(&graph, &mut refreshed_sources, &reqwest, self.prod, true) + .download_and_link( + &graph, + &Arc::new(Mutex::new(refreshed_sources)), + &reqwest, + self.prod, + true, + |graph| { + let graph = graph.clone(); + + async move { + try_join_all( + graph + .values() + .flat_map(|versions| versions.values()) + .filter(|node| node.target.bin_path().is_some()) + .filter_map(|node| node.node.direct.as_ref()) + .map(|(alias, _, _)| alias) + .filter(|alias| { + if *alias == env!("CARGO_BIN_NAME") { + log::warn!( + "package {alias} has the same name as the CLI, skipping bin link" + ); + return false; + } + + true + }) + .map(|alias| { + let bin_folder = bin_folder.clone(); + async move { + let bin_file = bin_folder.join(alias); + fs::write(&bin_file, bin_link_file(alias)) + .await + .context("failed to write bin link file")?; + + make_executable(&bin_file) + .await + .context("failed to make bin link executable")?; + + #[cfg(windows)] + { + let bin_file = bin_file.with_extension(std::env::consts::EXE_EXTENSION); + fs::copy( + std::env::current_exe() + .context("failed to get current executable path")?, + &bin_file, + ) + .await + .context("failed to copy bin link file")?; + } + + Ok::<_, CallbackError>(()) + } + }), + ) + .await + .map(|_| ()) + } + } + ) .await .context("failed to download dependencies")?; @@ -217,33 +283,15 @@ impl InstallCommand { ) .await?; - Arc::into_inner(downloaded_graph) - .unwrap() - .into_inner() - .unwrap() - }; - - let filtered_graph = if self.prod { downloaded_graph - .clone() - .into_iter() - .map(|(n, v)| { - ( - n, - v.into_iter() - .filter(|(_, n)| n.node.resolved_ty != DependencyType::Dev) - .collect(), - ) - }) - .collect() - } else { - downloaded_graph.clone() + .await + .context("failed to download & link dependencies")? }; #[cfg(feature = "patches")] { let rx = project - .apply_patches(&filtered_graph) + .apply_patches(&filter_graph(&downloaded_graph, self.prod)) .await .context("failed to apply patches")?; @@ -251,69 +299,13 @@ impl InstallCommand { manifest.patches.values().map(|v| v.len() as u64).sum(), rx, &multi, - format!("{} 🩹 ", job(4)), + format!("{} 🩹 ", job(JOBS - 1)), "applying patches".to_string(), "applied patches".to_string(), ) .await?; } - println!("{} 🗺️ linking dependencies", job(JOBS - 1)); - - let bin_folder = bin_dir().await?; - - try_join_all( - filtered_graph - .values() - .flat_map(|versions| versions.values()) - .filter(|node| node.target.bin_path().is_some()) - .filter_map(|node| node.node.direct.as_ref()) - .map(|(alias, _, _)| alias) - .filter(|alias| { - if *alias == env!("CARGO_BIN_NAME") { - log::warn!( - "package {alias} has the same name as the CLI, skipping bin link" - ); - return false; - } - - true - }) - .map(|alias| { - let bin_folder = bin_folder.clone(); - async move { - let bin_file = bin_folder.join(alias); - fs::write(&bin_file, bin_link_file(alias)) - .await - .context("failed to write bin link file")?; - - make_executable(&bin_file) - .await - .context("failed to make bin link executable")?; - - #[cfg(windows)] - { - let bin_file = bin_file.with_extension(std::env::consts::EXE_EXTENSION); - fs::copy( - std::env::current_exe() - .context("failed to get current executable path")?, - &bin_file, - ) - .await - .context("failed to copy bin link file")?; - } - - Ok::<_, anyhow::Error>(()) - } - }), - ) - .await?; - - project - .link_dependencies(&filtered_graph) - .await - .context("failed to link dependencies")?; - println!("{} 🧹 finishing up", job(JOBS)); project diff --git a/src/cli/commands/update.rs b/src/cli/commands/update.rs index ea0f866..863332b 100644 --- a/src/cli/commands/update.rs +++ b/src/cli/commands/update.rs @@ -5,6 +5,7 @@ use colored::Colorize; use indicatif::MultiProgress; use pesde::{lockfile::Lockfile, Project}; use std::{collections::HashSet, sync::Arc}; +use tokio::sync::Mutex; #[derive(Debug, Args, Copy, Clone)] pub struct UpdateCommand {} @@ -34,6 +35,7 @@ impl UpdateCommand { .dependency_graph(None, &mut refreshed_sources, false) .await .context("failed to build dependency graph")?; + let graph = Arc::new(graph); update_scripts(&project).await?; @@ -46,7 +48,14 @@ impl UpdateCommand { graph: { let (rx, downloaded_graph) = project - .download_graph(&graph, &mut refreshed_sources, &reqwest, false, false) + .download_and_link( + &graph, + &Arc::new(Mutex::new(refreshed_sources)), + &reqwest, + false, + false, + |_| async { Ok::<_, std::io::Error>(()) }, + ) .await .context("failed to download dependencies")?; @@ -60,10 +69,9 @@ impl UpdateCommand { ) .await?; - Arc::into_inner(downloaded_graph) - .unwrap() - .into_inner() - .unwrap() + downloaded_graph + .await + .context("failed to download dependencies")? }, workspace: run_on_workspace_members(&project, |project| { diff --git a/src/download.rs b/src/download.rs index 9b4b150..f0b4bcf 100644 --- a/src/download.rs +++ b/src/download.rs @@ -16,7 +16,7 @@ use std::{ type MultithreadedGraph = Arc>; -type MultithreadDownloadJob = ( +pub(crate) type MultithreadDownloadJob = ( tokio::sync::mpsc::Receiver>, MultithreadedGraph, ); @@ -30,6 +30,7 @@ impl Project { reqwest: &reqwest::Client, prod: bool, write: bool, + wally: bool, ) -> Result { let manifest = self.deser_manifest().await?; let manifest_target_kind = manifest.target.kind(); @@ -53,15 +54,22 @@ impl Project { ) .await?; + let project = Arc::new(self.clone()); + for (name, versions) in graph { for (version_id, node) in versions { + // we need to download pesde packages first, since scripts (for target finding for example) can depend on them + if node.pkg_ref.like_wally() != wally { + continue; + } + let tx = tx.clone(); let name = name.clone(); let version_id = version_id.clone(); let node = node.clone(); - let project = Arc::new(self.clone()); + let project = project.clone(); let reqwest = reqwest.clone(); let downloaded_graph = downloaded_graph.clone(); diff --git a/src/download_and_link.rs b/src/download_and_link.rs new file mode 100644 index 0000000..e3a33c2 --- /dev/null +++ b/src/download_and_link.rs @@ -0,0 +1,164 @@ +use crate::{ + lockfile::{DependencyGraph, DownloadedGraph}, + manifest::DependencyType, + source::PackageSources, + Project, +}; +use futures::FutureExt; +use std::{ + collections::HashSet, + future::Future, + sync::{Arc, Mutex as StdMutex}, +}; +use tokio::sync::Mutex; + +/// Filters a graph to only include production dependencies, if `prod` is `true` +pub fn filter_graph(graph: &DownloadedGraph, prod: bool) -> DownloadedGraph { + if !prod { + return graph.clone(); + } + + graph + .iter() + .map(|(name, versions)| { + ( + name.clone(), + versions + .iter() + .filter(|(_, node)| node.node.resolved_ty != DependencyType::Dev) + .map(|(v_id, node)| (v_id.clone(), node.clone())) + .collect(), + ) + }) + .collect() +} + +impl Project { + /// Downloads a graph of dependencies and links them in the correct order + pub async fn download_and_link< + F: FnOnce(&Arc) -> R + Send + 'static, + R: Future> + Send, + E: Send + Sync + 'static, + >( + &self, + graph: &Arc, + refreshed_sources: &Arc>>, + reqwest: &reqwest::Client, + prod: bool, + write: bool, + pesde_cb: F, + ) -> Result< + ( + tokio::sync::mpsc::Receiver< + Result, + >, + impl Future>>, + ), + errors::DownloadAndLinkError, + > { + let (tx, rx) = tokio::sync::mpsc::channel( + graph + .iter() + .map(|(_, versions)| versions.len()) + .sum::() + .max(1), + ); + let downloaded_graph = Arc::new(StdMutex::new(DownloadedGraph::default())); + + let this = self.clone(); + let graph = graph.clone(); + let reqwest = reqwest.clone(); + let refreshed_sources = refreshed_sources.clone(); + + Ok(( + rx, + tokio::spawn(async move { + let mut refreshed_sources = refreshed_sources.lock().await; + + // step 1. download pesde dependencies + let (mut pesde_rx, pesde_graph) = this + .download_graph(&graph, &mut refreshed_sources, &reqwest, prod, write, false) + .await?; + + while let Some(result) = pesde_rx.recv().await { + tx.send(result).await.unwrap(); + } + + let pesde_graph = Arc::into_inner(pesde_graph).unwrap().into_inner().unwrap(); + + // step 2. link pesde dependencies. do so without types + if write { + this.link_dependencies(&filter_graph(&pesde_graph, prod), false) + .await?; + } + + let pesde_graph = Arc::new(pesde_graph); + + pesde_cb(&pesde_graph) + .await + .map_err(errors::DownloadAndLinkError::PesdeCallback)?; + + let pesde_graph = Arc::into_inner(pesde_graph).unwrap(); + + // step 3. download wally dependencies + let (mut wally_rx, wally_graph) = this + .download_graph(&graph, &mut refreshed_sources, &reqwest, prod, write, true) + .await?; + + while let Some(result) = wally_rx.recv().await { + tx.send(result).await.unwrap(); + } + + let wally_graph = Arc::into_inner(wally_graph).unwrap().into_inner().unwrap(); + + { + let mut downloaded_graph = downloaded_graph.lock().unwrap(); + downloaded_graph.extend(pesde_graph); + for (name, versions) in wally_graph { + for (version_id, node) in versions { + downloaded_graph + .entry(name.clone()) + .or_default() + .insert(version_id, node); + } + } + } + + let graph = Arc::into_inner(downloaded_graph) + .unwrap() + .into_inner() + .unwrap(); + + // step 4. link ALL dependencies. do so with types + if write { + this.link_dependencies(&filter_graph(&graph, prod), true) + .await?; + } + + Ok(graph) + }) + .map(|r| r.unwrap()), + )) + } +} + +/// Errors that can occur when downloading and linking dependencies +pub mod errors { + use thiserror::Error; + + /// An error that can occur when downloading and linking dependencies + #[derive(Debug, Error)] + pub enum DownloadAndLinkError { + /// An error occurred while downloading the graph + #[error("error downloading graph")] + DownloadGraph(#[from] crate::download::errors::DownloadGraphError), + + /// An error occurred while linking dependencies + #[error("error linking dependencies")] + Linking(#[from] crate::linking::errors::LinkingError), + + /// An error occurred while executing the pesde callback + #[error("error executing pesde callback")] + PesdeCallback(#[source] E), + } +} diff --git a/src/lib.rs b/src/lib.rs index 837b003..c1812cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,6 +20,8 @@ use wax::Pattern; /// Downloading packages pub mod download; +/// Utility for downloading and linking in the correct order +pub mod download_and_link; /// Linking packages pub mod linking; /// Lockfile diff --git a/src/linking/mod.rs b/src/linking/mod.rs index 6474acc..a882aae 100644 --- a/src/linking/mod.rs +++ b/src/linking/mod.rs @@ -47,106 +47,114 @@ impl Project { pub async fn link_dependencies( &self, graph: &DownloadedGraph, + with_types: bool, ) -> Result<(), errors::LinkingError> { let manifest = self.deser_manifest().await?; let manifest_target_kind = manifest.target.kind(); let manifest = Arc::new(manifest); - // step 1. link all packages (and their dependencies) temporarily without types + // step 1. link all non-wally packages (and their dependencies) temporarily without types // we do this separately to allow the required tools for the scripts to be installed self.link(graph, &manifest, &Arc::new(Default::default())) .await?; - // step 2. extract the types from libraries + if !with_types { + return Ok(()); + } + + // step 2. extract the types from libraries, prepare Roblox packages for syncing let roblox_sync_config_gen_script = manifest .scripts .get(&ScriptName::RobloxSyncConfigGenerator.to_string()); - let package_types = try_join_all( - graph - .iter() - .map(|(name, versions)| async move { - Ok::<_, errors::LinkingError>((name, try_join_all(versions.iter().map(|(version_id, node)| async move { - let Some(lib_file) = node.target.lib_path() else { + let package_types = try_join_all(graph.iter().map(|(name, versions)| async move { + Ok::<_, errors::LinkingError>(( + name, + try_join_all(versions.iter().map(|(version_id, node)| async move { + let Some(lib_file) = node.target.lib_path() else { + return Ok((version_id, vec![])); + }; + + let container_folder = node.node.container_folder( + &self + .package_dir() + .join(manifest_target_kind.packages_folder(version_id.target())) + .join(PACKAGES_CONTAINER_NAME), + name, + version_id.version(), + ); + + let types = if lib_file.as_str() != LINK_LIB_NO_FILE_FOUND { + let lib_file = lib_file.to_path(&container_folder); + + let contents = match fs::read_to_string(&lib_file).await { + Ok(contents) => contents, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Err(errors::LinkingError::LibFileNotFound( + lib_file.display().to_string(), + )); + } + Err(e) => return Err(e.into()), + }; + + let types = match spawn_blocking(move || get_file_types(&contents)) + .await + .unwrap() + { + Ok(types) => types, + Err(e) => { + return Err(errors::LinkingError::FullMoon( + lib_file.display().to_string(), + e, + )) + } + }; + + log::debug!("{name}@{version_id} has {} exported types", types.len()); + + types + } else { + vec![] + }; + + if let Some(build_files) = Some(&node.target) + .filter(|_| !node.node.pkg_ref.like_wally()) + .and_then(|t| t.build_files()) + { + let Some(script_path) = roblox_sync_config_gen_script else { + log::warn!("not having a `{}` script in the manifest might cause issues with Roblox linking", ScriptName::RobloxSyncConfigGenerator); return Ok((version_id, vec![])); }; - let container_folder = node.node.container_folder( - &self - .package_dir() - .join(manifest_target_kind.packages_folder(version_id.target())) - .join(PACKAGES_CONTAINER_NAME), - name, - version_id.version(), - ); + execute_script( + ScriptName::RobloxSyncConfigGenerator, + &script_path.to_path(self.package_dir()), + std::iter::once(container_folder.as_os_str()) + .chain(build_files.iter().map(OsStr::new)), + self, + false, + ).await + .map_err(|e| { + errors::LinkingError::GenerateRobloxSyncConfig( + container_folder.display().to_string(), + e, + ) + })?; + } - let types = if lib_file.as_str() != LINK_LIB_NO_FILE_FOUND { - let lib_file = lib_file.to_path(&container_folder); - - let contents = match fs::read_to_string(&lib_file).await { - Ok(contents) => contents, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - return Err(errors::LinkingError::LibFileNotFound( - lib_file.display().to_string(), - )); - } - Err(e) => return Err(e.into()), - }; - - let types = match spawn_blocking(move || get_file_types(&contents)).await.unwrap() { - Ok(types) => types, - Err(e) => { - return Err(errors::LinkingError::FullMoon( - lib_file.display().to_string(), - e, - )) - } - }; - - log::debug!("{name}@{version_id} has {} exported types", types.len()); - - types - } else { - vec![] - }; - - if let Some(build_files) = Some(&node.target) - .filter(|_| !node.node.pkg_ref.like_wally()) - .and_then(|t| t.build_files()) - { - let Some(script_path) = roblox_sync_config_gen_script else { - log::warn!("not having a `{}` script in the manifest might cause issues with Roblox linking", ScriptName::RobloxSyncConfigGenerator); - return Ok((version_id, vec![])); - }; - - execute_script( - ScriptName::RobloxSyncConfigGenerator, - &script_path.to_path(self.package_dir()), - std::iter::once(container_folder.as_os_str()) - .chain(build_files.iter().map(OsStr::new)), - self, - false, - ) - .map_err(|e| { - errors::LinkingError::GenerateRobloxSyncConfig( - container_folder.display().to_string(), - e, - ) - })?; - } - - Ok((version_id, types)) - })).await?.into_iter().collect::>())) - } - ) - ) + Ok((version_id, types)) + })) + .await? + .into_iter() + .collect::>(), + )) + })) .await? .into_iter() .collect::>(); - let package_types = Arc::new(package_types); // step 3. link all packages (and their dependencies), this time with types - self.link(graph, &manifest, &package_types).await + self.link(graph, &manifest, &Arc::new(package_types)).await } async fn link( diff --git a/src/scripts.rs b/src/scripts.rs index 6527c30..b7de514 100644 --- a/src/scripts.rs +++ b/src/scripts.rs @@ -2,10 +2,12 @@ use crate::Project; use std::{ ffi::OsStr, fmt::{Display, Formatter}, - io::{BufRead, BufReader}, path::Path, - process::{Command, Stdio}, - thread::spawn, + process::Stdio, +}; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::Command, }; /// Script names used by pesde @@ -28,7 +30,7 @@ impl Display for ScriptName { } } -pub(crate) fn execute_script, S: AsRef>( +pub(crate) async fn execute_script, S: AsRef>( script_name: ScriptName, script_path: &Path, args: A, @@ -47,14 +49,14 @@ pub(crate) fn execute_script, S: AsRef>( .spawn() { Ok(mut child) => { - let stdout = BufReader::new(child.stdout.take().unwrap()); - let stderr = BufReader::new(child.stderr.take().unwrap()); + let mut stdout = BufReader::new(child.stdout.take().unwrap()).lines(); + let mut stderr = BufReader::new(child.stderr.take().unwrap()).lines(); let script = script_name.to_string(); let script_2 = script.to_string(); - spawn(move || { - for line in stderr.lines() { + tokio::spawn(async move { + while let Some(line) = stderr.next_line().await.transpose() { match line { Ok(line) => { log::error!("[{script}]: {line}"); @@ -69,7 +71,7 @@ pub(crate) fn execute_script, S: AsRef>( let mut stdout_str = String::new(); - for line in stdout.lines() { + while let Some(line) = stdout.next_line().await.transpose() { match line { Ok(line) => { if return_stdout { diff --git a/src/source/wally/compat_util.rs b/src/source/wally/compat_util.rs index befb877..d481ab4 100644 --- a/src/source/wally/compat_util.rs +++ b/src/source/wally/compat_util.rs @@ -39,7 +39,8 @@ pub(crate) async fn find_lib_path( [package_dir], project, true, - )?; + ) + .await?; if let Some(result) = result.filter(|result| !result.is_empty()) { let node: SourcemapNode = serde_json::from_str(&result)?;