feat: install pesde packages before wally

CHANGELOG.md

@@ -8,9 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
 - Add improved CLI styling by @daimond113
+- Install pesde dependencies before Wally to support scripts packages by @daimond113
 
 ### Fixed
-- Link dependencies before type extraction to support more use cases
+- Link dependencies before type extraction to support more use cases by @daimond113
 
 ## [0.5.0-rc.14] - 2024-11-30
 ### Fixed

src/cli/commands/execute.rs

@@ -5,7 +5,7 @@ use fs_err::tokio as fs;
 use indicatif::MultiProgress;
 use pesde::{
     linking::generator::generate_bin_linking_module,
-    manifest::{target::TargetKind, DependencyType},
+    manifest::target::TargetKind,
     names::PackageName,
     source::{
         pesde::{specifier::PesdeDependencySpecifier, PesdePackageSource},
@@ -17,6 +17,7 @@ use semver::VersionReq;
 use std::{
     collections::HashSet, env::current_dir, ffi::OsString, io::Write, process::Command, sync::Arc,
 };
+use tokio::sync::Mutex;
 
 #[derive(Debug, Args)]
 pub struct ExecuteCommand {
@@ -116,9 +117,17 @@ impl ExecuteCommand {
             .dependency_graph(None, &mut refreshed_sources, true)
             .await
             .context("failed to build dependency graph")?;
+        let graph = Arc::new(graph);
 
         let (rx, downloaded_graph) = project
-            .download_graph(&graph, &mut refreshed_sources, &reqwest, true, true)
+            .download_and_link(
+                &graph,
+                &Arc::new(Mutex::new(refreshed_sources)),
+                &reqwest,
+                true,
+                true,
+                |_| async { Ok::<_, std::io::Error>(()) },
+            )
             .await
             .context("failed to download dependencies")?;
 
@@ -132,27 +141,9 @@ impl ExecuteCommand {
         )
         .await?;
 
-        let downloaded_graph = Arc::into_inner(downloaded_graph)
+        downloaded_graph
-            .unwrap()
-            .into_inner()
-            .unwrap();
-
-        project
-            .link_dependencies(
-                &downloaded_graph
-                    .into_iter()
-                    .map(|(n, v)| {
-                        (
-                            n,
-                            v.into_iter()
-                                .filter(|(_, n)| n.node.resolved_ty != DependencyType::Dev)
-                                .collect(),
-                        )
-                    })
-                    .collect(),
-            )
             .await
-            .context("failed to link dependencies")?;
+            .context("failed to download & link dependencies")?;
 
         let mut caller =
             tempfile::NamedTempFile::new_in(tempdir.path()).context("failed to create tempfile")?;

src/cli/commands/install.rs

@@ -9,14 +9,14 @@ use fs_err::tokio as fs;
 use futures::future::try_join_all;
 use indicatif::MultiProgress;
 use pesde::{
-    lockfile::Lockfile,
+    download_and_link::filter_graph, lockfile::Lockfile, manifest::target::TargetKind, Project,
-    manifest::{target::TargetKind, DependencyType},
+    MANIFEST_FILE_NAME,
-    Project, MANIFEST_FILE_NAME,
 };
 use std::{
     collections::{BTreeSet, HashMap, HashSet},
     sync::Arc,
 };
+use tokio::sync::Mutex;
 
 #[derive(Debug, Args, Copy, Clone)]
 pub struct InstallCommand {
@@ -81,14 +81,18 @@ stdio.ewrite(stdio.color("red") .. "binary `{alias}` not found. are you in the r
 }
 
 #[cfg(feature = "patches")]
-const JOBS: u8 = 6;
-#[cfg(not(feature = "patches"))]
 const JOBS: u8 = 5;
+#[cfg(not(feature = "patches"))]
+const JOBS: u8 = 4;
 
 fn job(n: u8) -> ColoredString {
     format!("[{n}/{JOBS}]").dimmed().bold()
 }
 
+#[derive(Debug, thiserror::Error)]
+#[error(transparent)]
+struct CallbackError(#[from] anyhow::Error);
+
 impl InstallCommand {
     pub async fn run(
         self,
@@ -198,72 +202,26 @@ impl InstallCommand {
             .dependency_graph(old_graph.as_ref(), &mut refreshed_sources, false)
             .await
             .context("failed to build dependency graph")?;
+        let graph = Arc::new(graph);
 
         update_scripts_handle.await??;
 
-        let downloaded_graph = {
-            let (rx, downloaded_graph) = project
-                .download_graph(&graph, &mut refreshed_sources, &reqwest, self.prod, true)
-                .await
-                .context("failed to download dependencies")?;
-
-            progress_bar(
-                graph.values().map(|versions| versions.len() as u64).sum(),
-                rx,
-                &multi,
-                format!("{} 📥 ", job(3)),
-                "downloading dependencies".to_string(),
-                "downloaded dependencies".to_string(),
-            )
-            .await?;
-
-            Arc::into_inner(downloaded_graph)
-                .unwrap()
-                .into_inner()
-                .unwrap()
-        };
-
-        let filtered_graph = if self.prod {
-            downloaded_graph
-                .clone()
-                .into_iter()
-                .map(|(n, v)| {
-                    (
-                        n,
-                        v.into_iter()
-                            .filter(|(_, n)| n.node.resolved_ty != DependencyType::Dev)
-                            .collect(),
-                    )
-                })
-                .collect()
-        } else {
-            downloaded_graph.clone()
-        };
-
-        #[cfg(feature = "patches")]
-        {
-            let rx = project
-                .apply_patches(&filtered_graph)
-                .await
-                .context("failed to apply patches")?;
-
-            progress_bar(
-                manifest.patches.values().map(|v| v.len() as u64).sum(),
-                rx,
-                &multi,
-                format!("{} 🩹 ", job(4)),
-                "applying patches".to_string(),
-                "applied patches".to_string(),
-            )
-            .await?;
-        }
-
-        println!("{} 🗺️ linking dependencies", job(JOBS - 1));
-
         let bin_folder = bin_dir().await?;
 
+        let downloaded_graph = {
+            let (rx, downloaded_graph) = project
+                .download_and_link(
+                    &graph,
+                    &Arc::new(Mutex::new(refreshed_sources)),
+                    &reqwest,
+                    self.prod,
+                    true,
+                    |graph| {
+                        let graph = graph.clone();
+
+                        async move {
                             try_join_all(
-            filtered_graph
+                                graph
                                     .values()
                                     .flat_map(|versions| versions.values())
                                     .filter(|node| node.target.bin_path().is_some())
@@ -303,16 +261,50 @@ impl InstallCommand {
                                                     .context("failed to copy bin link file")?;
                                             }
 
-                        Ok::<_, anyhow::Error>(())
+                                            Ok::<_, CallbackError>(())
                                         }
                                     }),
+                            )
+                                .await
+                                .map(|_| ())
+                        }
+                    }
+                )
+                .await
+                .context("failed to download dependencies")?;
+
+            progress_bar(
+                graph.values().map(|versions| versions.len() as u64).sum(),
+                rx,
+                &multi,
+                format!("{} 📥 ", job(3)),
+                "downloading dependencies".to_string(),
+                "downloaded dependencies".to_string(),
             )
             .await?;
 
-        project
+            downloaded_graph
-            .link_dependencies(&filtered_graph)
                 .await
-            .context("failed to link dependencies")?;
+                .context("failed to download & link dependencies")?
+        };
+
+        #[cfg(feature = "patches")]
+        {
+            let rx = project
+                .apply_patches(&filter_graph(&downloaded_graph, self.prod))
+                .await
+                .context("failed to apply patches")?;
+
+            progress_bar(
+                manifest.patches.values().map(|v| v.len() as u64).sum(),
+                rx,
+                &multi,
+                format!("{} 🩹 ", job(JOBS - 1)),
+                "applying patches".to_string(),
+                "applied patches".to_string(),
+            )
+            .await?;
+        }
 
         println!("{} 🧹 finishing up", job(JOBS));
 

src/cli/commands/update.rs

@@ -5,6 +5,7 @@ use colored::Colorize;
 use indicatif::MultiProgress;
 use pesde::{lockfile::Lockfile, Project};
 use std::{collections::HashSet, sync::Arc};
+use tokio::sync::Mutex;
 
 #[derive(Debug, Args, Copy, Clone)]
 pub struct UpdateCommand {}
@@ -34,6 +35,7 @@ impl UpdateCommand {
             .dependency_graph(None, &mut refreshed_sources, false)
             .await
             .context("failed to build dependency graph")?;
+        let graph = Arc::new(graph);
 
         update_scripts(&project).await?;
 
@@ -46,7 +48,14 @@ impl UpdateCommand {
 
                 graph: {
                     let (rx, downloaded_graph) = project
-                        .download_graph(&graph, &mut refreshed_sources, &reqwest, false, false)
+                        .download_and_link(
+                            &graph,
+                            &Arc::new(Mutex::new(refreshed_sources)),
+                            &reqwest,
+                            false,
+                            false,
+                            |_| async { Ok::<_, std::io::Error>(()) },
+                        )
                         .await
                         .context("failed to download dependencies")?;
 
@@ -60,10 +69,9 @@ impl UpdateCommand {
                     )
                     .await?;
 
-                    Arc::into_inner(downloaded_graph)
+                    downloaded_graph
-                        .unwrap()
+                        .await
-                        .into_inner()
+                        .context("failed to download dependencies")?
-                        .unwrap()
                 },
 
                 workspace: run_on_workspace_members(&project, |project| {

src/download.rs

@@ -16,7 +16,7 @@ use std::{
 
 type MultithreadedGraph = Arc<Mutex<DownloadedGraph>>;
 
-type MultithreadDownloadJob = (
+pub(crate) type MultithreadDownloadJob = (
     tokio::sync::mpsc::Receiver<Result<String, errors::DownloadGraphError>>,
     MultithreadedGraph,
 );
@@ -30,6 +30,7 @@ impl Project {
         reqwest: &reqwest::Client,
         prod: bool,
         write: bool,
+        wally: bool,
     ) -> Result<MultithreadDownloadJob, errors::DownloadGraphError> {
         let manifest = self.deser_manifest().await?;
         let manifest_target_kind = manifest.target.kind();
@@ -53,15 +54,22 @@ impl Project {
         )
         .await?;
 
+        let project = Arc::new(self.clone());
+
         for (name, versions) in graph {
             for (version_id, node) in versions {
+                // we need to download pesde packages first, since scripts (for target finding for example) can depend on them
+                if node.pkg_ref.like_wally() != wally {
+                    continue;
+                }
+
                 let tx = tx.clone();
 
                 let name = name.clone();
                 let version_id = version_id.clone();
                 let node = node.clone();
 
-                let project = Arc::new(self.clone());
+                let project = project.clone();
                 let reqwest = reqwest.clone();
                 let downloaded_graph = downloaded_graph.clone();
 

src/download_and_link.rs

@@ -0,0 +1,164 @@
+use crate::{
+    lockfile::{DependencyGraph, DownloadedGraph},
+    manifest::DependencyType,
+    source::PackageSources,
+    Project,
+};
+use futures::FutureExt;
+use std::{
+    collections::HashSet,
+    future::Future,
+    sync::{Arc, Mutex as StdMutex},
+};
+use tokio::sync::Mutex;
+
+/// Filters a graph to only include production dependencies, if `prod` is `true`
+pub fn filter_graph(graph: &DownloadedGraph, prod: bool) -> DownloadedGraph {
+    if !prod {
+        return graph.clone();
+    }
+
+    graph
+        .iter()
+        .map(|(name, versions)| {
+            (
+                name.clone(),
+                versions
+                    .iter()
+                    .filter(|(_, node)| node.node.resolved_ty != DependencyType::Dev)
+                    .map(|(v_id, node)| (v_id.clone(), node.clone()))
+                    .collect(),
+            )
+        })
+        .collect()
+}
+
+impl Project {
+    /// Downloads a graph of dependencies and links them in the correct order
+    pub async fn download_and_link<
+        F: FnOnce(&Arc<DownloadedGraph>) -> R + Send + 'static,
+        R: Future<Output = Result<(), E>> + Send,
+        E: Send + Sync + 'static,
+    >(
+        &self,
+        graph: &Arc<DependencyGraph>,
+        refreshed_sources: &Arc<Mutex<HashSet<PackageSources>>>,
+        reqwest: &reqwest::Client,
+        prod: bool,
+        write: bool,
+        pesde_cb: F,
+    ) -> Result<
+        (
+            tokio::sync::mpsc::Receiver<
+                Result<String, crate::download::errors::DownloadGraphError>,
+            >,
+            impl Future<Output = Result<DownloadedGraph, errors::DownloadAndLinkError<E>>>,
+        ),
+        errors::DownloadAndLinkError<E>,
+    > {
+        let (tx, rx) = tokio::sync::mpsc::channel(
+            graph
+                .iter()
+                .map(|(_, versions)| versions.len())
+                .sum::<usize>()
+                .max(1),
+        );
+        let downloaded_graph = Arc::new(StdMutex::new(DownloadedGraph::default()));
+
+        let this = self.clone();
+        let graph = graph.clone();
+        let reqwest = reqwest.clone();
+        let refreshed_sources = refreshed_sources.clone();
+
+        Ok((
+            rx,
+            tokio::spawn(async move {
+                let mut refreshed_sources = refreshed_sources.lock().await;
+
+                // step 1. download pesde dependencies
+                let (mut pesde_rx, pesde_graph) = this
+                    .download_graph(&graph, &mut refreshed_sources, &reqwest, prod, write, false)
+                    .await?;
+
+                while let Some(result) = pesde_rx.recv().await {
+                    tx.send(result).await.unwrap();
+                }
+
+                let pesde_graph = Arc::into_inner(pesde_graph).unwrap().into_inner().unwrap();
+
+                // step 2. link pesde dependencies. do so without types
+                if write {
+                    this.link_dependencies(&filter_graph(&pesde_graph, prod), false)
+                        .await?;
+                }
+
+                let pesde_graph = Arc::new(pesde_graph);
+
+                pesde_cb(&pesde_graph)
+                    .await
+                    .map_err(errors::DownloadAndLinkError::PesdeCallback)?;
+
+                let pesde_graph = Arc::into_inner(pesde_graph).unwrap();
+
+                // step 3. download wally dependencies
+                let (mut wally_rx, wally_graph) = this
+                    .download_graph(&graph, &mut refreshed_sources, &reqwest, prod, write, true)
+                    .await?;
+
+                while let Some(result) = wally_rx.recv().await {
+                    tx.send(result).await.unwrap();
+                }
+
+                let wally_graph = Arc::into_inner(wally_graph).unwrap().into_inner().unwrap();
+
+                {
+                    let mut downloaded_graph = downloaded_graph.lock().unwrap();
+                    downloaded_graph.extend(pesde_graph);
+                    for (name, versions) in wally_graph {
+                        for (version_id, node) in versions {
+                            downloaded_graph
+                                .entry(name.clone())
+                                .or_default()
+                                .insert(version_id, node);
+                        }
+                    }
+                }
+
+                let graph = Arc::into_inner(downloaded_graph)
+                    .unwrap()
+                    .into_inner()
+                    .unwrap();
+
+                // step 4. link ALL dependencies. do so with types
+                if write {
+                    this.link_dependencies(&filter_graph(&graph, prod), true)
+                        .await?;
+                }
+
+                Ok(graph)
+            })
+            .map(|r| r.unwrap()),
+        ))
+    }
+}
+
+/// Errors that can occur when downloading and linking dependencies
+pub mod errors {
+    use thiserror::Error;
+
+    /// An error that can occur when downloading and linking dependencies
+    #[derive(Debug, Error)]
+    pub enum DownloadAndLinkError<E> {
+        /// An error occurred while downloading the graph
+        #[error("error downloading graph")]
+        DownloadGraph(#[from] crate::download::errors::DownloadGraphError),
+
+        /// An error occurred while linking dependencies
+        #[error("error linking dependencies")]
+        Linking(#[from] crate::linking::errors::LinkingError),
+
+        /// An error occurred while executing the pesde callback
+        #[error("error executing pesde callback")]
+        PesdeCallback(#[source] E),
+    }
+}

src/lib.rs

@@ -20,6 +20,8 @@ use wax::Pattern;
 
 /// Downloading packages
 pub mod download;
+/// Utility for downloading and linking in the correct order
+pub mod download_and_link;
 /// Linking packages
 pub mod linking;
 /// Lockfile

src/linking/mod.rs

@@ -47,26 +47,30 @@ impl Project {
     pub async fn link_dependencies(
         &self,
         graph: &DownloadedGraph,
+        with_types: bool,
     ) -> Result<(), errors::LinkingError> {
         let manifest = self.deser_manifest().await?;
         let manifest_target_kind = manifest.target.kind();
         let manifest = Arc::new(manifest);
 
-        // step 1. link all packages (and their dependencies) temporarily without types
+        // step 1. link all non-wally packages (and their dependencies) temporarily without types
         // we do this separately to allow the required tools for the scripts to be installed
         self.link(graph, &manifest, &Arc::new(Default::default()))
             .await?;
 
-        // step 2. extract the types from libraries
+        if !with_types {
+            return Ok(());
+        }
+
+        // step 2. extract the types from libraries, prepare Roblox packages for syncing
         let roblox_sync_config_gen_script = manifest
             .scripts
             .get(&ScriptName::RobloxSyncConfigGenerator.to_string());
 
-        let package_types = try_join_all(
+        let package_types = try_join_all(graph.iter().map(|(name, versions)| async move {
-            graph
+            Ok::<_, errors::LinkingError>((
-                .iter()
+                name,
-                .map(|(name, versions)| async move {
+                try_join_all(versions.iter().map(|(version_id, node)| async move {
-                    Ok::<_, errors::LinkingError>((name, try_join_all(versions.iter().map(|(version_id, node)| async move {
                     let Some(lib_file) = node.target.lib_path() else {
                         return Ok((version_id, vec![]));
                     };
@@ -93,7 +97,10 @@ impl Project {
                             Err(e) => return Err(e.into()),
                         };
 
-                            let types = match spawn_blocking(move || get_file_types(&contents)).await.unwrap() {
+                        let types = match spawn_blocking(move || get_file_types(&contents))
+                            .await
+                            .unwrap()
+                        {
                             Ok(types) => types,
                             Err(e) => {
                                 return Err(errors::LinkingError::FullMoon(
@@ -126,7 +133,7 @@ impl Project {
                                 .chain(build_files.iter().map(OsStr::new)),
                             self,
                             false,
-                            )
+                        ).await
                             .map_err(|e| {
                                 errors::LinkingError::GenerateRobloxSyncConfig(
                                     container_folder.display().to_string(),
@@ -136,17 +143,18 @@ impl Project {
                     }
 
                     Ok((version_id, types))
-                    })).await?.into_iter().collect::<HashMap<_, _>>()))
+                }))
-                }
+                    .await?
-                )
+                    .into_iter()
-        )
+                    .collect::<HashMap<_, _>>(),
+            ))
+        }))
             .await?
             .into_iter()
             .collect::<HashMap<_, _>>();
 
-        let package_types = Arc::new(package_types);
         // step 3. link all packages (and their dependencies), this time with types
-        self.link(graph, &manifest, &package_types).await
+        self.link(graph, &manifest, &Arc::new(package_types)).await
     }
 
     async fn link(

src/scripts.rs

@@ -2,10 +2,12 @@ use crate::Project;
 use std::{
     ffi::OsStr,
     fmt::{Display, Formatter},
-    io::{BufRead, BufReader},
     path::Path,
-    process::{Command, Stdio},
+    process::Stdio,
-    thread::spawn,
+};
+use tokio::{
+    io::{AsyncBufReadExt, BufReader},
+    process::Command,
 };
 
 /// Script names used by pesde
@@ -28,7 +30,7 @@ impl Display for ScriptName {
     }
 }
 
-pub(crate) fn execute_script<A: IntoIterator<Item = S>, S: AsRef<OsStr>>(
+pub(crate) async fn execute_script<A: IntoIterator<Item = S>, S: AsRef<OsStr>>(
     script_name: ScriptName,
     script_path: &Path,
     args: A,
@@ -47,14 +49,14 @@ pub(crate) fn execute_script<A: IntoIterator<Item = S>, S: AsRef<OsStr>>(
         .spawn()
     {
         Ok(mut child) => {
-            let stdout = BufReader::new(child.stdout.take().unwrap());
+            let mut stdout = BufReader::new(child.stdout.take().unwrap()).lines();
-            let stderr = BufReader::new(child.stderr.take().unwrap());
+            let mut stderr = BufReader::new(child.stderr.take().unwrap()).lines();
 
             let script = script_name.to_string();
             let script_2 = script.to_string();
 
-            spawn(move || {
+            tokio::spawn(async move {
-                for line in stderr.lines() {
+                while let Some(line) = stderr.next_line().await.transpose() {
                     match line {
                         Ok(line) => {
                             log::error!("[{script}]: {line}");
@@ -69,7 +71,7 @@ pub(crate) fn execute_script<A: IntoIterator<Item = S>, S: AsRef<OsStr>>(
 
             let mut stdout_str = String::new();
 
-            for line in stdout.lines() {
+            while let Some(line) = stdout.next_line().await.transpose() {
                 match line {
                     Ok(line) => {
                         if return_stdout {

src/source/wally/compat_util.rs

@@ -39,7 +39,8 @@ pub(crate) async fn find_lib_path(
         [package_dir],
         project,
         true,
-    )?;
+    )
+    .await?;
 
     if let Some(result) = result.filter(|result| !result.is_empty()) {
         let node: SourcemapNode = serde_json::from_str(&result)?;