perf: use arcs where possible, remove unnecessary cloning

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- Improve installation experience by @lukadev-0
+
+### Performance
+- Use `Arc` for more efficient cloning of multiple structs by @daimond113
+- Avoid cloning where possible by @daimond113
+
 ## [0.5.2] - 2024-12-19
 ### Fixed
 - Change dependency types for removed peer dependencies by @daimond113

registry/src/endpoints/publish_version.rs

@@ -18,6 +18,7 @@ use pesde::{
         git_index::{read_file, root_tree, GitBasedSource},
         pesde::{DocEntry, DocEntryKind, IndexFile, IndexFileEntry, ScopeInfo, SCOPE_INFO_FILE},
         specifiers::DependencySpecifiers,
+        traits::RefreshOptions,
         version_id::VersionId,
         IGNORED_DIRS, IGNORED_FILES,
     },
@@ -72,7 +73,12 @@ pub async fn publish_package(
     user_id: web::ReqData<UserId>,
 ) -> Result<impl Responder, Error> {
     let source = app_state.source.lock().await;
-    source.refresh(&app_state.project).await.map_err(Box::new)?;
+    source
+        .refresh(&RefreshOptions {
+            project: app_state.project.clone(),
+        })
+        .await
+        .map_err(Box::new)?;
     let config = source.config(&app_state.project).await?;
 
     let package_dir = tempfile::tempdir()?;

registry/src/main.rs

@@ -14,7 +14,10 @@ use actix_web::{
 };
 use fs_err::tokio as fs;
 use pesde::{
-    source::{pesde::PesdePackageSource, traits::PackageSource},
+    source::{
+        pesde::PesdePackageSource,
+        traits::{PackageSource, RefreshOptions},
+    },
     AuthConfig, Project,
 };
 use std::{env::current_dir, path::PathBuf};
@@ -106,7 +109,9 @@ async fn run() -> std::io::Result<()> {
     );
     let source = PesdePackageSource::new(benv!(required "INDEX_REPO_URL").try_into().unwrap());
     source
-        .refresh(&project)
+        .refresh(&RefreshOptions {
+            project: project.clone(),
+        })
         .await
         .expect("failed to refresh source");
     let config = source

src/cli/commands/add.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashSet, str::FromStr};
+use std::str::FromStr;
 
 use anyhow::Context;
 use clap::Args;
@@ -13,11 +13,11 @@ use pesde::{
         git::{specifier::GitDependencySpecifier, GitPackageSource},
         pesde::{specifier::PesdeDependencySpecifier, PesdePackageSource},
         specifiers::DependencySpecifiers,
-        traits::PackageSource,
+        traits::{PackageSource, RefreshOptions, ResolveOptions},
         workspace::WorkspacePackageSource,
         PackageSources,
     },
-    Project, DEFAULT_INDEX_NAME,
+    Project, RefreshedSources, DEFAULT_INDEX_NAME,
 };
 
 #[derive(Debug, Args)]
@@ -128,17 +128,27 @@ impl AddCommand {
                 ),
             ),
         };
-        source
+
-            .refresh(&project)
+        let refreshed_sources = RefreshedSources::new();
+
+        refreshed_sources
+            .refresh(
+                &source,
+                &RefreshOptions {
+                    project: project.clone(),
+                },
+            )
             .await
             .context("failed to refresh package source")?;
 
         let Some(version_id) = source
             .resolve(
                 &specifier,
-                &project,
+                &ResolveOptions {
-                manifest.target.kind(),
+                    project: project.clone(),
-                &mut HashSet::new(),
+                    target: manifest.target.kind(),
+                    refreshed_sources,
+                },
             )
             .await
             .context("failed to resolve package")?
@@ -167,7 +177,7 @@ impl AddCommand {
             "dependencies"
         };
 
-        let alias = self.alias.unwrap_or_else(|| match self.name.clone() {
+        let alias = self.alias.unwrap_or_else(|| match &self.name {
             AnyPackageIdentifier::PackageName(versioned) => versioned.0.as_str().1.to_string(),
             AnyPackageIdentifier::Url((url, _)) => url
                 .path
@@ -205,7 +215,8 @@ impl AddCommand {
             }
             #[cfg(feature = "wally-compat")]
             DependencySpecifiers::Wally(spec) => {
-                field["wally"] = toml_edit::value(spec.name.clone().to_string());
+                field["wally"] =
+                    toml_edit::value(spec.name.clone().to_string().trim_start_matches("wally#"));
                 field["version"] = toml_edit::value(format!("^{}", version_id.version()));
 
                 if let Some(index) = spec.index.filter(|i| i != DEFAULT_INDEX_NAME) {

src/cli/commands/auth/login.rs

@@ -6,13 +6,15 @@ use std::thread::spawn;
 use tokio::time::sleep;
 use url::Url;
 
+use crate::cli::auth::{get_token_login, set_token};
 use pesde::{
-    source::{pesde::PesdePackageSource, traits::PackageSource},
+    source::{
+        pesde::PesdePackageSource,
+        traits::{PackageSource, RefreshOptions},
+    },
     Project,
 };
 
-use crate::cli::auth::{get_token_login, set_token};
-
 #[derive(Debug, Args)]
 pub struct LoginCommand {
     /// The token to use for authentication, skipping login
@@ -57,7 +59,9 @@ impl LoginCommand {
 
         let source = PesdePackageSource::new(index_url.clone());
         source
-            .refresh(project)
+            .refresh(&RefreshOptions {
+                project: project.clone(),
+            })
             .await
             .context("failed to refresh index")?;
 

src/cli/commands/auth/mod.rs

@@ -1,4 +1,5 @@
 use crate::cli::config::read_config;
+use anyhow::Context;
 use clap::{Args, Subcommand};
 use pesde::{errors::ManifestReadError, Project, DEFAULT_INDEX_NAME};
 
@@ -56,10 +57,11 @@ impl AuthSubcommand {
             None => {
                 let index_name = self.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME);
 
-                match manifest.unwrap().indices.get(index_name) {
+                manifest
-                    Some(index) => index.clone(),
+                    .unwrap()
-                    None => anyhow::bail!("index {index_name} not found in manifest"),
+                    .indices
-                }
+                    .remove(index_name)
+                    .with_context(|| format!("index {index_name} not found in manifest"))?
             }
         };
 

src/cli/commands/execute.rs

@@ -15,20 +15,19 @@ use pesde::{
     names::PackageName,
     source::{
         pesde::{specifier::PesdeDependencySpecifier, PesdePackageSource},
-        traits::PackageSource,
+        traits::{DownloadOptions, PackageSource, RefreshOptions, ResolveOptions},
+        PackageSources,
     },
-    Project,
+    Project, RefreshedSources,
 };
 use semver::VersionReq;
 use std::{
-    collections::HashSet,
     env::current_dir,
     ffi::OsString,
     io::{Stderr, Write},
     process::Command,
     sync::Arc,
 };
-use tokio::sync::Mutex;
 
 #[derive(Debug, Args)]
 pub struct ExecuteCommand {
@@ -53,6 +52,8 @@ impl ExecuteCommand {
             .unwrap()
             .replace(multi_progress.clone());
 
+        let refreshed_sources = RefreshedSources::new();
+
         let (tempdir, bin_path) = reporters::run_with_reporter_and_writer(
             std::io::stderr(),
             |multi_progress, root_progress, reporter| async {
@@ -67,8 +68,13 @@ impl ExecuteCommand {
                 }
                 .context("no index specified")?;
                 let source = PesdePackageSource::new(index);
-                source
+                refreshed_sources
-                    .refresh(&project)
+                    .refresh(
+                        &PackageSources::Pesde(source.clone()),
+                        &RefreshOptions {
+                            project: project.clone(),
+                        },
+                    )
                     .await
                     .context("failed to refresh source")?;
 
@@ -82,7 +88,14 @@ impl ExecuteCommand {
                     };
 
                     if let Some(res) = source
-                        .resolve(&specifier, &project, TargetKind::Lune, &mut HashSet::new())
+                        .resolve(
+                            &specifier,
+                            &ResolveOptions {
+                                project: project.clone(),
+                                target: TargetKind::Lune,
+                                refreshed_sources: refreshed_sources.clone(),
+                            },
+                        )
                         .await
                         .context("failed to resolve package")?
                         .1
@@ -92,7 +105,14 @@ impl ExecuteCommand {
                     }
 
                     source
-                        .resolve(&specifier, &project, TargetKind::Luau, &mut HashSet::new())
+                        .resolve(
+                            &specifier,
+                            &ResolveOptions {
+                                project: project.clone(),
+                                target: TargetKind::Luau,
+                                refreshed_sources: refreshed_sources.clone(),
+                            },
+                        )
                         .await
                         .context("failed to resolve package")?
                         .1
@@ -120,7 +140,14 @@ impl ExecuteCommand {
                 );
 
                 let (fs, target) = source
-                    .download(&pkg_ref, &project, &reqwest, Arc::new(()))
+                    .download(
+                        &pkg_ref,
+                        &DownloadOptions {
+                            project: project.clone(),
+                            reqwest: reqwest.clone(),
+                            reporter: Arc::new(()),
+                        },
+                    )
                     .await
                     .context("failed to download package")?;
                 let bin_path = target.bin_path().context("package has no binary export")?;
@@ -129,10 +156,8 @@ impl ExecuteCommand {
                     .await
                     .context("failed to write package contents")?;
 
-                let mut refreshed_sources = HashSet::new();
-
                 let graph = project
-                    .dependency_graph(None, &mut refreshed_sources, true)
+                    .dependency_graph(None, refreshed_sources.clone(), true)
                     .await
                     .context("failed to build dependency graph")?;
 
@@ -152,7 +177,7 @@ impl ExecuteCommand {
                         &Arc::new(graph),
                         DownloadAndLinkOptions::<CliReporter<Stderr>, ()>::new(reqwest)
                             .reporter(reporter)
-                            .refreshed_sources(Mutex::new(refreshed_sources))
+                            .refreshed_sources(refreshed_sources)
                             .prod(true)
                             .write(true),
                     )

src/cli/commands/init.rs

@@ -11,12 +11,13 @@ use pesde::{
         git_index::GitBasedSource,
         pesde::{specifier::PesdeDependencySpecifier, PesdePackageSource},
         specifiers::DependencySpecifiers,
-        traits::PackageSource,
+        traits::{PackageSource, RefreshOptions, ResolveOptions},
+        PackageSources,
     },
-    Project, DEFAULT_INDEX_NAME, SCRIPTS_LINK_FOLDER,
+    Project, RefreshedSources, DEFAULT_INDEX_NAME, SCRIPTS_LINK_FOLDER,
 };
 use semver::VersionReq;
-use std::{collections::HashSet, fmt::Display, str::FromStr};
+use std::{fmt::Display, str::FromStr};
 
 #[derive(Debug, Args)]
 pub struct InitCommand {}
@@ -128,13 +129,21 @@ impl InitCommand {
         manifest["indices"].or_insert(toml_edit::Item::Table(toml_edit::Table::new()))
             [DEFAULT_INDEX_NAME] = toml_edit::value(source.repo_url().to_bstring().to_string());
 
+        let refreshed_sources = RefreshedSources::new();
+
         if target_env.is_roblox()
             || inquire::prompt_confirmation(
                 "would you like to setup default Roblox compatibility scripts?",
             )
             .unwrap()
         {
-            PackageSource::refresh(&source, &project)
+            refreshed_sources
+                .refresh(
+                    &PackageSources::Pesde(source.clone()),
+                    &RefreshOptions {
+                        project: project.clone(),
+                    },
+                )
                 .await
                 .context("failed to refresh package source")?;
             let config = source
@@ -193,9 +202,11 @@ impl InitCommand {
                             index: None,
                             target: None,
                         },
-                        &project,
+                        &ResolveOptions {
-                        TargetKind::Lune,
+                            project: project.clone(),
-                        &mut HashSet::new(),
+                            target: TargetKind::Lune,
+                            refreshed_sources,
+                        },
                     )
                     .await
                     .context("failed to resolve scripts package")?

src/cli/commands/outdated.rs

@@ -3,17 +3,14 @@ use anyhow::Context;
 use clap::Args;
 use futures::future::try_join_all;
 use pesde::{
-    refresh_sources,
     source::{
         refs::PackageRefs,
         specifiers::DependencySpecifiers,
-        traits::{PackageRef, PackageSource},
+        traits::{PackageRef, PackageSource, RefreshOptions, ResolveOptions},
     },
-    Project,
+    Project, RefreshedSources,
 };
 use semver::VersionReq;
-use std::{collections::HashSet, sync::Arc};
-use tokio::sync::Mutex;
 
 #[derive(Debug, Args)]
 pub struct OutdatedCommand {
@@ -40,19 +37,7 @@ impl OutdatedCommand {
             .context("failed to read manifest")?;
         let manifest_target_kind = manifest.target.kind();
 
-        let mut refreshed_sources = HashSet::new();
+        let refreshed_sources = RefreshedSources::new();
-
-        refresh_sources(
-            &project,
-            graph
-                .iter()
-                .flat_map(|(_, versions)| versions.iter())
-                .map(|(_, node)| node.node.pkg_ref.source()),
-            &mut refreshed_sources,
-        )
-        .await?;
-
-        let refreshed_sources = Arc::new(Mutex::new(refreshed_sources));
 
         if try_join_all(
             graph
@@ -74,14 +59,22 @@ impl OutdatedCommand {
                         }
 
                         let source = node.node.pkg_ref.source();
+                        refreshed_sources
+                            .refresh(
+                                &source,
+                                &RefreshOptions {
+                                    project: project.clone(),
+                                },
+                            )
+                            .await?;
 
                         if !self.strict {
-                            match specifier {
+                            match &mut specifier {
-                                DependencySpecifiers::Pesde(ref mut spec) => {
+                                DependencySpecifiers::Pesde(spec) => {
                                     spec.version = VersionReq::STAR;
                                 }
                                 #[cfg(feature = "wally-compat")]
-                                DependencySpecifiers::Wally(ref mut spec) => {
+                                DependencySpecifiers::Wally(spec) => {
                                     spec.version = VersionReq::STAR;
                                 }
                                 DependencySpecifiers::Git(_) => {}
@@ -92,9 +85,11 @@ impl OutdatedCommand {
                         let version_id = source
                             .resolve(
                                 &specifier,
-                                &project,
+                                &ResolveOptions {
-                                manifest_target_kind,
+                                    project: project.clone(),
-                                &mut *refreshed_sources.lock().await,
+                                    target: manifest_target_kind,
+                                    refreshed_sources: refreshed_sources.clone(),
+                                },
                             )
                             .await
                             .context("failed to resolve package versions")?

src/cli/commands/patch.rs

@@ -9,7 +9,7 @@ use pesde::{
     patches::setup_patches_repo,
     source::{
         refs::PackageRefs,
-        traits::{PackageRef, PackageSource},
+        traits::{DownloadOptions, PackageRef, PackageSource},
     },
     Project, MANIFEST_FILE_NAME,
 };
@@ -51,7 +51,14 @@ impl PatchCommand {
         fs::create_dir_all(&directory).await?;
 
         source
-            .download(&node.node.pkg_ref, &project, &reqwest, Arc::new(()))
+            .download(
+                &node.node.pkg_ref,
+                &DownloadOptions {
+                    project: project.clone(),
+                    reqwest,
+                    reporter: Arc::new(()),
+                },
+            )
             .await?
             .0
             .write_to(&directory, project.cas_dir(), false)

src/cli/commands/publish.rs

@@ -22,9 +22,16 @@ use pesde::{
     },
     Project, DEFAULT_INDEX_NAME, MANIFEST_FILE_NAME,
 };
+use pesde::{
+    source::{
+        traits::{RefreshOptions, ResolveOptions},
+        PackageSources,
+    },
+    RefreshedSources,
+};
 use reqwest::{header::AUTHORIZATION, StatusCode};
 use semver::VersionReq;
-use std::{collections::HashSet, path::PathBuf};
+use std::path::PathBuf;
 use tempfile::Builder;
 use tokio::io::{AsyncSeekExt, AsyncWriteExt};
 
@@ -365,6 +372,8 @@ info: otherwise, the file was deemed unnecessary, if you don't understand why, p
             }
         }
 
+        let refreshed_sources = RefreshedSources::new();
+
         for specifier in manifest
             .dependencies
             .values_mut()
@@ -406,7 +415,14 @@ info: otherwise, the file was deemed unnecessary, if you don't understand why, p
                 DependencySpecifiers::Git(_) => {}
                 DependencySpecifiers::Workspace(spec) => {
                     let pkg_ref = WorkspacePackageSource
-                        .resolve(spec, project, target_kind, &mut HashSet::new())
+                        .resolve(
+                            spec,
+                            &ResolveOptions {
+                                project: project.clone(),
+                                target: target_kind,
+                                refreshed_sources: refreshed_sources.clone(),
+                            },
+                        )
                         .await
                         .context("failed to resolve workspace package")?
                         .1
@@ -575,7 +591,13 @@ info: otherwise, the file was deemed unnecessary, if you don't understand why, p
             .get(&self.index)
             .context(format!("missing index {}", self.index))?;
         let source = PesdePackageSource::new(index_url.clone());
-        PackageSource::refresh(&source, project)
+        refreshed_sources
+            .refresh(
+                &PackageSources::Pesde(source.clone()),
+                &RefreshOptions {
+                    project: project.clone(),
+                },
+            )
             .await
             .context("failed to refresh source")?;
         let config = source

src/cli/commands/run.rs

@@ -9,8 +9,7 @@ use pesde::{
 };
 use relative_path::RelativePathBuf;
 use std::{
-    collections::HashSet, env::current_dir, ffi::OsString, io::Write, path::PathBuf,
+    collections::HashSet, env::current_dir, ffi::OsString, io::Write, path::Path, process::Command,
-    process::Command,
 };
 
 #[derive(Debug, Args)]
@@ -26,7 +25,7 @@ pub struct RunCommand {
 
 impl RunCommand {
     pub async fn run(self, project: Project) -> anyhow::Result<()> {
-        let run = |root: PathBuf, file_path: PathBuf| {
+        let run = |root: &Path, file_path: &Path| {
             let mut caller = tempfile::NamedTempFile::new().expect("failed to create tempfile");
             caller
                 .write_all(
@@ -55,8 +54,8 @@ impl RunCommand {
         let Some(package_or_script) = self.package_or_script else {
             if let Some(script_path) = project.deser_manifest().await?.target.bin_path() {
                 run(
-                    project.package_dir().to_owned(),
+                    project.package_dir(),
-                    script_path.to_path(project.package_dir()),
+                    &script_path.to_path(project.package_dir()),
                 );
                 return Ok(());
             }
@@ -99,7 +98,7 @@ impl RunCommand {
 
                 let path = bin_path.to_path(&container_folder);
 
-                run(path.clone(), path);
+                run(&path, &path);
                 return Ok(());
             }
         }
@@ -107,8 +106,8 @@ impl RunCommand {
         if let Ok(manifest) = project.deser_manifest().await {
             if let Some(script_path) = manifest.scripts.get(&package_or_script) {
                 run(
-                    project.package_dir().to_path_buf(),
+                    project.package_dir(),
-                    script_path.to_path(project.package_dir()),
+                    &script_path.to_path(project.package_dir()),
                 );
                 return Ok(());
             }
@@ -170,7 +169,7 @@ impl RunCommand {
             project.package_dir().to_path_buf()
         };
 
-        run(root, path);
+        run(&root, &path);
 
         Ok(())
     }

src/cli/commands/self_upgrade.rs

@@ -46,7 +46,7 @@ impl SelfUpgradeCommand {
             return Ok(());
         }
 
-        let path = get_or_download_version(&reqwest, &TagInfo::Complete(latest_version), true)
+        let path = get_or_download_version(&reqwest, TagInfo::Complete(latest_version), true)
             .await?
             .unwrap();
         update_bin_exe(&path).await?;

src/cli/install.rs

@@ -1,5 +1,5 @@
 use std::{
-    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
+    collections::{BTreeMap, BTreeSet, HashMap},
     num::NonZeroUsize,
     sync::Arc,
     time::Instant,
@@ -13,9 +13,9 @@ use pesde::{
     download_and_link::{filter_graph, DownloadAndLinkHooks, DownloadAndLinkOptions},
     lockfile::{DependencyGraph, DownloadedGraph, Lockfile},
     manifest::{target::TargetKind, DependencyType},
-    Project, MANIFEST_FILE_NAME,
+    Project, RefreshedSources, MANIFEST_FILE_NAME,
 };
-use tokio::{sync::Mutex, task::JoinSet};
+use tokio::task::JoinSet;
 
 use crate::cli::{
     bin_dir,
@@ -178,7 +178,7 @@ pub async fn install(
 ) -> anyhow::Result<()> {
     let start = Instant::now();
 
-    let mut refreshed_sources = HashSet::new();
+    let refreshed_sources = RefreshedSources::new();
 
     let manifest = project
         .deser_manifest()
@@ -276,7 +276,7 @@ pub async fn install(
             let graph = project
                 .dependency_graph(
                     old_graph.as_ref().filter(|_| options.use_lockfile),
-                    &mut refreshed_sources,
+                    refreshed_sources.clone(),
                     false,
                 )
                 .await
@@ -298,7 +298,7 @@ pub async fn install(
                     DownloadAndLinkOptions::<CliReporter, InstallHooks>::new(reqwest.clone())
                         .reporter(reporter.clone())
                         .hooks(hooks)
-                        .refreshed_sources(Mutex::new(refreshed_sources))
+                        .refreshed_sources(refreshed_sources)
                         .prod(options.prod)
                         .write(options.write)
                         .network_concurrency(options.network_concurrency),

src/cli/version.rs

@@ -251,7 +251,7 @@ pub enum TagInfo {
 #[instrument(skip(reqwest), level = "trace")]
 pub async fn get_or_download_version(
     reqwest: &reqwest::Client,
-    tag: &TagInfo,
+    tag: TagInfo,
     always_give_path: bool,
 ) -> anyhow::Result<Option<PathBuf>> {
     let path = home_dir()?.join("versions");
@@ -259,7 +259,7 @@ pub async fn get_or_download_version(
         .await
         .context("failed to create versions directory")?;
 
-    let version = match tag {
+    let version = match &tag {
         TagInfo::Complete(version) => version,
         // don't fetch the version since it could be cached
         TagInfo::Incomplete(version) => version,
@@ -290,9 +290,9 @@ pub async fn get_or_download_version(
             .context("failed to copy current executable to version directory")?;
     } else {
         let version = match tag {
-            TagInfo::Complete(version) => version.clone(),
+            TagInfo::Complete(version) => version,
             TagInfo::Incomplete(version) => {
-                get_remote_version(reqwest, VersionType::Specific(version.clone()))
+                get_remote_version(reqwest, VersionType::Specific(version))
                     .await
                     .context("failed to get remote version")?
             }

src/download.rs

@@ -2,19 +2,17 @@ use crate::{
     lockfile::{DependencyGraph, DownloadedDependencyGraphNode},
     manifest::DependencyType,
     names::PackageNames,
-    refresh_sources,
     reporters::{DownloadProgressReporter, DownloadsReporter},
     source::{
-        traits::{PackageRef, PackageSource},
+        traits::{DownloadOptions, PackageRef, PackageSource, RefreshOptions},
         version_id::VersionId,
-        PackageSources,
     },
-    Project, PACKAGES_CONTAINER_NAME,
+    Project, RefreshedSources, PACKAGES_CONTAINER_NAME,
 };
 use async_stream::try_stream;
 use fs_err::tokio as fs;
 use futures::Stream;
-use std::{collections::HashSet, num::NonZeroUsize, sync::Arc};
+use std::{num::NonZeroUsize, sync::Arc};
 use tokio::{sync::Semaphore, task::JoinSet};
 use tracing::{instrument, Instrument};
 
@@ -25,6 +23,8 @@ pub struct DownloadGraphOptions<Reporter> {
     pub reqwest: reqwest::Client,
     /// The downloads reporter.
     pub reporter: Option<Arc<Reporter>>,
+    /// The refreshed sources.
+    pub refreshed_sources: RefreshedSources,
     /// Whether to skip dev dependencies.
     pub prod: bool,
     /// Whether to write the downloaded packages to disk.
@@ -44,6 +44,7 @@ where
         Self {
             reqwest,
             reporter: None,
+            refreshed_sources: Default::default(),
             prod: false,
             write: false,
             wally: false,
@@ -57,6 +58,12 @@ where
         self
     }
 
+    /// Sets the refreshed sources.
+    pub fn refreshed_sources(mut self, refreshed_sources: RefreshedSources) -> Self {
+        self.refreshed_sources = refreshed_sources;
+        self
+    }
+
     /// Sets whether to skip dev dependencies.
     pub fn prod(mut self, prod: bool) -> Self {
         self.prod = prod;
@@ -87,6 +94,7 @@ impl<Reporter> Clone for DownloadGraphOptions<Reporter> {
         Self {
             reqwest: self.reqwest.clone(),
             reporter: self.reporter.clone(),
+            refreshed_sources: self.refreshed_sources.clone(),
             prod: self.prod,
             write: self.write,
             wally: self.wally,
@@ -101,7 +109,6 @@ impl Project {
     pub async fn download_graph<Reporter>(
         &self,
         graph: &DependencyGraph,
-        refreshed_sources: &mut HashSet<PackageSources>,
         options: DownloadGraphOptions<Reporter>,
     ) -> Result<
         impl Stream<
@@ -118,6 +125,7 @@ impl Project {
         let DownloadGraphOptions {
             reqwest,
             reporter,
+            refreshed_sources,
             prod,
             write,
             wally,
@@ -126,111 +134,111 @@ impl Project {
 
         let manifest = self.deser_manifest().await?;
         let manifest_target_kind = manifest.target.kind();
-        let project = Arc::new(self.clone());
 
-        refresh_sources(
-            self,
-            graph
-                .iter()
-                .flat_map(|(_, versions)| versions.iter())
-                .map(|(_, node)| node.pkg_ref.source()),
-            refreshed_sources,
-        )
-        .await?;
-
-        let mut tasks = JoinSet::<Result<_, errors::DownloadGraphError>>::new();
         let semaphore = Arc::new(Semaphore::new(network_concurrency.get()));
 
-        for (name, versions) in graph {
+        let mut tasks = graph
-            for (version_id, node) in versions {
+            .iter()
-                // we need to download pesde packages first, since scripts (for target finding for example) can depend on them
+            .flat_map(|(name, versions)| {
-                if node.pkg_ref.like_wally() != wally {
+                versions
-                    continue;
+                    .iter()
-                }
+                    .map(|(version_id, node)| (name.clone(), version_id.clone(), node.clone()))
-
+            })
-                let name = name.clone();
+            // we need to download pesde packages first, since scripts (for target finding for example) can depend on them
-                let version_id = version_id.clone();
+            .filter(|(_, _, node)| node.pkg_ref.like_wally() == wally)
-                let node = node.clone();
+            .map(|(name, version_id, node)| {
-
                 let span = tracing::info_span!(
                     "download",
                     name = name.to_string(),
                     version_id = version_id.to_string()
                 );
 
-                let project = project.clone();
+                let project = self.clone();
                 let reqwest = reqwest.clone();
                 let reporter = reporter.clone();
+                let refreshed_sources = refreshed_sources.clone();
                 let package_dir = project.package_dir().to_path_buf();
                 let semaphore = semaphore.clone();
 
-                tasks.spawn(
+                async move {
-                    async move {
+                    let display_name = format!("{name}@{version_id}");
-                        let display_name = format!("{name}@{version_id}");
+                    let progress_reporter = reporter
-                        let progress_reporter = reporter
+                        .as_deref()
-                            .as_deref()
+                        .map(|reporter| reporter.report_download(&display_name));
-                            .map(|reporter| reporter.report_download(&display_name));
 
-                        let _permit = semaphore.acquire().await;
+                    let _permit = semaphore.acquire().await;
 
-                        if let Some(ref progress_reporter) = progress_reporter {
+                    if let Some(ref progress_reporter) = progress_reporter {
-                            progress_reporter.report_start();
+                        progress_reporter.report_start();
-                        }
-
-                        let source = node.pkg_ref.source();
-                        let container_folder = node.container_folder(
-                            &package_dir
-                                .join(manifest_target_kind.packages_folder(version_id.target()))
-                                .join(PACKAGES_CONTAINER_NAME),
-                            &name,
-                            version_id.version(),
-                        );
-
-                        fs::create_dir_all(&container_folder).await?;
-
-                        let project = project.clone();
-
-                        tracing::debug!("downloading");
-
-                        let (fs, target) = match progress_reporter {
-                            Some(progress_reporter) => {
-                                source
-                                    .download(
-                                        &node.pkg_ref,
-                                        &project,
-                                        &reqwest,
-                                        Arc::new(progress_reporter),
-                                    )
-                                    .await
-                            }
-                            None => {
-                                source
-                                    .download(&node.pkg_ref, &project, &reqwest, Arc::new(()))
-                                    .await
-                            }
-                        }
-                        .map_err(Box::new)?;
-
-                        tracing::debug!("downloaded");
-
-                        if write {
-                            if !prod || node.resolved_ty != DependencyType::Dev {
-                                fs.write_to(container_folder, project.cas_dir(), true)
-                                    .await?;
-                            } else {
-                                tracing::debug!(
-                                    "skipping write to disk, dev dependency in prod mode"
-                                );
-                            }
-                        }
-
-                        let downloaded_node = DownloadedDependencyGraphNode { node, target };
-                        Ok((downloaded_node, name, version_id))
                     }
-                    .instrument(span),
+
-                );
+                    let source = node.pkg_ref.source();
-            }
+                    refreshed_sources
-        }
+                        .refresh(
+                            &source,
+                            &RefreshOptions {
+                                project: project.clone(),
+                            },
+                        )
+                        .await?;
+
+                    let container_folder = node.container_folder(
+                        &package_dir
+                            .join(manifest_target_kind.packages_folder(version_id.target()))
+                            .join(PACKAGES_CONTAINER_NAME),
+                        &name,
+                        version_id.version(),
+                    );
+
+                    fs::create_dir_all(&container_folder).await?;
+
+                    tracing::debug!("downloading");
+
+                    let (fs, target) = match progress_reporter {
+                        Some(progress_reporter) => {
+                            source
+                                .download(
+                                    &node.pkg_ref,
+                                    &DownloadOptions {
+                                        project: project.clone(),
+                                        reqwest,
+                                        reporter: Arc::new(progress_reporter),
+                                    },
+                                )
+                                .await
+                        }
+                        None => {
+                            source
+                                .download(
+                                    &node.pkg_ref,
+                                    &DownloadOptions {
+                                        project: project.clone(),
+                                        reqwest,
+                                        reporter: Arc::new(()),
+                                    },
+                                )
+                                .await
+                        }
+                    }
+                    .map_err(Box::new)?;
+
+                    tracing::debug!("downloaded");
+
+                    if write {
+                        if !prod || node.resolved_ty != DependencyType::Dev {
+                            fs.write_to(container_folder, project.cas_dir(), true)
+                                .await?;
+                        } else {
+                            tracing::debug!("skipping write to disk, dev dependency in prod mode");
+                        }
+                    }
+
+                    let downloaded_node = DownloadedDependencyGraphNode { node, target };
+                    Ok((downloaded_node, name, version_id))
+                }
+                .instrument(span)
+            })
+            .collect::<JoinSet<Result<_, errors::DownloadGraphError>>>();
 
         let stream = try_stream! {
             while let Some(res) = tasks.join_next().await {
@@ -256,7 +264,7 @@ pub mod errors {
 
         /// An error occurred refreshing a package source
         #[error("failed to refresh package source")]
-        RefreshFailed(#[from] Box<crate::source::errors::RefreshError>),
+        RefreshFailed(#[from] crate::source::errors::RefreshError),
 
         /// Error interacting with the filesystem
         #[error("error interacting with the filesystem")]

src/download_and_link.rs

@@ -3,18 +3,15 @@ use crate::{
     lockfile::{DependencyGraph, DownloadedGraph},
     manifest::DependencyType,
     reporters::DownloadsReporter,
-    source::PackageSources,
+    Project, RefreshedSources,
-    Project,
 };
 use futures::TryStreamExt;
 use std::{
-    collections::HashSet,
     convert::Infallible,
     future::{self, Future},
     num::NonZeroUsize,
     sync::Arc,
 };
-use tokio::sync::Mutex;
 use tracing::{instrument, Instrument};
 
 /// Filters a graph to only include production dependencies, if `prod` is `true`
@@ -90,7 +87,7 @@ pub struct DownloadAndLinkOptions<Reporter = (), Hooks = ()> {
     /// The download and link hooks.
     pub hooks: Option<Arc<Hooks>>,
     /// The refreshed sources.
-    pub refreshed_sources: Arc<Mutex<HashSet<PackageSources>>>,
+    pub refreshed_sources: RefreshedSources,
     /// Whether to skip dev dependencies.
     pub prod: bool,
     /// Whether to write the downloaded packages to disk.
@@ -130,11 +127,8 @@ where
     }
 
     /// Sets the refreshed sources.
-    pub fn refreshed_sources(
+    pub fn refreshed_sources(mut self, refreshed_sources: RefreshedSources) -> Self {
-        mut self,
+        self.refreshed_sources = refreshed_sources;
-        refreshed_sources: impl Into<Arc<Mutex<HashSet<PackageSources>>>>,
-    ) -> Self {
-        self.refreshed_sources = refreshed_sources.into();
         self
     }
 
@@ -196,10 +190,10 @@ impl Project {
         let graph = graph.clone();
         let reqwest = reqwest.clone();
 
-        let mut refreshed_sources = refreshed_sources.lock().await;
         let mut downloaded_graph = DownloadedGraph::new();
 
         let mut download_graph_options = DownloadGraphOptions::<Reporter>::new(reqwest.clone())
+            .refreshed_sources(refreshed_sources.clone())
             .prod(prod)
             .write(write)
             .network_concurrency(network_concurrency);
@@ -209,22 +203,18 @@ impl Project {
         }
 
         // step 1. download pesde dependencies
-        self.download_graph(
+        self.download_graph(&graph, download_graph_options.clone())
-            &graph,
+            .instrument(tracing::debug_span!("download (pesde)"))
-            &mut refreshed_sources,
+            .await?
-            download_graph_options.clone(),
+            .try_for_each(|(downloaded_node, name, version_id)| {
-        )
+                downloaded_graph
-        .instrument(tracing::debug_span!("download (pesde)"))
+                    .entry(name)
-        .await?
+                    .or_default()
-        .try_for_each(|(downloaded_node, name, version_id)| {
+                    .insert(version_id, downloaded_node);
-            downloaded_graph
-                .entry(name)
-                .or_default()
-                .insert(version_id, downloaded_node);
 
-            future::ready(Ok(()))
+                future::ready(Ok(()))
-        })
+            })
-        .await?;
+            .await?;
 
         // step 2. link pesde dependencies. do so without types
         if write {
@@ -246,22 +236,18 @@ impl Project {
         }
 
         // step 3. download wally dependencies
-        self.download_graph(
+        self.download_graph(&graph, download_graph_options.clone().wally(true))
-            &graph,
+            .instrument(tracing::debug_span!("download (wally)"))
-            &mut refreshed_sources,
+            .await?
-            download_graph_options.clone().wally(true),
+            .try_for_each(|(downloaded_node, name, version_id)| {
-        )
+                downloaded_graph
-        .instrument(tracing::debug_span!("download (wally)"))
+                    .entry(name)
-        .await?
+                    .or_default()
-        .try_for_each(|(downloaded_node, name, version_id)| {
+                    .insert(version_id, downloaded_node);
-            downloaded_graph
-                .entry(name)
-                .or_default()
-                .insert(version_id, downloaded_node);
 
-            future::ready(Ok(()))
+                future::ready(Ok(()))
-        })
+            })
-        .await?;
+            .await?;
 
         // step 4. link ALL dependencies. do so with types
         if write {

src/lib.rs

@@ -6,16 +6,21 @@
 use crate::{
     lockfile::Lockfile,
     manifest::Manifest,
-    source::{traits::PackageSource, PackageSources},
+    source::{
+        traits::{PackageSource, RefreshOptions},
+        PackageSources,
+    },
 };
 use async_stream::stream;
 use fs_err::tokio as fs;
-use futures::{future::try_join_all, Stream};
+use futures::Stream;
 use gix::sec::identity::Account;
 use std::{
     collections::{HashMap, HashSet},
     fmt::Debug,
+    hash::{Hash, Hasher},
     path::{Path, PathBuf},
+    sync::Arc,
 };
 use tracing::instrument;
 use wax::Pattern;
@@ -56,13 +61,18 @@ pub(crate) const LINK_LIB_NO_FILE_FOUND: &str = "____pesde_no_export_file_found"
 /// The folder in which scripts are linked
 pub const SCRIPTS_LINK_FOLDER: &str = ".pesde";
 
-/// Struct containing the authentication configuration
+#[derive(Debug, Default)]
-#[derive(Debug, Default, Clone)]
+struct AuthConfigShared {
-pub struct AuthConfig {
     tokens: HashMap<gix::Url, String>,
     git_credentials: Option<Account>,
 }
 
+/// Struct containing the authentication configuration
+#[derive(Debug, Clone, Default)]
+pub struct AuthConfig {
+    shared: Arc<AuthConfigShared>,
+}
+
 impl AuthConfig {
     /// Create a new `AuthConfig`
     pub fn new() -> Self {
@@ -70,11 +80,12 @@ impl AuthConfig {
     }
 
     /// Set the tokens
+    /// Panics if the `AuthConfig` is shared
     pub fn with_tokens<I: IntoIterator<Item = (gix::Url, S)>, S: AsRef<str>>(
         mut self,
         tokens: I,
     ) -> Self {
-        self.tokens = tokens
+        Arc::get_mut(&mut self.shared).unwrap().tokens = tokens
             .into_iter()
             .map(|(url, s)| (url, s.as_ref().to_string()))
             .collect();
@@ -82,25 +93,25 @@ impl AuthConfig {
     }
 
     /// Set the git credentials
+    /// Panics if the `AuthConfig` is shared
     pub fn with_git_credentials(mut self, git_credentials: Option<Account>) -> Self {
-        self.git_credentials = git_credentials;
+        Arc::get_mut(&mut self.shared).unwrap().git_credentials = git_credentials;
         self
     }
 
     /// Get the tokens
     pub fn tokens(&self) -> &HashMap<gix::Url, String> {
-        &self.tokens
+        &self.shared.tokens
     }
 
     /// Get the git credentials
     pub fn git_credentials(&self) -> Option<&Account> {
-        self.git_credentials.as_ref()
+        self.shared.git_credentials.as_ref()
     }
 }
 
-/// The main struct of the pesde library, representing a project
+#[derive(Debug)]
-#[derive(Debug, Clone)]
+struct ProjectShared {
-pub struct Project {
     package_dir: PathBuf,
     workspace_dir: Option<PathBuf>,
     data_dir: PathBuf,
@@ -108,6 +119,13 @@ pub struct Project {
     cas_dir: PathBuf,
 }
 
+/// The main struct of the pesde library, representing a project
+/// Unlike `ProjectShared`, this struct is `Send` and `Sync` and is cheap to clone because it is `Arc`-backed
+#[derive(Debug, Clone)]
+pub struct Project {
+    shared: Arc<ProjectShared>,
+}
+
 impl Project {
     /// Create a new `Project`
     pub fn new<P: AsRef<Path>, Q: AsRef<Path>, R: AsRef<Path>, S: AsRef<Path>>(
@@ -118,43 +136,45 @@ impl Project {
         auth_config: AuthConfig,
     ) -> Self {
         Project {
-            package_dir: package_dir.as_ref().to_path_buf(),
+            shared: Arc::new(ProjectShared {
-            workspace_dir: workspace_dir.map(|d| d.as_ref().to_path_buf()),
+                package_dir: package_dir.as_ref().to_path_buf(),
-            data_dir: data_dir.as_ref().to_path_buf(),
+                workspace_dir: workspace_dir.map(|d| d.as_ref().to_path_buf()),
-            auth_config,
+                data_dir: data_dir.as_ref().to_path_buf(),
-            cas_dir: cas_dir.as_ref().to_path_buf(),
+                auth_config,
+                cas_dir: cas_dir.as_ref().to_path_buf(),
+            }),
         }
     }
 
     /// The directory of the package
     pub fn package_dir(&self) -> &Path {
-        &self.package_dir
+        &self.shared.package_dir
     }
 
     /// The directory of the workspace this package belongs to, if any
     pub fn workspace_dir(&self) -> Option<&Path> {
-        self.workspace_dir.as_deref()
+        self.shared.workspace_dir.as_deref()
     }
 
     /// The directory to store general-purpose data
     pub fn data_dir(&self) -> &Path {
-        &self.data_dir
+        &self.shared.data_dir
     }
 
     /// The authentication configuration
     pub fn auth_config(&self) -> &AuthConfig {
-        &self.auth_config
+        &self.shared.auth_config
     }
 
     /// The CAS (content-addressable storage) directory
     pub fn cas_dir(&self) -> &Path {
-        &self.cas_dir
+        &self.shared.cas_dir
     }
 
     /// Read the manifest file
     #[instrument(skip(self), ret(level = "trace"), level = "debug")]
     pub async fn read_manifest(&self) -> Result<String, errors::ManifestReadError> {
-        let string = fs::read_to_string(self.package_dir.join(MANIFEST_FILE_NAME)).await?;
+        let string = fs::read_to_string(self.package_dir().join(MANIFEST_FILE_NAME)).await?;
         Ok(string)
     }
 
@@ -162,20 +182,24 @@ impl Project {
     /// Deserialize the manifest file
     #[instrument(skip(self), ret(level = "trace"), level = "debug")]
     pub async fn deser_manifest(&self) -> Result<Manifest, errors::ManifestReadError> {
-        let string = fs::read_to_string(self.package_dir.join(MANIFEST_FILE_NAME)).await?;
+        let string = fs::read_to_string(self.package_dir().join(MANIFEST_FILE_NAME)).await?;
         Ok(toml::from_str(&string)?)
     }
 
     /// Write the manifest file
     #[instrument(skip(self, manifest), level = "debug")]
     pub async fn write_manifest<S: AsRef<[u8]>>(&self, manifest: S) -> Result<(), std::io::Error> {
-        fs::write(self.package_dir.join(MANIFEST_FILE_NAME), manifest.as_ref()).await
+        fs::write(
+            self.package_dir().join(MANIFEST_FILE_NAME),
+            manifest.as_ref(),
+        )
+        .await
     }
 
     /// Deserialize the lockfile
     #[instrument(skip(self), ret(level = "trace"), level = "debug")]
     pub async fn deser_lockfile(&self) -> Result<Lockfile, errors::LockfileReadError> {
-        let string = fs::read_to_string(self.package_dir.join(LOCKFILE_FILE_NAME)).await?;
+        let string = fs::read_to_string(self.package_dir().join(LOCKFILE_FILE_NAME)).await?;
         Ok(toml::from_str(&string)?)
     }
 
@@ -186,7 +210,7 @@ impl Project {
         lockfile: &Lockfile,
     ) -> Result<(), errors::LockfileWriteError> {
         let string = toml::to_string(lockfile)?;
-        fs::write(self.package_dir.join(LOCKFILE_FILE_NAME), string).await?;
+        fs::write(self.package_dir().join(LOCKFILE_FILE_NAME), string).await?;
         Ok(())
     }
 
@@ -370,24 +394,35 @@ pub async fn matching_globs<'a, P: AsRef<Path> + Debug, I: IntoIterator<Item = &
     Ok(paths)
 }
 
-/// Refreshes the sources asynchronously
+/// A struct containing sources already having been refreshed
-pub async fn refresh_sources<I: Iterator<Item = PackageSources>>(
+#[derive(Debug, Clone, Default)]
-    project: &Project,
+pub struct RefreshedSources(Arc<tokio::sync::Mutex<HashSet<u64>>>);
-    sources: I,
+
-    refreshed_sources: &mut HashSet<PackageSources>,
+impl RefreshedSources {
-) -> Result<(), Box<source::errors::RefreshError>> {
+    /// Create a new empty `RefreshedSources`
-    try_join_all(sources.map(|source| {
+    pub fn new() -> Self {
-        let needs_refresh = refreshed_sources.insert(source.clone());
+        RefreshedSources::default()
-        async move {
+    }
-            if needs_refresh {
+
-                source.refresh(project).await.map_err(Box::new)
+    /// Refreshes the source asynchronously if it has not already been refreshed.
-            } else {
+    /// Will prevent more refreshes of the same source.
-                Ok(())
+    pub async fn refresh(
-            }
+        &self,
+        source: &PackageSources,
+        options: &RefreshOptions,
+    ) -> Result<(), source::errors::RefreshError> {
+        let mut hasher = std::hash::DefaultHasher::new();
+        source.hash(&mut hasher);
+        let hash = hasher.finish();
+
+        let mut refreshed_sources = self.0.lock().await;
+
+        if refreshed_sources.insert(hash) {
+            source.refresh(options).await
+        } else {
+            Ok(())
         }
-    }))
+    }
-    .await
-    .map(|_| ())
 }
 
 /// Errors that can occur when using the pesde library

src/main.rs

@@ -332,7 +332,7 @@ async fn run() -> anyhow::Result<()> {
             .and_then(|manifest| manifest.pesde_version);
 
         let exe_path = if let Some(version) = target_version {
-            get_or_download_version(&reqwest, &TagInfo::Incomplete(version), false).await?
+            get_or_download_version(&reqwest, TagInfo::Incomplete(version), false).await?
         } else {
             None
         };

src/patches.rs

@@ -9,7 +9,7 @@ use git2::{ApplyLocation, Diff, DiffFormat, DiffLineType, Repository, Signature}
 use relative_path::RelativePathBuf;
 use std::{path::Path, sync::Arc};
 use tokio::task::JoinSet;
-use tracing::instrument;
+use tracing::{instrument, Instrument};
 
 /// Set up a git repository for patches
 pub fn setup_patches_repo<P: AsRef<Path>>(dir: P) -> Result<Repository, git2::Error> {
@@ -91,7 +91,6 @@ impl Project {
 
         for (name, versions) in manifest.patches {
             for (version_id, patch_path) in versions {
-                let name = name.clone();
                 let patch_path = patch_path.to_path(self.package_dir());
 
                 let Some(node) = graph
@@ -114,62 +113,68 @@ impl Project {
                 );
 
                 let reporter = reporter.clone();
+                let span = tracing::info_span!(
+                    "apply patch",
+                    name = name.to_string(),
+                    version_id = version_id.to_string()
+                );
+                let display_name = format!("{name}@{version_id}");
 
-                tasks.spawn(async move {
+                tasks.spawn(
-                    tracing::debug!("applying patch to {name}@{version_id}");
+                    async move {
+                        tracing::debug!("applying patch");
 
-                    let display_name = format!("{name}@{version_id}");
+                        let progress_reporter = reporter.report_patch(&display_name);
-                    let progress_reporter = reporter.report_patch(&display_name);
 
-                    let patch = fs::read(&patch_path)
+                        let patch = fs::read(&patch_path)
-                        .await
+                            .await
-                        .map_err(errors::ApplyPatchesError::PatchRead)?;
+                            .map_err(errors::ApplyPatchesError::PatchRead)?;
-                    let patch = Diff::from_buffer(&patch)?;
+                        let patch = Diff::from_buffer(&patch)?;
 
-                    {
+                        {
-                        let repo = setup_patches_repo(&container_folder)?;
+                            let repo = setup_patches_repo(&container_folder)?;
 
-                        let mut apply_delta_tasks = patch
+                            let mut apply_delta_tasks = patch
-                            .deltas()
+                                .deltas()
-                            .filter(|delta| matches!(delta.status(), git2::Delta::Modified))
+                                .filter(|delta| matches!(delta.status(), git2::Delta::Modified))
-                            .filter_map(|delta| delta.new_file().path())
+                                .filter_map(|delta| delta.new_file().path())
-                            .map(|path| {
+                                .map(|path| {
-                                RelativePathBuf::from_path(path)
+                                    RelativePathBuf::from_path(path)
-                                    .unwrap()
+                                        .unwrap()
-                                    .to_path(&container_folder)
+                                        .to_path(&container_folder)
-                            })
+                                })
-                            .filter(|path| path.is_file())
+                                .filter(|path| path.is_file())
-                            .map(|path| {
+                                .map(|path| {
-                                async {
+                                    async {
-                                    // so, we always unlink it
+                                        // so, we always unlink it
-                                    let content = fs::read(&path).await?;
+                                        let content = fs::read(&path).await?;
-                                    fs::remove_file(&path).await?;
+                                        fs::remove_file(&path).await?;
-                                    fs::write(path, content).await?;
+                                        fs::write(path, content).await?;
-                                    Ok(())
+                                        Ok(())
-                                }
+                                    }
-                                .map_err(errors::ApplyPatchesError::File)
+                                    .map_err(errors::ApplyPatchesError::File)
-                            })
+                                })
-                            .collect::<JoinSet<_>>();
+                                .collect::<JoinSet<_>>();
 
-                        while let Some(res) = apply_delta_tasks.join_next().await {
+                            while let Some(res) = apply_delta_tasks.join_next().await {
-                            res.unwrap()?;
+                                res.unwrap()?;
+                            }
+
+                            repo.apply(&patch, ApplyLocation::Both, None)?;
                         }
 
-                        repo.apply(&patch, ApplyLocation::Both, None)?;
+                        tracing::debug!("patch applied");
+
+                        fs::remove_dir_all(container_folder.join(".git"))
+                            .await
+                            .map_err(errors::ApplyPatchesError::DotGitRemove)?;
+
+                        progress_reporter.report_done();
+
+                        Ok(())
                     }
-
+                    .instrument(span),
-                    tracing::debug!(
+                );
-                        "patch applied to {name}@{version_id}, removing .git directory"
-                    );
-
-                    fs::remove_dir_all(container_folder.join(".git"))
-                        .await
-                        .map_err(errors::ApplyPatchesError::DotGitRemove)?;
-
-                    progress_reporter.report_done();
-
-                    Ok(())
-                });
             }
         }
 

src/resolver.rs

@@ -5,19 +5,19 @@ use crate::{
     source::{
         pesde::PesdePackageSource,
         specifiers::DependencySpecifiers,
-        traits::{PackageRef, PackageSource},
+        traits::{PackageRef, PackageSource, RefreshOptions, ResolveOptions},
         version_id::VersionId,
         PackageSources,
     },
-    Project, DEFAULT_INDEX_NAME,
+    Project, RefreshedSources, DEFAULT_INDEX_NAME,
 };
-use std::collections::{btree_map::Entry, HashMap, HashSet, VecDeque};
+use std::collections::{btree_map::Entry, HashMap, VecDeque};
 use tracing::{instrument, Instrument};
 
 fn insert_node(
     graph: &mut DependencyGraph,
-    name: PackageNames,
+    name: &PackageNames,
-    version: VersionId,
+    version: &VersionId,
     mut node: DependencyGraphNode,
     is_top_level: bool,
 ) {
@@ -63,7 +63,7 @@ impl Project {
     pub async fn dependency_graph(
         &self,
         previous_graph: Option<&DependencyGraph>,
-        refreshed_sources: &mut HashSet<PackageSources>,
+        refreshed_sources: RefreshedSources,
         // used by `x` command - if true, specifier indices are expected to be URLs. will not do peer dependency checks
         is_published_package: bool,
     ) -> Result<DependencyGraph, Box<errors::DependencyGraphError>> {
@@ -108,8 +108,8 @@ impl Project {
                     tracing::debug!("resolved {}@{} from old dependency graph", name, version);
                     insert_node(
                         &mut graph,
-                        name.clone(),
+                        name,
-                        version.clone(),
+                        version,
                         DependencyGraphNode {
                             direct: Some((alias.clone(), specifier.clone(), *source_ty)),
                             ..node.clone()
@@ -138,13 +138,7 @@ impl Project {
                             .and_then(|v| v.get(dep_version))
                         {
                             tracing::debug!("resolved sub-dependency {dep_name}@{dep_version}");
-                            insert_node(
+                            insert_node(&mut graph, dep_name, dep_version, dep_node.clone(), false);
-                                &mut graph,
-                                dep_name.clone(),
-                                dep_version.clone(),
-                                dep_node.clone(),
-                                false,
-                            );
 
                             dep_node
                                 .dependencies
@@ -184,9 +178,13 @@ impl Project {
             })
             .collect::<VecDeque<_>>();
 
+        let refresh_options = RefreshOptions {
+            project: self.clone(),
+        };
+
         while let Some((specifier, ty, dependant, path, overridden, target)) = queue.pop_front() {
             async {
-                let alias = path.last().unwrap().clone();
+                let alias = path.last().unwrap();
                 let depth = path.len() - 1;
 
                 tracing::debug!("resolving {specifier} ({ty:?})");
@@ -203,10 +201,7 @@ impl Project {
                                 ))?
                                 .clone()
                         } else {
-                            let index_url = specifier.index.clone().unwrap();
+                            specifier.index.as_deref().unwrap()
-
-                            index_url
-                                .clone()
                                 .try_into()
                                 // specifiers in indices store the index url in this field
                                 .unwrap()
@@ -227,10 +222,7 @@ impl Project {
                                 ))?
                                 .clone()
                         } else {
-                            let index_url = specifier.index.clone().unwrap();
+                            specifier.index.as_deref().unwrap()
-
-                            index_url
-                                .clone()
                                 .try_into()
                                 // specifiers in indices store the index url in this field
                                 .unwrap()
@@ -246,12 +238,19 @@ impl Project {
                     }
                 };
 
-                if refreshed_sources.insert(source.clone()) {
+                refreshed_sources.refresh(
-                    source.refresh(self).await.map_err(|e| Box::new(e.into()))?;
+                    &source,
-                }
+                    &refresh_options,
+                )
+                .await
+                .map_err(|e| Box::new(e.into()))?;
 
                 let (name, resolved) = source
-                    .resolve(&specifier, self, target, refreshed_sources)
+                    .resolve(&specifier, &ResolveOptions {
+                        project: self.clone(),
+                        target,
+                        refreshed_sources: refreshed_sources.clone(),
+                    })
                     .await
                     .map_err(|e| Box::new(e.into()))?;
 
@@ -341,9 +340,9 @@ impl Project {
                 };
                 insert_node(
                     &mut graph,
-                    name.clone(),
+                    &name,
-                    target_version_id.clone(),
+                    &target_version_id,
-                    node.clone(),
+                    node,
                     depth == 0,
                 );
 

src/source/git/mod.rs

@@ -10,8 +10,8 @@ use crate::{
         git::{pkg_ref::GitPackageRef, specifier::GitDependencySpecifier},
         git_index::{read_file, GitBasedSource},
         specifiers::DependencySpecifiers,
-        traits::PackageRef,
+        traits::{DownloadOptions, PackageRef, RefreshOptions, ResolveOptions},
-        PackageSource, PackageSources, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES,
+        PackageSource, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES,
     },
     util::hash,
     Project, DEFAULT_INDEX_NAME, LOCKFILE_FILE_NAME, MANIFEST_FILE_NAME,
@@ -20,13 +20,7 @@ use fs_err::tokio as fs;
 use futures::future::try_join_all;
 use gix::{bstr::BStr, traverse::tree::Recorder, ObjectId, Url};
 use relative_path::RelativePathBuf;
-use std::{
+use std::{collections::BTreeMap, fmt::Debug, hash::Hash, path::PathBuf, sync::Arc};
-    collections::{BTreeMap, HashSet},
-    fmt::Debug,
-    hash::Hash,
-    path::PathBuf,
-    sync::Arc,
-};
 use tokio::{sync::Mutex, task::spawn_blocking};
 use tracing::instrument;
 
@@ -44,7 +38,7 @@ pub struct GitPackageSource {
 impl GitBasedSource for GitPackageSource {
     fn path(&self, project: &Project) -> PathBuf {
         project
-            .data_dir
+            .data_dir()
             .join("git_repos")
             .join(hash(self.as_bytes()))
     }
@@ -73,18 +67,18 @@ impl PackageSource for GitPackageSource {
     type DownloadError = errors::DownloadError;
 
     #[instrument(skip_all, level = "debug")]
-    async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> {
+    async fn refresh(&self, options: &RefreshOptions) -> Result<(), Self::RefreshError> {
-        GitBasedSource::refresh(self, project).await
+        GitBasedSource::refresh(self, options).await
     }
 
     #[instrument(skip_all, level = "debug")]
     async fn resolve(
         &self,
         specifier: &Self::Specifier,
-        project: &Project,
+        options: &ResolveOptions,
-        _project_target: TargetKind,
-        _refreshed_sources: &mut HashSet<PackageSources>,
     ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
+        let ResolveOptions { project, .. } = options;
+
         let repo = gix::open(self.path(project))
             .map_err(|e| errors::ResolveError::OpenRepo(Box::new(self.repo_url.clone()), e))?;
         let rev = repo
@@ -334,15 +328,15 @@ impl PackageSource for GitPackageSource {
     }
 
     #[instrument(skip_all, level = "debug")]
-    async fn download(
+    async fn download<R: DownloadProgressReporter>(
         &self,
         pkg_ref: &Self::Ref,
-        project: &Project,
+        options: &DownloadOptions<R>,
-        _reqwest: &reqwest::Client,
-        _reporter: Arc<impl DownloadProgressReporter>,
     ) -> Result<(PackageFS, Target), Self::DownloadError> {
+        let DownloadOptions { project, .. } = options;
+
         let index_file = project
-            .cas_dir
+            .cas_dir()
             .join("git_index")
             .join(hash(self.as_bytes()))
             .join(&pkg_ref.tree_id);

src/source/git_index.rs

@@ -1,6 +1,6 @@
 #![allow(async_fn_in_trait)]
 
-use crate::{util::authenticate_conn, Project};
+use crate::{source::traits::RefreshOptions, util::authenticate_conn, Project};
 use fs_err::tokio as fs;
 use gix::remote::Direction;
 use std::fmt::Debug;
@@ -16,10 +16,13 @@ pub trait GitBasedSource {
     fn repo_url(&self) -> &gix::Url;
 
     /// Refreshes the repository
-    async fn refresh(&self, project: &Project) -> Result<(), errors::RefreshError> {
+    async fn refresh(&self, options: &RefreshOptions) -> Result<(), errors::RefreshError> {
+        let RefreshOptions { project } = options;
+
         let path = self.path(project);
         let repo_url = self.repo_url().clone();
-        let auth_config = project.auth_config.clone();
+
+        let project = project.clone();
 
         if path.exists() {
             spawn_blocking(move || {
@@ -47,7 +50,7 @@ pub trait GitBasedSource {
                     }
                 };
 
-                authenticate_conn(&mut connection, &auth_config);
+                authenticate_conn(&mut connection, project.auth_config());
 
                 let fetch =
                     match connection.prepare_fetch(gix::progress::Discard, Default::default()) {
@@ -80,7 +83,7 @@ pub trait GitBasedSource {
             gix::prepare_clone_bare(repo_url.clone(), &path)
                 .map_err(|e| errors::RefreshError::Clone(repo_url.to_string(), Box::new(e)))?
                 .configure_connection(move |c| {
-                    authenticate_conn(c, &auth_config);
+                    authenticate_conn(c, project.auth_config());
                     Ok(())
                 })
                 .fetch_only(gix::progress::Discard, &false.into())
@@ -94,21 +97,16 @@ pub trait GitBasedSource {
 
 /// Reads a file from a tree
 #[instrument(skip(tree), ret, level = "trace")]
-pub fn read_file<
+pub fn read_file<I: IntoIterator<Item = P> + Debug, P: ToString + PartialEq<gix::bstr::BStr>>(
-    I: IntoIterator<Item = P> + Clone + Debug,
-    P: ToString + PartialEq<gix::bstr::BStr>,
->(
     tree: &gix::Tree,
     file_path: I,
 ) -> Result<Option<String>, errors::ReadFile> {
-    let file_path_str = file_path
+    let mut file_path_str = String::new();
-        .clone()
-        .into_iter()
-        .map(|s| s.to_string())
-        .collect::<Vec<_>>()
-        .join(std::path::MAIN_SEPARATOR_STR);
 
-    let entry = match tree.lookup_entry(file_path) {
+    let entry = match tree.lookup_entry(file_path.into_iter().inspect(|path| {
+        file_path_str.push_str(path.to_string().as_str());
+        file_path_str.push('/');
+    })) {
         Ok(Some(entry)) => entry,
         Ok(None) => return Ok(None),
         Err(e) => return Err(errors::ReadFile::Lookup(file_path_str, e)),

src/source/mod.rs

@@ -1,18 +1,13 @@
 use crate::{
-    manifest::target::{Target, TargetKind},
+    manifest::target::Target,
     names::PackageNames,
     reporters::DownloadProgressReporter,
     source::{
         fs::PackageFS, refs::PackageRefs, specifiers::DependencySpecifiers, traits::*,
         version_id::VersionId,
     },
-    Project,
-};
-use std::{
-    collections::{BTreeMap, HashSet},
-    fmt::Debug,
-    sync::Arc,
 };
+use std::{collections::BTreeMap, fmt::Debug};
 
 /// Packages' filesystems
 pub mod fs;
@@ -66,26 +61,33 @@ impl PackageSource for PackageSources {
     type ResolveError = errors::ResolveError;
     type DownloadError = errors::DownloadError;
 
-    async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> {
+    async fn refresh(&self, options: &RefreshOptions) -> Result<(), Self::RefreshError> {
         match self {
-            PackageSources::Pesde(source) => source.refresh(project).await.map_err(Into::into),
+            PackageSources::Pesde(source) => source
+                .refresh(options)
+                .await
+                .map_err(Self::RefreshError::Pesde),
             #[cfg(feature = "wally-compat")]
-            PackageSources::Wally(source) => source.refresh(project).await.map_err(Into::into),
+            PackageSources::Wally(source) => source
-            PackageSources::Git(source) => source.refresh(project).await.map_err(Into::into),
+                .refresh(options)
-            PackageSources::Workspace(source) => source.refresh(project).await.map_err(Into::into),
+                .await
+                .map_err(Self::RefreshError::Wally),
+            PackageSources::Git(source) => source
+                .refresh(options)
+                .await
+                .map_err(Self::RefreshError::Git),
+            PackageSources::Workspace(source) => source.refresh(options).await.map_err(Into::into),
         }
     }
 
     async fn resolve(
         &self,
         specifier: &Self::Specifier,
-        project: &Project,
+        options: &ResolveOptions,
-        project_target: TargetKind,
-        refreshed_sources: &mut HashSet<PackageSources>,
     ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
         match (self, specifier) {
             (PackageSources::Pesde(source), DependencySpecifiers::Pesde(specifier)) => source
-                .resolve(specifier, project, project_target, refreshed_sources)
+                .resolve(specifier, options)
                 .await
                 .map(|(name, results)| {
                     (
@@ -100,7 +102,7 @@ impl PackageSource for PackageSources {
 
             #[cfg(feature = "wally-compat")]
             (PackageSources::Wally(source), DependencySpecifiers::Wally(specifier)) => source
-                .resolve(specifier, project, project_target, refreshed_sources)
+                .resolve(specifier, options)
                 .await
                 .map(|(name, results)| {
                     (
@@ -114,7 +116,7 @@ impl PackageSource for PackageSources {
                 .map_err(Into::into),
 
             (PackageSources::Git(source), DependencySpecifiers::Git(specifier)) => source
-                .resolve(specifier, project, project_target, refreshed_sources)
+                .resolve(specifier, options)
                 .await
                 .map(|(name, results)| {
                     (
@@ -129,7 +131,7 @@ impl PackageSource for PackageSources {
 
             (PackageSources::Workspace(source), DependencySpecifiers::Workspace(specifier)) => {
                 source
-                    .resolve(specifier, project, project_target, refreshed_sources)
+                    .resolve(specifier, options)
                     .await
                     .map(|(name, results)| {
                         (
@@ -149,34 +151,28 @@ impl PackageSource for PackageSources {
         }
     }
 
-    async fn download(
+    async fn download<R: DownloadProgressReporter>(
         &self,
         pkg_ref: &Self::Ref,
-        project: &Project,
+        options: &DownloadOptions<R>,
-        reqwest: &reqwest::Client,
-        reporter: Arc<impl DownloadProgressReporter>,
     ) -> Result<(PackageFS, Target), Self::DownloadError> {
         match (self, pkg_ref) {
-            (PackageSources::Pesde(source), PackageRefs::Pesde(pkg_ref)) => source
+            (PackageSources::Pesde(source), PackageRefs::Pesde(pkg_ref)) => {
-                .download(pkg_ref, project, reqwest, reporter)
+                source.download(pkg_ref, options).await.map_err(Into::into)
-                .await
+            }
-                .map_err(Into::into),
 
             #[cfg(feature = "wally-compat")]
-            (PackageSources::Wally(source), PackageRefs::Wally(pkg_ref)) => source
+            (PackageSources::Wally(source), PackageRefs::Wally(pkg_ref)) => {
-                .download(pkg_ref, project, reqwest, reporter)
+                source.download(pkg_ref, options).await.map_err(Into::into)
-                .await
+            }
-                .map_err(Into::into),
 
-            (PackageSources::Git(source), PackageRefs::Git(pkg_ref)) => source
+            (PackageSources::Git(source), PackageRefs::Git(pkg_ref)) => {
-                .download(pkg_ref, project, reqwest, reporter)
+                source.download(pkg_ref, options).await.map_err(Into::into)
-                .await
+            }
-                .map_err(Into::into),
 
-            (PackageSources::Workspace(source), PackageRefs::Workspace(pkg_ref)) => source
+            (PackageSources::Workspace(source), PackageRefs::Workspace(pkg_ref)) => {
-                .download(pkg_ref, project, reqwest, reporter)
+                source.download(pkg_ref, options).await.map_err(Into::into)
-                .await
+            }
-                .map_err(Into::into),
 
             _ => Err(errors::DownloadError::Mismatch),
         }
@@ -191,9 +187,18 @@ pub mod errors {
     #[derive(Debug, Error)]
     #[non_exhaustive]
     pub enum RefreshError {
-        /// A git-based package source failed to refresh
+        /// A pesde package source failed to refresh
         #[error("error refreshing pesde package source")]
-        GitBased(#[from] crate::source::git_index::errors::RefreshError),
+        Pesde(#[source] crate::source::git_index::errors::RefreshError),
+
+        /// A Wally package source failed to refresh
+        #[cfg(feature = "wally-compat")]
+        #[error("error refreshing wally package source")]
+        Wally(#[source] crate::source::git_index::errors::RefreshError),
+
+        /// A Git package source failed to refresh
+        #[error("error refreshing git package source")]
+        Git(#[source] crate::source::git_index::errors::RefreshError),
 
         /// A workspace package source failed to refresh
         #[error("error refreshing workspace package source")]

src/source/pesde/mod.rs

@@ -7,7 +7,6 @@ use std::{
     fmt::Debug,
     hash::Hash,
     path::PathBuf,
-    sync::Arc,
 };
 use tokio_util::io::StreamReader;
 
@@ -15,17 +14,14 @@ use pkg_ref::PesdePackageRef;
 use specifier::PesdeDependencySpecifier;
 
 use crate::{
-    manifest::{
+    manifest::{target::Target, DependencyType},
-        target::{Target, TargetKind},
-        DependencyType,
-    },
     names::{PackageName, PackageNames},
     reporters::DownloadProgressReporter,
     source::{
         fs::{store_in_cas, FSEntry, PackageFS},
         git_index::{read_file, root_tree, GitBasedSource},
-        DependencySpecifiers, PackageSource, PackageSources, ResolveResult, VersionId,
+        traits::{DownloadOptions, RefreshOptions, ResolveOptions},
-        IGNORED_DIRS, IGNORED_FILES,
+        DependencySpecifiers, PackageSource, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES,
     },
     util::hash,
     Project,
@@ -58,7 +54,10 @@ pub struct ScopeInfo {
 
 impl GitBasedSource for PesdePackageSource {
     fn path(&self, project: &Project) -> PathBuf {
-        project.data_dir.join("indices").join(hash(self.as_bytes()))
+        project
+            .data_dir()
+            .join("indices")
+            .join(hash(self.as_bytes()))
     }
 
     fn repo_url(&self) -> &Url {
@@ -105,18 +104,22 @@ impl PackageSource for PesdePackageSource {
     type DownloadError = errors::DownloadError;
 
     #[instrument(skip_all, level = "debug")]
-    async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> {
+    async fn refresh(&self, options: &RefreshOptions) -> Result<(), Self::RefreshError> {
-        GitBasedSource::refresh(self, project).await
+        GitBasedSource::refresh(self, options).await
     }
 
     #[instrument(skip_all, level = "debug")]
     async fn resolve(
         &self,
         specifier: &Self::Specifier,
-        project: &Project,
+        options: &ResolveOptions,
-        project_target: TargetKind,
-        _refreshed_sources: &mut HashSet<PackageSources>,
     ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
+        let ResolveOptions {
+            project,
+            target: project_target,
+            ..
+        } = options;
+
         let (scope, name) = specifier.name.as_str();
         let repo = gix::open(self.path(project)).map_err(Box::new)?;
         let tree = root_tree(&repo).map_err(Box::new)?;
@@ -142,7 +145,7 @@ impl PackageSource for PesdePackageSource {
                 .into_iter()
                 .filter(|(VersionId(version, target), _)| {
                     specifier.version.matches(version)
-                        && specifier.target.unwrap_or(project_target) == *target
+                        && specifier.target.unwrap_or(*project_target) == *target
                 })
                 .map(|(id, entry)| {
                     let version = id.version().clone();
@@ -163,16 +166,20 @@ impl PackageSource for PesdePackageSource {
     }
 
     #[instrument(skip_all, level = "debug")]
-    async fn download(
+    async fn download<R: DownloadProgressReporter>(
         &self,
         pkg_ref: &Self::Ref,
-        project: &Project,
+        options: &DownloadOptions<R>,
-        reqwest: &reqwest::Client,
-        reporter: Arc<impl DownloadProgressReporter>,
     ) -> Result<(PackageFS, Target), Self::DownloadError> {
+        let DownloadOptions {
+            project,
+            reporter,
+            reqwest,
+        } = options;
+
         let config = self.config(project).await.map_err(Box::new)?;
         let index_file = project
-            .cas_dir
+            .cas_dir()
             .join("index")
             .join(pkg_ref.name.escaped())
             .join(pkg_ref.version.to_string())
@@ -200,7 +207,7 @@ impl PackageSource for PesdePackageSource {
 
         let mut request = reqwest.get(&url).header(ACCEPT, "application/octet-stream");
 
-        if let Some(token) = project.auth_config.tokens().get(&self.repo_url) {
+        if let Some(token) = project.auth_config().tokens().get(&self.repo_url) {
             tracing::debug!("using token for {}", self.repo_url);
             request = request.header(AUTHORIZATION, token);
         }

src/source/traits.rs

@@ -1,4 +1,3 @@
-#![allow(async_fn_in_trait)]
 use crate::{
     manifest::{
         target::{Target, TargetKind},
@@ -6,11 +5,12 @@ use crate::{
     },
     reporters::DownloadProgressReporter,
     source::{DependencySpecifiers, PackageFS, PackageSources, ResolveResult},
-    Project,
+    Project, RefreshedSources,
 };
 use std::{
-    collections::{BTreeMap, HashSet},
+    collections::BTreeMap,
     fmt::{Debug, Display},
+    future::Future,
     sync::Arc,
 };
 
@@ -27,6 +27,35 @@ pub trait PackageRef: Debug {
     fn source(&self) -> PackageSources;
 }
 
+/// Options for refreshing a source
+#[derive(Debug, Clone)]
+pub struct RefreshOptions {
+    /// The project to refresh for
+    pub project: Project,
+}
+
+/// Options for resolving a package
+#[derive(Debug, Clone)]
+pub struct ResolveOptions {
+    /// The project to resolve for
+    pub project: Project,
+    /// The target to resolve for
+    pub target: TargetKind,
+    /// The sources that have been refreshed
+    pub refreshed_sources: RefreshedSources,
+}
+
+/// Options for downloading a package
+#[derive(Debug, Clone)]
+pub struct DownloadOptions<R: DownloadProgressReporter> {
+    /// The project to download for
+    pub project: Project,
+    /// The reqwest client to use
+    pub reqwest: reqwest::Client,
+    /// The reporter to use
+    pub reporter: Arc<R>,
+}
+
 /// A source of packages
 pub trait PackageSource: Debug {
     /// The specifier type for this source
@@ -34,32 +63,31 @@ pub trait PackageSource: Debug {
     /// The reference type for this source
     type Ref: PackageRef;
     /// The error type for refreshing this source
-    type RefreshError: std::error::Error;
+    type RefreshError: std::error::Error + Send + Sync + 'static;
     /// The error type for resolving a package from this source
-    type ResolveError: std::error::Error;
+    type ResolveError: std::error::Error + Send + Sync + 'static;
     /// The error type for downloading a package from this source
-    type DownloadError: std::error::Error;
+    type DownloadError: std::error::Error + Send + Sync + 'static;
 
     /// Refreshes the source
-    async fn refresh(&self, _project: &Project) -> Result<(), Self::RefreshError> {
+    fn refresh(
-        Ok(())
+        &self,
+        _options: &RefreshOptions,
+    ) -> impl Future<Output = Result<(), Self::RefreshError>> + Send + Sync {
+        async { Ok(()) }
     }
 
     /// Resolves a specifier to a reference
-    async fn resolve(
+    fn resolve(
         &self,
         specifier: &Self::Specifier,
-        project: &Project,
+        options: &ResolveOptions,
-        project_target: TargetKind,
+    ) -> impl Future<Output = Result<ResolveResult<Self::Ref>, Self::ResolveError>>;
-        refreshed_sources: &mut HashSet<PackageSources>,
-    ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError>;
 
     /// Downloads a package
-    async fn download(
+    fn download<R: DownloadProgressReporter>(
         &self,
         pkg_ref: &Self::Ref,
-        project: &Project,
+        options: &DownloadOptions<R>,
-        reqwest: &reqwest::Client,
+    ) -> impl Future<Output = Result<(PackageFS, Target), Self::DownloadError>>;
-        reporter: Arc<impl DownloadProgressReporter>,
-    ) -> Result<(PackageFS, Target), Self::DownloadError>;
 }

src/source/wally/compat_util.rs

@@ -37,7 +37,7 @@ async fn find_lib_path(
 
     let result = execute_script(
         ScriptName::SourcemapGenerator,
-        &script_path.to_path(&project.package_dir),
+        &script_path.to_path(project.package_dir()),
         [package_dir],
         project,
         true,

src/source/wally/mod.rs

@@ -5,7 +5,7 @@ use crate::{
     source::{
         fs::{store_in_cas, FSEntry, PackageFS},
         git_index::{read_file, root_tree, GitBasedSource},
-        traits::PackageSource,
+        traits::{DownloadOptions, PackageSource, RefreshOptions, ResolveOptions},
         version_id::VersionId,
         wally::{
             compat_util::get_target,
@@ -23,11 +23,7 @@ use gix::Url;
 use relative_path::RelativePathBuf;
 use reqwest::header::AUTHORIZATION;
 use serde::Deserialize;
-use std::{
+use std::{collections::BTreeMap, path::PathBuf, sync::Arc};
-    collections::{BTreeMap, HashSet},
-    path::PathBuf,
-    sync::Arc,
-};
 use tempfile::tempdir;
 use tokio::{
     io::{AsyncReadExt, AsyncWriteExt},
@@ -53,7 +49,7 @@ pub struct WallyPackageSource {
 impl GitBasedSource for WallyPackageSource {
     fn path(&self, project: &Project) -> PathBuf {
         project
-            .data_dir
+            .data_dir()
             .join("wally_indices")
             .join(hash(self.as_bytes()))
     }
@@ -102,18 +98,22 @@ impl PackageSource for WallyPackageSource {
     type DownloadError = errors::DownloadError;
 
     #[instrument(skip_all, level = "debug")]
-    async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> {
+    async fn refresh(&self, options: &RefreshOptions) -> Result<(), Self::RefreshError> {
-        GitBasedSource::refresh(self, project).await
+        GitBasedSource::refresh(self, options).await
     }
 
     #[instrument(skip_all, level = "debug")]
     async fn resolve(
         &self,
         specifier: &Self::Specifier,
-        project: &Project,
+        options: &ResolveOptions,
-        project_target: TargetKind,
-        refreshed_sources: &mut HashSet<PackageSources>,
     ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
+        let ResolveOptions {
+            project,
+            refreshed_sources,
+            ..
+        } = options;
+
         let repo = gix::open(self.path(project)).map_err(Box::new)?;
         let tree = root_tree(&repo).map_err(Box::new)?;
         let (scope, name) = specifier.name.as_str();
@@ -127,23 +127,26 @@ impl PackageSource for WallyPackageSource {
 
                 let config = self.config(project).await.map_err(Box::new)?;
                 for registry in config.fallback_registries {
-                    let source = WallyPackageSource::new(registry.clone());
+                    let source = WallyPackageSource::new(registry);
-                    if refreshed_sources.insert(PackageSources::Wally(source.clone())) {
+                    match refreshed_sources
-                        GitBasedSource::refresh(&source, project)
+                        .refresh(
-                            .await
+                            &PackageSources::Wally(source.clone()),
-                            .map_err(Box::new)?;
+                            &RefreshOptions {
+                                project: project.clone(),
+                            },
+                        )
+                        .await
+                    {
+                        Ok(()) => {}
+                        Err(super::errors::RefreshError::Wally(e)) => {
+                            return Err(Self::ResolveError::Refresh(Box::new(e)));
+                        }
+                        Err(e) => unreachable!("unexpected error: {e:?}"),
                     }
 
-                    match Box::pin(source.resolve(
+                    match Box::pin(source.resolve(specifier, options)).await {
-                        specifier,
-                        project,
-                        project_target,
-                        refreshed_sources,
-                    ))
-                    .await
-                    {
                         Ok((name, results)) => {
-                            tracing::debug!("found {} in backup registry {registry}", name);
+                            tracing::debug!("found {name} in backup registry {}", source.repo_url);
                             return Ok((name, results));
                         }
                         Err(errors::ResolveError::NotFound(_)) => {
@@ -202,16 +205,20 @@ impl PackageSource for WallyPackageSource {
     }
 
     #[instrument(skip_all, level = "debug")]
-    async fn download(
+    async fn download<R: DownloadProgressReporter>(
         &self,
         pkg_ref: &Self::Ref,
-        project: &Project,
+        options: &DownloadOptions<R>,
-        reqwest: &reqwest::Client,
-        reporter: Arc<impl DownloadProgressReporter>,
     ) -> Result<(PackageFS, Target), Self::DownloadError> {
+        let DownloadOptions {
+            project,
+            reqwest,
+            reporter,
+        } = options;
+
         let config = self.config(project).await.map_err(Box::new)?;
         let index_file = project
-            .cas_dir
+            .cas_dir()
             .join("wally_index")
             .join(pkg_ref.name.escaped())
             .join(pkg_ref.version.to_string());
@@ -250,7 +257,7 @@ impl PackageSource for WallyPackageSource {
                     .unwrap_or("0.3.2"),
             );
 
-        if let Some(token) = project.auth_config.tokens().get(&self.repo_url) {
+        if let Some(token) = project.auth_config().tokens().get(&self.repo_url) {
             tracing::debug!("using token for {}", self.repo_url);
             request = request.header(AUTHORIZATION, token);
         }
@@ -291,6 +298,7 @@ impl PackageSource for WallyPackageSource {
 
         let archive = Arc::new(Mutex::new(archive));
 
+        // todo: remove this asyncification, since the Mutex makes it sequential anyway
         let entries = try_join_all(
             entries
                 .into_iter()
@@ -363,7 +371,7 @@ impl PackageSource for WallyPackageSource {
 pub struct WallyIndexConfig {
     api: url::Url,
     #[serde(default, deserialize_with = "crate::util::deserialize_gix_url_vec")]
-    fallback_registries: Vec<gix::Url>,
+    fallback_registries: Vec<Url>,
 }
 
 /// Errors that can occur when interacting with a Wally package source

src/source/workspace/mod.rs

@@ -1,21 +1,20 @@
 use crate::{
-    manifest::target::{Target, TargetKind},
+    manifest::target::Target,
     names::PackageNames,
     reporters::DownloadProgressReporter,
     source::{
-        fs::PackageFS, specifiers::DependencySpecifiers, traits::PackageSource,
+        fs::PackageFS,
-        version_id::VersionId, workspace::pkg_ref::WorkspacePackageRef, PackageSources,
+        specifiers::DependencySpecifiers,
+        traits::{DownloadOptions, PackageSource, ResolveOptions},
+        version_id::VersionId,
+        workspace::pkg_ref::WorkspacePackageRef,
         ResolveResult,
     },
-    Project, DEFAULT_INDEX_NAME,
+    DEFAULT_INDEX_NAME,
 };
 use futures::StreamExt;
 use relative_path::RelativePathBuf;
-use reqwest::Client;
+use std::collections::BTreeMap;
-use std::{
-    collections::{BTreeMap, HashSet},
-    sync::Arc,
-};
 use tokio::pin;
 use tracing::instrument;
 
@@ -35,25 +34,21 @@ impl PackageSource for WorkspacePackageSource {
     type ResolveError = errors::ResolveError;
     type DownloadError = errors::DownloadError;
 
-    async fn refresh(&self, _project: &Project) -> Result<(), Self::RefreshError> {
-        // no-op
-        Ok(())
-    }
-
     #[instrument(skip_all, level = "debug")]
     async fn resolve(
         &self,
         specifier: &Self::Specifier,
-        project: &Project,
+        options: &ResolveOptions,
-        project_target: TargetKind,
-        _refreshed_sources: &mut HashSet<PackageSources>,
     ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
+        let ResolveOptions {
+            project,
+            target: project_target,
+            ..
+        } = options;
+
         let (path, manifest) = 'finder: {
-            let workspace_dir = project
+            let workspace_dir = project.workspace_dir().unwrap_or(project.package_dir());
-                .workspace_dir
+            let target = specifier.target.unwrap_or(*project_target);
-                .as_ref()
-                .unwrap_or(&project.package_dir);
-            let target = specifier.target.unwrap_or(project_target);
 
             let members = project.workspace_members(workspace_dir, true).await?;
             pin!(members);
@@ -70,77 +65,77 @@ impl PackageSource for WorkspacePackageSource {
             ));
         };
 
+        let manifest_target_kind = manifest.target.kind();
+        let pkg_ref = WorkspacePackageRef {
+            // workspace_dir is guaranteed to be Some by the workspace_members method
+            // strip_prefix is guaranteed to be Some by same method
+            // from_path is guaranteed to be Ok because we just stripped the absolute path
+            path: RelativePathBuf::from_path(
+                path.strip_prefix(project.workspace_dir().unwrap()).unwrap(),
+            )
+            .unwrap(),
+            dependencies: manifest
+                .all_dependencies()?
+                .into_iter()
+                .map(|(alias, (mut spec, ty))| {
+                    match &mut spec {
+                        DependencySpecifiers::Pesde(spec) => {
+                            let index_name = spec.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME);
+
+                            spec.index = Some(
+                                manifest
+                                    .indices
+                                    .get(index_name)
+                                    .ok_or(errors::ResolveError::IndexNotFound(
+                                        index_name.to_string(),
+                                        manifest.name.to_string(),
+                                    ))?
+                                    .to_string(),
+                            )
+                        }
+                        #[cfg(feature = "wally-compat")]
+                        DependencySpecifiers::Wally(spec) => {
+                            let index_name = spec.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME);
+
+                            spec.index = Some(
+                                manifest
+                                    .wally_indices
+                                    .get(index_name)
+                                    .ok_or(errors::ResolveError::IndexNotFound(
+                                        index_name.to_string(),
+                                        manifest.name.to_string(),
+                                    ))?
+                                    .to_string(),
+                            )
+                        }
+                        DependencySpecifiers::Git(_) => {}
+                        DependencySpecifiers::Workspace(_) => {}
+                    }
+
+                    Ok((alias, (spec, ty)))
+                })
+                .collect::<Result<_, errors::ResolveError>>()?,
+            target: manifest.target,
+        };
+
         Ok((
-            PackageNames::Pesde(manifest.name.clone()),
+            PackageNames::Pesde(manifest.name),
             BTreeMap::from([(
-                VersionId::new(manifest.version.clone(), manifest.target.kind()),
+                VersionId::new(manifest.version, manifest_target_kind),
-                WorkspacePackageRef {
+                pkg_ref,
-                    // workspace_dir is guaranteed to be Some by the workspace_members method
-                    // strip_prefix is guaranteed to be Some by same method
-                    // from_path is guaranteed to be Ok because we just stripped the absolute path
-                    path: RelativePathBuf::from_path(
-                        path.strip_prefix(project.workspace_dir.clone().unwrap())
-                            .unwrap(),
-                    )
-                    .unwrap(),
-                    dependencies: manifest
-                        .all_dependencies()?
-                        .into_iter()
-                        .map(|(alias, (mut spec, ty))| {
-                            match &mut spec {
-                                DependencySpecifiers::Pesde(spec) => {
-                                    let index_name =
-                                        spec.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME);
-
-                                    spec.index = Some(
-                                        manifest
-                                            .indices
-                                            .get(index_name)
-                                            .ok_or(errors::ResolveError::IndexNotFound(
-                                                index_name.to_string(),
-                                                manifest.name.to_string(),
-                                            ))?
-                                            .to_string(),
-                                    )
-                                }
-                                #[cfg(feature = "wally-compat")]
-                                DependencySpecifiers::Wally(spec) => {
-                                    let index_name =
-                                        spec.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME);
-
-                                    spec.index = Some(
-                                        manifest
-                                            .wally_indices
-                                            .get(index_name)
-                                            .ok_or(errors::ResolveError::IndexNotFound(
-                                                index_name.to_string(),
-                                                manifest.name.to_string(),
-                                            ))?
-                                            .to_string(),
-                                    )
-                                }
-                                DependencySpecifiers::Git(_) => {}
-                                DependencySpecifiers::Workspace(_) => {}
-                            }
-
-                            Ok((alias, (spec, ty)))
-                        })
-                        .collect::<Result<_, errors::ResolveError>>()?,
-                    target: manifest.target,
-                },
             )]),
         ))
     }
 
     #[instrument(skip_all, level = "debug")]
-    async fn download(
+    async fn download<R: DownloadProgressReporter>(
         &self,
         pkg_ref: &Self::Ref,
-        project: &Project,
+        options: &DownloadOptions<R>,
-        _reqwest: &Client,
-        _reporter: Arc<impl DownloadProgressReporter>,
     ) -> Result<(PackageFS, Target), Self::DownloadError> {
-        let path = pkg_ref.path.to_path(project.workspace_dir.clone().unwrap());
+        let DownloadOptions { project, .. } = options;
+
+        let path = pkg_ref.path.to_path(project.workspace_dir().unwrap());
 
         Ok((
             PackageFS::Copy(path, pkg_ref.target.kind()),