pesde-pkg/pesde
1
1
Fork 0
mirror of https://github.com/pesde-pkg/pesde.git synced 2025-04-05 11:20:55 +01:00
Code Issues Projects Releases Packages Wiki Activity

perf: use arcs where possible, remove unnecessary cloning

Browse source
This commit is contained in:
daimond113 2024-12-28 16:50:14 +01:00
parent a41d9950f8
commit 8e6d877241
No known key found for this signature in database
GPG key ID: 3A8ECE51328B513C
29 changed files with 746 additions and 583 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
CHANGELOG.md
View file

@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Added
- Improve installation experience by @lukadev-0
### Performance
- Use `Arc` for more efficient cloning of multiple structs by @daimond113
- Avoid cloning where possible by @daimond113
## [0.5.2] - 2024-12-19
### Fixed
- Change dependency types for removed peer dependencies by @daimond113

8
registry/src/endpoints/publish_version.rs
View file

@ -18,6 +18,7 @@ use pesde::{
git_index::{read_file, root_tree, GitBasedSource},
pesde::{DocEntry, DocEntryKind, IndexFile, IndexFileEntry, ScopeInfo, SCOPE_INFO_FILE},
specifiers::DependencySpecifiers,
traits::RefreshOptions,
version_id::VersionId,
IGNORED_DIRS, IGNORED_FILES,
},
@ -72,7 +73,12 @@ pub async fn publish_package(
user_id: web::ReqData<UserId>,
) -> Result<impl Responder, Error> {
let source = app_state.source.lock().await;
source.refresh(&app_state.project).await.map_err(Box::new)?;
source
.refresh(&RefreshOptions {
project: app_state.project.clone(),
})
.await
.map_err(Box::new)?;
let config = source.config(&app_state.project).await?;
let package_dir = tempfile::tempdir()?;

9
registry/src/main.rs
View file

@ -14,7 +14,10 @@ use actix_web::{
};
use fs_err::tokio as fs;
use pesde::{
source::{pesde::PesdePackageSource, traits::PackageSource},
source::{
pesde::PesdePackageSource,
traits::{PackageSource, RefreshOptions},
},
AuthConfig, Project,
};
use std::{env::current_dir, path::PathBuf};
@ -106,7 +109,9 @@ async fn run() -> std::io::Result<()> {
);
let source = PesdePackageSource::new(benv!(required "INDEX_REPO_URL").try_into().unwrap());
source
.refresh(&project)
.refresh(&RefreshOptions {
project: project.clone(),
})
.await
.expect("failed to refresh source");
let config = source

31
src/cli/commands/add.rs
View file

@ -1,4 +1,4 @@
use std::{collections::HashSet, str::FromStr};
use std::str::FromStr;
use anyhow::Context;
use clap::Args;
@ -13,11 +13,11 @@ use pesde::{
git::{specifier::GitDependencySpecifier, GitPackageSource},
pesde::{specifier::PesdeDependencySpecifier, PesdePackageSource},
specifiers::DependencySpecifiers,
traits::PackageSource,
traits::{PackageSource, RefreshOptions, ResolveOptions},
workspace::WorkspacePackageSource,
PackageSources,
},
Project, DEFAULT_INDEX_NAME,
Project, RefreshedSources, DEFAULT_INDEX_NAME,
};
#[derive(Debug, Args)]
@ -128,17 +128,27 @@ impl AddCommand {
),
),
};
source
.refresh(&project)
let refreshed_sources = RefreshedSources::new();
refreshed_sources
.refresh(
&source,
&RefreshOptions {
project: project.clone(),
},
)
.await
.context("failed to refresh package source")?;
let Some(version_id) = source
.resolve(
&specifier,
&project,
manifest.target.kind(),
&mut HashSet::new(),
&ResolveOptions {
project: project.clone(),
target: manifest.target.kind(),
refreshed_sources,
},
)
.await
.context("failed to resolve package")?
@ -167,7 +177,7 @@ impl AddCommand {
"dependencies"
};
let alias = self.alias.unwrap_or_else(|| match self.name.clone() {
let alias = self.alias.unwrap_or_else(|| match &self.name {
AnyPackageIdentifier::PackageName(versioned) => versioned.0.as_str().1.to_string(),
AnyPackageIdentifier::Url((url, _)) => url
.path
@ -205,7 +215,8 @@ impl AddCommand {
}
#[cfg(feature = "wally-compat")]
DependencySpecifiers::Wally(spec) => {
field["wally"] = toml_edit::value(spec.name.clone().to_string());
field["wally"] =
toml_edit::value(spec.name.clone().to_string().trim_start_matches("wally#"));
field["version"] = toml_edit::value(format!("^{}", version_id.version()));
if let Some(index) = spec.index.filter(|i| i != DEFAULT_INDEX_NAME) {

12
src/cli/commands/auth/login.rs
View file

@ -6,13 +6,15 @@ use std::thread::spawn;
use tokio::time::sleep;
use url::Url;
use crate::cli::auth::{get_token_login, set_token};
use pesde::{
source::{pesde::PesdePackageSource, traits::PackageSource},
source::{
pesde::PesdePackageSource,
traits::{PackageSource, RefreshOptions},
},
Project,
};
use crate::cli::auth::{get_token_login, set_token};
#[derive(Debug, Args)]
pub struct LoginCommand {
/// The token to use for authentication, skipping login
@ -57,7 +59,9 @@ impl LoginCommand {
let source = PesdePackageSource::new(index_url.clone());
source
.refresh(project)
.refresh(&RefreshOptions {
project: project.clone(),
})
.await
.context("failed to refresh index")?;

10
src/cli/commands/auth/mod.rs
View file

@ -1,4 +1,5 @@
use crate::cli::config::read_config;
use anyhow::Context;
use clap::{Args, Subcommand};
use pesde::{errors::ManifestReadError, Project, DEFAULT_INDEX_NAME};
@ -56,10 +57,11 @@ impl AuthSubcommand {
None => {
let index_name = self.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME);
match manifest.unwrap().indices.get(index_name) {
Some(index) => index.clone(),
None => anyhow::bail!("index {index_name} not found in manifest"),
}
manifest
.unwrap()
.indices
.remove(index_name)
.with_context(|| format!("index {index_name} not found in manifest"))?
}
};

51
src/cli/commands/execute.rs
View file

@ -15,20 +15,19 @@ use pesde::{
names::PackageName,
source::{
pesde::{specifier::PesdeDependencySpecifier, PesdePackageSource},
traits::PackageSource,
traits::{DownloadOptions, PackageSource, RefreshOptions, ResolveOptions},
PackageSources,
},
Project,
Project, RefreshedSources,
};
use semver::VersionReq;
use std::{
collections::HashSet,
env::current_dir,
ffi::OsString,
io::{Stderr, Write},
process::Command,
sync::Arc,
};
use tokio::sync::Mutex;
#[derive(Debug, Args)]
pub struct ExecuteCommand {
@ -53,6 +52,8 @@ impl ExecuteCommand {
.unwrap()
.replace(multi_progress.clone());
let refreshed_sources = RefreshedSources::new();
let (tempdir, bin_path) = reporters::run_with_reporter_and_writer(
std::io::stderr(),
|multi_progress, root_progress, reporter| async {
@ -67,8 +68,13 @@ impl ExecuteCommand {
}
.context("no index specified")?;
let source = PesdePackageSource::new(index);
source
.refresh(&project)
refreshed_sources
.refresh(
&PackageSources::Pesde(source.clone()),
&RefreshOptions {
project: project.clone(),
},
)
.await
.context("failed to refresh source")?;
@ -82,7 +88,14 @@ impl ExecuteCommand {
};
if let Some(res) = source
.resolve(&specifier, &project, TargetKind::Lune, &mut HashSet::new())
.resolve(
&specifier,
&ResolveOptions {
project: project.clone(),
target: TargetKind::Lune,
refreshed_sources: refreshed_sources.clone(),
},
)
.await
.context("failed to resolve package")?
.1
@ -92,7 +105,14 @@ impl ExecuteCommand {
}
source
.resolve(&specifier, &project, TargetKind::Luau, &mut HashSet::new())
.resolve(
&specifier,
&ResolveOptions {
project: project.clone(),
target: TargetKind::Luau,
refreshed_sources: refreshed_sources.clone(),
},
)
.await
.context("failed to resolve package")?
.1
@ -120,7 +140,14 @@ impl ExecuteCommand {
);
let (fs, target) = source
.download(&pkg_ref, &project, &reqwest, Arc::new(()))
.download(
&pkg_ref,
&DownloadOptions {
project: project.clone(),
reqwest: reqwest.clone(),
reporter: Arc::new(()),
},
)
.await
.context("failed to download package")?;
let bin_path = target.bin_path().context("package has no binary export")?;
@ -129,10 +156,8 @@ impl ExecuteCommand {
.await
.context("failed to write package contents")?;
let mut refreshed_sources = HashSet::new();
let graph = project
.dependency_graph(None, &mut refreshed_sources, true)
.dependency_graph(None, refreshed_sources.clone(), true)
.await
.context("failed to build dependency graph")?;
@ -152,7 +177,7 @@ impl ExecuteCommand {
&Arc::new(graph),
DownloadAndLinkOptions::<CliReporter<Stderr>, ()>::new(reqwest)
.reporter(reporter)
.refreshed_sources(Mutex::new(refreshed_sources))
.refreshed_sources(refreshed_sources)
.prod(true)
.write(true),
)

25
src/cli/commands/init.rs
View file

@ -11,12 +11,13 @@ use pesde::{
git_index::GitBasedSource,
pesde::{specifier::PesdeDependencySpecifier, PesdePackageSource},
specifiers::DependencySpecifiers,
traits::PackageSource,
traits::{PackageSource, RefreshOptions, ResolveOptions},
PackageSources,
},
Project, DEFAULT_INDEX_NAME, SCRIPTS_LINK_FOLDER,
Project, RefreshedSources, DEFAULT_INDEX_NAME, SCRIPTS_LINK_FOLDER,
};
use semver::VersionReq;
use std::{collections::HashSet, fmt::Display, str::FromStr};
use std::{fmt::Display, str::FromStr};
#[derive(Debug, Args)]
pub struct InitCommand {}
@ -128,13 +129,21 @@ impl InitCommand {
manifest["indices"].or_insert(toml_edit::Item::Table(toml_edit::Table::new()))
[DEFAULT_INDEX_NAME] = toml_edit::value(source.repo_url().to_bstring().to_string());
let refreshed_sources = RefreshedSources::new();
if target_env.is_roblox()
|| inquire::prompt_confirmation(
"would you like to setup default Roblox compatibility scripts?",
)
.unwrap()
{
PackageSource::refresh(&source, &project)
refreshed_sources
.refresh(
&PackageSources::Pesde(source.clone()),
&RefreshOptions {
project: project.clone(),
},
)
.await
.context("failed to refresh package source")?;
let config = source
@ -193,9 +202,11 @@ impl InitCommand {
index: None,
target: None,
},
&project,
TargetKind::Lune,
&mut HashSet::new(),
&ResolveOptions {
project: project.clone(),
target: TargetKind::Lune,
refreshed_sources,
},
)
.await
.context("failed to resolve scripts package")?

43
src/cli/commands/outdated.rs
View file

@ -3,17 +3,14 @@ use anyhow::Context;
use clap::Args;
use futures::future::try_join_all;
use pesde::{
refresh_sources,
source::{
refs::PackageRefs,
specifiers::DependencySpecifiers,
traits::{PackageRef, PackageSource},
traits::{PackageRef, PackageSource, RefreshOptions, ResolveOptions},
},
Project,
Project, RefreshedSources,
};
use semver::VersionReq;
use std::{collections::HashSet, sync::Arc};
use tokio::sync::Mutex;
#[derive(Debug, Args)]
pub struct OutdatedCommand {
@ -40,19 +37,7 @@ impl OutdatedCommand {
.context("failed to read manifest")?;
let manifest_target_kind = manifest.target.kind();
let mut refreshed_sources = HashSet::new();
refresh_sources(
&project,
graph
.iter()
.flat_map(|(_, versions)| versions.iter())
.map(|(_, node)| node.node.pkg_ref.source()),
&mut refreshed_sources,
)
.await?;
let refreshed_sources = Arc::new(Mutex::new(refreshed_sources));
let refreshed_sources = RefreshedSources::new();
if try_join_all(
graph
@ -74,14 +59,22 @@ impl OutdatedCommand {
}
let source = node.node.pkg_ref.source();
refreshed_sources
.refresh(
&source,
&RefreshOptions {
project: project.clone(),
},
)
.await?;
if !self.strict {
match specifier {
DependencySpecifiers::Pesde(ref mut spec) => {
match &mut specifier {
DependencySpecifiers::Pesde(spec) => {
spec.version = VersionReq::STAR;
}
#[cfg(feature = "wally-compat")]
DependencySpecifiers::Wally(ref mut spec) => {
DependencySpecifiers::Wally(spec) => {
spec.version = VersionReq::STAR;
}
DependencySpecifiers::Git(_) => {}
@ -92,9 +85,11 @@ impl OutdatedCommand {
let version_id = source
.resolve(
&specifier,
&project,
manifest_target_kind,
&mut *refreshed_sources.lock().await,
&ResolveOptions {
project: project.clone(),
target: manifest_target_kind,
refreshed_sources: refreshed_sources.clone(),
},
)
.await
.context("failed to resolve package versions")?

11
src/cli/commands/patch.rs
View file

@ -9,7 +9,7 @@ use pesde::{
patches::setup_patches_repo,
source::{
refs::PackageRefs,
traits::{PackageRef, PackageSource},
traits::{DownloadOptions, PackageRef, PackageSource},
},
Project, MANIFEST_FILE_NAME,
};
@ -51,7 +51,14 @@ impl PatchCommand {
fs::create_dir_all(&directory).await?;
source
.download(&node.node.pkg_ref, &project, &reqwest, Arc::new(()))
.download(
&node.node.pkg_ref,
&DownloadOptions {
project: project.clone(),
reqwest,
reporter: Arc::new(()),
},
)
.await?
.0
.write_to(&directory, project.cas_dir(), false)

28
src/cli/commands/publish.rs
View file

@ -22,9 +22,16 @@ use pesde::{
},
Project, DEFAULT_INDEX_NAME, MANIFEST_FILE_NAME,
};
use pesde::{
source::{
traits::{RefreshOptions, ResolveOptions},
PackageSources,
},
RefreshedSources,
};
use reqwest::{header::AUTHORIZATION, StatusCode};
use semver::VersionReq;
use std::{collections::HashSet, path::PathBuf};
use std::path::PathBuf;
use tempfile::Builder;
use tokio::io::{AsyncSeekExt, AsyncWriteExt};
@ -365,6 +372,8 @@ info: otherwise, the file was deemed unnecessary, if you don't understand why, p
}
}
let refreshed_sources = RefreshedSources::new();
for specifier in manifest
.dependencies
.values_mut()
@ -406,7 +415,14 @@ info: otherwise, the file was deemed unnecessary, if you don't understand why, p
DependencySpecifiers::Git(_) => {}
DependencySpecifiers::Workspace(spec) => {
let pkg_ref = WorkspacePackageSource
.resolve(spec, project, target_kind, &mut HashSet::new())
.resolve(
spec,
&ResolveOptions {
project: project.clone(),
target: target_kind,
refreshed_sources: refreshed_sources.clone(),
},
)
.await
.context("failed to resolve workspace package")?
.1
@ -575,7 +591,13 @@ info: otherwise, the file was deemed unnecessary, if you don't understand why, p
.get(&self.index)
.context(format!("missing index {}", self.index))?;
let source = PesdePackageSource::new(index_url.clone());
PackageSource::refresh(&source, project)
refreshed_sources
.refresh(
&PackageSources::Pesde(source.clone()),
&RefreshOptions {
project: project.clone(),
},
)
.await
.context("failed to refresh source")?;
let config = source

17
src/cli/commands/run.rs
View file

@ -9,8 +9,7 @@ use pesde::{
};
use relative_path::RelativePathBuf;
use std::{
collections::HashSet, env::current_dir, ffi::OsString, io::Write, path::PathBuf,
process::Command,
collections::HashSet, env::current_dir, ffi::OsString, io::Write, path::Path, process::Command,
};
#[derive(Debug, Args)]
@ -26,7 +25,7 @@ pub struct RunCommand {
impl RunCommand {
pub async fn run(self, project: Project) -> anyhow::Result<()> {
let run = |root: PathBuf, file_path: PathBuf| {
let run = |root: &Path, file_path: &Path| {
let mut caller = tempfile::NamedTempFile::new().expect("failed to create tempfile");
caller
.write_all(
@ -55,8 +54,8 @@ impl RunCommand {
let Some(package_or_script) = self.package_or_script else {
if let Some(script_path) = project.deser_manifest().await?.target.bin_path() {
run(
project.package_dir().to_owned(),
script_path.to_path(project.package_dir()),
project.package_dir(),
&script_path.to_path(project.package_dir()),
);
return Ok(());
}
@ -99,7 +98,7 @@ impl RunCommand {
let path = bin_path.to_path(&container_folder);
run(path.clone(), path);
run(&path, &path);
return Ok(());
}
}
@ -107,8 +106,8 @@ impl RunCommand {
if let Ok(manifest) = project.deser_manifest().await {
if let Some(script_path) = manifest.scripts.get(&package_or_script) {
run(
project.package_dir().to_path_buf(),
script_path.to_path(project.package_dir()),
project.package_dir(),
&script_path.to_path(project.package_dir()),
);
return Ok(());
}
@ -170,7 +169,7 @@ impl RunCommand {
project.package_dir().to_path_buf()
};
run(root, path);
run(&root, &path);
Ok(())
}

2
src/cli/commands/self_upgrade.rs
View file

@ -46,7 +46,7 @@ impl SelfUpgradeCommand {
return Ok(());
}
let path = get_or_download_version(&reqwest, &TagInfo::Complete(latest_version), true)
let path = get_or_download_version(&reqwest, TagInfo::Complete(latest_version), true)
.await?
.unwrap();
update_bin_exe(&path).await?;

12
src/cli/install.rs
View file

@ -1,5 +1,5 @@
use std::{
collections::{BTreeMap, BTreeSet, HashMap, HashSet},
collections::{BTreeMap, BTreeSet, HashMap},
num::NonZeroUsize,
sync::Arc,
time::Instant,
@ -13,9 +13,9 @@ use pesde::{
download_and_link::{filter_graph, DownloadAndLinkHooks, DownloadAndLinkOptions},
lockfile::{DependencyGraph, DownloadedGraph, Lockfile},
manifest::{target::TargetKind, DependencyType},
Project, MANIFEST_FILE_NAME,
Project, RefreshedSources, MANIFEST_FILE_NAME,
};
use tokio::{sync::Mutex, task::JoinSet};
use tokio::task::JoinSet;
use crate::cli::{
bin_dir,
@ -178,7 +178,7 @@ pub async fn install(
) -> anyhow::Result<()> {
let start = Instant::now();
let mut refreshed_sources = HashSet::new();
let refreshed_sources = RefreshedSources::new();
let manifest = project
.deser_manifest()
@ -276,7 +276,7 @@ pub async fn install(
let graph = project
.dependency_graph(
old_graph.as_ref().filter(|_| options.use_lockfile),
&mut refreshed_sources,
refreshed_sources.clone(),
false,
)
.await
@ -298,7 +298,7 @@ pub async fn install(
DownloadAndLinkOptions::<CliReporter, InstallHooks>::new(reqwest.clone())
.reporter(reporter.clone())
.hooks(hooks)
.refreshed_sources(Mutex::new(refreshed_sources))
.refreshed_sources(refreshed_sources)
.prod(options.prod)
.write(options.write)
.network_concurrency(options.network_concurrency),

8
src/cli/version.rs
View file

@ -251,7 +251,7 @@ pub enum TagInfo {
#[instrument(skip(reqwest), level = "trace")]
pub async fn get_or_download_version(
reqwest: &reqwest::Client,
tag: &TagInfo,
tag: TagInfo,
always_give_path: bool,
) -> anyhow::Result<Option<PathBuf>> {
let path = home_dir()?.join("versions");
@ -259,7 +259,7 @@ pub async fn get_or_download_version(
.await
.context("failed to create versions directory")?;
let version = match tag {
let version = match &tag {
TagInfo::Complete(version) => version,
// don't fetch the version since it could be cached
TagInfo::Incomplete(version) => version,
@ -290,9 +290,9 @@ pub async fn get_or_download_version(
.context("failed to copy current executable to version directory")?;
} else {
let version = match tag {
TagInfo::Complete(version) => version.clone(),
TagInfo::Complete(version) => version,
TagInfo::Incomplete(version) => {
get_remote_version(reqwest, VersionType::Specific(version.clone()))
get_remote_version(reqwest, VersionType::Specific(version))
.await
.context("failed to get remote version")?
}

96
src/download.rs
View file

@ -2,19 +2,17 @@ use crate::{
lockfile::{DependencyGraph, DownloadedDependencyGraphNode},
manifest::DependencyType,
names::PackageNames,
refresh_sources,
reporters::{DownloadProgressReporter, DownloadsReporter},
source::{
traits::{PackageRef, PackageSource},
traits::{DownloadOptions, PackageRef, PackageSource, RefreshOptions},
version_id::VersionId,
PackageSources,
},
Project, PACKAGES_CONTAINER_NAME,
Project, RefreshedSources, PACKAGES_CONTAINER_NAME,
};
use async_stream::try_stream;
use fs_err::tokio as fs;
use futures::Stream;
use std::{collections::HashSet, num::NonZeroUsize, sync::Arc};
use std::{num::NonZeroUsize, sync::Arc};
use tokio::{sync::Semaphore, task::JoinSet};
use tracing::{instrument, Instrument};
@ -25,6 +23,8 @@ pub struct DownloadGraphOptions<Reporter> {
pub reqwest: reqwest::Client,
/// The downloads reporter.
pub reporter: Option<Arc<Reporter>>,
/// The refreshed sources.
pub refreshed_sources: RefreshedSources,
/// Whether to skip dev dependencies.
pub prod: bool,
/// Whether to write the downloaded packages to disk.
@ -44,6 +44,7 @@ where
Self {
reqwest,
reporter: None,
refreshed_sources: Default::default(),
prod: false,
write: false,
wally: false,
@ -57,6 +58,12 @@ where
self
}
/// Sets the refreshed sources.
pub fn refreshed_sources(mut self, refreshed_sources: RefreshedSources) -> Self {
self.refreshed_sources = refreshed_sources;
self
}
/// Sets whether to skip dev dependencies.
pub fn prod(mut self, prod: bool) -> Self {
self.prod = prod;
@ -87,6 +94,7 @@ impl<Reporter> Clone for DownloadGraphOptions<Reporter> {
Self {
reqwest: self.reqwest.clone(),
reporter: self.reporter.clone(),
refreshed_sources: self.refreshed_sources.clone(),
prod: self.prod,
write: self.write,
wally: self.wally,
@ -101,7 +109,6 @@ impl Project {
pub async fn download_graph<Reporter>(
&self,
graph: &DependencyGraph,
refreshed_sources: &mut HashSet<PackageSources>,
options: DownloadGraphOptions<Reporter>,
) -> Result<
impl Stream<
@ -118,6 +125,7 @@ impl Project {
let DownloadGraphOptions {
reqwest,
reporter,
refreshed_sources,
prod,
write,
wally,
@ -126,45 +134,32 @@ impl Project {
let manifest = self.deser_manifest().await?;
let manifest_target_kind = manifest.target.kind();
let project = Arc::new(self.clone());
refresh_sources(
self,
graph
.iter()
.flat_map(|(_, versions)| versions.iter())
.map(|(_, node)| node.pkg_ref.source()),
refreshed_sources,
)
.await?;
let mut tasks = JoinSet::<Result<_, errors::DownloadGraphError>>::new();
let semaphore = Arc::new(Semaphore::new(network_concurrency.get()));
for (name, versions) in graph {
for (version_id, node) in versions {
let mut tasks = graph
.iter()
.flat_map(|(name, versions)| {
versions
.iter()
.map(|(version_id, node)| (name.clone(), version_id.clone(), node.clone()))
})
// we need to download pesde packages first, since scripts (for target finding for example) can depend on them
if node.pkg_ref.like_wally() != wally {
continue;
}
let name = name.clone();
let version_id = version_id.clone();
let node = node.clone();
.filter(|(_, _, node)| node.pkg_ref.like_wally() == wally)
.map(|(name, version_id, node)| {
let span = tracing::info_span!(
"download",
name = name.to_string(),
version_id = version_id.to_string()
);
let project = project.clone();
let project = self.clone();
let reqwest = reqwest.clone();
let reporter = reporter.clone();
let refreshed_sources = refreshed_sources.clone();
let package_dir = project.package_dir().to_path_buf();
let semaphore = semaphore.clone();
tasks.spawn(
async move {
let display_name = format!("{name}@{version_id}");
let progress_reporter = reporter
@ -178,6 +173,15 @@ impl Project {
}
let source = node.pkg_ref.source();
refreshed_sources
.refresh(
&source,
&RefreshOptions {
project: project.clone(),
},
)
.await?;
let container_folder = node.container_folder(
&package_dir
.join(manifest_target_kind.packages_folder(version_id.target()))
@ -188,8 +192,6 @@ impl Project {
fs::create_dir_all(&container_folder).await?;
let project = project.clone();
tracing::debug!("downloading");
let (fs, target) = match progress_reporter {
@ -197,15 +199,24 @@ impl Project {
source
.download(
&node.pkg_ref,
&project,
&reqwest,
Arc::new(progress_reporter),
&DownloadOptions {
project: project.clone(),
reqwest,
reporter: Arc::new(progress_reporter),
},
)
.await
}
None => {
source
.download(&node.pkg_ref, &project, &reqwest, Arc::new(()))
.download(
&node.pkg_ref,
&DownloadOptions {
project: project.clone(),
reqwest,
reporter: Arc::new(()),
},
)
.await
}
}
@ -218,19 +229,16 @@ impl Project {
fs.write_to(container_folder, project.cas_dir(), true)
.await?;
} else {
tracing::debug!(
"skipping write to disk, dev dependency in prod mode"
);
tracing::debug!("skipping write to disk, dev dependency in prod mode");
}
}
let downloaded_node = DownloadedDependencyGraphNode { node, target };
Ok((downloaded_node, name, version_id))
}
.instrument(span),
);
}
}
.instrument(span)
})
.collect::<JoinSet<Result<_, errors::DownloadGraphError>>>();
let stream = try_stream! {
while let Some(res) = tasks.join_next().await {
@ -256,7 +264,7 @@ pub mod errors {
/// An error occurred refreshing a package source
#[error("failed to refresh package source")]
RefreshFailed(#[from] Box<crate::source::errors::RefreshError>),
RefreshFailed(#[from] crate::source::errors::RefreshError),
/// Error interacting with the filesystem
#[error("error interacting with the filesystem")]

28
src/download_and_link.rs
View file

@ -3,18 +3,15 @@ use crate::{
lockfile::{DependencyGraph, DownloadedGraph},
manifest::DependencyType,
reporters::DownloadsReporter,
source::PackageSources,
Project,
Project, RefreshedSources,
};
use futures::TryStreamExt;
use std::{
collections::HashSet,
convert::Infallible,
future::{self, Future},
num::NonZeroUsize,
sync::Arc,
};
use tokio::sync::Mutex;
use tracing::{instrument, Instrument};
/// Filters a graph to only include production dependencies, if `prod` is `true`
@ -90,7 +87,7 @@ pub struct DownloadAndLinkOptions<Reporter = (), Hooks = ()> {
/// The download and link hooks.
pub hooks: Option<Arc<Hooks>>,
/// The refreshed sources.
pub refreshed_sources: Arc<Mutex<HashSet<PackageSources>>>,
pub refreshed_sources: RefreshedSources,
/// Whether to skip dev dependencies.
pub prod: bool,
/// Whether to write the downloaded packages to disk.
@ -130,11 +127,8 @@ where
}
/// Sets the refreshed sources.
pub fn refreshed_sources(
mut self,
refreshed_sources: impl Into<Arc<Mutex<HashSet<PackageSources>>>>,
) -> Self {
self.refreshed_sources = refreshed_sources.into();
pub fn refreshed_sources(mut self, refreshed_sources: RefreshedSources) -> Self {
self.refreshed_sources = refreshed_sources;
self
}
@ -196,10 +190,10 @@ impl Project {
let graph = graph.clone();
let reqwest = reqwest.clone();
let mut refreshed_sources = refreshed_sources.lock().await;
let mut downloaded_graph = DownloadedGraph::new();
let mut download_graph_options = DownloadGraphOptions::<Reporter>::new(reqwest.clone())
.refreshed_sources(refreshed_sources.clone())
.prod(prod)
.write(write)
.network_concurrency(network_concurrency);
@ -209,11 +203,7 @@ impl Project {
}
// step 1. download pesde dependencies
self.download_graph(
&graph,
&mut refreshed_sources,
download_graph_options.clone(),
)
self.download_graph(&graph, download_graph_options.clone())
.instrument(tracing::debug_span!("download (pesde)"))
.await?
.try_for_each(|(downloaded_node, name, version_id)| {
@ -246,11 +236,7 @@ impl Project {
}
// step 3. download wally dependencies
self.download_graph(
&graph,
&mut refreshed_sources,
download_graph_options.clone().wally(true),
)
self.download_graph(&graph, download_graph_options.clone().wally(true))
.instrument(tracing::debug_span!("download (wally)"))
.await?
.try_for_each(|(downloaded_node, name, version_id)| {

107
src/lib.rs
View file

@ -6,16 +6,21 @@
use crate::{
lockfile::Lockfile,
manifest::Manifest,
source::{traits::PackageSource, PackageSources},
source::{
traits::{PackageSource, RefreshOptions},
PackageSources,
},
};
use async_stream::stream;
use fs_err::tokio as fs;
use futures::{future::try_join_all, Stream};
use futures::Stream;
use gix::sec::identity::Account;
use std::{
collections::{HashMap, HashSet},
fmt::Debug,
hash::{Hash, Hasher},
path::{Path, PathBuf},
sync::Arc,
};
use tracing::instrument;
use wax::Pattern;
@ -56,13 +61,18 @@ pub(crate) const LINK_LIB_NO_FILE_FOUND: &str = "____pesde_no_export_file_found"
/// The folder in which scripts are linked
pub const SCRIPTS_LINK_FOLDER: &str = ".pesde";
/// Struct containing the authentication configuration
#[derive(Debug, Default, Clone)]
pub struct AuthConfig {
#[derive(Debug, Default)]
struct AuthConfigShared {
tokens: HashMap<gix::Url, String>,
git_credentials: Option<Account>,
}
/// Struct containing the authentication configuration
#[derive(Debug, Clone, Default)]
pub struct AuthConfig {
shared: Arc<AuthConfigShared>,
}
impl AuthConfig {
/// Create a new `AuthConfig`
pub fn new() -> Self {
@ -70,11 +80,12 @@ impl AuthConfig {
}
/// Set the tokens
/// Panics if the `AuthConfig` is shared
pub fn with_tokens<I: IntoIterator<Item = (gix::Url, S)>, S: AsRef<str>>(
mut self,
tokens: I,
) -> Self {
self.tokens = tokens
Arc::get_mut(&mut self.shared).unwrap().tokens = tokens
.into_iter()
.map(|(url, s)| (url, s.as_ref().to_string()))
.collect();
@ -82,25 +93,25 @@ impl AuthConfig {
}
/// Set the git credentials
/// Panics if the `AuthConfig` is shared
pub fn with_git_credentials(mut self, git_credentials: Option<Account>) -> Self {
self.git_credentials = git_credentials;
Arc::get_mut(&mut self.shared).unwrap().git_credentials = git_credentials;
self
}
/// Get the tokens
pub fn tokens(&self) -> &HashMap<gix::Url, String> {
&self.tokens
&self.shared.tokens
}
/// Get the git credentials
pub fn git_credentials(&self) -> Option<&Account> {
self.git_credentials.as_ref()
self.shared.git_credentials.as_ref()
}
}
/// The main struct of the pesde library, representing a project
#[derive(Debug, Clone)]
pub struct Project {
#[derive(Debug)]
struct ProjectShared {
package_dir: PathBuf,
workspace_dir: Option<PathBuf>,
data_dir: PathBuf,
@ -108,6 +119,13 @@ pub struct Project {
cas_dir: PathBuf,
}
/// The main struct of the pesde library, representing a project
/// Unlike `ProjectShared`, this struct is `Send` and `Sync` and is cheap to clone because it is `Arc`-backed
#[derive(Debug, Clone)]
pub struct Project {
shared: Arc<ProjectShared>,
}
impl Project {
/// Create a new `Project`
pub fn new<P: AsRef<Path>, Q: AsRef<Path>, R: AsRef<Path>, S: AsRef<Path>>(
@ -118,43 +136,45 @@ impl Project {
auth_config: AuthConfig,
) -> Self {
Project {
shared: Arc::new(ProjectShared {
package_dir: package_dir.as_ref().to_path_buf(),
workspace_dir: workspace_dir.map(|d| d.as_ref().to_path_buf()),
data_dir: data_dir.as_ref().to_path_buf(),
auth_config,
cas_dir: cas_dir.as_ref().to_path_buf(),
}),
}
}
/// The directory of the package
pub fn package_dir(&self) -> &Path {
&self.package_dir
&self.shared.package_dir
}
/// The directory of the workspace this package belongs to, if any
pub fn workspace_dir(&self) -> Option<&Path> {
self.workspace_dir.as_deref()
self.shared.workspace_dir.as_deref()
}
/// The directory to store general-purpose data
pub fn data_dir(&self) -> &Path {
&self.data_dir
&self.shared.data_dir
}
/// The authentication configuration
pub fn auth_config(&self) -> &AuthConfig {
&self.auth_config
&self.shared.auth_config
}
/// The CAS (content-addressable storage) directory
pub fn cas_dir(&self) -> &Path {
&self.cas_dir
&self.shared.cas_dir
}
/// Read the manifest file
#[instrument(skip(self), ret(level = "trace"), level = "debug")]
pub async fn read_manifest(&self) -> Result<String, errors::ManifestReadError> {
let string = fs::read_to_string(self.package_dir.join(MANIFEST_FILE_NAME)).await?;
let string = fs::read_to_string(self.package_dir().join(MANIFEST_FILE_NAME)).await?;
Ok(string)
}
@ -162,20 +182,24 @@ impl Project {
/// Deserialize the manifest file
#[instrument(skip(self), ret(level = "trace"), level = "debug")]
pub async fn deser_manifest(&self) -> Result<Manifest, errors::ManifestReadError> {
let string = fs::read_to_string(self.package_dir.join(MANIFEST_FILE_NAME)).await?;
let string = fs::read_to_string(self.package_dir().join(MANIFEST_FILE_NAME)).await?;
Ok(toml::from_str(&string)?)
}
/// Write the manifest file
#[instrument(skip(self, manifest), level = "debug")]
pub async fn write_manifest<S: AsRef<[u8]>>(&self, manifest: S) -> Result<(), std::io::Error> {
fs::write(self.package_dir.join(MANIFEST_FILE_NAME), manifest.as_ref()).await
fs::write(
self.package_dir().join(MANIFEST_FILE_NAME),
manifest.as_ref(),
)
.await
}
/// Deserialize the lockfile
#[instrument(skip(self), ret(level = "trace"), level = "debug")]
pub async fn deser_lockfile(&self) -> Result<Lockfile, errors::LockfileReadError> {
let string = fs::read_to_string(self.package_dir.join(LOCKFILE_FILE_NAME)).await?;
let string = fs::read_to_string(self.package_dir().join(LOCKFILE_FILE_NAME)).await?;
Ok(toml::from_str(&string)?)
}
@ -186,7 +210,7 @@ impl Project {
lockfile: &Lockfile,
) -> Result<(), errors::LockfileWriteError> {
let string = toml::to_string(lockfile)?;
fs::write(self.package_dir.join(LOCKFILE_FILE_NAME), string).await?;
fs::write(self.package_dir().join(LOCKFILE_FILE_NAME), string).await?;
Ok(())
}
@ -370,24 +394,35 @@ pub async fn matching_globs<'a, P: AsRef<Path> + Debug, I: IntoIterator<Item = &
Ok(paths)
}
/// Refreshes the sources asynchronously
pub async fn refresh_sources<I: Iterator<Item = PackageSources>>(
project: &Project,
sources: I,
refreshed_sources: &mut HashSet<PackageSources>,
) -> Result<(), Box<source::errors::RefreshError>> {
try_join_all(sources.map(|source| {
let needs_refresh = refreshed_sources.insert(source.clone());
async move {
if needs_refresh {
source.refresh(project).await.map_err(Box::new)
/// A struct containing sources already having been refreshed
#[derive(Debug, Clone, Default)]
pub struct RefreshedSources(Arc<tokio::sync::Mutex<HashSet<u64>>>);
impl RefreshedSources {
/// Create a new empty `RefreshedSources`
pub fn new() -> Self {
RefreshedSources::default()
}
/// Refreshes the source asynchronously if it has not already been refreshed.
/// Will prevent more refreshes of the same source.
pub async fn refresh(
&self,
source: &PackageSources,
options: &RefreshOptions,
) -> Result<(), source::errors::RefreshError> {
let mut hasher = std::hash::DefaultHasher::new();
source.hash(&mut hasher);
let hash = hasher.finish();
let mut refreshed_sources = self.0.lock().await;
if refreshed_sources.insert(hash) {
source.refresh(options).await
} else {
Ok(())
}
}
}))
.await
.map(|_| ())
}
/// Errors that can occur when using the pesde library

2
src/main.rs
View file

@ -332,7 +332,7 @@ async fn run() -> anyhow::Result<()> {
.and_then(|manifest| manifest.pesde_version);
let exe_path = if let Some(version) = target_version {
get_or_download_version(&reqwest, &TagInfo::Incomplete(version), false).await?
get_or_download_version(&reqwest, TagInfo::Incomplete(version), false).await?
} else {
None
};

25
src/patches.rs
View file

@ -9,7 +9,7 @@ use git2::{ApplyLocation, Diff, DiffFormat, DiffLineType, Repository, Signature}
use relative_path::RelativePathBuf;
use std::{path::Path, sync::Arc};
use tokio::task::JoinSet;
use tracing::instrument;
use tracing::{instrument, Instrument};
/// Set up a git repository for patches
pub fn setup_patches_repo<P: AsRef<Path>>(dir: P) -> Result<Repository, git2::Error> {
@ -91,7 +91,6 @@ impl Project {
for (name, versions) in manifest.patches {
for (version_id, patch_path) in versions {
let name = name.clone();
let patch_path = patch_path.to_path(self.package_dir());
let Some(node) = graph
@ -114,11 +113,17 @@ impl Project {
);
let reporter = reporter.clone();
tasks.spawn(async move {
tracing::debug!("applying patch to {name}@{version_id}");
let span = tracing::info_span!(
"apply patch",
name = name.to_string(),
version_id = version_id.to_string()
);
let display_name = format!("{name}@{version_id}");
tasks.spawn(
async move {
tracing::debug!("applying patch");
let progress_reporter = reporter.report_patch(&display_name);
let patch = fs::read(&patch_path)
@ -158,9 +163,7 @@ impl Project {
repo.apply(&patch, ApplyLocation::Both, None)?;
}
tracing::debug!(
"patch applied to {name}@{version_id}, removing .git directory"
);
tracing::debug!("patch applied");
fs::remove_dir_all(container_folder.join(".git"))
.await
@ -169,7 +172,9 @@ impl Project {
progress_reporter.report_done();
Ok(())
});
}
.instrument(span),
);
}
}

61
src/resolver.rs
View file

@ -5,19 +5,19 @@ use crate::{
source::{
pesde::PesdePackageSource,
specifiers::DependencySpecifiers,
traits::{PackageRef, PackageSource},
traits::{PackageRef, PackageSource, RefreshOptions, ResolveOptions},
version_id::VersionId,
PackageSources,
},
Project, DEFAULT_INDEX_NAME,
Project, RefreshedSources, DEFAULT_INDEX_NAME,
};
use std::collections::{btree_map::Entry, HashMap, HashSet, VecDeque};
use std::collections::{btree_map::Entry, HashMap, VecDeque};
use tracing::{instrument, Instrument};
fn insert_node(
graph: &mut DependencyGraph,
name: PackageNames,
version: VersionId,
name: &PackageNames,
version: &VersionId,
mut node: DependencyGraphNode,
is_top_level: bool,
) {
@ -63,7 +63,7 @@ impl Project {
pub async fn dependency_graph(
&self,
previous_graph: Option<&DependencyGraph>,
refreshed_sources: &mut HashSet<PackageSources>,
refreshed_sources: RefreshedSources,
// used by `x` command - if true, specifier indices are expected to be URLs. will not do peer dependency checks
is_published_package: bool,
) -> Result<DependencyGraph, Box<errors::DependencyGraphError>> {
@ -108,8 +108,8 @@ impl Project {
tracing::debug!("resolved {}@{} from old dependency graph", name, version);
insert_node(
&mut graph,
name.clone(),
version.clone(),
name,
version,
DependencyGraphNode {
direct: Some((alias.clone(), specifier.clone(), *source_ty)),
..node.clone()
@ -138,13 +138,7 @@ impl Project {
.and_then(|v| v.get(dep_version))
{
tracing::debug!("resolved sub-dependency {dep_name}@{dep_version}");
insert_node(
&mut graph,
dep_name.clone(),
dep_version.clone(),
dep_node.clone(),
false,
);
insert_node(&mut graph, dep_name, dep_version, dep_node.clone(), false);
dep_node
.dependencies
@ -184,9 +178,13 @@ impl Project {
})
.collect::<VecDeque<_>>();
let refresh_options = RefreshOptions {
project: self.clone(),
};
while let Some((specifier, ty, dependant, path, overridden, target)) = queue.pop_front() {
async {
let alias = path.last().unwrap().clone();
let alias = path.last().unwrap();
let depth = path.len() - 1;
tracing::debug!("resolving {specifier} ({ty:?})");
@ -203,10 +201,7 @@ impl Project {
))?
.clone()
} else {
let index_url = specifier.index.clone().unwrap();
index_url
.clone()
specifier.index.as_deref().unwrap()
.try_into()
// specifiers in indices store the index url in this field
.unwrap()
@ -227,10 +222,7 @@ impl Project {
))?
.clone()
} else {
let index_url = specifier.index.clone().unwrap();
index_url
.clone()
specifier.index.as_deref().unwrap()
.try_into()
// specifiers in indices store the index url in this field
.unwrap()
@ -246,12 +238,19 @@ impl Project {
}
};
if refreshed_sources.insert(source.clone()) {
source.refresh(self).await.map_err(|e| Box::new(e.into()))?;
}
refreshed_sources.refresh(
&source,
&refresh_options,
)
.await
.map_err(|e| Box::new(e.into()))?;
let (name, resolved) = source
.resolve(&specifier, self, target, refreshed_sources)
.resolve(&specifier, &ResolveOptions {
project: self.clone(),
target,
refreshed_sources: refreshed_sources.clone(),
})
.await
.map_err(|e| Box::new(e.into()))?;
@ -341,9 +340,9 @@ impl Project {
};
insert_node(
&mut graph,
name.clone(),
target_version_id.clone(),
node.clone(),
&name,
&target_version_id,
node,
depth == 0,
);

34
src/source/git/mod.rs
View file

@ -10,8 +10,8 @@ use crate::{
git::{pkg_ref::GitPackageRef, specifier::GitDependencySpecifier},
git_index::{read_file, GitBasedSource},
specifiers::DependencySpecifiers,
traits::PackageRef,
PackageSource, PackageSources, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES,
traits::{DownloadOptions, PackageRef, RefreshOptions, ResolveOptions},
PackageSource, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES,
},
util::hash,
Project, DEFAULT_INDEX_NAME, LOCKFILE_FILE_NAME, MANIFEST_FILE_NAME,
@ -20,13 +20,7 @@ use fs_err::tokio as fs;
use futures::future::try_join_all;
use gix::{bstr::BStr, traverse::tree::Recorder, ObjectId, Url};
use relative_path::RelativePathBuf;
use std::{
collections::{BTreeMap, HashSet},
fmt::Debug,
hash::Hash,
path::PathBuf,
sync::Arc,
};
use std::{collections::BTreeMap, fmt::Debug, hash::Hash, path::PathBuf, sync::Arc};
use tokio::{sync::Mutex, task::spawn_blocking};
use tracing::instrument;
@ -44,7 +38,7 @@ pub struct GitPackageSource {
impl GitBasedSource for GitPackageSource {
fn path(&self, project: &Project) -> PathBuf {
project
.data_dir
.data_dir()
.join("git_repos")
.join(hash(self.as_bytes()))
}
@ -73,18 +67,18 @@ impl PackageSource for GitPackageSource {
type DownloadError = errors::DownloadError;
#[instrument(skip_all, level = "debug")]
async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> {
GitBasedSource::refresh(self, project).await
async fn refresh(&self, options: &RefreshOptions) -> Result<(), Self::RefreshError> {
GitBasedSource::refresh(self, options).await
}
#[instrument(skip_all, level = "debug")]
async fn resolve(
&self,
specifier: &Self::Specifier,
project: &Project,
_project_target: TargetKind,
_refreshed_sources: &mut HashSet<PackageSources>,
options: &ResolveOptions,
) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
let ResolveOptions { project, .. } = options;
let repo = gix::open(self.path(project))
.map_err(|e| errors::ResolveError::OpenRepo(Box::new(self.repo_url.clone()), e))?;
let rev = repo
@ -334,15 +328,15 @@ impl PackageSource for GitPackageSource {
}
#[instrument(skip_all, level = "debug")]
async fn download(
async fn download<R: DownloadProgressReporter>(
&self,
pkg_ref: &Self::Ref,
project: &Project,
_reqwest: &reqwest::Client,
_reporter: Arc<impl DownloadProgressReporter>,
options: &DownloadOptions<R>,
) -> Result<(PackageFS, Target), Self::DownloadError> {
let DownloadOptions { project, .. } = options;
let index_file = project
.cas_dir
.cas_dir()
.join("git_index")
.join(hash(self.as_bytes()))
.join(&pkg_ref.tree_id);

30
src/source/git_index.rs
View file

@ -1,6 +1,6 @@
#![allow(async_fn_in_trait)]
use crate::{util::authenticate_conn, Project};
use crate::{source::traits::RefreshOptions, util::authenticate_conn, Project};
use fs_err::tokio as fs;
use gix::remote::Direction;
use std::fmt::Debug;
@ -16,10 +16,13 @@ pub trait GitBasedSource {
fn repo_url(&self) -> &gix::Url;
/// Refreshes the repository
async fn refresh(&self, project: &Project) -> Result<(), errors::RefreshError> {
async fn refresh(&self, options: &RefreshOptions) -> Result<(), errors::RefreshError> {
let RefreshOptions { project } = options;
let path = self.path(project);
let repo_url = self.repo_url().clone();
let auth_config = project.auth_config.clone();
let project = project.clone();
if path.exists() {
spawn_blocking(move || {
@ -47,7 +50,7 @@ pub trait GitBasedSource {
}
};
authenticate_conn(&mut connection, &auth_config);
authenticate_conn(&mut connection, project.auth_config());
let fetch =
match connection.prepare_fetch(gix::progress::Discard, Default::default()) {
@ -80,7 +83,7 @@ pub trait GitBasedSource {
gix::prepare_clone_bare(repo_url.clone(), &path)
.map_err(|e| errors::RefreshError::Clone(repo_url.to_string(), Box::new(e)))?
.configure_connection(move |c| {
authenticate_conn(c, &auth_config);
authenticate_conn(c, project.auth_config());
Ok(())
})
.fetch_only(gix::progress::Discard, &false.into())
@ -94,21 +97,16 @@ pub trait GitBasedSource {
/// Reads a file from a tree
#[instrument(skip(tree), ret, level = "trace")]
pub fn read_file<
I: IntoIterator<Item = P> + Clone + Debug,
P: ToString + PartialEq<gix::bstr::BStr>,
>(
pub fn read_file<I: IntoIterator<Item = P> + Debug, P: ToString + PartialEq<gix::bstr::BStr>>(
tree: &gix::Tree,
file_path: I,
) -> Result<Option<String>, errors::ReadFile> {
let file_path_str = file_path
.clone()
.into_iter()
.map(|s| s.to_string())
.collect::<Vec<_>>()
.join(std::path::MAIN_SEPARATOR_STR);
let mut file_path_str = String::new();
let entry = match tree.lookup_entry(file_path) {
let entry = match tree.lookup_entry(file_path.into_iter().inspect(|path| {
file_path_str.push_str(path.to_string().as_str());
file_path_str.push('/');
})) {
Ok(Some(entry)) => entry,
Ok(None) => return Ok(None),
Err(e) => return Err(errors::ReadFile::Lookup(file_path_str, e)),

87
src/source/mod.rs
View file

@ -1,18 +1,13 @@
use crate::{
manifest::target::{Target, TargetKind},
manifest::target::Target,
names::PackageNames,
reporters::DownloadProgressReporter,
source::{
fs::PackageFS, refs::PackageRefs, specifiers::DependencySpecifiers, traits::*,
version_id::VersionId,
},
Project,
};
use std::{
collections::{BTreeMap, HashSet},
fmt::Debug,
sync::Arc,
};
use std::{collections::BTreeMap, fmt::Debug};
/// Packages' filesystems
pub mod fs;
@ -66,26 +61,33 @@ impl PackageSource for PackageSources {
type ResolveError = errors::ResolveError;
type DownloadError = errors::DownloadError;
async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> {
async fn refresh(&self, options: &RefreshOptions) -> Result<(), Self::RefreshError> {
match self {
PackageSources::Pesde(source) => source.refresh(project).await.map_err(Into::into),
PackageSources::Pesde(source) => source
.refresh(options)
.await
.map_err(Self::RefreshError::Pesde),
#[cfg(feature = "wally-compat")]
PackageSources::Wally(source) => source.refresh(project).await.map_err(Into::into),
PackageSources::Git(source) => source.refresh(project).await.map_err(Into::into),
PackageSources::Workspace(source) => source.refresh(project).await.map_err(Into::into),
PackageSources::Wally(source) => source
.refresh(options)
.await
.map_err(Self::RefreshError::Wally),
PackageSources::Git(source) => source
.refresh(options)
.await
.map_err(Self::RefreshError::Git),
PackageSources::Workspace(source) => source.refresh(options).await.map_err(Into::into),
}
}
async fn resolve(
&self,
specifier: &Self::Specifier,
project: &Project,
project_target: TargetKind,
refreshed_sources: &mut HashSet<PackageSources>,
options: &ResolveOptions,
) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
match (self, specifier) {
(PackageSources::Pesde(source), DependencySpecifiers::Pesde(specifier)) => source
.resolve(specifier, project, project_target, refreshed_sources)
.resolve(specifier, options)
.await
.map(|(name, results)| {
(
@ -100,7 +102,7 @@ impl PackageSource for PackageSources {
#[cfg(feature = "wally-compat")]
(PackageSources::Wally(source), DependencySpecifiers::Wally(specifier)) => source
.resolve(specifier, project, project_target, refreshed_sources)
.resolve(specifier, options)
.await
.map(|(name, results)| {
(
@ -114,7 +116,7 @@ impl PackageSource for PackageSources {
.map_err(Into::into),
(PackageSources::Git(source), DependencySpecifiers::Git(specifier)) => source
.resolve(specifier, project, project_target, refreshed_sources)
.resolve(specifier, options)
.await
.map(|(name, results)| {
(
@ -129,7 +131,7 @@ impl PackageSource for PackageSources {
(PackageSources::Workspace(source), DependencySpecifiers::Workspace(specifier)) => {
source
.resolve(specifier, project, project_target, refreshed_sources)
.resolve(specifier, options)
.await
.map(|(name, results)| {
(
@ -149,34 +151,28 @@ impl PackageSource for PackageSources {
}
}
async fn download(
async fn download<R: DownloadProgressReporter>(
&self,
pkg_ref: &Self::Ref,
project: &Project,
reqwest: &reqwest::Client,
reporter: Arc<impl DownloadProgressReporter>,
options: &DownloadOptions<R>,
) -> Result<(PackageFS, Target), Self::DownloadError> {
match (self, pkg_ref) {
(PackageSources::Pesde(source), PackageRefs::Pesde(pkg_ref)) => source
.download(pkg_ref, project, reqwest, reporter)
.await
.map_err(Into::into),
(PackageSources::Pesde(source), PackageRefs::Pesde(pkg_ref)) => {
source.download(pkg_ref, options).await.map_err(Into::into)
}
#[cfg(feature = "wally-compat")]
(PackageSources::Wally(source), PackageRefs::Wally(pkg_ref)) => source
.download(pkg_ref, project, reqwest, reporter)
.await
.map_err(Into::into),
(PackageSources::Wally(source), PackageRefs::Wally(pkg_ref)) => {
source.download(pkg_ref, options).await.map_err(Into::into)
}
(PackageSources::Git(source), PackageRefs::Git(pkg_ref)) => source
.download(pkg_ref, project, reqwest, reporter)
.await
.map_err(Into::into),
(PackageSources::Git(source), PackageRefs::Git(pkg_ref)) => {
source.download(pkg_ref, options).await.map_err(Into::into)
}
(PackageSources::Workspace(source), PackageRefs::Workspace(pkg_ref)) => source
.download(pkg_ref, project, reqwest, reporter)
.await
.map_err(Into::into),
(PackageSources::Workspace(source), PackageRefs::Workspace(pkg_ref)) => {
source.download(pkg_ref, options).await.map_err(Into::into)
}
_ => Err(errors::DownloadError::Mismatch),
}
@ -191,9 +187,18 @@ pub mod errors {
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum RefreshError {
/// A git-based package source failed to refresh
/// A pesde package source failed to refresh
#[error("error refreshing pesde package source")]
GitBased(#[from] crate::source::git_index::errors::RefreshError),
Pesde(#[source] crate::source::git_index::errors::RefreshError),
/// A Wally package source failed to refresh
#[cfg(feature = "wally-compat")]
#[error("error refreshing wally package source")]
Wally(#[source] crate::source::git_index::errors::RefreshError),
/// A Git package source failed to refresh
#[error("error refreshing git package source")]
Git(#[source] crate::source::git_index::errors::RefreshError),
/// A workspace package source failed to refresh
#[error("error refreshing workspace package source")]

47
src/source/pesde/mod.rs
View file

@ -7,7 +7,6 @@ use std::{
fmt::Debug,
hash::Hash,
path::PathBuf,
sync::Arc,
};
use tokio_util::io::StreamReader;
@ -15,17 +14,14 @@ use pkg_ref::PesdePackageRef;
use specifier::PesdeDependencySpecifier;
use crate::{
manifest::{
target::{Target, TargetKind},
DependencyType,
},
manifest::{target::Target, DependencyType},
names::{PackageName, PackageNames},
reporters::DownloadProgressReporter,
source::{
fs::{store_in_cas, FSEntry, PackageFS},
git_index::{read_file, root_tree, GitBasedSource},
DependencySpecifiers, PackageSource, PackageSources, ResolveResult, VersionId,
IGNORED_DIRS, IGNORED_FILES,
traits::{DownloadOptions, RefreshOptions, ResolveOptions},
DependencySpecifiers, PackageSource, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES,
},
util::hash,
Project,
@ -58,7 +54,10 @@ pub struct ScopeInfo {
impl GitBasedSource for PesdePackageSource {
fn path(&self, project: &Project) -> PathBuf {
project.data_dir.join("indices").join(hash(self.as_bytes()))
project
.data_dir()
.join("indices")
.join(hash(self.as_bytes()))
}
fn repo_url(&self) -> &Url {
@ -105,18 +104,22 @@ impl PackageSource for PesdePackageSource {
type DownloadError = errors::DownloadError;
#[instrument(skip_all, level = "debug")]
async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> {
GitBasedSource::refresh(self, project).await
async fn refresh(&self, options: &RefreshOptions) -> Result<(), Self::RefreshError> {
GitBasedSource::refresh(self, options).await
}
#[instrument(skip_all, level = "debug")]
async fn resolve(
&self,
specifier: &Self::Specifier,
project: &Project,
project_target: TargetKind,
_refreshed_sources: &mut HashSet<PackageSources>,
options: &ResolveOptions,
) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
let ResolveOptions {
project,
target: project_target,
..
} = options;
let (scope, name) = specifier.name.as_str();
let repo = gix::open(self.path(project)).map_err(Box::new)?;
let tree = root_tree(&repo).map_err(Box::new)?;
@ -142,7 +145,7 @@ impl PackageSource for PesdePackageSource {
.into_iter()
.filter(|(VersionId(version, target), _)| {
specifier.version.matches(version)
&& specifier.target.unwrap_or(project_target) == *target
&& specifier.target.unwrap_or(*project_target) == *target
})
.map(|(id, entry)| {
let version = id.version().clone();
@ -163,16 +166,20 @@ impl PackageSource for PesdePackageSource {
}
#[instrument(skip_all, level = "debug")]
async fn download(
async fn download<R: DownloadProgressReporter>(
&self,
pkg_ref: &Self::Ref,
project: &Project,
reqwest: &reqwest::Client,
reporter: Arc<impl DownloadProgressReporter>,
options: &DownloadOptions<R>,
) -> Result<(PackageFS, Target), Self::DownloadError> {
let DownloadOptions {
project,
reporter,
reqwest,
} = options;
let config = self.config(project).await.map_err(Box::new)?;
let index_file = project
.cas_dir
.cas_dir()
.join("index")
.join(pkg_ref.name.escaped())
.join(pkg_ref.version.to_string())
@ -200,7 +207,7 @@ impl PackageSource for PesdePackageSource {
let mut request = reqwest.get(&url).header(ACCEPT, "application/octet-stream");
if let Some(token) = project.auth_config.tokens().get(&self.repo_url) {
if let Some(token) = project.auth_config().tokens().get(&self.repo_url) {
tracing::debug!("using token for {}", self.repo_url);
request = request.header(AUTHORIZATION, token);
}

64
src/source/traits.rs
View file

@ -1,4 +1,3 @@
#![allow(async_fn_in_trait)]
use crate::{
manifest::{
target::{Target, TargetKind},
@ -6,11 +5,12 @@ use crate::{
},
reporters::DownloadProgressReporter,
source::{DependencySpecifiers, PackageFS, PackageSources, ResolveResult},
Project,
Project, RefreshedSources,
};
use std::{
collections::{BTreeMap, HashSet},
collections::BTreeMap,
fmt::{Debug, Display},
future::Future,
sync::Arc,
};
@ -27,6 +27,35 @@ pub trait PackageRef: Debug {
fn source(&self) -> PackageSources;
}
/// Options for refreshing a source
#[derive(Debug, Clone)]
pub struct RefreshOptions {
/// The project to refresh for
pub project: Project,
}
/// Options for resolving a package
#[derive(Debug, Clone)]
pub struct ResolveOptions {
/// The project to resolve for
pub project: Project,
/// The target to resolve for
pub target: TargetKind,
/// The sources that have been refreshed
pub refreshed_sources: RefreshedSources,
}
/// Options for downloading a package
#[derive(Debug, Clone)]
pub struct DownloadOptions<R: DownloadProgressReporter> {
/// The project to download for
pub project: Project,
/// The reqwest client to use
pub reqwest: reqwest::Client,
/// The reporter to use
pub reporter: Arc<R>,
}
/// A source of packages
pub trait PackageSource: Debug {
/// The specifier type for this source
@ -34,32 +63,31 @@ pub trait PackageSource: Debug {
/// The reference type for this source
type Ref: PackageRef;
/// The error type for refreshing this source
type RefreshError: std::error::Error;
type RefreshError: std::error::Error + Send + Sync + 'static;
/// The error type for resolving a package from this source
type ResolveError: std::error::Error;
type ResolveError: std::error::Error + Send + Sync + 'static;
/// The error type for downloading a package from this source
type DownloadError: std::error::Error;
type DownloadError: std::error::Error + Send + Sync + 'static;
/// Refreshes the source
async fn refresh(&self, _project: &Project) -> Result<(), Self::RefreshError> {
Ok(())
fn refresh(
&self,
_options: &RefreshOptions,
) -> impl Future<Output = Result<(), Self::RefreshError>> + Send + Sync {
async { Ok(()) }
}
/// Resolves a specifier to a reference
async fn resolve(
fn resolve(
&self,
specifier: &Self::Specifier,
project: &Project,
project_target: TargetKind,
refreshed_sources: &mut HashSet<PackageSources>,
) -> Result<ResolveResult<Self::Ref>, Self::ResolveError>;
options: &ResolveOptions,
) -> impl Future<Output = Result<ResolveResult<Self::Ref>, Self::ResolveError>>;
/// Downloads a package
async fn download(
fn download<R: DownloadProgressReporter>(
&self,
pkg_ref: &Self::Ref,
project: &Project,
reqwest: &reqwest::Client,
reporter: Arc<impl DownloadProgressReporter>,
) -> Result<(PackageFS, Target), Self::DownloadError>;
options: &DownloadOptions<R>,
) -> impl Future<Output = Result<(PackageFS, Target), Self::DownloadError>>;
}

2
src/source/wally/compat_util.rs
View file

@ -37,7 +37,7 @@ async fn find_lib_path(
let result = execute_script(
ScriptName::SourcemapGenerator,
&script_path.to_path(&project.package_dir),
&script_path.to_path(project.package_dir()),
[package_dir],
project,
true,

74
src/source/wally/mod.rs
View file

@ -5,7 +5,7 @@ use crate::{
source::{
fs::{store_in_cas, FSEntry, PackageFS},
git_index::{read_file, root_tree, GitBasedSource},
traits::PackageSource,
traits::{DownloadOptions, PackageSource, RefreshOptions, ResolveOptions},
version_id::VersionId,
wally::{
compat_util::get_target,
@ -23,11 +23,7 @@ use gix::Url;
use relative_path::RelativePathBuf;
use reqwest::header::AUTHORIZATION;
use serde::Deserialize;
use std::{
collections::{BTreeMap, HashSet},
path::PathBuf,
sync::Arc,
};
use std::{collections::BTreeMap, path::PathBuf, sync::Arc};
use tempfile::tempdir;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
@ -53,7 +49,7 @@ pub struct WallyPackageSource {
impl GitBasedSource for WallyPackageSource {
fn path(&self, project: &Project) -> PathBuf {
project
.data_dir
.data_dir()
.join("wally_indices")
.join(hash(self.as_bytes()))
}
@ -102,18 +98,22 @@ impl PackageSource for WallyPackageSource {
type DownloadError = errors::DownloadError;
#[instrument(skip_all, level = "debug")]
async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> {
GitBasedSource::refresh(self, project).await
async fn refresh(&self, options: &RefreshOptions) -> Result<(), Self::RefreshError> {
GitBasedSource::refresh(self, options).await
}
#[instrument(skip_all, level = "debug")]
async fn resolve(
&self,
specifier: &Self::Specifier,
project: &Project,
project_target: TargetKind,
refreshed_sources: &mut HashSet<PackageSources>,
options: &ResolveOptions,
) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
let ResolveOptions {
project,
refreshed_sources,
..
} = options;
let repo = gix::open(self.path(project)).map_err(Box::new)?;
let tree = root_tree(&repo).map_err(Box::new)?;
let (scope, name) = specifier.name.as_str();
@ -127,23 +127,26 @@ impl PackageSource for WallyPackageSource {
let config = self.config(project).await.map_err(Box::new)?;
for registry in config.fallback_registries {
let source = WallyPackageSource::new(registry.clone());
if refreshed_sources.insert(PackageSources::Wally(source.clone())) {
GitBasedSource::refresh(&source, project)
.await
.map_err(Box::new)?;
}
match Box::pin(source.resolve(
specifier,
project,
project_target,
refreshed_sources,
))
let source = WallyPackageSource::new(registry);
match refreshed_sources
.refresh(
&PackageSources::Wally(source.clone()),
&RefreshOptions {
project: project.clone(),
},
)
.await
{
Ok(()) => {}
Err(super::errors::RefreshError::Wally(e)) => {
return Err(Self::ResolveError::Refresh(Box::new(e)));
}
Err(e) => unreachable!("unexpected error: {e:?}"),
}
match Box::pin(source.resolve(specifier, options)).await {
Ok((name, results)) => {
tracing::debug!("found {} in backup registry {registry}", name);
tracing::debug!("found {name} in backup registry {}", source.repo_url);
return Ok((name, results));
}
Err(errors::ResolveError::NotFound(_)) => {
@ -202,16 +205,20 @@ impl PackageSource for WallyPackageSource {
}
#[instrument(skip_all, level = "debug")]
async fn download(
async fn download<R: DownloadProgressReporter>(
&self,
pkg_ref: &Self::Ref,
project: &Project,
reqwest: &reqwest::Client,
reporter: Arc<impl DownloadProgressReporter>,
options: &DownloadOptions<R>,
) -> Result<(PackageFS, Target), Self::DownloadError> {
let DownloadOptions {
project,
reqwest,
reporter,
} = options;
let config = self.config(project).await.map_err(Box::new)?;
let index_file = project
.cas_dir
.cas_dir()
.join("wally_index")
.join(pkg_ref.name.escaped())
.join(pkg_ref.version.to_string());
@ -250,7 +257,7 @@ impl PackageSource for WallyPackageSource {
.unwrap_or("0.3.2"),
);
if let Some(token) = project.auth_config.tokens().get(&self.repo_url) {
if let Some(token) = project.auth_config().tokens().get(&self.repo_url) {
tracing::debug!("using token for {}", self.repo_url);
request = request.header(AUTHORIZATION, token);
}
@ -291,6 +298,7 @@ impl PackageSource for WallyPackageSource {
let archive = Arc::new(Mutex::new(archive));
// todo: remove this asyncification, since the Mutex makes it sequential anyway
let entries = try_join_all(
entries
.into_iter()
@ -363,7 +371,7 @@ impl PackageSource for WallyPackageSource {
pub struct WallyIndexConfig {
api: url::Url,
#[serde(default, deserialize_with = "crate::util::deserialize_gix_url_vec")]
fallback_registries: Vec<gix::Url>,
fallback_registries: Vec<Url>,
}
/// Errors that can occur when interacting with a Wally package source

73
src/source/workspace/mod.rs
View file

@ -1,21 +1,20 @@
use crate::{
manifest::target::{Target, TargetKind},
manifest::target::Target,
names::PackageNames,
reporters::DownloadProgressReporter,
source::{
fs::PackageFS, specifiers::DependencySpecifiers, traits::PackageSource,
version_id::VersionId, workspace::pkg_ref::WorkspacePackageRef, PackageSources,
fs::PackageFS,
specifiers::DependencySpecifiers,
traits::{DownloadOptions, PackageSource, ResolveOptions},
version_id::VersionId,
workspace::pkg_ref::WorkspacePackageRef,
ResolveResult,
},
Project, DEFAULT_INDEX_NAME,
DEFAULT_INDEX_NAME,
};
use futures::StreamExt;
use relative_path::RelativePathBuf;
use reqwest::Client;
use std::{
collections::{BTreeMap, HashSet},
sync::Arc,
};
use std::collections::BTreeMap;
use tokio::pin;
use tracing::instrument;
@ -35,25 +34,21 @@ impl PackageSource for WorkspacePackageSource {
type ResolveError = errors::ResolveError;
type DownloadError = errors::DownloadError;
async fn refresh(&self, _project: &Project) -> Result<(), Self::RefreshError> {
// no-op
Ok(())
}
#[instrument(skip_all, level = "debug")]
async fn resolve(
&self,
specifier: &Self::Specifier,
project: &Project,
project_target: TargetKind,
_refreshed_sources: &mut HashSet<PackageSources>,
options: &ResolveOptions,
) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
let ResolveOptions {
project,
target: project_target,
..
} = options;
let (path, manifest) = 'finder: {
let workspace_dir = project
.workspace_dir
.as_ref()
.unwrap_or(&project.package_dir);
let target = specifier.target.unwrap_or(project_target);
let workspace_dir = project.workspace_dir().unwrap_or(project.package_dir());
let target = specifier.target.unwrap_or(*project_target);
let members = project.workspace_members(workspace_dir, true).await?;
pin!(members);
@ -70,17 +65,13 @@ impl PackageSource for WorkspacePackageSource {
));
};
Ok((
PackageNames::Pesde(manifest.name.clone()),
BTreeMap::from([(
VersionId::new(manifest.version.clone(), manifest.target.kind()),
WorkspacePackageRef {
let manifest_target_kind = manifest.target.kind();
let pkg_ref = WorkspacePackageRef {
// workspace_dir is guaranteed to be Some by the workspace_members method
// strip_prefix is guaranteed to be Some by same method
// from_path is guaranteed to be Ok because we just stripped the absolute path
path: RelativePathBuf::from_path(
path.strip_prefix(project.workspace_dir.clone().unwrap())
.unwrap(),
path.strip_prefix(project.workspace_dir().unwrap()).unwrap(),
)
.unwrap(),
dependencies: manifest
@ -89,8 +80,7 @@ impl PackageSource for WorkspacePackageSource {
.map(|(alias, (mut spec, ty))| {
match &mut spec {
DependencySpecifiers::Pesde(spec) => {
let index_name =
spec.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME);
let index_name = spec.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME);
spec.index = Some(
manifest
@ -105,8 +95,7 @@ impl PackageSource for WorkspacePackageSource {
}
#[cfg(feature = "wally-compat")]
DependencySpecifiers::Wally(spec) => {
let index_name =
spec.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME);
let index_name = spec.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME);
spec.index = Some(
manifest
@ -127,20 +116,26 @@ impl PackageSource for WorkspacePackageSource {
})
.collect::<Result<_, errors::ResolveError>>()?,
target: manifest.target,
},
};
Ok((
PackageNames::Pesde(manifest.name),
BTreeMap::from([(
VersionId::new(manifest.version, manifest_target_kind),
pkg_ref,
)]),
))
}
#[instrument(skip_all, level = "debug")]
async fn download(
async fn download<R: DownloadProgressReporter>(
&self,
pkg_ref: &Self::Ref,
project: &Project,
_reqwest: &Client,
_reporter: Arc<impl DownloadProgressReporter>,
options: &DownloadOptions<R>,
) -> Result<(PackageFS, Target), Self::DownloadError> {
let path = pkg_ref.path.to_path(project.workspace_dir.clone().unwrap());
let DownloadOptions { project, .. } = options;
let path = pkg_ref.path.to_path(project.workspace_dir().unwrap());
Ok((
PackageFS::Copy(path, pkg_ref.target.kind()),