From bb92a06d64fca612f5cd8f8fb157c943ffe50869 Mon Sep 17 00:00:00 2001 From: daimond113 <72147841+daimond113@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:40:20 +0100 Subject: [PATCH] feat: support fallback wally registries --- CHANGELOG.md | 3 ++ src/cli/commands/add.rs | 9 +++-- src/cli/commands/execute.rs | 6 ++-- src/cli/commands/outdated.rs | 19 +++++++---- src/cli/commands/publish.rs | 4 +-- src/resolver.rs | 2 +- src/source/git/mod.rs | 16 ++++++--- src/source/mod.rs | 17 ++++++---- src/source/pesde/mod.rs | 21 ++++++------ src/source/traits.rs | 10 +++--- src/source/wally/mod.rs | 66 +++++++++++++++++++++++++++++++++--- src/source/workspace/mod.rs | 10 +++--- src/util.rs | 9 +++++ 13 files changed, 142 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 699d394..0b034a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Support fallback Wally registries by @daimond113 + ### Fixed - Fix peer dependencies being resolved incorrectly by @daimond113 diff --git a/src/cli/commands/add.rs b/src/cli/commands/add.rs index e1c2b61..91bb585 100644 --- a/src/cli/commands/add.rs +++ b/src/cli/commands/add.rs @@ -1,4 +1,4 @@ -use std::str::FromStr; +use std::{collections::HashSet, str::FromStr}; use anyhow::Context; use clap::Args; @@ -133,7 +133,12 @@ impl AddCommand { .context("failed to refresh package source")?; let Some(version_id) = source - .resolve(&specifier, &project, manifest.target.kind()) + .resolve( + &specifier, + &project, + manifest.target.kind(), + &mut HashSet::new(), + ) .await .context("failed to resolve package")? .1 diff --git a/src/cli/commands/execute.rs b/src/cli/commands/execute.rs index 443d0b4..93c41fe 100644 --- a/src/cli/commands/execute.rs +++ b/src/cli/commands/execute.rs @@ -13,7 +13,7 @@ use pesde::{ Project, }; use semver::VersionReq; -use std::{env::current_dir, ffi::OsString, io::Write, process::Command}; +use std::{collections::HashSet, env::current_dir, ffi::OsString, io::Write, process::Command}; #[derive(Debug, Args)] pub struct ExecuteCommand { @@ -53,7 +53,7 @@ impl ExecuteCommand { }; if let Some(res) = source - .resolve(&specifier, &project, TargetKind::Lune) + .resolve(&specifier, &project, TargetKind::Lune, &mut HashSet::new()) .await .context("failed to resolve package")? .1 @@ -63,7 +63,7 @@ impl ExecuteCommand { } source - .resolve(&specifier, &project, TargetKind::Luau) + .resolve(&specifier, &project, TargetKind::Luau, &mut HashSet::new()) .await .context("failed to resolve package")? .1 diff --git a/src/cli/commands/outdated.rs b/src/cli/commands/outdated.rs index 828493b..9d37ece 100644 --- a/src/cli/commands/outdated.rs +++ b/src/cli/commands/outdated.rs @@ -1,11 +1,7 @@ -use std::collections::HashSet; - +use crate::cli::up_to_date_lockfile; use anyhow::Context; use clap::Args; use futures::future::try_join_all; -use semver::VersionReq; - -use crate::cli::up_to_date_lockfile; use pesde::{ refresh_sources, source::{ @@ -15,6 +11,9 @@ use pesde::{ }, Project, }; +use semver::VersionReq; +use std::{collections::HashSet, sync::Arc}; +use tokio::sync::Mutex; #[derive(Debug, Args)] pub struct OutdatedCommand { @@ -53,12 +52,15 @@ impl OutdatedCommand { ) .await?; + let refreshed_sources = Arc::new(Mutex::new(refreshed_sources)); + try_join_all( graph .into_iter() .flat_map(|(_, versions)| versions.into_iter()) .map(|(current_version_id, node)| { let project = project.clone(); + let refreshed_sources = refreshed_sources.clone(); async move { let Some((alias, mut specifier, _)) = node.node.direct else { return Ok::<(), anyhow::Error>(()); @@ -88,7 +90,12 @@ impl OutdatedCommand { } let version_id = source - .resolve(&specifier, &project, manifest_target_kind) + .resolve( + &specifier, + &project, + manifest_target_kind, + &mut *refreshed_sources.lock().await, + ) .await .context("failed to resolve package versions")? .1 diff --git a/src/cli/commands/publish.rs b/src/cli/commands/publish.rs index 57872c8..a493ae8 100644 --- a/src/cli/commands/publish.rs +++ b/src/cli/commands/publish.rs @@ -21,7 +21,7 @@ use pesde::{ }; use reqwest::{header::AUTHORIZATION, StatusCode}; use semver::VersionReq; -use std::path::Component; +use std::{collections::HashSet, path::Component}; use tempfile::Builder; use tokio::io::{AsyncSeekExt, AsyncWriteExt}; @@ -334,7 +334,7 @@ impl PublishCommand { } DependencySpecifiers::Workspace(spec) => { let pkg_ref = WorkspacePackageSource - .resolve(spec, project, target_kind) + .resolve(spec, project, target_kind, &mut HashSet::new()) .await .context("failed to resolve workspace package")? .1 diff --git a/src/resolver.rs b/src/resolver.rs index c11c392..b9ac644 100644 --- a/src/resolver.rs +++ b/src/resolver.rs @@ -198,7 +198,7 @@ impl Project { } let (name, resolved) = source - .resolve(&specifier, self, target) + .resolve(&specifier, self, target, refreshed_sources) .await .map_err(|e| Box::new(e.into()))?; diff --git a/src/source/git/mod.rs b/src/source/git/mod.rs index 413bdba..82f1302 100644 --- a/src/source/git/mod.rs +++ b/src/source/git/mod.rs @@ -1,7 +1,3 @@ -use gix::{bstr::BStr, traverse::tree::Recorder, ObjectId, Url}; -use relative_path::RelativePathBuf; -use std::{collections::BTreeMap, fmt::Debug, hash::Hash, path::PathBuf, sync::Arc}; - use crate::{ manifest::{ target::{Target, TargetKind}, @@ -14,13 +10,22 @@ use crate::{ git_index::{read_file, GitBasedSource}, specifiers::DependencySpecifiers, traits::PackageRef, - PackageSource, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES, + PackageSource, PackageSources, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES, }, util::hash, Project, DEFAULT_INDEX_NAME, LOCKFILE_FILE_NAME, MANIFEST_FILE_NAME, }; use fs_err::tokio as fs; use futures::future::try_join_all; +use gix::{bstr::BStr, traverse::tree::Recorder, ObjectId, Url}; +use relative_path::RelativePathBuf; +use std::{ + collections::{BTreeMap, HashSet}, + fmt::Debug, + hash::Hash, + path::PathBuf, + sync::Arc, +}; use tokio::{sync::Mutex, task::spawn_blocking}; /// The Git package reference @@ -74,6 +79,7 @@ impl PackageSource for GitPackageSource { specifier: &Self::Specifier, project: &Project, _project_target: TargetKind, + _refreshed_sources: &mut HashSet, ) -> Result, Self::ResolveError> { let repo = gix::open(self.path(project)) .map_err(|e| errors::ResolveError::OpenRepo(Box::new(self.repo_url.clone()), e))?; diff --git a/src/source/mod.rs b/src/source/mod.rs index 3294731..e02dd10 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -1,5 +1,3 @@ -use std::{collections::BTreeMap, fmt::Debug}; - use crate::{ manifest::target::{Target, TargetKind}, names::PackageNames, @@ -9,6 +7,10 @@ use crate::{ }, Project, }; +use std::{ + collections::{BTreeMap, HashSet}, + fmt::Debug, +}; /// Packages' filesystems pub mod fs; @@ -76,11 +78,12 @@ impl PackageSource for PackageSources { &self, specifier: &Self::Specifier, project: &Project, - package_target: TargetKind, + project_target: TargetKind, + refreshed_sources: &mut HashSet, ) -> Result, Self::ResolveError> { match (self, specifier) { (PackageSources::Pesde(source), DependencySpecifiers::Pesde(specifier)) => source - .resolve(specifier, project, package_target) + .resolve(specifier, project, project_target, refreshed_sources) .await .map(|(name, results)| { ( @@ -95,7 +98,7 @@ impl PackageSource for PackageSources { #[cfg(feature = "wally-compat")] (PackageSources::Wally(source), DependencySpecifiers::Wally(specifier)) => source - .resolve(specifier, project, package_target) + .resolve(specifier, project, project_target, refreshed_sources) .await .map(|(name, results)| { ( @@ -109,7 +112,7 @@ impl PackageSource for PackageSources { .map_err(Into::into), (PackageSources::Git(source), DependencySpecifiers::Git(specifier)) => source - .resolve(specifier, project, package_target) + .resolve(specifier, project, project_target, refreshed_sources) .await .map(|(name, results)| { ( @@ -124,7 +127,7 @@ impl PackageSource for PackageSources { (PackageSources::Workspace(source), DependencySpecifiers::Workspace(specifier)) => { source - .resolve(specifier, project, package_target) + .resolve(specifier, project, project_target, refreshed_sources) .await .map(|(name, results)| { ( diff --git a/src/source/pesde/mod.rs b/src/source/pesde/mod.rs index 512ae31..e8c4667 100644 --- a/src/source/pesde/mod.rs +++ b/src/source/pesde/mod.rs @@ -1,14 +1,13 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - fmt::Debug, - hash::Hash, - path::PathBuf, -}; - use gix::Url; use relative_path::RelativePathBuf; use reqwest::header::{ACCEPT, AUTHORIZATION}; use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, BTreeSet, HashSet}, + fmt::Debug, + hash::Hash, + path::PathBuf, +}; use pkg_ref::PesdePackageRef; use specifier::PesdeDependencySpecifier; @@ -22,7 +21,8 @@ use crate::{ source::{ fs::{store_in_cas, FSEntry, PackageFS}, git_index::{read_file, root_tree, GitBasedSource}, - DependencySpecifiers, PackageSource, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES, + DependencySpecifiers, PackageSource, PackageSources, ResolveResult, VersionId, + IGNORED_DIRS, IGNORED_FILES, }, util::hash, Project, @@ -115,7 +115,8 @@ impl PackageSource for PesdePackageSource { &self, specifier: &Self::Specifier, project: &Project, - package_target: TargetKind, + project_target: TargetKind, + _refreshed_sources: &mut HashSet, ) -> Result, Self::ResolveError> { let (scope, name) = specifier.name.as_str(); let repo = gix::open(self.path(project)).map_err(Box::new)?; @@ -142,7 +143,7 @@ impl PackageSource for PesdePackageSource { .into_iter() .filter(|(VersionId(version, target), _)| { specifier.version.matches(version) - && specifier.target.unwrap_or(package_target) == *target + && specifier.target.unwrap_or(project_target) == *target }) .map(|(id, entry)| { let version = id.version().clone(); diff --git a/src/source/traits.rs b/src/source/traits.rs index 4dc9bd9..53fd066 100644 --- a/src/source/traits.rs +++ b/src/source/traits.rs @@ -1,9 +1,4 @@ #![allow(async_fn_in_trait)] -use std::{ - collections::BTreeMap, - fmt::{Debug, Display}, -}; - use crate::{ manifest::{ target::{Target, TargetKind}, @@ -12,6 +7,10 @@ use crate::{ source::{DependencySpecifiers, PackageFS, PackageSources, ResolveResult}, Project, }; +use std::{ + collections::{BTreeMap, HashSet}, + fmt::{Debug, Display}, +}; /// A specifier for a dependency pub trait DependencySpecifier: Debug + Display {} @@ -50,6 +49,7 @@ pub trait PackageSource: Debug { specifier: &Self::Specifier, project: &Project, project_target: TargetKind, + refreshed_sources: &mut HashSet, ) -> Result, Self::ResolveError>; /// Downloads a package diff --git a/src/source/wally/mod.rs b/src/source/wally/mod.rs index f366985..7cfb931 100644 --- a/src/source/wally/mod.rs +++ b/src/source/wally/mod.rs @@ -11,7 +11,7 @@ use crate::{ manifest::{Realm, WallyManifest}, pkg_ref::WallyPackageRef, }, - IGNORED_DIRS, IGNORED_FILES, + PackageSources, ResolveResult, IGNORED_DIRS, IGNORED_FILES, }, util::hash, Project, @@ -22,7 +22,11 @@ use gix::Url; use relative_path::RelativePathBuf; use reqwest::header::AUTHORIZATION; use serde::Deserialize; -use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; +use std::{ + collections::{BTreeMap, HashSet}, + path::PathBuf, + sync::Arc, +}; use tempfile::tempdir; use tokio::{io::AsyncWriteExt, sync::Mutex, task::spawn_blocking}; use tokio_util::compat::FuturesAsyncReadCompatExt; @@ -98,14 +102,52 @@ impl PackageSource for WallyPackageSource { &self, specifier: &Self::Specifier, project: &Project, - _package_target: TargetKind, - ) -> Result, Self::ResolveError> { + project_target: TargetKind, + refreshed_sources: &mut HashSet, + ) -> Result, Self::ResolveError> { let repo = gix::open(self.path(project)).map_err(Box::new)?; let tree = root_tree(&repo).map_err(Box::new)?; let (scope, name) = specifier.name.as_str(); let string = match read_file(&tree, [scope, name]) { Ok(Some(s)) => s, - Ok(None) => return Err(Self::ResolveError::NotFound(specifier.name.to_string())), + Ok(None) => { + log::debug!( + "{} not found in wally registry. searching in backup registries", + specifier.name + ); + + let config = self.config(project).await.map_err(Box::new)?; + for registry in config.fallback_registries { + let source = WallyPackageSource::new(registry.clone()); + if refreshed_sources.insert(PackageSources::Wally(source.clone())) { + GitBasedSource::refresh(&source, project) + .await + .map_err(Box::new)?; + } + + match Box::pin(source.resolve( + specifier, + project, + project_target, + refreshed_sources, + )) + .await + { + Ok((name, results)) => { + log::debug!("found {} in backup registry {registry}", name); + return Ok((name, results)); + } + Err(errors::ResolveError::NotFound(_)) => { + continue; + } + Err(e) => { + return Err(e); + } + } + } + + return Err(Self::ResolveError::NotFound(specifier.name.to_string())); + } Err(e) => { return Err(Self::ResolveError::Read( specifier.name.to_string(), @@ -289,6 +331,8 @@ impl PackageSource for WallyPackageSource { #[derive(Debug, Clone, Deserialize)] pub struct WallyIndexConfig { api: url::Url, + #[serde(default, deserialize_with = "crate::util::deserialize_gix_url_vec")] + fallback_registries: Vec, } /// Errors that can occur when interacting with a Wally package source @@ -327,6 +371,18 @@ pub mod errors { String, #[source] crate::manifest::errors::AllDependenciesError, ), + + /// Error reading config file + #[error("error reading config file")] + Config(#[from] Box), + + /// Error refreshing backup registry source + #[error("error refreshing backup registry source")] + Refresh(#[from] Box), + + /// Error resolving package in backup registries + #[error("error resolving package in backup registries")] + BackupResolve(#[from] Box), } /// Errors that can occur when reading the config file for a Wally package source diff --git a/src/source/workspace/mod.rs b/src/source/workspace/mod.rs index c421e4d..9fbf976 100644 --- a/src/source/workspace/mod.rs +++ b/src/source/workspace/mod.rs @@ -3,14 +3,15 @@ use crate::{ names::PackageNames, source::{ fs::PackageFS, specifiers::DependencySpecifiers, traits::PackageSource, - version_id::VersionId, workspace::pkg_ref::WorkspacePackageRef, ResolveResult, + version_id::VersionId, workspace::pkg_ref::WorkspacePackageRef, PackageSources, + ResolveResult, }, Project, DEFAULT_INDEX_NAME, }; use futures::StreamExt; use relative_path::RelativePathBuf; use reqwest::Client; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use tokio::pin; /// The workspace package reference @@ -38,14 +39,15 @@ impl PackageSource for WorkspacePackageSource { &self, specifier: &Self::Specifier, project: &Project, - package_target: TargetKind, + project_target: TargetKind, + _refreshed_sources: &mut HashSet, ) -> Result, Self::ResolveError> { let (path, manifest) = 'finder: { let workspace_dir = project .workspace_dir .as_ref() .unwrap_or(&project.package_dir); - let target = specifier.target.unwrap_or(package_target); + let target = specifier.target.unwrap_or(project_target); let members = project.workspace_members(workspace_dir).await?; pin!(members); diff --git a/src/util.rs b/src/util.rs index d6bdd39..7a5e28f 100644 --- a/src/util.rs +++ b/src/util.rs @@ -61,6 +61,15 @@ pub fn deserialize_gix_url_map<'de, D: Deserializer<'de>>( .collect() } +pub fn deserialize_gix_url_vec<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + Vec::::deserialize(deserializer)? + .into_iter() + .map(|v| gix::Url::from_bytes(BStr::new(&v)).map_err(serde::de::Error::custom)) + .collect() +} + pub fn deserialize_git_like_url<'de, D: Deserializer<'de>>( deserializer: D, ) -> Result {