feat: support fallback wally registries

CHANGELOG.md

@@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Support fallback Wally registries by @daimond113
+
 ### Fixed
 - Fix peer dependencies being resolved incorrectly by @daimond113
 

src/cli/commands/add.rs

@@ -1,4 +1,4 @@
-use std::str::FromStr;
+use std::{collections::HashSet, str::FromStr};
 
 use anyhow::Context;
 use clap::Args;
@@ -133,7 +133,12 @@ impl AddCommand {
             .context("failed to refresh package source")?;
 
         let Some(version_id) = source
-            .resolve(&specifier, &project, manifest.target.kind())
+            .resolve(
+                &specifier,
+                &project,
+                manifest.target.kind(),
+                &mut HashSet::new(),
+            )
             .await
             .context("failed to resolve package")?
             .1

src/cli/commands/execute.rs

@@ -13,7 +13,7 @@ use pesde::{
     Project,
 };
 use semver::VersionReq;
-use std::{env::current_dir, ffi::OsString, io::Write, process::Command};
+use std::{collections::HashSet, env::current_dir, ffi::OsString, io::Write, process::Command};
 
 #[derive(Debug, Args)]
 pub struct ExecuteCommand {
@@ -53,7 +53,7 @@ impl ExecuteCommand {
             };
 
             if let Some(res) = source
-                .resolve(&specifier, &project, TargetKind::Lune)
+                .resolve(&specifier, &project, TargetKind::Lune, &mut HashSet::new())
                 .await
                 .context("failed to resolve package")?
                 .1
@@ -63,7 +63,7 @@ impl ExecuteCommand {
             }
 
             source
-                .resolve(&specifier, &project, TargetKind::Luau)
+                .resolve(&specifier, &project, TargetKind::Luau, &mut HashSet::new())
                 .await
                 .context("failed to resolve package")?
                 .1

src/cli/commands/outdated.rs

@@ -1,11 +1,7 @@
-use std::collections::HashSet;
+use crate::cli::up_to_date_lockfile;
-
 use anyhow::Context;
 use clap::Args;
 use futures::future::try_join_all;
-use semver::VersionReq;
-
-use crate::cli::up_to_date_lockfile;
 use pesde::{
     refresh_sources,
     source::{
@@ -15,6 +11,9 @@ use pesde::{
     },
     Project,
 };
+use semver::VersionReq;
+use std::{collections::HashSet, sync::Arc};
+use tokio::sync::Mutex;
 
 #[derive(Debug, Args)]
 pub struct OutdatedCommand {
@@ -53,12 +52,15 @@ impl OutdatedCommand {
         )
         .await?;
 
+        let refreshed_sources = Arc::new(Mutex::new(refreshed_sources));
+
         try_join_all(
             graph
                 .into_iter()
                 .flat_map(|(_, versions)| versions.into_iter())
                 .map(|(current_version_id, node)| {
                     let project = project.clone();
+                    let refreshed_sources = refreshed_sources.clone();
                     async move {
                         let Some((alias, mut specifier, _)) = node.node.direct else {
                             return Ok::<(), anyhow::Error>(());
@@ -88,7 +90,12 @@ impl OutdatedCommand {
                         }
 
                         let version_id = source
-                            .resolve(&specifier, &project, manifest_target_kind)
+                            .resolve(
+                                &specifier,
+                                &project,
+                                manifest_target_kind,
+                                &mut *refreshed_sources.lock().await,
+                            )
                             .await
                             .context("failed to resolve package versions")?
                             .1

src/cli/commands/publish.rs

@@ -21,7 +21,7 @@ use pesde::{
 };
 use reqwest::{header::AUTHORIZATION, StatusCode};
 use semver::VersionReq;
-use std::path::Component;
+use std::{collections::HashSet, path::Component};
 use tempfile::Builder;
 use tokio::io::{AsyncSeekExt, AsyncWriteExt};
 
@@ -334,7 +334,7 @@ impl PublishCommand {
                 }
                 DependencySpecifiers::Workspace(spec) => {
                     let pkg_ref = WorkspacePackageSource
-                        .resolve(spec, project, target_kind)
+                        .resolve(spec, project, target_kind, &mut HashSet::new())
                         .await
                         .context("failed to resolve workspace package")?
                         .1

src/resolver.rs

@@ -198,7 +198,7 @@ impl Project {
             }
 
             let (name, resolved) = source
-                .resolve(&specifier, self, target)
+                .resolve(&specifier, self, target, refreshed_sources)
                 .await
                 .map_err(|e| Box::new(e.into()))?;
 

src/source/git/mod.rs

@@ -1,7 +1,3 @@
-use gix::{bstr::BStr, traverse::tree::Recorder, ObjectId, Url};
-use relative_path::RelativePathBuf;
-use std::{collections::BTreeMap, fmt::Debug, hash::Hash, path::PathBuf, sync::Arc};
-
 use crate::{
     manifest::{
         target::{Target, TargetKind},
@@ -14,13 +10,22 @@ use crate::{
         git_index::{read_file, GitBasedSource},
         specifiers::DependencySpecifiers,
         traits::PackageRef,
-        PackageSource, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES,
+        PackageSource, PackageSources, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES,
     },
     util::hash,
     Project, DEFAULT_INDEX_NAME, LOCKFILE_FILE_NAME, MANIFEST_FILE_NAME,
 };
 use fs_err::tokio as fs;
 use futures::future::try_join_all;
+use gix::{bstr::BStr, traverse::tree::Recorder, ObjectId, Url};
+use relative_path::RelativePathBuf;
+use std::{
+    collections::{BTreeMap, HashSet},
+    fmt::Debug,
+    hash::Hash,
+    path::PathBuf,
+    sync::Arc,
+};
 use tokio::{sync::Mutex, task::spawn_blocking};
 
 /// The Git package reference
@@ -74,6 +79,7 @@ impl PackageSource for GitPackageSource {
         specifier: &Self::Specifier,
         project: &Project,
         _project_target: TargetKind,
+        _refreshed_sources: &mut HashSet<PackageSources>,
     ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
         let repo = gix::open(self.path(project))
             .map_err(|e| errors::ResolveError::OpenRepo(Box::new(self.repo_url.clone()), e))?;

src/source/mod.rs

@@ -1,5 +1,3 @@
-use std::{collections::BTreeMap, fmt::Debug};
-
 use crate::{
     manifest::target::{Target, TargetKind},
     names::PackageNames,
@@ -9,6 +7,10 @@ use crate::{
     },
     Project,
 };
+use std::{
+    collections::{BTreeMap, HashSet},
+    fmt::Debug,
+};
 
 /// Packages' filesystems
 pub mod fs;
@@ -76,11 +78,12 @@ impl PackageSource for PackageSources {
         &self,
         specifier: &Self::Specifier,
         project: &Project,
-        package_target: TargetKind,
+        project_target: TargetKind,
+        refreshed_sources: &mut HashSet<PackageSources>,
     ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
         match (self, specifier) {
             (PackageSources::Pesde(source), DependencySpecifiers::Pesde(specifier)) => source
-                .resolve(specifier, project, package_target)
+                .resolve(specifier, project, project_target, refreshed_sources)
                 .await
                 .map(|(name, results)| {
                     (
@@ -95,7 +98,7 @@ impl PackageSource for PackageSources {
 
             #[cfg(feature = "wally-compat")]
             (PackageSources::Wally(source), DependencySpecifiers::Wally(specifier)) => source
-                .resolve(specifier, project, package_target)
+                .resolve(specifier, project, project_target, refreshed_sources)
                 .await
                 .map(|(name, results)| {
                     (
@@ -109,7 +112,7 @@ impl PackageSource for PackageSources {
                 .map_err(Into::into),
 
             (PackageSources::Git(source), DependencySpecifiers::Git(specifier)) => source
-                .resolve(specifier, project, package_target)
+                .resolve(specifier, project, project_target, refreshed_sources)
                 .await
                 .map(|(name, results)| {
                     (
@@ -124,7 +127,7 @@ impl PackageSource for PackageSources {
 
             (PackageSources::Workspace(source), DependencySpecifiers::Workspace(specifier)) => {
                 source
-                    .resolve(specifier, project, package_target)
+                    .resolve(specifier, project, project_target, refreshed_sources)
                     .await
                     .map(|(name, results)| {
                         (

src/source/pesde/mod.rs

@@ -1,14 +1,13 @@
-use std::{
-    collections::{BTreeMap, BTreeSet},
-    fmt::Debug,
-    hash::Hash,
-    path::PathBuf,
-};
-
 use gix::Url;
 use relative_path::RelativePathBuf;
 use reqwest::header::{ACCEPT, AUTHORIZATION};
 use serde::{Deserialize, Serialize};
+use std::{
+    collections::{BTreeMap, BTreeSet, HashSet},
+    fmt::Debug,
+    hash::Hash,
+    path::PathBuf,
+};
 
 use pkg_ref::PesdePackageRef;
 use specifier::PesdeDependencySpecifier;
@@ -22,7 +21,8 @@ use crate::{
     source::{
         fs::{store_in_cas, FSEntry, PackageFS},
         git_index::{read_file, root_tree, GitBasedSource},
-        DependencySpecifiers, PackageSource, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES,
+        DependencySpecifiers, PackageSource, PackageSources, ResolveResult, VersionId,
+        IGNORED_DIRS, IGNORED_FILES,
     },
     util::hash,
     Project,
@@ -115,7 +115,8 @@ impl PackageSource for PesdePackageSource {
         &self,
         specifier: &Self::Specifier,
         project: &Project,
-        package_target: TargetKind,
+        project_target: TargetKind,
+        _refreshed_sources: &mut HashSet<PackageSources>,
     ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
         let (scope, name) = specifier.name.as_str();
         let repo = gix::open(self.path(project)).map_err(Box::new)?;
@@ -142,7 +143,7 @@ impl PackageSource for PesdePackageSource {
                 .into_iter()
                 .filter(|(VersionId(version, target), _)| {
                     specifier.version.matches(version)
-                        && specifier.target.unwrap_or(package_target) == *target
+                        && specifier.target.unwrap_or(project_target) == *target
                 })
                 .map(|(id, entry)| {
                     let version = id.version().clone();

src/source/traits.rs

@@ -1,9 +1,4 @@
 #![allow(async_fn_in_trait)]
-use std::{
-    collections::BTreeMap,
-    fmt::{Debug, Display},
-};
-
 use crate::{
     manifest::{
         target::{Target, TargetKind},
@@ -12,6 +7,10 @@ use crate::{
     source::{DependencySpecifiers, PackageFS, PackageSources, ResolveResult},
     Project,
 };
+use std::{
+    collections::{BTreeMap, HashSet},
+    fmt::{Debug, Display},
+};
 
 /// A specifier for a dependency
 pub trait DependencySpecifier: Debug + Display {}
@@ -50,6 +49,7 @@ pub trait PackageSource: Debug {
         specifier: &Self::Specifier,
         project: &Project,
         project_target: TargetKind,
+        refreshed_sources: &mut HashSet<PackageSources>,
     ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError>;
 
     /// Downloads a package

src/source/wally/mod.rs

@@ -11,7 +11,7 @@ use crate::{
             manifest::{Realm, WallyManifest},
             pkg_ref::WallyPackageRef,
         },
-        IGNORED_DIRS, IGNORED_FILES,
+        PackageSources, ResolveResult, IGNORED_DIRS, IGNORED_FILES,
     },
     util::hash,
     Project,
@@ -22,7 +22,11 @@ use gix::Url;
 use relative_path::RelativePathBuf;
 use reqwest::header::AUTHORIZATION;
 use serde::Deserialize;
-use std::{collections::BTreeMap, path::PathBuf, sync::Arc};
+use std::{
+    collections::{BTreeMap, HashSet},
+    path::PathBuf,
+    sync::Arc,
+};
 use tempfile::tempdir;
 use tokio::{io::AsyncWriteExt, sync::Mutex, task::spawn_blocking};
 use tokio_util::compat::FuturesAsyncReadCompatExt;
@@ -98,14 +102,52 @@ impl PackageSource for WallyPackageSource {
         &self,
         specifier: &Self::Specifier,
         project: &Project,
-        _package_target: TargetKind,
+        project_target: TargetKind,
-    ) -> Result<crate::source::ResolveResult<Self::Ref>, Self::ResolveError> {
+        refreshed_sources: &mut HashSet<PackageSources>,
+    ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
         let repo = gix::open(self.path(project)).map_err(Box::new)?;
         let tree = root_tree(&repo).map_err(Box::new)?;
         let (scope, name) = specifier.name.as_str();
         let string = match read_file(&tree, [scope, name]) {
             Ok(Some(s)) => s,
-            Ok(None) => return Err(Self::ResolveError::NotFound(specifier.name.to_string())),
+            Ok(None) => {
+                log::debug!(
+                    "{} not found in wally registry. searching in backup registries",
+                    specifier.name
+                );
+
+                let config = self.config(project).await.map_err(Box::new)?;
+                for registry in config.fallback_registries {
+                    let source = WallyPackageSource::new(registry.clone());
+                    if refreshed_sources.insert(PackageSources::Wally(source.clone())) {
+                        GitBasedSource::refresh(&source, project)
+                            .await
+                            .map_err(Box::new)?;
+                    }
+
+                    match Box::pin(source.resolve(
+                        specifier,
+                        project,
+                        project_target,
+                        refreshed_sources,
+                    ))
+                    .await
+                    {
+                        Ok((name, results)) => {
+                            log::debug!("found {} in backup registry {registry}", name);
+                            return Ok((name, results));
+                        }
+                        Err(errors::ResolveError::NotFound(_)) => {
+                            continue;
+                        }
+                        Err(e) => {
+                            return Err(e);
+                        }
+                    }
+                }
+
+                return Err(Self::ResolveError::NotFound(specifier.name.to_string()));
+            }
             Err(e) => {
                 return Err(Self::ResolveError::Read(
                     specifier.name.to_string(),
@@ -289,6 +331,8 @@ impl PackageSource for WallyPackageSource {
 #[derive(Debug, Clone, Deserialize)]
 pub struct WallyIndexConfig {
     api: url::Url,
+    #[serde(default, deserialize_with = "crate::util::deserialize_gix_url_vec")]
+    fallback_registries: Vec<gix::Url>,
 }
 
 /// Errors that can occur when interacting with a Wally package source
@@ -327,6 +371,18 @@ pub mod errors {
             String,
             #[source] crate::manifest::errors::AllDependenciesError,
         ),
+
+        /// Error reading config file
+        #[error("error reading config file")]
+        Config(#[from] Box<ConfigError>),
+
+        /// Error refreshing backup registry source
+        #[error("error refreshing backup registry source")]
+        Refresh(#[from] Box<crate::source::git_index::errors::RefreshError>),
+
+        /// Error resolving package in backup registries
+        #[error("error resolving package in backup registries")]
+        BackupResolve(#[from] Box<ResolveError>),
     }
 
     /// Errors that can occur when reading the config file for a Wally package source

src/source/workspace/mod.rs

@@ -3,14 +3,15 @@ use crate::{
     names::PackageNames,
     source::{
         fs::PackageFS, specifiers::DependencySpecifiers, traits::PackageSource,
-        version_id::VersionId, workspace::pkg_ref::WorkspacePackageRef, ResolveResult,
+        version_id::VersionId, workspace::pkg_ref::WorkspacePackageRef, PackageSources,
+        ResolveResult,
     },
     Project, DEFAULT_INDEX_NAME,
 };
 use futures::StreamExt;
 use relative_path::RelativePathBuf;
 use reqwest::Client;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashSet};
 use tokio::pin;
 
 /// The workspace package reference
@@ -38,14 +39,15 @@ impl PackageSource for WorkspacePackageSource {
         &self,
         specifier: &Self::Specifier,
         project: &Project,
-        package_target: TargetKind,
+        project_target: TargetKind,
+        _refreshed_sources: &mut HashSet<PackageSources>,
     ) -> Result<ResolveResult<Self::Ref>, Self::ResolveError> {
         let (path, manifest) = 'finder: {
             let workspace_dir = project
                 .workspace_dir
                 .as_ref()
                 .unwrap_or(&project.package_dir);
-            let target = specifier.target.unwrap_or(package_target);
+            let target = specifier.target.unwrap_or(project_target);
 
             let members = project.workspace_members(workspace_dir).await?;
             pin!(members);

src/util.rs

@@ -61,6 +61,15 @@ pub fn deserialize_gix_url_map<'de, D: Deserializer<'de>>(
         .collect()
 }
 
+pub fn deserialize_gix_url_vec<'de, D: Deserializer<'de>>(
+    deserializer: D,
+) -> Result<Vec<gix::Url>, D::Error> {
+    Vec::<String>::deserialize(deserializer)?
+        .into_iter()
+        .map(|v| gix::Url::from_bytes(BStr::new(&v)).map_err(serde::de::Error::custom))
+        .collect()
+}
+
 pub fn deserialize_git_like_url<'de, D: Deserializer<'de>>(
     deserializer: D,
 ) -> Result<gix::Url, D::Error> {