From de35d5906aa4432bec4e8b6d506bd3c90440d6bd Mon Sep 17 00:00:00 2001 From: daimond113 <72147841+daimond113@users.noreply.github.com> Date: Sun, 31 Mar 2024 14:23:08 +0200 Subject: [PATCH] feat: support manifest-less repos & running local package bin export --- Cargo.toml | 6 +- src/cli/mod.rs | 2 +- src/cli/root.rs | 75 ++++++++--------- src/dependencies/git.rs | 143 +++++++++++++++++++++++---------- src/dependencies/mod.rs | 27 +++++-- src/dependencies/resolution.rs | 15 +++- src/manifest.rs | 68 +++++++++++----- src/project.rs | 12 ++- 8 files changed, 235 insertions(+), 113 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2b5e055..0494d17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,8 +10,8 @@ repository = "https://github.com/daimond113/pesde" include = ["src/**/*", "Cargo.toml", "Cargo.lock", "README.md", "LICENSE", "CHANGELOG.md"] [features] -bin = ["clap", "directories", "keyring", "anyhow", "ignore", "pretty_env_logger", "serde_json", "reqwest/json", "reqwest/multipart", "lune", "futures-executor", "indicatif", "auth-git2", "indicatif-log-bridge", "inquire", "once_cell"] -wally = ["toml", "zip", "serde_json"] +bin = ["clap", "directories", "keyring", "anyhow", "ignore", "pretty_env_logger", "reqwest/json", "reqwest/multipart", "lune", "futures-executor", "indicatif", "auth-git2", "indicatif-log-bridge", "inquire", "once_cell"] +wally = ["toml", "zip"] [[bin]] name = "pesde" @@ -21,6 +21,7 @@ required-features = ["bin"] [dependencies] serde = { version = "1.0.197", features = ["derive"] } serde_yaml = "0.9.33" +serde_json = "1.0.114" git2 = "0.18.3" semver = { version = "1.0.22", features = ["serde"] } reqwest = { version = "0.12.1", default-features = false, features = ["rustls-tls", "blocking"] } @@ -47,7 +48,6 @@ keyring = { version = "2.3.2", optional = true } anyhow = { version = "1.0.81", optional = true } ignore = { version = "0.4.22", optional = true } pretty_env_logger = { version = "0.5.0", optional = true } -serde_json = { version = "1.0.114", optional = true } lune = { version = "0.8.2", optional = true } futures-executor = { version = "0.3.30", optional = true } indicatif = { version = "0.17.8", optional = true } diff --git a/src/cli/mod.rs b/src/cli/mod.rs index f799f26..df1fafd 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -90,7 +90,7 @@ pub enum Command { Run { /// The package to run #[clap(value_name = "PACKAGE")] - package: StandardPackageName, + package: Option, /// The arguments to pass to the package #[clap(last = true)] diff --git a/src/cli/root.rs b/src/cli/root.rs index e3d1b4f..b3612de 100644 --- a/src/cli/root.rs +++ b/src/cli/root.rs @@ -2,7 +2,7 @@ use cfg_if::cfg_if; use chrono::Utc; use std::{ collections::{BTreeMap, HashMap}, - fs::{create_dir_all, read, remove_dir_all, write, File}, + fs::{create_dir_all, read, remove_dir_all, write}, str::FromStr, time::Duration, }; @@ -105,7 +105,7 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { multithreaded_bar( download_job, - lockfile.children.len() as u64, + lockfile.children.values().map(|v| v.len() as u64).sum(), "Downloading packages".to_string(), )?; @@ -144,36 +144,47 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { )?; } Command::Run { package, args } => { - let lockfile = project - .lockfile()? - .ok_or(anyhow::anyhow!("lockfile not found"))?; + let bin_path = if let Some(package) = package { + let lockfile = project + .lockfile()? + .ok_or(anyhow::anyhow!("lockfile not found"))?; - let resolved_pkg = lockfile - .children - .get(&package.into()) - .and_then(|versions| { - versions - .values() - .find(|pkg_ref| lockfile.root_specifier(pkg_ref).is_some()) - }) - .ok_or(anyhow::anyhow!( - "package not found in lockfile (or isn't root)" - ))?; + let resolved_pkg = lockfile + .children + .get(&package.clone().into()) + .and_then(|versions| { + versions + .values() + .find(|pkg_ref| lockfile.root_specifier(pkg_ref).is_some()) + }) + .ok_or(anyhow::anyhow!( + "package not found in lockfile (or isn't root)" + ))?; - let pkg_path = resolved_pkg.directory(project.path()).1; - let manifest = Manifest::from_path(&pkg_path)?; + let pkg_path = resolved_pkg.directory(project.path()).1; + let manifest = Manifest::from_path(&pkg_path)?; - let Some(bin_path) = manifest.exports.bin else { - anyhow::bail!("no bin found in package"); + let Some(bin_path) = manifest.exports.bin else { + anyhow::bail!("no bin found in package"); + }; + + bin_path.to_path(pkg_path) + } else { + let manifest = project.manifest(); + let bin_path = manifest + .exports + .bin + .clone() + .ok_or(anyhow::anyhow!("no bin found in package"))?; + + bin_path.to_path(project.path()) }; - let absolute_bin_path = bin_path.to_path(pkg_path); - let mut runtime = Runtime::new().with_args(args); block_on(runtime.run( - resolved_pkg.pkg_ref.name().to_string(), - &read(absolute_bin_path)?, + bin_path.with_extension("").display().to_string(), + &read(bin_path)?, ))?; } Command::Search { query } => { @@ -347,9 +358,7 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { ); } Command::Init => { - let manifest_path = CWD.join(MANIFEST_FILE_NAME); - - if manifest_path.exists() { + if CWD.join(MANIFEST_FILE_NAME).exists() { anyhow::bail!("manifest already exists"); } @@ -427,7 +436,7 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { repository: none_if_empty!(repository), }; - serde_yaml::to_writer(File::create(manifest_path)?, &manifest)?; + manifest.write(CWD.to_path_buf())?; } Command::Add { package, @@ -484,10 +493,7 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { insert_into(&mut manifest.dependencies, specifier, package.0.clone()); } - serde_yaml::to_writer( - File::create(project.path().join(MANIFEST_FILE_NAME))?, - &manifest, - )?; + manifest.write(CWD.to_path_buf())? } Command::Remove { package } => { let mut manifest = project.manifest().clone(); @@ -520,10 +526,7 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { }); } - serde_yaml::to_writer( - File::create(project.path().join(MANIFEST_FILE_NAME))?, - &manifest, - )?; + manifest.write(project.path())? } Command::Outdated => { let project = Lazy::force_mut(&mut project); diff --git a/src/dependencies/git.rs b/src/dependencies/git.rs index 71ef4b7..33bf6ce 100644 --- a/src/dependencies/git.rs +++ b/src/dependencies/git.rs @@ -1,4 +1,9 @@ -use std::{fs::create_dir_all, path::Path, sync::Arc}; +use std::{ + fs::create_dir_all, + hash::{DefaultHasher, Hash, Hasher}, + path::Path, + sync::Arc, +}; use git2::{build::RepoBuilder, Repository}; use log::{debug, error, warn}; @@ -9,7 +14,7 @@ use url::Url; use crate::{ index::{remote_callbacks, CredentialsFn}, - manifest::{Manifest, ManifestConvertError, Realm}, + manifest::{update_sync_tool_files, Manifest, ManifestConvertError, Realm}, package_name::StandardPackageName, project::{get_index, Indices}, }; @@ -60,10 +65,87 @@ pub enum GitDownloadError { #[error("invalid URL")] InvalidUrl(#[from] url::ParseError), - /// An error that occurred because the manifest is not present in the git repository, and the wally feature is not enabled - #[cfg(not(feature = "wally"))] - #[error("wally feature is not enabled, but the manifest is not present in the git repository")] - ManifestNotPresent, + /// An error that occurred while resolving a git dependency's manifest + #[error("error resolving git dependency manifest")] + Resolve(#[from] GitManifestResolveError), +} + +/// An error that occurred while resolving a git dependency's manifest +#[derive(Debug, Error)] +pub enum GitManifestResolveError { + /// An error that occurred because the scope and name could not be extracted from the URL + #[error("could not extract scope and name from URL: {0}")] + ScopeAndNameFromUrl(Url), + + /// An error that occurred because the package name is invalid + #[error("invalid package name")] + InvalidPackageName(#[from] crate::package_name::StandardPackageNameValidationError), + + /// An error that occurred while interacting with the file system + #[error("error interacting with the file system")] + Io(#[from] std::io::Error), +} + +fn to_snake_case(s: &str) -> String { + s.chars() + .enumerate() + .map(|(i, c)| { + if c.is_uppercase() { + format!("{}{}", if i == 0 { "" } else { "_" }, c.to_lowercase()) + } else if c == '-' { + "_".to_string() + } else { + c.to_string() + } + }) + .collect() +} + +pub(crate) fn manifest(path: &Path, url: &Url) -> Result { + Manifest::from_path_or_convert(path).or_else(|_| { + let (scope, name) = url + .path_segments() + .and_then(|mut s| { + let scope = s.next(); + let name = s.next(); + + if let (Some(scope), Some(name)) = (scope, name) { + Some((scope.to_string(), name.to_string())) + } else { + None + } + }) + .ok_or_else(|| GitManifestResolveError::ScopeAndNameFromUrl(url.clone()))?; + + let manifest = Manifest { + name: StandardPackageName::new( + &to_snake_case(&scope), + &to_snake_case(name.trim_end_matches(".git")), + )?, + version: Version::new(0, 1, 0), + description: None, + license: None, + authors: None, + repository: None, + exports: Default::default(), + path_style: Default::default(), + private: true, + realm: None, + indices: Default::default(), + #[cfg(feature = "wally")] + sourcemap_generator: None, + overrides: Default::default(), + + dependencies: Default::default(), + peer_dependencies: Default::default(), + }; + + manifest.write(path).unwrap(); + + update_sync_tool_files(path, manifest.name.name().to_string())?; + + Ok(manifest) + }) } impl GitDependencySpecifier { @@ -75,41 +157,22 @@ impl GitDependencySpecifier { debug!("resolving git dependency {}", self.repo); // should also work with ssh urls - let is_url = self.repo.contains(':'); - - let repo_name = if !is_url { - self.repo.to_string() - } else { - let parts: Vec<&str> = self.repo.split('/').collect(); - format!( - "{}/{}", - parts[parts.len() - 2], - parts[parts.len() - 1].trim_end_matches(".git") - ) - }; - - if is_url { - debug!("resolved git repository name to: {}", &repo_name); - } else { - debug!("assuming git repository is a name: {}", &repo_name); - } - - let repo_url = if !is_url { - Url::parse(&format!("https://github.com/{}.git", &self.repo)) - } else { + let repo_url = if self.repo.contains(':') { + debug!("resolved git repository name to: {}", self.repo); Url::parse(&self.repo) + } else { + debug!("assuming git repository is a name: {}", self.repo); + Url::parse(&format!("https://github.com/{}.git", &self.repo)) }?; - if is_url { - debug!("assuming git repository is a url: {}", &repo_url); - } else { - debug!("resolved git repository url to: {}", &repo_url); - } + debug!("resolved git repository url to: {}", &repo_url); - let dest = cache_dir - .join("git") - .join(repo_name.replace('/', "_")) - .join(&self.rev); + let mut hasher = DefaultHasher::new(); + repo_url.hash(&mut hasher); + self.rev.hash(&mut hasher); + let repo_hash = hasher.finish(); + + let dest = cache_dir.join("git").join(repo_hash.to_string()); let repo = if !dest.exists() { create_dir_all(&dest)?; @@ -129,11 +192,7 @@ impl GitDependencySpecifier { repo.reset(&obj, git2::ResetType::Hard, None)?; - Ok(( - Manifest::from_path_or_convert(dest)?, - repo_url, - obj.id().to_string(), - )) + Ok((manifest(&dest, &repo_url)?, repo_url, obj.id().to_string())) } } diff --git a/src/dependencies/mod.rs b/src/dependencies/mod.rs index e6fdaa9..449b717 100644 --- a/src/dependencies/mod.rs +++ b/src/dependencies/mod.rs @@ -21,7 +21,7 @@ use crate::{ resolution::RootLockfileNode, }, index::{CredentialsFn, Index}, - manifest::{Manifest, Realm}, + manifest::{ManifestWriteError, Realm}, multithread::MultithreadedJob, package_name::PackageName, project::{get_index, get_index_by_url, InstallProjectError, Project}, @@ -251,6 +251,10 @@ pub enum ConvertManifestsError { #[error("error converting the manifest")] Manifest(#[from] crate::manifest::ManifestConvertError), + /// An error that occurred while converting a git dependency's manifest + #[error("error converting a git dependency's manifest")] + Git(#[from] crate::dependencies::git::GitManifestResolveError), + /// An error that occurred while reading the sourcemap #[error("error reading the sourcemap")] Sourcemap(#[from] std::io::Error), @@ -262,7 +266,7 @@ pub enum ConvertManifestsError { /// An error that occurred while writing the manifest #[error("error writing the manifest")] - Write(#[from] serde_yaml::Error), + Write(#[from] ManifestWriteError), /// A manifest is not present in a dependency, and the wally feature is not enabled #[cfg(not(feature = "wally"))] @@ -338,7 +342,12 @@ impl Project { _ => continue, }; - let mut manifest = Manifest::from_path_or_convert(&source)?; + let mut manifest = match &resolved_package.pkg_ref { + PackageRef::Git(git) => { + crate::dependencies::git::manifest(&source, &git.repo_url)? + } + _ => crate::manifest::Manifest::from_path_or_convert(&source)?, + }; generate_sourcemap(source.to_path_buf()); @@ -359,10 +368,7 @@ impl Project { }) .or_else(|| Some(relative_path::RelativePathBuf::from("true"))); - serde_yaml::to_writer( - &std::fs::File::create(&source.join(crate::MANIFEST_FILE_NAME))?, - &manifest, - )?; + manifest.write(&source)?; } } @@ -383,7 +389,12 @@ impl Project { _ => continue, }; - if Manifest::from_path_or_convert(&source).is_err() { + if match &resolved_package.pkg_ref { + PackageRef::Git(git) => { + crate::dependencies::git::manifest(&source, &git.repo_url).is_err() + } + _ => crate::manifest::Manifest::from_path_or_convert(&source).is_err(), + } { return Err(ConvertManifestsError::ManifestNotPresent); } } diff --git a/src/dependencies/resolution.rs b/src/dependencies/resolution.rs index d13c165..461a7ee 100644 --- a/src/dependencies/resolution.rs +++ b/src/dependencies/resolution.rs @@ -17,7 +17,7 @@ use crate::{ }, index::{Index, IndexFileEntry, IndexPackageError}, manifest::{DependencyType, Manifest, OverrideKey, Realm}, - package_name::PackageName, + package_name::{PackageName, StandardPackageName}, project::{get_index, get_index_by_url, Project, ReadLockfileError}, DEV_PACKAGES_FOLDER, INDEX_FOLDER, PACKAGES_FOLDER, SERVER_PACKAGES_FOLDER, }; @@ -26,9 +26,12 @@ use crate::{ pub type PackageMap = BTreeMap>; /// The root node of the dependency graph -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash, Default)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] #[serde(deny_unknown_fields)] pub struct RootLockfileNode { + /// The name of the package + pub name: StandardPackageName, + /// Dependency overrides #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub overrides: BTreeMap, @@ -214,6 +217,10 @@ impl Manifest { project: &Project, ) -> Result, ResolveError> { Ok(if let Some(old_root) = project.lockfile()? { + if self.name != old_root.name && locked { + return Err(ResolveError::OutOfDateLockfile); + } + if self.overrides != old_root.overrides { // TODO: resolve only the changed dependencies (will this be worth it?) debug!("overrides have changed, resolving all dependencies"); @@ -331,8 +338,10 @@ impl Manifest { debug!("resolving dependency graph for project {}", self.name); // try to reuse versions (according to semver specifiers) to decrease the amount of downloads and storage let mut root = RootLockfileNode { + name: self.name.clone(), overrides: self.overrides.clone(), - ..Default::default() + specifiers: Default::default(), + children: Default::default(), }; let missing_dependencies = self.missing_dependencies(&mut root, locked, project)?; diff --git a/src/manifest.rs b/src/manifest.rs index 8f46dc3..33a37b7 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, fmt::Display, fs::read, str::FromStr}; +use std::{collections::BTreeMap, fmt::Display, fs::read, path::Path, str::FromStr}; use cfg_if::cfg_if; use relative_path::RelativePathBuf; @@ -177,12 +177,13 @@ pub struct Manifest { #[serde(default)] pub private: bool, /// The realm of the package + #[serde(default, skip_serializing_if = "Option::is_none")] pub realm: Option, /// Indices of the package pub indices: BTreeMap, /// The command to generate a `sourcemap.json` #[cfg(feature = "wally")] - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub sourcemap_generator: Option, /// Dependency overrides #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] @@ -229,18 +230,9 @@ cfg_if! { #[error("error interacting with the file system")] Io(#[from] std::io::Error), - /// An error that occurred while making a package name from a string - #[error("error making a package name from a string")] - PackageName( - #[from] - crate::package_name::FromStrPackageNameParseError< - crate::package_name::StandardPackageNameValidationError, - >, - ), - /// An error that occurred while writing the manifest #[error("error writing the manifest")] - ManifestWrite(#[from] serde_yaml::Error), + ManifestWrite(#[from] crate::manifest::ManifestWriteError), /// An error that occurred while parsing the dependencies #[error("error parsing the dependencies")] @@ -252,6 +244,18 @@ cfg_if! { } } +/// An error that occurred while writing the manifest +#[derive(Debug, Error)] +pub enum ManifestWriteError { + /// An error that occurred while interacting with the file system + #[error("error interacting with the file system")] + Io(#[from] std::io::Error), + + /// An error that occurred while serializing the manifest + #[error("error serializing manifest")] + ManifestSer(#[from] serde_yaml::Error), +} + /// The type of dependency #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] #[serde(rename_all = "snake_case")] @@ -263,6 +267,25 @@ pub enum DependencyType { Peer, } +pub(crate) fn update_sync_tool_files(project_path: &Path, name: String) -> std::io::Result<()> { + if let Ok(file) = std::fs::File::open(project_path.join("default.project.json")) { + let mut project: serde_json::Value = serde_json::from_reader(file)?; + + if project["name"].as_str() == Some(&name) { + return Ok(()); + } + + project["name"] = serde_json::Value::String(name); + + serde_json::to_writer_pretty( + std::fs::File::create(project_path.join("default.project.json"))?, + &project, + )?; + } + + Ok(()) +} + impl Manifest { /// Reads a manifest from a path (if the path is a directory, it will look for the manifest file inside it, otherwise it will read the file directly) pub fn from_path>(path: P) -> Result { @@ -318,7 +341,7 @@ impl Manifest { } let manifest = Self { - name: wally_manifest.package.name.replace('-', "_").parse()?, + name: wally_manifest.package.name.into(), version: wally_manifest.package.version, exports: Exports { lib: Some(RelativePathBuf::from("true")), @@ -326,10 +349,7 @@ impl Manifest { }, path_style: PathStyle::Roblox { place }, private: wally_manifest.package.private.unwrap_or(false), - realm: wally_manifest - .package - .realm - .map(|r| r.parse().unwrap_or(Realm::Shared)), + realm: wally_manifest.package.realm, indices: BTreeMap::from([( crate::project::DEFAULT_INDEX_NAME.to_string(), "".to_string(), @@ -345,8 +365,9 @@ impl Manifest { repository: None, }; - let manifest_path = dir_path.join(MANIFEST_FILE_NAME); - serde_yaml::to_writer(std::fs::File::create(manifest_path)?, &manifest)?; + manifest.write(&dir_path)?; + + update_sync_tool_files(&dir_path, manifest.name.name().to_string())?; Ok(manifest) }) @@ -382,4 +403,13 @@ impl Manifest { ) .collect() } + + /// Writes the manifest to a path + pub fn write>(&self, to: P) -> Result<(), ManifestWriteError> { + let manifest_path = to.as_ref().join(MANIFEST_FILE_NAME); + + serde_yaml::to_writer(std::fs::File::create(manifest_path)?, self)?; + + Ok(()) + } } diff --git a/src/project.rs b/src/project.rs index 7a8a9b1..5c50763 100644 --- a/src/project.rs +++ b/src/project.rs @@ -13,7 +13,7 @@ use crate::{ dependencies::{resolution::RootLockfileNode, DownloadError, UrlResolveError}, index::Index, linking_file::LinkingDependenciesError, - manifest::{Manifest, ManifestReadError}, + manifest::{update_sync_tool_files, Manifest, ManifestReadError}, LOCKFILE_FILE_NAME, }; @@ -124,6 +124,10 @@ pub enum InstallProjectError { /// An error that occurred while resolving the url of a package #[error("failed to resolve package URL")] UrlResolve(#[from] UrlResolveError), + + /// An error that occurred while reading the lockfile + #[error("failed to read lockfile")] + ReadLockfile(#[from] ReadLockfileError), } /// The name of the default index to use @@ -289,6 +293,8 @@ impl Project { /// Downloads the project's dependencies, applies patches, and links the dependencies pub fn install(&mut self, install_options: InstallOptions) -> Result<(), InstallProjectError> { + let old_lockfile = self.lockfile()?; + let lockfile = match install_options.lockfile { Some(map) => map, None => { @@ -311,6 +317,10 @@ impl Project { .map_err(InstallProjectError::LockfileSer)?; } + if !old_lockfile.is_some_and(|old| old.name == lockfile.name) { + update_sync_tool_files(self.path(), lockfile.name.name().to_string())?; + } + Ok(()) } }