pesde-pkg/pesde
1
1
Fork 0
mirror of https://github.com/pesde-pkg/pesde.git synced 2024-12-12 11:00:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

feat: content addressable storage

Browse source
This commit is contained in:
daimond113 2024-07-28 18:19:54 +02:00
parent d8cd78e7e2
commit 37cc86f028
No known key found for this signature in database
GPG key ID: 3A8ECE51328B513C
36 changed files with 574 additions and 311 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -2736,6 +2736,7 @@ dependencies = [
"serde",
"serde_json",
"serde_with",
"sha2",
"tar",
"thiserror",
"threadpool",

1
Cargo.toml
View file

@ -61,6 +61,7 @@ url = { version = "2.5.2", features = ["serde"] }
# TODO: reevaluate whether to use this
# secrecy = "0.8.0"
chrono = { version = "0.4.38", features = ["serde"] }
sha2 = "0.10.8"
# TODO: remove this when gitoxide adds support for: committing, pushing, adding
git2 = { version = "0.19.0", optional = true }

11
src/cli/auth.rs
View file

@ -2,16 +2,15 @@ use crate::cli::config::{read_config, write_config};
use anyhow::Context;
use keyring::Entry;
use serde::Deserialize;
use std::path::Path;
pub fn get_token<P: AsRef<Path>>(data_dir: P) -> anyhow::Result<Option<String>> {
pub fn get_token() -> anyhow::Result<Option<String>> {
match std::env::var("PESDE_TOKEN") {
Ok(token) => return Ok(Some(token)),
Err(std::env::VarError::NotPresent) => {}
Err(e) => return Err(e.into()),
}
let config = read_config(data_dir)?;
let config = read_config()?;
if let Some(token) = config.token {
return Ok(Some(token));
}
@ -29,7 +28,7 @@ pub fn get_token<P: AsRef<Path>>(data_dir: P) -> anyhow::Result<Option<String>>
Ok(None)
}
pub fn set_token<P: AsRef<Path>>(data_dir: P, token: Option<&str>) -> anyhow::Result<()> {
pub fn set_token(token: Option<&str>) -> anyhow::Result<()> {
let entry = match Entry::new("token", env!("CARGO_PKG_NAME")) {
Ok(entry) => entry,
Err(e) => return Err(e.into()),
@ -47,9 +46,9 @@ pub fn set_token<P: AsRef<Path>>(data_dir: P, token: Option<&str>) -> anyhow::Re
Err(e) => return Err(e.into()),
}
let mut config = read_config(&data_dir)?;
let mut config = read_config()?;
config.token = token.map(|s| s.to_string());
write_config(data_dir, &config)?;
write_config(&config)?;
Ok(())
}

6
src/cli/commands/auth/login.rs
View file

@ -7,7 +7,7 @@ use clap::Args;
use colored::Colorize;
use pesde::{
errors::ManifestReadError,
source::{pesde::PesdePackageSource, PackageSource},
source::{pesde::PesdePackageSource, traits::PackageSource},
Project,
};
use serde::Deserialize;
@ -71,7 +71,7 @@ impl LoginCommand {
},
None => match manifest {
Some(_) => None,
None => Some(read_config(project.data_dir())?.default_index),
None => Some(read_config()?.default_index),
},
};
@ -182,7 +182,7 @@ impl LoginCommand {
println!("logged in as {}", get_token_login(&reqwest, &token)?.bold());
set_token(project.data_dir(), Some(&token))?;
set_token(Some(&token))?;
Ok(())
}

5
src/cli/commands/auth/logout.rs
View file

@ -1,13 +1,12 @@
use crate::cli::auth::set_token;
use clap::Args;
use pesde::Project;
#[derive(Debug, Args)]
pub struct LogoutCommand {}
impl LogoutCommand {
pub fn run(self, project: Project) -> anyhow::Result<()> {
set_token(project.data_dir(), None)?;
pub fn run(self) -> anyhow::Result<()> {
set_token(None)?;
println!("logged out");

4
src/cli/commands/auth/mod.rs
View file

@ -20,8 +20,8 @@ impl AuthCommands {
pub fn run(self, project: Project, reqwest: reqwest::blocking::Client) -> anyhow::Result<()> {
match self {
AuthCommands::Login(login) => login.run(project, reqwest),
AuthCommands::Logout(logout) => logout.run(project),
AuthCommands::WhoAmI(whoami) => whoami.run(project, reqwest),
AuthCommands::Logout(logout) => logout.run(),
AuthCommands::WhoAmI(whoami) => whoami.run(reqwest),
}
}
}

5
src/cli/commands/auth/whoami.rs
View file

@ -1,14 +1,13 @@
use crate::cli::{auth::get_token_login, get_token};
use clap::Args;
use colored::Colorize;
use pesde::Project;
#[derive(Debug, Args)]
pub struct WhoAmICommand {}
impl WhoAmICommand {
pub fn run(self, project: Project, reqwest: reqwest::blocking::Client) -> anyhow::Result<()> {
let token = match get_token(project.data_dir())? {
pub fn run(self, reqwest: reqwest::blocking::Client) -> anyhow::Result<()> {
let token = match get_token()? {
Some(token) => token,
None => {
println!("not logged in");

7
src/cli/commands/config/default_index.rs
View file

@ -1,6 +1,5 @@
use crate::cli::config::{read_config, write_config, CliConfig};
use clap::Args;
use pesde::Project;
#[derive(Debug, Args)]
pub struct DefaultIndexCommand {
@ -14,8 +13,8 @@ pub struct DefaultIndexCommand {
}
impl DefaultIndexCommand {
pub fn run(self, project: Project) -> anyhow::Result<()> {
let mut config = read_config(project.data_dir())?;
pub fn run(self) -> anyhow::Result<()> {
let mut config = read_config()?;
let index = if self.reset {
Some(CliConfig::default().default_index)
@ -26,7 +25,7 @@ impl DefaultIndexCommand {
match index {
Some(index) => {
config.default_index = index.clone();
write_config(project.data_dir(), &config)?;
write_config(&config)?;
println!("default index set to: {index}");
}
None => {

7
src/cli/commands/config/mod.rs
View file

@ -1,5 +1,4 @@
use clap::Subcommand;
use pesde::Project;
mod default_index;
mod scripts_repo;
@ -14,10 +13,10 @@ pub enum ConfigCommands {
}
impl ConfigCommands {
pub fn run(self, project: Project) -> anyhow::Result<()> {
pub fn run(self) -> anyhow::Result<()> {
match self {
ConfigCommands::DefaultIndex(default_index) => default_index.run(project),
ConfigCommands::ScriptsRepo(scripts_repo) => scripts_repo.run(project),
ConfigCommands::DefaultIndex(default_index) => default_index.run(),
ConfigCommands::ScriptsRepo(scripts_repo) => scripts_repo.run(),
}
}
}

7
src/cli/commands/config/scripts_repo.rs
View file

@ -1,6 +1,5 @@
use crate::cli::config::{read_config, write_config, CliConfig};
use clap::Args;
use pesde::Project;
#[derive(Debug, Args)]
pub struct ScriptsRepoCommand {
@ -14,8 +13,8 @@ pub struct ScriptsRepoCommand {
}
impl ScriptsRepoCommand {
pub fn run(self, project: Project) -> anyhow::Result<()> {
let mut config = read_config(project.data_dir())?;
pub fn run(self) -> anyhow::Result<()> {
let mut config = read_config()?;
let repo = if self.reset {
Some(CliConfig::default().scripts_repo)
@ -26,7 +25,7 @@ impl ScriptsRepoCommand {
match repo {
Some(repo) => {
config.scripts_repo = repo.clone();
write_config(project.data_dir(), &config)?;
write_config(&config)?;
println!("scripts repo set to: {repo}");
}
None => {

8
src/cli/commands/init.rs
View file

@ -144,12 +144,8 @@ impl InitCommand {
));
}
manifest["indices"][DEFAULT_INDEX_NAME] = toml_edit::value(
read_config(project.data_dir())?
.default_index
.to_bstring()
.to_string(),
);
manifest["indices"][DEFAULT_INDEX_NAME] =
toml_edit::value(read_config()?.default_index.to_bstring().to_string());
project.write_manifest(manifest.to_string())?;

4
src/cli/commands/install.rs
View file

@ -33,9 +33,9 @@ fn bin_link_file(alias: &str) -> String {
.collect::<Vec<_>>()
.join(", ");
#[cfg(windows)]
#[cfg(not(unix))]
let prefix = String::new();
#[cfg(not(windows))]
#[cfg(unix)]
let prefix = "#!/usr/bin/env -S lune run\n";
format!(

4
src/cli/commands/mod.rs
View file

@ -60,7 +60,7 @@ impl Subcommand {
) -> anyhow::Result<()> {
match self {
Subcommand::Auth(auth) => auth.run(project, reqwest),
Subcommand::Config(config) => config.run(project),
Subcommand::Config(config) => config.run(),
Subcommand::Init(init) => init.run(project),
Subcommand::Run(run) => run.run(project),
Subcommand::Install(install) => install.run(project, multi, reqwest),
@ -70,7 +70,7 @@ impl Subcommand {
Subcommand::Patch(patch) => patch.run(project, reqwest),
#[cfg(feature = "patches")]
Subcommand::PatchCommit(patch_commit) => patch_commit.run(project),
Subcommand::SelfUpgrade(self_upgrade) => self_upgrade.run(project, reqwest),
Subcommand::SelfUpgrade(self_upgrade) => self_upgrade.run(reqwest),
}
}
}

8
src/cli/commands/patch.rs
View file

@ -4,7 +4,7 @@ use clap::Args;
use colored::Colorize;
use pesde::{
patches::setup_patches_repo,
source::{PackageRef, PackageSource},
source::traits::{PackageRef, PackageSource},
Project, MANIFEST_FILE_NAME,
};
@ -39,7 +39,11 @@ impl PatchCommand {
.join(chrono::Utc::now().timestamp().to_string());
std::fs::create_dir_all(&directory)?;
source.download(&node.node.pkg_ref, &directory, &project, &reqwest)?;
source
.download(&node.node.pkg_ref, &project, &reqwest)?
.0
.write_to(&directory, project.cas_dir(), false)
.context("failed to write package contents")?;
// TODO: if MANIFEST_FILE_NAME does not exist, try to convert it

4
src/cli/commands/patch_commit.rs
View file

@ -2,8 +2,8 @@ use crate::cli::IsUpToDate;
use anyhow::Context;
use clap::Args;
use pesde::{
manifest::Manifest, names::PackageNames, patches::create_patch, source::VersionId, Project,
MANIFEST_FILE_NAME,
manifest::Manifest, names::PackageNames, patches::create_patch, source::version_id::VersionId,
Project, MANIFEST_FILE_NAME,
};
use std::{path::PathBuf, str::FromStr};

5
src/cli/commands/self_upgrade.rs
View file

@ -1,6 +1,5 @@
use crate::cli::{config::read_config, version::get_or_download_version};
use clap::Args;
use pesde::Project;
#[derive(Debug, Args)]
pub struct SelfUpgradeCommand {
@ -10,8 +9,8 @@ pub struct SelfUpgradeCommand {
}
impl SelfUpgradeCommand {
pub fn run(self, project: Project, reqwest: reqwest::blocking::Client) -> anyhow::Result<()> {
let config = read_config(project.data_dir())?;
pub fn run(self, reqwest: reqwest::blocking::Client) -> anyhow::Result<()> {
let config = read_config()?;
get_or_download_version(&reqwest, &config.last_checked_updates.unwrap().1)?;

11
src/cli/config.rs
View file

@ -1,6 +1,7 @@
use anyhow::Context;
use serde::{Deserialize, Serialize};
use std::path::Path;
use crate::cli::home_dir;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CliConfig {
@ -37,8 +38,8 @@ impl Default for CliConfig {
}
}
pub fn read_config<P: AsRef<Path>>(data_dir: P) -> anyhow::Result<CliConfig> {
let config_string = match std::fs::read_to_string(data_dir.as_ref().join("config.toml")) {
pub fn read_config() -> anyhow::Result<CliConfig> {
let config_string = match std::fs::read_to_string(home_dir()?.join("config.toml")) {
Ok(config_string) => config_string,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Ok(CliConfig::default());
@ -51,9 +52,9 @@ pub fn read_config<P: AsRef<Path>>(data_dir: P) -> anyhow::Result<CliConfig> {
Ok(config)
}
pub fn write_config<P: AsRef<Path>>(data_dir: P, config: &CliConfig) -> anyhow::Result<()> {
pub fn write_config(config: &CliConfig) -> anyhow::Result<()> {
let config_string = toml::to_string(config).context("failed to serialize config")?;
std::fs::write(data_dir.as_ref().join("config.toml"), config_string)
std::fs::write(home_dir()?.join("config.toml"), config_string)
.context("failed to write config file")?;
Ok(())

4
src/cli/mod.rs
View file

@ -7,7 +7,9 @@ pub mod version;
use crate::cli::auth::get_token;
use anyhow::Context;
use pesde::{lockfile::DownloadedGraph, names::PackageNames, source::VersionId, Project};
use pesde::{
lockfile::DownloadedGraph, names::PackageNames, source::version_id::VersionId, Project,
};
use std::{collections::HashSet, str::FromStr};
pub const HOME_DIR: &str = concat!(".", env!("CARGO_PKG_NAME"));

2
src/cli/scripts.rs
View file

@ -79,7 +79,7 @@ pub fn update_scripts_folder(project: &Project) -> anyhow::Result<()> {
} else {
std::fs::create_dir_all(&scripts_dir).context("failed to create scripts directory")?;
let cli_config = read_config(project.data_dir())?;
let cli_config = read_config()?;
gix::prepare_clone(cli_config.scripts_repo, &scripts_dir)
.context("failed to prepare scripts repository clone")?

24
src/cli/version.rs
View file

@ -1,8 +1,4 @@
use std::{
fs::create_dir_all,
io::Read,
path::{Path, PathBuf},
};
use std::{fs::create_dir_all, io::Read, path::PathBuf};
use anyhow::Context;
use colored::Colorize;
@ -42,13 +38,10 @@ fn get_repo() -> (String, String) {
const CHECK_INTERVAL: chrono::Duration = chrono::Duration::seconds(30);
pub fn check_for_updates<P: AsRef<Path>>(
reqwest: &reqwest::blocking::Client,
data_dir: P,
) -> anyhow::Result<()> {
pub fn check_for_updates(reqwest: &reqwest::blocking::Client) -> anyhow::Result<()> {
let (owner, repo) = get_repo();
let config = read_config(&data_dir)?;
let config = read_config()?;
let version = if let Some((_, version)) = config
.last_checked_updates
@ -71,13 +64,10 @@ pub fn check_for_updates<P: AsRef<Path>>(
.max()
.context("failed to find latest version")?;
write_config(
&data_dir,
&CliConfig {
last_checked_updates: Some((chrono::Utc::now(), version.clone())),
..config
},
)?;
write_config(&CliConfig {
last_checked_updates: Some((chrono::Utc::now(), version.clone())),
..config
})?;
version
};

42
src/download.rs
View file

@ -1,15 +1,17 @@
use crate::{
lockfile::{DependencyGraph, DownloadedDependencyGraphNode, DownloadedGraph},
source::{
traits::{PackageRef, PackageSource},
PackageSources,
},
Project, PACKAGES_CONTAINER_NAME,
};
use std::{
collections::HashSet,
fs::create_dir_all,
sync::{mpsc::Receiver, Arc, Mutex},
};
use crate::{
lockfile::{DependencyGraph, DownloadedDependencyGraphNode, DownloadedGraph},
source::{PackageRef, PackageSource, PackageSources},
Project, PACKAGES_CONTAINER_NAME,
};
type MultithreadedGraph = Arc<Mutex<DownloadedGraph>>;
type MultithreadDownloadJob = (
@ -65,18 +67,25 @@ impl Project {
log::debug!("downloading {name}@{version_id}");
let target =
match source.download(&node.pkg_ref, &container_folder, &project, &reqwest)
{
Ok(target) => target,
Err(e) => {
tx.send(Err(e.into())).unwrap();
return;
}
};
let (fs, target) = match source.download(&node.pkg_ref, &project, &reqwest) {
Ok(target) => target,
Err(e) => {
tx.send(Err(e.into())).unwrap();
return;
}
};
log::debug!("downloaded {name}@{version_id}");
match fs.write_to(container_folder, project.cas_dir(), true) {
Ok(_) => {}
Err(e) => {
tx.send(Err(errors::DownloadGraphError::WriteFailed(e)))
.unwrap();
return;
}
};
let mut downloaded_graph = downloaded_graph.lock().unwrap();
downloaded_graph
.entry(name)
@ -109,5 +118,8 @@ pub mod errors {
#[error("failed to download package")]
DownloadFailed(#[from] crate::source::errors::DownloadError),
#[error("failed to write package contents")]
WriteFailed(std::io::Error),
}
}

9
src/lib.rs
View file

@ -62,18 +62,21 @@ pub struct Project {
path: PathBuf,
data_dir: PathBuf,
auth_config: AuthConfig,
cas_dir: PathBuf,
}
impl Project {
pub fn new<P: AsRef<Path>, Q: AsRef<Path>>(
pub fn new<P: AsRef<Path>, Q: AsRef<Path>, R: AsRef<Path>>(
path: P,
data_dir: Q,
cas_dir: R,
auth_config: AuthConfig,
) -> Self {
Project {
path: path.as_ref().to_path_buf(),
data_dir: data_dir.as_ref().to_path_buf(),
auth_config,
cas_dir: cas_dir.as_ref().to_path_buf(),
}
}
@ -89,6 +92,10 @@ impl Project {
&self.auth_config
}
pub fn cas_dir(&self) -> &Path {
&self.cas_dir
}
pub fn read_manifest(&self) -> Result<String, errors::ManifestReadError> {
let string = std::fs::read_to_string(self.path.join(MANIFEST_FILE_NAME))?;
Ok(string)

102
src/linking/mod.rs
View file

@ -4,21 +4,30 @@ use crate::{
manifest::target::Target,
names::PackageNames,
scripts::{execute_script, ScriptName},
source::{PackageRef, VersionId},
source::{fs::store_in_cas, traits::PackageRef, version_id::VersionId},
util::hash,
Project, PACKAGES_CONTAINER_NAME,
};
use std::{collections::BTreeMap, fs::create_dir_all};
use std::{
collections::BTreeMap,
fs::create_dir_all,
path::{Path, PathBuf},
};
pub mod generator;
fn create_and_canonicalize<P: AsRef<std::path::Path>>(
path: P,
) -> std::io::Result<std::path::PathBuf> {
fn create_and_canonicalize<P: AsRef<Path>>(path: P) -> std::io::Result<PathBuf> {
let p = path.as_ref();
create_dir_all(p)?;
p.canonicalize()
}
fn write_cas(destination: PathBuf, cas_dir: &Path, contents: &str) -> std::io::Result<()> {
let cas_path = store_in_cas(cas_dir, contents)?.1;
std::fs::hard_link(cas_path, destination)
}
impl Project {
pub fn link_dependencies(&self, graph: &DownloadedGraph) -> Result<(), errors::LinkingError> {
let manifest = self.deser_manifest()?;
@ -117,34 +126,34 @@ impl Project {
.and_then(|types| node.node.direct.as_ref().map(|(alias, _)| (alias, types)))
{
if let Some(lib_file) = node.target.lib_path() {
let linker_file = base_folder.join(format!("{alias}.luau"));
let module = generator::generate_lib_linking_module(
&generator::get_lib_require_path(
&node.target.kind(),
&base_folder,
lib_file,
&container_folder,
node.node.pkg_ref.use_new_structure(),
write_cas(
base_folder.join(format!("{alias}.luau")),
self.cas_dir(),
&generator::generate_lib_linking_module(
&generator::get_lib_require_path(
&node.target.kind(),
&base_folder,
lib_file,
&container_folder,
node.node.pkg_ref.use_new_structure(),
),
types,
),
types,
);
std::fs::write(linker_file, module)?;
)?;
};
if let Some(bin_file) = node.target.bin_path() {
let linker_file = base_folder.join(format!("{alias}.bin.luau"));
let module = generator::generate_bin_linking_module(
&generator::get_bin_require_path(
&base_folder,
bin_file,
&container_folder,
write_cas(
base_folder.join(format!("{alias}.bin.luau")),
self.cas_dir(),
&generator::generate_bin_linking_module(
&generator::get_bin_require_path(
&base_folder,
bin_file,
&container_folder,
),
),
);
std::fs::write(linker_file, module)?;
)?;
}
}
@ -169,27 +178,28 @@ impl Project {
container_folder
.join(dependency_node.node.base_folder(node.target.kind(), false)),
)?;
let linker_file = linker_folder.join(format!("{dependency_alias}.luau"));
let module = generator::generate_lib_linking_module(
&generator::get_lib_require_path(
&dependency_node.target.kind(),
&linker_folder,
lib_file,
&dependency_node.node.container_folder(
&packages_container_folder,
dependency_name,
dependency_version_id.version(),
write_cas(
linker_folder.join(format!("{dependency_alias}.luau")),
self.cas_dir(),
&generator::generate_lib_linking_module(
&generator::get_lib_require_path(
&dependency_node.target.kind(),
&linker_folder,
lib_file,
&dependency_node.node.container_folder(
&packages_container_folder,
dependency_name,
dependency_version_id.version(),
),
node.node.pkg_ref.use_new_structure(),
),
node.node.pkg_ref.use_new_structure(),
package_types
.get(dependency_name)
.and_then(|v| v.get(dependency_version_id))
.unwrap(),
),
package_types
.get(dependency_name)
.and_then(|v| v.get(dependency_version_id))
.unwrap(),
);
std::fs::write(linker_file, module)?;
)?;
}
}
}

5
src/lockfile.rs
View file

@ -5,7 +5,10 @@ use crate::{
DependencyType,
},
names::{PackageName, PackageNames},
source::{DependencySpecifiers, PackageRef, PackageRefs, VersionId},
source::{
refs::PackageRefs, specifiers::DependencySpecifiers, traits::PackageRef,
version_id::VersionId,
},
};
use semver::Version;
use serde::{Deserialize, Serialize};

51
src/main.rs
View file

@ -2,6 +2,7 @@ use crate::cli::{
auth::get_token,
home_dir,
version::{check_for_updates, current_version, get_or_download_version, max_installed_version},
HOME_DIR,
};
use anyhow::Context;
use clap::Parser;
@ -9,7 +10,7 @@ use colored::Colorize;
use indicatif::MultiProgress;
use indicatif_log_bridge::LogWrapper;
use pesde::{AuthConfig, Project};
use std::fs::create_dir_all;
use std::{fs::create_dir_all, path::PathBuf};
mod cli;
pub mod util;
@ -26,6 +27,39 @@ struct Cli {
subcommand: cli::commands::Subcommand,
}
#[cfg(windows)]
fn get_root(path: &std::path::Path) -> PathBuf {
match path.components().next().unwrap() {
std::path::Component::Prefix(prefix) => {
let mut string = prefix.as_os_str().to_string_lossy().to_string();
if string.ends_with(':') {
string.push(std::path::MAIN_SEPARATOR);
}
std::path::PathBuf::from(&string)
}
_ => unreachable!(),
}
}
#[cfg(unix)]
fn get_root(path: &std::path::Path) -> PathBuf {
use std::os::unix::fs::MetadataExt;
let path = std::fs::canonicalize(path).unwrap();
let mut current = path.as_path();
while let Some(parent) = current.parent() {
if std::fs::metadata(parent).unwrap().dev() != std::fs::metadata(current).unwrap().dev() {
break;
}
current = parent;
}
current.to_path_buf()
}
fn run() -> anyhow::Result<()> {
#[cfg(windows)]
'scripts: {
@ -73,11 +107,20 @@ fn run() -> anyhow::Result<()> {
let data_dir = home_dir()?.join("data");
create_dir_all(&data_dir).expect("failed to create data directory");
let token = get_token(&data_dir)?;
let token = get_token()?;
let home_cas_dir = data_dir.join("cas");
let project_root = get_root(&cwd);
let cas_dir = if get_root(&home_cas_dir) == project_root {
home_cas_dir
} else {
project_root.join(HOME_DIR).join("cas")
};
let project = Project::new(
cwd,
&data_dir,
data_dir,
cas_dir,
AuthConfig::new().with_pesde_token(token.as_ref()),
);
@ -109,7 +152,7 @@ fn run() -> anyhow::Result<()> {
.build()?
};
check_for_updates(&reqwest, &data_dir)?;
check_for_updates(&reqwest)?;
let target_version = project
.deser_manifest()

2
src/manifest/mod.rs
View file

@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
use crate::{
manifest::{overrides::OverrideKey, target::Target},
names::{PackageName, PackageNames},
source::{DependencySpecifiers, VersionId},
source::{specifiers::DependencySpecifiers, version_id::VersionId},
};
pub mod overrides;

37
src/patches.rs
View file

@ -1,5 +1,6 @@
use crate::{lockfile::DownloadedGraph, Project, MANIFEST_FILE_NAME, PACKAGES_CONTAINER_NAME};
use git2::{ApplyLocation, Diff, DiffFormat, DiffLineType, Repository, Signature};
use git2::{ApplyLocation, ApplyOptions, Diff, DiffFormat, DiffLineType, Repository, Signature};
use relative_path::RelativePathBuf;
use std::{fs::read, path::Path};
pub fn setup_patches_repo<P: AsRef<Path>>(dir: P) -> Result<Repository, git2::Error> {
@ -94,7 +95,37 @@ impl Project {
{
let repo = setup_patches_repo(&container_folder)?;
repo.apply(&patch, ApplyLocation::Both, None)?;
let mut apply_opts = ApplyOptions::new();
apply_opts.delta_callback(|delta| {
let Some(delta) = delta else {
return true;
};
if !matches!(delta.status(), git2::Delta::Modified) {
return true;
}
let file = delta.new_file();
let Some(relative_path) = file.path() else {
return true;
};
let relative_path = RelativePathBuf::from_path(relative_path).unwrap();
let path = relative_path.to_path(&container_folder);
if !path.is_file() {
return true;
}
// there is no way (as far as I know) to check if it's hardlinked
// so, we always unlink it
let content = read(&path).unwrap();
std::fs::remove_file(&path).unwrap();
std::fs::write(path, content).unwrap();
true
});
repo.apply(&patch, ApplyLocation::Both, Some(&mut apply_opts))?;
}
log::debug!("patch applied to {name}@{version_id}, removing .git directory");
@ -112,7 +143,7 @@ impl Project {
pub mod errors {
use std::path::PathBuf;
use crate::{names::PackageNames, source::VersionId};
use crate::{names::PackageNames, source::version_id::VersionId};
use thiserror::Error;
#[derive(Debug, Error)]

7
src/resolver.rs
View file

@ -3,8 +3,11 @@ use crate::{
manifest::DependencyType,
names::PackageNames,
source::{
pesde::PesdePackageSource, DependencySpecifiers, PackageRef, PackageSource, PackageSources,
VersionId,
pesde::PesdePackageSource,
specifiers::DependencySpecifiers,
traits::{PackageRef, PackageSource},
version_id::VersionId,
PackageSources,
},
Project, DEFAULT_INDEX_NAME,
};

72
src/source/fs.rs Normal file
View file

@ -0,0 +1,72 @@
use crate::util::hash;
use relative_path::RelativePathBuf;
use serde::{Deserialize, Serialize};
use std::{
collections::BTreeMap,
path::{Path, PathBuf},
};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum FSEntry {
#[serde(rename = "f")]
File(String),
#[serde(rename = "d")]
Directory,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(transparent)]
pub struct PackageFS(pub(crate) BTreeMap<RelativePathBuf, FSEntry>);
pub(crate) fn store_in_cas<P: AsRef<Path>>(
cas_dir: P,
contents: &str,
) -> std::io::Result<(String, PathBuf)> {
let hash = hash(contents.as_bytes());
let (prefix, rest) = hash.split_at(2);
let folder = cas_dir.as_ref().join(prefix);
std::fs::create_dir_all(&folder)?;
let cas_path = folder.join(rest);
if !cas_path.exists() {
std::fs::write(&cas_path, contents)?;
}
Ok((hash, cas_path))
}
impl PackageFS {
pub fn write_to<P: AsRef<Path>, Q: AsRef<Path>>(
&self,
destination: P,
cas_path: Q,
link: bool,
) -> std::io::Result<()> {
for (path, entry) in &self.0 {
let path = path.to_path(destination.as_ref());
match entry {
FSEntry::File(hash) => {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let (prefix, rest) = hash.split_at(2);
let cas_file_path = cas_path.as_ref().join(prefix).join(rest);
if link {
std::fs::hard_link(cas_file_path, path)?;
} else {
std::fs::copy(cas_file_path, path)?;
}
}
FSEntry::Directory => {
std::fs::create_dir_all(path)?;
}
}
}
Ok(())
}
}

173
src/source/mod.rs
View file

@ -1,127 +1,21 @@
use std::{collections::BTreeMap, fmt::Debug};
use crate::{
manifest::{
target::{Target, TargetKind},
DependencyType,
},
manifest::target::{Target, TargetKind},
names::PackageNames,
source::{
fs::PackageFS, refs::PackageRefs, specifiers::DependencySpecifiers, traits::*,
version_id::VersionId,
},
Project,
};
use semver::Version;
use serde::{Deserialize, Serialize};
use serde_with::{DeserializeFromStr, SerializeDisplay};
use std::{
collections::BTreeMap,
fmt::{Debug, Display},
path::Path,
str::FromStr,
};
pub mod fs;
pub mod pesde;
pub(crate) fn hash<S: std::hash::Hash>(struc: &S) -> String {
use std::{collections::hash_map::DefaultHasher, hash::Hasher};
let mut hasher = DefaultHasher::new();
struc.hash(&mut hasher);
hasher.finish().to_string()
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
#[serde(untagged)]
pub enum DependencySpecifiers {
Pesde(pesde::specifier::PesdeDependencySpecifier),
}
pub trait DependencySpecifier: Debug + Display {}
impl DependencySpecifier for DependencySpecifiers {}
impl Display for DependencySpecifiers {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
DependencySpecifiers::Pesde(specifier) => write!(f, "{specifier}"),
}
}
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "snake_case", tag = "ref_ty")]
pub enum PackageRefs {
Pesde(pesde::pkg_ref::PesdePackageRef),
}
pub trait PackageRef: Debug {
fn dependencies(&self) -> &BTreeMap<String, (DependencySpecifiers, DependencyType)>;
fn use_new_structure(&self) -> bool;
fn target_kind(&self) -> TargetKind;
fn source(&self) -> PackageSources;
}
impl PackageRef for PackageRefs {
fn dependencies(&self) -> &BTreeMap<String, (DependencySpecifiers, DependencyType)> {
match self {
PackageRefs::Pesde(pkg_ref) => pkg_ref.dependencies(),
}
}
fn use_new_structure(&self) -> bool {
match self {
PackageRefs::Pesde(pkg_ref) => pkg_ref.use_new_structure(),
}
}
fn target_kind(&self) -> TargetKind {
match self {
PackageRefs::Pesde(pkg_ref) => pkg_ref.target_kind(),
}
}
fn source(&self) -> PackageSources {
match self {
PackageRefs::Pesde(pkg_ref) => pkg_ref.source(),
}
}
}
#[derive(
Debug, SerializeDisplay, DeserializeFromStr, Clone, PartialEq, Eq, Hash, PartialOrd, Ord,
)]
pub struct VersionId(Version, TargetKind);
impl VersionId {
pub fn new(version: Version, target: TargetKind) -> Self {
VersionId(version, target)
}
pub fn version(&self) -> &Version {
&self.0
}
pub fn target(&self) -> &TargetKind {
&self.1
}
pub fn escaped(&self) -> String {
format!("{}+{}", self.0, self.1)
}
}
impl Display for VersionId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} {}", self.0, self.1)
}
}
impl FromStr for VersionId {
type Err = errors::VersionIdParseError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let Some((version, target)) = s.split_once(' ') else {
return Err(errors::VersionIdParseError::Malformed(s.to_string()));
};
let version = version.parse()?;
let target = target.parse()?;
Ok(VersionId(version, target))
}
}
pub mod refs;
pub mod specifiers;
pub mod traits;
pub mod version_id;
pub type ResolveResult<Ref> = (PackageNames, BTreeMap<VersionId, Ref>);
@ -129,32 +23,7 @@ pub type ResolveResult<Ref> = (PackageNames, BTreeMap<VersionId, Ref>);
pub enum PackageSources {
Pesde(pesde::PesdePackageSource),
}
pub trait PackageSource: Debug {
type Ref: PackageRef;
type Specifier: DependencySpecifier;
type RefreshError: std::error::Error;
type ResolveError: std::error::Error;
type DownloadError: std::error::Error;
fn refresh(&self, _project: &Project) -> Result<(), Self::RefreshError> {
Ok(())
}
fn resolve(
&self,
specifier: &Self::Specifier,
project: &Project,
project_target: TargetKind,
) -> Result<ResolveResult<Self::Ref>, Self::ResolveError>;
fn download(
&self,
pkg_ref: &Self::Ref,
destination: &Path,
project: &Project,
reqwest: &reqwest::blocking::Client,
) -> Result<Target, Self::DownloadError>;
}
impl PackageSource for PackageSources {
type Ref = PackageRefs;
type Specifier = DependencySpecifiers;
@ -195,13 +64,12 @@ impl PackageSource for PackageSources {
fn download(
&self,
pkg_ref: &Self::Ref,
destination: &Path,
project: &Project,
reqwest: &reqwest::blocking::Client,
) -> Result<Target, Self::DownloadError> {
) -> Result<(PackageFS, Target), Self::DownloadError> {
match (self, pkg_ref) {
(PackageSources::Pesde(source), PackageRefs::Pesde(pkg_ref)) => source
.download(pkg_ref, destination, project, reqwest)
.download(pkg_ref, project, reqwest)
.map_err(Into::into),
_ => Err(errors::DownloadError::Mismatch),
@ -238,17 +106,4 @@ pub mod errors {
#[error("error downloading pesde package")]
Pesde(#[from] crate::source::pesde::errors::DownloadError),
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum VersionIdParseError {
#[error("malformed entry key {0}")]
Malformed(String),
#[error("malformed version")]
Version(#[from] semver::Error),
#[error("malformed target")]
Target(#[from] crate::manifest::target::errors::TargetKindFromStr),
}
}

82
src/source/pesde/mod.rs
View file

@ -1,7 +1,7 @@
use std::{collections::BTreeMap, fmt::Debug, hash::Hash, path::Path};
use gix::remote::Direction;
use relative_path::RelativePathBuf;
use serde::{Deserialize, Serialize};
use std::{collections::BTreeMap, fmt::Debug, hash::Hash, io::Read};
use pkg_ref::PesdePackageRef;
use specifier::PesdeDependencySpecifier;
@ -12,8 +12,11 @@ use crate::{
DependencyType,
},
names::{PackageName, PackageNames},
source::{hash, DependencySpecifiers, PackageSource, ResolveResult, VersionId},
util::authenticate_conn,
source::{
fs::{store_in_cas, FSEntry, PackageFS},
DependencySpecifiers, PackageSource, ResolveResult, VersionId,
},
util::{authenticate_conn, hash},
Project,
};
@ -32,8 +35,12 @@ impl PesdePackageSource {
Self { repo_url }
}
fn as_bytes(&self) -> Vec<u8> {
self.repo_url.to_bstring().to_vec()
}
pub fn path(&self, project: &Project) -> std::path::PathBuf {
project.data_dir.join("indices").join(hash(self))
project.data_dir.join("indices").join(hash(self.as_bytes()))
}
pub(crate) fn tree<'a>(
@ -349,11 +356,30 @@ impl PackageSource for PesdePackageSource {
fn download(
&self,
pkg_ref: &Self::Ref,
destination: &Path,
project: &Project,
reqwest: &reqwest::blocking::Client,
) -> Result<Target, Self::DownloadError> {
) -> Result<(PackageFS, Target), Self::DownloadError> {
let config = self.config(project)?;
let index_file = project
.cas_dir
.join("index")
.join(pkg_ref.name.escaped())
.join(pkg_ref.version.to_string())
.join(pkg_ref.target.to_string());
match std::fs::read_to_string(&index_file) {
Ok(s) => {
log::debug!(
"using cached index file for package {}@{} {}",
pkg_ref.name,
pkg_ref.version,
pkg_ref.target
);
return Ok((toml::from_str::<PackageFS>(&s)?, pkg_ref.target.clone()));
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(errors::DownloadError::ReadIndex(e)),
}
let (scope, name) = pkg_ref.name.as_str();
let url = config
@ -375,9 +401,35 @@ impl PackageSource for PesdePackageSource {
let mut decoder = flate2::read::GzDecoder::new(bytes.as_ref());
let mut archive = tar::Archive::new(&mut decoder);
archive.unpack(destination)?;
let mut entries = BTreeMap::new();
Ok(pkg_ref.target.clone())
for entry in archive.entries()? {
let mut entry = entry?;
let path = RelativePathBuf::from_path(entry.path()?).unwrap();
if entry.header().entry_type().is_dir() {
entries.insert(path, FSEntry::Directory);
continue;
}
let mut contents = String::new();
entry.read_to_string(&mut contents)?;
let hash = store_in_cas(&project.cas_dir, &contents)?.0;
entries.insert(path, FSEntry::File(hash));
}
let fs = PackageFS(entries);
if let Some(parent) = index_file.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::write(&index_file, toml::to_string(&fs)?)
.map_err(errors::DownloadError::WriteIndex)?;
Ok((fs, pkg_ref.target.clone()))
}
}
@ -575,5 +627,17 @@ pub mod errors {
#[error("error unpacking package")]
Unpack(#[from] std::io::Error),
#[error("error writing index file")]
WriteIndex(#[source] std::io::Error),
#[error("error serializing index file")]
SerializeIndex(#[from] toml::ser::Error),
#[error("error deserializing index file")]
DeserializeIndex(#[from] toml::de::Error),
#[error("error reading index file")]
ReadIndex(#[source] std::io::Error),
}
}

38
src/source/refs.rs Normal file
View file

@ -0,0 +1,38 @@
use crate::{
manifest::{target::TargetKind, DependencyType},
source::{pesde, specifiers::DependencySpecifiers, traits::PackageRef, PackageSources},
};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "snake_case", tag = "ref_ty")]
pub enum PackageRefs {
Pesde(pesde::pkg_ref::PesdePackageRef),
}
impl PackageRef for PackageRefs {
fn dependencies(&self) -> &BTreeMap<String, (DependencySpecifiers, DependencyType)> {
match self {
PackageRefs::Pesde(pkg_ref) => pkg_ref.dependencies(),
}
}
fn use_new_structure(&self) -> bool {
match self {
PackageRefs::Pesde(pkg_ref) => pkg_ref.use_new_structure(),
}
}
fn target_kind(&self) -> TargetKind {
match self {
PackageRefs::Pesde(pkg_ref) => pkg_ref.target_kind(),
}
}
fn source(&self) -> PackageSources {
match self {
PackageRefs::Pesde(pkg_ref) => pkg_ref.source(),
}
}
}

18
src/source/specifiers.rs Normal file
View file

@ -0,0 +1,18 @@
use crate::source::{pesde, traits::DependencySpecifier};
use serde::{Deserialize, Serialize};
use std::fmt::Display;
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
#[serde(untagged)]
pub enum DependencySpecifiers {
Pesde(pesde::specifier::PesdeDependencySpecifier),
}
impl DependencySpecifier for DependencySpecifiers {}
impl Display for DependencySpecifiers {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
DependencySpecifiers::Pesde(specifier) => write!(f, "{specifier}"),
}
}
}

47
src/source/traits.rs Normal file
View file

@ -0,0 +1,47 @@
use crate::{
manifest::{
target::{Target, TargetKind},
DependencyType,
},
source::{DependencySpecifiers, PackageFS, PackageSources, ResolveResult},
Project,
};
use std::{
collections::BTreeMap,
fmt::{Debug, Display},
};
pub trait DependencySpecifier: Debug + Display {}
pub trait PackageRef: Debug {
fn dependencies(&self) -> &BTreeMap<String, (DependencySpecifiers, DependencyType)>;
fn use_new_structure(&self) -> bool;
fn target_kind(&self) -> TargetKind;
fn source(&self) -> PackageSources;
}
pub trait PackageSource: Debug {
type Ref: PackageRef;
type Specifier: DependencySpecifier;
type RefreshError: std::error::Error;
type ResolveError: std::error::Error;
type DownloadError: std::error::Error;
fn refresh(&self, _project: &Project) -> Result<(), Self::RefreshError> {
Ok(())
}
fn resolve(
&self,
specifier: &Self::Specifier,
project: &Project,
project_target: TargetKind,
) -> Result<ResolveResult<Self::Ref>, Self::ResolveError>;
fn download(
&self,
pkg_ref: &Self::Ref,
project: &Project,
reqwest: &reqwest::blocking::Client,
) -> Result<(PackageFS, Target), Self::DownloadError>;
}

65
src/source/version_id.rs Normal file
View file

@ -0,0 +1,65 @@
use crate::manifest::target::TargetKind;
use semver::Version;
use serde_with::{DeserializeFromStr, SerializeDisplay};
use std::{fmt::Display, str::FromStr};
#[derive(
Debug, SerializeDisplay, DeserializeFromStr, Clone, PartialEq, Eq, Hash, PartialOrd, Ord,
)]
pub struct VersionId(pub(crate) Version, pub(crate) TargetKind);
impl VersionId {
pub fn new(version: Version, target: TargetKind) -> Self {
VersionId(version, target)
}
pub fn version(&self) -> &Version {
&self.0
}
pub fn target(&self) -> &TargetKind {
&self.1
}
pub fn escaped(&self) -> String {
format!("{}+{}", self.0, self.1)
}
}
impl Display for VersionId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} {}", self.0, self.1)
}
}
impl FromStr for VersionId {
type Err = errors::VersionIdParseError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let Some((version, target)) = s.split_once(' ') else {
return Err(errors::VersionIdParseError::Malformed(s.to_string()));
};
let version = version.parse()?;
let target = target.parse()?;
Ok(VersionId(version, target))
}
}
pub mod errors {
use thiserror::Error;
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum VersionIdParseError {
#[error("malformed entry key {0}")]
Malformed(String),
#[error("malformed version")]
Version(#[from] semver::Error),
#[error("malformed target")]
Target(#[from] crate::manifest::target::errors::TargetKindFromStr),
}
}

7
src/util.rs
View file

@ -1,6 +1,7 @@
use crate::AuthConfig;
use gix::bstr::BStr;
use serde::{ser::SerializeMap, Deserialize, Deserializer, Serializer};
use sha2::{Digest, Sha256};
use std::collections::BTreeMap;
pub fn authenticate_conn(
@ -59,3 +60,9 @@ pub fn deserialize_gix_url_map<'de, D: Deserializer<'de>>(
})
.collect()
}
pub fn hash<S: AsRef<[u8]>>(struc: S) -> String {
let mut hasher = Sha256::new();
hasher.update(struc.as_ref());
format!("{:x}", hasher.finalize())
}