From a38da43670b7d19b2debe639945b1d97a36b3e5d Mon Sep 17 00:00:00 2001 From: daimond113 Date: Sun, 26 Jan 2025 23:02:52 +0100 Subject: [PATCH] feat: add cas pruning command Removes unused files from the CAS. Still needs to remove individual package index entries to be complete. --- Cargo.lock | 52 ++++++++++- Cargo.toml | 2 + src/cli/commands/cas/mod.rs | 18 ++++ src/cli/commands/cas/prune.rs | 161 ++++++++++++++++++++++++++++++++++ src/cli/commands/mod.rs | 6 ++ src/cli/mod.rs | 2 +- 6 files changed, 237 insertions(+), 4 deletions(-) create mode 100644 src/cli/commands/cas/mod.rs create mode 100644 src/cli/commands/cas/prune.rs diff --git a/Cargo.lock b/Cargo.lock index be156bd..d230342 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2502,7 +2502,7 @@ checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" dependencies = [ "cfg-if", "libc", - "windows", + "windows 0.52.0", ] [[package]] @@ -2652,7 +2652,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -3656,6 +3656,7 @@ dependencies = [ "url", "urlencoding", "wax", + "windows 0.59.0", "windows-registry 0.4.0", ] @@ -5669,10 +5670,20 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core", + "windows-core 0.52.0", "windows-targets 0.52.6", ] +[[package]] +name = "windows" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f919aee0a93304be7f62e8e5027811bbba96bcb1de84d6618be56e43f8a32a1" +dependencies = [ + "windows-core 0.59.0", + "windows-targets 0.53.0", +] + [[package]] name = "windows-core" version = "0.52.0" @@ -5682,6 +5693,41 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "810ce18ed2112484b0d4e15d022e5f598113e220c53e373fb31e67e21670c1ce" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result 0.3.0", + "windows-strings 0.3.0", + "windows-targets 0.53.0", +] + +[[package]] +name = "windows-implement" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "windows-interface" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26fd936d991781ea39e87c3a27285081e3c0da5ca0fcbc02d368cc6f52ff01" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "windows-registry" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 2e2feb4..c6b6971 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ bin = [ "dep:paste", "dep:serde_json", "dep:windows-registry", + "dep:windows", "gix/worktree-mutation", "fs-err/expose_original_error", "tokio/rt", @@ -91,6 +92,7 @@ paste = { version = "1.0.15", optional = true } [target.'cfg(target_os = "windows")'.dependencies] windows-registry = { version = "0.4.0", optional = true } +windows = { version = "0.59.0", features = ["Win32_Storage", "Win32_Storage_FileSystem", "Win32_Security"], optional = true } [workspace] resolver = "2" diff --git a/src/cli/commands/cas/mod.rs b/src/cli/commands/cas/mod.rs new file mode 100644 index 0000000..51d64d8 --- /dev/null +++ b/src/cli/commands/cas/mod.rs @@ -0,0 +1,18 @@ +use clap::Subcommand; +use pesde::Project; + +mod prune; + +#[derive(Debug, Subcommand)] +pub enum CasCommands { + /// Removes unused files from the CAS + Prune(prune::PruneCommand), +} + +impl CasCommands { + pub async fn run(self, project: Project) -> anyhow::Result<()> { + match self { + CasCommands::Prune(prune) => prune.run(project).await, + } + } +} diff --git a/src/cli/commands/cas/prune.rs b/src/cli/commands/cas/prune.rs new file mode 100644 index 0000000..0ab32b8 --- /dev/null +++ b/src/cli/commands/cas/prune.rs @@ -0,0 +1,161 @@ +use anyhow::Context; +use clap::Args; +use fs_err::tokio as fs; +use pesde::Project; +use std::{collections::HashSet, path::Path}; +use tokio::task::{spawn_blocking, JoinSet}; + +#[derive(Debug, Args)] +pub struct PruneCommand {} + +#[allow(unreachable_code)] +async fn get_nlinks(path: &Path) -> anyhow::Result { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = fs::metadata(path).await?; + return Ok(metadata.nlink()); + } + // life if rust stabilized the nightly feature from 2019 + #[cfg(windows)] + { + use std::os::windows::ffi::OsStrExt; + use windows::{ + core::PWSTR, + Win32::{ + Foundation::CloseHandle, + Storage::FileSystem::{ + CreateFileW, GetFileInformationByHandle, FILE_ATTRIBUTE_NORMAL, + FILE_GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING, + }, + }, + }; + + let path = path.to_path_buf(); + return spawn_blocking(move || unsafe { + let handle = CreateFileW( + PWSTR( + path.as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect::>() + .as_mut_ptr(), + ), + FILE_GENERIC_READ.0, + FILE_SHARE_READ, + None, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + None, + )?; + + let mut info = + windows::Win32::Storage::FileSystem::BY_HANDLE_FILE_INFORMATION::default(); + let res = GetFileInformationByHandle(handle, &mut info); + CloseHandle(handle)?; + res?; + + Ok(info.nNumberOfLinks as u64) + }) + .await + .unwrap(); + } + #[cfg(not(any(unix, windows)))] + { + compile_error!("unsupported platform"); + } + anyhow::bail!("unsupported platform") +} + +async fn remove_hashes(cas_dir: &Path) -> anyhow::Result> { + let mut tasks = JoinSet::new(); + + let mut cas_entries = fs::read_dir(cas_dir) + .await + .context("failed to read directory")?; + + while let Some(cas_entry) = cas_entries + .next_entry() + .await + .context("failed to read dir entry")? + { + let prefix = cas_entry.file_name(); + let Some(prefix) = prefix.to_str() else { + continue; + }; + // we only want hash directories + if prefix.len() != 2 { + continue; + } + + let prefix = prefix.to_string(); + + tasks.spawn(async move { + let mut hash_entries = fs::read_dir(cas_entry.path()) + .await + .context("failed to read hash directory")?; + + let mut tasks = JoinSet::new(); + + while let Some(hash_entry) = hash_entries + .next_entry() + .await + .context("failed to read hash dir entry")? + { + let hash = hash_entry.file_name(); + let hash = hash.to_str().expect("non-UTF-8 hash").to_string(); + let hash = format!("{prefix}{hash}"); + + let path = hash_entry.path(); + tasks.spawn(async move { + let nlinks = get_nlinks(&path) + .await + .context("failed to count file usage")?; + if nlinks != 1 { + return Ok::<_, anyhow::Error>(None); + } + + fs::remove_file(path) + .await + .context("failed to remove unused file")?; + + Ok::<_, anyhow::Error>(Some(hash)) + }); + } + + let mut removed_hashes = HashSet::new(); + + while let Some(removed_hash) = tasks.join_next().await { + let Some(hash) = removed_hash.unwrap()? else { + continue; + }; + + removed_hashes.insert(hash); + } + + Ok::<_, anyhow::Error>(removed_hashes) + }); + } + + let mut res = HashSet::new(); + + while let Some(removed_hashes) = tasks.join_next().await { + res.extend(removed_hashes.unwrap()?); + } + + Ok(res) +} + +impl PruneCommand { + pub async fn run(self, project: Project) -> anyhow::Result<()> { + // CAS structure: + // /2 first chars of hash/rest of hash + // /index/hash/name/version/target + // /wally_index/hash/name/version + // /git_index/hash/hash + // the last thing in the path is the serialized PackageFs + let _ = remove_hashes(project.cas_dir()).await?; + + todo!("remove unused index entries"); + } +} diff --git a/src/cli/commands/mod.rs b/src/cli/commands/mod.rs index 04229ed..c9fd1ce 100644 --- a/src/cli/commands/mod.rs +++ b/src/cli/commands/mod.rs @@ -2,6 +2,7 @@ use pesde::Project; mod add; mod auth; +mod cas; mod config; mod deprecate; mod execute; @@ -30,6 +31,10 @@ pub enum Subcommand { #[command(subcommand)] Config(config::ConfigCommands), + /// CAS-related commands + #[command(subcommand)] + Cas(cas::CasCommands), + /// Initializes a manifest file in the current directory Init(init::InitCommand), @@ -83,6 +88,7 @@ impl Subcommand { match self { Subcommand::Auth(auth) => auth.run(project, reqwest).await, Subcommand::Config(config) => config.run().await, + Subcommand::Cas(cas) => cas.run(project).await, Subcommand::Init(init) => init.run(project).await, Subcommand::Run(run) => run.run(project).await, Subcommand::Install(install) => install.run(project, reqwest).await, diff --git a/src/cli/mod.rs b/src/cli/mod.rs index d298bfb..0503518 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -297,7 +297,7 @@ pub fn display_err(result: anyhow::Result<()>, prefix: &str) { if !cause.is_empty() { eprintln!("{}:", ERROR_STYLE.apply_to("caused by")); for err in cause { - eprintln!("\t- {}", ERROR_STYLE.apply_to(err)); + eprintln!("\t- {err}"); } }