diff --git a/Cargo.lock b/Cargo.lock index ab46c50..93c5a95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "actix-codec" @@ -389,15 +389,6 @@ version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" -[[package]] -name = "arbitrary" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "arc-swap" version = "1.7.1" @@ -428,6 +419,45 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-compression" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" +dependencies = [ + "deflate64", + "flate2", + "futures-core", + "futures-io", + "memchr", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-executor" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ebdfa2ebdab6b1760375fa7d6f382b9f486eac35fc994625a00e89280bdbb7" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "slab", +] + +[[package]] +name = "async-fs" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcd09b382f40fcd159c2d695175b2ae620ffa5f3bd6f664131efff4e8b9e04a" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + [[package]] name = "async-io" version = "2.3.4" @@ -506,6 +536,28 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "async-task" version = "4.7.1" @@ -523,6 +575,21 @@ dependencies = [ "syn 2.0.85", ] +[[package]] +name = "async_zip" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b9f7252833d5ed4b00aa9604b563529dd5e11de9c23615de2dcdf91eb87b52" +dependencies = [ + "async-compression", + "crc32fast", + "futures-lite", + "pin-project", + "thiserror", + "tokio", + "tokio-util", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -688,27 +755,6 @@ dependencies = [ "bytes", ] -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "cbc" version = "0.1.2" @@ -923,21 +969,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32fast" version = "1.4.2" @@ -1125,17 +1156,6 @@ dependencies = [ "serde", ] -[[package]] -name = "derive_arbitrary" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.85", -] - [[package]] name = "derive_more" version = "0.99.18" @@ -1202,17 +1222,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.85", -] - [[package]] name = "dotenvy" version = "0.15.7" @@ -1433,6 +1442,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb60e7409f34ef959985bc9d9c5ee8f5db24ee46ed9775850548021710f807f" dependencies = [ "autocfg", + "tokio", ] [[package]] @@ -1867,12 +1877,14 @@ checksum = "8e0eb9efdf96c35c0bed7596d1bef2d4ce6360a1d09738001f9d3e402aa7ba3e" dependencies = [ "bytes", "crc32fast", + "crossbeam-channel", "flate2", "gix-hash", "gix-trace", "gix-utils", "libc", "once_cell", + "parking_lot", "prodash", "sha1_smol", "thiserror", @@ -2936,6 +2948,7 @@ dependencies = [ "security-framework 2.11.1", "security-framework 3.0.0", "windows-sys 0.59.0", + "zbus", ] [[package]] @@ -3008,7 +3021,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", - "redox_syscall", + "redox_syscall 0.5.7", ] [[package]] @@ -3070,12 +3083,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "lockfree-object-pool" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" - [[package]] name = "log" version = "0.4.22" @@ -3097,16 +3104,6 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" -[[package]] -name = "lzma-rs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" -dependencies = [ - "byteorder", - "crc", -] - [[package]] name = "maybe-async" version = "0.2.10" @@ -3517,7 +3514,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.7", "smallvec", "windows-targets 0.52.6", ] @@ -3540,16 +3537,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d61c5ce1153ab5b689d0c074c4e7fc613e942dfb7dd9eea5ab202d2ad91fe361" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest", - "hmac", -] - [[package]] name = "percent-encoding" version = "2.3.1" @@ -3561,13 +3548,16 @@ name = "pesde" version = "0.5.0-rc.7" dependencies = [ "anyhow", + "async-compression", + "async-stream", + "async_zip", "chrono", "clap", "colored", "dirs", - "flate2", "fs-err", "full_moon", + "futures", "git2", "gix", "glob", @@ -3586,15 +3576,15 @@ dependencies = [ "serde_json", "serde_with", "sha2", - "tar", "tempfile", "thiserror", - "threadpool", + "tokio", + "tokio-tar", + "tokio-util", "toml", "toml_edit", "url", "winreg", - "zip", ] [[package]] @@ -3605,11 +3595,11 @@ dependencies = [ "actix-governor", "actix-multipart", "actix-web", + "async-compression", "chrono", "constant_time_eq", "convert_case 0.6.0", "dotenvy", - "flate2", "fs-err", "futures", "git2", @@ -3628,13 +3618,34 @@ dependencies = [ "serde_yaml", "sha2", "tantivy", - "tar", "tempfile", "thiserror", + "tokio", + "tokio-tar", "toml", "url", ] +[[package]] +name = "pin-project" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -3889,6 +3900,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.7" @@ -4536,12 +4556,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simd-adler32" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" - [[package]] name = "sketches-ddsketch" version = "0.2.2" @@ -4818,17 +4832,6 @@ dependencies = [ "serde", ] -[[package]] -name = "tar" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "tempfile" version = "3.13.0" @@ -4881,15 +4884,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - [[package]] name = "time" version = "0.3.36" @@ -4938,9 +4932,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", @@ -4950,9 +4944,21 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "socket2", + "tokio-macros", "windows-sys 0.52.0", ] +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.85", +] + [[package]] name = "tokio-native-tls" version = "0.3.1" @@ -4974,6 +4980,32 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "tokio-util" version = "0.7.12" @@ -4982,6 +5014,7 @@ checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -5618,9 +5651,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb97012beadd29e654708a0fdb4c84bc046f537aecfde2c3ee0a9e4b4d48c725" dependencies = [ "async-broadcast", + "async-executor", + "async-fs", + "async-io", + "async-lock", "async-process", "async-recursion", + "async-task", "async-trait", + "blocking", "enumflags2", "event-listener", "futures-core", @@ -5693,63 +5732,6 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.85", -] - -[[package]] -name = "zip" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" -dependencies = [ - "aes", - "arbitrary", - "bzip2", - "constant_time_eq", - "crc32fast", - "crossbeam-utils", - "deflate64", - "displaydoc", - "flate2", - "hmac", - "indexmap 2.6.0", - "lzma-rs", - "memchr", - "pbkdf2", - "rand", - "sha1", - "thiserror", - "time", - "zeroize", - "zopfli", - "zstd", -] - -[[package]] -name = "zopfli" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" -dependencies = [ - "bumpalo", - "crc32fast", - "lockfree-object-pool", - "log", - "once_cell", - "simd-adler32", -] [[package]] name = "zstd" diff --git a/Cargo.toml b/Cargo.toml index f32778a..128872f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,9 +27,12 @@ bin = [ "gix/worktree-mutation", "serde_json", "winreg", - "fs-err/expose_original_error" + "fs-err/expose_original_error", + "tokio/rt", + "tokio/rt-multi-thread", + "tokio/macros", ] -wally-compat = ["zip", "serde_json"] +wally-compat = ["async_zip", "serde_json"] patches = ["git2"] version-management = ["bin"] @@ -45,33 +48,36 @@ uninlined_format_args = "warn" serde = { version = "1.0.213", features = ["derive"] } toml = "0.8.19" serde_with = "3.11.0" -gix = { version = "0.67.0", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls", "revparse-regex", "credentials"] } +gix = { version = "0.67.0", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls", "revparse-regex", "credentials", "parallel"] } semver = { version = "1.0.23", features = ["serde"] } -reqwest = { version = "0.12.8", default-features = false, features = ["rustls-tls", "blocking"] } -tar = "0.4.42" -flate2 = "1.0.34" +reqwest = { version = "0.12.8", default-features = false, features = ["rustls-tls"] } +tokio-tar = "0.3.1" +async-compression = { version = "0.4.17", features = ["tokio", "gzip"] } pathdiff = "0.2.2" relative-path = { version = "1.9.3", features = ["serde"] } log = "0.4.22" thiserror = "1.0.65" -threadpool = "1.8.1" +tokio = "1.41.0" +tokio-util = "0.7.12" +async-stream = "0.3.6" +futures = "0.3.31" full_moon = { version = "1.1.0", features = ["luau"] } url = { version = "2.5.2", features = ["serde"] } chrono = { version = "0.4.38", features = ["serde"] } sha2 = "0.10.8" tempfile = "3.13.0" glob = "0.3.1" -fs-err = "3.0.0" +fs-err = { version = "3.0.0", features = ["tokio"] } # TODO: remove this when gitoxide adds support for: committing, pushing, adding git2 = { version = "0.19.0", optional = true } -zip = { version = "2.2.0", optional = true } +async_zip = { version = "0.0.17", features = ["tokio", "deflate", "deflate64", "tokio-fs"], optional = true } serde_json = { version = "1.0.132", optional = true } anyhow = { version = "1.0.91", optional = true } open = { version = "5.3.0", optional = true } -keyring = { version = "3.5.0", features = ["crypto-rust", "windows-native", "apple-native", "sync-secret-service"], optional = true } +keyring = { version = "3.5.0", features = ["crypto-rust", "windows-native", "apple-native", "async-secret-service", "async-io"], optional = true } colored = { version = "2.1.0", optional = true } toml_edit = { version = "0.22.22", optional = true } clap = { version = "4.5.20", features = ["derive"], optional = true } diff --git a/registry/Cargo.toml b/registry/Cargo.toml index bf11480..d0f81c4 100644 --- a/registry/Cargo.toml +++ b/registry/Cargo.toml @@ -17,8 +17,9 @@ semver = "1.0.23" chrono = { version = "0.4.38", features = ["serde"] } url = "2.5.2" futures = "0.3.31" +tokio = "1.41.0" tempfile = "3.13.0" -fs-err = "3.0.0" +fs-err = { version = "3.0.0", features = ["tokio"] } git2 = "0.19.0" gix = { version = "0.67.0", default-features = false, features = [ @@ -37,8 +38,8 @@ rusty-s3 = "0.5.0" reqwest = { version = "0.12.8", features = ["json", "rustls-tls"] } constant_time_eq = "0.3.1" -tar = "0.4.42" -flate2 = "1.0.34" +tokio-tar = "0.3.1" +async-compression = { version = "0.4.17", features = ["tokio", "gzip"] } log = "0.4.22" pretty_env_logger = "0.5.0" diff --git a/registry/src/endpoints/package_version.rs b/registry/src/endpoints/package_version.rs index fea4db0..685a749 100644 --- a/registry/src/endpoints/package_version.rs +++ b/registry/src/endpoints/package_version.rs @@ -72,7 +72,7 @@ pub async fn get_package_version( let (scope, name_part) = name.as_str(); let entries: IndexFile = { - let source = app_state.source.lock().unwrap(); + let source = app_state.source.lock().await; match source.read_file([scope, name_part], &app_state.project, None)? { Some(versions) => toml::de::from_str(&versions)?, diff --git a/registry/src/endpoints/package_versions.rs b/registry/src/endpoints/package_versions.rs index 8e635bd..ef39f93 100644 --- a/registry/src/endpoints/package_versions.rs +++ b/registry/src/endpoints/package_versions.rs @@ -17,7 +17,7 @@ pub async fn get_package_versions( let (scope, name_part) = name.as_str(); - let source = app_state.source.lock().unwrap(); + let source = app_state.source.lock().await; let versions: IndexFile = match source.read_file([scope, name_part], &app_state.project, None)? { Some(versions) => toml::de::from_str(&versions)?, diff --git a/registry/src/endpoints/publish_version.rs b/registry/src/endpoints/publish_version.rs index d1fa7a2..5c9e446 100644 --- a/registry/src/endpoints/publish_version.rs +++ b/registry/src/endpoints/publish_version.rs @@ -1,17 +1,3 @@ -use actix_multipart::Multipart; -use actix_web::{web, HttpResponse, Responder}; -use convert_case::{Case, Casing}; -use flate2::read::GzDecoder; -use futures::{future::join_all, join, StreamExt}; -use git2::{Remote, Repository, Signature}; -use serde::Deserialize; -use sha2::{Digest, Sha256}; -use std::{ - collections::{BTreeSet, HashMap}, - io::{Cursor, Read, Write}, -}; -use tar::Archive; - use crate::{ auth::UserId, benv, @@ -20,6 +6,13 @@ use crate::{ storage::StorageImpl, AppState, }; +use actix_multipart::Multipart; +use actix_web::{web, HttpResponse, Responder}; +use async_compression::Level; +use convert_case::{Case, Casing}; +use fs_err::tokio as fs; +use futures::{future::join_all, join, StreamExt}; +use git2::{Remote, Repository, Signature}; use pesde::{ manifest::Manifest, source::{ @@ -31,6 +24,13 @@ use pesde::{ }, MANIFEST_FILE_NAME, }; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::{ + collections::{BTreeSet, HashMap}, + io::{Cursor, Write}, +}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; fn signature<'a>() -> Signature<'a> { Signature::now( @@ -71,18 +71,16 @@ pub async fn publish_package( mut body: Multipart, user_id: web::ReqData, ) -> Result { - let max_archive_size = { - let source = app_state.source.lock().unwrap(); - source.refresh(&app_state.project).map_err(Box::new)?; - source.config(&app_state.project)?.max_archive_size - }; + let source = app_state.source.lock().await; + source.refresh(&app_state.project).await.map_err(Box::new)?; + let config = source.config(&app_state.project)?; let bytes = body .next() .await .ok_or(Error::InvalidArchive)? .map_err(|_| Error::InvalidArchive)? - .bytes(max_archive_size) + .bytes(config.max_archive_size) .await .map_err(|_| Error::InvalidArchive)? .map_err(|_| Error::InvalidArchive)?; @@ -90,10 +88,10 @@ pub async fn publish_package( let package_dir = tempfile::tempdir()?; { - let mut decoder = GzDecoder::new(Cursor::new(&bytes)); - let mut archive = Archive::new(&mut decoder); + let mut decoder = async_compression::tokio::bufread::GzipDecoder::new(Cursor::new(&bytes)); + let mut archive = tokio_tar::Archive::new(&mut decoder); - archive.unpack(package_dir.path())?; + archive.unpack(package_dir.path()).await?; } let mut manifest = None::; @@ -101,15 +99,15 @@ pub async fn publish_package( let mut docs = BTreeSet::new(); let mut docs_pages = HashMap::new(); - for entry in fs_err::read_dir(package_dir.path())? { - let entry = entry?; + let mut read_dir = fs::read_dir(package_dir.path()).await?; + while let Some(entry) = read_dir.next_entry().await? { let file_name = entry .file_name() .to_str() .ok_or(Error::InvalidArchive)? .to_string(); - if entry.file_type()?.is_dir() { + if entry.file_type().await?.is_dir() { if IGNORED_DIRS.contains(&file_name.as_str()) { return Err(Error::InvalidArchive); } @@ -117,23 +115,22 @@ pub async fn publish_package( if file_name == "docs" { let mut stack = vec![( BTreeSet::new(), - fs_err::read_dir(entry.path())?, + fs::read_dir(entry.path()).await?, None::, )]; 'outer: while let Some((set, iter, category_info)) = stack.last_mut() { - for entry in iter { - let entry = entry?; + while let Some(entry) = iter.next_entry().await? { let file_name = entry .file_name() .to_str() .ok_or(Error::InvalidArchive)? .to_string(); - if entry.file_type()?.is_dir() { + if entry.file_type().await?.is_dir() { stack.push(( BTreeSet::new(), - fs_err::read_dir(entry.path())?, + fs::read_dir(entry.path()).await?, Some(DocEntryInfo { label: Some(file_name.to_case(Case::Title)), ..Default::default() @@ -143,7 +140,7 @@ pub async fn publish_package( } if file_name == "_category_.json" { - let info = fs_err::read_to_string(entry.path())?; + let info = fs::read_to_string(entry.path()).await?; let mut info: DocEntryInfo = serde_json::from_str(&info)?; let old_info = category_info.take(); info.label = info.label.or(old_info.and_then(|i| i.label)); @@ -155,16 +152,16 @@ pub async fn publish_package( continue; }; - let content = fs_err::read_to_string(entry.path())?; + let content = fs::read_to_string(entry.path()).await?; let content = content.trim(); let hash = format!("{:x}", Sha256::digest(content.as_bytes())); - let mut gz = flate2::read::GzEncoder::new( + let mut gz = async_compression::tokio::bufread::GzipEncoder::with_quality( Cursor::new(content.as_bytes().to_vec()), - flate2::Compression::best(), + Level::Best, ); let mut bytes = vec![]; - gz.read_to_end(&mut bytes)?; + gz.read_to_end(&mut bytes).await?; docs_pages.insert(hash.to_string(), bytes); let mut lines = content.lines().peekable(); @@ -245,7 +242,7 @@ pub async fn publish_package( } if file_name == MANIFEST_FILE_NAME { - let content = fs_err::read_to_string(entry.path())?; + let content = fs::read_to_string(entry.path()).await?; manifest = Some(toml::de::from_str(&content)?); } else if file_name @@ -258,12 +255,12 @@ pub async fn publish_package( return Err(Error::InvalidArchive); } - let file = fs_err::File::open(entry.path())?; + let mut file = fs::File::open(entry.path()).await?; - let mut gz = flate2::read::GzEncoder::new(file, flate2::Compression::best()); - let mut bytes = vec![]; - gz.read_to_end(&mut bytes)?; - readme = Some(bytes); + let mut gz = async_compression::tokio::write::GzipEncoder::new(vec![]); + tokio::io::copy(&mut file, &mut gz).await?; + gz.shutdown().await?; + readme = Some(gz.into_inner()); } } @@ -272,10 +269,6 @@ pub async fn publish_package( }; { - let source = app_state.source.lock().unwrap(); - source.refresh(&app_state.project).map_err(Box::new)?; - let config = source.config(&app_state.project)?; - let dependencies = manifest .all_dependencies() .map_err(|_| Error::InvalidArchive)?; diff --git a/registry/src/endpoints/search.rs b/registry/src/endpoints/search.rs index f9946b7..7e331be 100644 --- a/registry/src/endpoints/search.rs +++ b/registry/src/endpoints/search.rs @@ -59,7 +59,7 @@ pub async fn search_packages( ) .unwrap(); - let source = app_state.source.lock().unwrap(); + let source = app_state.source.lock().await; let top_docs = top_docs .into_iter() diff --git a/registry/src/main.rs b/registry/src/main.rs index a866576..984e501 100644 --- a/registry/src/main.rs +++ b/registry/src/main.rs @@ -5,13 +5,13 @@ use actix_web::{ rt::System, web, App, HttpServer, }; +use fs_err::tokio as fs; use log::info; -use std::{env::current_dir, path::PathBuf, sync::Mutex}; - use pesde::{ source::{pesde::PesdePackageSource, traits::PackageSource}, AuthConfig, Project, }; +use std::{env::current_dir, path::PathBuf}; use crate::{ auth::{get_auth_from_env, Auth, UserIdExtractor}, @@ -38,13 +38,13 @@ pub fn make_reqwest() -> reqwest::Client { } pub struct AppState { - pub source: Mutex, + pub source: tokio::sync::Mutex, pub project: Project, pub storage: Storage, pub auth: Auth, pub search_reader: tantivy::IndexReader, - pub search_writer: Mutex, + pub search_writer: std::sync::Mutex, } #[macro_export] @@ -87,7 +87,7 @@ async fn run() -> std::io::Result<()> { let cwd = current_dir().unwrap(); let data_dir = PathBuf::from(benv!("DATA_DIR" => "{CWD}/data").replace("{CWD}", cwd.to_str().unwrap())); - fs_err::create_dir_all(&data_dir).unwrap(); + fs::create_dir_all(&data_dir).await.unwrap(); let project = Project::new( &cwd, @@ -100,7 +100,10 @@ async fn run() -> std::io::Result<()> { })), ); let source = PesdePackageSource::new(benv!(required "INDEX_REPO_URL").try_into().unwrap()); - source.refresh(&project).expect("failed to refresh source"); + source + .refresh(&project) + .await + .expect("failed to refresh source"); let (search_reader, search_writer) = make_search(&project, &source); @@ -116,11 +119,11 @@ async fn run() -> std::io::Result<()> { info!("auth: {auth}"); auth }, - source: Mutex::new(source), + source: tokio::sync::Mutex::new(source), project, search_reader, - search_writer: Mutex::new(search_writer), + search_writer: std::sync::Mutex::new(search_writer), }); let publish_governor_config = GovernorConfigBuilder::default() diff --git a/registry/src/storage/fs.rs b/registry/src/storage/fs.rs index 8f050f7..1cc5857 100644 --- a/registry/src/storage/fs.rs +++ b/registry/src/storage/fs.rs @@ -3,6 +3,7 @@ use actix_web::{ http::header::{CONTENT_ENCODING, CONTENT_TYPE}, HttpResponse, }; +use fs_err::tokio as fs; use pesde::{names::PackageName, source::version_id::VersionId}; use std::{ fmt::Display, @@ -14,8 +15,8 @@ pub struct FSStorage { pub root: PathBuf, } -fn read_file_to_response(path: &Path, content_type: &str) -> Result { - Ok(match fs_err::read(path) { +async fn read_file_to_response(path: &Path, content_type: &str) -> Result { + Ok(match fs::read(path).await { Ok(contents) => HttpResponse::Ok() .append_header((CONTENT_TYPE, content_type)) .append_header((CONTENT_ENCODING, "gzip")) @@ -40,9 +41,9 @@ impl StorageImpl for FSStorage { .join(name) .join(version.version().to_string()) .join(version.target().to_string()); - fs_err::create_dir_all(&path)?; + fs::create_dir_all(&path).await?; - fs_err::write(path.join("pkg.tar.gz"), &contents)?; + fs::write(path.join("pkg.tar.gz"), &contents).await?; Ok(()) } @@ -61,7 +62,7 @@ impl StorageImpl for FSStorage { .join(version.version().to_string()) .join(version.target().to_string()); - read_file_to_response(&path.join("pkg.tar.gz"), "application/gzip") + read_file_to_response(&path.join("pkg.tar.gz"), "application/gzip").await } async fn store_readme( @@ -78,9 +79,9 @@ impl StorageImpl for FSStorage { .join(name) .join(version.version().to_string()) .join(version.target().to_string()); - fs_err::create_dir_all(&path)?; + fs::create_dir_all(&path).await?; - fs_err::write(path.join("readme.gz"), &contents)?; + fs::write(path.join("readme.gz"), &contents).await?; Ok(()) } @@ -99,14 +100,14 @@ impl StorageImpl for FSStorage { .join(version.version().to_string()) .join(version.target().to_string()); - read_file_to_response(&path.join("readme.gz"), "text/plain") + read_file_to_response(&path.join("readme.gz"), "text/plain").await } async fn store_doc(&self, doc_hash: String, contents: Vec) -> Result<(), Error> { let path = self.root.join("Doc"); - fs_err::create_dir_all(&path)?; + fs::create_dir_all(&path).await?; - fs_err::write(path.join(format!("{doc_hash}.gz")), &contents)?; + fs::write(path.join(format!("{doc_hash}.gz")), &contents).await?; Ok(()) } @@ -114,7 +115,7 @@ impl StorageImpl for FSStorage { async fn get_doc(&self, doc_hash: &str) -> Result { let path = self.root.join("Doc"); - read_file_to_response(&path.join(format!("{doc_hash}.gz")), "text/plain") + read_file_to_response(&path.join(format!("{doc_hash}.gz")), "text/plain").await } } diff --git a/src/cli/auth.rs b/src/cli/auth.rs index e940483..54ed228 100644 --- a/src/cli/auth.rs +++ b/src/cli/auth.rs @@ -37,8 +37,8 @@ impl<'de> Deserialize<'de> for Tokens { } } -pub fn get_tokens() -> anyhow::Result { - let config = read_config()?; +pub async fn get_tokens() -> anyhow::Result { + let config = read_config().await?; if !config.tokens.0.is_empty() { return Ok(config.tokens); } @@ -56,7 +56,7 @@ pub fn get_tokens() -> anyhow::Result { Ok(Tokens(BTreeMap::new())) } -pub fn set_tokens(tokens: Tokens) -> anyhow::Result<()> { +pub async fn set_tokens(tokens: Tokens) -> anyhow::Result<()> { let entry = Entry::new("tokens", env!("CARGO_PKG_NAME"))?; let json = serde_json::to_string(&tokens).context("failed to serialize tokens")?; @@ -66,19 +66,19 @@ pub fn set_tokens(tokens: Tokens) -> anyhow::Result<()> { Err(e) => return Err(e.into()), } - let mut config = read_config()?; + let mut config = read_config().await?; config.tokens = tokens; - write_config(&config).map_err(Into::into) + write_config(&config).await.map_err(Into::into) } -pub fn set_token(repo: &gix::Url, token: Option<&str>) -> anyhow::Result<()> { - let mut tokens = get_tokens()?; +pub async fn set_token(repo: &gix::Url, token: Option<&str>) -> anyhow::Result<()> { + let mut tokens = get_tokens().await?; if let Some(token) = token { tokens.0.insert(repo.clone(), token.to_string()); } else { tokens.0.remove(repo); } - set_tokens(tokens) + set_tokens(tokens).await } #[derive(Debug, Deserialize)] @@ -86,18 +86,20 @@ struct UserResponse { login: String, } -pub fn get_token_login( - reqwest: &reqwest::blocking::Client, +pub async fn get_token_login( + reqwest: &reqwest::Client, access_token: &str, ) -> anyhow::Result { let response = reqwest .get("https://api.github.com/user") .header(AUTHORIZATION, access_token) .send() + .await .context("failed to send user request")? .error_for_status() .context("failed to get user")? .json::() + .await .context("failed to parse user response")?; Ok(response.login) diff --git a/src/cli/commands/add.rs b/src/cli/commands/add.rs index fa55912..e1c2b61 100644 --- a/src/cli/commands/add.rs +++ b/src/cli/commands/add.rs @@ -47,9 +47,10 @@ pub struct AddCommand { } impl AddCommand { - pub fn run(self, project: Project) -> anyhow::Result<()> { + pub async fn run(self, project: Project) -> anyhow::Result<()> { let manifest = project .deser_manifest() + .await .context("failed to read manifest")?; let (source, specifier) = match &self.name { @@ -65,7 +66,10 @@ impl AddCommand { return Ok(()); } - let index = index.unwrap_or(read_config()?.default_index); + let index = match index { + Some(index) => index, + None => read_config().await?.default_index, + }; let source = PackageSources::Pesde(PesdePackageSource::new(index)); let specifier = DependencySpecifiers::Pesde(PesdeDependencySpecifier { @@ -89,7 +93,7 @@ impl AddCommand { return Ok(()); } - let index = index.unwrap_or(read_config()?.default_index); + let index = index.context("no wally index found")?; let source = PackageSources::Wally(pesde::source::wally::WallyPackageSource::new(index)); @@ -125,10 +129,12 @@ impl AddCommand { }; source .refresh(&project) + .await .context("failed to refresh package source")?; let Some(version_id) = source .resolve(&specifier, &project, manifest.target.kind()) + .await .context("failed to resolve package")? .1 .pop_last() @@ -141,7 +147,10 @@ impl AddCommand { let project_target = manifest.target.kind(); let mut manifest = toml_edit::DocumentMut::from_str( - &project.read_manifest().context("failed to read manifest")?, + &project + .read_manifest() + .await + .context("failed to read manifest")?, ) .context("failed to parse manifest")?; let dependency_key = if self.peer { @@ -227,6 +236,7 @@ impl AddCommand { project .write_manifest(manifest.to_string()) + .await .context("failed to write manifest")?; Ok(()) diff --git a/src/cli/commands/auth/login.rs b/src/cli/commands/auth/login.rs index 52d3dff..c569ea2 100644 --- a/src/cli/commands/auth/login.rs +++ b/src/cli/commands/auth/login.rs @@ -3,6 +3,7 @@ use clap::Args; use colored::Colorize; use serde::Deserialize; use std::thread::spawn; +use tokio::time::sleep; use url::Url; use pesde::{ @@ -46,16 +47,19 @@ enum AccessTokenResponse { } impl LoginCommand { - pub fn authenticate_device_flow( + pub async fn authenticate_device_flow( &self, index_url: &gix::Url, project: &Project, - reqwest: &reqwest::blocking::Client, + reqwest: &reqwest::Client, ) -> anyhow::Result { println!("logging in into {index_url}"); let source = PesdePackageSource::new(index_url.clone()); - source.refresh(project).context("failed to refresh index")?; + source + .refresh(project) + .await + .context("failed to refresh index")?; let config = source .config(project) @@ -70,10 +74,12 @@ impl LoginCommand { &[("client_id", &client_id)], )?) .send() + .await .context("failed to send device code request")? .error_for_status() .context("failed to get device code response")? .json::() + .await .context("failed to parse device code response")?; println!( @@ -102,7 +108,7 @@ impl LoginCommand { let mut interval = std::time::Duration::from_secs(response.interval); while time_left > 0 { - std::thread::sleep(interval); + sleep(interval).await; time_left = time_left.saturating_sub(interval.as_secs()); let response = reqwest @@ -118,10 +124,12 @@ impl LoginCommand { ], )?) .send() + .await .context("failed to send access token request")? .error_for_status() .context("failed to get access token response")? .json::() + .await .context("failed to parse access token response")?; match response { @@ -149,16 +157,19 @@ impl LoginCommand { anyhow::bail!("code expired, please re-run the login command"); } - pub fn run( + pub async fn run( self, index_url: gix::Url, project: Project, - reqwest: reqwest::blocking::Client, + reqwest: reqwest::Client, ) -> anyhow::Result<()> { let token_given = self.token.is_some(); let token = match self.token { Some(token) => token, - None => self.authenticate_device_flow(&index_url, &project, &reqwest)?, + None => { + self.authenticate_device_flow(&index_url, &project, &reqwest) + .await? + } }; let token = if token_given { @@ -168,13 +179,13 @@ impl LoginCommand { let token = format!("Bearer {token}"); println!( "logged in as {} for {index_url}", - get_token_login(&reqwest, &token)?.bold() + get_token_login(&reqwest, &token).await?.bold() ); token }; - set_token(&index_url, Some(&token))?; + set_token(&index_url, Some(&token)).await?; Ok(()) } diff --git a/src/cli/commands/auth/logout.rs b/src/cli/commands/auth/logout.rs index 3c6a155..e7363cc 100644 --- a/src/cli/commands/auth/logout.rs +++ b/src/cli/commands/auth/logout.rs @@ -5,8 +5,8 @@ use clap::Args; pub struct LogoutCommand {} impl LogoutCommand { - pub fn run(self, index_url: gix::Url) -> anyhow::Result<()> { - set_token(&index_url, None)?; + pub async fn run(self, index_url: gix::Url) -> anyhow::Result<()> { + set_token(&index_url, None).await?; println!("logged out of {index_url}"); diff --git a/src/cli/commands/auth/mod.rs b/src/cli/commands/auth/mod.rs index bcde22d..cbb1160 100644 --- a/src/cli/commands/auth/mod.rs +++ b/src/cli/commands/auth/mod.rs @@ -28,8 +28,8 @@ pub enum AuthCommands { } impl AuthSubcommand { - pub fn run(self, project: Project, reqwest: reqwest::blocking::Client) -> anyhow::Result<()> { - let manifest = match project.deser_manifest() { + pub async fn run(self, project: Project, reqwest: reqwest::Client) -> anyhow::Result<()> { + let manifest = match project.deser_manifest().await { Ok(manifest) => Some(manifest), Err(e) => match e { ManifestReadError::Io(e) if e.kind() == std::io::ErrorKind::NotFound => None, @@ -44,7 +44,7 @@ impl AuthSubcommand { }, None => match manifest { Some(_) => None, - None => Some(read_config()?.default_index), + None => Some(read_config().await?.default_index), }, }; @@ -61,9 +61,9 @@ impl AuthSubcommand { }; match self.command { - AuthCommands::Login(login) => login.run(index_url, project, reqwest), - AuthCommands::Logout(logout) => logout.run(index_url), - AuthCommands::WhoAmI(whoami) => whoami.run(index_url, reqwest), + AuthCommands::Login(login) => login.run(index_url, project, reqwest).await, + AuthCommands::Logout(logout) => logout.run(index_url).await, + AuthCommands::WhoAmI(whoami) => whoami.run(index_url, reqwest).await, } } } diff --git a/src/cli/commands/auth/whoami.rs b/src/cli/commands/auth/whoami.rs index b89104a..b0459cf 100644 --- a/src/cli/commands/auth/whoami.rs +++ b/src/cli/commands/auth/whoami.rs @@ -6,12 +6,8 @@ use colored::Colorize; pub struct WhoAmICommand {} impl WhoAmICommand { - pub fn run( - self, - index_url: gix::Url, - reqwest: reqwest::blocking::Client, - ) -> anyhow::Result<()> { - let tokens = get_tokens()?; + pub async fn run(self, index_url: gix::Url, reqwest: reqwest::Client) -> anyhow::Result<()> { + let tokens = get_tokens().await?; let token = match tokens.0.get(&index_url) { Some(token) => token, None => { @@ -22,7 +18,7 @@ impl WhoAmICommand { println!( "logged in as {} into {index_url}", - get_token_login(&reqwest, token)?.bold() + get_token_login(&reqwest, token).await?.bold() ); Ok(()) diff --git a/src/cli/commands/config/default_index.rs b/src/cli/commands/config/default_index.rs index 895c763..bc93ae0 100644 --- a/src/cli/commands/config/default_index.rs +++ b/src/cli/commands/config/default_index.rs @@ -13,8 +13,8 @@ pub struct DefaultIndexCommand { } impl DefaultIndexCommand { - pub fn run(self) -> anyhow::Result<()> { - let mut config = read_config()?; + pub async fn run(self) -> anyhow::Result<()> { + let mut config = read_config().await?; let index = if self.reset { Some(CliConfig::default().default_index) @@ -25,7 +25,7 @@ impl DefaultIndexCommand { match index { Some(index) => { config.default_index = index.clone(); - write_config(&config)?; + write_config(&config).await?; println!("default index set to: {index}"); } None => { diff --git a/src/cli/commands/config/mod.rs b/src/cli/commands/config/mod.rs index 5cecba5..a10570a 100644 --- a/src/cli/commands/config/mod.rs +++ b/src/cli/commands/config/mod.rs @@ -13,10 +13,10 @@ pub enum ConfigCommands { } impl ConfigCommands { - pub fn run(self) -> anyhow::Result<()> { + pub async fn run(self) -> anyhow::Result<()> { match self { - ConfigCommands::DefaultIndex(default_index) => default_index.run(), - ConfigCommands::ScriptsRepo(scripts_repo) => scripts_repo.run(), + ConfigCommands::DefaultIndex(default_index) => default_index.run().await, + ConfigCommands::ScriptsRepo(scripts_repo) => scripts_repo.run().await, } } } diff --git a/src/cli/commands/config/scripts_repo.rs b/src/cli/commands/config/scripts_repo.rs index 2c559e6..16d6525 100644 --- a/src/cli/commands/config/scripts_repo.rs +++ b/src/cli/commands/config/scripts_repo.rs @@ -4,6 +4,7 @@ use crate::cli::{ }; use anyhow::Context; use clap::Args; +use fs_err::tokio as fs; #[derive(Debug, Args)] pub struct ScriptsRepoCommand { @@ -17,8 +18,8 @@ pub struct ScriptsRepoCommand { } impl ScriptsRepoCommand { - pub fn run(self) -> anyhow::Result<()> { - let mut config = read_config()?; + pub async fn run(self) -> anyhow::Result<()> { + let mut config = read_config().await?; let repo = if self.reset { Some(CliConfig::default().scripts_repo) @@ -29,9 +30,10 @@ impl ScriptsRepoCommand { match repo { Some(repo) => { config.scripts_repo = repo.clone(); - write_config(&config)?; + write_config(&config).await?; - fs_err::remove_dir_all(home_dir()?.join("scripts")) + fs::remove_dir_all(home_dir()?.join("scripts")) + .await .context("failed to remove scripts directory")?; println!("scripts repo set to: {repo}"); diff --git a/src/cli/commands/execute.rs b/src/cli/commands/execute.rs index cd178d5..443d0b4 100644 --- a/src/cli/commands/execute.rs +++ b/src/cli/commands/execute.rs @@ -1,6 +1,7 @@ use crate::cli::{config::read_config, VersionedPackageName}; use anyhow::Context; use clap::Args; +use fs_err::tokio as fs; use pesde::{ linking::generator::generate_bin_linking_module, manifest::target::TargetKind, @@ -30,14 +31,16 @@ pub struct ExecuteCommand { } impl ExecuteCommand { - pub fn run(self, project: Project, reqwest: reqwest::blocking::Client) -> anyhow::Result<()> { - let index = self - .index - .or_else(|| read_config().ok().map(|c| c.default_index)) - .context("no index specified")?; + pub async fn run(self, project: Project, reqwest: reqwest::Client) -> anyhow::Result<()> { + let index = match self.index { + Some(index) => Some(index), + None => read_config().await.ok().map(|c| c.default_index), + } + .context("no index specified")?; let source = PesdePackageSource::new(index); source .refresh(&project) + .await .context("failed to refresh source")?; let version_req = self.package.1.unwrap_or(VersionReq::STAR); @@ -51,6 +54,7 @@ impl ExecuteCommand { if let Some(res) = source .resolve(&specifier, &project, TargetKind::Lune) + .await .context("failed to resolve package")? .1 .pop_last() @@ -60,6 +64,7 @@ impl ExecuteCommand { source .resolve(&specifier, &project, TargetKind::Luau) + .await .context("failed to resolve package")? .1 .pop_last() @@ -74,16 +79,20 @@ impl ExecuteCommand { let (fs, target) = source .download(&pkg_ref, &project, &reqwest) + .await .context("failed to download package")?; let bin_path = target.bin_path().context("package has no binary export")?; let tmp_dir = project.cas_dir().join(".tmp"); - fs_err::create_dir_all(&tmp_dir).context("failed to create temporary directory")?; + fs::create_dir_all(&tmp_dir) + .await + .context("failed to create temporary directory")?; let tempdir = tempfile::tempdir_in(tmp_dir).context("failed to create temporary directory")?; fs.write_to(tempdir.path(), project.cas_dir(), true) + .await .context("failed to write package contents")?; let mut caller = diff --git a/src/cli/commands/init.rs b/src/cli/commands/init.rs index 154550f..e3aec28 100644 --- a/src/cli/commands/init.rs +++ b/src/cli/commands/init.rs @@ -10,6 +10,7 @@ use pesde::{ }; use crate::cli::{config::read_config, HOME_DIR}; +use fs_err::tokio as fs; #[derive(Debug, Args)] pub struct InitCommand {} @@ -25,8 +26,8 @@ require(home_dir .. {:?})"#, } impl InitCommand { - pub fn run(self, project: Project) -> anyhow::Result<()> { - match project.read_manifest() { + pub async fn run(self, project: Project) -> anyhow::Result<()> { + match project.read_manifest().await { Ok(_) => { println!("{}", "project already initialized".red()); return Ok(()); @@ -125,25 +126,29 @@ impl InitCommand { let folder = project .package_dir() .join(concat!(".", env!("CARGO_PKG_NAME"))); - fs_err::create_dir_all(&folder).context("failed to create scripts folder")?; + fs::create_dir_all(&folder) + .await + .context("failed to create scripts folder")?; - fs_err::write( + fs::write( folder.join(format!("{}.luau", ScriptName::RobloxSyncConfigGenerator)), script_contents(Path::new(&format!( "lune/rojo/{}.luau", ScriptName::RobloxSyncConfigGenerator ))), ) + .await .context("failed to write sync config generator script file")?; #[cfg(feature = "wally-compat")] - fs_err::write( + fs::write( folder.join(format!("{}.luau", ScriptName::SourcemapGenerator)), script_contents(Path::new(&format!( "lune/rojo/{}.luau", ScriptName::SourcemapGenerator ))), ) + .await .context("failed to write sourcemap generator script file")?; let scripts = @@ -166,9 +171,9 @@ impl InitCommand { manifest["indices"].or_insert(toml_edit::Item::Table(toml_edit::Table::new())) [DEFAULT_INDEX_NAME] = - toml_edit::value(read_config()?.default_index.to_bstring().to_string()); + toml_edit::value(read_config().await?.default_index.to_bstring().to_string()); - project.write_manifest(manifest.to_string())?; + project.write_manifest(manifest.to_string()).await?; println!("{}", "initialized project".green()); Ok(()) diff --git a/src/cli/commands/install.rs b/src/cli/commands/install.rs index ec9f9cf..7dd001b 100644 --- a/src/cli/commands/install.rs +++ b/src/cli/commands/install.rs @@ -1,26 +1,21 @@ use crate::cli::{ - bin_dir, download_graph, files::make_executable, run_on_workspace_members, up_to_date_lockfile, + bin_dir, download_graph, files::make_executable, repos::update_scripts, + run_on_workspace_members, up_to_date_lockfile, }; use anyhow::Context; use clap::Args; use colored::{ColoredString, Colorize}; +use fs_err::tokio as fs; use indicatif::MultiProgress; use pesde::{ lockfile::Lockfile, manifest::{target::TargetKind, DependencyType}, Project, MANIFEST_FILE_NAME, }; -use std::{ - collections::{BTreeSet, HashSet}, - thread::JoinHandle, -}; +use std::collections::{BTreeSet, HashSet}; #[derive(Debug, Args, Copy, Clone)] pub struct InstallCommand { - /// The amount of threads to use for downloading - #[arg(short, long, default_value_t = 6, value_parser = clap::value_parser!(u64).range(1..=128))] - threads: u64, - /// Whether to error on changes in the lockfile #[arg(long)] locked: bool, @@ -91,21 +86,21 @@ fn job(n: u8) -> ColoredString { } impl InstallCommand { - pub fn run( + pub async fn run( self, project: Project, multi: MultiProgress, - reqwest: reqwest::blocking::Client, - update_task: &mut Option>, + reqwest: reqwest::Client, ) -> anyhow::Result<()> { let mut refreshed_sources = HashSet::new(); let manifest = project .deser_manifest() + .await .context("failed to read manifest")?; let lockfile = if self.locked { - match up_to_date_lockfile(&project)? { + match up_to_date_lockfile(&project).await? { None => { anyhow::bail!( "lockfile is out of sync, run `{} install` to update it", @@ -115,7 +110,7 @@ impl InstallCommand { file => file, } } else { - match project.deser_lockfile() { + match project.deser_lockfile().await { Ok(lockfile) => { if lockfile.overrides != manifest.overrides { log::debug!("overrides are different"); @@ -154,7 +149,8 @@ impl InstallCommand { if deleted_folders.insert(folder.to_string()) { log::debug!("deleting the {folder} folder"); - if let Some(e) = fs_err::remove_dir_all(project.package_dir().join(&folder)) + if let Some(e) = fs::remove_dir_all(project.package_dir().join(&folder)) + .await .err() .filter(|e| e.kind() != std::io::ErrorKind::NotFound) { @@ -184,12 +180,10 @@ impl InstallCommand { let graph = project .dependency_graph(old_graph.as_ref(), &mut refreshed_sources) + .await .context("failed to build dependency graph")?; - if let Some(task) = update_task.take() { - log::debug!("waiting for update task to finish"); - task.join().expect("failed to join update task"); - } + update_scripts(&project).await?; let downloaded_graph = download_graph( &project, @@ -197,12 +191,12 @@ impl InstallCommand { &graph, &multi, &reqwest, - self.threads as usize, self.prod, true, format!("{} 📥 downloading dependencies", job(3)), format!("{} 📥 downloaded dependencies", job(3)), - )?; + ) + .await?; let filtered_graph = if self.prod { downloaded_graph @@ -225,9 +219,10 @@ impl InstallCommand { project .link_dependencies(&filtered_graph) + .await .context("failed to link dependencies")?; - let bin_folder = bin_dir()?; + let bin_folder = bin_dir().await?; for versions in filtered_graph.values() { for node in versions.values() { @@ -245,18 +240,22 @@ impl InstallCommand { } let bin_file = bin_folder.join(alias); - fs_err::write(&bin_file, bin_link_file(alias)) + fs::write(&bin_file, bin_link_file(alias)) + .await .context("failed to write bin link file")?; - make_executable(&bin_file).context("failed to make bin link executable")?; + make_executable(&bin_file) + .await + .context("failed to make bin link executable")?; #[cfg(windows)] { let bin_file = bin_file.with_extension(std::env::consts::EXE_EXTENSION); - fs_err::copy( + fs::copy( std::env::current_exe().context("failed to get current executable path")?, &bin_file, ) + .await .context("failed to copy bin link file")?; } } @@ -268,6 +267,7 @@ impl InstallCommand { project .apply_patches(&filtered_graph) + .await .context("failed to apply patches")?; } @@ -283,9 +283,13 @@ impl InstallCommand { graph: downloaded_graph, workspace: run_on_workspace_members(&project, |project| { - self.run(project, multi.clone(), reqwest.clone(), &mut None) - })?, + let multi = multi.clone(); + let reqwest = reqwest.clone(); + async move { Box::pin(self.run(project, multi, reqwest)).await } + }) + .await?, }) + .await .context("failed to write lockfile")?; Ok(()) diff --git a/src/cli/commands/mod.rs b/src/cli/commands/mod.rs index feacfb8..e32b042 100644 --- a/src/cli/commands/mod.rs +++ b/src/cli/commands/mod.rs @@ -1,6 +1,5 @@ use indicatif::MultiProgress; use pesde::Project; -use std::thread::JoinHandle; mod add; mod auth; @@ -73,40 +72,31 @@ pub enum Subcommand { } impl Subcommand { - pub fn run( + pub async fn run( self, project: Project, multi: MultiProgress, - reqwest: reqwest::blocking::Client, - update_task: JoinHandle<()>, + reqwest: reqwest::Client, ) -> anyhow::Result<()> { - let mut update_task = Some(update_task); - - let res = match self { - Subcommand::Auth(auth) => auth.run(project, reqwest), - Subcommand::Config(config) => config.run(), - Subcommand::Init(init) => init.run(project), - Subcommand::Run(run) => run.run(project, &mut update_task), - Subcommand::Install(install) => install.run(project, multi, reqwest, &mut update_task), - Subcommand::Publish(publish) => publish.run(project, reqwest), + match self { + Subcommand::Auth(auth) => auth.run(project, reqwest).await, + Subcommand::Config(config) => config.run().await, + Subcommand::Init(init) => init.run(project).await, + Subcommand::Run(run) => run.run(project).await, + Subcommand::Install(install) => install.run(project, multi, reqwest).await, + Subcommand::Publish(publish) => publish.run(project, reqwest).await, #[cfg(feature = "version-management")] - Subcommand::SelfInstall(self_install) => self_install.run(), + Subcommand::SelfInstall(self_install) => self_install.run().await, #[cfg(feature = "patches")] - Subcommand::Patch(patch) => patch.run(project, reqwest), + Subcommand::Patch(patch) => patch.run(project, reqwest).await, #[cfg(feature = "patches")] - Subcommand::PatchCommit(patch_commit) => patch_commit.run(project), + Subcommand::PatchCommit(patch_commit) => patch_commit.run(project).await, #[cfg(feature = "version-management")] - Subcommand::SelfUpgrade(self_upgrade) => self_upgrade.run(reqwest), - Subcommand::Add(add) => add.run(project), - Subcommand::Update(update) => update.run(project, multi, reqwest, &mut update_task), - Subcommand::Outdated(outdated) => outdated.run(project), - Subcommand::Execute(execute) => execute.run(project, reqwest), - }; - - if let Some(handle) = update_task.take() { - handle.join().expect("failed to join update task"); + Subcommand::SelfUpgrade(self_upgrade) => self_upgrade.run(reqwest).await, + Subcommand::Add(add) => add.run(project).await, + Subcommand::Update(update) => update.run(project, multi, reqwest).await, + Subcommand::Outdated(outdated) => outdated.run(project).await, + Subcommand::Execute(execute) => execute.run(project, reqwest).await, } - - res } } diff --git a/src/cli/commands/outdated.rs b/src/cli/commands/outdated.rs index d9dde2e..dae004d 100644 --- a/src/cli/commands/outdated.rs +++ b/src/cli/commands/outdated.rs @@ -20,11 +20,12 @@ pub struct OutdatedCommand { } impl OutdatedCommand { - pub fn run(self, project: Project) -> anyhow::Result<()> { - let graph = project.deser_lockfile()?.graph; + pub async fn run(self, project: Project) -> anyhow::Result<()> { + let graph = project.deser_lockfile().await?.graph; let manifest = project .deser_manifest() + .await .context("failed to read manifest")?; let mut refreshed_sources = HashSet::new(); @@ -45,7 +46,7 @@ impl OutdatedCommand { let source = node.node.pkg_ref.source(); if refreshed_sources.insert(source.clone()) { - source.refresh(&project)?; + source.refresh(&project).await?; } if !self.strict { @@ -64,6 +65,7 @@ impl OutdatedCommand { let version_id = source .resolve(&specifier, &project, manifest.target.kind()) + .await .context("failed to resolve package versions")? .1 .pop_last() diff --git a/src/cli/commands/patch.rs b/src/cli/commands/patch.rs index 2790d1e..ea2518d 100644 --- a/src/cli/commands/patch.rs +++ b/src/cli/commands/patch.rs @@ -2,6 +2,7 @@ use crate::cli::{up_to_date_lockfile, VersionedPackageName}; use anyhow::Context; use clap::Args; use colored::Colorize; +use fs_err::tokio as fs; use pesde::{ patches::setup_patches_repo, source::{ @@ -19,8 +20,8 @@ pub struct PatchCommand { } impl PatchCommand { - pub fn run(self, project: Project, reqwest: reqwest::blocking::Client) -> anyhow::Result<()> { - let graph = if let Some(lockfile) = up_to_date_lockfile(&project)? { + pub async fn run(self, project: Project, reqwest: reqwest::Client) -> anyhow::Result<()> { + let graph = if let Some(lockfile) = up_to_date_lockfile(&project).await? { lockfile.graph } else { anyhow::bail!("outdated lockfile, please run the install command first") @@ -45,12 +46,14 @@ impl PatchCommand { .join(name.escaped()) .join(version_id.escaped()) .join(chrono::Utc::now().timestamp().to_string()); - fs_err::create_dir_all(&directory)?; + fs::create_dir_all(&directory).await?; source - .download(&node.node.pkg_ref, &project, &reqwest)? + .download(&node.node.pkg_ref, &project, &reqwest) + .await? .0 .write_to(&directory, project.cas_dir(), false) + .await .context("failed to write package contents")?; setup_patches_repo(&directory)?; diff --git a/src/cli/commands/patch_commit.rs b/src/cli/commands/patch_commit.rs index 4c8be10..763462f 100644 --- a/src/cli/commands/patch_commit.rs +++ b/src/cli/commands/patch_commit.rs @@ -1,6 +1,7 @@ use crate::cli::up_to_date_lockfile; use anyhow::Context; use clap::Args; +use fs_err::tokio as fs; use pesde::{names::PackageNames, patches::create_patch, source::version_id::VersionId, Project}; use std::{path::PathBuf, str::FromStr}; @@ -12,8 +13,8 @@ pub struct PatchCommitCommand { } impl PatchCommitCommand { - pub fn run(self, project: Project) -> anyhow::Result<()> { - let graph = if let Some(lockfile) = up_to_date_lockfile(&project)? { + pub async fn run(self, project: Project) -> anyhow::Result<()> { + let graph = if let Some(lockfile) = up_to_date_lockfile(&project).await? { lockfile.graph } else { anyhow::bail!("outdated lockfile, please run the install command first") @@ -48,15 +49,22 @@ impl PatchCommitCommand { .context("package not found in graph")?; let mut manifest = toml_edit::DocumentMut::from_str( - &project.read_manifest().context("failed to read manifest")?, + &project + .read_manifest() + .await + .context("failed to read manifest")?, ) .context("failed to parse manifest")?; let patch = create_patch(&self.directory).context("failed to create patch")?; - fs_err::remove_dir_all(self.directory).context("failed to remove patch directory")?; + fs::remove_dir_all(self.directory) + .await + .context("failed to remove patch directory")?; let patches_dir = project.package_dir().join("patches"); - fs_err::create_dir_all(&patches_dir).context("failed to create patches directory")?; + fs::create_dir_all(&patches_dir) + .await + .context("failed to create patches directory")?; let patch_file_name = format!("{}-{}.patch", name.escaped(), version_id.escaped()); @@ -65,7 +73,9 @@ impl PatchCommitCommand { anyhow::bail!("patch file already exists: {}", patch_file.display()); } - fs_err::write(&patch_file, patch).context("failed to write patch file")?; + fs::write(&patch_file, patch) + .await + .context("failed to write patch file")?; manifest["patches"].or_insert(toml_edit::Item::Table(toml_edit::Table::new())) [&name.to_string()][&version_id.to_string()] = @@ -73,6 +83,7 @@ impl PatchCommitCommand { project .write_manifest(manifest.to_string()) + .await .context("failed to write manifest")?; println!(concat!( diff --git a/src/cli/commands/publish.rs b/src/cli/commands/publish.rs index 49d5fbc..233e148 100644 --- a/src/cli/commands/publish.rs +++ b/src/cli/commands/publish.rs @@ -1,15 +1,9 @@ +use crate::cli::{display_err, run_on_workspace_members, up_to_date_lockfile}; use anyhow::Context; +use async_compression::Level; use clap::Args; use colored::Colorize; -use reqwest::{header::AUTHORIZATION, StatusCode}; -use semver::VersionReq; -use std::{ - io::{Seek, Write}, - path::Component, -}; -use tempfile::tempfile; - -use crate::cli::{run_on_workspace_members, up_to_date_lockfile}; +use fs_err::tokio as fs; use pesde::{ manifest::{target::Target, DependencyType}, scripts::ScriptName, @@ -25,6 +19,11 @@ use pesde::{ }, Project, DEFAULT_INDEX_NAME, MANIFEST_FILE_NAME, }; +use reqwest::{header::AUTHORIZATION, StatusCode}; +use semver::VersionReq; +use std::path::Component; +use tempfile::Builder; +use tokio::io::{AsyncSeekExt, AsyncWriteExt}; #[derive(Debug, Args, Clone)] pub struct PublishCommand { @@ -42,9 +41,10 @@ pub struct PublishCommand { } impl PublishCommand { - fn run_impl(self, project: &Project, reqwest: reqwest::blocking::Client) -> anyhow::Result<()> { + async fn run_impl(self, project: &Project, reqwest: reqwest::Client) -> anyhow::Result<()> { let mut manifest = project .deser_manifest() + .await .context("failed to read manifest")?; println!( @@ -72,7 +72,7 @@ impl PublishCommand { anyhow::bail!("no build files found in target"); } - match up_to_date_lockfile(project)? { + match up_to_date_lockfile(project).await? { Some(lockfile) => { if lockfile .graph @@ -93,10 +93,9 @@ impl PublishCommand { } } - let mut archive = tar::Builder::new(flate2::write::GzEncoder::new( - vec![], - flate2::Compression::best(), - )); + let mut archive = tokio_tar::Builder::new( + async_compression::tokio::write::GzipEncoder::with_quality(vec![], Level::Best), + ); let mut display_includes: Vec = vec![MANIFEST_FILE_NAME.to_string()]; let mut display_build_files: Vec = vec![]; @@ -179,8 +178,9 @@ impl PublishCommand { anyhow::bail!("{name} must point to a file"); } - let contents = - fs_err::read_to_string(&export_path).context(format!("failed to read {name}"))?; + let contents = fs::read_to_string(&export_path) + .await + .context(format!("failed to read {name}"))?; if let Err(err) = full_moon::parse(&contents).map_err(|errs| { errs.into_iter() @@ -237,17 +237,21 @@ impl PublishCommand { if included_path.is_file() { display_includes.push(included_name.clone()); - archive.append_file( - included_name, - fs_err::File::open(&included_path) - .context(format!("failed to read {included_name}"))? - .file_mut(), - )?; + archive + .append_file( + included_name, + fs::File::open(&included_path) + .await + .context(format!("failed to read {included_name}"))? + .file_mut(), + ) + .await?; } else { display_includes.push(format!("{included_name}/*")); archive .append_dir_all(included_name, &included_path) + .await .context(format!("failed to include directory {included_name}"))?; } } @@ -331,6 +335,7 @@ impl PublishCommand { DependencySpecifiers::Workspace(spec) => { let pkg_ref = WorkspacePackageSource .resolve(spec, project, target_kind) + .await .context("failed to resolve workspace package")? .1 .pop_last() @@ -345,7 +350,8 @@ impl PublishCommand { .context("failed to get workspace directory")?, ) .join(MANIFEST_FILE_NAME); - let manifest = fs_err::read_to_string(&manifest) + let manifest = fs::read_to_string(&manifest) + .await .context("failed to read workspace package manifest")?; let manifest = toml::from_str::(&manifest) .context("failed to parse workspace package manifest")?; @@ -442,25 +448,41 @@ impl PublishCommand { println!(); } - let mut temp_manifest = tempfile().context("failed to create temp manifest file")?; + let temp_path = Builder::new().make(|_| Ok(()))?.into_temp_path(); + let mut temp_manifest = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .read(true) + .open(temp_path.to_path_buf()) + .await?; + temp_manifest .write_all( toml::to_string(&manifest) .context("failed to serialize manifest")? .as_bytes(), ) + .await .context("failed to write temp manifest file")?; temp_manifest .rewind() + .await .context("failed to rewind temp manifest file")?; - archive.append_file(MANIFEST_FILE_NAME, &mut temp_manifest)?; + archive + .append_file(MANIFEST_FILE_NAME, temp_manifest.file_mut()) + .await?; - let archive = archive + let mut encoder = archive .into_inner() - .context("failed to encode archive")? - .finish() - .context("failed to get archive bytes")?; + .await + .context("failed to finish archive")?; + encoder + .shutdown() + .await + .context("failed to finish archive")?; + let archive = encoder.into_inner(); let index_url = manifest .indices @@ -469,6 +491,7 @@ impl PublishCommand { let source = PesdePackageSource::new(index_url.clone()); source .refresh(project) + .await .context("failed to refresh source")?; let config = source .config(project) @@ -494,7 +517,7 @@ impl PublishCommand { } if self.dry_run { - fs_err::write("package.tar.gz", archive)?; + fs::write("package.tar.gz", archive).await?; println!( "{}", @@ -506,9 +529,9 @@ impl PublishCommand { let mut request = reqwest .post(format!("{}/v0/packages", config.api())) - .multipart(reqwest::blocking::multipart::Form::new().part( + .multipart(reqwest::multipart::Form::new().part( "tarball", - reqwest::blocking::multipart::Part::bytes(archive).file_name("package.tar.gz"), + reqwest::multipart::Part::bytes(archive).file_name("package.tar.gz"), )); if let Some(token) = project.auth_config().tokens().get(index_url) { @@ -516,10 +539,13 @@ impl PublishCommand { request = request.header(AUTHORIZATION, token); } - let response = request.send().context("failed to send request")?; + let response = request.send().await.context("failed to send request")?; let status = response.status(); - let text = response.text().context("failed to get response text")?; + let text = response + .text() + .await + .context("failed to get response text")?; match status { StatusCode::CONFLICT => { println!("{}", "package version already exists".red().bold()); @@ -544,17 +570,20 @@ impl PublishCommand { Ok(()) } - pub fn run(self, project: Project, reqwest: reqwest::blocking::Client) -> anyhow::Result<()> { - let result = self.clone().run_impl(&project, reqwest.clone()); + pub async fn run(self, project: Project, reqwest: reqwest::Client) -> anyhow::Result<()> { + let result = self.clone().run_impl(&project, reqwest.clone()).await; if project.workspace_dir().is_some() { return result; - } else if let Err(result) = result { - println!("an error occurred publishing workspace root: {result}"); + } else { + display_err(result, " occurred publishing workspace root"); } run_on_workspace_members(&project, |project| { - self.clone().run_impl(&project, reqwest.clone()) + let reqwest = reqwest.clone(); + let this = self.clone(); + async move { this.run_impl(&project, reqwest).await } }) + .await .map(|_| ()) } } diff --git a/src/cli/commands/run.rs b/src/cli/commands/run.rs index c07370d..bda06e9 100644 --- a/src/cli/commands/run.rs +++ b/src/cli/commands/run.rs @@ -1,4 +1,4 @@ -use crate::cli::up_to_date_lockfile; +use crate::cli::{repos::update_scripts, up_to_date_lockfile}; use anyhow::Context; use clap::Args; use pesde::{ @@ -8,9 +8,8 @@ use pesde::{ Project, PACKAGES_CONTAINER_NAME, }; use relative_path::RelativePathBuf; -use std::{ - env::current_dir, ffi::OsString, io::Write, path::PathBuf, process::Command, thread::JoinHandle, -}; +use std::{env::current_dir, ffi::OsString, io::Write, path::PathBuf, process::Command}; +use tokio::runtime::Handle; #[derive(Debug, Args)] pub struct RunCommand { @@ -24,15 +23,11 @@ pub struct RunCommand { } impl RunCommand { - pub fn run( - self, - project: Project, - update_task: &mut Option>, - ) -> anyhow::Result<()> { - let mut run = |path: PathBuf| { - if let Some(handle) = update_task.take() { - handle.join().expect("failed to join update task"); - } + pub async fn run(self, project: Project) -> anyhow::Result<()> { + let run = |path: PathBuf| { + Handle::current() + .block_on(update_scripts(&project)) + .expect("failed to update scripts"); let mut caller = tempfile::NamedTempFile::new().expect("failed to create tempfile"); caller @@ -62,7 +57,7 @@ impl RunCommand { let package_or_script = match self.package_or_script { Some(package_or_script) => package_or_script, None => { - if let Some(script_path) = project.deser_manifest()?.target.bin_path() { + if let Some(script_path) = project.deser_manifest().await?.target.bin_path() { run(script_path.to_path(project.package_dir())); } @@ -71,7 +66,7 @@ impl RunCommand { }; if let Ok(pkg_name) = package_or_script.parse::() { - let graph = if let Some(lockfile) = up_to_date_lockfile(&project)? { + let graph = if let Some(lockfile) = up_to_date_lockfile(&project).await? { lockfile.graph } else { anyhow::bail!("outdated lockfile, please run the install command first") @@ -89,7 +84,8 @@ impl RunCommand { }; let base_folder = project - .deser_manifest()? + .deser_manifest() + .await? .target .kind() .packages_folder(&node.node.pkg_ref.target_kind()); @@ -106,7 +102,7 @@ impl RunCommand { } } - if let Ok(manifest) = project.deser_manifest() { + if let Ok(manifest) = project.deser_manifest().await { if let Some(script_path) = manifest.scripts.get(&package_or_script) { run(script_path.to_path(project.package_dir())) } diff --git a/src/cli/commands/self_install.rs b/src/cli/commands/self_install.rs index a122baa..7b68a41 100644 --- a/src/cli/commands/self_install.rs +++ b/src/cli/commands/self_install.rs @@ -10,7 +10,7 @@ pub struct SelfInstallCommand { } impl SelfInstallCommand { - pub fn run(self) -> anyhow::Result<()> { + pub async fn run(self) -> anyhow::Result<()> { #[cfg(windows)] { if !self.skip_add_to_path { @@ -24,7 +24,7 @@ impl SelfInstallCommand { .0; let path: String = env.get_value("Path").context("failed to get Path value")?; - let bin_dir = crate::cli::bin_dir()?; + let bin_dir = crate::cli::bin_dir().await?; let bin_dir = bin_dir.to_string_lossy(); let exists = path.split(';').any(|part| *part == bin_dir); @@ -68,7 +68,7 @@ and then restart your shell. ); } - update_bin_exe()?; + update_bin_exe().await?; Ok(()) } diff --git a/src/cli/commands/self_upgrade.rs b/src/cli/commands/self_upgrade.rs index 5c66051..2644d01 100644 --- a/src/cli/commands/self_upgrade.rs +++ b/src/cli/commands/self_upgrade.rs @@ -5,10 +5,10 @@ use clap::Args; pub struct SelfUpgradeCommand {} impl SelfUpgradeCommand { - pub fn run(self, reqwest: reqwest::blocking::Client) -> anyhow::Result<()> { - let config = read_config()?; + pub async fn run(self, reqwest: reqwest::Client) -> anyhow::Result<()> { + let config = read_config().await?; - get_or_download_version(&reqwest, &config.last_checked_updates.unwrap().1)?; + get_or_download_version(&reqwest, &config.last_checked_updates.unwrap().1).await?; // a call to `update_bin_exe` or other similar function *should* be here, in case new versions // have fixes to bugs in executing other versions, but that would cause // the current file to be overwritten by itself, so this needs more thought diff --git a/src/cli/commands/update.rs b/src/cli/commands/update.rs index 5ac1b42..8f2674b 100644 --- a/src/cli/commands/update.rs +++ b/src/cli/commands/update.rs @@ -1,30 +1,26 @@ -use crate::cli::{download_graph, run_on_workspace_members}; +use crate::cli::{download_graph, repos::update_scripts, run_on_workspace_members}; use anyhow::Context; use clap::Args; use colored::Colorize; use indicatif::MultiProgress; use pesde::{lockfile::Lockfile, Project}; -use std::{collections::HashSet, thread::JoinHandle}; +use std::collections::HashSet; #[derive(Debug, Args, Copy, Clone)] -pub struct UpdateCommand { - /// The amount of threads to use for downloading - #[arg(short, long, default_value_t = 6, value_parser = clap::value_parser!(u64).range(1..=128))] - threads: u64, -} +pub struct UpdateCommand {} impl UpdateCommand { - pub fn run( + pub async fn run( self, project: Project, multi: MultiProgress, - reqwest: reqwest::blocking::Client, - update_task: &mut Option>, + reqwest: reqwest::Client, ) -> anyhow::Result<()> { let mut refreshed_sources = HashSet::new(); let manifest = project .deser_manifest() + .await .context("failed to read manifest")?; println!( @@ -36,11 +32,10 @@ impl UpdateCommand { let graph = project .dependency_graph(None, &mut refreshed_sources) + .await .context("failed to build dependency graph")?; - if let Some(handle) = update_task.take() { - handle.join().expect("failed to join update task"); - } + update_scripts(&project).await?; project .write_lockfile(Lockfile { @@ -55,17 +50,21 @@ impl UpdateCommand { &graph, &multi, &reqwest, - self.threads as usize, false, false, "📥 downloading dependencies".to_string(), "📥 downloaded dependencies".to_string(), - )?, + ) + .await?, workspace: run_on_workspace_members(&project, |project| { - self.run(project, multi.clone(), reqwest.clone(), &mut None) - })?, + let multi = multi.clone(); + let reqwest = reqwest.clone(); + async move { Box::pin(self.run(project, multi, reqwest)).await } + }) + .await?, }) + .await .context("failed to write lockfile")?; Ok(()) diff --git a/src/cli/config.rs b/src/cli/config.rs index a793e6d..2049d51 100644 --- a/src/cli/config.rs +++ b/src/cli/config.rs @@ -1,5 +1,6 @@ use crate::cli::{auth::Tokens, home_dir}; use anyhow::Context; +use fs_err::tokio as fs; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -38,8 +39,8 @@ impl Default for CliConfig { } } -pub fn read_config() -> anyhow::Result { - let config_string = match fs_err::read_to_string(home_dir()?.join("config.toml")) { +pub async fn read_config() -> anyhow::Result { + let config_string = match fs::read_to_string(home_dir()?.join("config.toml")).await { Ok(config_string) => config_string, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { return Ok(CliConfig::default()); @@ -52,9 +53,10 @@ pub fn read_config() -> anyhow::Result { Ok(config) } -pub fn write_config(config: &CliConfig) -> anyhow::Result<()> { +pub async fn write_config(config: &CliConfig) -> anyhow::Result<()> { let config_string = toml::to_string(config).context("failed to serialize config")?; - fs_err::write(home_dir()?.join("config.toml"), config_string) + fs::write(home_dir()?.join("config.toml"), config_string) + .await .context("failed to write config file")?; Ok(()) diff --git a/src/cli/files.rs b/src/cli/files.rs index f41706f..c24b57c 100644 --- a/src/cli/files.rs +++ b/src/cli/files.rs @@ -1,16 +1,19 @@ use std::path::Path; -pub fn make_executable>(_path: P) -> anyhow::Result<()> { +pub async fn make_executable>(_path: P) -> anyhow::Result<()> { #[cfg(unix)] { use anyhow::Context; + use fs_err::tokio as fs; use std::os::unix::fs::PermissionsExt; - let mut perms = fs_err::metadata(&_path) + let mut perms = fs::metadata(&_path) + .await .context("failed to get bin link file metadata")? .permissions(); perms.set_mode(perms.mode() | 0o111); - fs_err::set_permissions(&_path, perms) + fs::set_permissions(&_path, perms) + .await .context("failed to set bin link file permissions")?; } diff --git a/src/cli/mod.rs b/src/cli/mod.rs index a15488c..e705f10 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1,4 +1,7 @@ use anyhow::Context; +use colored::Colorize; +use fs_err::tokio as fs; +use futures::StreamExt; use indicatif::MultiProgress; use pesde::{ lockfile::{DependencyGraph, DownloadedGraph, Lockfile}, @@ -10,11 +13,13 @@ use pesde::{ use relative_path::RelativePathBuf; use std::{ collections::{BTreeMap, HashSet}, + future::Future, path::PathBuf, str::FromStr, sync::Arc, time::Duration, }; +use tokio::pin; pub mod auth; pub mod commands; @@ -32,15 +37,17 @@ pub fn home_dir() -> anyhow::Result { .join(HOME_DIR)) } -pub fn bin_dir() -> anyhow::Result { +pub async fn bin_dir() -> anyhow::Result { let bin_dir = home_dir()?.join("bin"); - fs_err::create_dir_all(&bin_dir).context("failed to create bin folder")?; + fs::create_dir_all(&bin_dir) + .await + .context("failed to create bin folder")?; Ok(bin_dir) } -pub fn up_to_date_lockfile(project: &Project) -> anyhow::Result> { - let manifest = project.deser_manifest()?; - let lockfile = match project.deser_lockfile() { +pub async fn up_to_date_lockfile(project: &Project) -> anyhow::Result> { + let manifest = project.deser_manifest().await?; + let lockfile = match project.deser_lockfile().await { Ok(lockfile) => lockfile, Err(pesde::errors::LockfileReadError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => @@ -185,13 +192,12 @@ pub fn parse_gix_url(s: &str) -> Result { } #[allow(clippy::too_many_arguments)] -pub fn download_graph( +pub async fn download_graph( project: &Project, refreshed_sources: &mut HashSet, graph: &DependencyGraph, multi: &MultiProgress, - reqwest: &reqwest::blocking::Client, - threads: usize, + reqwest: &reqwest::Client, prod: bool, write: bool, progress_msg: String, @@ -207,11 +213,12 @@ pub fn download_graph( ); bar.enable_steady_tick(Duration::from_millis(100)); - let (rx, downloaded_graph) = project - .download_graph(graph, refreshed_sources, reqwest, threads, prod, write) + let (mut rx, downloaded_graph) = project + .download_graph(graph, refreshed_sources, reqwest, prod, write) + .await .context("failed to download dependencies")?; - while let Ok(result) = rx.recv() { + while let Some(result) = rx.recv().await { bar.inc(1); match result { @@ -238,41 +245,63 @@ pub fn shift_project_dir(project: &Project, pkg_dir: PathBuf) -> Project { ) } -pub fn run_on_workspace_members( +pub async fn run_on_workspace_members>>( project: &Project, - f: impl Fn(Project) -> anyhow::Result<()>, + f: impl Fn(Project) -> F, ) -> anyhow::Result>> { - Ok(match project.workspace_dir() { - Some(_) => { - // this might seem counterintuitive, but remember that - // the presence of a workspace dir means that this project is a member of one - Default::default() - } - None => project - .workspace_members(project.package_dir()) - .context("failed to get workspace members")? - .into_iter() - .map(|(path, manifest)| { - ( - manifest.name, - manifest.target.kind(), - RelativePathBuf::from_path(path.strip_prefix(project.package_dir()).unwrap()) - .unwrap(), - ) - }) - .map(|(name, target, path)| { - f(shift_project_dir( - project, - path.to_path(project.package_dir()), - )) - .map(|_| (name, target, path)) - }) - .collect::, _>>() - .context("failed to install workspace member's dependencies")? - .into_iter() - .fold(BTreeMap::new(), |mut map, (name, target, path)| { - map.entry(name).or_default().insert(target, path); - map - }), - }) + // this might seem counterintuitive, but remember that + // the presence of a workspace dir means that this project is a member of one + if project.workspace_dir().is_some() { + return Ok(Default::default()); + } + + let members_future = project.workspace_members(project.package_dir()).await?; + pin!(members_future); + + let mut results = BTreeMap::>::new(); + + while let Some((path, manifest)) = members_future.next().await.transpose()? { + let relative_path = + RelativePathBuf::from_path(path.strip_prefix(project.package_dir()).unwrap()).unwrap(); + + f(shift_project_dir(project, path)).await?; + + results + .entry(manifest.name) + .or_default() + .insert(manifest.target.kind(), relative_path); + } + + Ok(results) +} + +pub fn display_err(result: anyhow::Result<()>, prefix: &str) { + if let Err(err) = result { + eprintln!("{}: {err}\n", format!("error{prefix}").red().bold()); + + let cause = err.chain().skip(1).collect::>(); + + if !cause.is_empty() { + eprintln!("{}:", "caused by".red().bold()); + for err in cause { + eprintln!(" - {err}"); + } + } + + let backtrace = err.backtrace(); + match backtrace.status() { + std::backtrace::BacktraceStatus::Disabled => { + eprintln!( + "\n{}: set RUST_BACKTRACE=1 for a backtrace", + "help".yellow().bold() + ); + } + std::backtrace::BacktraceStatus::Captured => { + eprintln!("\n{}:\n{backtrace}", "backtrace".yellow().bold()); + } + _ => { + eprintln!("\n{}: not captured", "backtrace".yellow().bold()); + } + } + } } diff --git a/src/cli/repos.rs b/src/cli/repos.rs index c2bf3f9..695c4e4 100644 --- a/src/cli/repos.rs +++ b/src/cli/repos.rs @@ -3,18 +3,22 @@ use crate::{ util::authenticate_conn, }; use anyhow::Context; +use fs_err::tokio as fs; use gix::remote::{fetch::Shallow, Direction}; use pesde::Project; -use std::path::Path; +use std::{path::Path, sync::atomic::AtomicBool}; +use tokio::{runtime::Handle, task::spawn_blocking}; -fn update_repo>( +async fn update_repo>( name: &str, path: P, url: gix::Url, project: &Project, ) -> anyhow::Result<()> { let path = path.as_ref(); - if path.exists() { + let should_update = path.exists(); + + let (repo, oid) = if should_update { let repo = gix::open(path).context(format!("failed to open {name} repository"))?; let remote = repo @@ -49,68 +53,91 @@ fn update_repo>( .or(unpacked.2) .context("couldn't find oid in remote ref")?; - let tree = repo - .find_object(oid) - .context(format!("failed to find {name} repository tree"))? - .peel_to_tree() - .context(format!("failed to peel {name} repository object to tree"))?; - - let mut index = gix::index::File::from_state( - gix::index::State::from_tree(&tree.id, &repo.objects, Default::default()).context( - format!("failed to create index state from {name} repository tree"), - )?, - repo.index_path(), - ); - - let opts = gix::worktree::state::checkout::Options { - overwrite_existing: true, - destination_is_initially_empty: false, - ..Default::default() - }; - - gix::worktree::state::checkout( - &mut index, - repo.work_dir().context(format!("{name} repo is bare"))?, - repo.objects - .clone() - .into_arc() - .context("failed to clone objects")?, - &gix::progress::Discard, - &gix::progress::Discard, - &false.into(), - opts, - ) - .context(format!("failed to checkout {name} repository"))?; - - index - .write(gix::index::write::Options::default()) - .context("failed to write index")?; + (repo, gix::ObjectId::from(oid)) } else { - fs_err::create_dir_all(path).context(format!("failed to create {name} directory"))?; + fs::create_dir_all(path) + .await + .context(format!("failed to create {name} directory"))?; - gix::prepare_clone(url, path) + let repo = gix::prepare_clone(url, path) .context(format!("failed to prepare {name} repository clone"))? .with_shallow(Shallow::Deepen(1)) - .fetch_then_checkout(gix::progress::Discard, &false.into()) + .fetch_only(gix::progress::Discard, &false.into()) .context(format!("failed to fetch and checkout {name} repository"))? - .0 - .main_worktree(gix::progress::Discard, &false.into()) - .context(format!("failed to set {name} repository as main worktree"))?; + .0; + + let oid = { + let mut head = repo + .head() + .context(format!("failed to get {name} repository head"))?; + let obj = head + .peel_to_object_in_place() + .context(format!("failed to peel {name} repository head to object"))?; + + obj.id + }; + + (repo, oid) }; - Ok(()) + let tree = repo + .find_object(oid) + .context(format!("failed to find {name} repository tree"))? + .peel_to_tree() + .context(format!("failed to peel {name} repository object to tree"))?; + + let mut index = gix::index::File::from_state( + gix::index::State::from_tree(&tree.id, &repo.objects, Default::default()).context( + format!("failed to create index state from {name} repository tree"), + )?, + repo.index_path(), + ); + + let opts = gix::worktree::state::checkout::Options { + overwrite_existing: true, + destination_is_initially_empty: !should_update, + ..Default::default() + }; + + gix::worktree::state::checkout( + &mut index, + repo.work_dir().context(format!("{name} repo is bare"))?, + repo.objects + .clone() + .into_arc() + .context("failed to clone objects")?, + &gix::progress::Discard, + &gix::progress::Discard, + &false.into(), + opts, + ) + .context(format!("failed to checkout {name} repository"))?; + + index + .write(gix::index::write::Options::default()) + .context("failed to write index") } -pub fn update_repo_dependencies(project: &Project) -> anyhow::Result<()> { +static SCRIPTS_UPDATED: AtomicBool = AtomicBool::new(false); + +pub async fn update_scripts(project: &Project) -> anyhow::Result<()> { + if SCRIPTS_UPDATED.swap(true, std::sync::atomic::Ordering::Relaxed) { + return Ok(()); + } + let home_dir = home_dir()?; - let config = read_config()?; + let config = read_config().await?; - update_repo( - "scripts", - home_dir.join("scripts"), - config.scripts_repo, - project, - )?; + let project = project.clone(); + spawn_blocking(move || { + Handle::current().block_on(update_repo( + "scripts", + home_dir.join("scripts"), + config.scripts_repo, + &project, + )) + }) + .await??; Ok(()) } diff --git a/src/cli/version.rs b/src/cli/version.rs index 6e9700f..2633491 100644 --- a/src/cli/version.rs +++ b/src/cli/version.rs @@ -1,16 +1,18 @@ -use anyhow::Context; -use colored::Colorize; -use reqwest::header::ACCEPT; -use semver::Version; -use serde::Deserialize; -use std::{io::Read, path::PathBuf}; - use crate::cli::{ bin_dir, config::{read_config, write_config, CliConfig}, files::make_executable, home_dir, }; +use anyhow::Context; +use colored::Colorize; +use fs_err::tokio as fs; +use futures::StreamExt; +use reqwest::header::ACCEPT; +use semver::Version; +use serde::Deserialize; +use std::path::PathBuf; +use tokio::io::AsyncReadExt; pub fn current_version() -> Version { Version::parse(env!("CARGO_PKG_VERSION")).unwrap() @@ -38,10 +40,10 @@ fn get_repo() -> (String, String) { const CHECK_INTERVAL: chrono::Duration = chrono::Duration::hours(6); -pub fn check_for_updates(reqwest: &reqwest::blocking::Client) -> anyhow::Result<()> { +pub async fn check_for_updates(reqwest: &reqwest::Client) -> anyhow::Result<()> { let (owner, repo) = get_repo(); - let config = read_config()?; + let config = read_config().await?; let version = if let Some((_, version)) = config .last_checked_updates @@ -54,10 +56,12 @@ pub fn check_for_updates(reqwest: &reqwest::blocking::Client) -> anyhow::Result< "https://api.github.com/repos/{owner}/{repo}/releases", )) .send() + .await .context("failed to send request to GitHub API")? .error_for_status() .context("failed to get GitHub API response")? .json::>() + .await .context("failed to parse GitHub API response")?; let version = releases @@ -69,7 +73,8 @@ pub fn check_for_updates(reqwest: &reqwest::blocking::Client) -> anyhow::Result< write_config(&CliConfig { last_checked_updates: Some((chrono::Utc::now(), version.clone())), ..config - })?; + }) + .await?; version }; @@ -97,8 +102,8 @@ pub fn check_for_updates(reqwest: &reqwest::blocking::Client) -> anyhow::Result< Ok(()) } -pub fn download_github_release( - reqwest: &reqwest::blocking::Client, +pub async fn download_github_release( + reqwest: &reqwest::Client, version: &Version, ) -> anyhow::Result> { let (owner, repo) = get_repo(); @@ -108,10 +113,12 @@ pub fn download_github_release( "https://api.github.com/repos/{owner}/{repo}/releases/tags/v{version}", )) .send() + .await .context("failed to send request to GitHub API")? .error_for_status() .context("failed to get GitHub API response")? .json::() + .await .context("failed to parse GitHub API response")?; let asset = release @@ -130,34 +137,43 @@ pub fn download_github_release( .get(asset.url) .header(ACCEPT, "application/octet-stream") .send() + .await .context("failed to send request to download asset")? .error_for_status() .context("failed to download asset")? .bytes() + .await .context("failed to download asset")?; - let mut decoder = flate2::read::GzDecoder::new(bytes.as_ref()); - let mut archive = tar::Archive::new(&mut decoder); + let mut decoder = async_compression::tokio::bufread::GzipDecoder::new(bytes.as_ref()); + let mut archive = tokio_tar::Archive::new(&mut decoder); - let entry = archive + let mut entry = archive .entries() .context("failed to read archive entries")? .next() + .await .context("archive has no entry")? .context("failed to get first archive entry")?; + let mut result = Vec::new(); + entry - .bytes() - .collect::, std::io::Error>>() - .context("failed to read archive entry bytes") + .read_to_end(&mut result) + .await + .context("failed to read archive entry bytes")?; + + Ok(result) } -pub fn get_or_download_version( - reqwest: &reqwest::blocking::Client, +pub async fn get_or_download_version( + reqwest: &reqwest::Client, version: &Version, ) -> anyhow::Result> { let path = home_dir()?.join("versions"); - fs_err::create_dir_all(&path).context("failed to create versions directory")?; + fs::create_dir_all(&path) + .await + .context("failed to create versions directory")?; let path = path.join(format!("{version}{}", std::env::consts::EXE_SUFFIX)); @@ -172,14 +188,19 @@ pub fn get_or_download_version( } if is_requested_version { - fs_err::copy(std::env::current_exe()?, &path) + fs::copy(std::env::current_exe()?, &path) + .await .context("failed to copy current executable to version directory")?; } else { - let bytes = download_github_release(reqwest, version)?; - fs_err::write(&path, bytes).context("failed to write downloaded version file")?; + let bytes = download_github_release(reqwest, version).await?; + fs::write(&path, bytes) + .await + .context("failed to write downloaded version file")?; } - make_executable(&path).context("failed to make downloaded version executable")?; + make_executable(&path) + .await + .context("failed to make downloaded version executable")?; Ok(if is_requested_version { None @@ -188,48 +209,52 @@ pub fn get_or_download_version( }) } -pub fn max_installed_version() -> anyhow::Result { +pub async fn max_installed_version() -> anyhow::Result { let versions_dir = home_dir()?.join("versions"); - fs_err::create_dir_all(&versions_dir).context("failed to create versions directory")?; + fs::create_dir_all(&versions_dir) + .await + .context("failed to create versions directory")?; - let max_version = fs_err::read_dir(versions_dir) - .context("failed to read versions directory")? - .collect::, _>>()? - .into_iter() - .map(|entry| { - #[cfg(not(windows))] - let name = entry - .path() - .file_name() - .unwrap() - .to_string_lossy() - .to_string(); - #[cfg(windows)] - let name = entry - .path() - .file_stem() - .unwrap() - .to_string_lossy() - .to_string(); + let mut read_dir = fs::read_dir(versions_dir) + .await + .context("failed to read versions directory")?; + let mut max_version = current_version(); - Version::parse(&name).unwrap() - }) - .max() - .filter(|v| v >= ¤t_version()) - .unwrap_or_else(current_version); + while let Some(entry) = read_dir.next_entry().await? { + #[cfg(not(windows))] + let name = entry + .path() + .file_name() + .unwrap() + .to_string_lossy() + .to_string(); + #[cfg(windows)] + let name = entry + .path() + .file_stem() + .unwrap() + .to_string_lossy() + .to_string(); + + let version = Version::parse(&name).unwrap(); + if version > max_version { + max_version = version; + } + } Ok(max_version) } -pub fn update_bin_exe() -> anyhow::Result<()> { - let copy_to = bin_dir()?.join(format!( +pub async fn update_bin_exe() -> anyhow::Result<()> { + let copy_to = bin_dir().await?.join(format!( "{}{}", env!("CARGO_BIN_NAME"), std::env::consts::EXE_SUFFIX )); - fs_err::copy(std::env::current_exe()?, ©_to) + fs::copy(std::env::current_exe()?, ©_to) + .await .context("failed to copy executable to bin folder")?; - make_executable(©_to) + make_executable(©_to).await } diff --git a/src/download.rs b/src/download.rs index b5f529d..c9eb0ce 100644 --- a/src/download.rs +++ b/src/download.rs @@ -7,42 +7,41 @@ use crate::{ }, Project, PACKAGES_CONTAINER_NAME, }; -use fs_err::create_dir_all; +use fs_err::tokio as fs; use std::{ collections::HashSet, - sync::{mpsc::Receiver, Arc, Mutex}, + sync::{Arc, Mutex}, }; type MultithreadedGraph = Arc>; type MultithreadDownloadJob = ( - Receiver>, + tokio::sync::mpsc::Receiver>, MultithreadedGraph, ); impl Project { /// Downloads a graph of dependencies - pub fn download_graph( + pub async fn download_graph( &self, graph: &DependencyGraph, refreshed_sources: &mut HashSet, - reqwest: &reqwest::blocking::Client, - threads: usize, + reqwest: &reqwest::Client, prod: bool, write: bool, ) -> Result { - let manifest = self.deser_manifest()?; + let manifest = self.deser_manifest().await?; let downloaded_graph: MultithreadedGraph = Arc::new(Mutex::new(Default::default())); - let threadpool = threadpool::ThreadPool::new(threads); - let (tx, rx) = std::sync::mpsc::channel(); + let (tx, rx) = + tokio::sync::mpsc::channel(graph.iter().map(|(_, versions)| versions.len()).sum()); for (name, versions) in graph { for (version_id, node) in versions { let source = node.pkg_ref.source(); if refreshed_sources.insert(source.clone()) { - source.refresh(self).map_err(Box::new)?; + source.refresh(self).await.map_err(Box::new)?; } let container_folder = node.container_folder( @@ -59,7 +58,7 @@ impl Project { version_id.version(), ); - create_dir_all(&container_folder)?; + fs::create_dir_all(&container_folder).await?; let tx = tx.clone(); @@ -71,27 +70,29 @@ impl Project { let reqwest = reqwest.clone(); let downloaded_graph = downloaded_graph.clone(); - threadpool.execute(move || { + tokio::spawn(async move { let project = project.clone(); log::debug!("downloading {name}@{version_id}"); - let (fs, target) = match source.download(&node.pkg_ref, &project, &reqwest) { - Ok(target) => target, - Err(e) => { - tx.send(Err(Box::new(e).into())).unwrap(); - return; - } - }; + let (fs, target) = + match source.download(&node.pkg_ref, &project, &reqwest).await { + Ok(target) => target, + Err(e) => { + tx.send(Err(Box::new(e).into())).await.unwrap(); + return; + } + }; log::debug!("downloaded {name}@{version_id}"); if write { if !prod || node.ty != DependencyType::Dev { - match fs.write_to(container_folder, project.cas_dir(), true) { + match fs.write_to(container_folder, project.cas_dir(), true).await { Ok(_) => {} Err(e) => { tx.send(Err(errors::DownloadGraphError::WriteFailed(e))) + .await .unwrap(); return; } @@ -101,13 +102,15 @@ impl Project { } } - let mut downloaded_graph = downloaded_graph.lock().unwrap(); - downloaded_graph - .entry(name) - .or_default() - .insert(version_id, DownloadedDependencyGraphNode { node, target }); + { + let mut downloaded_graph = downloaded_graph.lock().unwrap(); + downloaded_graph + .entry(name) + .or_default() + .insert(version_id, DownloadedDependencyGraphNode { node, target }); + } - tx.send(Ok(())).unwrap(); + tx.send(Ok(())).await.unwrap(); }); } } diff --git a/src/lib.rs b/src/lib.rs index 3a9c21d..870d94d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,9 @@ //! It has been designed with multiple targets in mind, namely Roblox, Lune, and Luau. use crate::{lockfile::Lockfile, manifest::Manifest}; +use async_stream::stream; +use fs_err::tokio as fs; +use futures::Stream; use gix::sec::identity::Account; use std::{ collections::HashMap, @@ -137,42 +140,49 @@ impl Project { } /// Read the manifest file - pub fn read_manifest(&self) -> Result { - let string = fs_err::read_to_string(self.package_dir.join(MANIFEST_FILE_NAME))?; + pub async fn read_manifest(&self) -> Result { + let string = fs::read_to_string(self.package_dir.join(MANIFEST_FILE_NAME)).await?; Ok(string) } /// Deserialize the manifest file - pub fn deser_manifest(&self) -> Result { - let string = fs_err::read_to_string(self.package_dir.join(MANIFEST_FILE_NAME))?; + pub async fn deser_manifest(&self) -> Result { + let string = fs::read_to_string(self.package_dir.join(MANIFEST_FILE_NAME)).await?; Ok(toml::from_str(&string)?) } /// Write the manifest file - pub fn write_manifest>(&self, manifest: S) -> Result<(), std::io::Error> { - fs_err::write(self.package_dir.join(MANIFEST_FILE_NAME), manifest.as_ref()) + pub async fn write_manifest>(&self, manifest: S) -> Result<(), std::io::Error> { + fs::write(self.package_dir.join(MANIFEST_FILE_NAME), manifest.as_ref()).await } /// Deserialize the lockfile - pub fn deser_lockfile(&self) -> Result { - let string = fs_err::read_to_string(self.package_dir.join(LOCKFILE_FILE_NAME))?; + pub async fn deser_lockfile(&self) -> Result { + let string = fs::read_to_string(self.package_dir.join(LOCKFILE_FILE_NAME)).await?; Ok(toml::from_str(&string)?) } /// Write the lockfile - pub fn write_lockfile(&self, lockfile: Lockfile) -> Result<(), errors::LockfileWriteError> { + pub async fn write_lockfile( + &self, + lockfile: Lockfile, + ) -> Result<(), errors::LockfileWriteError> { let string = toml::to_string(&lockfile)?; - fs_err::write(self.package_dir.join(LOCKFILE_FILE_NAME), string)?; + fs::write(self.package_dir.join(LOCKFILE_FILE_NAME), string).await?; Ok(()) } /// Get the workspace members - pub fn workspace_members>( + pub async fn workspace_members>( &self, dir: P, - ) -> Result, errors::WorkspaceMembersError> { + ) -> Result< + impl Stream>, + errors::WorkspaceMembersError, + > { let dir = dir.as_ref().to_path_buf(); - let manifest = fs_err::read_to_string(dir.join(MANIFEST_FILE_NAME)) + let manifest = fs::read_to_string(dir.join(MANIFEST_FILE_NAME)) + .await .map_err(errors::WorkspaceMembersError::ManifestMissing)?; let manifest = toml::from_str::(&manifest).map_err(|e| { errors::WorkspaceMembersError::ManifestDeser(dir.to_path_buf(), Box::new(e)) @@ -188,17 +198,18 @@ impl Project { .flat_map(|paths| paths.into_iter()) .collect::, _>>()?; - members - .into_iter() - .map(|path| { - let manifest = fs_err::read_to_string(path.join(MANIFEST_FILE_NAME)) + Ok(stream! { + for path in members { + let manifest = fs::read_to_string(path.join(MANIFEST_FILE_NAME)) + .await .map_err(errors::WorkspaceMembersError::ManifestMissing)?; let manifest = toml::from_str::(&manifest).map_err(|e| { errors::WorkspaceMembersError::ManifestDeser(path.clone(), Box::new(e)) })?; - Ok((path, manifest)) - }) - .collect::>() + + yield Ok((path, manifest)); + } + }) } } diff --git a/src/linking/mod.rs b/src/linking/mod.rs index 56281c1..10cbfff 100644 --- a/src/linking/mod.rs +++ b/src/linking/mod.rs @@ -3,9 +3,14 @@ use crate::{ lockfile::DownloadedGraph, names::PackageNames, scripts::{execute_script, ScriptName}, - source::{fs::store_in_cas, traits::PackageRef, version_id::VersionId}, + source::{ + fs::{cas_path, store_in_cas}, + traits::PackageRef, + version_id::VersionId, + }, Project, LINK_LIB_NO_FILE_FOUND, PACKAGES_CONTAINER_NAME, }; +use fs_err::tokio as fs; use std::{ collections::BTreeMap, ffi::OsStr, @@ -15,22 +20,25 @@ use std::{ /// Generates linking modules for a project pub mod generator; -fn create_and_canonicalize>(path: P) -> std::io::Result { +async fn create_and_canonicalize>(path: P) -> std::io::Result { let p = path.as_ref(); - fs_err::create_dir_all(p)?; + fs::create_dir_all(p).await?; p.canonicalize() } -fn write_cas(destination: PathBuf, cas_dir: &Path, contents: &str) -> std::io::Result<()> { - let cas_path = store_in_cas(cas_dir, contents.as_bytes())?.1; +async fn write_cas(destination: PathBuf, cas_dir: &Path, contents: &str) -> std::io::Result<()> { + let hash = store_in_cas(cas_dir, contents.as_bytes(), |_| async { Ok(()) }).await?; - fs_err::hard_link(cas_path, destination) + fs::hard_link(cas_path(&hash, cas_dir), destination).await } impl Project { /// Links the dependencies of the project - pub fn link_dependencies(&self, graph: &DownloadedGraph) -> Result<(), errors::LinkingError> { - let manifest = self.deser_manifest()?; + pub async fn link_dependencies( + &self, + graph: &DownloadedGraph, + ) -> Result<(), errors::LinkingError> { + let manifest = self.deser_manifest().await?; let mut package_types = BTreeMap::<&PackageNames, BTreeMap<&VersionId, Vec>>::new(); @@ -57,7 +65,7 @@ impl Project { let types = if lib_file.as_str() != LINK_LIB_NO_FILE_FOUND { let lib_file = lib_file.to_path(&container_folder); - let contents = match fs_err::read_to_string(&lib_file) { + let contents = match fs::read_to_string(&lib_file).await { Ok(contents) => contents, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { return Err(errors::LinkingError::LibFileNotFound( @@ -128,7 +136,8 @@ impl Project { .kind() .packages_folder(&node.node.pkg_ref.target_kind()), ), - )?; + ) + .await?; let packages_container_folder = base_folder.join(PACKAGES_CONTAINER_NAME); let container_folder = node.node.container_folder( @@ -162,7 +171,8 @@ impl Project { )?, types, ), - )?; + ) + .await?; }; if let Some(bin_file) = node.target.bin_path() { @@ -177,7 +187,8 @@ impl Project { &container_folder, ), ), - )?; + ) + .await?; } } @@ -208,7 +219,8 @@ impl Project { .target_kind() .packages_folder(&dependency_node.node.pkg_ref.target_kind()), ), - )?; + ) + .await?; let packages_container_folder = base_folder.join(PACKAGES_CONTAINER_NAME); let container_folder = dependency_node.node.container_folder( @@ -220,7 +232,8 @@ impl Project { let linker_folder = create_and_canonicalize( node_container_folder .join(node.node.base_folder(dependency_node.target.kind())), - )?; + ) + .await?; write_cas( linker_folder.join(format!("{dependency_alias}.luau")), @@ -241,7 +254,8 @@ impl Project { .and_then(|v| v.get(dependency_version_id)) .unwrap(), ), - )?; + ) + .await?; } } } diff --git a/src/main.rs b/src/main.rs index 3b1602c..8687bfe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,17 +2,16 @@ use crate::cli::version::{ check_for_updates, current_version, get_or_download_version, max_installed_version, }; -use crate::cli::{auth::get_tokens, home_dir, repos::update_repo_dependencies, HOME_DIR}; +use crate::cli::{auth::get_tokens, display_err, home_dir, HOME_DIR}; use anyhow::Context; use clap::Parser; -use colored::Colorize; +use fs_err::tokio as fs; use indicatif::MultiProgress; use indicatif_log_bridge::LogWrapper; use pesde::{AuthConfig, Project, MANIFEST_FILE_NAME}; use std::{ collections::HashSet, path::{Path, PathBuf}, - thread::spawn, }; use tempfile::NamedTempFile; @@ -34,7 +33,7 @@ struct Cli { subcommand: cli::commands::Subcommand, } -fn get_linkable_dir(path: &Path) -> PathBuf { +async fn get_linkable_dir(path: &Path) -> PathBuf { let mut curr_path = PathBuf::new(); let file_to_try = NamedTempFile::new_in(path).expect("failed to create temporary file"); let temp_file_name = file_to_try.path().file_name().unwrap(); @@ -44,8 +43,8 @@ fn get_linkable_dir(path: &Path) -> PathBuf { let try_path = curr_path.join(temp_file_name); - if fs_err::hard_link(file_to_try.path(), &try_path).is_ok() { - if let Err(err) = fs_err::remove_file(&try_path) { + if fs::hard_link(file_to_try.path(), &try_path).await.is_ok() { + if let Err(err) = fs::remove_file(&try_path).await { log::warn!( "failed to remove temporary file at {}: {err}", try_path.display() @@ -62,7 +61,7 @@ fn get_linkable_dir(path: &Path) -> PathBuf { ); } -fn run() -> anyhow::Result<()> { +async fn run() -> anyhow::Result<()> { let cwd = std::env::current_dir().expect("failed to get current working directory"); #[cfg(windows)] @@ -107,8 +106,9 @@ fn run() -> anyhow::Result<()> { let mut project_root = None::; let mut workspace_dir = None::; - fn get_workspace_members(path: &Path) -> anyhow::Result> { - let manifest = fs_err::read_to_string(path.join(MANIFEST_FILE_NAME)) + async fn get_workspace_members(path: &Path) -> anyhow::Result> { + let manifest = fs::read_to_string(path.join(MANIFEST_FILE_NAME)) + .await .context("failed to read manifest")?; let manifest: pesde::manifest::Manifest = toml::from_str(&manifest).context("failed to parse manifest")?; @@ -143,13 +143,13 @@ fn run() -> anyhow::Result<()> { } (Some(project_root), None) => { - if get_workspace_members(&path)?.contains(project_root) { + if get_workspace_members(&path).await?.contains(project_root) { workspace_dir = Some(path); } } (None, None) => { - if get_workspace_members(&path)?.contains(&cwd) { + if get_workspace_members(&path).await?.contains(&cwd) { // initializing a new member of a workspace break 'finder (cwd, Some(path)); } else { @@ -179,9 +179,11 @@ fn run() -> anyhow::Result<()> { let home_dir = home_dir()?; let data_dir = home_dir.join("data"); - fs_err::create_dir_all(&data_dir).expect("failed to create data directory"); + fs::create_dir_all(&data_dir) + .await + .expect("failed to create data directory"); - let cas_dir = get_linkable_dir(&project_root_dir).join(HOME_DIR); + let cas_dir = get_linkable_dir(&project_root_dir).await.join(HOME_DIR); let cas_dir = if cas_dir == home_dir { &data_dir @@ -197,7 +199,7 @@ fn run() -> anyhow::Result<()> { project_workspace_dir, data_dir, cas_dir, - AuthConfig::new().with_tokens(get_tokens()?.0), + AuthConfig::new().with_tokens(get_tokens().await?.0), ); let reqwest = { @@ -210,7 +212,7 @@ fn run() -> anyhow::Result<()> { .context("failed to create accept header")?, ); - reqwest::blocking::Client::builder() + reqwest::Client::builder() .user_agent(concat!( env!("CARGO_PKG_NAME"), "/", @@ -224,21 +226,22 @@ fn run() -> anyhow::Result<()> { { let target_version = project .deser_manifest() + .await .ok() .and_then(|manifest| manifest.pesde_version); // store the current version in case it needs to be used later - get_or_download_version(&reqwest, ¤t_version())?; + get_or_download_version(&reqwest, ¤t_version()).await?; let exe_path = if let Some(version) = target_version { - Some(get_or_download_version(&reqwest, &version)?) + Some(get_or_download_version(&reqwest, &version).await?) } else { None }; let exe_path = if let Some(exe_path) = exe_path { exe_path } else { - get_or_download_version(&reqwest, &max_installed_version()?)? + get_or_download_version(&reqwest, &max_installed_version().await?).await? }; if let Some(exe_path) = exe_path { @@ -250,62 +253,26 @@ fn run() -> anyhow::Result<()> { std::process::exit(status.code().unwrap()); } - display_err(check_for_updates(&reqwest), " while checking for updates"); - } - - let project_2 = project.clone(); - let update_task = spawn(move || { display_err( - update_repo_dependencies(&project_2), - " while updating repository dependencies", + check_for_updates(&reqwest).await, + " while checking for updates", ); - }); + } let cli = match Cli::try_parse() { Ok(cli) => cli, Err(err) => { let _ = err.print(); - update_task.join().expect("failed to join update task"); std::process::exit(err.exit_code()); } }; - cli.subcommand.run(project, multi, reqwest, update_task) + cli.subcommand.run(project, multi, reqwest).await } -fn display_err(result: anyhow::Result<()>, prefix: &str) { - if let Err(err) = result { - eprintln!("{}: {err}\n", format!("error{prefix}").red().bold()); - - let cause = err.chain().skip(1).collect::>(); - - if !cause.is_empty() { - eprintln!("{}:", "caused by".red().bold()); - for err in cause { - eprintln!(" - {err}"); - } - } - - let backtrace = err.backtrace(); - match backtrace.status() { - std::backtrace::BacktraceStatus::Disabled => { - eprintln!( - "\n{}: set RUST_BACKTRACE=1 for a backtrace", - "help".yellow().bold() - ); - } - std::backtrace::BacktraceStatus::Captured => { - eprintln!("\n{}:\n{backtrace}", "backtrace".yellow().bold()); - } - _ => { - eprintln!("\n{}: not captured", "backtrace".yellow().bold()); - } - } - } -} - -fn main() { - let result = run(); +#[tokio::main] +async fn main() { + let result = run().await; let is_err = result.is_err(); display_err(result, ""); if is_err { diff --git a/src/patches.rs b/src/patches.rs index 14d6ac9..cf2c4a1 100644 --- a/src/patches.rs +++ b/src/patches.rs @@ -2,8 +2,8 @@ use crate::{ lockfile::DownloadedGraph, source::traits::PackageRef, Project, MANIFEST_FILE_NAME, PACKAGES_CONTAINER_NAME, }; -use fs_err::read; -use git2::{ApplyLocation, ApplyOptions, Diff, DiffFormat, DiffLineType, Repository, Signature}; +use fs_err::tokio as fs; +use git2::{ApplyLocation, Diff, DiffFormat, DiffLineType, Repository, Signature}; use relative_path::RelativePathBuf; use std::path::Path; @@ -72,14 +72,19 @@ pub fn create_patch>(dir: P) -> Result, git2::Error> { impl Project { /// Apply patches to the project's dependencies - pub fn apply_patches(&self, graph: &DownloadedGraph) -> Result<(), errors::ApplyPatchesError> { - let manifest = self.deser_manifest()?; + pub async fn apply_patches( + &self, + graph: &DownloadedGraph, + ) -> Result<(), errors::ApplyPatchesError> { + let manifest = self.deser_manifest().await?; for (name, versions) in manifest.patches { for (version_id, patch_path) in versions { let patch_path = patch_path.to_path(self.package_dir()); let patch = Diff::from_buffer( - &read(&patch_path).map_err(errors::ApplyPatchesError::PatchRead)?, + &fs::read(&patch_path) + .await + .map_err(errors::ApplyPatchesError::PatchRead)?, )?; let Some(node) = graph @@ -110,42 +115,37 @@ impl Project { { let repo = setup_patches_repo(&container_folder)?; - let mut apply_opts = ApplyOptions::new(); - apply_opts.delta_callback(|delta| { - let Some(delta) = delta else { - return true; - }; - + for delta in patch.deltas() { if !matches!(delta.status(), git2::Delta::Modified) { - return true; + continue; } let file = delta.new_file(); let Some(relative_path) = file.path() else { - return true; + continue; }; let relative_path = RelativePathBuf::from_path(relative_path).unwrap(); let path = relative_path.to_path(&container_folder); if !path.is_file() { - return true; + continue; } // there is no way (as far as I know) to check if it's hardlinked // so, we always unlink it - let content = read(&path).unwrap(); - fs_err::remove_file(&path).unwrap(); - fs_err::write(path, content).unwrap(); + let content = fs::read(&path).await.unwrap(); + fs::remove_file(&path).await.unwrap(); + fs::write(path, content).await.unwrap(); + } - true - }); - repo.apply(&patch, ApplyLocation::Both, Some(&mut apply_opts))?; + repo.apply(&patch, ApplyLocation::Both, None)?; } log::debug!("patch applied to {name}@{version_id}, removing .git directory"); - fs_err::remove_dir_all(container_folder.join(".git")) + fs::remove_dir_all(container_folder.join(".git")) + .await .map_err(errors::ApplyPatchesError::DotGitRemove)?; } } diff --git a/src/resolver.rs b/src/resolver.rs index 9d5a5ef..5545190 100644 --- a/src/resolver.rs +++ b/src/resolver.rs @@ -15,12 +15,15 @@ use std::collections::{HashMap, HashSet, VecDeque}; impl Project { /// Create a dependency graph from the project's manifest - pub fn dependency_graph( + pub async fn dependency_graph( &self, previous_graph: Option<&DependencyGraph>, refreshed_sources: &mut HashSet, ) -> Result> { - let manifest = self.deser_manifest().map_err(|e| Box::new(e.into()))?; + let manifest = self + .deser_manifest() + .await + .map_err(|e| Box::new(e.into()))?; let mut all_specifiers = manifest .all_dependencies() @@ -190,11 +193,12 @@ impl Project { }; if refreshed_sources.insert(source.clone()) { - source.refresh(self).map_err(|e| Box::new(e.into()))?; + source.refresh(self).await.map_err(|e| Box::new(e.into()))?; } let (name, resolved) = source .resolve(&specifier, self, target) + .await .map_err(|e| Box::new(e.into()))?; let Some(target_version_id) = graph diff --git a/src/source/fs.rs b/src/source/fs.rs index 812ec92..f4471ac 100644 --- a/src/source/fs.rs +++ b/src/source/fs.rs @@ -1,17 +1,21 @@ -use std::{ - collections::BTreeMap, - io::{BufWriter, Read, Write}, - path::{Path, PathBuf}, -}; - use crate::{ manifest::target::TargetKind, source::{IGNORED_DIRS, IGNORED_FILES}, - util::hash, }; +use fs_err::tokio as fs; use relative_path::RelativePathBuf; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use std::{ + collections::{BTreeMap, VecDeque}, + future::Future, + path::{Path, PathBuf}, +}; +use tempfile::Builder; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt, BufWriter}, + pin, +}; /// A file system entry #[derive(Debug, Clone, Serialize, Deserialize)] @@ -35,7 +39,7 @@ pub enum PackageFS { Copy(PathBuf, TargetKind), } -fn make_readonly(_file: &fs_err::File) -> std::io::Result<()> { +fn make_readonly(_file: &fs::File) -> std::io::Result<()> { // on Windows, file deletion is disallowed if the file is read-only which breaks patching #[cfg(not(windows))] { @@ -48,58 +52,54 @@ fn make_readonly(_file: &fs_err::File) -> std::io::Result<()> { Ok(()) } -pub(crate) fn store_in_cas>( - cas_dir: P, - contents: &[u8], -) -> std::io::Result<(String, PathBuf)> { - let hash = hash(contents); +pub(crate) fn cas_path(hash: &str, cas_dir: &Path) -> PathBuf { let (prefix, rest) = hash.split_at(2); - - let folder = cas_dir.as_ref().join(prefix); - fs_err::create_dir_all(&folder)?; - - let cas_path = folder.join(rest); - if !cas_path.exists() { - let mut file = fs_err::File::create(&cas_path)?; - file.write_all(contents)?; - - make_readonly(&file)?; - } - - Ok((hash, cas_path)) + cas_dir.join(prefix).join(rest) } -pub(crate) fn store_reader_in_cas>( +pub(crate) async fn store_in_cas< + R: tokio::io::AsyncRead + Unpin, + P: AsRef, + C: FnMut(Vec) -> F, + F: Future>, +>( cas_dir: P, - contents: &mut dyn Read, + mut contents: R, + mut bytes_cb: C, ) -> std::io::Result { let tmp_dir = cas_dir.as_ref().join(".tmp"); - fs_err::create_dir_all(&tmp_dir)?; + fs::create_dir_all(&tmp_dir).await?; let mut hasher = Sha256::new(); let mut buf = [0; 8 * 1024]; - let mut file_writer = BufWriter::new(tempfile::NamedTempFile::new_in(&tmp_dir)?); + + let temp_path = Builder::new() + .make_in(&tmp_dir, |_| Ok(()))? + .into_temp_path(); + let mut file_writer = BufWriter::new(fs::File::create(temp_path.to_path_buf()).await?); loop { - let bytes_read = contents.read(&mut buf)?; + let bytes_future = contents.read(&mut buf); + pin!(bytes_future); + let bytes_read = bytes_future.await?; + if bytes_read == 0 { break; } let bytes = &buf[..bytes_read]; hasher.update(bytes); - file_writer.write_all(bytes)?; + bytes_cb(bytes.to_vec()).await?; + file_writer.write_all(bytes).await?; } let hash = format!("{:x}", hasher.finalize()); - let (prefix, rest) = hash.split_at(2); - let folder = cas_dir.as_ref().join(prefix); - fs_err::create_dir_all(&folder)?; + let cas_path = cas_path(&hash, cas_dir.as_ref()); + fs::create_dir_all(cas_path.parent().unwrap()).await?; - let cas_path = folder.join(rest); - match file_writer.into_inner()?.persist_noclobber(&cas_path) { + match temp_path.persist_noclobber(&cas_path) { Ok(_) => { - make_readonly(&fs_err::File::open(cas_path)?)?; + make_readonly(&file_writer.into_inner())?; } Err(e) if e.error.kind() == std::io::ErrorKind::AlreadyExists => {} Err(e) => return Err(e.error), @@ -108,43 +108,9 @@ pub(crate) fn store_reader_in_cas>( Ok(hash) } -fn copy_dir_all( - src: impl AsRef, - dst: impl AsRef, - target: TargetKind, -) -> std::io::Result<()> { - fs_err::create_dir_all(&dst)?; - 'outer: for entry in fs_err::read_dir(src.as_ref().to_path_buf())? { - let entry = entry?; - let ty = entry.file_type()?; - let file_name = entry.file_name().to_string_lossy().to_string(); - - if ty.is_dir() { - if IGNORED_DIRS.contains(&file_name.as_ref()) { - continue; - } - - for other_target in TargetKind::VARIANTS { - if target.packages_folder(other_target) == file_name { - continue 'outer; - } - } - - copy_dir_all(entry.path(), dst.as_ref().join(&file_name), target)?; - } else { - if IGNORED_FILES.contains(&file_name.as_ref()) { - continue; - } - - fs_err::copy(entry.path(), dst.as_ref().join(file_name))?; - } - } - Ok(()) -} - impl PackageFS { /// Write the package to the given destination - pub fn write_to, Q: AsRef>( + pub async fn write_to, Q: AsRef>( &self, destination: P, cas_path: Q, @@ -158,17 +124,17 @@ impl PackageFS { match entry { FSEntry::File(hash) => { if let Some(parent) = path.parent() { - fs_err::create_dir_all(parent)?; + fs::create_dir_all(parent).await?; } let (prefix, rest) = hash.split_at(2); let cas_file_path = cas_path.as_ref().join(prefix).join(rest); if link { - fs_err::hard_link(cas_file_path, path)?; + fs::hard_link(cas_file_path, path).await?; } else { - let mut f = fs_err::File::create(&path)?; - f.write_all(&fs_err::read(cas_file_path)?)?; + let mut f = fs::File::create(&path).await?; + f.write_all(&fs::read(cas_file_path).await?).await?; #[cfg(unix)] { @@ -180,13 +146,45 @@ impl PackageFS { } } FSEntry::Directory => { - fs_err::create_dir_all(path)?; + fs::create_dir_all(path).await?; } } } } PackageFS::Copy(src, target) => { - copy_dir_all(src, destination, *target)?; + fs::create_dir_all(destination.as_ref()).await?; + + let mut read_dirs = VecDeque::from([fs::read_dir(src.to_path_buf())]); + while let Some(read_dir) = read_dirs.pop_front() { + let mut read_dir = read_dir.await?; + while let Some(entry) = read_dir.next_entry().await? { + let relative_path = + RelativePathBuf::from_path(entry.path().strip_prefix(src).unwrap()) + .unwrap(); + let file_name = relative_path.file_name().unwrap(); + + if entry.file_type().await?.is_dir() { + if IGNORED_DIRS.contains(&file_name) { + continue; + } + + for other_target in TargetKind::VARIANTS { + if target.packages_folder(other_target) == file_name { + continue; + } + } + + read_dirs.push_back(fs::read_dir(entry.path())); + continue; + } + + if IGNORED_FILES.contains(&file_name) { + continue; + } + + fs::copy(entry.path(), relative_path.to_path(destination.as_ref())).await?; + } + } } } @@ -194,7 +192,7 @@ impl PackageFS { } /// Returns the contents of the file with the given hash - pub fn read_file, H: AsRef>( + pub async fn read_file, H: AsRef>( &self, file_hash: H, cas_path: P, @@ -205,6 +203,6 @@ impl PackageFS { let (prefix, rest) = file_hash.as_ref().split_at(2); let cas_file_path = cas_path.as_ref().join(prefix).join(rest); - fs_err::read_to_string(cas_file_path).ok() + fs::read_to_string(cas_file_path).await.ok() } } diff --git a/src/source/git/mod.rs b/src/source/git/mod.rs index 5c4d1d0..bcdbf3c 100644 --- a/src/source/git/mod.rs +++ b/src/source/git/mod.rs @@ -10,7 +10,7 @@ use crate::{ }, names::PackageNames, source::{ - fs::{store_in_cas, FSEntry, PackageFS}, + fs::{FSEntry, PackageFS}, git::{pkg_ref::GitPackageRef, specifier::GitDependencySpecifier}, git_index::GitBasedSource, specifiers::DependencySpecifiers, @@ -20,6 +20,7 @@ use crate::{ util::hash, Project, DEFAULT_INDEX_NAME, LOCKFILE_FILE_NAME, MANIFEST_FILE_NAME, }; +use fs_err::tokio as fs; /// The Git package reference pub mod pkg_ref; @@ -63,11 +64,11 @@ impl PackageSource for GitPackageSource { type ResolveError = errors::ResolveError; type DownloadError = errors::DownloadError; - fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> { - GitBasedSource::refresh(self, project) + async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> { + GitBasedSource::refresh(self, project).await } - fn resolve( + async fn resolve( &self, specifier: &Self::Specifier, project: &Project, @@ -324,11 +325,11 @@ impl PackageSource for GitPackageSource { )) } - fn download( + async fn download( &self, pkg_ref: &Self::Ref, project: &Project, - _reqwest: &reqwest::blocking::Client, + _reqwest: &reqwest::Client, ) -> Result<(PackageFS, Target), Self::DownloadError> { let index_file = project .cas_dir @@ -336,7 +337,7 @@ impl PackageSource for GitPackageSource { .join(hash(self.as_bytes())) .join(&pkg_ref.tree_id); - match fs_err::read_to_string(&index_file) { + match fs::read_to_string(&index_file).await { Ok(s) => { log::debug!( "using cached index file for package {}#{} {}", @@ -354,6 +355,7 @@ impl PackageSource for GitPackageSource { match entries.get(&RelativePathBuf::from(MANIFEST_FILE_NAME)) { Some(FSEntry::File(hash)) => match fs .read_file(hash, project.cas_dir()) + .await .map(|m| toml::de::from_str::(&m)) { Some(Ok(m)) => Some(m), @@ -376,9 +378,10 @@ impl PackageSource for GitPackageSource { #[cfg(feature = "wally-compat")] None if !pkg_ref.new_structure => { let tempdir = tempfile::tempdir()?; - fs.write_to(tempdir.path(), project.cas_dir(), false)?; + fs.write_to(tempdir.path(), project.cas_dir(), false) + .await?; - crate::source::wally::compat_util::get_target(project, &tempdir)? + crate::source::wally::compat_util::get_target(project, &tempdir).await? } None => { return Err(errors::DownloadError::NoManifest(Box::new( @@ -393,47 +396,50 @@ impl PackageSource for GitPackageSource { Err(e) => return Err(errors::DownloadError::Io(e)), } - let repo = gix::open(self.path(project)) - .map_err(|e| errors::DownloadError::OpenRepo(Box::new(self.repo_url.clone()), e))?; - let rev = repo - .rev_parse_single(BStr::new(&pkg_ref.tree_id)) - .map_err(|e| { - errors::DownloadError::ParseRev( - pkg_ref.tree_id.clone(), - Box::new(self.repo_url.clone()), - e, - ) - })?; - let tree = rev - .object() - .map_err(|e| { - errors::DownloadError::ParseEntryToObject(Box::new(self.repo_url.clone()), e) - })? - .peel_to_tree() - .map_err(|e| { - errors::DownloadError::ParseObjectToTree(Box::new(self.repo_url.clone()), e) - })?; - - let mut recorder = Recorder::default(); - tree.traverse() - .breadthfirst(&mut recorder) - .map_err(|e| errors::DownloadError::TraverseTree(Box::new(self.repo_url.clone()), e))?; - let mut entries = BTreeMap::new(); let mut manifest = None; + let repo = gix::open(self.path(project)) + .map_err(|e| errors::DownloadError::OpenRepo(Box::new(self.repo_url.clone()), e))?; + + let recorder = { + let rev = repo + .rev_parse_single(BStr::new(&pkg_ref.tree_id)) + .map_err(|e| { + errors::DownloadError::ParseRev( + pkg_ref.tree_id.clone(), + Box::new(self.repo_url.clone()), + e, + ) + })?; + + let tree = rev + .object() + .map_err(|e| { + errors::DownloadError::ParseEntryToObject(Box::new(self.repo_url.clone()), e) + })? + .peel_to_tree() + .map_err(|e| { + errors::DownloadError::ParseObjectToTree(Box::new(self.repo_url.clone()), e) + })?; + + let mut recorder = Recorder::default(); + tree.traverse().breadthfirst(&mut recorder).map_err(|e| { + errors::DownloadError::TraverseTree(Box::new(self.repo_url.clone()), e) + })?; + + recorder + }; + for entry in recorder.records { let path = RelativePathBuf::from(entry.filepath.to_string()); + let name = path.file_name().unwrap_or(""); let object = repo.find_object(entry.oid).map_err(|e| { errors::DownloadError::ParseEntryToObject(Box::new(self.repo_url.clone()), e) })?; if matches!(object.kind, gix::object::Kind::Tree) { - if path - .components() - .next() - .is_some_and(|ct| IGNORED_DIRS.contains(&ct.as_str())) - { + if IGNORED_DIRS.contains(&name) { continue; } @@ -442,13 +448,13 @@ impl PackageSource for GitPackageSource { continue; } - if IGNORED_FILES.contains(&path.as_str()) { + if IGNORED_FILES.contains(&name) { continue; } - if pkg_ref.use_new_structure() && path == "default.project.json" { + if pkg_ref.use_new_structure() && name == "default.project.json" { log::debug!( - "removing default.project.json from {}#{} - using new structure", + "removing default.project.json from {}#{} at {path} - using new structure", pkg_ref.repo, pkg_ref.tree_id ); @@ -456,13 +462,15 @@ impl PackageSource for GitPackageSource { } let data = object.into_blob().data.clone(); - let hash = store_in_cas(project.cas_dir(), &data)?.0; + // let hash = + // store_reader_in_cas(project.cas_dir(), data.as_slice(), |_| async { Ok(()) }) + // .await?; if path == MANIFEST_FILE_NAME { manifest = Some(data); } - entries.insert(path, FSEntry::File(hash)); + // entries.insert(path, FSEntry::File(hash)); } let manifest = match manifest { @@ -488,9 +496,10 @@ impl PackageSource for GitPackageSource { #[cfg(feature = "wally-compat")] None if !pkg_ref.new_structure => { let tempdir = tempfile::tempdir()?; - fs.write_to(tempdir.path(), project.cas_dir(), false)?; + fs.write_to(tempdir.path(), project.cas_dir(), false) + .await?; - crate::source::wally::compat_util::get_target(project, &tempdir)? + crate::source::wally::compat_util::get_target(project, &tempdir).await? } None => { return Err(errors::DownloadError::NoManifest(Box::new( @@ -500,15 +509,16 @@ impl PackageSource for GitPackageSource { }; if let Some(parent) = index_file.parent() { - fs_err::create_dir_all(parent)?; + fs::create_dir_all(parent).await?; } - fs_err::write( + fs::write( &index_file, toml::to_string(&fs).map_err(|e| { errors::DownloadError::SerializeIndex(Box::new(self.repo_url.clone()), e) })?, ) + .await .map_err(errors::DownloadError::Io)?; Ok((fs, target)) diff --git a/src/source/git_index.rs b/src/source/git_index.rs index b2dca21..a8aec66 100644 --- a/src/source/git_index.rs +++ b/src/source/git_index.rs @@ -1,6 +1,8 @@ -use gix::remote::Direction; - +#![allow(async_fn_in_trait)] use crate::{util::authenticate_conn, Project}; +use fs_err::tokio as fs; +use gix::remote::Direction; +use tokio::task::spawn_blocking; /// A trait for sources that are based on Git repositories pub trait GitBasedSource { @@ -104,56 +106,79 @@ pub trait GitBasedSource { } /// Refreshes the repository - fn refresh(&self, project: &Project) -> Result<(), errors::RefreshError> { + async fn refresh(&self, project: &Project) -> Result<(), errors::RefreshError> { let path = self.path(project); + let repo_url = self.repo_url().clone(); + let auth_config = project.auth_config.clone(); + if path.exists() { - let repo = match gix::open(&path) { - Ok(repo) => repo, - Err(e) => return Err(errors::RefreshError::Open(path, Box::new(e))), - }; - let remote = match repo.find_default_remote(Direction::Fetch) { - Some(Ok(remote)) => remote, - Some(Err(e)) => { - return Err(errors::RefreshError::GetDefaultRemote(path, Box::new(e))) + spawn_blocking(move || { + let repo = match gix::open(&path) { + Ok(repo) => repo, + Err(e) => return Err(errors::RefreshError::Open(path, Box::new(e))), + }; + let remote = match repo.find_default_remote(Direction::Fetch) { + Some(Ok(remote)) => remote, + Some(Err(e)) => { + return Err(errors::RefreshError::GetDefaultRemote(path, Box::new(e))) + } + None => { + return Err(errors::RefreshError::NoDefaultRemote(path)); + } + }; + + let mut connection = match remote.connect(Direction::Fetch) { + Ok(connection) => connection, + Err(e) => { + return Err(errors::RefreshError::Connect( + repo_url.to_string(), + Box::new(e), + )) + } + }; + + authenticate_conn(&mut connection, &auth_config); + + let fetch = + match connection.prepare_fetch(gix::progress::Discard, Default::default()) { + Ok(fetch) => fetch, + Err(e) => { + return Err(errors::RefreshError::PrepareFetch( + repo_url.to_string(), + Box::new(e), + )) + } + }; + + match fetch.receive(gix::progress::Discard, &false.into()) { + Ok(_) => Ok(()), + Err(e) => Err(errors::RefreshError::Read( + repo_url.to_string(), + Box::new(e), + )), } - None => { - return Err(errors::RefreshError::NoDefaultRemote(path)); - } - }; - - let mut connection = remote.connect(Direction::Fetch).map_err(|e| { - errors::RefreshError::Connect(self.repo_url().to_string(), Box::new(e)) - })?; - - authenticate_conn(&mut connection, &project.auth_config); - - connection - .prepare_fetch(gix::progress::Discard, Default::default()) - .map_err(|e| { - errors::RefreshError::PrepareFetch(self.repo_url().to_string(), Box::new(e)) - })? - .receive(gix::progress::Discard, &false.into()) - .map_err(|e| { - errors::RefreshError::Read(self.repo_url().to_string(), Box::new(e)) - })?; + }) + .await + .unwrap()?; return Ok(()); } - fs_err::create_dir_all(&path)?; + fs::create_dir_all(&path).await?; - let auth_config = project.auth_config.clone(); - - gix::prepare_clone_bare(self.repo_url().clone(), &path) - .map_err(|e| errors::RefreshError::Clone(self.repo_url().to_string(), Box::new(e)))? - .configure_connection(move |c| { - authenticate_conn(c, &auth_config); - Ok(()) - }) - .fetch_only(gix::progress::Discard, &false.into()) - .map_err(|e| errors::RefreshError::Fetch(self.repo_url().to_string(), Box::new(e)))?; - - Ok(()) + spawn_blocking(move || { + gix::prepare_clone_bare(repo_url.clone(), &path) + .map_err(|e| errors::RefreshError::Clone(repo_url.to_string(), Box::new(e)))? + .configure_connection(move |c| { + authenticate_conn(c, &auth_config); + Ok(()) + }) + .fetch_only(gix::progress::Discard, &false.into()) + .map_err(|e| errors::RefreshError::Fetch(repo_url.to_string(), Box::new(e))) + }) + .await + .unwrap() + .map(|_| ()) } } diff --git a/src/source/mod.rs b/src/source/mod.rs index 7254db8..3294731 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -62,17 +62,17 @@ impl PackageSource for PackageSources { type ResolveError = errors::ResolveError; type DownloadError = errors::DownloadError; - fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> { + async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> { match self { - PackageSources::Pesde(source) => source.refresh(project).map_err(Into::into), + PackageSources::Pesde(source) => source.refresh(project).await.map_err(Into::into), #[cfg(feature = "wally-compat")] - PackageSources::Wally(source) => source.refresh(project).map_err(Into::into), - PackageSources::Git(source) => source.refresh(project).map_err(Into::into), - PackageSources::Workspace(source) => source.refresh(project).map_err(Into::into), + PackageSources::Wally(source) => source.refresh(project).await.map_err(Into::into), + PackageSources::Git(source) => source.refresh(project).await.map_err(Into::into), + PackageSources::Workspace(source) => source.refresh(project).await.map_err(Into::into), } } - fn resolve( + async fn resolve( &self, specifier: &Self::Specifier, project: &Project, @@ -81,6 +81,7 @@ impl PackageSource for PackageSources { match (self, specifier) { (PackageSources::Pesde(source), DependencySpecifiers::Pesde(specifier)) => source .resolve(specifier, project, package_target) + .await .map(|(name, results)| { ( name, @@ -95,6 +96,7 @@ impl PackageSource for PackageSources { #[cfg(feature = "wally-compat")] (PackageSources::Wally(source), DependencySpecifiers::Wally(specifier)) => source .resolve(specifier, project, package_target) + .await .map(|(name, results)| { ( name, @@ -108,6 +110,7 @@ impl PackageSource for PackageSources { (PackageSources::Git(source), DependencySpecifiers::Git(specifier)) => source .resolve(specifier, project, package_target) + .await .map(|(name, results)| { ( name, @@ -122,6 +125,7 @@ impl PackageSource for PackageSources { (PackageSources::Workspace(source), DependencySpecifiers::Workspace(specifier)) => { source .resolve(specifier, project, package_target) + .await .map(|(name, results)| { ( name, @@ -140,28 +144,32 @@ impl PackageSource for PackageSources { } } - fn download( + async fn download( &self, pkg_ref: &Self::Ref, project: &Project, - reqwest: &reqwest::blocking::Client, + reqwest: &reqwest::Client, ) -> Result<(PackageFS, Target), Self::DownloadError> { match (self, pkg_ref) { (PackageSources::Pesde(source), PackageRefs::Pesde(pkg_ref)) => source .download(pkg_ref, project, reqwest) + .await .map_err(Into::into), #[cfg(feature = "wally-compat")] (PackageSources::Wally(source), PackageRefs::Wally(pkg_ref)) => source .download(pkg_ref, project, reqwest) + .await .map_err(Into::into), (PackageSources::Git(source), PackageRefs::Git(pkg_ref)) => source .download(pkg_ref, project, reqwest) + .await .map_err(Into::into), (PackageSources::Workspace(source), PackageRefs::Workspace(pkg_ref)) => source .download(pkg_ref, project, reqwest) + .await .map_err(Into::into), _ => Err(errors::DownloadError::Mismatch), diff --git a/src/source/pesde/mod.rs b/src/source/pesde/mod.rs index de8a242..22f7371 100644 --- a/src/source/pesde/mod.rs +++ b/src/source/pesde/mod.rs @@ -20,13 +20,17 @@ use crate::{ }, names::{PackageName, PackageNames}, source::{ - fs::{store_reader_in_cas, FSEntry, PackageFS}, + fs::{store_in_cas, FSEntry, PackageFS}, git_index::GitBasedSource, DependencySpecifiers, PackageSource, ResolveResult, VersionId, IGNORED_DIRS, IGNORED_FILES, }, util::hash, Project, }; +use fs_err::tokio as fs; +use futures::StreamExt; + +// TODO: make more of these functions async /// The pesde package reference pub mod pkg_ref; @@ -186,11 +190,11 @@ impl PackageSource for PesdePackageSource { type ResolveError = errors::ResolveError; type DownloadError = errors::DownloadError; - fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> { - GitBasedSource::refresh(self, project) + async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> { + GitBasedSource::refresh(self, project).await } - fn resolve( + async fn resolve( &self, specifier: &Self::Specifier, project: &Project, @@ -239,11 +243,11 @@ impl PackageSource for PesdePackageSource { )) } - fn download( + async fn download( &self, pkg_ref: &Self::Ref, project: &Project, - reqwest: &reqwest::blocking::Client, + reqwest: &reqwest::Client, ) -> Result<(PackageFS, Target), Self::DownloadError> { let config = self.config(project).map_err(Box::new)?; let index_file = project @@ -253,7 +257,7 @@ impl PackageSource for PesdePackageSource { .join(pkg_ref.version.to_string()) .join(pkg_ref.target.to_string()); - match fs_err::read_to_string(&index_file) { + match fs::read_to_string(&index_file).await { Ok(s) => { log::debug!( "using cached index file for package {}@{} {}", @@ -280,26 +284,29 @@ impl PackageSource for PesdePackageSource { request = request.header(AUTHORIZATION, token); } - let response = request.send()?.error_for_status()?; - let bytes = response.bytes()?; + let response = request.send().await?.error_for_status()?; + let bytes = response.bytes().await?; - let mut decoder = flate2::read::GzDecoder::new(bytes.as_ref()); - let mut archive = tar::Archive::new(&mut decoder); + let mut decoder = async_compression::tokio::bufread::GzipDecoder::new(bytes.as_ref()); + let mut archive = tokio_tar::Archive::new(&mut decoder); let mut entries = BTreeMap::new(); - for entry in archive.entries().map_err(errors::DownloadError::Unpack)? { - let mut entry = entry.map_err(errors::DownloadError::Unpack)?; + while let Some(entry) = archive + .entries() + .map_err(errors::DownloadError::Unpack)? + .next() + .await + .transpose() + .map_err(errors::DownloadError::Unpack)? + { let path = RelativePathBuf::from_path(entry.path().map_err(errors::DownloadError::Unpack)?) .unwrap(); + let name = path.file_name().unwrap_or(""); if entry.header().entry_type().is_dir() { - if path - .components() - .next() - .is_some_and(|ct| IGNORED_DIRS.contains(&ct.as_str())) - { + if IGNORED_DIRS.contains(&name) { continue; } @@ -308,11 +315,12 @@ impl PackageSource for PesdePackageSource { continue; } - if IGNORED_FILES.contains(&path.as_str()) { + if IGNORED_FILES.contains(&name) { continue; } - let hash = store_reader_in_cas(project.cas_dir(), &mut entry) + let hash = store_in_cas(project.cas_dir(), entry, |_| async { Ok(()) }) + .await .map_err(errors::DownloadError::Store)?; entries.insert(path, FSEntry::File(hash)); } @@ -320,10 +328,13 @@ impl PackageSource for PesdePackageSource { let fs = PackageFS::CAS(entries); if let Some(parent) = index_file.parent() { - fs_err::create_dir_all(parent).map_err(errors::DownloadError::WriteIndex)?; + fs::create_dir_all(parent) + .await + .map_err(errors::DownloadError::WriteIndex)?; } - fs_err::write(&index_file, toml::to_string(&fs)?) + fs::write(&index_file, toml::to_string(&fs)?) + .await .map_err(errors::DownloadError::WriteIndex)?; Ok((fs, pkg_ref.target.clone())) diff --git a/src/source/traits.rs b/src/source/traits.rs index e432729..bcd487a 100644 --- a/src/source/traits.rs +++ b/src/source/traits.rs @@ -1,3 +1,4 @@ +#![allow(async_fn_in_trait)] use std::{ collections::BTreeMap, fmt::{Debug, Display}, @@ -41,12 +42,12 @@ pub trait PackageSource: Debug { type DownloadError: std::error::Error; /// Refreshes the source - fn refresh(&self, _project: &Project) -> Result<(), Self::RefreshError> { + async fn refresh(&self, _project: &Project) -> Result<(), Self::RefreshError> { Ok(()) } /// Resolves a specifier to a reference - fn resolve( + async fn resolve( &self, specifier: &Self::Specifier, project: &Project, @@ -54,10 +55,10 @@ pub trait PackageSource: Debug { ) -> Result, Self::ResolveError>; /// Downloads a package - fn download( + async fn download( &self, pkg_ref: &Self::Ref, project: &Project, - reqwest: &reqwest::blocking::Client, + reqwest: &reqwest::Client, ) -> Result<(PackageFS, Target), Self::DownloadError>; } diff --git a/src/source/wally/compat_util.rs b/src/source/wally/compat_util.rs index b11629e..befb877 100644 --- a/src/source/wally/compat_util.rs +++ b/src/source/wally/compat_util.rs @@ -10,6 +10,7 @@ use crate::{ source::wally::manifest::{Realm, WallyManifest}, Project, LINK_LIB_NO_FILE_FOUND, }; +use fs_err::tokio as fs; #[derive(Deserialize)] #[serde(rename_all = "camelCase")] @@ -18,11 +19,11 @@ struct SourcemapNode { file_paths: Vec, } -pub(crate) fn find_lib_path( +pub(crate) async fn find_lib_path( project: &Project, package_dir: &Path, ) -> Result, errors::FindLibPathError> { - let manifest = project.deser_manifest()?; + let manifest = project.deser_manifest().await?; let Some(script_path) = manifest .scripts @@ -53,16 +54,17 @@ pub(crate) fn find_lib_path( pub(crate) const WALLY_MANIFEST_FILE_NAME: &str = "wally.toml"; -pub(crate) fn get_target( +pub(crate) async fn get_target( project: &Project, tempdir: &TempDir, ) -> Result { - let lib = find_lib_path(project, tempdir.path())? + let lib = find_lib_path(project, tempdir.path()) + .await? .or_else(|| Some(RelativePathBuf::from(LINK_LIB_NO_FILE_FOUND))); let build_files = Default::default(); let manifest = tempdir.path().join(WALLY_MANIFEST_FILE_NAME); - let manifest = fs_err::read_to_string(&manifest)?; + let manifest = fs::read_to_string(&manifest).await?; let manifest: WallyManifest = toml::from_str(&manifest)?; Ok(if matches!(manifest.package.realm, Realm::Shared) { diff --git a/src/source/wally/mod.rs b/src/source/wally/mod.rs index b0d03ee..cff5295 100644 --- a/src/source/wally/mod.rs +++ b/src/source/wally/mod.rs @@ -1,19 +1,8 @@ -use std::{ - collections::{BTreeMap, VecDeque}, - path::PathBuf, -}; - -use gix::Url; -use relative_path::RelativePathBuf; -use reqwest::header::AUTHORIZATION; -use serde::Deserialize; -use tempfile::tempdir; - use crate::{ manifest::target::{Target, TargetKind}, names::PackageNames, source::{ - fs::{store_reader_in_cas, FSEntry, PackageFS}, + fs::{store_in_cas, FSEntry, PackageFS}, git_index::GitBasedSource, traits::PackageSource, version_id::VersionId, @@ -23,6 +12,15 @@ use crate::{ util::hash, Project, }; +use fs_err::tokio as fs; +use gix::Url; +use relative_path::RelativePathBuf; +use reqwest::header::AUTHORIZATION; +use serde::Deserialize; +use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; +use tempfile::tempdir; +use tokio::{io::AsyncWriteExt, sync::Mutex}; +use tokio_util::compat::FuturesAsyncReadCompatExt; pub(crate) mod compat_util; pub(crate) mod manifest; @@ -86,11 +84,11 @@ impl PackageSource for WallyPackageSource { type ResolveError = errors::ResolveError; type DownloadError = errors::DownloadError; - fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> { - GitBasedSource::refresh(self, project) + async fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> { + GitBasedSource::refresh(self, project).await } - fn resolve( + async fn resolve( &self, specifier: &Self::Specifier, project: &Project, @@ -138,11 +136,11 @@ impl PackageSource for WallyPackageSource { )) } - fn download( + async fn download( &self, pkg_ref: &Self::Ref, project: &Project, - reqwest: &reqwest::blocking::Client, + reqwest: &reqwest::Client, ) -> Result<(PackageFS, Target), Self::DownloadError> { let config = self.config(project).map_err(Box::new)?; let index_file = project @@ -151,7 +149,7 @@ impl PackageSource for WallyPackageSource { .join(pkg_ref.name.escaped()) .join(pkg_ref.version.to_string()); - let tempdir = match fs_err::read_to_string(&index_file) { + let tempdir = match fs::read_to_string(&index_file).await { Ok(s) => { log::debug!( "using cached index file for package {}@{}", @@ -162,9 +160,9 @@ impl PackageSource for WallyPackageSource { let tempdir = tempdir()?; let fs = toml::from_str::(&s)?; - fs.write_to(&tempdir, project.cas_dir(), false)?; + fs.write_to(&tempdir, project.cas_dir(), false).await?; - return Ok((fs, get_target(project, &tempdir)?)); + return Ok((fs, get_target(project, &tempdir).await?)); } Err(e) if e.kind() == std::io::ErrorKind::NotFound => tempdir()?, Err(e) => return Err(errors::DownloadError::ReadIndex(e)), @@ -190,50 +188,67 @@ impl PackageSource for WallyPackageSource { request = request.header(AUTHORIZATION, token); } - let response = request.send()?.error_for_status()?; - let bytes = response.bytes()?; - - let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes))?; - archive.extract(tempdir.path())?; + let response = request.send().await?.error_for_status()?; + let mut bytes = response.bytes().await?; let mut entries = BTreeMap::new(); + let mut archive = async_zip::tokio::read::seek::ZipFileReader::with_tokio( + std::io::Cursor::new(&mut bytes), + ) + .await?; - let mut dir_entries = fs_err::read_dir(tempdir.path())?.collect::>(); - while let Some(entry) = dir_entries.pop_front() { - let entry = entry?; - let path = - RelativePathBuf::from_path(entry.path().strip_prefix(tempdir.path())?).unwrap(); + for index in 0..archive.file().entries().len() { + let entry = archive.file().entries().get(index).unwrap(); - if entry.file_type()?.is_dir() { - if IGNORED_DIRS.contains(&path.as_str()) { + let relative_path = RelativePathBuf::from_path(entry.filename().as_str()?).unwrap(); + let path = relative_path.to_path(tempdir.path()); + let name = relative_path.file_name().unwrap_or(""); + + let entry_is_dir = entry.dir()?; + let entry_reader = archive.reader_without_entry(index).await?; + + if entry_is_dir { + if IGNORED_DIRS.contains(&name) { continue; } - entries.insert(path, FSEntry::Directory); - dir_entries.extend(fs_err::read_dir(entry.path())?); + entries.insert(relative_path, FSEntry::Directory); + fs::create_dir_all(&path).await?; continue; } - if IGNORED_FILES.contains(&path.as_str()) { + if IGNORED_FILES.contains(&name) { continue; } - let mut file = fs_err::File::open(entry.path())?; - let hash = store_reader_in_cas(project.cas_dir(), &mut file)?; - entries.insert(path, FSEntry::File(hash)); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).await?; + } + + let writer = Arc::new(Mutex::new(fs::File::create(&path).await?)); + let hash = store_in_cas(project.cas_dir(), entry_reader.compat(), |bytes| { + let writer = writer.clone(); + async move { writer.lock().await.write_all(&bytes).await } + }) + .await?; + + entries.insert(relative_path, FSEntry::File(hash)); } let fs = PackageFS::CAS(entries); if let Some(parent) = index_file.parent() { - fs_err::create_dir_all(parent).map_err(errors::DownloadError::WriteIndex)?; + fs::create_dir_all(parent) + .await + .map_err(errors::DownloadError::WriteIndex)?; } - fs_err::write(&index_file, toml::to_string(&fs)?) + fs::write(&index_file, toml::to_string(&fs)?) + .await .map_err(errors::DownloadError::WriteIndex)?; - Ok((fs, get_target(project, &tempdir)?)) + Ok((fs, get_target(project, &tempdir).await?)) } } @@ -320,7 +335,7 @@ pub mod errors { /// Error decompressing archive #[error("error decompressing archive")] - Decompress(#[from] zip::result::ZipError), + Decompress(#[from] async_zip::error::ZipError), /// Error interacting with the filesystem #[error("error interacting with the filesystem")] diff --git a/src/source/workspace/mod.rs b/src/source/workspace/mod.rs index 9a01a03..c421e4d 100644 --- a/src/source/workspace/mod.rs +++ b/src/source/workspace/mod.rs @@ -7,9 +7,11 @@ use crate::{ }, Project, DEFAULT_INDEX_NAME, }; +use futures::StreamExt; use relative_path::RelativePathBuf; -use reqwest::blocking::Client; +use reqwest::Client; use std::collections::BTreeMap; +use tokio::pin; /// The workspace package reference pub mod pkg_ref; @@ -27,12 +29,12 @@ impl PackageSource for WorkspacePackageSource { type ResolveError = errors::ResolveError; type DownloadError = errors::DownloadError; - fn refresh(&self, _project: &Project) -> Result<(), Self::RefreshError> { + async fn refresh(&self, _project: &Project) -> Result<(), Self::RefreshError> { // no-op Ok(()) } - fn resolve( + async fn resolve( &self, specifier: &Self::Specifier, project: &Project, @@ -45,7 +47,10 @@ impl PackageSource for WorkspacePackageSource { .unwrap_or(&project.package_dir); let target = specifier.target.unwrap_or(package_target); - for (path, manifest) in project.workspace_members(workspace_dir)? { + let members = project.workspace_members(workspace_dir).await?; + pin!(members); + + while let Some((path, manifest)) = members.next().await.transpose()? { if manifest.name == specifier.name && manifest.target.kind() == target { break 'finder (path, manifest); } @@ -119,7 +124,7 @@ impl PackageSource for WorkspacePackageSource { )) } - fn download( + async fn download( &self, pkg_ref: &Self::Ref, project: &Project,