From a8a8ffcbe2f54f72a60c831a79ed3a2c1a8bcea8 Mon Sep 17 00:00:00 2001 From: daimond113 <72147841+daimond113@users.noreply.github.com> Date: Thu, 8 Aug 2024 17:59:59 +0200 Subject: [PATCH] feat: implement wally support --- Cargo.lock | 222 ++++++++------ Cargo.toml | 15 +- registry/Cargo.toml | 4 +- registry/src/endpoints/package_version.rs | 7 +- registry/src/endpoints/package_versions.rs | 6 +- registry/src/endpoints/publish_version.rs | 34 +- registry/src/endpoints/search.rs | 6 +- registry/src/error.rs | 4 +- src/cli/commands/add.rs | 42 +++ src/cli/commands/mod.rs | 2 +- src/cli/commands/outdated.rs | 4 + src/cli/commands/publish.rs | 77 ++--- src/cli/commands/self_install.rs | 14 +- src/cli/scripts.rs | 101 ++---- src/download.rs | 4 +- src/lib.rs | 17 +- src/linking/mod.rs | 67 ++-- src/main.rs | 12 +- src/names.rs | 95 +++++- src/resolver.rs | 35 ++- src/scripts.rs | 8 +- src/source/fs.rs | 15 +- src/source/git_index.rs | 268 ++++++++++++++++ src/source/mod.rs | 47 ++- src/source/pesde/mod.rs | 337 +++----------------- src/source/refs.rs | 22 ++ src/source/specifiers.rs | 5 + src/source/wally/compat_util.rs | 83 +++++ src/source/wally/manifest.rs | 82 +++++ src/source/wally/mod.rs | 341 +++++++++++++++++++++ src/source/wally/pkg_ref.rs | 58 ++++ src/source/wally/specifier.rs | 26 ++ 32 files changed, 1466 insertions(+), 594 deletions(-) create mode 100644 src/source/git_index.rs create mode 100644 src/source/wally/compat_util.rs create mode 100644 src/source/wally/manifest.rs create mode 100644 src/source/wally/mod.rs create mode 100644 src/source/wally/pkg_ref.rs create mode 100644 src/source/wally/specifier.rs diff --git a/Cargo.lock b/Cargo.lock index b231dc4..7f09ee4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -160,16 +160,16 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b02303ce8d4e8be5b855af6cf3c3a08f3eff26880faad82bab679c22d3650cb5" +checksum = "7ca2549781d8dd6d75c40cf6b6051260a2cc2f3c62343d761a969a0640646894" dependencies = [ "actix-rt", "actix-service", "actix-utils", "futures-core", "futures-util", - "mio 0.8.11", + "mio 1.0.1", "socket2", "tokio", "tracing", @@ -251,9 +251,9 @@ dependencies = [ [[package]] name = "actix-web-lab" -version = "0.20.2" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7675c1a84eec1b179c844cdea8488e3e409d8e4984026e92fa96c87dd86f33c6" +checksum = "993cb5477d926300f11d7daede86368dbf797ccae3564aadfb41cb5b3aee2d23" dependencies = [ "actix-http", "actix-router", @@ -263,7 +263,6 @@ dependencies = [ "actix-web-lab-derive", "ahash", "arc-swap", - "async-trait", "bytes", "bytestring", "csv", @@ -272,7 +271,7 @@ dependencies = [ "futures-util", "http 0.2.12", "impl-more", - "itertools", + "itertools 0.13.0", "local-channel", "mediatype", "mime", @@ -289,9 +288,9 @@ dependencies = [ [[package]] name = "actix-web-lab-derive" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aa0b287c8de4a76b691f29dbb5451e8dd5b79d777eaf87350c9b0cbfdb5e968" +checksum = "5aa1bc8506ff10e35419d82d2502e182b94bafa1a68f5651e8e1e6c6717fe1d3" dependencies = [ "proc-macro2", "quote", @@ -715,9 +714,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.1" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "bytestring" @@ -760,9 +759,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.6" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" +checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549" dependencies = [ "jobserver", "libc", @@ -813,9 +812,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" +checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc" dependencies = [ "clap_builder", "clap_derive", @@ -823,9 +822,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" +checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99" dependencies = [ "anstream", "anstyle", @@ -835,9 +834,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ "heck", "proc-macro2", @@ -1247,9 +1246,9 @@ checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" [[package]] name = "dunce" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "dyn-clone" @@ -1399,9 +1398,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920" dependencies = [ "crc32fast", "miniz_oxide", @@ -1707,7 +1706,7 @@ dependencies = [ "gix-utils", "itoa", "thiserror", - "winnow 0.6.16", + "winnow 0.6.18", ] [[package]] @@ -1789,7 +1788,7 @@ dependencies = [ "smallvec", "thiserror", "unicode-bom", - "winnow 0.6.16", + "winnow 0.6.18", ] [[package]] @@ -2042,7 +2041,7 @@ dependencies = [ "itoa", "smallvec", "thiserror", - "winnow 0.6.16", + "winnow 0.6.18", ] [[package]] @@ -2165,7 +2164,7 @@ dependencies = [ "gix-utils", "maybe-async", "thiserror", - "winnow 0.6.16", + "winnow 0.6.18", ] [[package]] @@ -2197,7 +2196,7 @@ dependencies = [ "gix-validate", "memmap2", "thiserror", - "winnow 0.6.16", + "winnow 0.6.18", ] [[package]] @@ -2432,7 +2431,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.2.6", + "indexmap 2.3.0", "slab", "tokio", "tokio-util", @@ -2451,7 +2450,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.2.6", + "indexmap 2.3.0", "slab", "tokio", "tokio-util", @@ -2661,9 +2660,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" dependencies = [ "bytes", "futures-channel", @@ -2737,9 +2736,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -2869,6 +2868,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -2895,9 +2903,9 @@ dependencies = [ [[package]] name = "keyring" -version = "3.0.4" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c118b1bc529b034aad851808f41f49a69a337d10e112039e7f342e5fd514635b" +checksum = "0c163ef0b9da5ccf44ae4d7c9d24fb1a8750aa1969d484865fc1eedc44b26c09" dependencies = [ "byteorder", "dbus-secret-service", @@ -3062,9 +3070,9 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" dependencies = [ "hashbrown 0.14.5", ] @@ -3197,6 +3205,7 @@ checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ "hermit-abi 0.3.9", "libc", + "log", "wasi", "windows-sys 0.52.0", ] @@ -3375,9 +3384,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.2" +version = "0.36.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" +checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" dependencies = [ "memchr", ] @@ -3577,10 +3586,11 @@ dependencies = [ "serde_with", "sha2", "tar", + "tempfile", "thiserror", "threadpool", "toml", - "toml_edit 0.22.17", + "toml_edit 0.22.20", "url", "winreg", "zip", @@ -3695,9 +3705,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "pretty_env_logger" @@ -3760,16 +3773,17 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.0.0", "rustls", + "socket2", "thiserror", "tokio", "tracing", @@ -3777,14 +3791,14 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" dependencies = [ "bytes", "rand", "ring", - "rustc-hash", + "rustc-hash 2.0.0", "rustls", "slab", "thiserror", @@ -3801,6 +3815,7 @@ dependencies = [ "libc", "once_cell", "socket2", + "tracing", "windows-sys 0.52.0", ] @@ -3913,9 +3928,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", @@ -4042,6 +4057,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" version = "0.4.0" @@ -4080,9 +4101,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ "base64 0.22.1", "rustls-pki-types", @@ -4090,9 +4111,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" @@ -4336,18 +4357,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.204" +version = "1.0.205" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.205" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1" dependencies = [ "proc-macro2", "quote", @@ -4361,7 +4382,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" dependencies = [ "form_urlencoded", - "indexmap 2.2.6", + "indexmap 2.3.0", "itoa", "ryu", "serde", @@ -4369,9 +4390,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.121" +version = "1.0.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609" +checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" dependencies = [ "itoa", "memchr", @@ -4430,7 +4451,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.2.6", + "indexmap 2.3.0", "serde", "serde_derive", "serde_json", @@ -4496,9 +4517,9 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" dependencies = [ "libc", "mio 0.8.11", @@ -4670,7 +4691,7 @@ dependencies = [ "fnv", "fs4", "htmlescape", - "itertools", + "itertools 0.12.1", "levenshtein_automata", "log", "lru", @@ -4683,7 +4704,7 @@ dependencies = [ "rayon", "regex", "rust-stemmers", - "rustc-hash", + "rustc-hash 1.1.0", "serde", "serde_json", "sketches-ddsketch", @@ -4719,7 +4740,7 @@ checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" dependencies = [ "downcast-rs", "fastdivide", - "itertools", + "itertools 0.12.1", "serde", "tantivy-bitpacker", "tantivy-common", @@ -4805,14 +4826,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4913,9 +4935,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.1" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes", @@ -4987,21 +5009,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.17", + "toml_edit 0.22.20", ] [[package]] name = "toml_datetime" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] @@ -5012,22 +5034,22 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.3.0", "toml_datetime", "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.22.17" +version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16" +checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.3.0", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.16", + "winnow 0.6.18", ] [[package]] @@ -5378,11 +5400,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5428,6 +5450,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -5560,9 +5591,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.16" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b480ae9340fc261e6be3e95a1ba86d54ae3f9171132a73ce8d4bbaf68339507c" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" dependencies = [ "memchr", ] @@ -5660,6 +5691,7 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] @@ -5696,9 +5728,9 @@ dependencies = [ [[package]] name = "zip" -version = "2.1.5" +version = "2.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b895748a3ebcb69b9d38dcfdf21760859a4b0d0b0015277640c2ef4c69640e6f" +checksum = "40dd8c92efc296286ce1fbd16657c5dbefff44f1b4ca01cc5f517d8b7b3d3e2e" dependencies = [ "aes", "arbitrary", @@ -5710,7 +5742,7 @@ dependencies = [ "displaydoc", "flate2", "hmac", - "indexmap 2.2.6", + "indexmap 2.3.0", "lzma-rs", "memchr", "pbkdf2", @@ -5748,18 +5780,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.12+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 3558c5d..8323537 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,13 +44,13 @@ uninlined_format_args = "warn" [dependencies] serde = { version = "1.0.204", features = ["derive"] } -toml = "0.8.16" +toml = "0.8.19" serde_with = "3.9.0" gix = { version = "0.64.0", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls", "revparse-regex", "credentials"] } semver = { version = "1.0.23", features = ["serde"] } reqwest = { version = "0.12.5", default-features = false, features = ["rustls-tls", "blocking"] } tar = "0.4.41" -flate2 = "1.0.30" +flate2 = "1.0.31" pathdiff = "0.2.1" relative-path = { version = "1.9.3", features = ["serde"] } log = "0.4.22" @@ -62,19 +62,20 @@ url = { version = "2.5.2", features = ["serde"] } # secrecy = "0.8.0" chrono = { version = "0.4.38", features = ["serde"] } sha2 = "0.10.8" +tempfile = "3.12.0" # TODO: remove this when gitoxide adds support for: committing, pushing, adding git2 = { version = "0.19.0", optional = true } -zip = { version = "2.1.5", optional = true } -serde_json = { version = "1.0.121", optional = true } +zip = { version = "2.1.6", optional = true } +serde_json = { version = "1.0.122", optional = true } anyhow = { version = "1.0.86", optional = true } open = { version = "5.3.0", optional = true } -keyring = { version = "3.0.4", features = ["crypto-rust", "windows-native", "apple-native", "linux-native"], optional = true } +keyring = { version = "3.0.5", features = ["crypto-rust", "windows-native", "apple-native", "linux-native"], optional = true } colored = { version = "2.1.0", optional = true } -toml_edit = { version = "0.22.17", optional = true } -clap = { version = "4.5.11", features = ["derive"], optional = true } +toml_edit = { version = "0.22.20", optional = true } +clap = { version = "4.5.13", features = ["derive"], optional = true } dirs = { version = "5.0.1", optional = true } pretty_env_logger = { version = "0.5.0", optional = true } indicatif = { version = "0.17.8", optional = true } diff --git a/registry/Cargo.toml b/registry/Cargo.toml index 372c68f..dfe1cd0 100644 --- a/registry/Cargo.toml +++ b/registry/Cargo.toml @@ -7,8 +7,8 @@ publish = false [dependencies] actix-web = "4.8.0" -actix-web-lab = "0.20.2" -actix-multipart = { version = "0.7.2", features = ["derive"] } +actix-web-lab = "0.21.0" +actix-multipart = "0.7.2" actix-cors = "0.7.0" actix-governor = "0.5.0" dotenvy = "0.15.7" diff --git a/registry/src/endpoints/package_version.rs b/registry/src/endpoints/package_version.rs index df80c6a..d991ec1 100644 --- a/registry/src/endpoints/package_version.rs +++ b/registry/src/endpoints/package_version.rs @@ -5,13 +5,16 @@ use rusty_s3::{actions::GetObject, S3Action}; use semver::Version; use serde::{Deserialize, Deserializer}; -use pesde::{manifest::target::TargetKind, names::PackageName, source::pesde::IndexFile}; - use crate::{ error::Error, package::{s3_name, PackageResponse, S3_SIGN_DURATION}, AppState, }; +use pesde::{ + manifest::target::TargetKind, + names::PackageName, + source::{git_index::GitBasedSource, pesde::IndexFile}, +}; #[derive(Debug)] pub enum VersionRequest { diff --git a/registry/src/endpoints/package_versions.rs b/registry/src/endpoints/package_versions.rs index 26ebf78..82dc7c3 100644 --- a/registry/src/endpoints/package_versions.rs +++ b/registry/src/endpoints/package_versions.rs @@ -1,8 +1,10 @@ use actix_web::{web, HttpResponse, Responder}; -use pesde::{names::PackageName, source::pesde::IndexFile}; - use crate::{error::Error, package::PackageResponse, AppState}; +use pesde::{ + names::PackageName, + source::{git_index::GitBasedSource, pesde::IndexFile}, +}; pub async fn get_package_versions( app_state: web::Data, diff --git a/registry/src/endpoints/publish_version.rs b/registry/src/endpoints/publish_version.rs index 065c7da..55373e0 100644 --- a/registry/src/endpoints/publish_version.rs +++ b/registry/src/endpoints/publish_version.rs @@ -3,8 +3,9 @@ use std::{ io::{Cursor, Read, Write}, }; -use actix_multipart::form::{bytes::Bytes, MultipartForm}; +use actix_multipart::Multipart; use actix_web::{web, HttpResponse, Responder}; +use actix_web_lab::__reexports::futures_util::StreamExt; use flate2::read::GzDecoder; use git2::{Remote, Repository, Signature}; use rusty_s3::{actions::PutObject, S3Action}; @@ -13,9 +14,9 @@ use tar::Archive; use pesde::{ manifest::Manifest, source::{ + git_index::GitBasedSource, pesde::{IndexFile, IndexFileEntry, ScopeInfo, SCOPE_INFO_FILE}, specifiers::DependencySpecifiers, - traits::PackageSource, version_id::VersionId, }, DEFAULT_INDEX_NAME, MANIFEST_FILE_NAME, @@ -30,12 +31,6 @@ use crate::{ AppState, }; -#[derive(MultipartForm)] -pub struct PublishBody { - #[multipart(limit = "4 MiB")] - tarball: Bytes, -} - fn signature<'a>() -> Signature<'a> { Signature::now( &benv!(required "COMMITTER_GIT_NAME"), @@ -63,10 +58,24 @@ const FORBIDDEN_DIRECTORIES: &[&str] = &[".git"]; pub async fn publish_package( app_state: web::Data, - body: MultipartForm, + mut body: Multipart, user_id: web::ReqData, ) -> Result { - let bytes = body.tarball.data.to_vec(); + let max_archive_size = { + let source = app_state.source.lock().unwrap(); + source.refresh(&app_state.project).map_err(Box::new)?; + source.config(&app_state.project)?.max_archive_size + }; + + let bytes = body + .next() + .await + .ok_or(Error::InvalidArchive)? + .map_err(|_| Error::InvalidArchive)? + .bytes(max_archive_size) + .await + .map_err(|_| Error::InvalidArchive)? + .map_err(|_| Error::InvalidArchive)?; let mut decoder = GzDecoder::new(Cursor::new(&bytes)); let mut archive = Archive::new(&mut decoder); @@ -139,6 +148,11 @@ pub async fn publish_package( return Err(Error::InvalidArchive); } } + DependencySpecifiers::Wally(_) => { + if !config.wally_allowed { + return Err(Error::InvalidArchive); + } + } }; } diff --git a/registry/src/endpoints/search.rs b/registry/src/endpoints/search.rs index d161e6a..cf8c4bd 100644 --- a/registry/src/endpoints/search.rs +++ b/registry/src/endpoints/search.rs @@ -4,9 +4,11 @@ use actix_web::{web, HttpResponse, Responder}; use serde::Deserialize; use tantivy::{query::AllQuery, schema::Value, DateTime, Order}; -use pesde::{names::PackageName, source::pesde::IndexFile}; - use crate::{error::Error, package::PackageResponse, AppState}; +use pesde::{ + names::PackageName, + source::{git_index::GitBasedSource, pesde::IndexFile}, +}; #[derive(Deserialize)] pub struct Request { diff --git a/registry/src/error.rs b/registry/src/error.rs index 064067e..1324059 100644 --- a/registry/src/error.rs +++ b/registry/src/error.rs @@ -1,6 +1,6 @@ use actix_web::{body::BoxBody, HttpResponse, ResponseError}; use log::error; -use pesde::source::pesde::errors::ReadFile; +use pesde::source::git_index::errors::{ReadFile, RefreshError}; use serde::Serialize; use thiserror::Error; @@ -31,7 +31,7 @@ pub enum Error { Git(#[from] git2::Error), #[error("failed to refresh source")] - Refresh(#[from] Box), + Refresh(#[from] Box), #[error("failed to serialize struct")] Serialize(#[from] toml::ser::Error), diff --git a/src/cli/commands/add.rs b/src/cli/commands/add.rs index 806afcb..c0997ba 100644 --- a/src/cli/commands/add.rs +++ b/src/cli/commands/add.rs @@ -67,6 +67,22 @@ impl AddCommand { PackageSources::Pesde(PesdePackageSource::new(index)) } + #[cfg(feature = "wally-compat")] + PackageNames::Wally(_) => { + let index = manifest + .wally_indices + .get(self.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME)) + .cloned(); + + if let Some(index) = self.index.as_ref().filter(|_| index.is_none()) { + log::error!("wally index {index} not found"); + return Ok(()); + } + + let index = index.unwrap_or(read_config()?.default_index); + + PackageSources::Wally(pesde::source::wally::WallyPackageSource::new(index)) + } }; source .refresh(&project) @@ -79,6 +95,14 @@ impl AddCommand { index: self.index, target: self.target, }), + #[cfg(feature = "wally-compat")] + PackageNames::Wally(name) => DependencySpecifiers::Wally( + pesde::source::wally::specifier::WallyDependencySpecifier { + name: name.clone(), + version: self.name.1.unwrap_or(VersionReq::STAR), + index: self.index, + }, + ), }; let Some(version_id) = source @@ -139,6 +163,24 @@ impl AddCommand { dependency_key ); } + #[cfg(feature = "wally-compat")] + DependencySpecifiers::Wally(spec) => { + manifest[dependency_key][alias]["name"] = + toml_edit::value(spec.name.clone().to_string()); + manifest[dependency_key][alias]["version"] = + toml_edit::value(format!("^{}", version_id.version())); + + if let Some(index) = spec.index.filter(|i| i != DEFAULT_INDEX_NAME) { + manifest[dependency_key][alias]["index"] = toml_edit::value(index); + } + + println!( + "added wally {}@{} to {}", + spec.name, + version_id.version(), + dependency_key + ); + } } project diff --git a/src/cli/commands/mod.rs b/src/cli/commands/mod.rs index 332dbb7..ec5dbb1 100644 --- a/src/cli/commands/mod.rs +++ b/src/cli/commands/mod.rs @@ -76,7 +76,7 @@ impl Subcommand { Subcommand::Run(run) => run.run(project), Subcommand::Install(install) => install.run(project, multi, reqwest), Subcommand::Publish(publish) => publish.run(project, reqwest), - Subcommand::SelfInstall(self_install) => self_install.run(project), + Subcommand::SelfInstall(self_install) => self_install.run(), #[cfg(feature = "patches")] Subcommand::Patch(patch) => patch.run(project, reqwest), #[cfg(feature = "patches")] diff --git a/src/cli/commands/outdated.rs b/src/cli/commands/outdated.rs index 9979cdc..07cb6ce 100644 --- a/src/cli/commands/outdated.rs +++ b/src/cli/commands/outdated.rs @@ -46,6 +46,10 @@ impl OutdatedCommand { DependencySpecifiers::Pesde(ref mut spec) => { spec.version = VersionReq::STAR; } + #[cfg(feature = "wally-compat")] + DependencySpecifiers::Wally(ref mut spec) => { + spec.version = VersionReq::STAR; + } }; } diff --git a/src/cli/commands/publish.rs b/src/cli/commands/publish.rs index a1f6162..9f845ae 100644 --- a/src/cli/commands/publish.rs +++ b/src/cli/commands/publish.rs @@ -1,14 +1,20 @@ +use std::{ + io::{Seek, Write}, + path::Component, +}; + use anyhow::Context; use clap::Args; use colored::Colorize; +use reqwest::StatusCode; +use tempfile::tempfile; + use pesde::{ manifest::target::Target, scripts::ScriptName, source::{pesde::PesdePackageSource, traits::PackageSource}, - Project, DEFAULT_INDEX_NAME, MANIFEST_FILE_NAME, MAX_ARCHIVE_SIZE, + Project, DEFAULT_INDEX_NAME, MANIFEST_FILE_NAME, }; -use reqwest::StatusCode; -use std::path::Component; #[derive(Debug, Args)] pub struct PublishCommand { @@ -270,50 +276,26 @@ impl PublishCommand { println!(); } - let temp_manifest_path = project - .data_dir() - .join(format!("temp_manifest_{}", chrono::Utc::now().timestamp())); - - std::fs::write( - &temp_manifest_path, - toml::to_string(&manifest).context("failed to serialize manifest")?, - ) - .context("failed to write temp manifest file")?; - - let mut temp_manifest = std::fs::File::open(&temp_manifest_path) - .context("failed to open temp manifest file")?; + let mut temp_manifest = tempfile().context("failed to create temp manifest file")?; + temp_manifest + .write_all( + toml::to_string(&manifest) + .context("failed to serialize manifest")? + .as_bytes(), + ) + .context("failed to write temp manifest file")?; + temp_manifest + .rewind() + .context("failed to rewind temp manifest file")?; archive.append_file(MANIFEST_FILE_NAME, &mut temp_manifest)?; - drop(temp_manifest); - - std::fs::remove_file(temp_manifest_path)?; - let archive = archive .into_inner() .context("failed to encode archive")? .finish() .context("failed to get archive bytes")?; - if archive.len() > MAX_ARCHIVE_SIZE { - anyhow::bail!( - "archive size exceeds maximum size of {} bytes by {} bytes", - MAX_ARCHIVE_SIZE, - archive.len() - MAX_ARCHIVE_SIZE - ); - } - - if self.dry_run { - std::fs::write("package.tar.gz", archive)?; - - println!( - "{}", - "(dry run) package written to package.tar.gz".green().bold() - ); - - return Ok(()); - } - let source = PesdePackageSource::new( manifest .indices @@ -328,6 +310,25 @@ impl PublishCommand { .config(&project) .context("failed to get source config")?; + if archive.len() > config.max_archive_size { + anyhow::bail!( + "archive size exceeds maximum size of {} bytes by {} bytes", + config.max_archive_size, + archive.len() - config.max_archive_size + ); + } + + if self.dry_run { + std::fs::write("package.tar.gz", archive)?; + + println!( + "{}", + "(dry run) package written to package.tar.gz".green().bold() + ); + + return Ok(()); + } + match reqwest .post(format!("{}/v0/packages", config.api())) .multipart(reqwest::blocking::multipart::Form::new().part( diff --git a/src/cli/commands/self_install.rs b/src/cli/commands/self_install.rs index 254f5e4..7e7c236 100644 --- a/src/cli/commands/self_install.rs +++ b/src/cli/commands/self_install.rs @@ -1,9 +1,7 @@ -use crate::cli::{bin_dir, scripts::update_scripts_folder, version::update_bin_exe, HOME_DIR}; +use crate::cli::{bin_dir, version::update_bin_exe, HOME_DIR}; use anyhow::Context; use clap::Args; use colored::Colorize; -use pesde::Project; - #[derive(Debug, Args)] pub struct SelfInstallCommand { /// Skip adding the bin directory to the PATH @@ -13,9 +11,7 @@ pub struct SelfInstallCommand { } impl SelfInstallCommand { - pub fn run(self, project: Project) -> anyhow::Result<()> { - update_scripts_folder(&project)?; - + pub fn run(self) -> anyhow::Result<()> { let bin_dir = bin_dir()?; #[cfg(windows)] @@ -30,10 +26,8 @@ impl SelfInstallCommand { let path: String = env.get_value("Path").context("failed to get Path value")?; let bin_dir = bin_dir.to_string_lossy(); - - let exists = path - .split(';') - .any(|part| *part == bin_dir); + + let exists = path.split(';').any(|part| *part == bin_dir); if !exists { let new_path = format!("{path};{bin_dir}"); diff --git a/src/cli/scripts.rs b/src/cli/scripts.rs index 546712a..ca9e242 100644 --- a/src/cli/scripts.rs +++ b/src/cli/scripts.rs @@ -1,94 +1,31 @@ -use crate::{ - cli::{config::read_config, home_dir}, - util::authenticate_conn, -}; +use std::fs::remove_dir_all; + use anyhow::Context; -use gix::remote::Direction; + use pesde::Project; -pub fn update_scripts_folder(project: &Project) -> anyhow::Result<()> { +use crate::cli::{config::read_config, home_dir}; + +pub fn update_scripts_folder(_project: &Project) -> anyhow::Result<()> { let scripts_dir = home_dir()?.join("scripts"); if scripts_dir.exists() { - let repo = gix::open(&scripts_dir).context("failed to open scripts repository")?; + // checking out the repository seems to be corrupting the repository contents + // TODO: add actual `git pull`-esque functionality + remove_dir_all(&scripts_dir).context("failed to remove scripts directory")?; + } - let remote = repo - .find_default_remote(Direction::Fetch) - .context("missing default remote of scripts repository")? - .context("failed to find default remote of scripts repository")?; + std::fs::create_dir_all(&scripts_dir).context("failed to create scripts directory")?; - let mut connection = remote - .connect(Direction::Fetch) - .context("failed to connect to default remote of scripts repository")?; + let cli_config = read_config()?; - authenticate_conn(&mut connection, project.auth_config()); - - let results = connection - .prepare_fetch(gix::progress::Discard, Default::default()) - .context("failed to prepare scripts repository fetch")? - .receive(gix::progress::Discard, &false.into()) - .context("failed to receive new scripts repository contents")?; - - let remote_ref = results - .ref_map - .remote_refs - .first() - .context("failed to get remote refs of scripts repository")?; - - let unpacked = remote_ref.unpack(); - let oid = unpacked - .1 - .or(unpacked.2) - .context("couldn't find oid in remote ref")?; - - let tree = repo - .find_object(oid) - .context("failed to find scripts repository tree")? - .peel_to_tree() - .context("failed to peel scripts repository object to tree")?; - - let mut index = gix::index::File::from_state( - gix::index::State::from_tree(&tree.id, &repo.objects, Default::default()) - .context("failed to create index state from scripts repository tree")?, - repo.index_path(), - ); - - let opts = gix::worktree::state::checkout::Options { - overwrite_existing: true, - destination_is_initially_empty: false, - ..Default::default() - }; - - gix::worktree::state::checkout( - &mut index, - repo.work_dir().context("scripts repo is bare")?, - repo.objects - .clone() - .into_arc() - .context("failed to clone objects")?, - &gix::progress::Discard, - &gix::progress::Discard, - &false.into(), - opts, - ) - .context("failed to checkout scripts repository")?; - - index - .write(gix::index::write::Options::default()) - .context("failed to write index")?; - } else { - std::fs::create_dir_all(&scripts_dir).context("failed to create scripts directory")?; - - let cli_config = read_config()?; - - gix::prepare_clone(cli_config.scripts_repo, &scripts_dir) - .context("failed to prepare scripts repository clone")? - .fetch_then_checkout(gix::progress::Discard, &false.into()) - .context("failed to fetch and checkout scripts repository")? - .0 - .main_worktree(gix::progress::Discard, &false.into()) - .context("failed to set scripts repository as main worktree")?; - }; + gix::prepare_clone(cli_config.scripts_repo, &scripts_dir) + .context("failed to prepare scripts repository clone")? + .fetch_then_checkout(gix::progress::Discard, &false.into()) + .context("failed to fetch and checkout scripts repository")? + .0 + .main_worktree(gix::progress::Discard, &false.into()) + .context("failed to set scripts repository as main worktree")?; Ok(()) } diff --git a/src/download.rs b/src/download.rs index 364020f..a9865fd 100644 --- a/src/download.rs +++ b/src/download.rs @@ -110,11 +110,11 @@ pub mod errors { #[derive(Debug, Error)] #[non_exhaustive] pub enum DownloadGraphError { - /// Error occurred deserializing the project manifest + /// An error occurred deserializing the project manifest #[error("error deserializing project manifest")] ManifestDeserializationFailed(#[from] crate::errors::ManifestReadError), - /// Error occurred refreshing a package source + /// An error occurred refreshing a package source #[error("failed to refresh package source")] RefreshFailed(#[from] Box), diff --git a/src/lib.rs b/src/lib.rs index 6e53390..d6cf126 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,13 +38,12 @@ pub const LOCKFILE_FILE_NAME: &str = "pesde.lock"; pub const DEFAULT_INDEX_NAME: &str = "default"; /// The name of the packages container pub const PACKAGES_CONTAINER_NAME: &str = ".pesde"; -/// Maximum size of a package's archive -pub const MAX_ARCHIVE_SIZE: usize = 4 * 1024 * 1024; +pub(crate) const LINK_LIB_NO_FILE_FOUND: &str = "____pesde_no_export_file_found"; /// Struct containing the authentication configuration #[derive(Debug, Default, Clone)] pub struct AuthConfig { - pesde_token: Option, + github_token: Option, git_credentials: Option, } @@ -54,9 +53,9 @@ impl AuthConfig { AuthConfig::default() } - /// Access the pesde token - pub fn pesde_token(&self) -> Option<&str> { - self.pesde_token.as_deref() + /// Access the GitHub token + pub fn github_token(&self) -> Option<&str> { + self.github_token.as_deref() } /// Access the git credentials @@ -64,9 +63,9 @@ impl AuthConfig { self.git_credentials.as_ref() } - /// Set the pesde token - pub fn with_pesde_token>(mut self, token: Option) -> Self { - self.pesde_token = token.map(|s| s.as_ref().to_string()); + /// Set the GitHub token + pub fn with_github_token>(mut self, token: Option) -> Self { + self.github_token = token.map(|s| s.as_ref().to_string()); self } diff --git a/src/linking/mod.rs b/src/linking/mod.rs index 00054aa..4706214 100644 --- a/src/linking/mod.rs +++ b/src/linking/mod.rs @@ -1,3 +1,9 @@ +use std::{ + collections::BTreeMap, + fs::create_dir_all, + path::{Path, PathBuf}, +}; + use crate::{ linking::generator::get_file_types, lockfile::DownloadedGraph, @@ -5,12 +11,7 @@ use crate::{ names::PackageNames, scripts::{execute_script, ScriptName}, source::{fs::store_in_cas, traits::PackageRef, version_id::VersionId}, - Project, PACKAGES_CONTAINER_NAME, -}; -use std::{ - collections::BTreeMap, - fs::create_dir_all, - path::{Path, PathBuf}, + Project, LINK_LIB_NO_FILE_FOUND, PACKAGES_CONTAINER_NAME, }; /// Generates linking modules for a project @@ -23,7 +24,7 @@ fn create_and_canonicalize>(path: P) -> std::io::Result } fn write_cas(destination: PathBuf, cas_dir: &Path, contents: &str) -> std::io::Result<()> { - let cas_path = store_in_cas(cas_dir, contents)?.1; + let cas_path = store_in_cas(cas_dir, contents.as_bytes())?.1; std::fs::hard_link(cas_path, destination) } @@ -50,37 +51,45 @@ impl Project { version_id.version(), ); - let lib_file = lib_file.to_path(&container_folder); + let types = if lib_file.as_str() != LINK_LIB_NO_FILE_FOUND { + let lib_file = lib_file.to_path(&container_folder); - let contents = match std::fs::read_to_string(&lib_file) { - Ok(contents) => contents, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - return Err(errors::LinkingError::LibFileNotFound( - lib_file.display().to_string(), - )); - } - Err(e) => return Err(e.into()), + let contents = match std::fs::read_to_string(&lib_file) { + Ok(contents) => contents, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Err(errors::LinkingError::LibFileNotFound( + lib_file.display().to_string(), + )); + } + Err(e) => return Err(e.into()), + }; + + let types = match get_file_types(&contents) { + Ok(types) => types, + Err(e) => { + return Err(errors::LinkingError::FullMoon( + lib_file.display().to_string(), + e, + )) + } + }; + + log::debug!("{name}@{version_id} has {} exported types", types.len()); + + types + } else { + vec![] }; - let types = match get_file_types(&contents) { - Ok(types) => types, - Err(e) => { - return Err(errors::LinkingError::FullMoon( - lib_file.display().to_string(), - e, - )) - } - }; - - log::debug!("{name}@{version_id} has {} exported types", types.len()); - package_types .entry(name) .or_default() .insert(version_id, types); #[cfg(feature = "roblox")] - if let Target::Roblox { build_files, .. } = &node.target { + if let Some(Target::Roblox { build_files, .. }) = + Some(&node.target).filter(|_| !node.node.pkg_ref.is_wally()) + { let script_name = ScriptName::RobloxSyncConfigGenerator.to_string(); let Some(script_path) = manifest.scripts.get(&script_name) else { diff --git a/src/main.rs b/src/main.rs index 4b35ba9..86c3374 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ use crate::cli::{ auth::get_token, home_dir, + scripts::update_scripts_folder, version::{check_for_updates, current_version, get_or_download_version, max_installed_version}, HOME_DIR, }; @@ -121,7 +122,7 @@ fn run() -> anyhow::Result<()> { cwd, data_dir, cas_dir, - AuthConfig::new().with_pesde_token(token.as_ref()), + AuthConfig::new().with_github_token(token.as_ref()), ); let reqwest = { @@ -161,6 +162,15 @@ fn run() -> anyhow::Result<()> { ); } } + match update_scripts_folder(&project) { + Ok(_) => {} + Err(e) => { + println!( + "{}", + format!("failed to update scripts: {e}\n\n").red().bold() + ); + } + } let target_version = project .deser_manifest() diff --git a/src/names.rs b/src/names.rs index bb8c886..7ed675f 100644 --- a/src/names.rs +++ b/src/names.rs @@ -81,6 +81,9 @@ impl PackageName { pub enum PackageNames { /// A pesde package name Pesde(PackageName), + /// A Wally package name + #[cfg(feature = "wally-compat")] + Wally(wally::WallyPackageName), } impl PackageNames { @@ -88,6 +91,8 @@ impl PackageNames { pub fn as_str(&self) -> (&str, &str) { match self { PackageNames::Pesde(name) => name.as_str(), + #[cfg(feature = "wally-compat")] + PackageNames::Wally(name) => name.as_str(), } } @@ -95,6 +100,8 @@ impl PackageNames { pub fn escaped(&self) -> String { match self { PackageNames::Pesde(name) => name.escaped(), + #[cfg(feature = "wally-compat")] + PackageNames::Wally(name) => name.escaped(), } } } @@ -103,6 +110,8 @@ impl Display for PackageNames { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { PackageNames::Pesde(name) => write!(f, "{name}"), + #[cfg(feature = "wally-compat")] + PackageNames::Wally(name) => write!(f, "{name}"), } } } @@ -111,6 +120,15 @@ impl FromStr for PackageNames { type Err = errors::PackageNamesError; fn from_str(s: &str) -> Result { + #[cfg(feature = "wally-compat")] + if let Some(wally_name) = s + .strip_prefix("wally#") + .or_else(|| if s.contains('-') { Some(s) } else { None }) + .and_then(|s| wally::WallyPackageName::from_str(s).ok()) + { + return Ok(PackageNames::Wally(wally_name)); + } + if let Ok(name) = PackageName::from_str(s) { Ok(PackageNames::Pesde(name)) } else { @@ -119,6 +137,64 @@ impl FromStr for PackageNames { } } +/// Wally package names +#[cfg(feature = "wally-compat")] +pub mod wally { + use std::{fmt::Display, str::FromStr}; + + use serde_with::{DeserializeFromStr, SerializeDisplay}; + + use crate::names::{errors, ErrorReason}; + + /// A Wally package name + #[derive( + Debug, DeserializeFromStr, SerializeDisplay, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, + )] + pub struct WallyPackageName(String, String); + + impl FromStr for WallyPackageName { + type Err = errors::WallyPackageNameError; + + fn from_str(s: &str) -> Result { + let (scope, name) = s + .strip_prefix("wally#") + .unwrap_or(s) + .split_once('/') + .ok_or(Self::Err::InvalidFormat(s.to_string()))?; + + for (reason, part) in [(ErrorReason::Scope, scope), (ErrorReason::Name, name)] { + if part.is_empty() || part.len() > 64 { + return Err(Self::Err::InvalidLength(reason, part.to_string())); + } + + if !part.chars().all(|c| c.is_ascii_alphanumeric() || c == '-') { + return Err(Self::Err::InvalidCharacters(reason, part.to_string())); + } + } + + Ok(Self(scope.to_string(), name.to_string())) + } + } + + impl Display for WallyPackageName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "wally#{}/{}", self.0, self.1) + } + } + + impl WallyPackageName { + /// Returns the parts of the package name + pub fn as_str(&self) -> (&str, &str) { + (&self.0, &self.1) + } + + /// Returns the package name as a string suitable for use in the filesystem + pub fn escaped(&self) -> String { + format!("wally#{}+{}", self.0, self.1) + } + } +} + /// Errors that can occur when working with package names pub mod errors { use thiserror::Error; @@ -149,11 +225,28 @@ pub mod errors { InvalidLength(ErrorReason, String), } + /// Errors that can occur when working with Wally package names + #[cfg(feature = "wally-compat")] + #[derive(Debug, Error)] + pub enum WallyPackageNameError { + /// The package name is not in the format `scope/name` + #[error("wally package name `{0}` is not in the format `scope/name`")] + InvalidFormat(String), + + /// The package name is outside the allowed characters: a-z, 0-9, and - + #[error("wally package {0} `{1}` contains characters outside a-z, 0-9, and -")] + InvalidCharacters(ErrorReason, String), + + /// The package name is not within 1-64 characters long + #[error("wally package {0} `{1}` is not within 1-64 characters long")] + InvalidLength(ErrorReason, String), + } + /// Errors that can occur when working with package names #[derive(Debug, Error)] #[non_exhaustive] pub enum PackageNamesError { - /// The pesde package name is invalid + /// The package name is invalid #[error("invalid package name {0}")] InvalidPackageName(String), } diff --git a/src/resolver.rs b/src/resolver.rs index e45f097..089a330 100644 --- a/src/resolver.rs +++ b/src/resolver.rs @@ -152,6 +152,30 @@ impl Project { PackageSources::Pesde(PesdePackageSource::new(index_url)) } + #[cfg(feature = "wally-compat")] + DependencySpecifiers::Wally(specifier) => { + let index_url = if depth == 0 || overridden { + let index_name = specifier.index.as_deref().unwrap_or(DEFAULT_INDEX_NAME); + + manifest + .wally_indices + .get(index_name) + .ok_or(errors::DependencyGraphError::WallyIndexNotFound( + index_name.to_string(), + ))? + .clone() + } else { + let index_url = specifier.index.clone().unwrap(); + + index_url + .clone() + .try_into() + // specifiers in indices store the index url in this field + .unwrap() + }; + + PackageSources::Wally(crate::source::wally::WallyPackageSource::new(index_url)) + } }; if refreshed_sources.insert(source.clone()) { @@ -305,19 +329,24 @@ pub mod errors { /// An error occurred while deserializing the manifest #[error("failed to deserialize manifest")] ManifestRead(#[from] crate::errors::ManifestReadError), - + /// An error occurred while reading all dependencies from the manifest #[error("error getting all project dependencies")] AllDependencies(#[from] crate::manifest::errors::AllDependenciesError), /// An index was not found in the manifest - #[error("index named {0} not found in manifest")] + #[error("index named `{0}` not found in manifest")] IndexNotFound(String), + /// A Wally index was not found in the manifest + #[cfg(feature = "wally-compat")] + #[error("wally index named `{0}` not found in manifest")] + WallyIndexNotFound(String), + /// An error occurred while refreshing a package source #[error("error refreshing package source")] Refresh(#[from] crate::source::errors::RefreshError), - + /// An error occurred while resolving a package #[error("error resolving package")] Resolve(#[from] crate::source::errors::ResolveError), diff --git a/src/scripts.rs b/src/scripts.rs index 5d4a3aa..6fef047 100644 --- a/src/scripts.rs +++ b/src/scripts.rs @@ -1,13 +1,12 @@ use std::{ ffi::OsStr, + fmt::{Display, Formatter}, io::{BufRead, BufReader}, path::Path, process::{Command, Stdio}, thread::spawn, }; -use std::fmt::{Display, Formatter}; - /// Script names used by pesde #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum ScriptName { @@ -41,6 +40,7 @@ pub fn execute_script, S: AsRef, P: AsRef match Command::new("lune") .arg("run") .arg(script_path.as_os_str()) + .arg("--") .args(args) .current_dir(cwd) .stdin(Stdio::null()) @@ -78,11 +78,11 @@ pub fn execute_script, S: AsRef, P: AsRef for line in stdout.lines() { match line { Ok(line) => { - log::info!("[{script_2}]: {line}"); - if return_stdout { stdout_str.push_str(&line); stdout_str.push('\n'); + } else { + log::info!("[{script_2}]: {line}"); } } Err(e) => { diff --git a/src/source/fs.rs b/src/source/fs.rs index e6420d9..e0e1f58 100644 --- a/src/source/fs.rs +++ b/src/source/fs.rs @@ -24,9 +24,9 @@ pub struct PackageFS(pub(crate) BTreeMap); pub(crate) fn store_in_cas>( cas_dir: P, - contents: &str, + contents: &[u8], ) -> std::io::Result<(String, PathBuf)> { - let hash = hash(contents.as_bytes()); + let hash = hash(contents); let (prefix, rest) = hash.split_at(2); let folder = cas_dir.as_ref().join(prefix); @@ -74,4 +74,15 @@ impl PackageFS { Ok(()) } + + /// Returns the contents of the file with the given hash + pub fn read_file, H: AsRef>( + &self, + file_hash: H, + cas_path: P, + ) -> Option { + let (prefix, rest) = file_hash.as_ref().split_at(2); + let cas_file_path = cas_path.as_ref().join(prefix).join(rest); + std::fs::read_to_string(cas_file_path).ok() + } } diff --git a/src/source/git_index.rs b/src/source/git_index.rs new file mode 100644 index 0000000..e98bc96 --- /dev/null +++ b/src/source/git_index.rs @@ -0,0 +1,268 @@ +use gix::remote::Direction; + +use crate::{util::authenticate_conn, Project}; + +/// A trait for sources that are based on Git repositories +pub trait GitBasedSource { + /// The path to the index + fn path(&self, project: &Project) -> std::path::PathBuf; + + /// The URL of the repository + fn repo_url(&self) -> &gix::Url; + + /// Gets the tree of the repository + fn tree<'a>(&'a self, repo: &'a gix::Repository) -> Result { + // this is a bare repo, so this is the actual path + let path = repo.path().to_path_buf(); + + let remote = match repo.find_default_remote(Direction::Fetch) { + Some(Ok(remote)) => remote, + Some(Err(e)) => return Err(errors::TreeError::GetDefaultRemote(path, Box::new(e))), + None => { + return Err(errors::TreeError::NoDefaultRemote(path)); + } + }; + + let refspec = match remote.refspecs(Direction::Fetch).first() { + Some(head) => head, + None => return Err(errors::TreeError::NoRefSpecs(path)), + }; + + let spec_ref = refspec.to_ref(); + let local_ref = match spec_ref.local() { + Some(local) => local + .to_string() + .replace('*', repo.branch_names().first().unwrap_or(&"main")), + None => return Err(errors::TreeError::NoLocalRefSpec(path)), + }; + + let reference = match repo.find_reference(&local_ref) { + Ok(reference) => reference, + Err(e) => return Err(errors::TreeError::NoReference(local_ref.to_string(), e)), + }; + + let reference_name = reference.name().as_bstr().to_string(); + let id = match reference.into_fully_peeled_id() { + Ok(id) => id, + Err(e) => return Err(errors::TreeError::CannotPeel(reference_name, e)), + }; + + let id_str = id.to_string(); + let object = match id.object() { + Ok(object) => object, + Err(e) => return Err(errors::TreeError::CannotConvertToObject(id_str, e)), + }; + + match object.peel_to_tree() { + Ok(tree) => Ok(tree), + Err(e) => Err(errors::TreeError::CannotPeelToTree(id_str, e)), + } + } + + /// Reads a file from the repository + fn read_file + Clone, P: ToString + PartialEq>( + &self, + file_path: I, + project: &Project, + ) -> Result, errors::ReadFile> { + let path = self.path(project); + + let repo = match gix::open(&path) { + Ok(repo) => repo, + Err(e) => return Err(errors::ReadFile::Open(path, Box::new(e))), + }; + + let tree = match self.tree(&repo) { + Ok(tree) => tree, + Err(e) => return Err(errors::ReadFile::Tree(path, Box::new(e))), + }; + + let file_path_str = file_path + .clone() + .into_iter() + .map(|s| s.to_string()) + .collect::>() + .join(std::path::MAIN_SEPARATOR_STR); + + let mut lookup_buf = vec![]; + let entry = match tree.lookup_entry(file_path, &mut lookup_buf) { + Ok(Some(entry)) => entry, + Ok(None) => return Ok(None), + Err(e) => return Err(errors::ReadFile::Lookup(file_path_str, e)), + }; + + let object = match entry.object() { + Ok(object) => object, + Err(e) => return Err(errors::ReadFile::Lookup(file_path_str, e)), + }; + + let blob = object.into_blob(); + let string = String::from_utf8(blob.data.clone()) + .map_err(|e| errors::ReadFile::Utf8(file_path_str, e))?; + + Ok(Some(string)) + } + + /// Refreshes the repository + fn refresh(&self, project: &Project) -> Result<(), errors::RefreshError> { + let path = self.path(project); + if path.exists() { + let repo = match gix::open(&path) { + Ok(repo) => repo, + Err(e) => return Err(errors::RefreshError::Open(path, Box::new(e))), + }; + let remote = match repo.find_default_remote(Direction::Fetch) { + Some(Ok(remote)) => remote, + Some(Err(e)) => { + return Err(errors::RefreshError::GetDefaultRemote(path, Box::new(e))) + } + None => { + return Err(errors::RefreshError::NoDefaultRemote(path)); + } + }; + + let mut connection = remote.connect(Direction::Fetch).map_err(|e| { + errors::RefreshError::Connect(self.repo_url().to_string(), Box::new(e)) + })?; + + authenticate_conn(&mut connection, &project.auth_config); + + connection + .prepare_fetch(gix::progress::Discard, Default::default()) + .map_err(|e| { + errors::RefreshError::PrepareFetch(self.repo_url().to_string(), Box::new(e)) + })? + .receive(gix::progress::Discard, &false.into()) + .map_err(|e| { + errors::RefreshError::Read(self.repo_url().to_string(), Box::new(e)) + })?; + + return Ok(()); + } + + std::fs::create_dir_all(&path)?; + + let auth_config = project.auth_config.clone(); + + gix::prepare_clone_bare(self.repo_url().clone(), &path) + .map_err(|e| errors::RefreshError::Clone(self.repo_url().to_string(), Box::new(e)))? + .configure_connection(move |c| { + authenticate_conn(c, &auth_config); + Ok(()) + }) + .fetch_only(gix::progress::Discard, &false.into()) + .map_err(|e| errors::RefreshError::Fetch(self.repo_url().to_string(), Box::new(e)))?; + + Ok(()) + } +} + +/// Errors that can occur when interacting with a git-based package source +pub mod errors { + use std::path::PathBuf; + + use thiserror::Error; + + /// Errors that can occur when refreshing a git-based package source + #[derive(Debug, Error)] + #[non_exhaustive] + pub enum RefreshError { + /// Error interacting with the filesystem + #[error("error interacting with the filesystem")] + Io(#[from] std::io::Error), + + /// Error opening the repository + #[error("error opening repository at {0}")] + Open(PathBuf, #[source] Box), + + /// No default remote found in repository + #[error("no default remote found in repository at {0}")] + NoDefaultRemote(PathBuf), + + /// Error getting default remote from repository + #[error("error getting default remote from repository at {0}")] + GetDefaultRemote(PathBuf, #[source] Box), + + /// Error connecting to remote repository + #[error("error connecting to remote repository at {0}")] + Connect(String, #[source] Box), + + /// Error preparing fetch from remote repository + #[error("error preparing fetch from remote repository at {0}")] + PrepareFetch(String, #[source] Box), + + /// Error reading from remote repository + #[error("error reading from remote repository at {0}")] + Read(String, #[source] Box), + + /// Error cloning repository + #[error("error cloning repository from {0}")] + Clone(String, #[source] Box), + + /// Error fetching repository + #[error("error fetching repository from {0}")] + Fetch(String, #[source] Box), + } + + /// Errors that can occur when reading a git-based package source's tree + #[derive(Debug, Error)] + #[non_exhaustive] + pub enum TreeError { + /// Error interacting with the filesystem + #[error("error interacting with the filesystem")] + Io(#[from] std::io::Error), + + /// No default remote found in repository + #[error("no default remote found in repository at {0}")] + NoDefaultRemote(PathBuf), + + /// Error getting default remote from repository + #[error("error getting default remote from repository at {0}")] + GetDefaultRemote(PathBuf, #[source] Box), + + /// Error getting refspec from remote repository + #[error("no refspecs found in repository at {0}")] + NoRefSpecs(PathBuf), + + /// Error getting local refspec from remote repository + #[error("no local refspec found in repository at {0}")] + NoLocalRefSpec(PathBuf), + + /// Error finding reference in repository + #[error("no reference found for local refspec {0}")] + NoReference(String, #[source] gix::reference::find::existing::Error), + + /// Error peeling reference in repository + #[error("cannot peel reference {0}")] + CannotPeel(String, #[source] gix::reference::peel::Error), + + /// Error converting id to object in repository + #[error("error converting id {0} to object")] + CannotConvertToObject(String, #[source] gix::object::find::existing::Error), + + /// Error peeling object to tree in repository + #[error("error peeling object {0} to tree")] + CannotPeelToTree(String, #[source] gix::object::peel::to_kind::Error), + } + + /// Errors that can occur when reading a file from a git-based package source + #[derive(Debug, Error)] + #[non_exhaustive] + pub enum ReadFile { + /// Error opening the repository + #[error("error opening repository at {0}")] + Open(PathBuf, #[source] Box), + + /// Error reading tree from repository + #[error("error getting tree from repository at {0}")] + Tree(PathBuf, #[source] Box), + + /// Error looking up entry in tree + #[error("error looking up entry {0} in tree")] + Lookup(String, #[source] gix::object::find::existing::Error), + + /// Error reading file as utf8 + #[error("error parsing file for {0} as utf8")] + Utf8(String, #[source] std::string::FromUtf8Error), + } +} diff --git a/src/source/mod.rs b/src/source/mod.rs index 7ad41d7..32945f9 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -12,6 +12,8 @@ use crate::{ /// Packages' filesystems pub mod fs; +/// Git index-based package source utilities +pub mod git_index; /// The pesde package source pub mod pesde; /// Package references @@ -22,6 +24,9 @@ pub mod specifiers; pub mod traits; /// Version IDs pub mod version_id; +/// The Wally package source +#[cfg(feature = "wally-compat")] +pub mod wally; /// The result of resolving a package pub type ResolveResult = (PackageNames, BTreeMap); @@ -31,6 +36,9 @@ pub type ResolveResult = (PackageNames, BTreeMap); pub enum PackageSources { /// A pesde package source Pesde(pesde::PesdePackageSource), + /// A Wally package source + #[cfg(feature = "wally-compat")] + Wally(wally::WallyPackageSource), } impl PackageSource for PackageSources { @@ -43,6 +51,8 @@ impl PackageSource for PackageSources { fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> { match self { PackageSources::Pesde(source) => source.refresh(project).map_err(Into::into), + #[cfg(feature = "wally-compat")] + PackageSources::Wally(source) => source.refresh(project).map_err(Into::into), } } @@ -66,6 +76,20 @@ impl PackageSource for PackageSources { }) .map_err(Into::into), + #[cfg(feature = "wally-compat")] + (PackageSources::Wally(source), DependencySpecifiers::Wally(specifier)) => source + .resolve(specifier, project, project_target) + .map(|(name, results)| { + ( + name, + results + .into_iter() + .map(|(version, pkg_ref)| (version, PackageRefs::Wally(pkg_ref))) + .collect(), + ) + }) + .map_err(Into::into), + _ => Err(errors::ResolveError::Mismatch), } } @@ -81,6 +105,11 @@ impl PackageSource for PackageSources { .download(pkg_ref, project, reqwest) .map_err(Into::into), + #[cfg(feature = "wally-compat")] + (PackageSources::Wally(source), PackageRefs::Wally(pkg_ref)) => source + .download(pkg_ref, project, reqwest) + .map_err(Into::into), + _ => Err(errors::DownloadError::Mismatch), } } @@ -94,9 +123,9 @@ pub mod errors { #[derive(Debug, Error)] #[non_exhaustive] pub enum RefreshError { - /// The pesde package source failed to refresh + /// A git-based package source failed to refresh #[error("error refreshing pesde package source")] - Pesde(#[from] crate::source::pesde::errors::RefreshError), + GitBased(#[from] crate::source::git_index::errors::RefreshError), } /// Errors that can occur when resolving a package @@ -107,9 +136,14 @@ pub mod errors { #[error("mismatched dependency specifier for source")] Mismatch, - /// The pesde package source failed to resolve + /// A pesde package source failed to resolve #[error("error resolving pesde package")] Pesde(#[from] crate::source::pesde::errors::ResolveError), + + /// A Wally package source failed to resolve + #[cfg(feature = "wally-compat")] + #[error("error resolving wally package")] + Wally(#[from] crate::source::wally::errors::ResolveError), } /// Errors that can occur when downloading a package @@ -120,8 +154,13 @@ pub mod errors { #[error("mismatched package ref for source")] Mismatch, - /// The pesde package source failed to download + /// A pesde package source failed to download #[error("error downloading pesde package")] Pesde(#[from] crate::source::pesde::errors::DownloadError), + + /// A Wally package source failed to download + #[cfg(feature = "wally-compat")] + #[error("error downloading wally package")] + Wally(#[from] crate::source::wally::errors::DownloadError), } } diff --git a/src/source/pesde/mod.rs b/src/source/pesde/mod.rs index f1318ed..1d84d7b 100644 --- a/src/source/pesde/mod.rs +++ b/src/source/pesde/mod.rs @@ -1,4 +1,18 @@ -use gix::remote::Direction; +use crate::{ + manifest::{ + target::{Target, TargetKind}, + DependencyType, + }, + names::{PackageName, PackageNames}, + source::{ + fs::{store_in_cas, FSEntry, PackageFS}, + git_index::GitBasedSource, + DependencySpecifiers, PackageSource, ResolveResult, VersionId, + }, + util::hash, + Project, +}; +use gix::Url; use pkg_ref::PesdePackageRef; use relative_path::RelativePathBuf; use reqwest::header::ACCEPT; @@ -9,20 +23,7 @@ use std::{ fmt::Debug, hash::Hash, io::Read, -}; - -use crate::{ - manifest::{ - target::{Target, TargetKind}, - DependencyType, - }, - names::{PackageName, PackageNames}, - source::{ - fs::{store_in_cas, FSEntry, PackageFS}, - DependencySpecifiers, PackageSource, ResolveResult, VersionId, - }, - util::{authenticate_conn, hash}, - Project, + path::PathBuf, }; /// The pesde package reference @@ -33,7 +34,7 @@ pub mod specifier; /// The pesde package source #[derive(Debug, Hash, PartialEq, Eq, Clone)] pub struct PesdePackageSource { - repo_url: gix::Url, + repo_url: Url, } /// The file containing scope information @@ -46,9 +47,19 @@ pub struct ScopeInfo { pub owners: BTreeSet, } +impl GitBasedSource for PesdePackageSource { + fn path(&self, project: &Project) -> PathBuf { + project.data_dir.join("indices").join(hash(self.as_bytes())) + } + + fn repo_url(&self) -> &Url { + &self.repo_url + } +} + impl PesdePackageSource { /// Creates a new pesde package source - pub fn new(repo_url: gix::Url) -> Self { + pub fn new(repo_url: Url) -> Self { Self { repo_url } } @@ -56,114 +67,6 @@ impl PesdePackageSource { self.repo_url.to_bstring().to_vec() } - /// The path to the index - pub fn path(&self, project: &Project) -> std::path::PathBuf { - project.data_dir.join("indices").join(hash(self.as_bytes())) - } - - /// The URL of the repository - pub fn repo_url(&self) -> &gix::Url { - &self.repo_url - } - - pub(crate) fn tree<'a>( - &'a self, - repo: &'a gix::Repository, - ) -> Result { - // this is a bare repo, so this is the actual path - let path = repo.path().to_path_buf(); - - let remote = match repo.find_default_remote(Direction::Fetch) { - Some(Ok(remote)) => remote, - Some(Err(e)) => return Err(errors::TreeError::GetDefaultRemote(path, Box::new(e))), - None => { - return Err(errors::TreeError::NoDefaultRemote(path)); - } - }; - - let refspec = match remote.refspecs(Direction::Fetch).first() { - Some(head) => head, - None => return Err(errors::TreeError::NoRefSpecs(path)), - }; - - let spec_ref = refspec.to_ref(); - let local_ref = match spec_ref.local() { - Some(local) => local - .to_string() - .replace('*', repo.branch_names().first().unwrap_or(&"main")), - None => return Err(errors::TreeError::NoLocalRefSpec(path)), - }; - - let reference = match repo.find_reference(&local_ref) { - Ok(reference) => reference, - Err(e) => return Err(errors::TreeError::NoReference(local_ref.to_string(), e)), - }; - - let reference_name = reference.name().as_bstr().to_string(); - let id = match reference.into_fully_peeled_id() { - Ok(id) => id, - Err(e) => return Err(errors::TreeError::CannotPeel(reference_name, e)), - }; - - let id_str = id.to_string(); - let object = match id.object() { - Ok(object) => object, - Err(e) => return Err(errors::TreeError::CannotConvertToObject(id_str, e)), - }; - - match object.peel_to_tree() { - Ok(tree) => Ok(tree), - Err(e) => Err(errors::TreeError::CannotPeelToTree(id_str, e)), - } - } - - /// Reads a file from the index - pub fn read_file< - I: IntoIterator + Clone, - P: ToString + PartialEq, - >( - &self, - file_path: I, - project: &Project, - ) -> Result, errors::ReadFile> { - let path = self.path(project); - - let repo = match gix::open(&path) { - Ok(repo) => repo, - Err(e) => return Err(errors::ReadFile::Open(path, Box::new(e))), - }; - - let tree = match self.tree(&repo) { - Ok(tree) => tree, - Err(e) => return Err(errors::ReadFile::Tree(path, Box::new(e))), - }; - - let file_path_str = file_path - .clone() - .into_iter() - .map(|s| s.to_string()) - .collect::>() - .join(std::path::MAIN_SEPARATOR_STR); - - let mut lookup_buf = vec![]; - let entry = match tree.lookup_entry(file_path, &mut lookup_buf) { - Ok(Some(entry)) => entry, - Ok(None) => return Ok(None), - Err(e) => return Err(errors::ReadFile::Lookup(file_path_str, e)), - }; - - let object = match entry.object() { - Ok(object) => object, - Err(e) => return Err(errors::ReadFile::Lookup(file_path_str, e)), - }; - - let blob = object.into_blob(); - let string = String::from_utf8(blob.data.clone()) - .map_err(|e| errors::ReadFile::Utf8(file_path_str, e))?; - - Ok(Some(string)) - } - /// Reads the config file pub fn config(&self, project: &Project) -> Result { let file = self.read_file(["config.toml"], project).map_err(Box::new)?; @@ -177,9 +80,7 @@ impl PesdePackageSource { } }; - let config: IndexConfig = toml::from_str(&string)?; - - Ok(config) + toml::from_str(&string).map_err(Into::into) } /// Reads all packages from the index @@ -277,56 +178,12 @@ impl PesdePackageSource { impl PackageSource for PesdePackageSource { type Specifier = PesdeDependencySpecifier; type Ref = PesdePackageRef; - type RefreshError = errors::RefreshError; + type RefreshError = crate::source::git_index::errors::RefreshError; type ResolveError = errors::ResolveError; type DownloadError = errors::DownloadError; fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> { - log::debug!("refreshing pesde index at {}", self.repo_url); - - let path = self.path(project); - if path.exists() { - let repo = match gix::open(&path) { - Ok(repo) => repo, - Err(e) => return Err(Self::RefreshError::Open(path, e)), - }; - let remote = match repo.find_default_remote(Direction::Fetch) { - Some(Ok(remote)) => remote, - Some(Err(e)) => return Err(Self::RefreshError::GetDefaultRemote(path, e)), - None => { - return Err(Self::RefreshError::NoDefaultRemote(path)); - } - }; - - let mut connection = remote - .connect(Direction::Fetch) - .map_err(|e| Self::RefreshError::Connect(self.repo_url.clone(), e))?; - - authenticate_conn(&mut connection, &project.auth_config); - - connection - .prepare_fetch(gix::progress::Discard, Default::default()) - .map_err(|e| Self::RefreshError::PrepareFetch(self.repo_url.clone(), e))? - .receive(gix::progress::Discard, &false.into()) - .map_err(|e| Self::RefreshError::Read(self.repo_url.clone(), e))?; - - return Ok(()); - } - - std::fs::create_dir_all(&path)?; - - let auth_config = project.auth_config.clone(); - - gix::prepare_clone_bare(self.repo_url.clone(), &path) - .map_err(|e| Self::RefreshError::Clone(self.repo_url.clone(), e))? - .configure_connection(move |c| { - authenticate_conn(c, &auth_config); - Ok(()) - }) - .fetch_only(gix::progress::Discard, &false.into()) - .map_err(|e| Self::RefreshError::Fetch(self.repo_url.clone(), e))?; - - Ok(()) + GitBasedSource::refresh(self, project) } fn resolve( @@ -416,7 +273,7 @@ impl PackageSource for PesdePackageSource { let mut response = reqwest.get(url).header(ACCEPT, "application/octet-stream"); - if let Some(token) = &project.auth_config.pesde_token { + if let Some(token) = &project.auth_config.github_token { log::debug!("using token for pesde package download"); response = response.header("Authorization", format!("Bearer {token}")); } @@ -439,8 +296,8 @@ impl PackageSource for PesdePackageSource { continue; } - let mut contents = String::new(); - entry.read_to_string(&mut contents)?; + let mut contents = vec![]; + entry.read_to_end(&mut contents)?; let hash = store_in_cas(&project.cas_dir, &contents)?.0; entries.insert(path, FSEntry::File(hash)); @@ -459,22 +316,32 @@ impl PackageSource for PesdePackageSource { } } +fn default_archive_size() -> usize { + 4 * 1024 * 1024 +} + /// The configuration for the pesde index -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] pub struct IndexConfig { /// The URL of the API pub api: url::Url, /// The URL to download packages from pub download: Option, - /// Whether git is allowed as a source for publishing packages + /// Whether Git is allowed as a source for publishing packages #[serde(default)] pub git_allowed: bool, /// Whether other registries are allowed as a source for publishing packages #[serde(default)] pub other_registries_allowed: bool, + /// Whether Wally is allowed as a source for publishing packages + #[serde(default)] + pub wally_allowed: bool, /// The OAuth client ID for GitHub pub github_oauth_client_id: String, + /// The maximum size of an archive in bytes + #[serde(default = "default_archive_size")] + pub max_archive_size: usize, } impl IndexConfig { @@ -520,112 +387,10 @@ pub type IndexFile = BTreeMap; pub mod errors { use std::path::PathBuf; + use crate::source::git_index::errors::{ReadFile, TreeError}; use thiserror::Error; - /// Errors that can occur when refreshing the pesde package source - #[derive(Debug, Error)] - #[non_exhaustive] - pub enum RefreshError { - /// Error interacting with the filesystem - #[error("error interacting with the filesystem")] - Io(#[from] std::io::Error), - - /// Error opening the repository - #[error("error opening repository at {0}")] - Open(PathBuf, #[source] gix::open::Error), - - /// No default remote found in repository - #[error("no default remote found in repository at {0}")] - NoDefaultRemote(PathBuf), - - /// Error getting default remote from repository - #[error("error getting default remote from repository at {0}")] - GetDefaultRemote(PathBuf, #[source] gix::remote::find::existing::Error), - - /// Error connecting to remote repository - #[error("error connecting to remote repository at {0}")] - Connect(gix::Url, #[source] gix::remote::connect::Error), - - /// Error preparing fetch from remote repository - #[error("error preparing fetch from remote repository at {0}")] - PrepareFetch(gix::Url, #[source] gix::remote::fetch::prepare::Error), - - /// Error reading from remote repository - #[error("error reading from remote repository at {0}")] - Read(gix::Url, #[source] gix::remote::fetch::Error), - - /// Error cloning repository - #[error("error cloning repository from {0}")] - Clone(gix::Url, #[source] gix::clone::Error), - - /// Error fetching repository - #[error("error fetching repository from {0}")] - Fetch(gix::Url, #[source] gix::clone::fetch::Error), - } - - /// Errors that can occur when reading the pesde package source's tree - #[derive(Debug, Error)] - #[non_exhaustive] - pub enum TreeError { - /// Error interacting with the filesystem - #[error("error interacting with the filesystem")] - Io(#[from] std::io::Error), - - /// No default remote found in repository - #[error("no default remote found in repository at {0}")] - NoDefaultRemote(PathBuf), - - /// Error getting default remote from repository - #[error("error getting default remote from repository at {0}")] - GetDefaultRemote(PathBuf, #[source] Box), - - /// Error getting refspec from remote repository - #[error("no refspecs found in repository at {0}")] - NoRefSpecs(PathBuf), - - /// Error getting local refspec from remote repository - #[error("no local refspec found in repository at {0}")] - NoLocalRefSpec(PathBuf), - - /// Error finding reference in repository - #[error("no reference found for local refspec {0}")] - NoReference(String, #[source] gix::reference::find::existing::Error), - - /// Error peeling reference in repository - #[error("cannot peel reference {0}")] - CannotPeel(String, #[source] gix::reference::peel::Error), - - /// Error converting id to object in repository - #[error("error converting id {0} to object")] - CannotConvertToObject(String, #[source] gix::object::find::existing::Error), - - /// Error peeling object to tree in repository - #[error("error peeling object {0} to tree")] - CannotPeelToTree(String, #[source] gix::object::peel::to_kind::Error), - } - - /// Errors that can occur when reading a file from the pesde package source - #[derive(Debug, Error)] - #[non_exhaustive] - pub enum ReadFile { - /// Error opening the repository - #[error("error opening repository at {0}")] - Open(PathBuf, #[source] Box), - - /// Error reading tree from repository - #[error("error getting tree from repository at {0}")] - Tree(PathBuf, #[source] Box), - - /// Error looking up entry in tree - #[error("error looking up entry {0} in tree")] - Lookup(String, #[source] gix::object::find::existing::Error), - - /// Error reading file as utf8 - #[error("error parsing file for {0} as utf8")] - Utf8(String, #[source] std::string::FromUtf8Error), - } - - /// Errors that can occur when resolving a package from the pesde package source + /// Errors that can occur when resolving a package from a pesde package source #[derive(Debug, Error)] #[non_exhaustive] pub enum ResolveError { @@ -650,7 +415,7 @@ pub mod errors { Utf8(String, #[source] std::string::FromUtf8Error), } - /// Errors that can occur when reading the config file for the pesde package source + /// Errors that can occur when reading the config file for a pesde package source #[derive(Debug, Error)] #[non_exhaustive] pub enum ConfigError { @@ -667,7 +432,7 @@ pub mod errors { Missing(Box), } - /// Errors that can occur when reading all packages from the pesde package source + /// Errors that can occur when reading all packages from a pesde package source #[derive(Debug, Error)] #[non_exhaustive] pub enum AllPackagesError { @@ -696,7 +461,7 @@ pub mod errors { Utf8(String, #[source] std::string::FromUtf8Error), } - /// Errors that can occur when downloading a package from the pesde package source + /// Errors that can occur when downloading a package from a pesde package source #[derive(Debug, Error)] #[non_exhaustive] pub enum DownloadError { diff --git a/src/source/refs.rs b/src/source/refs.rs index 93c8ec0..0cdadd8 100644 --- a/src/source/refs.rs +++ b/src/source/refs.rs @@ -11,30 +11,52 @@ use std::collections::BTreeMap; pub enum PackageRefs { /// A pesde package reference Pesde(pesde::pkg_ref::PesdePackageRef), + /// A Wally package reference + #[cfg(feature = "wally-compat")] + Wally(crate::source::wally::pkg_ref::WallyPackageRef), +} + +impl PackageRefs { + /// Returns whether this package reference is a Wally package reference + pub fn is_wally(&self) -> bool { + match self { + #[cfg(feature = "wally-compat")] + PackageRefs::Wally(_) => true, + _ => false, + } + } } impl PackageRef for PackageRefs { fn dependencies(&self) -> &BTreeMap { match self { PackageRefs::Pesde(pkg_ref) => pkg_ref.dependencies(), + #[cfg(feature = "wally-compat")] + PackageRefs::Wally(pkg_ref) => pkg_ref.dependencies(), } } fn use_new_structure(&self) -> bool { match self { PackageRefs::Pesde(pkg_ref) => pkg_ref.use_new_structure(), + #[cfg(feature = "wally-compat")] + PackageRefs::Wally(pkg_ref) => pkg_ref.use_new_structure(), } } fn target_kind(&self) -> TargetKind { match self { PackageRefs::Pesde(pkg_ref) => pkg_ref.target_kind(), + #[cfg(feature = "wally-compat")] + PackageRefs::Wally(pkg_ref) => pkg_ref.target_kind(), } } fn source(&self) -> PackageSources { match self { PackageRefs::Pesde(pkg_ref) => pkg_ref.source(), + #[cfg(feature = "wally-compat")] + PackageRefs::Wally(pkg_ref) => pkg_ref.source(), } } } diff --git a/src/source/specifiers.rs b/src/source/specifiers.rs index 6dbd04b..063603c 100644 --- a/src/source/specifiers.rs +++ b/src/source/specifiers.rs @@ -8,6 +8,9 @@ use std::fmt::Display; pub enum DependencySpecifiers { /// A pesde dependency specifier Pesde(pesde::specifier::PesdeDependencySpecifier), + /// A Wally dependency specifier + #[cfg(feature = "wally-compat")] + Wally(crate::source::wally::specifier::WallyDependencySpecifier), } impl DependencySpecifier for DependencySpecifiers {} @@ -15,6 +18,8 @@ impl Display for DependencySpecifiers { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { DependencySpecifiers::Pesde(specifier) => write!(f, "{specifier}"), + #[cfg(feature = "wally-compat")] + DependencySpecifiers::Wally(specifier) => write!(f, "{specifier}"), } } } diff --git a/src/source/wally/compat_util.rs b/src/source/wally/compat_util.rs new file mode 100644 index 0000000..1b5dcb7 --- /dev/null +++ b/src/source/wally/compat_util.rs @@ -0,0 +1,83 @@ +use std::path::Path; + +use relative_path::RelativePathBuf; +use serde::Deserialize; +use tempfile::TempDir; + +use crate::{ + manifest::target::Target, + scripts::{execute_script, ScriptName}, + Project, LINK_LIB_NO_FILE_FOUND, +}; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct SourcemapNode { + #[serde(default)] + file_paths: Vec, +} + +pub(crate) fn find_lib_path( + project: &Project, + cwd: &Path, +) -> Result, errors::FindLibPathError> { + let manifest = project.deser_manifest()?; + + let Some(script_path) = manifest + .scripts + .get(&ScriptName::SourcemapGenerator.to_string()) + else { + log::warn!("no sourcemap generator script found in manifest"); + return Ok(None); + }; + + let result = execute_script( + Some(&ScriptName::SourcemapGenerator.to_string()), + &script_path.to_path(&project.path), + ["--wally"], + cwd, + true, + )?; + + if let Some(result) = result { + let node: SourcemapNode = serde_json::from_str(&result)?; + Ok(node.file_paths.into_iter().find(|path| { + path.extension() + .is_some_and(|ext| ext == "lua" || ext == "luau") + })) + } else { + Ok(None) + } +} + +pub(crate) fn get_target( + project: &Project, + tempdir: &TempDir, +) -> Result { + Ok(Target::Roblox { + lib: find_lib_path(project, tempdir.path())? + .or_else(|| Some(RelativePathBuf::from(LINK_LIB_NO_FILE_FOUND))), + build_files: Default::default(), + }) +} + +pub mod errors { + use thiserror::Error; + + /// Errors that can occur when finding the lib path + #[derive(Debug, Error)] + #[non_exhaustive] + pub enum FindLibPathError { + /// An error occurred deserializing the project manifest + #[error("error deserializing manifest")] + Manifest(#[from] crate::errors::ManifestReadError), + + /// An error occurred while executing the sourcemap generator script + #[error("error executing sourcemap generator script")] + Script(#[from] std::io::Error), + + /// An error occurred while deserializing the sourcemap result + #[error("error deserializing sourcemap result")] + Serde(#[from] serde_json::Error), + } +} diff --git a/src/source/wally/manifest.rs b/src/source/wally/manifest.rs new file mode 100644 index 0000000..64fe24a --- /dev/null +++ b/src/source/wally/manifest.rs @@ -0,0 +1,82 @@ +use std::collections::BTreeMap; + +use semver::{Version, VersionReq}; +use serde::{Deserialize, Deserializer}; + +use crate::{ + manifest::{errors, DependencyType}, + source::{specifiers::DependencySpecifiers, wally::specifier::WallyDependencySpecifier}, +}; + +#[derive(Deserialize, Clone, Debug)] +#[serde(rename_all = "kebab-case")] +pub struct WallyPackage { + pub version: Version, +} + +pub fn deserialize_specifiers<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + // specifier is in form of `name@version_req` + BTreeMap::::deserialize(deserializer)? + .into_iter() + .map(|(k, v)| { + let (name, version) = v.split_once('@').ok_or_else(|| { + serde::de::Error::custom("invalid specifier format, expected `name@version_req`") + })?; + + Ok(( + k, + WallyDependencySpecifier { + name: name.parse().map_err(serde::de::Error::custom)?, + version: VersionReq::parse(version).map_err(serde::de::Error::custom)?, + index: None, + }, + )) + }) + .collect() +} + +#[derive(Deserialize, Clone, Debug)] +#[serde(rename_all = "kebab-case")] +pub struct WallyManifest { + pub package: WallyPackage, + #[serde(default, deserialize_with = "deserialize_specifiers")] + pub dependencies: BTreeMap, + #[serde(default, deserialize_with = "deserialize_specifiers")] + pub server_dependencies: BTreeMap, + #[serde(default, deserialize_with = "deserialize_specifiers")] + pub dev_dependencies: BTreeMap, +} + +impl WallyManifest { + /// Get all dependencies from the manifest + pub fn all_dependencies( + &self, + ) -> Result< + BTreeMap, + errors::AllDependenciesError, + > { + let mut all_deps = BTreeMap::new(); + + for (deps, ty) in [ + (&self.dependencies, DependencyType::Standard), + (&self.server_dependencies, DependencyType::Standard), + (&self.dev_dependencies, DependencyType::Dev), + ] { + for (alias, spec) in deps { + if all_deps + .insert( + alias.clone(), + (DependencySpecifiers::Wally(spec.clone()), ty), + ) + .is_some() + { + return Err(errors::AllDependenciesError::AliasConflict(alias.clone())); + } + } + } + + Ok(all_deps) + } +} diff --git a/src/source/wally/mod.rs b/src/source/wally/mod.rs new file mode 100644 index 0000000..bce3eae --- /dev/null +++ b/src/source/wally/mod.rs @@ -0,0 +1,341 @@ +use std::{ + collections::{BTreeMap, VecDeque}, + io::Read, + path::PathBuf, +}; + +use gix::Url; +use relative_path::RelativePathBuf; +use serde::Deserialize; +use tempfile::tempdir; + +use crate::{ + manifest::target::{Target, TargetKind}, + names::PackageNames, + source::{ + fs::{store_in_cas, FSEntry, PackageFS}, + git_index::GitBasedSource, + traits::PackageSource, + version_id::VersionId, + wally::{compat_util::get_target, manifest::WallyManifest, pkg_ref::WallyPackageRef}, + }, + util::hash, + Project, +}; + +mod compat_util; +pub(crate) mod manifest; +/// The Wally package reference +pub mod pkg_ref; +/// The Wally dependency specifier +pub mod specifier; + +/// The Wally package source +#[derive(Debug, Hash, PartialEq, Eq, Clone)] +pub struct WallyPackageSource { + repo_url: Url, +} + +impl GitBasedSource for WallyPackageSource { + fn path(&self, project: &Project) -> PathBuf { + project + .data_dir + .join("wally_indices") + .join(hash(self.as_bytes())) + } + + fn repo_url(&self) -> &Url { + &self.repo_url + } +} + +impl WallyPackageSource { + /// Creates a new Wally package source + pub fn new(repo_url: Url) -> Self { + Self { repo_url } + } + + fn as_bytes(&self) -> Vec { + self.repo_url.to_bstring().to_vec() + } + + /// Reads the config file + pub fn config(&self, project: &Project) -> Result { + let file = self.read_file(["config.json"], project).map_err(Box::new)?; + + let string = match file { + Some(s) => s, + None => { + return Err(errors::ConfigError::Missing(Box::new( + self.repo_url.clone(), + ))) + } + }; + + serde_json::from_str(&string).map_err(Into::into) + } +} + +impl PackageSource for WallyPackageSource { + type Specifier = specifier::WallyDependencySpecifier; + type Ref = WallyPackageRef; + type RefreshError = crate::source::git_index::errors::RefreshError; + type ResolveError = errors::ResolveError; + type DownloadError = errors::DownloadError; + + fn refresh(&self, project: &Project) -> Result<(), Self::RefreshError> { + GitBasedSource::refresh(self, project) + } + + fn resolve( + &self, + specifier: &Self::Specifier, + project: &Project, + _project_target: TargetKind, + ) -> Result, Self::ResolveError> { + let (scope, name) = specifier.name.as_str(); + let string = match self.read_file([scope, name], project) { + Ok(Some(s)) => s, + Ok(None) => return Err(Self::ResolveError::NotFound(specifier.name.to_string())), + Err(e) => { + return Err(Self::ResolveError::Read( + specifier.name.to_string(), + Box::new(e), + )) + } + }; + + let entries: Vec = string + .lines() + .map(serde_json::from_str) + .collect::>() + .map_err(|e| Self::ResolveError::Parse(specifier.name.to_string(), e))?; + + log::debug!("{} has {} possible entries", specifier.name, entries.len()); + + Ok(( + PackageNames::Wally(specifier.name.clone()), + entries + .into_iter() + .filter(|manifest| specifier.version.matches(&manifest.package.version)) + .map(|manifest| { + Ok(( + VersionId(manifest.package.version.clone(), TargetKind::Roblox), + WallyPackageRef { + name: specifier.name.clone(), + index_url: self.repo_url.clone(), + dependencies: manifest.all_dependencies().map_err(|e| { + Self::ResolveError::AllDependencies(specifier.to_string(), e) + })?, + version: manifest.package.version, + }, + )) + }) + .collect::>()?, + )) + } + + fn download( + &self, + pkg_ref: &Self::Ref, + project: &Project, + reqwest: &reqwest::blocking::Client, + ) -> Result<(PackageFS, Target), Self::DownloadError> { + let config = self.config(project).map_err(Box::new)?; + let index_file = project + .cas_dir + .join("wally_index") + .join(pkg_ref.name.escaped()) + .join(pkg_ref.version.to_string()); + + let tempdir = match std::fs::read_to_string(&index_file) { + Ok(s) => { + log::debug!( + "using cached index file for package {}@{}", + pkg_ref.name, + pkg_ref.version + ); + + let tempdir = tempdir()?; + let fs = toml::from_str::(&s)?; + + fs.write_to(&tempdir, project.cas_dir(), false)?; + + return Ok((fs, get_target(project, &tempdir)?)); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => tempdir()?, + Err(e) => return Err(errors::DownloadError::ReadIndex(e)), + }; + + let (scope, name) = pkg_ref.name.as_str(); + + let url = format!( + "{}/v1/package-contents/{scope}/{name}/{}", + config.api.as_str().trim_end_matches('/'), + pkg_ref.version + ); + + let mut response = reqwest.get(url).header( + "Wally-Version", + std::env::var("PESDE_WALLY_VERSION") + .as_deref() + .unwrap_or("0.3.2"), + ); + + if let Some(token) = &project.auth_config.github_token { + log::debug!("using token for wally package download"); + response = response.header("Authorization", format!("Bearer {token}")); + } + + let response = response.send()?.error_for_status()?; + let bytes = response.bytes()?; + + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes))?; + archive.extract(tempdir.path())?; + + let mut entries = BTreeMap::new(); + + let mut dir_entries = std::fs::read_dir(tempdir.path())?.collect::>(); + while let Some(entry) = dir_entries.pop_front() { + let entry = entry?; + let path = + RelativePathBuf::from_path(entry.path().strip_prefix(tempdir.path())?).unwrap(); + + if path == ".git" { + continue; + } + + if entry.file_type()?.is_dir() { + entries.insert(path, FSEntry::Directory); + dir_entries.extend(std::fs::read_dir(entry.path())?); + + continue; + } + + let mut file = std::fs::File::open(entry.path())?; + let mut contents = vec![]; + file.read_to_end(&mut contents)?; + + let hash = store_in_cas(&project.cas_dir, &contents)?.0; + entries.insert(path, FSEntry::File(hash)); + } + + let fs = PackageFS(entries); + + if let Some(parent) = index_file.parent() { + std::fs::create_dir_all(parent).map_err(errors::DownloadError::WriteIndex)?; + } + + std::fs::write(&index_file, toml::to_string(&fs)?) + .map_err(errors::DownloadError::WriteIndex)?; + + Ok((fs, get_target(project, &tempdir)?)) + } +} + +/// A Wally index config +#[derive(Debug, Clone, Deserialize)] +pub struct WallyIndexConfig { + api: url::Url, +} + +/// Errors that can occur when interacting with a Wally package source +pub mod errors { + use thiserror::Error; + + use crate::source::git_index::errors::ReadFile; + + /// Errors that can occur when resolving a package from a Wally package source + #[derive(Debug, Error)] + #[non_exhaustive] + pub enum ResolveError { + /// Error interacting with the filesystem + #[error("error interacting with the filesystem")] + Io(#[from] std::io::Error), + + /// Package not found in index + #[error("package {0} not found")] + NotFound(String), + + /// Error reading file for package + #[error("error reading file for {0}")] + Read(String, #[source] Box), + + /// Error parsing file for package + #[error("error parsing file for {0}")] + Parse(String, #[source] serde_json::Error), + + /// Error parsing file for package as utf8 + #[error("error parsing file for {0} to utf8")] + Utf8(String, #[source] std::string::FromUtf8Error), + + /// Error parsing all dependencies + #[error("error parsing all dependencies for {0}")] + AllDependencies( + String, + #[source] crate::manifest::errors::AllDependenciesError, + ), + } + + /// Errors that can occur when reading the config file for a Wally package source + #[derive(Debug, Error)] + #[non_exhaustive] + pub enum ConfigError { + /// Error reading file + #[error("error reading config file")] + ReadFile(#[from] Box), + + /// Error parsing config file + #[error("error parsing config file")] + Parse(#[from] serde_json::Error), + + /// The config file is missing + #[error("missing config file for index at {0}")] + Missing(Box), + } + + /// Errors that can occur when downloading a package from a Wally package source + #[derive(Debug, Error)] + #[non_exhaustive] + pub enum DownloadError { + /// Error reading index file + #[error("error reading config file")] + ReadFile(#[from] Box), + + /// Error downloading package + #[error("error downloading package")] + Download(#[from] reqwest::Error), + + /// Error deserializing index file + #[error("error deserializing index file")] + Deserialize(#[from] toml::de::Error), + + /// Error reading index file + #[error("error reading index file")] + ReadIndex(#[source] std::io::Error), + + /// Error decompressing archive + #[error("error decompressing archive")] + Decompress(#[from] zip::result::ZipError), + + /// Error interacting with the filesystem + #[error("error interacting with the filesystem")] + Io(#[from] std::io::Error), + + /// Error stripping prefix from path + #[error("error stripping prefix from path")] + StripPrefix(#[from] std::path::StripPrefixError), + + /// Error serializing index file + #[error("error serializing index file")] + SerializeIndex(#[from] toml::ser::Error), + + /// Error getting lib path + #[error("error getting lib path")] + LibPath(#[from] crate::source::wally::compat_util::errors::FindLibPathError), + + /// Error writing index file + #[error("error writing index file")] + WriteIndex(#[source] std::io::Error), + } +} diff --git a/src/source/wally/pkg_ref.rs b/src/source/wally/pkg_ref.rs new file mode 100644 index 0000000..6ed8f17 --- /dev/null +++ b/src/source/wally/pkg_ref.rs @@ -0,0 +1,58 @@ +use std::collections::BTreeMap; + +use semver::Version; +use serde::{Deserialize, Serialize}; + +use crate::{ + manifest::{target::TargetKind, DependencyType}, + names::wally::WallyPackageName, + source::{wally::WallyPackageSource, DependencySpecifiers, PackageRef, PackageSources}, +}; + +/// A Wally package reference +#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)] +pub struct WallyPackageRef { + /// The name of the package + #[serde(rename = "wally")] + pub name: WallyPackageName, + /// The version of the package + pub version: Version, + /// The index of the package + #[serde( + serialize_with = "crate::util::serialize_gix_url", + deserialize_with = "crate::util::deserialize_gix_url" + )] + pub index_url: gix::Url, + /// The dependencies of the package + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub dependencies: BTreeMap, +} +impl PackageRef for WallyPackageRef { + fn dependencies(&self) -> &BTreeMap { + &self.dependencies + } + + fn use_new_structure(&self) -> bool { + false + } + + fn target_kind(&self) -> TargetKind { + TargetKind::Roblox + } + + fn source(&self) -> PackageSources { + PackageSources::Wally(WallyPackageSource::new(self.index_url.clone())) + } +} + +impl Ord for WallyPackageRef { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.version.cmp(&other.version) + } +} + +impl PartialOrd for WallyPackageRef { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} diff --git a/src/source/wally/specifier.rs b/src/source/wally/specifier.rs new file mode 100644 index 0000000..4f0d1d7 --- /dev/null +++ b/src/source/wally/specifier.rs @@ -0,0 +1,26 @@ +use std::fmt::Display; + +use semver::VersionReq; +use serde::{Deserialize, Serialize}; + +use crate::{names::wally::WallyPackageName, source::DependencySpecifier}; + +/// The specifier for a Wally dependency +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)] +pub struct WallyDependencySpecifier { + /// The name of the package + #[serde(rename = "wally")] + pub name: WallyPackageName, + /// The version requirement for the package + pub version: VersionReq, + /// The index to use for the package + #[serde(default, skip_serializing_if = "Option::is_none")] + pub index: Option, +} +impl DependencySpecifier for WallyDependencySpecifier {} + +impl Display for WallyDependencySpecifier { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}@{}", self.name, self.version) + } +}