diff --git a/Cargo.lock b/Cargo.lock index 8e9a791..4013619 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "bytes", "futures-core", "futures-sink", @@ -58,7 +58,7 @@ dependencies = [ "actix-utils", "ahash", "base64 0.21.7", - "bitflags 2.4.2", + "bitflags 2.5.0", "brotli", "bytes", "bytestring", @@ -92,7 +92,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -130,7 +130,7 @@ dependencies = [ "parse-size", "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -243,7 +243,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -283,11 +283,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ "cfg-if", - "cipher", + "cipher 0.3.0", "cpufeatures", "opaque-debug", ] +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher 0.4.4", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.8.11" @@ -303,9 +314,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -402,9 +413,9 @@ checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" [[package]] name = "arc-swap" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b3d0060af21e8d11a926981cc00c6c1541aa91dd64b9f881985c3da1094425f" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "arrayref" @@ -470,8 +481,8 @@ dependencies = [ "async-lock 3.3.0", "async-task", "concurrent-queue", - "fastrand 2.0.1", - "futures-lite 2.2.0", + "fastrand 2.0.2", + "futures-lite 2.3.0", "slab", ] @@ -517,10 +528,10 @@ dependencies = [ "cfg-if", "concurrent-queue", "futures-io", - "futures-lite 2.2.0", + "futures-lite 2.3.0", "parking", - "polling 3.5.0", - "rustix 0.38.31", + "polling 3.6.0", + "rustix 0.38.32", "slab", "tracing", "windows-sys 0.52.0", @@ -559,19 +570,19 @@ dependencies = [ "cfg-if", "event-listener 3.1.0", "futures-lite 1.13.0", - "rustix 0.38.31", + "rustix 0.38.32", "windows-sys 0.48.0", ] [[package]] name = "async-recursion" -version = "1.0.5" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +checksum = "30c5ef0ede93efbf733c1a727f3b6b5a1060bbedd5600183e66f6e4be4af0ec5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -586,7 +597,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix 0.38.31", + "rustix 0.38.32", "signal-hook-registry", "slab", "windows-sys 0.48.0", @@ -600,13 +611,13 @@ checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" -version = "0.1.78" +version = "0.1.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "461abc97219de0eaaf81fe3ef974a540158f3d079c2ab200f891f1a2ef201e85" +checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -634,9 +645,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -665,6 +676,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "beef" version = "0.5.2" @@ -679,9 +696,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "bitpacking" @@ -732,7 +749,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e" dependencies = [ "block-padding", - "cipher", + "cipher 0.3.0", ] [[package]] @@ -750,18 +767,18 @@ dependencies = [ "async-channel", "async-lock 3.3.0", "async-task", - "fastrand 2.0.1", + "fastrand 2.0.2", "futures-io", - "futures-lite 2.2.0", + "futures-lite 2.3.0", "piper", "tracing", ] [[package]] name = "brotli" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -814,9 +831,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "bytestring" @@ -827,6 +844,27 @@ dependencies = [ "bytes", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.90" @@ -894,6 +932,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.3" @@ -925,7 +973,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1146,7 +1194,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1157,7 +1205,7 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1363,7 +1411,7 @@ checksum = "5c785274071b1b420972453b306eeca06acf4633829db4223b58a2a8c5953bc4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1509,9 +1557,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" [[package]] name = "fd-lock" @@ -1520,7 +1568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e5768da2206272c81ef0b5e951a41862938a6070da63bcea197899942d3b947" dependencies = [ "cfg-if", - "rustix 0.38.31", + "rustix 0.38.32", "windows-sys 0.52.0", ] @@ -1594,7 +1642,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" dependencies = [ - "rustix 0.38.31", + "rustix 0.38.32", "windows-sys 0.48.0", ] @@ -1692,11 +1740,11 @@ dependencies = [ [[package]] name = "futures-lite" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba" +checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" dependencies = [ - "fastrand 2.0.1", + "fastrand 2.0.2", "futures-core", "futures-io", "parking", @@ -1711,7 +1759,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1821,11 +1869,11 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "git2" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b3ba52851e73b46a4c3df1d89343741112003f0f6f13beb0dfac9e457c3fdcd" +checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "libc", "libgit2-sys", "log", @@ -1891,7 +1939,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.2.5", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -1910,7 +1958,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.1.0", - "indexmap 2.2.5", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -2128,6 +2176,23 @@ dependencies = [ "tokio-rustls 0.24.1", ] +[[package]] +name = "hyper-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.2.0", + "hyper-util", + "rustls 0.22.2", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.25.0", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -2141,6 +2206,22 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.2.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-tungstenite" version = "0.13.0" @@ -2263,9 +2344,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.5" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2295,12 +2376,21 @@ dependencies = [ ] [[package]] -name = "inquire" -version = "0.7.1" +name = "inout" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d98c2ec0b4c8e12455b249aedacfda205df26ee9c6498945b00353a4e682ea6" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ - "bitflags 2.4.2", + "generic-array", +] + +[[package]] +name = "inquire" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fca231a35040487041f975afae9272ecd3701cc95a5efbbda36c6ecc22a269c" +dependencies = [ + "bitflags 2.5.0", "crossterm", "dyn-clone", "fuzzy-matcher", @@ -2461,7 +2551,7 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "libc", "redox_syscall 0.4.1", ] @@ -2482,9 +2572,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.15" +version = "1.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" +checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" dependencies = [ "cc", "libc", @@ -2494,12 +2584,9 @@ dependencies = [ [[package]] name = "line-wrap" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9" -dependencies = [ - "safemem", -] +checksum = "dd1bc4d24ad230d21fb898d1116b1801d7adfc449d42026475862ab48b11e70e" [[package]] name = "linux-keyutils" @@ -2507,7 +2594,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "761e49ec5fd8a5a463f9b84e877c373d888935b71c6be78f3767fe2ae6bed18e" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "libc", ] @@ -2654,7 +2741,7 @@ dependencies = [ "rbx_reflection_database", "rbx_xml", "regex", - "reqwest", + "reqwest 0.11.27", "rustyline", "serde", "serde_json", @@ -2837,7 +2924,7 @@ dependencies = [ "concurrent-queue", "derive_more", "event-listener 4.0.3", - "futures-lite 2.2.0", + "futures-lite 2.3.0", "mlua", "rustc-hash", "tracing", @@ -2915,7 +3002,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cfg-if", "cfg_aliases", "libc", @@ -3086,7 +3173,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -3103,7 +3190,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -3151,9 +3238,9 @@ dependencies = [ [[package]] name = "os_info" -version = "3.8.0" +version = "3.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a07930afc1bd77ac9e1101dc18d3fc4986c6568e939c31d1c26657eb0ccbf5" +checksum = "ae99c7fa6dd38c7cafe1ec085e804f8f555a2f8659b0dbe03f1f9963a9b51092" dependencies = [ "log", "serde", @@ -3219,6 +3306,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "944553dd59c802559559161f9816429058b869003836120e262e8caec061b7ae" +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + [[package]] name = "paste" version = "1.0.14" @@ -3237,6 +3335,18 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -3249,6 +3359,7 @@ version = "0.2.0" dependencies = [ "anyhow", "auth-git2", + "cfg-if", "chrono", "clap", "directories", @@ -3267,7 +3378,7 @@ dependencies = [ "pathdiff", "pretty_env_logger", "relative-path", - "reqwest", + "reqwest 0.12.1", "semver 1.0.22", "serde", "serde_json", @@ -3277,6 +3388,8 @@ dependencies = [ "thiserror", "threadpool", "toml", + "url", + "zip", ] [[package]] @@ -3295,7 +3408,7 @@ dependencies = [ "log", "pesde", "pretty_env_logger", - "reqwest", + "reqwest 0.12.1", "rusty-s3", "semver 1.0.22", "sentry", @@ -3326,7 +3439,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -3348,7 +3461,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" dependencies = [ "atomic-waker", - "fastrand 2.0.1", + "fastrand 2.0.2", "futures-io", ] @@ -3360,12 +3473,12 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plist" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5699cc8a63d1aa2b1ee8e12b9ad70ac790d65788cd36101fa37f87ea46c4cef" +checksum = "d9d34169e64b3c7a80c8621a48adaf44e0cf62c78a9b25dd9dd35f1881a17cf9" dependencies = [ "base64 0.21.7", - "indexmap 2.2.5", + "indexmap 2.2.6", "line-wrap", "quick-xml 0.31.0", "serde", @@ -3390,14 +3503,15 @@ dependencies = [ [[package]] name = "polling" -version = "3.5.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24f040dee2588b4963afb4e420540439d126f73fdacf4a9c486a96d840bac3c9" +checksum = "e0c976a60b2d7e99d6f229e414670a9b85d13ac305cc6d1e9c134de58c5aaaf6" dependencies = [ "cfg-if", "concurrent-queue", + "hermit-abi", "pin-project-lite", - "rustix 0.38.31", + "rustix 0.38.32", "tracing", "windows-sys 0.52.0", ] @@ -3471,7 +3585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8021cf59c8ec9c432cfc2526ac6b8aa508ecaf29cd415f271b8406c1b851c3fd" dependencies = [ "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -3572,14 +3686,14 @@ version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", ] [[package]] name = "rayon" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -3726,9 +3840,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", @@ -3779,9 +3893,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.26" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "base64 0.21.7", "bytes", @@ -3792,13 +3906,12 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "hyper 0.14.28", - "hyper-rustls", - "hyper-tls", + "hyper-rustls 0.24.2", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", "mime", - "mime_guess", "native-tls", "once_cell", "percent-encoding", @@ -3822,6 +3935,55 @@ dependencies = [ "winreg 0.50.0", ] +[[package]] +name = "reqwest" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e333b1eb9fe677f6893a9efcb0d277a2d3edd83f358a236b657c32301dc6e5f6" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.2.0", + "hyper-rustls 0.26.0", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "mime_guess", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls 0.22.2", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-rustls 0.25.0", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 0.26.1", + "winreg 0.50.0", +] + [[package]] name = "ring" version = "0.17.8" @@ -3927,11 +4089,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.31" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys 0.4.13", @@ -3975,9 +4137,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ede67b28608b4c60685c7d54122d4400d90f62b40caee7700e700380a390fa8" +checksum = "868e20fada228fefaf6b652e00cc73623d54f8171e7352c18bb281571f2d92da" [[package]] name = "rustls-webpki" @@ -4031,7 +4193,7 @@ version = "14.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cfg-if", "clipboard-win", "fd-lock", @@ -4053,12 +4215,6 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" -[[package]] -name = "safemem" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" - [[package]] name = "same-file" version = "1.0.6" @@ -4105,7 +4261,7 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5da1a5ad4d28c03536f82f77d9f36603f5e37d8869ac98f0a750d5b5686d8d95" dependencies = [ - "aes", + "aes 0.7.5", "block-modes", "futures-util", "generic-array", @@ -4173,7 +4329,7 @@ checksum = "766448f12e44d68e675d5789a261515c46ac6ccd240abdd451a9c46c84a49523" dependencies = [ "httpdate", "native-tls", - "reqwest", + "reqwest 0.11.27", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -4321,7 +4477,7 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -4330,7 +4486,7 @@ version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ - "indexmap 2.2.5", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -4353,7 +4509,7 @@ checksum = "0b2e6b945e9d3df726b65d6ee24060aff8e3533d431f677a9695db04eff9dfdb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -4379,11 +4535,11 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.32" +version = "0.9.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd075d994154d4a774f95b51fb96bdc2832b0ea48425c92546073816cda1f2f" +checksum = "a0623d197252096520c6f2a5e1171ee436e5af99a5d7caa2891e55e61950e6d9" dependencies = [ - "indexmap 2.2.5", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -4492,9 +4648,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "smol_str" @@ -4654,9 +4810,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.53" +version = "2.0.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" +checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0" dependencies = [ "proc-macro2", "quote", @@ -4848,8 +5004,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", - "fastrand 2.0.1", - "rustix 0.38.31", + "fastrand 2.0.2", + "rustix 0.38.32", "windows-sys 0.52.0", ] @@ -4889,7 +5045,7 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -5023,7 +5179,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -5089,15 +5245,15 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af06656561d28735e9c1cd63dfd57132c8155426aa6af24f36a00a351f88c48e" +checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" dependencies = [ - "indexmap 2.2.5", + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.7", + "toml_edit 0.22.9", ] [[package]] @@ -5115,18 +5271,18 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.5", + "indexmap 2.2.6", "toml_datetime", "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.22.7" +version = "0.22.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18769cd1cec395d70860ceb4d932812a0b4d06b1a4bb336745a4d21b9496e992" +checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" dependencies = [ - "indexmap 2.2.5", + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", @@ -5181,7 +5337,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -5330,9 +5486,9 @@ checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unsafe-libyaml" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" @@ -5391,9 +5547,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ "getrandom 0.2.12", "serde", @@ -5475,7 +5631,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", "wasm-bindgen-shared", ] @@ -5509,7 +5665,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5771,7 +5927,7 @@ checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" dependencies = [ "libc", "linux-raw-sys 0.4.13", - "rustix 0.38.31", + "rustix 0.38.32", ] [[package]] @@ -5873,7 +6029,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -5882,6 +6038,35 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes 0.8.4", + "byteorder 1.5.0", + "bzip2", + "constant_time_eq 0.1.5", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2", + "sha1 0.10.6", + "time 0.3.34", + "zstd 0.11.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe 5.0.2+zstd.1.5.2", +] + [[package]] name = "zstd" version = "0.12.4" @@ -5900,6 +6085,16 @@ dependencies = [ "zstd-safe 7.0.0", ] +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + [[package]] name = "zstd-safe" version = "6.0.6" diff --git a/Cargo.toml b/Cargo.toml index 1cc49b1..2d4a5e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ include = ["src/**/*", "Cargo.toml", "Cargo.lock", "README.md", "LICENSE", "CHAN [features] bin = ["clap", "directories", "keyring", "anyhow", "ignore", "pretty_env_logger", "serde_json", "reqwest/json", "reqwest/multipart", "lune", "futures-executor", "indicatif", "auth-git2", "indicatif-log-bridge", "inquire", "once_cell"] +wally = ["toml", "zip", "serde_json"] [[bin]] name = "pesde" @@ -18,11 +19,10 @@ required-features = ["bin"] [dependencies] serde = { version = "1.0.197", features = ["derive"] } -serde_yaml = "0.9.32" -toml = "0.8.11" -git2 = "0.18.2" +serde_yaml = "0.9.33" +git2 = "0.18.3" semver = { version = "1.0.22", features = ["serde"] } -reqwest = { version = "0.11.26", default-features = false, features = ["rustls-tls", "blocking"] } +reqwest = { version = "0.12.1", default-features = false, features = ["rustls-tls", "blocking"] } tar = "0.4.40" flate2 = "1.0.28" pathdiff = "0.2.1" @@ -31,6 +31,11 @@ log = "0.4.21" thiserror = "1.0.58" threadpool = "1.8.1" full_moon = { version = "0.19.0", features = ["stacker", "roblox"] } +url = { version = "2.5.0", features = ["serde"] } +cfg-if = "1.0.0" + +toml = { version = "0.8.12", optional = true } +zip = { version = "0.6.6", optional = true } # chrono-lc breaks because of https://github.com/chronotope/chrono/compare/v0.4.34...v0.4.35#diff-67de5678fb5c14378bbff7ecf7f8bfab17cc223c4726f8da3afca183a4e59543 chrono = { version = "=0.4.34", features = ["serde"] } @@ -47,7 +52,7 @@ futures-executor = { version = "0.3.30", optional = true } indicatif = { version = "0.17.8", optional = true } auth-git2 = { version = "0.5.4", optional = true } indicatif-log-bridge = { version = "0.2.2", optional = true } -inquire = { version = "0.7.1", optional = true } +inquire = { version = "0.7.3", optional = true } once_cell = { version = "1.19.0", optional = true } [dev-dependencies] diff --git a/README.md b/README.md index 523760b..ba2cbde 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Currently, pesde is in a very early stage of development, but already supports t - Re-exporting types - `bin` exports (ran with Lune) - Patching packages +- Downloading packages from Wally registries ## Installation diff --git a/registry/Cargo.toml b/registry/Cargo.toml index 9383e82..4cd1416 100644 --- a/registry/Cargo.toml +++ b/registry/Cargo.toml @@ -11,17 +11,17 @@ actix-multipart = "0.6.1" actix-multipart-derive = "0.6.1" actix-governor = "0.5.0" dotenvy = "0.15.7" -reqwest = { version = "0.11.24", features = ["json", "blocking"] } +reqwest = { version = "0.12.1", features = ["json", "blocking"] } rusty-s3 = "0.5.0" serde = { version = "1.0.197", features = ["derive"] } serde_json = "1.0.114" -serde_yaml = "0.9.32" +serde_yaml = "0.9.33" flate2 = "1.0.28" tar = "0.4.40" pesde = { path = ".." } semver = "1.0.22" -git2 = "0.18.2" -thiserror = "1.0.57" +git2 = "0.18.3" +thiserror = "1.0.58" tantivy = "0.21.1" log = "0.4.21" pretty_env_logger = "0.5.0" diff --git a/registry/src/endpoints/packages.rs b/registry/src/endpoints/packages.rs index e4d301f..ee72e8d 100644 --- a/registry/src/endpoints/packages.rs +++ b/registry/src/endpoints/packages.rs @@ -8,8 +8,9 @@ use tantivy::{doc, DateTime, Term}; use tar::Archive; use pesde::{ - dependencies::DependencySpecifier, index::Index, manifest::Manifest, package_name::PackageName, - IGNORED_FOLDERS, MANIFEST_FILE_NAME, + dependencies::DependencySpecifier, index::Index, manifest::Manifest, + package_name::StandardPackageName, project::DEFAULT_INDEX_NAME, IGNORED_FOLDERS, + MANIFEST_FILE_NAME, }; use crate::{commit_signature, errors, AppState, UserId, S3_EXPIRY}; @@ -83,7 +84,7 @@ pub async fn create_package( let mut index = app_state.index.lock().unwrap(); let config = index.config()?; - for (dependency, _) in manifest.dependencies().iter() { + for (dependency, _) in manifest.dependencies() { match dependency { DependencySpecifier::Git(_) => { if !config.git_allowed { @@ -93,12 +94,24 @@ pub async fn create_package( } } DependencySpecifier::Registry(registry) => { - if index.package(®istry.name).unwrap().is_none() { + if index + .package(®istry.name.clone().into()) + .unwrap() + .is_none() + { return Ok(HttpResponse::BadRequest().json(errors::ErrorResponse { error: format!("Dependency {} not found", registry.name), })); } + + if registry.index != DEFAULT_INDEX_NAME && !config.custom_registry_allowed { + return Ok(HttpResponse::BadRequest().json(errors::ErrorResponse { + error: "Custom registries are not allowed on this registry".to_string(), + })); + } } + #[allow(unreachable_patterns)] + _ => {} }; } @@ -166,12 +179,12 @@ pub async fn get_package_version( ) -> Result { let (scope, name, mut version) = path.into_inner(); - let package_name = PackageName::new(&scope, &name)?; + let package_name = StandardPackageName::new(&scope, &name)?; { let index = app_state.index.lock().unwrap(); - match index.package(&package_name)? { + match index.package(&package_name.clone().into())? { Some(package) => { if version == "latest" { version = package.last().map(|v| v.version.to_string()).unwrap(); @@ -223,12 +236,12 @@ pub async fn get_package_versions( ) -> Result { let (scope, name) = path.into_inner(); - let package_name = PackageName::new(&scope, &name)?; + let package_name = StandardPackageName::new(&scope, &name)?; { let index = app_state.index.lock().unwrap(); - match index.package(&package_name)? { + match index.package(&package_name.into())? { Some(package) => { let versions = package .iter() diff --git a/registry/src/endpoints/search.rs b/registry/src/endpoints/search.rs index 7606bc5..7fcff38 100644 --- a/registry/src/endpoints/search.rs +++ b/registry/src/endpoints/search.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use serde_json::{json, Value}; use tantivy::{query::AllQuery, DateTime, DocAddress, Order}; -use pesde::{index::Index, package_name::PackageName}; +use pesde::{index::Index, package_name::StandardPackageName}; use crate::{errors, AppState}; @@ -50,7 +50,7 @@ pub async fn search_packages( .into_iter() .map(|(published_at, doc_address)| { let retrieved_doc = searcher.doc(doc_address).unwrap(); - let name: PackageName = retrieved_doc + let name: StandardPackageName = retrieved_doc .get_first(name) .and_then(|v| v.as_text()) .and_then(|v| v.parse().ok()) @@ -63,7 +63,7 @@ pub async fn search_packages( .unwrap(); let entry = index - .package(&name) + .package(&name.clone().into()) .unwrap() .and_then(|v| v.into_iter().find(|v| v.version == version)) .unwrap(); diff --git a/registry/src/errors.rs b/registry/src/errors.rs index 7212f90..712681f 100644 --- a/registry/src/errors.rs +++ b/registry/src/errors.rs @@ -1,5 +1,6 @@ use actix_web::{HttpResponse, ResponseError}; use log::error; +use pesde::index::CreatePackageVersionError; use serde::Serialize; use thiserror::Error; @@ -20,13 +21,13 @@ pub enum Errors { Reqwest(#[from] reqwest::Error), #[error("package name invalid")] - PackageName(#[from] pesde::package_name::PackageNameValidationError), + PackageName(#[from] pesde::package_name::StandardPackageNameValidationError), #[error("config error")] Config(#[from] pesde::index::ConfigError), #[error("create package version error")] - CreatePackageVersion(#[from] pesde::index::CreatePackageVersionError), + CreatePackageVersion(#[from] CreatePackageVersionError), #[error("commit and push error")] CommitAndPush(#[from] pesde::index::CommitAndPushError), @@ -43,11 +44,16 @@ impl ResponseError for Errors { match self { Errors::UserYaml(_) | Errors::PackageName(_) | Errors::QueryParser(_) => {} Errors::CreatePackageVersion(err) => match err { - pesde::index::CreatePackageVersionError::MissingScopeOwnership => { + CreatePackageVersionError::MissingScopeOwnership => { return HttpResponse::Unauthorized().json(ErrorResponse { error: "You do not have permission to publish this scope".to_string(), }); } + CreatePackageVersionError::FromManifestIndexFileEntry(err) => { + return HttpResponse::BadRequest().json(ErrorResponse { + error: format!("Error in manifest: {err:?}"), + }); + } _ => error!("{err:?}"), }, err => { diff --git a/registry/src/main.rs b/registry/src/main.rs index 400c31c..e2b3da2 100644 --- a/registry/src/main.rs +++ b/registry/src/main.rs @@ -18,8 +18,8 @@ use rusty_s3::{Bucket, Credentials, UrlStyle}; use tantivy::{doc, DateTime, IndexReader, IndexWriter}; use pesde::{ - index::{GitIndex, IndexFile}, - package_name::PackageName, + index::{GitIndex, Index, IndexFile}, + package_name::StandardPackageName, }; mod endpoints; @@ -157,7 +157,7 @@ fn search_index(index: &GitIndex) -> (IndexReader, IndexWriter) { let package = path.file_name().and_then(|v| v.to_str()).unwrap(); - let package_name = PackageName::new(scope, package).unwrap(); + let package_name = StandardPackageName::new(scope, package).unwrap(); let entries: IndexFile = serde_yaml::from_slice(&std::fs::read(&path).unwrap()).unwrap(); let entry = entries.last().unwrap().clone(); @@ -216,7 +216,7 @@ fn main() -> std::io::Result<()> { let index = GitIndex::new( current_dir.join("cache"), - &get_env!("INDEX_REPO_URL"), + &get_env!("INDEX_REPO_URL", "p"), Some(Box::new(|| { Box::new(|_, _, _| { let username = get_env!("GITHUB_USERNAME"); @@ -225,6 +225,7 @@ fn main() -> std::io::Result<()> { Cred::userpass_plaintext(&username, &pat) }) })), + None, ); index.refresh().expect("failed to refresh index"); diff --git a/src/cli/api_token.rs b/src/cli/api_token.rs index f5ec99d..952e241 100644 --- a/src/cli/api_token.rs +++ b/src/cli/api_token.rs @@ -1,25 +1,15 @@ use std::path::PathBuf; +use crate::cli::DEFAULT_INDEX_DATA; use keyring::Entry; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; -use crate::cli::INDEX_DIR; - -pub trait ApiTokenSource: Send + Sync { - fn get_api_token(&self) -> anyhow::Result>; - fn set_api_token(&self, api_token: &str) -> anyhow::Result<()>; - fn delete_api_token(&self) -> anyhow::Result<()>; - fn persists(&self) -> bool { - true - } -} - -pub struct EnvVarApiTokenSource; +struct EnvVarApiTokenSource; const API_TOKEN_ENV_VAR: &str = "PESDE_API_TOKEN"; -impl ApiTokenSource for EnvVarApiTokenSource { +impl EnvVarApiTokenSource { fn get_api_token(&self) -> anyhow::Result> { match std::env::var(API_TOKEN_ENV_VAR) { Ok(token) => Ok(Some(token)), @@ -27,51 +17,10 @@ impl ApiTokenSource for EnvVarApiTokenSource { Err(e) => Err(e.into()), } } - - // don't need to implement set_api_token or delete_api_token - fn set_api_token(&self, _api_token: &str) -> anyhow::Result<()> { - Ok(()) - } - - fn delete_api_token(&self) -> anyhow::Result<()> { - Ok(()) - } - - fn persists(&self) -> bool { - false - } } -static KEYRING_ENTRY: Lazy = - Lazy::new(|| Entry::new(env!("CARGO_BIN_NAME"), "api_token").unwrap()); - -pub struct KeyringApiTokenSource; - -impl ApiTokenSource for KeyringApiTokenSource { - fn get_api_token(&self) -> anyhow::Result> { - match KEYRING_ENTRY.get_password() { - Ok(api_token) => Ok(Some(api_token)), - Err(err) => match err { - keyring::Error::NoEntry | keyring::Error::PlatformFailure(_) => Ok(None), - _ => Err(err.into()), - }, - } - } - - fn set_api_token(&self, api_token: &str) -> anyhow::Result<()> { - KEYRING_ENTRY.set_password(api_token)?; - - Ok(()) - } - - fn delete_api_token(&self) -> anyhow::Result<()> { - KEYRING_ENTRY.delete_password()?; - - Ok(()) - } -} - -static AUTH_FILE_PATH: Lazy = Lazy::new(|| INDEX_DIR.join("auth.yaml")); +static AUTH_FILE_PATH: Lazy = + Lazy::new(|| DEFAULT_INDEX_DATA.0.parent().unwrap().join("auth.yaml")); static AUTH_FILE: Lazy = Lazy::new( || match std::fs::read_to_string(AUTH_FILE_PATH.to_path_buf()) { @@ -87,9 +36,9 @@ struct AuthFile { api_token: Option, } -pub struct ConfigFileApiTokenSource; +struct ConfigFileApiTokenSource; -impl ApiTokenSource for ConfigFileApiTokenSource { +impl ConfigFileApiTokenSource { fn get_api_token(&self) -> anyhow::Result> { Ok(AUTH_FILE.api_token.clone()) } @@ -120,11 +69,77 @@ impl ApiTokenSource for ConfigFileApiTokenSource { } } -pub static API_TOKEN_SOURCE: Lazy> = Lazy::new(|| { - let sources: Vec> = vec![ - Box::new(EnvVarApiTokenSource), - Box::new(KeyringApiTokenSource), - Box::new(ConfigFileApiTokenSource), +static KEYRING_ENTRY: Lazy = + Lazy::new(|| Entry::new(env!("CARGO_PKG_NAME"), "api_token").unwrap()); + +struct KeyringApiTokenSource; + +impl KeyringApiTokenSource { + fn get_api_token(&self) -> anyhow::Result> { + match KEYRING_ENTRY.get_password() { + Ok(api_token) => Ok(Some(api_token)), + Err(err) => match err { + keyring::Error::NoEntry | keyring::Error::PlatformFailure(_) => Ok(None), + _ => Err(err.into()), + }, + } + } + + fn set_api_token(&self, api_token: &str) -> anyhow::Result<()> { + KEYRING_ENTRY.set_password(api_token)?; + + Ok(()) + } + + fn delete_api_token(&self) -> anyhow::Result<()> { + KEYRING_ENTRY.delete_password()?; + + Ok(()) + } +} + +#[derive(Debug)] +pub enum ApiTokenSource { + EnvVar, + ConfigFile, + Keyring, +} + +impl ApiTokenSource { + pub fn get_api_token(&self) -> anyhow::Result> { + match self { + ApiTokenSource::EnvVar => EnvVarApiTokenSource.get_api_token(), + ApiTokenSource::ConfigFile => ConfigFileApiTokenSource.get_api_token(), + ApiTokenSource::Keyring => KeyringApiTokenSource.get_api_token(), + } + } + + pub fn set_api_token(&self, api_token: &str) -> anyhow::Result<()> { + match self { + ApiTokenSource::EnvVar => Ok(()), + ApiTokenSource::ConfigFile => ConfigFileApiTokenSource.set_api_token(api_token), + ApiTokenSource::Keyring => KeyringApiTokenSource.set_api_token(api_token), + } + } + + pub fn delete_api_token(&self) -> anyhow::Result<()> { + match self { + ApiTokenSource::EnvVar => Ok(()), + ApiTokenSource::ConfigFile => ConfigFileApiTokenSource.delete_api_token(), + ApiTokenSource::Keyring => KeyringApiTokenSource.delete_api_token(), + } + } + + fn persists(&self) -> bool { + !matches!(self, ApiTokenSource::EnvVar) + } +} + +pub static API_TOKEN_SOURCE: Lazy = Lazy::new(|| { + let sources: [ApiTokenSource; 3] = [ + ApiTokenSource::EnvVar, + ApiTokenSource::ConfigFile, + ApiTokenSource::Keyring, ]; let mut valid_sources = vec![]; diff --git a/src/cli/auth.rs b/src/cli/auth.rs index 470dcaa..87725a7 100644 --- a/src/cli/auth.rs +++ b/src/cli/auth.rs @@ -2,7 +2,7 @@ use clap::Subcommand; use pesde::index::Index; use reqwest::{header::AUTHORIZATION, Url}; -use crate::cli::{api_token::API_TOKEN_SOURCE, send_request, INDEX, REQWEST_CLIENT}; +use crate::cli::{api_token::API_TOKEN_SOURCE, send_request, DEFAULT_INDEX, REQWEST_CLIENT}; #[derive(Subcommand, Clone)] pub enum AuthCommand { @@ -15,7 +15,7 @@ pub enum AuthCommand { pub fn auth_command(cmd: AuthCommand) -> anyhow::Result<()> { match cmd { AuthCommand::Login => { - let github_oauth_client_id = INDEX.config()?.github_oauth_client_id; + let github_oauth_client_id = DEFAULT_INDEX.config()?.github_oauth_client_id; let response = send_request(REQWEST_CLIENT.post(Url::parse_with_params( "https://github.com/login/device/code", diff --git a/src/cli/config.rs b/src/cli/config.rs index 1eb98c0..cddecb1 100644 --- a/src/cli/config.rs +++ b/src/cli/config.rs @@ -6,15 +6,6 @@ use crate::{cli::CLI_CONFIG, CliConfig}; #[derive(Subcommand, Clone)] pub enum ConfigCommand { - /// Sets the index repository URL - SetIndexRepo { - /// The URL of the index repository - #[clap(value_name = "URL")] - url: String, - }, - /// Gets the index repository URL - GetIndexRepo, - /// Sets the cache directory SetCacheDir { /// The directory to use as the cache directory @@ -27,26 +18,9 @@ pub enum ConfigCommand { pub fn config_command(cmd: ConfigCommand) -> anyhow::Result<()> { match cmd { - ConfigCommand::SetIndexRepo { url } => { - let cli_config = CliConfig { - index_repo_url: url.clone(), - ..CLI_CONFIG.clone() - }; - - cli_config.write()?; - - println!("index repository url set to: `{url}`"); - } - ConfigCommand::GetIndexRepo => { - println!( - "current index repository url: `{}`", - CLI_CONFIG.index_repo_url - ); - } ConfigCommand::SetCacheDir { directory } => { let cli_config = CliConfig { cache_dir: directory, - ..CLI_CONFIG.clone() }; cli_config.write()?; diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 8261112..a1177c0 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -6,7 +6,12 @@ use indicatif::MultiProgress; use indicatif_log_bridge::LogWrapper; use log::error; use once_cell::sync::Lazy; -use pesde::{index::GitIndex, manifest::Realm, package_name::PackageName}; +use pesde::{ + index::{GitIndex, Index}, + manifest::{Manifest, Realm}, + package_name::{PackageName, StandardPackageName}, + project::DEFAULT_INDEX_NAME, +}; use pretty_env_logger::env_logger::Env; use reqwest::{ blocking::{RequestBuilder, Response}, @@ -84,7 +89,7 @@ pub enum Command { Run { /// The package to run #[clap(value_name = "PACKAGE")] - package: PackageName, + package: StandardPackageName, /// The arguments to pass to the package #[clap(last = true)] @@ -102,6 +107,7 @@ pub enum Command { Publish, /// Converts a `wally.toml` file to a `pesde.yaml` file + #[cfg(feature = "wally")] Convert, /// Begins a new patch @@ -141,21 +147,11 @@ pub struct Cli { pub directory: Option, } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, Default)] pub struct CliConfig { - pub index_repo_url: String, pub cache_dir: Option, } -impl Default for CliConfig { - fn default() -> Self { - Self { - index_repo_url: "https://github.com/daimond113/pesde-index".to_string(), - cache_dir: None, - } - } -} - impl CliConfig { pub fn cache_dir(&self) -> PathBuf { self.cache_dir @@ -201,40 +197,12 @@ pub fn send_request(request_builder: RequestBuilder) -> anyhow::Result pub static CLI: Lazy = Lazy::new(Cli::parse); pub static DIRS: Lazy = Lazy::new(|| { - ProjectDirs::from("com", env!("CARGO_BIN_NAME"), env!("CARGO_BIN_NAME")) + ProjectDirs::from("com", env!("CARGO_PKG_NAME"), env!("CARGO_BIN_NAME")) .expect("couldn't get home directory") }); pub static CLI_CONFIG: Lazy = Lazy::new(|| CliConfig::open().unwrap()); -pub static INDEX_DIR: Lazy = Lazy::new(|| { - let mut hasher = DefaultHasher::new(); - CLI_CONFIG.index_repo_url.hash(&mut hasher); - let hash = hasher.finish().to_string(); - - CLI_CONFIG.cache_dir().join("indices").join(hash) -}); - -pub static INDEX: Lazy = Lazy::new(|| { - let index = GitIndex::new( - INDEX_DIR.join("index"), - &CLI_CONFIG.index_repo_url, - Some(Box::new(|| { - Box::new(|a, b, c| { - let git_authenticator = GitAuthenticator::new(); - let config = git2::Config::open_default().unwrap(); - let mut cred = git_authenticator.credentials(&config); - - cred(a, b, c) - }) - })), - ); - - index.refresh().unwrap(); - - index -}); - pub static CWD: Lazy = Lazy::new(|| { CLI.directory .clone() @@ -275,3 +243,50 @@ pub static MULTI: Lazy = Lazy::new(|| { multi }); + +pub const DEFAULT_INDEX_URL: &str = "https://github.com/daimond113/pesde-index"; +#[cfg(feature = "wally")] +pub const DEFAULT_WALLY_INDEX_URL: &str = "https://github.com/UpliftGames/wally-index"; + +pub fn index_dir(url: &str) -> PathBuf { + let mut hasher = DefaultHasher::new(); + url.hash(&mut hasher); + let hash = hasher.finish().to_string(); + + CLI_CONFIG + .cache_dir() + .join("indices") + .join(hash) + .join("index") +} + +pub fn clone_index(url: &str) -> GitIndex { + let index = GitIndex::new( + index_dir(url), + &url.parse().unwrap(), + Some(Box::new(|| { + Box::new(|a, b, c| { + let git_authenticator = GitAuthenticator::new(); + let config = git2::Config::open_default().unwrap(); + let mut cred = git_authenticator.credentials(&config); + + cred(a, b, c) + }) + })), + API_TOKEN_SOURCE.get_api_token().unwrap(), + ); + + index.refresh().unwrap(); + + index +} + +pub static DEFAULT_INDEX_DATA: Lazy<(PathBuf, String)> = Lazy::new(|| { + let manifest = Manifest::from_path(CWD.to_path_buf()) + .map(|m| m.indices.get(DEFAULT_INDEX_NAME).unwrap().clone()); + let url = &manifest.unwrap_or(DEFAULT_INDEX_URL.to_string()); + + (index_dir(url), url.clone()) +}); + +pub static DEFAULT_INDEX: Lazy = Lazy::new(|| clone_index(&DEFAULT_INDEX_DATA.1)); diff --git a/src/cli/root.rs b/src/cli/root.rs index 41b6344..8ed43fd 100644 --- a/src/cli/root.rs +++ b/src/cli/root.rs @@ -1,4 +1,7 @@ +use cfg_if::cfg_if; +use chrono::Utc; use std::{ + collections::{BTreeMap, HashMap}, fs::{create_dir_all, read, remove_dir_all, write, File}, str::FromStr, time::Duration, @@ -18,19 +21,19 @@ use tar::Builder as TarBuilder; use pesde::{ dependencies::{registry::RegistryDependencySpecifier, DependencySpecifier, PackageRef}, - index::{GitIndex, Index}, + index::Index, manifest::{Manifest, PathStyle, Realm}, multithread::MultithreadedJob, - package_name::PackageName, + package_name::{PackageName, StandardPackageName}, patches::{create_patch, setup_patches_repo}, - project::{InstallOptions, Project}, + project::{InstallOptions, Project, DEFAULT_INDEX_NAME}, DEV_PACKAGES_FOLDER, IGNORED_FOLDERS, MANIFEST_FILE_NAME, PACKAGES_FOLDER, PATCHES_FOLDER, SERVER_PACKAGES_FOLDER, }; use crate::cli::{ - api_token::API_TOKEN_SOURCE, send_request, Command, CLI_CONFIG, CWD, DIRS, INDEX, MULTI, - REQWEST_CLIENT, + clone_index, send_request, Command, CLI_CONFIG, CWD, DEFAULT_INDEX, DEFAULT_INDEX_URL, DIRS, + MULTI, REQWEST_CLIENT, }; pub const MAX_ARCHIVE_SIZE: usize = 4 * 1024 * 1024; @@ -71,12 +74,19 @@ macro_rules! none_if_empty { } pub fn root_command(cmd: Command) -> anyhow::Result<()> { - let project: Lazy> = Lazy::new(|| { - Project::from_path( + let mut project: Lazy = Lazy::new(|| { + let manifest = Manifest::from_path(CWD.to_path_buf()).unwrap(); + let indices = manifest + .indices + .clone() + .into_iter() + .map(|(k, v)| (k, Box::new(clone_index(&v)) as Box)); + + Project::new( CWD.to_path_buf(), CLI_CONFIG.cache_dir(), - INDEX.clone(), - API_TOKEN_SOURCE.get_api_token().ok().flatten(), + HashMap::from_iter(indices), + manifest, ) .unwrap() }); @@ -93,9 +103,10 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { }; } - let resolved_versions_map = project.manifest().dependency_tree(&project, locked)?; + let manifest = project.manifest().clone(); + let resolved_versions_map = manifest.dependency_tree(&mut project, locked)?; - let download_job = project.download(&resolved_versions_map)?; + let download_job = project.download(resolved_versions_map.clone())?; multithreaded_bar( download_job, @@ -103,6 +114,8 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { "Downloading packages".to_string(), )?; + let project = Lazy::force_mut(&mut project); + project.install( InstallOptions::new() .locked(locked) @@ -116,7 +129,7 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { .ok_or(anyhow::anyhow!("lockfile not found"))?; let (_, resolved_pkg) = lockfile - .get(&package) + .get(&package.into()) .and_then(|versions| versions.iter().find(|(_, pkg_ref)| pkg_ref.is_root)) .ok_or(anyhow::anyhow!( "package not found in lockfile (or isn't root)" @@ -143,7 +156,7 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { ))?; } Command::Search { query } => { - let config = INDEX.config()?; + let config = DEFAULT_INDEX.config()?; let api_url = config.api(); let response = send_request(REQWEST_CLIENT.get(Url::parse_with_params( @@ -220,11 +233,13 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { .file_name("tarball.tar.gz") .mime_str("application/gzip")?; + let index = project.indices().get(DEFAULT_INDEX_NAME).unwrap(); + let mut request = REQWEST_CLIENT - .post(format!("{}/v0/packages", project.index().config()?.api())) + .post(format!("{}/v0/packages", index.config()?.api())) .multipart(reqwest::blocking::multipart::Form::new().part("tarball", part)); - if let Some(token) = project.registry_auth_token() { + if let Some(token) = index.registry_auth_token() { request = request.header(AUTHORIZATION, format!("Bearer {token}")); } else { request = request.header(AUTHORIZATION, ""); @@ -242,11 +257,11 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { .and_then(|versions| versions.get(&package.1)) .ok_or(anyhow::anyhow!("package not found in lockfile"))?; - let dir = DIRS.data_dir().join("patches").join(format!( - "{}_{}", - package.0.escaped(), - package.1 - )); + let dir = DIRS + .data_dir() + .join("patches") + .join(package.0.escaped()) + .join(Utc::now().timestamp().to_string()); if dir.exists() { anyhow::bail!( @@ -257,8 +272,20 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { create_dir_all(&dir)?; - resolved_pkg.pkg_ref.download(&project, &dir)?; - match resolved_pkg.pkg_ref { + let project = Lazy::force_mut(&mut project); + let url = resolved_pkg.pkg_ref.resolve_url(project)?; + + let index = project.indices().get(DEFAULT_INDEX_NAME).unwrap(); + + resolved_pkg.pkg_ref.download( + &REQWEST_CLIENT, + index.registry_auth_token().map(|t| t.to_string()), + url.as_ref(), + index.credentials_fn().cloned(), + &dir, + )?; + + match &resolved_pkg.pkg_ref { PackageRef::Git(_) => {} _ => { setup_patches_repo(&dir)?; @@ -268,13 +295,17 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { println!("done! modify the files in {} and run `{} patch-commit ` to commit the changes", dir.display(), env!("CARGO_BIN_NAME")); } Command::PatchCommit { dir } => { - let manifest = Manifest::from_path(&dir)?; - let patch_path = project.path().join(PATCHES_FOLDER).join(format!( - "{}@{}.patch", - manifest.name.escaped(), - manifest.version - )); + let name = dir + .parent() + .and_then(|p| p.file_name()) + .and_then(|f| f.to_str()) + .unwrap(); + let manifest = Manifest::from_path(&dir)?; + let patch_path = project.path().join(PATCHES_FOLDER); + create_dir_all(&patch_path)?; + + let patch_path = patch_path.join(format!("{name}@{}.patch", manifest.version)); if patch_path.exists() { anyhow::bail!( "patch already exists. remove the file {} to create a new patch", @@ -304,7 +335,7 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { let mut name = Text::new("What is the name of the package?").with_validator(|name: &str| { - Ok(match PackageName::from_str(name) { + Ok(match StandardPackageName::from_str(name) { Ok(_) => Validation::Valid, Err(e) => Validation::Invalid(e.into()), }) @@ -358,6 +389,10 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { path_style, private, realm: Some(realm), + indices: BTreeMap::from([( + DEFAULT_INDEX_NAME.to_string(), + DEFAULT_INDEX_URL.to_string(), + )]), dependencies: Default::default(), peer_dependencies: Default::default(), description: none_if_empty!(description), @@ -375,11 +410,25 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { } => { let mut manifest = project.manifest().clone(); - let specifier = DependencySpecifier::Registry(RegistryDependencySpecifier { - name: package.0, - version: package.1, - realm, - }); + let specifier = match package.0 { + PackageName::Standard(name) => { + DependencySpecifier::Registry(RegistryDependencySpecifier { + name, + version: package.1, + realm, + index: DEFAULT_INDEX_NAME.to_string(), + }) + } + #[cfg(feature = "wally")] + PackageName::Wally(name) => DependencySpecifier::Wally( + pesde::dependencies::wally::WallyDependencySpecifier { + name, + version: package.1, + realm, + index_url: crate::cli::DEFAULT_WALLY_INDEX_URL.parse().unwrap(), + }, + ), + }; if peer { manifest.peer_dependencies.push(specifier); @@ -398,9 +447,27 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { for dependencies in [&mut manifest.dependencies, &mut manifest.peer_dependencies] { dependencies.retain(|d| { if let DependencySpecifier::Registry(registry) = d { - registry.name != package + match &package { + PackageName::Standard(name) => ®istry.name != name, + #[cfg(feature = "wally")] + PackageName::Wally(_) => true, + } } else { - true + cfg_if! { + if #[cfg(feature = "wally")] { + #[allow(clippy::collapsible_else_if)] + if let DependencySpecifier::Wally(wally) = d { + match &package { + PackageName::Standard(_) => true, + PackageName::Wally(name) => &wally.name != name, + } + } else { + true + } + } else { + true + } + } } }); } @@ -411,8 +478,10 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { )?; } Command::Outdated => { - let manifest = project.manifest(); - let dependency_tree = manifest.dependency_tree(&project, false)?; + let project = Lazy::force_mut(&mut project); + + let manifest = project.manifest().clone(); + let dependency_tree = manifest.dependency_tree(project, false)?; for (name, versions) in dependency_tree { for (version, resolved_pkg) in versions { @@ -420,10 +489,10 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { continue; } - if let PackageRef::Registry(registry) = resolved_pkg.pkg_ref { + if let PackageRef::Registry(ref registry) = resolved_pkg.pkg_ref { let latest_version = send_request(REQWEST_CLIENT.get(format!( "{}/v0/packages/{}/{}/versions", - project.index().config()?.api(), + resolved_pkg.pkg_ref.get_index(project).config()?.api(), registry.name.scope(), registry.name.name() )))? @@ -445,6 +514,7 @@ pub fn root_command(cmd: Command) -> anyhow::Result<()> { } } } + #[cfg(feature = "wally")] Command::Convert => { Manifest::from_path_or_convert(CWD.to_path_buf())?; } diff --git a/src/dependencies/git.rs b/src/dependencies/git.rs index 735702e..8d4bfca 100644 --- a/src/dependencies/git.rs +++ b/src/dependencies/git.rs @@ -1,16 +1,18 @@ -use std::{fs::create_dir_all, path::Path}; +use cfg_if::cfg_if; +use std::{fs::create_dir_all, path::Path, sync::Arc}; use git2::{build::RepoBuilder, Repository}; -use log::{debug, warn}; +use log::{debug, error, warn}; use semver::Version; use serde::{Deserialize, Serialize}; use thiserror::Error; +use url::Url; use crate::{ - index::{remote_callbacks, Index}, + index::{remote_callbacks, CredentialsFn}, manifest::{Manifest, ManifestConvertError, Realm}, - package_name::PackageName, - project::Project, + package_name::StandardPackageName, + project::{get_index, Indices}, }; /// A dependency of a package that can be downloaded from a git repository @@ -31,11 +33,11 @@ pub struct GitDependencySpecifier { #[serde(deny_unknown_fields)] pub struct GitPackageRef { /// The name of the package - pub name: PackageName, + pub name: StandardPackageName, /// The version of the package pub version: Version, /// The URL of the git repository - pub repo_url: String, + pub repo_url: Url, /// The revision of the git repository to use pub rev: String, } @@ -54,13 +56,23 @@ pub enum GitDownloadError { /// An error that occurred while reading the manifest of the git repository #[error("error reading manifest")] ManifestRead(#[from] ManifestConvertError), + + /// An error that occurred because the URL is invalid + #[error("invalid URL")] + InvalidUrl(#[from] url::ParseError), + + /// An error that occurred because the manifest is not present in the git repository, and the wally feature is not enabled + #[cfg(not(feature = "wally"))] + #[error("wally feature is not enabled, but the manifest is not present in the git repository")] + ManifestNotPresent, } impl GitDependencySpecifier { - pub(crate) fn resolve( + pub(crate) fn resolve( &self, - project: &Project, - ) -> Result<(Manifest, String, String), GitDownloadError> { + cache_dir: &Path, + indices: &Indices, + ) -> Result<(Manifest, Url, String), GitDownloadError> { debug!("resolving git dependency {}", self.repo); // should also work with ssh urls @@ -84,10 +96,10 @@ impl GitDependencySpecifier { } let repo_url = if !is_url { - format!("https://github.com/{}.git", &self.repo) + Url::parse(&format!("https://github.com/{}.git", &self.repo)) } else { - self.repo.to_string() - }; + Url::parse(&self.repo) + }?; if is_url { debug!("assuming git repository is a url: {}", &repo_url); @@ -95,8 +107,7 @@ impl GitDependencySpecifier { debug!("resolved git repository url to: {}", &repo_url); } - let dest = project - .cache_dir() + let dest = cache_dir .join("git") .join(repo_name.replace('/', "_")) .join(&self.rev); @@ -105,11 +116,11 @@ impl GitDependencySpecifier { create_dir_all(&dest)?; let mut fetch_options = git2::FetchOptions::new(); - fetch_options.remote_callbacks(remote_callbacks(project.index())); + fetch_options.remote_callbacks(remote_callbacks!(get_index(indices, None))); RepoBuilder::new() .fetch_options(fetch_options) - .clone(&repo_url, &dest)? + .clone(repo_url.as_ref(), &dest)? } else { Repository::open(&dest)? }; @@ -121,7 +132,7 @@ impl GitDependencySpecifier { Ok(( Manifest::from_path_or_convert(dest)?, - repo_url.to_string(), + repo_url, obj.id().to_string(), )) } @@ -129,17 +140,27 @@ impl GitDependencySpecifier { impl GitPackageRef { /// Downloads the package to the specified destination - pub fn download, I: Index>( + pub fn download>( &self, - project: &Project, dest: P, + credentials_fn: Option>, ) -> Result<(), GitDownloadError> { let mut fetch_options = git2::FetchOptions::new(); - fetch_options.remote_callbacks(remote_callbacks(project.index())); + let mut remote_callbacks = git2::RemoteCallbacks::new(); + let credentials_fn = credentials_fn.map(|f| f()); + + if let Some(credentials_fn) = credentials_fn { + debug!("authenticating this git clone with credentials"); + remote_callbacks.credentials(credentials_fn); + } else { + debug!("no credentials provided for this git clone"); + } + + fetch_options.remote_callbacks(remote_callbacks); let repo = RepoBuilder::new() .fetch_options(fetch_options) - .clone(&self.repo_url, dest.as_ref())?; + .clone(self.repo_url.as_ref(), dest.as_ref())?; let obj = repo.revparse_single(&self.rev)?; @@ -153,7 +174,15 @@ impl GitPackageRef { repo.reset(&obj, git2::ResetType::Hard, None)?; - Manifest::from_path_or_convert(dest)?; + cfg_if! { + if #[cfg(feature = "wally")] { + Manifest::from_path_or_convert(dest)?; + } else { + if Manifest::from_path(dest).is_err() { + return Err(GitDownloadError::ManifestNotPresent); + } + } + } Ok(()) } diff --git a/src/dependencies/mod.rs b/src/dependencies/mod.rs index a7bd914..2e9bba9 100644 --- a/src/dependencies/mod.rs +++ b/src/dependencies/mod.rs @@ -1,9 +1,13 @@ +use cfg_if::cfg_if; use log::debug; -use std::{fmt::Display, fs::create_dir_all, path::Path}; +use reqwest::header::AUTHORIZATION; +use std::{fmt::Display, fs::create_dir_all, path::Path, sync::Arc}; use semver::Version; use serde::{de::IntoDeserializer, Deserialize, Deserializer, Serialize}; +use serde_yaml::Value; use thiserror::Error; +use url::Url; use crate::{ dependencies::{ @@ -11,11 +15,11 @@ use crate::{ registry::{RegistryDependencySpecifier, RegistryPackageRef}, resolution::ResolvedVersionsMap, }, - index::Index, + index::{CredentialsFn, Index}, manifest::Realm, multithread::MultithreadedJob, package_name::PackageName, - project::{InstallProjectError, Project}, + project::{get_index, get_index_by_url, InstallProjectError, Project}, }; /// Git dependency related stuff @@ -24,6 +28,9 @@ pub mod git; pub mod registry; /// Resolution pub mod resolution; +/// Wally dependency related stuff +#[cfg(feature = "wally")] +pub mod wally; // To improve developer experience, we resolve the type of the dependency specifier with a custom deserializer, so that the user doesn't have to specify the type of the dependency /// A dependency of a package @@ -34,6 +41,9 @@ pub enum DependencySpecifier { Registry(RegistryDependencySpecifier), /// A dependency that can be downloaded from a git repository Git(GitDependencySpecifier), + /// A dependency that can be downloaded from a wally registry + #[cfg(feature = "wally")] + Wally(wally::WallyDependencySpecifier), } impl DependencySpecifier { @@ -42,6 +52,8 @@ impl DependencySpecifier { match self { DependencySpecifier::Registry(registry) => registry.name.to_string(), DependencySpecifier::Git(git) => git.repo.to_string(), + #[cfg(feature = "wally")] + DependencySpecifier::Wally(wally) => wally.name.to_string(), } } @@ -50,6 +62,8 @@ impl DependencySpecifier { match self { DependencySpecifier::Registry(registry) => registry.version.to_string(), DependencySpecifier::Git(git) => git.rev.clone(), + #[cfg(feature = "wally")] + DependencySpecifier::Wally(wally) => wally.version.to_string(), } } @@ -58,13 +72,15 @@ impl DependencySpecifier { match self { DependencySpecifier::Registry(registry) => registry.realm.as_ref(), DependencySpecifier::Git(git) => git.realm.as_ref(), + #[cfg(feature = "wally")] + DependencySpecifier::Wally(wally) => wally.realm.as_ref(), } } } impl<'de> Deserialize<'de> for DependencySpecifier { fn deserialize>(deserializer: D) -> Result { - let yaml = serde_yaml::Value::deserialize(deserializer)?; + let yaml = Value::deserialize(deserializer)?; let result = if yaml.get("repo").is_some() { GitDependencySpecifier::deserialize(yaml.into_deserializer()) @@ -72,6 +88,15 @@ impl<'de> Deserialize<'de> for DependencySpecifier { } else if yaml.get("name").is_some() { RegistryDependencySpecifier::deserialize(yaml.into_deserializer()) .map(DependencySpecifier::Registry) + } else if yaml.get("wally").is_some() { + cfg_if! { + if #[cfg(feature = "wally")] { + wally::WallyDependencySpecifier::deserialize(yaml.into_deserializer()) + .map(DependencySpecifier::Wally) + } else { + Err(serde::de::Error::custom("wally is not enabled")) + } + } } else { Err(serde::de::Error::custom("invalid dependency")) }; @@ -89,6 +114,9 @@ pub enum PackageRef { Registry(RegistryPackageRef), /// A reference to a package that can be downloaded from a git repository Git(GitPackageRef), + /// A reference to a package that can be downloaded from a wally registry + #[cfg(feature = "wally")] + Wally(wally::WallyPackageRef), } /// An error that occurred while downloading a package @@ -101,14 +129,38 @@ pub enum DownloadError { /// An error that occurred while downloading a package from a git repository #[error("error downloading package {1} from git repository")] Git(#[source] git::GitDownloadError, Box), + + /// An error that occurred while downloading a package from a wally registry + #[cfg(feature = "wally")] + #[error("error downloading package {1} from wally registry")] + Wally(#[source] wally::WallyDownloadError, Box), + + /// A URL is required for this type of package reference + #[error("a URL is required for this type of package reference")] + UrlRequired, +} + +/// An error that occurred while resolving a URL +#[derive(Debug, Error)] +pub enum UrlResolveError { + /// An error that occurred while resolving a URL of a registry package + #[error("error resolving URL of registry package")] + Registry(#[from] registry::RegistryUrlResolveError), + + /// An error that occurred while resolving a URL of a wally package + #[cfg(feature = "wally")] + #[error("error resolving URL of wally package")] + Wally(#[from] wally::ResolveWallyUrlError), } impl PackageRef { /// Gets the name of the package - pub fn name(&self) -> &PackageName { + pub fn name(&self) -> PackageName { match self { - PackageRef::Registry(registry) => ®istry.name, - PackageRef::Git(git) => &git.name, + PackageRef::Registry(registry) => PackageName::Standard(registry.name.clone()), + PackageRef::Git(git) => PackageName::Standard(git.name.clone()), + #[cfg(feature = "wally")] + PackageRef::Wally(wally) => PackageName::Wally(wally.name.clone()), } } @@ -117,31 +169,81 @@ impl PackageRef { match self { PackageRef::Registry(registry) => ®istry.version, PackageRef::Git(git) => &git.version, + #[cfg(feature = "wally")] + PackageRef::Wally(wally) => &wally.version, + } + } + + /// Returns the URL of the index + pub fn index_url(&self) -> Option { + match self { + PackageRef::Registry(registry) => Some(registry.index_url.clone()), + PackageRef::Git(_) => None, + #[cfg(feature = "wally")] + PackageRef::Wally(wally) => Some(wally.index_url.clone()), + } + } + + /// Resolves the URL of the package + pub fn resolve_url(&self, project: &mut Project) -> Result, UrlResolveError> { + Ok(match &self { + PackageRef::Registry(registry) => Some(registry.resolve_url(project.indices())?), + PackageRef::Git(_) => None, + #[cfg(feature = "wally")] + PackageRef::Wally(wally) => { + let cache_dir = project.cache_dir().to_path_buf(); + Some(wally.resolve_url(&cache_dir, project.indices_mut())?) + } + }) + } + + /// Gets the index of the package + pub fn get_index<'a>(&self, project: &'a Project) -> &'a dyn Index { + match &self.index_url() { + Some(url) => get_index_by_url(project.indices(), url), + None => get_index(project.indices(), None), } } /// Downloads the package to the specified destination - pub fn download, I: Index>( + pub fn download>( &self, - project: &Project, + reqwest_client: &reqwest::blocking::Client, + registry_auth_token: Option, + url: Option<&Url>, + credentials_fn: Option>, dest: P, ) -> Result<(), DownloadError> { match self { PackageRef::Registry(registry) => registry - .download(project, dest) + .download( + reqwest_client, + url.ok_or(DownloadError::UrlRequired)?, + registry_auth_token, + dest, + ) .map_err(|e| DownloadError::Registry(e, Box::new(self.clone()))), PackageRef::Git(git) => git - .download(project, dest) + .download(dest, credentials_fn) .map_err(|e| DownloadError::Git(e, Box::new(self.clone()))), + #[cfg(feature = "wally")] + PackageRef::Wally(wally) => wally + .download( + reqwest_client, + url.ok_or(DownloadError::UrlRequired)?, + registry_auth_token, + dest, + ) + .map_err(|e| DownloadError::Wally(e, Box::new(self.clone()))), } } } -impl Project { +impl Project { /// Downloads the project's dependencies pub fn download( - &self, - map: &ResolvedVersionsMap, + &mut self, + map: ResolvedVersionsMap, ) -> Result, InstallProjectError> { let (job, tx) = MultithreadedJob::new(); @@ -161,10 +263,20 @@ impl Project { create_dir_all(&source)?; - let project = self.clone(); + let reqwest_client = self.reqwest_client.clone(); + let url = resolved_package.pkg_ref.resolve_url(self)?; + let index = resolved_package.pkg_ref.get_index(self); + let registry_auth_token = index.registry_auth_token().map(|t| t.to_string()); + let credentials_fn = index.credentials_fn().cloned(); job.execute(&tx, move || { - resolved_package.pkg_ref.download(&project, source) + resolved_package.pkg_ref.download( + &reqwest_client, + registry_auth_token, + url.as_ref(), + credentials_fn, + source, + ) }); } } @@ -178,3 +290,24 @@ impl Display for PackageRef { write!(f, "{}@{}", self.name(), self.version()) } } + +pub(crate) fn maybe_authenticated_request( + reqwest_client: &reqwest::blocking::Client, + url: &str, + registry_auth_token: Option, +) -> reqwest::blocking::RequestBuilder { + let mut builder = reqwest_client.get(url); + debug!("sending request to {}", url); + + if let Some(token) = registry_auth_token { + let hidden_token = token + .chars() + .enumerate() + .map(|(i, c)| if i <= 8 { c } else { '*' }) + .collect::(); + debug!("with registry token {hidden_token}"); + builder = builder.header(AUTHORIZATION, format!("Bearer {token}")); + } + + builder +} diff --git a/src/dependencies/registry.rs b/src/dependencies/registry.rs index 37101ae..1b8aa13 100644 --- a/src/dependencies/registry.rs +++ b/src/dependencies/registry.rs @@ -1,26 +1,33 @@ use std::path::Path; use log::{debug, error}; -use reqwest::header::{AUTHORIZATION, USER_AGENT as USER_AGENT_HEADER}; use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; use thiserror::Error; +use url::Url; use crate::{ - index::Index, manifest::Realm, package_name::PackageName, project::Project, USER_AGENT, + dependencies::maybe_authenticated_request, + manifest::Realm, + package_name::StandardPackageName, + project::{get_index_by_url, Indices, DEFAULT_INDEX_NAME}, }; +fn default_index_name() -> String { + DEFAULT_INDEX_NAME.to_string() +} + /// A dependency of a package that can be downloaded from a registry #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] #[serde(deny_unknown_fields)] pub struct RegistryDependencySpecifier { /// The name of the package - pub name: PackageName, + pub name: StandardPackageName, /// The version requirement of the package pub version: VersionReq, - // TODO: support per-package registries - // #[serde(skip_serializing_if = "Option::is_none")] - // pub registry: Option, + /// The name of the index to use + #[serde(default = "default_index_name")] + pub index: String, /// The realm of the package #[serde(skip_serializing_if = "Option::is_none")] pub realm: Option, @@ -31,12 +38,11 @@ pub struct RegistryDependencySpecifier { #[serde(deny_unknown_fields)] pub struct RegistryPackageRef { /// The name of the package - pub name: PackageName, + pub name: StandardPackageName, /// The version of the package pub version: Version, - // TODO: support per-package registries - // #[serde(skip_serializing_if = "Option::is_none")] - // pub index_url: Option, + /// The index URL of the package + pub index_url: Url, } /// An error that occurred while downloading a package from a registry @@ -56,48 +62,64 @@ pub enum RegistryDownloadError { /// The package was not found on the registry #[error("package {0} not found on the registry, but found in the index")] - NotFound(PackageName), + NotFound(StandardPackageName), /// The user is unauthorized to download the package #[error("unauthorized to download package {0}")] - Unauthorized(PackageName), + Unauthorized(StandardPackageName), /// An HTTP error occurred #[error("http error {0}: the server responded with {1}")] Http(reqwest::StatusCode, String), + + /// An error occurred while parsing the api URL + #[error("error parsing the API URL")] + UrlParse(#[from] url::ParseError), +} + +/// An error that occurred while resolving the url of a registry package +#[derive(Debug, Error)] +pub enum RegistryUrlResolveError { + /// An error that occurred while reading the index config + #[error("error with the index config")] + IndexConfig(#[from] crate::index::ConfigError), + + /// An error occurred while parsing the api URL + #[error("error parsing the API URL")] + UrlParse(#[from] url::ParseError), } impl RegistryPackageRef { - /// Downloads the package to the specified destination - pub fn download, I: Index>( - &self, - project: &Project, - dest: P, - ) -> Result<(), RegistryDownloadError> { - let url = project - .index() - .config()? + /// Resolves the download URL of the package + pub fn resolve_url(&self, indices: &Indices) -> Result { + let index = get_index_by_url(indices, &self.index_url); + let config = index.config()?; + + let url = config .download() .replace("{PACKAGE_AUTHOR}", self.name.scope()) .replace("{PACKAGE_NAME}", self.name.name()) .replace("{PACKAGE_VERSION}", &self.version.to_string()); + Ok(Url::parse(&url)?) + } + + /// Downloads the package to the specified destination + pub fn download>( + &self, + reqwest_client: &reqwest::blocking::Client, + url: &Url, + registry_auth_token: Option, + dest: P, + ) -> Result<(), RegistryDownloadError> { debug!( "downloading registry package {}@{} from {}", self.name, self.version, url ); - let client = reqwest::blocking::Client::new(); - let response = { - let mut builder = client.get(&url).header(USER_AGENT_HEADER, USER_AGENT); - if let Some(token) = project.registry_auth_token() { - let visible_tokens = token.chars().take(8).collect::(); - let hidden_tokens = "*".repeat(token.len() - 8); - debug!("using registry token {visible_tokens}{hidden_tokens}"); - builder = builder.header(AUTHORIZATION, format!("Bearer {}", token)); - } - builder.send()? - }; + let response = + maybe_authenticated_request(reqwest_client, url.as_str(), registry_auth_token) + .send()?; if !response.status().is_success() { return match response.status() { diff --git a/src/dependencies/resolution.rs b/src/dependencies/resolution.rs index 92ab2d4..e9b2095 100644 --- a/src/dependencies/resolution.rs +++ b/src/dependencies/resolution.rs @@ -9,16 +9,18 @@ use semver::Version; use serde::{Deserialize, Serialize}; use thiserror::Error; +#[cfg(feature = "wally")] +use crate::index::Index; use crate::{ dependencies::{ git::{GitDownloadError, GitPackageRef}, - registry::{RegistryDependencySpecifier, RegistryPackageRef}, + registry::RegistryPackageRef, DependencySpecifier, PackageRef, }, - index::{Index, IndexPackageError}, + index::IndexPackageError, manifest::{DependencyType, Manifest, Realm}, package_name::PackageName, - project::{Project, ReadLockfileError}, + project::{get_index, get_index_by_url, Project, ReadLockfileError}, DEV_PACKAGES_FOLDER, INDEX_FOLDER, PACKAGES_FOLDER, SERVER_PACKAGES_FOLDER, }; @@ -135,15 +137,15 @@ pub enum ResolveError { /// An error that occurred because a registry dependency conflicts with a git dependency #[error("registry dependency {0}@{1} conflicts with git dependency")] - RegistryConflict(PackageName, Version), + RegistryConflict(String, Version), /// An error that occurred because a git dependency conflicts with a registry dependency #[error("git dependency {0}@{1} conflicts with registry dependency")] - GitConflict(PackageName, Version), + GitConflict(String, Version), /// An error that occurred because no satisfying version was found for a dependency #[error("no satisfying version found for dependency {0:?}")] - NoSatisfyingVersion(RegistryDependencySpecifier), + NoSatisfyingVersion(Box), /// An error that occurred while downloading a package from a git repository #[error("error downloading git package")] @@ -151,11 +153,11 @@ pub enum ResolveError { /// An error that occurred because a package was not found in the index #[error("package {0} not found in index")] - PackageNotFound(PackageName), + PackageNotFound(String), /// An error that occurred while getting a package from the index #[error("failed to get package {1} from index")] - IndexPackage(#[source] IndexPackageError, PackageName), + IndexPackage(#[source] IndexPackageError, String), /// An error that occurred while reading the lockfile #[error("failed to read lockfile")] @@ -167,18 +169,27 @@ pub enum ResolveError { /// An error that occurred because two realms are incompatible #[error("incompatible realms for package {0} (package specified {1}, user specified {2})")] - IncompatibleRealms(PackageName, Realm, Realm), + IncompatibleRealms(String, Realm, Realm), /// An error that occurred because a peer dependency is not installed #[error("peer dependency {0}@{1} is not installed")] - PeerNotInstalled(PackageName, Version), + PeerNotInstalled(String, Version), + + /// An error that occurred while cloning a wally index + #[cfg(feature = "wally")] + #[error("error cloning wally index")] + CloneWallyIndex(#[from] crate::dependencies::wally::CloneWallyIndexError), + + /// An error that occurred while parsing a URL + #[error("error parsing URL")] + UrlParse(#[from] url::ParseError), } impl Manifest { /// Resolves the dependency tree for the project - pub fn dependency_tree( + pub fn dependency_tree( &self, - project: &Project, + project: &mut Project, locked: bool, ) -> Result { debug!("resolving dependency tree for project {}", self.name); @@ -253,19 +264,23 @@ impl Manifest { while let Some(((specifier, dep_type), dependant)) = queue.pop_front() { let (pkg_ref, default_realm, dependencies) = match &specifier { DependencySpecifier::Registry(registry_dependency) => { - let index_entries = project - .index() - .package(®istry_dependency.name) + let index = if dependant.is_none() { + get_index(project.indices(), Some(®istry_dependency.index)) + } else { + get_index_by_url(project.indices(), ®istry_dependency.index.parse()?) + }; + let pkg_name: PackageName = registry_dependency.name.clone().into(); + + let index_entries = index + .package(&pkg_name) .map_err(|e| { - ResolveError::IndexPackage(e, registry_dependency.name.clone()) + ResolveError::IndexPackage(e, registry_dependency.name.to_string()) })? .ok_or_else(|| { - ResolveError::PackageNotFound(registry_dependency.name.clone()) + ResolveError::PackageNotFound(registry_dependency.name.to_string()) })?; - let resolved_versions = resolved_versions_map - .entry(registry_dependency.name.clone()) - .or_default(); + let resolved_versions = resolved_versions_map.entry(pkg_name).or_default(); // try to find the highest already downloaded version that satisfies the requirement, otherwise find the highest satisfying version in the index let Some(version) = @@ -278,9 +293,9 @@ impl Manifest { }, ) else { - return Err(ResolveError::NoSatisfyingVersion( - registry_dependency.clone(), - )); + return Err(ResolveError::NoSatisfyingVersion(Box::new( + specifier.clone(), + ))); }; let entry = index_entries @@ -297,13 +312,15 @@ impl Manifest { PackageRef::Registry(RegistryPackageRef { name: registry_dependency.name.clone(), version: version.clone(), + index_url: index.url().clone(), }), entry.realm, entry.dependencies, ) } DependencySpecifier::Git(git_dependency) => { - let (manifest, url, rev) = git_dependency.resolve(project)?; + let (manifest, url, rev) = + git_dependency.resolve(project.cache_dir(), project.indices())?; debug!( "resolved git dependency {} to {url}#{rev}", @@ -321,6 +338,61 @@ impl Manifest { manifest.dependencies(), ) } + #[cfg(feature = "wally")] + DependencySpecifier::Wally(wally_dependency) => { + let cache_dir = project.cache_dir().to_path_buf(); + let index = crate::dependencies::wally::clone_wally_index( + &cache_dir, + project.indices_mut(), + &wally_dependency.index_url, + )?; + let pkg_name = wally_dependency.name.clone().into(); + + let index_entries = index + .package(&pkg_name) + .map_err(|e| { + ResolveError::IndexPackage(e, wally_dependency.name.to_string()) + })? + .ok_or_else(|| { + ResolveError::PackageNotFound(wally_dependency.name.to_string()) + })?; + + let resolved_versions = resolved_versions_map.entry(pkg_name).or_default(); + + // try to find the highest already downloaded version that satisfies the requirement, otherwise find the highest satisfying version in the index + let Some(version) = find_highest!(resolved_versions.keys(), wally_dependency) + .or_else(|| { + find_highest!( + index_entries.iter().map(|v| &v.version), + wally_dependency + ) + }) + else { + return Err(ResolveError::NoSatisfyingVersion(Box::new( + specifier.clone(), + ))); + }; + + let entry = index_entries + .into_iter() + .find(|e| e.version.eq(&version)) + .unwrap(); + + debug!( + "resolved registry dependency {} to {}", + wally_dependency.name, version + ); + + ( + PackageRef::Wally(crate::dependencies::wally::WallyPackageRef { + name: wally_dependency.name.clone(), + version: version.clone(), + index_url: index.url().clone(), + }), + entry.realm, + entry.dependencies, + ) + } }; let is_root = dependant.is_none(); @@ -337,7 +409,7 @@ impl Manifest { .and_then(|v| v.get_mut(&dependant_version)) .unwrap() .dependencies - .insert((pkg_ref.name().clone(), pkg_ref.version().clone())); + .insert((pkg_ref.name(), pkg_ref.version().clone())); } let resolved_versions = resolved_versions_map @@ -348,12 +420,15 @@ impl Manifest { match (&pkg_ref, &previously_resolved.pkg_ref) { (PackageRef::Registry(r), PackageRef::Git(_g)) => { return Err(ResolveError::RegistryConflict( - r.name.clone(), + r.name.to_string(), r.version.clone(), )); } (PackageRef::Git(g), PackageRef::Registry(_r)) => { - return Err(ResolveError::GitConflict(g.name.clone(), g.version.clone())); + return Err(ResolveError::GitConflict( + g.name.to_string(), + g.version.clone(), + )); } _ => (), } @@ -374,7 +449,7 @@ impl Manifest { && default_realm.is_some_and(|realm| realm == Realm::Server) { return Err(ResolveError::IncompatibleRealms( - pkg_ref.name().clone(), + pkg_ref.name().to_string(), default_realm.unwrap(), *specifier.realm().unwrap(), )); @@ -410,7 +485,7 @@ impl Manifest { for (version, resolved_package) in versions { if resolved_package.dep_type == DependencyType::Peer { return Err(ResolveError::PeerNotInstalled( - resolved_package.pkg_ref.name().clone(), + resolved_package.pkg_ref.name().to_string(), resolved_package.pkg_ref.version().clone(), )); } diff --git a/src/dependencies/wally.rs b/src/dependencies/wally.rs new file mode 100644 index 0000000..00d48d3 --- /dev/null +++ b/src/dependencies/wally.rs @@ -0,0 +1,364 @@ +use std::{ + collections::BTreeMap, + fs::{create_dir_all, read}, + hash::{DefaultHasher, Hash, Hasher}, + io::Cursor, + path::Path, +}; + +use git2::build::RepoBuilder; +use log::{debug, error}; +use semver::{Version, VersionReq}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use url::Url; + +use crate::{ + dependencies::{maybe_authenticated_request, DependencySpecifier}, + index::{remote_callbacks, IndexFileEntry, WallyIndex}, + manifest::{DependencyType, Manifest, ManifestConvertError, Realm}, + package_name::{ + FromStrPackageNameParseError, WallyPackageName, WallyPackageNameValidationError, + }, + project::{get_wally_index, Indices}, +}; + +/// A dependency of a package that can be downloaded from a registry +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] +#[serde(deny_unknown_fields)] +pub struct WallyDependencySpecifier { + /// The name of the package + #[serde(rename = "wally")] + pub name: WallyPackageName, + /// The version requirement of the package + pub version: VersionReq, + /// The url of the index + pub index_url: Url, + /// The realm of the package + #[serde(skip_serializing_if = "Option::is_none")] + pub realm: Option, +} + +/// A reference to a package that can be downloaded from a registry +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] +#[serde(deny_unknown_fields)] +pub struct WallyPackageRef { + /// The name of the package + pub name: WallyPackageName, + /// The version of the package + pub version: Version, + /// The index URL of the package + pub index_url: Url, +} + +/// An error that occurred while downloading a package from a wally registry +#[derive(Debug, Error)] +pub enum WallyDownloadError { + /// An error that occurred while interacting with reqwest + #[error("error interacting with reqwest")] + Reqwest(#[from] reqwest::Error), + + /// An error that occurred while interacting with the file system + #[error("error interacting with the file system")] + Io(#[from] std::io::Error), + + /// The package was not found on the registry + #[error("package {0} not found on the registry, but found in the index")] + NotFound(WallyPackageName), + + /// The user is unauthorized to download the package + #[error("unauthorized to download package {0}")] + Unauthorized(WallyPackageName), + + /// An HTTP error occurred + #[error("http error {0}: the server responded with {1}")] + Http(reqwest::StatusCode, String), + + /// An error occurred while extracting the archive + #[error("error extracting archive")] + Zip(#[from] zip::result::ZipError), + + /// An error occurred while interacting with git + #[error("error interacting with git")] + Git(#[from] git2::Error), + + /// An error occurred while interacting with serde + #[error("error interacting with serde")] + Serde(#[from] serde_json::Error), + + /// An error occurred while parsing the api URL + #[error("error parsing URL")] + Url(#[from] url::ParseError), + + /// An error occurred while refreshing the index + #[error("error refreshing index")] + RefreshIndex(#[from] crate::index::RefreshError), + + /// An error occurred while converting the manifest + #[error("error converting manifest")] + Manifest(#[from] ManifestConvertError), +} + +/// An error that occurred while cloning a wally index +#[derive(Error, Debug)] +pub enum CloneWallyIndexError { + /// An error that occurred while interacting with git + #[error("error interacting with git")] + Git(#[from] git2::Error), + + /// An error that occurred while interacting with the file system + #[error("error interacting with the file system")] + Io(#[from] std::io::Error), + + /// An error that occurred while refreshing the index + #[error("error refreshing index")] + RefreshIndex(#[from] crate::index::RefreshError), +} + +pub(crate) fn clone_wally_index( + cache_dir: &Path, + indices: &mut Indices, + index_url: &Url, +) -> Result { + let mut hasher = DefaultHasher::new(); + index_url.hash(&mut hasher); + let url_hash = hasher.finish().to_string(); + + let index_path = cache_dir.join("wally_indices").join(url_hash); + + if index_path.exists() { + debug!("wally index already exists at {}", index_path.display()); + + return Ok(get_wally_index(indices, index_url, Some(&index_path))?.clone()); + } + + debug!( + "cloning wally index from {} to {}", + index_url, + index_path.display() + ); + + create_dir_all(&index_path)?; + + let mut fetch_options = git2::FetchOptions::new(); + fetch_options.remote_callbacks(remote_callbacks!(get_wally_index( + indices, + index_url, + Some(&index_path) + )?)); + + RepoBuilder::new() + .fetch_options(fetch_options) + .clone(index_url.as_ref(), &index_path)?; + + Ok(get_wally_index(indices, index_url, Some(&index_path))?.clone()) +} + +/// The configuration of a wally index +#[derive(Serialize, Deserialize, Debug)] +struct WallyIndexConfig { + /// The URL of the wally API + api: String, +} + +/// An error that occurred while resolving the URL of a wally package +#[derive(Error, Debug)] +pub enum ResolveWallyUrlError { + /// An error that occurred while interacting with the file system + #[error("error interacting with the file system")] + Io(#[from] std::io::Error), + + /// An error that occurred while interacting with the index + #[error("error interacting with the index")] + Index(#[from] crate::index::ConfigError), + + /// An error that occurred while parsing the URL + #[error("error parsing URL")] + Url(#[from] url::ParseError), + + /// An error that occurred while cloning the index + #[error("error cloning index")] + CloneIndex(#[from] CloneWallyIndexError), + + /// An error that occurred while reading the index config + #[error("error reading index config")] + ReadConfig(#[from] serde_json::Error), +} + +fn read_api_url(index_path: &Path) -> Result { + let config_path = index_path.join("config.json"); + let raw_config_contents = read(config_path)?; + let config: WallyIndexConfig = serde_json::from_slice(&raw_config_contents)?; + + Ok(config.api) +} + +impl WallyPackageRef { + /// Resolves the download URL of the package + pub fn resolve_url( + &self, + cache_dir: &Path, + indices: &mut Indices, + ) -> Result { + let index = clone_wally_index(cache_dir, indices, &self.index_url)?; + + let api_url = Url::parse(&read_api_url(&index.path)?)?; + + let url = format!( + "{}/v1/package-contents/{}/{}/{}", + api_url.to_string().trim_end_matches('/'), + self.name.scope(), + self.name.name(), + self.version + ); + + Ok(Url::parse(&url)?) + } + + /// Downloads the package to the specified destination + pub fn download>( + &self, + reqwest_client: &reqwest::blocking::Client, + url: &Url, + registry_auth_token: Option, + dest: P, + ) -> Result<(), WallyDownloadError> { + let response = + maybe_authenticated_request(reqwest_client, url.as_str(), registry_auth_token) + .header("Wally-Version", "0.3.2") + .send()?; + + if !response.status().is_success() { + return match response.status() { + reqwest::StatusCode::NOT_FOUND => { + Err(WallyDownloadError::NotFound(self.name.clone())) + } + reqwest::StatusCode::UNAUTHORIZED => { + Err(WallyDownloadError::Unauthorized(self.name.clone())) + } + _ => Err(WallyDownloadError::Http( + response.status(), + response.text()?, + )), + }; + } + + let bytes = response.bytes()?; + + let mut archive = zip::read::ZipArchive::new(Cursor::new(bytes))?; + archive.extract(dest.as_ref())?; + + Manifest::from_path_or_convert(dest.as_ref())?; + + Ok(()) + } +} + +#[derive(Deserialize, Clone, Debug)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct WallyPackage { + pub(crate) name: String, + pub(crate) version: Version, + pub(crate) registry: Url, + #[serde(default)] + pub(crate) realm: Option, + #[serde(default)] + pub(crate) description: Option, + #[serde(default)] + pub(crate) license: Option, + #[serde(default)] + pub(crate) authors: Option>, + #[serde(default)] + pub(crate) private: Option, +} + +#[derive(Deserialize, Default, Clone, Debug)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct WallyPlace { + #[serde(default)] + pub(crate) shared_packages: Option, + #[serde(default)] + pub(crate) server_packages: Option, +} + +#[derive(Deserialize, Clone, Debug)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct WallyManifest { + pub(crate) package: WallyPackage, + #[serde(default)] + pub(crate) place: WallyPlace, + #[serde(default)] + pub(crate) dependencies: BTreeMap, + #[serde(default)] + pub(crate) server_dependencies: BTreeMap, + #[serde(default)] + pub(crate) dev_dependencies: BTreeMap, +} + +/// An error that occurred while converting a wally manifest's dependencies +#[derive(Debug, Error)] +pub enum WallyManifestDependencyError { + /// An error that occurred because the dependency specifier is invalid + #[error("invalid dependency specifier: {0}")] + InvalidDependencySpecifier(String), + + /// An error that occurred while parsing a package name + #[error("error parsing package name")] + PackageName(#[from] FromStrPackageNameParseError), + + /// An error that occurred while parsing a version requirement + #[error("error parsing version requirement")] + VersionReq(#[from] semver::Error), +} + +pub(crate) fn parse_wally_dependencies( + manifest: WallyManifest, +) -> Result, WallyManifestDependencyError> { + [ + (manifest.dependencies, Realm::Shared), + (manifest.server_dependencies, Realm::Server), + (manifest.dev_dependencies, Realm::Development), + ] + .into_iter() + .flat_map(|(deps, realm)| { + deps.into_values() + .map(|specifier| { + let (name, req) = specifier.split_once('@').ok_or_else(|| { + WallyManifestDependencyError::InvalidDependencySpecifier(specifier.clone()) + })?; + let name: WallyPackageName = name.parse()?; + let req: VersionReq = req.parse()?; + + Ok(DependencySpecifier::Wally(WallyDependencySpecifier { + name, + version: req, + index_url: manifest.package.registry.clone(), + realm: Some(realm), + })) + }) + .collect::>() + }) + .collect() +} + +impl TryFrom for IndexFileEntry { + type Error = WallyManifestDependencyError; + + fn try_from(value: WallyManifest) -> Result { + let dependencies = parse_wally_dependencies(value.clone())? + .into_iter() + .map(|d| (d, DependencyType::Normal)) + .collect(); + + Ok(IndexFileEntry { + version: value.package.version, + realm: value + .package + .realm + .map(|r| r.parse().unwrap_or(Realm::Shared)), + published_at: Default::default(), + description: value.package.description, + dependencies, + }) + } +} diff --git a/src/index.rs b/src/index.rs index 2bb6be0..3a1eaa0 100644 --- a/src/index.rs +++ b/src/index.rs @@ -1,5 +1,5 @@ -use chrono::{DateTime, Utc}; use std::{ + any::Any, collections::BTreeSet, fmt::Debug, fs::create_dir_all, @@ -8,11 +8,13 @@ use std::{ sync::Arc, }; +use chrono::{DateTime, Utc}; use git2::{build::RepoBuilder, Remote, Repository, Signature}; use log::debug; use semver::Version; use serde::{Deserialize, Serialize}; use thiserror::Error; +use url::Url; use crate::{ dependencies::DependencySpecifier, @@ -24,7 +26,7 @@ use crate::{ pub type ScopeOwners = BTreeSet; /// A packages index -pub trait Index: Send + Sync + Debug + Clone + 'static { +pub trait Index: Send + Sync + Debug + Any + 'static { /// Gets the owners of a scope fn scope_owners(&self, scope: &str) -> Result, ScopeOwnersError>; @@ -50,6 +52,22 @@ pub trait Index: Send + Sync + Debug + Clone + 'static { /// Returns a function that gets the credentials for a git repository fn credentials_fn(&self) -> Option<&Arc>; + + /// Returns the URL of the index's repository + fn url(&self) -> &Url; + + /// Returns the token to this index's registry + fn registry_auth_token(&self) -> Option<&str> { + None + } + + /// Updates the index + fn refresh(&self) -> Result<(), RefreshError> { + Ok(()) + } + + /// Returns this as Any + fn as_any(&self) -> &dyn Any; } /// A function that gets the credentials for a git repository @@ -64,7 +82,8 @@ pub type CredentialsFn = Box< #[derive(Clone)] pub struct GitIndex { path: PathBuf, - repo_url: String, + repo_url: Url, + registry_auth_token: Option, pub(crate) credentials_fn: Option>, } @@ -174,6 +193,10 @@ pub enum IndexPackageError { /// An error that occurred while deserializing the index file #[error("error deserializing index file")] FileDeser(#[source] serde_yaml::Error), + + /// An unknown error occurred + #[error("unknown error")] + Other(#[source] Box), } /// An error that occurred while creating a package version @@ -202,6 +225,10 @@ pub enum CreatePackageVersionError { /// The scope is missing ownership #[error("missing scope ownership")] MissingScopeOwnership, + + /// An error that occurred while converting a manifest to an index file entry + #[error("error converting manifest to index file entry")] + FromManifestIndexFileEntry(#[from] FromManifestIndexFileEntry), } /// An error that occurred while getting the index's configuration @@ -247,29 +274,36 @@ fn get_refspec( Ok((refspec.to_string(), upstream_branch.to_string())) } -pub(crate) fn remote_callbacks(index: &I) -> git2::RemoteCallbacks { - let mut remote_callbacks = git2::RemoteCallbacks::new(); +macro_rules! remote_callbacks { + ($index:expr) => {{ + #[allow(unused_imports)] + use crate::index::Index; + let mut remote_callbacks = git2::RemoteCallbacks::new(); - if let Some(credentials) = &index.credentials_fn() { - let credentials = std::sync::Arc::clone(credentials); + if let Some(credentials) = &$index.credentials_fn() { + let credentials = std::sync::Arc::clone(credentials); - remote_callbacks.credentials(move |a, b, c| credentials()(a, b, c)); - } + remote_callbacks.credentials(move |a, b, c| credentials()(a, b, c)); + } - remote_callbacks + remote_callbacks + }}; } +pub(crate) use remote_callbacks; impl GitIndex { /// Creates a new git index. The `refresh` method must be called before using the index, preferably immediately after creating it. pub fn new>( path: P, - repo_url: &str, + repo_url: &Url, credentials: Option, + registry_auth_token: Option, ) -> Self { Self { path: path.as_ref().to_path_buf(), - repo_url: repo_url.to_string(), + repo_url: repo_url.clone(), credentials_fn: credentials.map(Arc::new), + registry_auth_token, } } @@ -278,58 +312,6 @@ impl GitIndex { &self.path } - /// Gets the URL of the index's repository - pub fn repo_url(&self) -> &str { - &self.repo_url - } - - /// Refreshes the index - pub fn refresh(&self) -> Result<(), RefreshError> { - let repo = if self.path.exists() { - Repository::open(&self.path).ok() - } else { - None - }; - - if let Some(repo) = repo { - let mut remote = repo.find_remote("origin")?; - let (refspec, upstream_branch) = get_refspec(&repo, &mut remote)?; - - remote.fetch( - &[&refspec], - Some(git2::FetchOptions::new().remote_callbacks(remote_callbacks(self))), - None, - )?; - - let commit = repo.find_reference(&upstream_branch)?.peel_to_commit()?; - - debug!( - "refreshing index, fetching {refspec}#{} from origin", - commit.id().to_string() - ); - - repo.reset(&commit.into_object(), git2::ResetType::Hard, None)?; - - Ok(()) - } else { - debug!( - "refreshing index - first time, cloning {} into {}", - self.repo_url, - self.path.display() - ); - create_dir_all(&self.path)?; - - let mut fetch_options = git2::FetchOptions::new(); - fetch_options.remote_callbacks(remote_callbacks(self)); - - RepoBuilder::new() - .fetch_options(fetch_options) - .clone(&self.repo_url, &self.path)?; - - Ok(()) - } - } - /// Commits and pushes to the index pub fn commit_and_push( &self, @@ -362,13 +344,61 @@ impl GitIndex { remote.push( &[&refspec], - Some(git2::PushOptions::new().remote_callbacks(remote_callbacks(self))), + Some(git2::PushOptions::new().remote_callbacks(remote_callbacks!(self))), )?; Ok(()) } } +macro_rules! refresh_git_based_index { + ($index:expr) => {{ + let repo = if $index.path.exists() { + Repository::open(&$index.path).ok() + } else { + None + }; + + if let Some(repo) = repo { + let mut remote = repo.find_remote("origin")?; + let (refspec, upstream_branch) = get_refspec(&repo, &mut remote)?; + + remote.fetch( + &[&refspec], + Some(git2::FetchOptions::new().remote_callbacks(remote_callbacks!($index))), + None, + )?; + + let commit = repo.find_reference(&upstream_branch)?.peel_to_commit()?; + + debug!( + "refreshing index, fetching {refspec}#{} from origin", + commit.id().to_string() + ); + + repo.reset(&commit.into_object(), git2::ResetType::Hard, None)?; + + Ok(()) + } else { + debug!( + "refreshing index - first time, cloning {} into {}", + $index.repo_url, + $index.path.display() + ); + create_dir_all(&$index.path)?; + + let mut fetch_options = git2::FetchOptions::new(); + fetch_options.remote_callbacks(remote_callbacks!($index)); + + RepoBuilder::new() + .fetch_options(fetch_options) + .clone(&$index.repo_url.to_string(), &$index.path)?; + + Ok(()) + } + }}; +} + impl Index for GitIndex { fn scope_owners(&self, scope: &str) -> Result, ScopeOwnersError> { let path = self.path.join(scope).join("owners.yaml"); @@ -434,16 +464,17 @@ impl Index for GitIndex { let path = self.path.join(scope); - let mut file = if let Some(file) = self.package(&manifest.name)? { - if file.iter().any(|e| e.version == manifest.version) { - return Ok(None); - } - file - } else { - BTreeSet::new() - }; + let mut file = + if let Some(file) = self.package(&PackageName::Standard(manifest.name.clone()))? { + if file.iter().any(|e| e.version == manifest.version) { + return Ok(None); + } + file + } else { + BTreeSet::new() + }; - let entry: IndexFileEntry = manifest.clone().into(); + let entry: IndexFileEntry = manifest.clone().try_into()?; file.insert(entry.clone()); serde_yaml::to_writer( @@ -472,6 +503,22 @@ impl Index for GitIndex { fn credentials_fn(&self) -> Option<&Arc> { self.credentials_fn.as_ref() } + + fn url(&self) -> &Url { + &self.repo_url + } + + fn registry_auth_token(&self) -> Option<&str> { + self.registry_auth_token.as_deref() + } + + fn refresh(&self) -> Result<(), RefreshError> { + refresh_git_based_index!(self) + } + + fn as_any(&self) -> &dyn Any { + self + } } /// The configuration of the index @@ -479,10 +526,10 @@ impl Index for GitIndex { #[serde(deny_unknown_fields)] pub struct IndexConfig { /// The URL of the index's API - pub api: String, + pub api: Url, /// The URL of the index's download API, defaults to `{API_URL}/v0/packages/{PACKAGE_AUTHOR}/{PACKAGE_NAME}/{PACKAGE_VERSION}`. /// Has the following variables: - /// - `{API_URL}`: The URL of the index's API + /// - `{API_URL}`: The URL of the index's API (without trailing `/`) /// - `{PACKAGE_AUTHOR}`: The author of the package /// - `{PACKAGE_NAME}`: The name of the package /// - `{PACKAGE_VERSION}`: The version of the package @@ -500,7 +547,7 @@ pub struct IndexConfig { impl IndexConfig { /// Gets the URL of the index's API pub fn api(&self) -> &str { - self.api.strip_suffix('/').unwrap_or(&self.api) + self.api.as_str().trim_end_matches('/') } /// Gets the URL of the index's download API @@ -535,19 +582,48 @@ pub struct IndexFileEntry { pub dependencies: Vec<(DependencySpecifier, DependencyType)>, } -impl From for IndexFileEntry { - fn from(manifest: Manifest) -> IndexFileEntry { - let dependencies = manifest.dependencies(); +/// An error that occurred while converting a manifest to an index file entry +#[derive(Debug, Error)] +pub enum FromManifestIndexFileEntry { + /// An error that occurred because an index is not specified + #[error("index {0} is not specified")] + IndexNotSpecified(String), +} - IndexFileEntry { +impl TryFrom for IndexFileEntry { + type Error = FromManifestIndexFileEntry; + + fn try_from(manifest: Manifest) -> Result { + let dependencies = manifest.dependencies(); + let indices = manifest.indices; + + Ok(Self { version: manifest.version, realm: manifest.realm, published_at: Utc::now(), description: manifest.description, - dependencies, - } + dependencies: dependencies + .into_iter() + .map(|(dep, ty)| { + Ok(match dep { + DependencySpecifier::Registry(mut registry) => { + registry.index = indices + .get(®istry.index) + .ok_or_else(|| { + FromManifestIndexFileEntry::IndexNotSpecified( + registry.index.clone(), + ) + })? + .clone(); + (DependencySpecifier::Registry(registry), ty) + } + d => (d, ty), + }) + }) + .collect::>()?, + }) } } @@ -565,3 +641,110 @@ impl Ord for IndexFileEntry { /// An index file pub type IndexFile = BTreeSet; + +#[cfg(feature = "wally")] +#[derive(Clone)] +pub(crate) struct WallyIndex { + repo_url: Url, + registry_auth_token: Option, + credentials_fn: Option>, + pub(crate) path: PathBuf, +} + +#[cfg(feature = "wally")] +impl Debug for WallyIndex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WallyIndex") + .field("path", &self.path) + .field("repo_url", &self.repo_url) + .finish() + } +} + +#[cfg(feature = "wally")] +impl WallyIndex { + pub(crate) fn new( + repo_url: Url, + registry_auth_token: Option, + path: &Path, + credentials_fn: Option>, + ) -> Self { + Self { + repo_url, + registry_auth_token, + path: path.to_path_buf(), + credentials_fn, + } + } +} + +#[cfg(feature = "wally")] +impl Index for WallyIndex { + fn scope_owners(&self, _scope: &str) -> Result, ScopeOwnersError> { + unimplemented!("wally index is a virtual index meant for wally compatibility only") + } + + fn create_scope_for( + &mut self, + _scope: &str, + _owners: &ScopeOwners, + ) -> Result { + unimplemented!("wally index is a virtual index meant for wally compatibility only") + } + + fn package(&self, name: &PackageName) -> Result, IndexPackageError> { + let path = self.path.join(name.scope()).join(name.name()); + + if !path.exists() { + return Ok(None); + } + + let file = std::fs::File::open(&path)?; + let file = std::io::BufReader::new(file); + + let manifest_stream = serde_json::Deserializer::from_reader(file) + .into_iter::() + .collect::, _>>() + .map_err(|e| IndexPackageError::Other(Box::new(e)))?; + + Ok(Some(BTreeSet::from_iter( + manifest_stream + .into_iter() + .map(|m| m.try_into()) + .collect::, _>>() + .map_err(|e| IndexPackageError::Other(Box::new(e)))?, + ))) + } + + fn create_package_version( + &mut self, + _manifest: &Manifest, + _uploader: &u64, + ) -> Result, CreatePackageVersionError> { + unimplemented!("wally index is a virtual index meant for wally compatibility only") + } + + fn config(&self) -> Result { + unimplemented!("wally index is a virtual index meant for wally compatibility only") + } + + fn credentials_fn(&self) -> Option<&Arc> { + self.credentials_fn.as_ref() + } + + fn url(&self) -> &Url { + &self.repo_url + } + + fn registry_auth_token(&self) -> Option<&str> { + self.registry_auth_token.as_deref() + } + + fn refresh(&self) -> Result<(), RefreshError> { + refresh_git_based_index!(self) + } + + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/src/lib.rs b/src/lib.rs index 242ac6a..3d7cbf4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,7 @@ //! - Re-exporting types //! - `bin` exports (ran with Lune) //! - Patching packages +//! - Downloading packages from Wally registries /// Resolving, downloading and managing dependencies pub mod dependencies; @@ -44,5 +45,3 @@ pub const IGNORED_FOLDERS: &[&str] = &[ SERVER_PACKAGES_FOLDER, ".git", ]; - -const USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); diff --git a/src/linking_file.rs b/src/linking_file.rs index 937f376..d44bbf0 100644 --- a/src/linking_file.rs +++ b/src/linking_file.rs @@ -1,8 +1,7 @@ use std::{ - fs::{read, write}, + fs::{read_to_string, write}, iter, path::{Component, Path, PathBuf}, - str::from_utf8, }; use full_moon::{ @@ -16,7 +15,6 @@ use thiserror::Error; use crate::{ dependencies::resolution::{packages_folder, ResolvedPackage, ResolvedVersionsMap}, - index::Index, manifest::{Manifest, ManifestReadError, PathStyle, Realm}, package_name::PackageName, project::Project, @@ -124,8 +122,8 @@ pub enum LinkingError { InvalidLuau(#[from] full_moon::Error), } -pub(crate) fn link, Q: AsRef, I: Index>( - project: &Project, +pub(crate) fn link, Q: AsRef>( + project: &Project, resolved_pkg: &ResolvedPackage, destination_dir: P, parent_dependency_packages_dir: Q, @@ -133,18 +131,19 @@ pub(crate) fn link, Q: AsRef, I: Index>( let (_, source_dir) = resolved_pkg.directory(project.path()); let file = Manifest::from_path(&source_dir)?; - let Some(lib_export) = file.exports.lib else { + let Some(relative_lib_export) = file.exports.lib else { return Ok(()); }; - let lib_export = lib_export.to_path(&source_dir); + let lib_export = relative_lib_export.to_path(&source_dir); let path_style = &project.manifest().path_style; let PathStyle::Roblox { place } = &path_style; debug!("linking {resolved_pkg} using `{}` path style", path_style); - let name = resolved_pkg.pkg_ref.name().name(); + let pkg_name = resolved_pkg.pkg_ref.name(); + let name = pkg_name.name(); let destination_dir = if resolved_pkg.is_root { project.path().join(packages_folder( @@ -154,7 +153,7 @@ pub(crate) fn link, Q: AsRef, I: Index>( destination_dir.as_ref().to_path_buf() }; - let destination_file = destination_dir.join(format!("{name}.lua")); + let destination_file = destination_dir.join(format!("{}{}.lua", pkg_name.prefix(), name)); let realm_folder = project.path().join(resolved_pkg.packages_folder()); let in_different_folders = realm_folder != parent_dependency_packages_dir.as_ref(); @@ -199,10 +198,12 @@ pub(crate) fn link, Q: AsRef, I: Index>( destination_file.display() ); - let raw_file_contents = read(lib_export)?; - let file_contents = from_utf8(&raw_file_contents)?; + let file_contents = match relative_lib_export.as_str() { + "true" => "".to_string(), + _ => read_to_string(lib_export)?, + }; - let linking_file_contents = linking_file(file_contents, &path)?; + let linking_file_contents = linking_file(&file_contents, &path)?; write(&destination_file, linking_file_contents)?; @@ -220,7 +221,7 @@ pub struct LinkingDependenciesError( Version, ); -impl Project { +impl Project { /// Links the dependencies of the project pub fn link_dependencies( &self, diff --git a/src/manifest.rs b/src/manifest.rs index 92bea9f..caf1e5b 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -1,15 +1,14 @@ -use std::fs::read_to_string; -use std::path::PathBuf; -use std::str::FromStr; -use std::{collections::BTreeMap, fmt::Display, fs::read}; +use cfg_if::cfg_if; +use std::{collections::BTreeMap, fmt::Display, fs::read, str::FromStr}; use relative_path::RelativePathBuf; -use semver::{Version, VersionReq}; +use semver::Version; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::dependencies::registry::RegistryDependencySpecifier; -use crate::{dependencies::DependencySpecifier, package_name::PackageName, MANIFEST_FILE_NAME}; +use crate::{ + dependencies::DependencySpecifier, package_name::StandardPackageName, MANIFEST_FILE_NAME, +}; /// The files exported by the package #[derive(Serialize, Deserialize, Debug, Clone, Default)] @@ -18,7 +17,7 @@ pub struct Exports { /// Points to the file which exports the package. As of currently this is only used for re-exporting types. /// Libraries must have a structure in Roblox where the main file becomes the folder, for example: /// A package called pesde/lib has a file called src/main.lua. - /// Pesde puts this package in a folder called pesde_lib. + /// pesde puts this package in a folder called pesde_lib. /// The package has to have set up configuration for file-syncing tools such as Rojo so that src/main.lua becomes the pesde_lib and turns it into a ModuleScript #[serde(default, skip_serializing_if = "Option::is_none")] pub lib: Option, @@ -114,7 +113,7 @@ impl FromStr for Realm { // #[serde(deny_unknown_fields)] pub struct Manifest { /// The name of the package - pub name: PackageName, + pub name: StandardPackageName, /// The version of the package. Must be [semver](https://semver.org) compatible. The registry will not accept non-semver versions and the CLI will not handle such packages pub version: Version, /// The files exported by the package @@ -128,6 +127,8 @@ pub struct Manifest { pub private: bool, /// The realm of the package pub realm: Option, + /// Indices of the package + pub indices: BTreeMap, /// The dependencies of the package #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -162,40 +163,48 @@ pub enum ManifestReadError { ManifestDeser(#[source] serde_yaml::Error), } -/// An error that occurred while converting the manifest -#[derive(Debug, Error)] -pub enum ManifestConvertError { - /// An error that occurred while reading the manifest - #[error("error reading the manifest")] - ManifestRead(#[from] ManifestReadError), +cfg_if! { + if #[cfg(feature = "wally")] { + /// An error that occurred while converting the manifest + #[derive(Debug, Error)] + pub enum ManifestConvertError { + /// An error that occurred while reading the manifest + #[error("error reading the manifest")] + ManifestRead(#[from] ManifestReadError), - /// An error that occurred while converting the manifest - #[error("error converting the manifest")] - ManifestConvert(#[source] toml::de::Error), + /// An error that occurred while converting the manifest + #[error("error converting the manifest")] + ManifestConvert(#[source] toml::de::Error), - /// The given path does not have a parent - #[error("the path {0} does not have a parent")] - NoParent(PathBuf), + /// The given path does not have a parent + #[error("the path {0} does not have a parent")] + NoParent(std::path::PathBuf), - /// An error that occurred while interacting with the file system - #[error("error interacting with the file system")] - Io(#[from] std::io::Error), + /// An error that occurred while interacting with the file system + #[error("error interacting with the file system")] + Io(#[from] std::io::Error), - /// An error that occurred while making a package name from a string - #[error("error making a package name from a string")] - PackageName(#[from] crate::package_name::FromStrPackageNameParseError), + /// An error that occurred while making a package name from a string + #[error("error making a package name from a string")] + PackageName( + #[from] + crate::package_name::FromStrPackageNameParseError< + crate::package_name::StandardPackageNameValidationError, + >, + ), - /// An error that occurred while writing the manifest - #[error("error writing the manifest")] - ManifestWrite(#[from] serde_yaml::Error), + /// An error that occurred while writing the manifest + #[error("error writing the manifest")] + ManifestWrite(#[from] serde_yaml::Error), - /// An error that occurred while converting a dependency specifier's version - #[error("error converting a dependency specifier's version")] - Version(#[from] semver::Error), - - /// The dependency specifier isn't in the format of `scope/name@version` - #[error("the dependency specifier {0} isn't in the format of `scope/name@version`")] - InvalidDependencySpecifier(String), + /// An error that occurred while parsing the dependencies + #[error("error parsing the dependencies")] + DependencyParse(#[from] crate::dependencies::wally::WallyManifestDependencyError), + } + } else { + /// An error that occurred while converting the manifest + pub type ManifestConvertError = ManifestReadError; + } } /// The type of dependency @@ -227,6 +236,7 @@ impl Manifest { } /// Tries to read the manifest from the given path, and if it fails, tries converting the `wally.toml` and writes a `pesde.yaml` in the same directory + #[cfg(feature = "wally")] pub fn from_path_or_convert>( path: P, ) -> Result { @@ -240,69 +250,14 @@ impl Manifest { }; Self::from_path(path).or_else(|_| { - #[derive(Deserialize)] - struct WallyPackage { - name: String, - version: Version, - #[serde(default)] - realm: Option, - #[serde(default)] - description: Option, - #[serde(default)] - license: Option, - #[serde(default)] - authors: Option>, - #[serde(default)] - private: Option, - } - - #[derive(Deserialize, Default)] - struct WallyPlace { - #[serde(default)] - shared_packages: Option, - #[serde(default)] - server_packages: Option, - } - - #[derive(Deserialize)] - struct WallyDependencySpecifier(String); - - impl TryFrom for DependencySpecifier { - type Error = ManifestConvertError; - - fn try_from(specifier: WallyDependencySpecifier) -> Result { - let (name, req) = specifier.0.split_once('@').ok_or_else(|| { - ManifestConvertError::InvalidDependencySpecifier(specifier.0.clone()) - })?; - let name: PackageName = name.replace('-', "_").parse()?; - let req: VersionReq = req.parse()?; - - Ok(DependencySpecifier::Registry(RegistryDependencySpecifier { - name, - version: req, - realm: None, - })) - } - } - - #[derive(Deserialize)] - struct WallyManifest { - package: WallyPackage, - #[serde(default)] - place: WallyPlace, - #[serde(default)] - dependencies: BTreeMap, - #[serde(default)] - server_dependencies: BTreeMap, - #[serde(default)] - dev_dependencies: BTreeMap, - } - let toml_path = dir_path.join("wally.toml"); - let toml_contents = read_to_string(toml_path)?; - let wally_manifest: WallyManifest = + let toml_contents = std::fs::read_to_string(toml_path)?; + let wally_manifest: crate::dependencies::wally::WallyManifest = toml::from_str(&toml_contents).map_err(ManifestConvertError::ManifestConvert)?; + let dependencies = + crate::dependencies::wally::parse_wally_dependencies(wally_manifest.clone())?; + let mut place = BTreeMap::new(); if let Some(shared) = wally_manifest.place.shared_packages { @@ -320,36 +275,21 @@ impl Manifest { let manifest = Self { name: wally_manifest.package.name.replace('-', "_").parse()?, version: wally_manifest.package.version, - exports: Exports::default(), + exports: Exports { + lib: Some(RelativePathBuf::from("true")), + bin: None, + }, path_style: PathStyle::Roblox { place }, private: wally_manifest.package.private.unwrap_or(false), realm: wally_manifest .package .realm .map(|r| r.parse().unwrap_or(Realm::Shared)), - dependencies: [ - (wally_manifest.dependencies, Realm::Shared), - (wally_manifest.server_dependencies, Realm::Server), - (wally_manifest.dev_dependencies, Realm::Development), - ] - .into_iter() - .flat_map(|(deps, realm)| { - deps.into_values() - .map(|specifier| { - specifier.try_into().map(|mut specifier| { - match specifier { - DependencySpecifier::Registry(ref mut specifier) => { - specifier.realm = Some(realm); - } - _ => unreachable!(), - } - - specifier - }) - }) - .collect::>() - }) - .collect::>()?, + indices: BTreeMap::from([( + crate::project::DEFAULT_INDEX_NAME.to_string(), + "".to_string(), + )]), + dependencies, peer_dependencies: Vec::new(), description: wally_manifest.package.description, license: wally_manifest.package.license, @@ -364,6 +304,14 @@ impl Manifest { }) } + /// Same as `from_path`, enable the `wally` feature to add support for converting `wally.toml` to `pesde.yaml` + #[cfg(not(feature = "wally"))] + pub fn from_path_or_convert>( + path: P, + ) -> Result { + Self::from_path(path) + } + /// Returns all dependencies pub fn dependencies(&self) -> Vec<(DependencySpecifier, DependencyType)> { self.dependencies diff --git a/src/package_name.rs b/src/package_name.rs index 57bd899..28c1627 100644 --- a/src/package_name.rs +++ b/src/package_name.rs @@ -1,15 +1,23 @@ -use std::{fmt::Display, str::FromStr}; +use std::{ + fmt::Debug, + hash::Hash, + {fmt::Display, str::FromStr}, +}; -use serde::{de::Visitor, Deserialize, Serialize}; +use cfg_if::cfg_if; +use serde::{ + de::{IntoDeserializer, Visitor}, + Deserialize, Serialize, +}; use thiserror::Error; /// A package name #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct PackageName(String, String); +pub struct StandardPackageName(String, String); /// An error that occurred while validating a package name part (scope or name) #[derive(Debug, Error)] -pub enum PackageNameValidationError { +pub enum StandardPackageNameValidationError { /// The package name part is empty #[error("package name part cannot be empty")] EmptyPart, @@ -22,130 +30,386 @@ pub enum PackageNameValidationError { } /// Validates a package name part (scope or name) -pub fn validate_part(part: &str) -> Result<(), PackageNameValidationError> { +pub fn validate_part(part: &str) -> Result<(), StandardPackageNameValidationError> { if part.is_empty() { - return Err(PackageNameValidationError::EmptyPart); + return Err(StandardPackageNameValidationError::EmptyPart); } if !part .chars() .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') { - return Err(PackageNameValidationError::InvalidPart(part.to_string())); + return Err(StandardPackageNameValidationError::InvalidPart( + part.to_string(), + )); } if part.len() > 24 { - return Err(PackageNameValidationError::PartTooLong(part.to_string())); + return Err(StandardPackageNameValidationError::PartTooLong( + part.to_string(), + )); } Ok(()) } -const SEPARATOR: char = '/'; -const ESCAPED_SEPARATOR: char = '-'; +/// A wally package name +#[cfg(feature = "wally")] +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct WallyPackageName(String, String); + +/// An error that occurred while validating a wally package name part (scope or name) +#[cfg(feature = "wally")] +#[derive(Debug, Error)] +pub enum WallyPackageNameValidationError { + /// The package name part is empty + #[error("wally package name part cannot be empty")] + EmptyPart, + /// The package name part contains invalid characters (only lowercase ASCII characters, numbers, and dashes are allowed) + #[error("wally package name {0} part can only contain lowercase ASCII characters, numbers, and dashes")] + InvalidPart(String), + /// The package name part is too long (it cannot be longer than 64 characters) + #[error("wally package name {0} part cannot be longer than 64 characters")] + PartTooLong(String), +} + +/// Validates a wally package name part (scope or name) +#[cfg(feature = "wally")] +pub fn validate_wally_part(part: &str) -> Result<(), WallyPackageNameValidationError> { + if part.is_empty() { + return Err(WallyPackageNameValidationError::EmptyPart); + } + + if !part + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') + { + return Err(WallyPackageNameValidationError::InvalidPart( + part.to_string(), + )); + } + + if part.len() > 64 { + return Err(WallyPackageNameValidationError::PartTooLong( + part.to_string(), + )); + } + + Ok(()) +} /// An error that occurred while parsing an escaped package name #[derive(Debug, Error)] -pub enum EscapedPackageNameError { +pub enum EscapedPackageNameError { + /// This package name is missing a prefix + #[error("package name is missing prefix {0}")] + MissingPrefix(String), + /// This is not a valid escaped package name - #[error("package name is not in the format `scope{ESCAPED_SEPARATOR}name`")] - Invalid, + #[error("package name {0} is not in the format `scope{ESCAPED_SEPARATOR}name`")] + Invalid(String), /// The package name is invalid #[error("invalid package name")] - InvalidName(#[from] PackageNameValidationError), + InvalidName(#[from] E), +} + +/// An error that occurred while parsing a package name +#[derive(Debug, Error)] +pub enum FromStrPackageNameParseError { + /// This is not a valid package name + #[error("package name {0} is not in the format `scope{SEPARATOR}name`")] + Invalid(String), + + /// The package name is invalid + #[error("invalid name part")] + InvalidPart(#[from] E), +} + +const SEPARATOR: char = '/'; +const ESCAPED_SEPARATOR: char = '+'; + +macro_rules! name_impl { + ($Name:ident, $Error:ident, $Visitor:ident, $validate:expr, $prefix:expr) => { + impl $Name { + /// Creates a new package name + pub fn new(scope: &str, name: &str) -> Result { + $validate(scope)?; + $validate(name)?; + + Ok(Self(scope.to_string(), name.to_string())) + } + + /// Parses an escaped package name + pub fn from_escaped(s: &str) -> Result> { + if !s.starts_with($prefix) { + return Err(EscapedPackageNameError::MissingPrefix($prefix.to_string())); + } + + let (scope, name) = &s[$prefix.len()..] + .split_once(ESCAPED_SEPARATOR) + .ok_or_else(|| EscapedPackageNameError::Invalid(s.to_string()))?; + Ok(Self::new(scope, name)?) + } + + /// Gets the scope of the package name + pub fn scope(&self) -> &str { + &self.0 + } + + /// Gets the name of the package name + pub fn name(&self) -> &str { + &self.1 + } + + /// Gets the escaped form (for use in file names, etc.) of the package name + pub fn escaped(&self) -> String { + format!("{}{}{ESCAPED_SEPARATOR}{}", $prefix, self.0, self.1) + } + + /// Gets the parts of the package name + pub fn parts(&self) -> (&str, &str) { + (&self.0, &self.1) + } + + /// Returns the prefix for this package name + pub fn prefix() -> &'static str { + $prefix + } + } + + impl Display for $Name { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}{SEPARATOR}{}", $prefix, self.0, self.1) + } + } + + impl FromStr for $Name { + type Err = FromStrPackageNameParseError<$Error>; + + fn from_str(s: &str) -> Result { + let len = if s.starts_with($prefix) { + $prefix.len() + } else { + 0 + }; + + let parts: Vec<&str> = s[len..].split(SEPARATOR).collect(); + if parts.len() != 2 { + return Err(FromStrPackageNameParseError::Invalid(s.to_string())); + } + + Ok($Name::new(parts[0], parts[1])?) + } + } + + impl Serialize for $Name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.to_string()) + } + } + + impl<'de> Visitor<'de> for $Visitor { + type Value = $Name; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + formatter, + "a string in the format `{}scope{SEPARATOR}name`", + $prefix + ) + } + + fn visit_str(self, v: &str) -> Result { + v.parse().map_err(|e| E::custom(e)) + } + } + + impl<'de> Deserialize<'de> for $Name { + fn deserialize>( + deserializer: D, + ) -> Result<$Name, D::Error> { + deserializer.deserialize_str($Visitor) + } + } + }; +} + +struct StandardPackageNameVisitor; +#[cfg(feature = "wally")] +struct WallyPackageNameVisitor; + +/// A package name +#[derive(Serialize, Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[serde(untagged)] +pub enum PackageName { + /// A standard package name + Standard(StandardPackageName), + /// A wally package name + #[cfg(feature = "wally")] + Wally(WallyPackageName), } impl PackageName { - /// Creates a new package name - pub fn new(scope: &str, name: &str) -> Result { - validate_part(scope)?; - validate_part(name)?; - - Ok(Self(scope.to_string(), name.to_string())) - } - - /// Parses an escaped package name - pub fn from_escaped(s: &str) -> Result { - let (scope, name) = s - .split_once(ESCAPED_SEPARATOR) - .ok_or(EscapedPackageNameError::Invalid)?; - Ok(Self::new(scope, name)?) - } - /// Gets the scope of the package name pub fn scope(&self) -> &str { - &self.0 + match self { + PackageName::Standard(name) => name.scope(), + #[cfg(feature = "wally")] + PackageName::Wally(name) => name.scope(), + } } /// Gets the name of the package name pub fn name(&self) -> &str { - &self.1 + match self { + PackageName::Standard(name) => name.name(), + #[cfg(feature = "wally")] + PackageName::Wally(name) => name.name(), + } } /// Gets the escaped form (for use in file names, etc.) of the package name pub fn escaped(&self) -> String { - format!("{}{ESCAPED_SEPARATOR}{}", self.0, self.1) + match self { + PackageName::Standard(name) => name.escaped(), + #[cfg(feature = "wally")] + PackageName::Wally(name) => name.escaped(), + } } /// Gets the parts of the package name pub fn parts(&self) -> (&str, &str) { - (&self.0, &self.1) + match self { + PackageName::Standard(name) => name.parts(), + #[cfg(feature = "wally")] + PackageName::Wally(name) => name.parts(), + } + } + + /// Returns the prefix for this package name + pub fn prefix(&self) -> &'static str { + match self { + PackageName::Standard(_) => StandardPackageName::prefix(), + #[cfg(feature = "wally")] + PackageName::Wally(_) => WallyPackageName::prefix(), + } } } impl Display for PackageName { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}{SEPARATOR}{}", self.0, self.1) + match self { + PackageName::Standard(name) => write!(f, "{name}"), + #[cfg(feature = "wally")] + PackageName::Wally(name) => write!(f, "{name}"), + } + } +} + +impl From for PackageName { + fn from(name: StandardPackageName) -> Self { + PackageName::Standard(name) + } +} + +#[cfg(feature = "wally")] +impl From for PackageName { + fn from(name: WallyPackageName) -> Self { + PackageName::Wally(name) + } +} + +name_impl!( + StandardPackageName, + StandardPackageNameValidationError, + StandardPackageNameVisitor, + validate_part, + "" +); + +#[cfg(feature = "wally")] +name_impl!( + WallyPackageName, + WallyPackageNameValidationError, + WallyPackageNameVisitor, + validate_wally_part, + "wally#" +); + +impl<'de> Deserialize<'de> for PackageName { + fn deserialize>(deserializer: D) -> Result { + let s = String::deserialize(deserializer)?; + + cfg_if! { + if #[cfg(feature = "wally")] { + if s.starts_with(WallyPackageName::prefix()) { + return Ok(PackageName::Wally( + WallyPackageName::deserialize(s.into_deserializer())?, + )); + } + } + } + + Ok(PackageName::Standard(StandardPackageName::deserialize( + s.into_deserializer(), + )?)) } } /// An error that occurred while parsing a package name #[derive(Debug, Error)] -pub enum FromStrPackageNameParseError { - /// This is not a valid package name - #[error("package name is not in the format `scope{SEPARATOR}name`")] - Invalid, - /// The package name is invalid - #[error("invalid name part")] - InvalidPart(#[from] PackageNameValidationError), +pub enum FromStrPackageNameError { + /// Error parsing the package name as a standard package name + #[error("error parsing standard package name")] + Standard(#[from] FromStrPackageNameParseError), + + /// Error parsing the package name as a wally package name + #[cfg(feature = "wally")] + #[error("error parsing wally package name")] + Wally(#[from] FromStrPackageNameParseError), } impl FromStr for PackageName { - type Err = FromStrPackageNameParseError; + type Err = FromStrPackageNameError; fn from_str(s: &str) -> Result { - let parts: Vec<&str> = s.split(SEPARATOR).collect(); - if parts.len() != 2 { - return Err(FromStrPackageNameParseError::Invalid); + cfg_if! { + if #[cfg(feature = "wally")] { + if s.starts_with(WallyPackageName::prefix()) { + return Ok(PackageName::Wally(WallyPackageName::from_str(s)?)); + } + } } - Ok(PackageName::new(parts[0], parts[1])?) + Ok(PackageName::Standard(StandardPackageName::from_str(s)?)) } } -impl Serialize for PackageName { - fn serialize(&self, serializer: S) -> Result { - serializer.serialize_str(&self.to_string()) - } +/// An error that occurred while parsing an escaped package name +#[derive(Debug, Error)] +pub enum FromEscapedStrPackageNameError { + /// Error parsing the package name as a standard package name + #[error("error parsing standard package name")] + Standard(#[from] EscapedPackageNameError), + + /// Error parsing the package name as a wally package name + #[cfg(feature = "wally")] + #[error("error parsing wally package name")] + Wally(#[from] EscapedPackageNameError), } -struct PackageNameVisitor; +impl PackageName { + /// Like `from_str`, but for escaped package names + pub fn from_escaped_str(s: &str) -> Result { + cfg_if! { + if #[cfg(feature = "wally")] { + if s.starts_with(WallyPackageName::prefix()) { + return Ok(PackageName::Wally(WallyPackageName::from_escaped(s)?)); + } + } + } -impl<'de> Visitor<'de> for PackageNameVisitor { - type Value = PackageName; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(formatter, "a string in the format `scope{SEPARATOR}name`") - } - - fn visit_str(self, v: &str) -> Result { - v.parse().map_err(|e| E::custom(e)) - } -} - -impl<'de> Deserialize<'de> for PackageName { - fn deserialize>(deserializer: D) -> Result { - deserializer.deserialize_str(PackageNameVisitor) + Ok(PackageName::Standard(StandardPackageName::from_escaped(s)?)) } } diff --git a/src/patches.rs b/src/patches.rs index d56e60e..2097b8d 100644 --- a/src/patches.rs +++ b/src/patches.rs @@ -9,8 +9,10 @@ use semver::Version; use thiserror::Error; use crate::{ - dependencies::resolution::ResolvedVersionsMap, index::Index, package_name::PackageName, - project::Project, PATCHES_FOLDER, + dependencies::resolution::ResolvedVersionsMap, + package_name::{FromEscapedStrPackageNameError, PackageName}, + project::Project, + PATCHES_FOLDER, }; fn make_signature<'a>() -> Result, git2::Error> { @@ -106,11 +108,11 @@ pub enum ApplyPatchesError { /// An error that occurred because a patch name was malformed #[error("malformed patch name {0}")] - MalformedPatch(String), + MalformedPatchName(String), /// An error that occurred while parsing a package name #[error("failed to parse package name {0}")] - PackageNameParse(#[from] crate::package_name::EscapedPackageNameError), + PackageNameParse(#[from] FromEscapedStrPackageNameError), /// An error that occurred while getting a file stem #[error("failed to get file stem")] @@ -137,7 +139,7 @@ pub enum ApplyPatchesError { StripPrefixFail(#[from] std::path::StripPrefixError), } -impl Project { +impl Project { /// Applies patches for the project pub fn apply_patches(&self, map: &ResolvedVersionsMap) -> Result<(), ApplyPatchesError> { let patches_dir = self.path().join(PATCHES_FOLDER); @@ -153,27 +155,28 @@ impl Project { let path = file.path(); - let dir_name = path + let file_name = path .file_name() .ok_or_else(|| ApplyPatchesError::FileNameFail(path.clone()))?; - let dir_name = dir_name.to_str().ok_or(ApplyPatchesError::ToStringFail)?; + let file_name = file_name.to_str().ok_or(ApplyPatchesError::ToStringFail)?; - let (package_name, version) = dir_name + let (package_name, version) = file_name .strip_suffix(".patch") - .unwrap_or(dir_name) + .unwrap_or(file_name) .split_once('@') - .ok_or_else(|| ApplyPatchesError::MalformedPatch(dir_name.to_string()))?; + .ok_or_else(|| ApplyPatchesError::MalformedPatchName(file_name.to_string()))?; + + let package_name = PackageName::from_escaped_str(package_name)?; - let package_name = PackageName::from_escaped(package_name)?; let version = Version::parse(version)?; - let versions = map + let resolved_pkg = map .get(&package_name) - .ok_or_else(|| ApplyPatchesError::PackageNotFound(package_name.clone()))?; - - let resolved_pkg = versions.get(&version).ok_or_else(|| { - ApplyPatchesError::VersionNotFound(version.clone(), package_name.clone()) - })?; + .ok_or_else(|| ApplyPatchesError::PackageNotFound(package_name.clone()))? + .get(&version) + .ok_or_else(|| { + ApplyPatchesError::VersionNotFound(version.clone(), package_name.clone()) + })?; debug!("resolved package {package_name}@{version} to {resolved_pkg}"); diff --git a/src/project.rs b/src/project.rs index cfdf724..ff7ce47 100644 --- a/src/project.rs +++ b/src/project.rs @@ -1,27 +1,33 @@ +use log::{error, warn}; use std::{ + collections::HashMap, fmt::Debug, fs::{read, File}, path::{Path, PathBuf}, }; -use thiserror::Error; -use crate::dependencies::DownloadError; -use crate::index::Index; -use crate::linking_file::LinkingDependenciesError; +use thiserror::Error; +use url::Url; + use crate::{ - dependencies::resolution::ResolvedVersionsMap, + dependencies::{resolution::ResolvedVersionsMap, DownloadError, UrlResolveError}, + index::Index, + linking_file::LinkingDependenciesError, manifest::{Manifest, ManifestReadError}, LOCKFILE_FILE_NAME, }; +/// A map of indices +pub type Indices = HashMap>; + /// A pesde project -#[derive(Clone, Debug)] -pub struct Project { +#[derive(Debug)] +pub struct Project { path: PathBuf, cache_path: PathBuf, - index: I, + indices: Indices, manifest: Manifest, - registry_auth_token: Option, + pub(crate) reqwest_client: reqwest::blocking::Client, } /// Options for installing a project @@ -114,47 +120,141 @@ pub enum InstallProjectError { /// An error that occurred while writing the lockfile #[error("failed to write lockfile")] LockfileSer(#[source] serde_yaml::Error), + + /// An error that occurred while resolving the url of a package + #[error("failed to resolve package URL")] + UrlResolve(#[from] UrlResolveError), } -impl Project { +/// The name of the default index to use +pub const DEFAULT_INDEX_NAME: &str = "default"; + +pub(crate) fn get_index<'a>(indices: &'a Indices, index_name: Option<&str>) -> &'a dyn Index { + indices + .get(index_name.unwrap_or(DEFAULT_INDEX_NAME)) + .or_else(|| { + warn!( + "index `{}` not found, using default index", + index_name.unwrap_or("") + ); + indices.get(DEFAULT_INDEX_NAME) + }) + .unwrap() + .as_ref() +} + +pub(crate) fn get_index_by_url<'a>(indices: &'a Indices, url: &Url) -> &'a dyn Index { + indices + .values() + .find(|index| index.url() == url) + .map(|index| index.as_ref()) + .unwrap_or_else(|| get_index(indices, None)) +} + +#[cfg(feature = "wally")] +pub(crate) fn get_wally_index<'a>( + indices: &'a mut Indices, + url: &Url, + path: Option<&Path>, +) -> Result<&'a crate::index::WallyIndex, crate::index::RefreshError> { + if !indices.contains_key(url.as_str()) { + let default_index = indices.get(DEFAULT_INDEX_NAME).unwrap(); + let default_token = default_index.registry_auth_token().map(|t| t.to_string()); + let default_credentials_fn = default_index.credentials_fn().cloned(); + + let index = crate::index::WallyIndex::new( + url.clone(), + default_token, + path.expect("index should already exist by now"), + default_credentials_fn, + ); + + match index.refresh() { + Ok(_) => { + indices.insert(url.as_str().to_string(), Box::new(index)); + } + Err(e) => { + error!("failed to refresh wally index: {e}"); + return Err(e); + } + } + } + + Ok(indices + .get(url.as_str()) + .unwrap() + .as_any() + .downcast_ref() + .unwrap()) +} + +/// An error that occurred while creating a new project +#[derive(Debug, Error)] +pub enum NewProjectError { + /// A default index was not provided + #[error("default index not provided")] + DefaultIndexNotProvided, +} + +/// An error that occurred while creating a project from a path +#[derive(Debug, Error)] +pub enum ProjectFromPathError { + /// An error that occurred while reading the manifest + #[error("error reading manifest")] + ManifestRead(#[from] ManifestReadError), + + /// An error that occurred while creating the project + #[error("error creating project")] + NewProject(#[from] NewProjectError), +} + +impl Project { /// Creates a new project pub fn new, Q: AsRef>( path: P, cache_path: Q, - index: I, + indices: Indices, manifest: Manifest, - registry_auth_token: Option, - ) -> Self { - Self { + ) -> Result { + if !indices.contains_key(DEFAULT_INDEX_NAME) { + return Err(NewProjectError::DefaultIndexNotProvided); + } + + Ok(Self { path: path.as_ref().to_path_buf(), cache_path: cache_path.as_ref().to_path_buf(), - index, + indices, manifest, - registry_auth_token, - } + reqwest_client: reqwest::blocking::ClientBuilder::new() + .user_agent(concat!( + env!("CARGO_PKG_NAME"), + "/", + env!("CARGO_PKG_VERSION") + )) + .build() + .unwrap(), + }) } /// Creates a new project from a path (manifest will be read from the path) pub fn from_path, Q: AsRef>( path: P, cache_path: Q, - index: I, - registry_auth_token: Option, - ) -> Result { + indices: Indices, + ) -> Result { let manifest = Manifest::from_path(path.as_ref())?; - Ok(Self::new( - path, - cache_path, - index, - manifest, - registry_auth_token, - )) + Ok(Self::new(path, cache_path, indices, manifest)?) } - /// Returns the index of the project - pub fn index(&self) -> &I { - &self.index + /// Returns the indices of the project + pub fn indices(&self) -> &HashMap> { + &self.indices + } + + #[cfg(feature = "wally")] + pub(crate) fn indices_mut(&mut self) -> &mut HashMap> { + &mut self.indices } /// Returns the manifest of the project @@ -172,11 +272,6 @@ impl Project { &self.path } - /// Returns the registry auth token of the project - pub fn registry_auth_token(&self) -> Option<&String> { - self.registry_auth_token.as_ref() - } - /// Returns the lockfile of the project pub fn lockfile(&self) -> Result, ReadLockfileError> { let lockfile_path = self.path.join(LOCKFILE_FILE_NAME); @@ -193,16 +288,18 @@ impl Project { } /// Downloads the project's dependencies, applies patches, and links the dependencies - pub fn install(&self, install_options: InstallOptions) -> Result<(), InstallProjectError> { + pub fn install(&mut self, install_options: InstallOptions) -> Result<(), InstallProjectError> { let map = match install_options.resolved_versions_map { Some(map) => map, - None => self - .manifest - .dependency_tree(self, install_options.locked)?, + None => { + let manifest = self.manifest.clone(); + + manifest.dependency_tree(self, install_options.locked)? + } }; if install_options.auto_download { - self.download(&map)?.wait()?; + self.download(map.clone())?.wait()?; } self.apply_patches(&map)?; diff --git a/tests/prelude.rs b/tests/prelude.rs index 38ff3fe..fdf52c9 100644 --- a/tests/prelude.rs +++ b/tests/prelude.rs @@ -1,7 +1,9 @@ use std::{ + any::Any, collections::{BTreeSet, HashMap}, sync::Arc, }; +use url::Url; use pesde::{ index::{ @@ -13,9 +15,19 @@ use pesde::{ }; /// An in-memory implementation of the [`Index`] trait. Used for testing. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct InMemoryIndex { packages: HashMap, IndexFile)>, + url: Url, +} + +impl Default for InMemoryIndex { + fn default() -> Self { + Self { + packages: HashMap::new(), + url: Url::parse("https://example.com").unwrap(), + } + } } impl InMemoryIndex { @@ -78,7 +90,7 @@ impl Index for InMemoryIndex { let package = self.packages.get_mut(scope).unwrap(); - let entry: IndexFileEntry = manifest.clone().into(); + let entry: IndexFileEntry = manifest.clone().try_into()?; package.1.insert(entry.clone()); Ok(Some(entry)) @@ -87,7 +99,7 @@ impl Index for InMemoryIndex { fn config(&self) -> Result { Ok(IndexConfig { download: None, - api: "http://127.0.0.1:8080".to_string(), + api: "http://127.0.0.1:8080".parse().unwrap(), github_oauth_client_id: "".to_string(), custom_registry_allowed: false, git_allowed: false, @@ -97,4 +109,12 @@ impl Index for InMemoryIndex { fn credentials_fn(&self) -> Option<&Arc> { None } + + fn url(&self) -> &Url { + &self.url + } + + fn as_any(&self) -> &dyn Any { + self + } } diff --git a/tests/resolver.rs b/tests/resolver.rs index 2aecbe6..e1a7eda 100644 --- a/tests/resolver.rs +++ b/tests/resolver.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use semver::Version; use tempfile::tempdir; @@ -9,9 +9,10 @@ use pesde::{ resolution::ResolvedPackage, DependencySpecifier, PackageRef, }, + index::Index, manifest::{DependencyType, Manifest, Realm}, - package_name::PackageName, - project::Project, + package_name::StandardPackageName, + project::{Project, DEFAULT_INDEX_NAME}, }; use prelude::*; @@ -30,7 +31,7 @@ fn test_resolves_package() { let description = "test package"; - let pkg_name = PackageName::new("test", "test").unwrap(); + let pkg_name = StandardPackageName::new("test", "test").unwrap(); let pkg_manifest = Manifest { name: pkg_name.clone(), @@ -39,6 +40,7 @@ fn test_resolves_package() { path_style: Default::default(), private: true, realm: None, + indices: Default::default(), dependencies: vec![], peer_dependencies: vec![], description: Some(description.to_string()), @@ -52,18 +54,20 @@ fn test_resolves_package() { let index = index .with_scope(pkg_name.scope(), BTreeSet::from([0])) - .with_package(pkg_name.scope(), pkg_manifest.into()) - .with_package(pkg_name.scope(), pkg_2_manifest.into()); + .with_package(pkg_name.scope(), pkg_manifest.try_into().unwrap()) + .with_package(pkg_name.scope(), pkg_2_manifest.try_into().unwrap()); let specifier = DependencySpecifier::Registry(RegistryDependencySpecifier { name: pkg_name.clone(), version: format!("={version_str}").parse().unwrap(), realm: None, + index: DEFAULT_INDEX_NAME.to_string(), }); let specifier_2 = DependencySpecifier::Registry(RegistryDependencySpecifier { name: pkg_name.clone(), version: format!(">{version_str}").parse().unwrap(), realm: None, + index: DEFAULT_INDEX_NAME.to_string(), }); let user_manifest = Manifest { @@ -73,6 +77,7 @@ fn test_resolves_package() { path_style: Default::default(), private: true, realm: None, + indices: Default::default(), dependencies: vec![specifier.clone()], peer_dependencies: vec![specifier_2.clone()], description: Some(description.to_string()), @@ -81,11 +86,21 @@ fn test_resolves_package() { repository: None, }; - let project = Project::new(&dir_path, &dir_path, index, user_manifest, None); + let mut project = Project::new( + &dir_path, + &dir_path, + HashMap::from([( + DEFAULT_INDEX_NAME.to_string(), + Box::new(index.clone()) as Box, + )]), + user_manifest, + ) + .unwrap(); - let tree = project.manifest().dependency_tree(&project, false).unwrap(); + let manifest = project.manifest().clone(); + let tree = manifest.dependency_tree(&mut project, false).unwrap(); assert_eq!(tree.len(), 1); - let versions = tree.get(&pkg_name).unwrap(); + let versions = tree.get(&pkg_name.clone().into()).unwrap(); assert_eq!(versions.len(), 2); let resolved_pkg = versions.get(&version).unwrap(); assert_eq!( @@ -94,6 +109,7 @@ fn test_resolves_package() { pkg_ref: PackageRef::Registry(RegistryPackageRef { name: pkg_name.clone(), version: version.clone(), + index_url: index.url().clone(), }), specifier, dependencies: Default::default(), @@ -109,6 +125,7 @@ fn test_resolves_package() { pkg_ref: PackageRef::Registry(RegistryPackageRef { name: pkg_name.clone(), version: version_2.clone(), + index_url: index.url().clone(), }), specifier: specifier_2, dependencies: Default::default(),