diff --git a/Cargo.lock b/Cargo.lock index e41f9e3..649a4a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,7 +92,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -130,7 +130,7 @@ dependencies = [ "parse-size", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -247,7 +247,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -492,7 +492,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -527,7 +527,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -718,9 +718,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.14" +version = "1.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d2eb3cd3d1bf4529e31c215ee6f93ec5a3d536d9f578f93d9d33ee19562932" +checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" dependencies = [ "jobserver", "libc", @@ -801,7 +801,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -856,9 +856,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "convert_case" @@ -866,6 +866,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.16.2" @@ -1022,7 +1031,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1033,7 +1042,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1112,7 +1121,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1121,11 +1130,11 @@ version = "0.99.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ - "convert_case", + "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1168,7 +1177,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1240,7 +1249,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1307,15 +1316,15 @@ checksum = "a2a2b11eda1d40935b26cf18f6833c526845ae8c41e58d09af6adeb6f0269183" [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "filetime" -version = "0.2.24" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ "cfg-if", "libc", @@ -1337,9 +1346,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", "miniz_oxide 0.8.0", @@ -1481,7 +1490,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1644,7 +1653,7 @@ dependencies = [ "gix-utils", "itoa", "thiserror", - "winnow 0.6.18", + "winnow", ] [[package]] @@ -1726,7 +1735,7 @@ dependencies = [ "smallvec", "thiserror", "unicode-bom", - "winnow 0.6.18", + "winnow", ] [[package]] @@ -1968,7 +1977,7 @@ dependencies = [ "itoa", "smallvec", "thiserror", - "winnow 0.6.18", + "winnow", ] [[package]] @@ -2091,7 +2100,7 @@ dependencies = [ "gix-utils", "maybe-async", "thiserror", - "winnow 0.6.18", + "winnow", ] [[package]] @@ -2123,7 +2132,7 @@ dependencies = [ "gix-validate", "memmap2", "thiserror", - "winnow 0.6.18", + "winnow", ] [[package]] @@ -2687,9 +2696,9 @@ dependencies = [ [[package]] name = "indicatif-log-bridge" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2963046f28a204e3e3fd7e754fd90a6235da05b5378f24707ff0ec9513725ce3" +checksum = "63703cf9069b85dbe6fe26e1c5230d013dee99d3559cd3d02ba39e099ef7ab02" dependencies = [ "indicatif", "log", @@ -2803,9 +2812,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jiff" -version = "0.1.8" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2b7379a75544c94b3da32821b0bf41f9062e9970e23b78cc577d0d89676d16" +checksum = "362be9c702bada57298130d0565e9c389e6d4024a541692f01819e21abc3e26a" dependencies = [ "jiff-tzdb-platform", "windows-sys 0.59.0", @@ -2852,7 +2861,6 @@ checksum = "73b9af47ded4df3067484d7d45758ca2b36bd083bf6d024c2952bbd8af1cdaa4" dependencies = [ "byteorder", "dbus-secret-service", - "linux-keyutils", "secret-service", "security-framework", "windows-sys 0.59.0", @@ -2947,9 +2955,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.19" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc53a7799a7496ebc9fd29f31f7df80e83c9bda5299768af5f9e59eeea74647" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "libc", @@ -2957,16 +2965,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "linux-keyutils" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "761e49ec5fd8a5a463f9b84e877c373d888935b71c6be78f3767fe2ae6bed18e" -dependencies = [ - "bitflags 2.6.0", - "libc", -] - [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -3045,7 +3043,7 @@ checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -3375,7 +3373,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -3528,7 +3526,7 @@ dependencies = [ "thiserror", "threadpool", "toml", - "toml_edit 0.22.20", + "toml_edit", "url", "winreg", "zip", @@ -3543,6 +3541,7 @@ dependencies = [ "actix-multipart", "actix-web", "chrono", + "convert_case 0.6.0", "dotenvy", "flate2", "futures", @@ -3559,8 +3558,11 @@ dependencies = [ "sentry-log", "serde", "serde_json", + "serde_yaml", + "sha2", "tantivy", "tar", + "tempfile", "thiserror", "toml", "url", @@ -3583,7 +3585,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -3663,11 +3665,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.21.1", + "toml_edit", ] [[package]] @@ -3995,18 +3997,18 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f" dependencies = [ "bitflags 2.6.0", "errno", @@ -4047,9 +4049,9 @@ checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" dependencies = [ "ring", "rustls-pki-types", @@ -4287,29 +4289,29 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.208" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" +checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.208" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" +checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] name = "serde_json" -version = "1.0.125" +version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" +checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" dependencies = [ "itoa", "memchr", @@ -4334,7 +4336,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -4385,7 +4387,20 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.4.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", ] [[package]] @@ -4559,9 +4574,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.75" +version = "2.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" +checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" dependencies = [ "proc-macro2", "quote", @@ -4789,7 +4804,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -4917,7 +4932,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.20", + "toml_edit", ] [[package]] @@ -4929,17 +4944,6 @@ dependencies = [ "serde", ] -[[package]] -name = "toml_edit" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" -dependencies = [ - "indexmap 2.4.0", - "toml_datetime", - "winnow 0.5.40", -] - [[package]] name = "toml_edit" version = "0.22.20" @@ -4950,7 +4954,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.18", + "winnow", ] [[package]] @@ -5000,7 +5004,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -5102,6 +5106,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -5220,7 +5230,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", "wasm-bindgen-shared", ] @@ -5254,7 +5264,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5512,15 +5522,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "0.6.18" @@ -5602,7 +5603,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", "zvariant_utils", ] @@ -5635,7 +5636,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -5655,7 +5656,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -5751,7 +5752,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", "zvariant_utils", ] @@ -5763,5 +5764,5 @@ checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] diff --git a/Cargo.toml b/Cargo.toml index 6e5a633..4ee0141 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ serde_json = { version = "1.0.122", optional = true } anyhow = { version = "1.0.86", optional = true } open = { version = "5.3.0", optional = true } -keyring = { version = "3.0.5", features = ["crypto-rust", "windows-native", "apple-native", "linux-native"], optional = true } +keyring = { version = "3.0.5", features = ["crypto-rust", "windows-native", "apple-native", "sync-secret-service"], optional = true } colored = { version = "2.1.0", optional = true } toml_edit = { version = "0.22.20", optional = true } clap = { version = "4.5.13", features = ["derive"], optional = true } diff --git a/registry/Cargo.toml b/registry/Cargo.toml index 9d5c295..dbd95a8 100644 --- a/registry/Cargo.toml +++ b/registry/Cargo.toml @@ -17,19 +17,26 @@ semver = "1.0.23" chrono = { version = "0.4.38", features = ["serde"] } url = "2.5.2" futures = "0.3.30" +tempfile = "3.12.0" git2 = "0.19.0" -gix = { version = "0.66.0", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls", "credentials"] } +gix = { version = "0.66.0", default-features = false, features = [ + "blocking-http-transport-reqwest-rust-tls", + "credentials", +] } -serde = "1.0.206" -serde_json = "1.0.124" -toml = "0.8.16" +serde = "1.0.209" +serde_json = "1.0.127" +serde_yaml = "0.9.34" +toml = "0.8.19" +convert_case = "0.6.0" +sha2 = "0.10.8" rusty-s3 = "0.5.0" -reqwest = { version = "0.12.5", features = ["json", "rustls-tls"] } +reqwest = { version = "0.12.7", features = ["json", "rustls-tls"] } tar = "0.4.41" -flate2 = "1.0.30" +flate2 = "1.0.33" log = "0.4.22" pretty_env_logger = "0.5.0" @@ -38,4 +45,10 @@ sentry = "0.34.0" sentry-log = "0.34.0" sentry-actix = "0.34.0" -pesde = { path = "..", features = ["roblox", "lune", "luau", "wally-compat", "git2"] } +pesde = { path = "..", features = [ + "roblox", + "lune", + "luau", + "wally-compat", + "git2", +] } diff --git a/registry/src/endpoints/package_version.rs b/registry/src/endpoints/package_version.rs index 67b98bf..997ad31 100644 --- a/registry/src/endpoints/package_version.rs +++ b/registry/src/endpoints/package_version.rs @@ -6,16 +6,18 @@ use rusty_s3::{actions::GetObject, S3Action}; use semver::Version; use serde::{Deserialize, Deserializer}; +use crate::{ + error::Error, + package::{s3_doc_name, s3_name, PackageResponse, S3_SIGN_DURATION}, + AppState, +}; use pesde::{ manifest::target::TargetKind, names::PackageName, - source::{git_index::GitBasedSource, pesde::IndexFile}, -}; - -use crate::{ - error::Error, - package::{s3_name, PackageResponse, S3_SIGN_DURATION}, - AppState, + source::{ + git_index::GitBasedSource, + pesde::{DocEntryKind, IndexFile}, + }, }; #[derive(Debug)] @@ -62,10 +64,16 @@ impl<'de> Deserialize<'de> for TargetRequest { } } +#[derive(Debug, Deserialize)] +pub struct Query { + doc: Option, +} + pub async fn get_package_version( request: HttpRequest, app_state: web::Data, path: web::Path<(PackageName, VersionRequest, TargetRequest)>, + query: web::Query, ) -> Result { let (name, version, target) = path.into_inner(); @@ -110,6 +118,36 @@ pub async fn get_package_version( return Ok(HttpResponse::NotFound().finish()); }; + if let Some(doc_name) = query.doc.as_deref() { + let hash = 'finder: { + let mut hash = entry.docs.iter().map(|doc| &doc.kind).collect::>(); + while let Some(doc) = hash.pop() { + match doc { + DocEntryKind::Page { name, hash } if name == doc_name => { + break 'finder hash.clone() + } + DocEntryKind::Category { items, .. } => { + hash.extend(items.iter().map(|item| &item.kind)) + } + _ => continue, + }; + } + + return Ok(HttpResponse::NotFound().finish()); + }; + + let object_url = GetObject::new( + &app_state.s3_bucket, + Some(&app_state.s3_credentials), + &s3_doc_name(&hash), + ) + .sign(S3_SIGN_DURATION); + + return Ok(HttpResponse::TemporaryRedirect() + .append_header((LOCATION, object_url.as_str())) + .finish()); + } + let accept = request .headers() .get(ACCEPT) @@ -133,7 +171,7 @@ pub async fn get_package_version( .finish()); } - Ok(HttpResponse::Ok().json(PackageResponse { + let response = PackageResponse { name: name.to_string(), version: v_id.version().to_string(), targets, @@ -142,5 +180,10 @@ pub async fn get_package_version( license: entry.license.clone().unwrap_or_default(), authors: entry.authors.clone(), repository: entry.repository.clone().map(|url| url.to_string()), - })) + }; + + let mut value = serde_json::to_value(response)?; + value["docs"] = serde_json::to_value(entry.docs.clone())?; + + Ok(HttpResponse::Ok().json(value)) } diff --git a/registry/src/endpoints/publish_version.rs b/registry/src/endpoints/publish_version.rs index 829ec4e..337a219 100644 --- a/registry/src/endpoints/publish_version.rs +++ b/registry/src/endpoints/publish_version.rs @@ -1,37 +1,39 @@ -use std::{ - collections::BTreeSet, - io::{Cursor, Read, Write}, -}; - use actix_multipart::Multipart; use actix_web::{web, HttpResponse, Responder}; +use convert_case::{Case, Casing}; use flate2::read::GzDecoder; -use futures::StreamExt; +use futures::{future::join_all, StreamExt}; use git2::{Remote, Repository, Signature}; use reqwest::header::{CONTENT_ENCODING, CONTENT_TYPE}; use rusty_s3::{actions::PutObject, S3Action}; -use tar::Archive; - -use pesde::{ - manifest::Manifest, - source::{ - git_index::GitBasedSource, - pesde::{IndexFile, IndexFileEntry, ScopeInfo, SCOPE_INFO_FILE}, - specifiers::DependencySpecifiers, - version_id::VersionId, - IGNORED_DIRS, IGNORED_FILES, - }, - MANIFEST_FILE_NAME, +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::{ + collections::{BTreeSet, HashMap}, + fs::read_dir, + io::{Cursor, Read, Write}, }; +use tar::Archive; use crate::{ auth::UserId, benv, error::{Error, ErrorResponse}, - package::{s3_name, S3_SIGN_DURATION}, + package::{s3_doc_name, s3_name, S3_SIGN_DURATION}, search::update_version, AppState, }; +use pesde::{ + manifest::Manifest, + source::{ + git_index::GitBasedSource, + pesde::{DocEntry, DocEntryKind, IndexFile, IndexFileEntry, ScopeInfo, SCOPE_INFO_FILE}, + specifiers::DependencySpecifiers, + version_id::VersionId, + IGNORED_DIRS, IGNORED_FILES, + }, + MANIFEST_FILE_NAME, +}; fn signature<'a>() -> Signature<'a> { Signature::now( @@ -57,6 +59,16 @@ fn get_refspec(repo: &Repository, remote: &mut Remote) -> Result, + #[serde(default)] + sidebar_position: Option, + #[serde(default)] + collapsed: bool, +} + pub async fn publish_package( app_state: web::Data, mut body: Multipart, @@ -77,51 +89,181 @@ pub async fn publish_package( .await .map_err(|_| Error::InvalidArchive)? .map_err(|_| Error::InvalidArchive)?; - let mut decoder = GzDecoder::new(Cursor::new(&bytes)); - let mut archive = Archive::new(&mut decoder); - let entries = archive.entries()?; + let package_dir = tempfile::tempdir()?; + + { + let mut decoder = GzDecoder::new(Cursor::new(&bytes)); + let mut archive = Archive::new(&mut decoder); + + archive.unpack(package_dir.path())?; + } + let mut manifest = None::; let mut readme = None::>; + let mut docs = BTreeSet::new(); + let mut docs_pages = HashMap::new(); - for entry in entries { - let mut entry = entry?; - let path = entry.path()?; + for entry in read_dir(package_dir.path())? { + let entry = entry?; + let file_name = entry + .file_name() + .to_str() + .ok_or(Error::InvalidArchive)? + .to_string(); - if entry.header().entry_type().is_dir() { - if path.components().next().is_some_and(|ct| { - ct.as_os_str() - .to_str() - .map_or(true, |s| IGNORED_DIRS.contains(&s)) - }) { + if entry.file_type()?.is_dir() { + if IGNORED_DIRS.contains(&file_name.as_str()) { return Err(Error::InvalidArchive); } + if file_name == "docs" { + let mut stack = vec![( + BTreeSet::new(), + read_dir(entry.path())?, + None::, + )]; + + 'outer: while let Some((set, iter, category_info)) = stack.last_mut() { + for entry in iter { + let entry = entry?; + let file_name = entry + .file_name() + .to_str() + .ok_or(Error::InvalidArchive)? + .to_string(); + + if entry.file_type()?.is_dir() { + stack.push(( + BTreeSet::new(), + read_dir(entry.path())?, + Some(DocEntryInfo { + label: Some(file_name.to_case(Case::Title)), + ..Default::default() + }), + )); + continue 'outer; + } + + if file_name == "_category_.json" { + let info = std::fs::read_to_string(entry.path())?; + let mut info: DocEntryInfo = serde_json::from_str(&info)?; + let old_info = category_info.take(); + info.label = info.label.or(old_info.and_then(|i| i.label)); + *category_info = Some(info); + continue; + } + + let Some(file_name) = file_name.strip_suffix(".md") else { + continue; + }; + + let content = std::fs::read_to_string(entry.path())?; + let content = content.trim(); + let hash = format!("{:x}", Sha256::digest(content.as_bytes())); + + let mut gz = flate2::read::GzEncoder::new( + Cursor::new(content.as_bytes().to_vec()), + flate2::Compression::best(), + ); + let mut bytes = vec![]; + gz.read_to_end(&mut bytes)?; + docs_pages.insert(hash.to_string(), bytes); + + let mut lines = content.lines().peekable(); + let front_matter = if lines.peek().filter(|l| **l == "---").is_some() { + lines.next(); // skip the first `---` + + let front_matter = lines + .by_ref() + .take_while(|l| *l != "---") + .collect::>() + .join("\n"); + + lines.next(); // skip the last `---` + + front_matter + } else { + "".to_string() + }; + + let h1 = lines + .find(|l| !l.trim().is_empty()) + .and_then(|l| l.strip_prefix("# ")) + .map(|s| s.to_string()); + + let info: DocEntryInfo = serde_yaml::from_str(&front_matter) + .map_err(|_| Error::InvalidArchive)?; + + set.insert(DocEntry { + label: info.label.or(h1).unwrap_or(file_name.to_case(Case::Title)), + position: info.sidebar_position, + kind: DocEntryKind::Page { + name: entry + .path() + .strip_prefix(package_dir.path().join("docs")) + .unwrap() + .with_extension("") + .to_str() + .ok_or(Error::InvalidArchive)? + // ensure that the path is always using forward slashes + .replace("\\", "/"), + hash, + }, + }); + } + + // should never be None + let (popped, _, category_info) = stack.pop().unwrap(); + docs = popped; + + if let Some((set, _, _)) = stack.last_mut() { + let category_info = category_info.unwrap_or_default(); + + set.insert(DocEntry { + label: category_info.label.unwrap(), + position: category_info.sidebar_position, + kind: DocEntryKind::Category { + items: { + let curr_docs = docs; + docs = BTreeSet::new(); + curr_docs + }, + collapsed: category_info.collapsed, + }, + }); + } + } + } + continue; } - let path = path.to_str().ok_or(Error::InvalidArchive)?; - - if IGNORED_FILES.contains(&path) || ADDITIONAL_FORBIDDEN_FILES.contains(&path) { + if IGNORED_FILES.contains(&file_name.as_str()) { return Err(Error::InvalidArchive); } - if path == MANIFEST_FILE_NAME { - let mut content = String::new(); - entry.read_to_string(&mut content)?; - manifest = Some(toml::de::from_str(&content).map_err(|_| Error::InvalidArchive)?); - } else if path.to_lowercase() == "readme" - || path - .to_lowercase() - .split_once('.') - .filter(|(file, ext)| *file == "readme" && (*ext == "md" || *ext == "txt")) - .is_some() + if ADDITIONAL_FORBIDDEN_FILES.contains(&file_name.as_str()) { + return Err(Error::InvalidArchive); + } + + if file_name == MANIFEST_FILE_NAME { + let content = std::fs::read_to_string(entry.path())?; + + manifest = Some(toml::de::from_str(&content)?); + } else if file_name + .to_lowercase() + .split_once('.') + .filter(|(file, ext)| *file == "readme" && (*ext == "md" || *ext == "txt")) + .is_some() { if readme.is_some() { return Err(Error::InvalidArchive); } - let mut gz = flate2::read::GzEncoder::new(entry, flate2::Compression::best()); + let file = std::fs::File::open(entry.path())?; + + let mut gz = flate2::read::GzEncoder::new(file, flate2::Compression::best()); let mut bytes = vec![]; gz.read_to_end(&mut bytes)?; readme = Some(bytes); @@ -222,6 +364,7 @@ pub async fn publish_package( license: manifest.license.clone(), authors: manifest.authors.clone(), repository: manifest.repository.clone(), + docs, dependencies, }; @@ -314,39 +457,57 @@ pub async fn publish_package( let version_id = VersionId::new(manifest.version.clone(), manifest.target.kind()); - let object_url = PutObject::new( - &app_state.s3_bucket, - Some(&app_state.s3_credentials), - &s3_name(&manifest.name, &version_id, false), + join_all( + std::iter::once({ + let object_url = PutObject::new( + &app_state.s3_bucket, + Some(&app_state.s3_credentials), + &s3_name(&manifest.name, &version_id, false), + ) + .sign(S3_SIGN_DURATION); + + app_state + .reqwest_client + .put(object_url) + .header(CONTENT_TYPE, "application/gzip") + .header(CONTENT_ENCODING, "gzip") + .body(bytes) + .send() + }) + .chain(docs_pages.into_iter().map(|(hash, content)| { + let object_url = PutObject::new( + &app_state.s3_bucket, + Some(&app_state.s3_credentials), + &s3_doc_name(&hash), + ) + .sign(S3_SIGN_DURATION); + + app_state + .reqwest_client + .put(object_url) + .header(CONTENT_TYPE, "text/plain") + .header(CONTENT_ENCODING, "gzip") + .body(content) + .send() + })) + .chain(readme.map(|readme| { + let object_url = PutObject::new( + &app_state.s3_bucket, + Some(&app_state.s3_credentials), + &s3_name(&manifest.name, &version_id, true), + ) + .sign(S3_SIGN_DURATION); + + app_state + .reqwest_client + .put(object_url) + .header(CONTENT_TYPE, "text/plain") + .header(CONTENT_ENCODING, "gzip") + .body(readme) + .send() + })), ) - .sign(S3_SIGN_DURATION); - - app_state - .reqwest_client - .put(object_url) - .header(CONTENT_TYPE, "application/gzip") - .header(CONTENT_ENCODING, "gzip") - .body(bytes) - .send() - .await?; - - if let Some(readme) = readme { - let object_url = PutObject::new( - &app_state.s3_bucket, - Some(&app_state.s3_credentials), - &s3_name(&manifest.name, &version_id, true), - ) - .sign(S3_SIGN_DURATION); - - app_state - .reqwest_client - .put(object_url) - .header(CONTENT_TYPE, "text/plain") - .header(CONTENT_ENCODING, "gzip") - .body(readme) - .send() - .await?; - } + .await; Ok(HttpResponse::Ok().body(format!( "published {}@{} {}", diff --git a/registry/src/package.rs b/registry/src/package.rs index 3d18950..6eb1045 100644 --- a/registry/src/package.rs +++ b/registry/src/package.rs @@ -11,13 +11,17 @@ pub const S3_SIGN_DURATION: Duration = Duration::from_secs(60 * 3); pub fn s3_name(name: &PackageName, version_id: &VersionId, is_readme: bool) -> String { format!( - "{}+{}{}", - name.escaped(), - version_id.escaped(), - if is_readme { "+readme.gz" } else { ".tar.gz" } + "{name}/{}/{}/{}.gz", + version_id.version(), + version_id.target(), + if is_readme { "readme" } else { "pkg.tar" }, ) } +pub fn s3_doc_name(doc_hash: &str) -> String { + format!("doc/{}.gz", doc_hash) +} + #[derive(Debug, Serialize, Eq, PartialEq)] pub struct TargetInfo { kind: TargetKind, diff --git a/src/cli/commands/publish.rs b/src/cli/commands/publish.rs index 65c5e9f..05d42f6 100644 --- a/src/cli/commands/publish.rs +++ b/src/cli/commands/publish.rs @@ -93,6 +93,13 @@ impl PublishCommand { ); } + if !manifest.includes.iter().any(|f| f == "docs") { + println!( + "{}: no docs directory in includes, consider adding one", + "warn".yellow().bold() + ); + } + if manifest.includes.remove("default.project.json") { println!( "{}: default.project.json was in includes, this should be generated by the {} script upon dependants installation", diff --git a/src/source/fs.rs b/src/source/fs.rs index b60902d..f4f5d86 100644 --- a/src/source/fs.rs +++ b/src/source/fs.rs @@ -40,7 +40,7 @@ pub(crate) fn store_in_cas>( if !cas_path.exists() { let mut file = std::fs::File::create(&cas_path)?; file.write_all(contents)?; - + // prevent the CAS from being corrupted due to accidental modifications let mut permissions = file.metadata()?.permissions(); permissions.set_readonly(true); diff --git a/src/source/pesde/mod.rs b/src/source/pesde/mod.rs index 4f62dbf..3f1c482 100644 --- a/src/source/pesde/mod.rs +++ b/src/source/pesde/mod.rs @@ -387,6 +387,59 @@ impl IndexConfig { } } +/// An entry in a package's documentation +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(untagged)] +pub enum DocEntryKind { + /// A page in the documentation + Page { + /// The name of the page + name: String, + /// The hash of the page's content + hash: String, + }, + /// A category in the documentation + Category { + /// The items in the section + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + items: BTreeSet, + /// Whether this category is collapsed by default + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + collapsed: bool, + }, +} + +/// An entry in a package's documentation +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct DocEntry { + /// The label for this entry + pub label: String, + /// The position of this entry + #[serde(default, skip_serializing_if = "Option::is_none")] + pub position: Option, + /// The kind of this entry + #[serde(flatten)] + pub kind: DocEntryKind, +} + +impl Ord for DocEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + match (self.position, other.position) { + (Some(l), Some(r)) => l.cmp(&r), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + } + .then(self.label.cmp(&other.label)) + } +} + +impl PartialOrd for DocEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + /// The entry in a package's index file #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct IndexFileEntry { @@ -409,6 +462,10 @@ pub struct IndexFileEntry { #[serde(default, skip_serializing_if = "Option::is_none")] pub repository: Option, + /// The documentation for this package + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub docs: BTreeSet, + /// The dependencies of this package #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub dependencies: BTreeMap,