feat(registry): store package docs

This commit is contained in:
daimond113 2024-09-02 16:49:40 +02:00
parent f631c1deb9
commit bd7e1452b0
No known key found for this signature in database
GPG key ID: 3A8ECE51328B513C
9 changed files with 486 additions and 200 deletions

203
Cargo.lock generated
View file

@ -92,7 +92,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
dependencies = [
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -130,7 +130,7 @@ dependencies = [
"parse-size",
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -247,7 +247,7 @@ dependencies = [
"actix-router",
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -492,7 +492,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -527,7 +527,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -718,9 +718,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.1.14"
version = "1.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50d2eb3cd3d1bf4529e31c215ee6f93ec5a3d536d9f578f93d9d33ee19562932"
checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
dependencies = [
"jobserver",
"libc",
@ -801,7 +801,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -856,9 +856,9 @@ dependencies = [
[[package]]
name = "constant_time_eq"
version = "0.3.0"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
[[package]]
name = "convert_case"
@ -866,6 +866,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
[[package]]
name = "convert_case"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca"
dependencies = [
"unicode-segmentation",
]
[[package]]
name = "cookie"
version = "0.16.2"
@ -1022,7 +1031,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -1033,7 +1042,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
dependencies = [
"darling_core",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -1112,7 +1121,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -1121,11 +1130,11 @@ version = "0.99.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"
dependencies = [
"convert_case",
"convert_case 0.4.0",
"proc-macro2",
"quote",
"rustc_version",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -1168,7 +1177,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -1240,7 +1249,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -1307,15 +1316,15 @@ checksum = "a2a2b11eda1d40935b26cf18f6833c526845ae8c41e58d09af6adeb6f0269183"
[[package]]
name = "fastrand"
version = "2.1.0"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
[[package]]
name = "filetime"
version = "0.2.24"
version = "0.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550"
checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586"
dependencies = [
"cfg-if",
"libc",
@ -1337,9 +1346,9 @@ dependencies = [
[[package]]
name = "flate2"
version = "1.0.32"
version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666"
checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253"
dependencies = [
"crc32fast",
"miniz_oxide 0.8.0",
@ -1481,7 +1490,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -1644,7 +1653,7 @@ dependencies = [
"gix-utils",
"itoa",
"thiserror",
"winnow 0.6.18",
"winnow",
]
[[package]]
@ -1726,7 +1735,7 @@ dependencies = [
"smallvec",
"thiserror",
"unicode-bom",
"winnow 0.6.18",
"winnow",
]
[[package]]
@ -1968,7 +1977,7 @@ dependencies = [
"itoa",
"smallvec",
"thiserror",
"winnow 0.6.18",
"winnow",
]
[[package]]
@ -2091,7 +2100,7 @@ dependencies = [
"gix-utils",
"maybe-async",
"thiserror",
"winnow 0.6.18",
"winnow",
]
[[package]]
@ -2123,7 +2132,7 @@ dependencies = [
"gix-validate",
"memmap2",
"thiserror",
"winnow 0.6.18",
"winnow",
]
[[package]]
@ -2687,9 +2696,9 @@ dependencies = [
[[package]]
name = "indicatif-log-bridge"
version = "0.2.2"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2963046f28a204e3e3fd7e754fd90a6235da05b5378f24707ff0ec9513725ce3"
checksum = "63703cf9069b85dbe6fe26e1c5230d013dee99d3559cd3d02ba39e099ef7ab02"
dependencies = [
"indicatif",
"log",
@ -2803,9 +2812,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
[[package]]
name = "jiff"
version = "0.1.8"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db2b7379a75544c94b3da32821b0bf41f9062e9970e23b78cc577d0d89676d16"
checksum = "362be9c702bada57298130d0565e9c389e6d4024a541692f01819e21abc3e26a"
dependencies = [
"jiff-tzdb-platform",
"windows-sys 0.59.0",
@ -2852,7 +2861,6 @@ checksum = "73b9af47ded4df3067484d7d45758ca2b36bd083bf6d024c2952bbd8af1cdaa4"
dependencies = [
"byteorder",
"dbus-secret-service",
"linux-keyutils",
"secret-service",
"security-framework",
"windows-sys 0.59.0",
@ -2947,9 +2955,9 @@ dependencies = [
[[package]]
name = "libz-sys"
version = "1.1.19"
version = "1.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdc53a7799a7496ebc9fd29f31f7df80e83c9bda5299768af5f9e59eeea74647"
checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472"
dependencies = [
"cc",
"libc",
@ -2957,16 +2965,6 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "linux-keyutils"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "761e49ec5fd8a5a463f9b84e877c373d888935b71c6be78f3767fe2ae6bed18e"
dependencies = [
"bitflags 2.6.0",
"libc",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.14"
@ -3045,7 +3043,7 @@ checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -3375,7 +3373,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -3528,7 +3526,7 @@ dependencies = [
"thiserror",
"threadpool",
"toml",
"toml_edit 0.22.20",
"toml_edit",
"url",
"winreg",
"zip",
@ -3543,6 +3541,7 @@ dependencies = [
"actix-multipart",
"actix-web",
"chrono",
"convert_case 0.6.0",
"dotenvy",
"flate2",
"futures",
@ -3559,8 +3558,11 @@ dependencies = [
"sentry-log",
"serde",
"serde_json",
"serde_yaml",
"sha2",
"tantivy",
"tar",
"tempfile",
"thiserror",
"toml",
"url",
@ -3583,7 +3585,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -3663,11 +3665,11 @@ dependencies = [
[[package]]
name = "proc-macro-crate"
version = "3.1.0"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284"
checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b"
dependencies = [
"toml_edit 0.21.1",
"toml_edit",
]
[[package]]
@ -3995,18 +3997,18 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152"
[[package]]
name = "rustc_version"
version = "0.4.0"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
dependencies = [
"semver",
]
[[package]]
name = "rustix"
version = "0.38.34"
version = "0.38.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f"
dependencies = [
"bitflags 2.6.0",
"errno",
@ -4047,9 +4049,9 @@ checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
[[package]]
name = "rustls-webpki"
version = "0.102.6"
version = "0.102.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e"
checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56"
dependencies = [
"ring",
"rustls-pki-types",
@ -4287,29 +4289,29 @@ dependencies = [
[[package]]
name = "serde"
version = "1.0.208"
version = "1.0.209"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2"
checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.208"
version = "1.0.209"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf"
checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
name = "serde_json"
version = "1.0.125"
version = "1.0.127"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed"
checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad"
dependencies = [
"itoa",
"memchr",
@ -4334,7 +4336,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -4385,7 +4387,20 @@ dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
name = "serde_yaml"
version = "0.9.34+deprecated"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
dependencies = [
"indexmap 2.4.0",
"itoa",
"ryu",
"serde",
"unsafe-libyaml",
]
[[package]]
@ -4559,9 +4574,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.75"
version = "2.0.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9"
checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525"
dependencies = [
"proc-macro2",
"quote",
@ -4789,7 +4804,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -4917,7 +4932,7 @@ dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit 0.22.20",
"toml_edit",
]
[[package]]
@ -4929,17 +4944,6 @@ dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
dependencies = [
"indexmap 2.4.0",
"toml_datetime",
"winnow 0.5.40",
]
[[package]]
name = "toml_edit"
version = "0.22.20"
@ -4950,7 +4954,7 @@ dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"winnow 0.6.18",
"winnow",
]
[[package]]
@ -5000,7 +5004,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -5102,6 +5106,12 @@ version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
[[package]]
name = "unsafe-libyaml"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
[[package]]
name = "untrusted"
version = "0.9.0"
@ -5220,7 +5230,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
"wasm-bindgen-shared",
]
@ -5254,7 +5264,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@ -5512,15 +5522,6 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
version = "0.5.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876"
dependencies = [
"memchr",
]
[[package]]
name = "winnow"
version = "0.6.18"
@ -5602,7 +5603,7 @@ dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
"zvariant_utils",
]
@ -5635,7 +5636,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -5655,7 +5656,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]
[[package]]
@ -5751,7 +5752,7 @@ dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
"zvariant_utils",
]
@ -5763,5 +5764,5 @@ checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.75",
"syn 2.0.76",
]

View file

@ -72,7 +72,7 @@ serde_json = { version = "1.0.122", optional = true }
anyhow = { version = "1.0.86", optional = true }
open = { version = "5.3.0", optional = true }
keyring = { version = "3.0.5", features = ["crypto-rust", "windows-native", "apple-native", "linux-native"], optional = true }
keyring = { version = "3.0.5", features = ["crypto-rust", "windows-native", "apple-native", "sync-secret-service"], optional = true }
colored = { version = "2.1.0", optional = true }
toml_edit = { version = "0.22.20", optional = true }
clap = { version = "4.5.13", features = ["derive"], optional = true }

View file

@ -17,19 +17,26 @@ semver = "1.0.23"
chrono = { version = "0.4.38", features = ["serde"] }
url = "2.5.2"
futures = "0.3.30"
tempfile = "3.12.0"
git2 = "0.19.0"
gix = { version = "0.66.0", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls", "credentials"] }
gix = { version = "0.66.0", default-features = false, features = [
"blocking-http-transport-reqwest-rust-tls",
"credentials",
] }
serde = "1.0.206"
serde_json = "1.0.124"
toml = "0.8.16"
serde = "1.0.209"
serde_json = "1.0.127"
serde_yaml = "0.9.34"
toml = "0.8.19"
convert_case = "0.6.0"
sha2 = "0.10.8"
rusty-s3 = "0.5.0"
reqwest = { version = "0.12.5", features = ["json", "rustls-tls"] }
reqwest = { version = "0.12.7", features = ["json", "rustls-tls"] }
tar = "0.4.41"
flate2 = "1.0.30"
flate2 = "1.0.33"
log = "0.4.22"
pretty_env_logger = "0.5.0"
@ -38,4 +45,10 @@ sentry = "0.34.0"
sentry-log = "0.34.0"
sentry-actix = "0.34.0"
pesde = { path = "..", features = ["roblox", "lune", "luau", "wally-compat", "git2"] }
pesde = { path = "..", features = [
"roblox",
"lune",
"luau",
"wally-compat",
"git2",
] }

View file

@ -6,16 +6,18 @@ use rusty_s3::{actions::GetObject, S3Action};
use semver::Version;
use serde::{Deserialize, Deserializer};
use crate::{
error::Error,
package::{s3_doc_name, s3_name, PackageResponse, S3_SIGN_DURATION},
AppState,
};
use pesde::{
manifest::target::TargetKind,
names::PackageName,
source::{git_index::GitBasedSource, pesde::IndexFile},
};
use crate::{
error::Error,
package::{s3_name, PackageResponse, S3_SIGN_DURATION},
AppState,
source::{
git_index::GitBasedSource,
pesde::{DocEntryKind, IndexFile},
},
};
#[derive(Debug)]
@ -62,10 +64,16 @@ impl<'de> Deserialize<'de> for TargetRequest {
}
}
#[derive(Debug, Deserialize)]
pub struct Query {
doc: Option<String>,
}
pub async fn get_package_version(
request: HttpRequest,
app_state: web::Data<AppState>,
path: web::Path<(PackageName, VersionRequest, TargetRequest)>,
query: web::Query<Query>,
) -> Result<impl Responder, Error> {
let (name, version, target) = path.into_inner();
@ -110,6 +118,36 @@ pub async fn get_package_version(
return Ok(HttpResponse::NotFound().finish());
};
if let Some(doc_name) = query.doc.as_deref() {
let hash = 'finder: {
let mut hash = entry.docs.iter().map(|doc| &doc.kind).collect::<Vec<_>>();
while let Some(doc) = hash.pop() {
match doc {
DocEntryKind::Page { name, hash } if name == doc_name => {
break 'finder hash.clone()
}
DocEntryKind::Category { items, .. } => {
hash.extend(items.iter().map(|item| &item.kind))
}
_ => continue,
};
}
return Ok(HttpResponse::NotFound().finish());
};
let object_url = GetObject::new(
&app_state.s3_bucket,
Some(&app_state.s3_credentials),
&s3_doc_name(&hash),
)
.sign(S3_SIGN_DURATION);
return Ok(HttpResponse::TemporaryRedirect()
.append_header((LOCATION, object_url.as_str()))
.finish());
}
let accept = request
.headers()
.get(ACCEPT)
@ -133,7 +171,7 @@ pub async fn get_package_version(
.finish());
}
Ok(HttpResponse::Ok().json(PackageResponse {
let response = PackageResponse {
name: name.to_string(),
version: v_id.version().to_string(),
targets,
@ -142,5 +180,10 @@ pub async fn get_package_version(
license: entry.license.clone().unwrap_or_default(),
authors: entry.authors.clone(),
repository: entry.repository.clone().map(|url| url.to_string()),
}))
};
let mut value = serde_json::to_value(response)?;
value["docs"] = serde_json::to_value(entry.docs.clone())?;
Ok(HttpResponse::Ok().json(value))
}

View file

@ -1,37 +1,39 @@
use std::{
collections::BTreeSet,
io::{Cursor, Read, Write},
};
use actix_multipart::Multipart;
use actix_web::{web, HttpResponse, Responder};
use convert_case::{Case, Casing};
use flate2::read::GzDecoder;
use futures::StreamExt;
use futures::{future::join_all, StreamExt};
use git2::{Remote, Repository, Signature};
use reqwest::header::{CONTENT_ENCODING, CONTENT_TYPE};
use rusty_s3::{actions::PutObject, S3Action};
use tar::Archive;
use pesde::{
manifest::Manifest,
source::{
git_index::GitBasedSource,
pesde::{IndexFile, IndexFileEntry, ScopeInfo, SCOPE_INFO_FILE},
specifiers::DependencySpecifiers,
version_id::VersionId,
IGNORED_DIRS, IGNORED_FILES,
},
MANIFEST_FILE_NAME,
use serde::Deserialize;
use sha2::{Digest, Sha256};
use std::{
collections::{BTreeSet, HashMap},
fs::read_dir,
io::{Cursor, Read, Write},
};
use tar::Archive;
use crate::{
auth::UserId,
benv,
error::{Error, ErrorResponse},
package::{s3_name, S3_SIGN_DURATION},
package::{s3_doc_name, s3_name, S3_SIGN_DURATION},
search::update_version,
AppState,
};
use pesde::{
manifest::Manifest,
source::{
git_index::GitBasedSource,
pesde::{DocEntry, DocEntryKind, IndexFile, IndexFileEntry, ScopeInfo, SCOPE_INFO_FILE},
specifiers::DependencySpecifiers,
version_id::VersionId,
IGNORED_DIRS, IGNORED_FILES,
},
MANIFEST_FILE_NAME,
};
fn signature<'a>() -> Signature<'a> {
Signature::now(
@ -57,6 +59,16 @@ fn get_refspec(repo: &Repository, remote: &mut Remote) -> Result<String, git2::E
const ADDITIONAL_FORBIDDEN_FILES: &[&str] = &["default.project.json"];
#[derive(Debug, Deserialize, Default)]
struct DocEntryInfo {
#[serde(default)]
label: Option<String>,
#[serde(default)]
sidebar_position: Option<usize>,
#[serde(default)]
collapsed: bool,
}
pub async fn publish_package(
app_state: web::Data<AppState>,
mut body: Multipart,
@ -77,51 +89,181 @@ pub async fn publish_package(
.await
.map_err(|_| Error::InvalidArchive)?
.map_err(|_| Error::InvalidArchive)?;
let mut decoder = GzDecoder::new(Cursor::new(&bytes));
let mut archive = Archive::new(&mut decoder);
let entries = archive.entries()?;
let package_dir = tempfile::tempdir()?;
{
let mut decoder = GzDecoder::new(Cursor::new(&bytes));
let mut archive = Archive::new(&mut decoder);
archive.unpack(package_dir.path())?;
}
let mut manifest = None::<Manifest>;
let mut readme = None::<Vec<u8>>;
let mut docs = BTreeSet::new();
let mut docs_pages = HashMap::new();
for entry in entries {
let mut entry = entry?;
let path = entry.path()?;
for entry in read_dir(package_dir.path())? {
let entry = entry?;
let file_name = entry
.file_name()
.to_str()
.ok_or(Error::InvalidArchive)?
.to_string();
if entry.header().entry_type().is_dir() {
if path.components().next().is_some_and(|ct| {
ct.as_os_str()
.to_str()
.map_or(true, |s| IGNORED_DIRS.contains(&s))
}) {
if entry.file_type()?.is_dir() {
if IGNORED_DIRS.contains(&file_name.as_str()) {
return Err(Error::InvalidArchive);
}
if file_name == "docs" {
let mut stack = vec![(
BTreeSet::new(),
read_dir(entry.path())?,
None::<DocEntryInfo>,
)];
'outer: while let Some((set, iter, category_info)) = stack.last_mut() {
for entry in iter {
let entry = entry?;
let file_name = entry
.file_name()
.to_str()
.ok_or(Error::InvalidArchive)?
.to_string();
if entry.file_type()?.is_dir() {
stack.push((
BTreeSet::new(),
read_dir(entry.path())?,
Some(DocEntryInfo {
label: Some(file_name.to_case(Case::Title)),
..Default::default()
}),
));
continue 'outer;
}
if file_name == "_category_.json" {
let info = std::fs::read_to_string(entry.path())?;
let mut info: DocEntryInfo = serde_json::from_str(&info)?;
let old_info = category_info.take();
info.label = info.label.or(old_info.and_then(|i| i.label));
*category_info = Some(info);
continue;
}
let Some(file_name) = file_name.strip_suffix(".md") else {
continue;
};
let content = std::fs::read_to_string(entry.path())?;
let content = content.trim();
let hash = format!("{:x}", Sha256::digest(content.as_bytes()));
let mut gz = flate2::read::GzEncoder::new(
Cursor::new(content.as_bytes().to_vec()),
flate2::Compression::best(),
);
let mut bytes = vec![];
gz.read_to_end(&mut bytes)?;
docs_pages.insert(hash.to_string(), bytes);
let mut lines = content.lines().peekable();
let front_matter = if lines.peek().filter(|l| **l == "---").is_some() {
lines.next(); // skip the first `---`
let front_matter = lines
.by_ref()
.take_while(|l| *l != "---")
.collect::<Vec<_>>()
.join("\n");
lines.next(); // skip the last `---`
front_matter
} else {
"".to_string()
};
let h1 = lines
.find(|l| !l.trim().is_empty())
.and_then(|l| l.strip_prefix("# "))
.map(|s| s.to_string());
let info: DocEntryInfo = serde_yaml::from_str(&front_matter)
.map_err(|_| Error::InvalidArchive)?;
set.insert(DocEntry {
label: info.label.or(h1).unwrap_or(file_name.to_case(Case::Title)),
position: info.sidebar_position,
kind: DocEntryKind::Page {
name: entry
.path()
.strip_prefix(package_dir.path().join("docs"))
.unwrap()
.with_extension("")
.to_str()
.ok_or(Error::InvalidArchive)?
// ensure that the path is always using forward slashes
.replace("\\", "/"),
hash,
},
});
}
// should never be None
let (popped, _, category_info) = stack.pop().unwrap();
docs = popped;
if let Some((set, _, _)) = stack.last_mut() {
let category_info = category_info.unwrap_or_default();
set.insert(DocEntry {
label: category_info.label.unwrap(),
position: category_info.sidebar_position,
kind: DocEntryKind::Category {
items: {
let curr_docs = docs;
docs = BTreeSet::new();
curr_docs
},
collapsed: category_info.collapsed,
},
});
}
}
}
continue;
}
let path = path.to_str().ok_or(Error::InvalidArchive)?;
if IGNORED_FILES.contains(&path) || ADDITIONAL_FORBIDDEN_FILES.contains(&path) {
if IGNORED_FILES.contains(&file_name.as_str()) {
return Err(Error::InvalidArchive);
}
if path == MANIFEST_FILE_NAME {
let mut content = String::new();
entry.read_to_string(&mut content)?;
manifest = Some(toml::de::from_str(&content).map_err(|_| Error::InvalidArchive)?);
} else if path.to_lowercase() == "readme"
|| path
.to_lowercase()
.split_once('.')
.filter(|(file, ext)| *file == "readme" && (*ext == "md" || *ext == "txt"))
.is_some()
if ADDITIONAL_FORBIDDEN_FILES.contains(&file_name.as_str()) {
return Err(Error::InvalidArchive);
}
if file_name == MANIFEST_FILE_NAME {
let content = std::fs::read_to_string(entry.path())?;
manifest = Some(toml::de::from_str(&content)?);
} else if file_name
.to_lowercase()
.split_once('.')
.filter(|(file, ext)| *file == "readme" && (*ext == "md" || *ext == "txt"))
.is_some()
{
if readme.is_some() {
return Err(Error::InvalidArchive);
}
let mut gz = flate2::read::GzEncoder::new(entry, flate2::Compression::best());
let file = std::fs::File::open(entry.path())?;
let mut gz = flate2::read::GzEncoder::new(file, flate2::Compression::best());
let mut bytes = vec![];
gz.read_to_end(&mut bytes)?;
readme = Some(bytes);
@ -222,6 +364,7 @@ pub async fn publish_package(
license: manifest.license.clone(),
authors: manifest.authors.clone(),
repository: manifest.repository.clone(),
docs,
dependencies,
};
@ -314,39 +457,57 @@ pub async fn publish_package(
let version_id = VersionId::new(manifest.version.clone(), manifest.target.kind());
let object_url = PutObject::new(
&app_state.s3_bucket,
Some(&app_state.s3_credentials),
&s3_name(&manifest.name, &version_id, false),
join_all(
std::iter::once({
let object_url = PutObject::new(
&app_state.s3_bucket,
Some(&app_state.s3_credentials),
&s3_name(&manifest.name, &version_id, false),
)
.sign(S3_SIGN_DURATION);
app_state
.reqwest_client
.put(object_url)
.header(CONTENT_TYPE, "application/gzip")
.header(CONTENT_ENCODING, "gzip")
.body(bytes)
.send()
})
.chain(docs_pages.into_iter().map(|(hash, content)| {
let object_url = PutObject::new(
&app_state.s3_bucket,
Some(&app_state.s3_credentials),
&s3_doc_name(&hash),
)
.sign(S3_SIGN_DURATION);
app_state
.reqwest_client
.put(object_url)
.header(CONTENT_TYPE, "text/plain")
.header(CONTENT_ENCODING, "gzip")
.body(content)
.send()
}))
.chain(readme.map(|readme| {
let object_url = PutObject::new(
&app_state.s3_bucket,
Some(&app_state.s3_credentials),
&s3_name(&manifest.name, &version_id, true),
)
.sign(S3_SIGN_DURATION);
app_state
.reqwest_client
.put(object_url)
.header(CONTENT_TYPE, "text/plain")
.header(CONTENT_ENCODING, "gzip")
.body(readme)
.send()
})),
)
.sign(S3_SIGN_DURATION);
app_state
.reqwest_client
.put(object_url)
.header(CONTENT_TYPE, "application/gzip")
.header(CONTENT_ENCODING, "gzip")
.body(bytes)
.send()
.await?;
if let Some(readme) = readme {
let object_url = PutObject::new(
&app_state.s3_bucket,
Some(&app_state.s3_credentials),
&s3_name(&manifest.name, &version_id, true),
)
.sign(S3_SIGN_DURATION);
app_state
.reqwest_client
.put(object_url)
.header(CONTENT_TYPE, "text/plain")
.header(CONTENT_ENCODING, "gzip")
.body(readme)
.send()
.await?;
}
.await;
Ok(HttpResponse::Ok().body(format!(
"published {}@{} {}",

View file

@ -11,13 +11,17 @@ pub const S3_SIGN_DURATION: Duration = Duration::from_secs(60 * 3);
pub fn s3_name(name: &PackageName, version_id: &VersionId, is_readme: bool) -> String {
format!(
"{}+{}{}",
name.escaped(),
version_id.escaped(),
if is_readme { "+readme.gz" } else { ".tar.gz" }
"{name}/{}/{}/{}.gz",
version_id.version(),
version_id.target(),
if is_readme { "readme" } else { "pkg.tar" },
)
}
pub fn s3_doc_name(doc_hash: &str) -> String {
format!("doc/{}.gz", doc_hash)
}
#[derive(Debug, Serialize, Eq, PartialEq)]
pub struct TargetInfo {
kind: TargetKind,

View file

@ -93,6 +93,13 @@ impl PublishCommand {
);
}
if !manifest.includes.iter().any(|f| f == "docs") {
println!(
"{}: no docs directory in includes, consider adding one",
"warn".yellow().bold()
);
}
if manifest.includes.remove("default.project.json") {
println!(
"{}: default.project.json was in includes, this should be generated by the {} script upon dependants installation",

View file

@ -40,7 +40,7 @@ pub(crate) fn store_in_cas<P: AsRef<Path>>(
if !cas_path.exists() {
let mut file = std::fs::File::create(&cas_path)?;
file.write_all(contents)?;
// prevent the CAS from being corrupted due to accidental modifications
let mut permissions = file.metadata()?.permissions();
permissions.set_readonly(true);

View file

@ -387,6 +387,59 @@ impl IndexConfig {
}
}
/// An entry in a package's documentation
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
#[serde(untagged)]
pub enum DocEntryKind {
/// A page in the documentation
Page {
/// The name of the page
name: String,
/// The hash of the page's content
hash: String,
},
/// A category in the documentation
Category {
/// The items in the section
#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
items: BTreeSet<DocEntry>,
/// Whether this category is collapsed by default
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
collapsed: bool,
},
}
/// An entry in a package's documentation
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub struct DocEntry {
/// The label for this entry
pub label: String,
/// The position of this entry
#[serde(default, skip_serializing_if = "Option::is_none")]
pub position: Option<usize>,
/// The kind of this entry
#[serde(flatten)]
pub kind: DocEntryKind,
}
impl Ord for DocEntry {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
match (self.position, other.position) {
(Some(l), Some(r)) => l.cmp(&r),
(Some(_), None) => std::cmp::Ordering::Less,
(None, Some(_)) => std::cmp::Ordering::Greater,
(None, None) => std::cmp::Ordering::Equal,
}
.then(self.label.cmp(&other.label))
}
}
impl PartialOrd for DocEntry {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
/// The entry in a package's index file
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub struct IndexFileEntry {
@ -409,6 +462,10 @@ pub struct IndexFileEntry {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub repository: Option<url::Url>,
/// The documentation for this package
#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
pub docs: BTreeSet<DocEntry>,
/// The dependencies of this package
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub dependencies: BTreeMap<String, (DependencySpecifiers, DependencyType)>,