From 0685e62a8f81da55194d5862e2864a6f0398b1da Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 20 Apr 2024 17:19:08 +0200 Subject: [PATCH] Refactor downloading lune binary to cache, some fixes + formatting --- Cargo.lock | 251 +++++++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 11 +-- src/cli/build.rs | 180 +++++++++++++++------------------ 3 files changed, 311 insertions(+), 131 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index de30793..8ff4e5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -110,6 +121,15 @@ version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arrayref" version = "0.3.7" @@ -150,7 +170,6 @@ dependencies = [ "brotli", "flate2", "futures-core", - "futures-io", "memchr", "pin-project-lite", "tokio", @@ -197,21 +216,6 @@ dependencies = [ "syn 2.0.59", ] -[[package]] -name = "async_zip" -version = "0.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527207465fb6dcafbf661b0d4a51d0d2306c9d0c2975423079a6caa807930daf" -dependencies = [ - "async-compression", - "crc32fast", - "futures-lite", - "pin-project", - "thiserror", - "tokio", - "tokio-util", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -374,11 +378,36 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" +dependencies = [ + "jobserver", + "libc", +] [[package]] name = "cfg-if" @@ -421,6 +450,16 @@ dependencies = [ "walkdir", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.4" @@ -470,6 +509,15 @@ dependencies = [ "error-code", ] +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -557,6 +605,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.0" @@ -588,6 +651,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" +[[package]] +name = "deflate64" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83ace6c86376be0b6cdcf3fb41882e81d94b31587573d1cfa9d01cd06bba210d" + [[package]] name = "deranged" version = "0.3.11" @@ -597,6 +666,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.59", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -631,6 +711,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -824,6 +905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", + "libz-ng-sys", "miniz_oxide", ] @@ -1020,6 +1102,15 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.9" @@ -1260,6 +1351,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "ipnet" version = "2.9.0" @@ -1281,6 +1381,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.69" @@ -1322,6 +1431,16 @@ dependencies = [ "libc", ] +[[package]] +name = "libz-ng-sys" +version = "1.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6409efc61b12687963e602df8ecf70e8ddacf95bc6576bcf16e3ac6328083c5" +dependencies = [ + "cmake", + "libc", +] + [[package]] name = "line-wrap" version = "0.2.0" @@ -1366,7 +1485,6 @@ dependencies = [ "anyhow", "async-compression", "async-trait", - "async_zip", "blocking", "bstr", "chrono", @@ -1410,11 +1528,11 @@ dependencies = [ "thiserror", "tokio", "tokio-tungstenite", - "tokio-util", "toml", "tracing", "tracing-subscriber", "urlencoding", + "zip_next", ] [[package]] @@ -1446,6 +1564,16 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder 1.5.0", + "crc", +] + [[package]] name = "matchers" version = "0.1.0" @@ -1691,6 +1819,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -2432,6 +2570,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "slab" version = "0.4.9" @@ -2790,7 +2934,6 @@ checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", - "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -2959,6 +3102,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + [[package]] name = "typenum" version = "1.17.0" @@ -3391,3 +3540,67 @@ name = "zeroize" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + +[[package]] +name = "zip_next" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9519d1479ea50c3b79f1e00eacbb58e311c72c721e08313ebe64d8617a31b5d1" +dependencies = [ + "aes", + "arbitrary", + "byteorder 1.5.0", + "bzip2", + "constant_time_eq 0.3.0", + "crc32fast", + "crossbeam-utils", + "deflate64", + "flate2", + "hmac", + "lzma-rs", + "pbkdf2", + "sha1 0.10.6", + "time 0.3.36", + "zopfli", + "zstd", +] + +[[package]] +name = "zopfli" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1f48f3508a3a3f2faee01629564400bc12260f6214a056d06a3aaaa6ef0736" +dependencies = [ + "crc32fast", + "log", + "simd-adler32", + "typed-arena", +] + +[[package]] +name = "zstd" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.10+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 2eace8d..9e35a0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,8 +26,7 @@ cli = [ "dep:include_dir", "dep:regex", "dep:rustyline", - "dep:async_zip", - "dep:tokio-util", + "dep:zip_next", ] roblox = [ "dep:glam", @@ -139,6 +138,7 @@ regex = { optional = true, version = "1.7", default-features = false, features = "unicode-perl", ] } rustyline = { optional = true, version = "14.0" } +zip_next = { optional = true, version = "1.1" } ### ROBLOX @@ -152,10 +152,3 @@ rbx_dom_weak = { optional = true, version = "2.6.0" } rbx_reflection = { optional = true, version = "4.4.0" } rbx_reflection_database = { optional = true, version = "0.2.9" } rbx_xml = { optional = true, version = "0.13.2" } - -### CROSS COMPILATION -async_zip = { optional = true, version = "0.0.16", features = [ - "tokio", - "deflate", -] } -tokio-util = { optional = true, version = "0.7", features = ["io-util"] } diff --git a/src/cli/build.rs b/src/cli/build.rs index a9b6bc8..f5471d1 100644 --- a/src/cli/build.rs +++ b/src/cli/build.rs @@ -1,22 +1,17 @@ use std::{ env::consts, - io::Cursor, + io::{Cursor, Read}, path::{Path, PathBuf}, process::ExitCode, }; use anyhow::{Context, Result}; -use async_zip::base::read::seek::ZipFileReader; use clap::Parser; use console::style; use directories::BaseDirs; use once_cell::sync::Lazy; use thiserror::Error; -use tokio::{ - fs, - io::{AsyncReadExt, AsyncWriteExt}, -}; -use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt}; +use tokio::{fs, io::AsyncWriteExt, task::spawn_blocking}; use crate::standalone::metadata::{Metadata, CURRENT_EXE}; @@ -30,7 +25,7 @@ const TARGET_BASE_DIR: Lazy = Lazy::new(|| { .join(env!("CARGO_PKG_VERSION")) }); -// Build a standalone executable +/// Build a standalone executable #[derive(Debug, Clone, Parser)] pub struct BuildCommand { /// The path to the input file @@ -64,9 +59,8 @@ impl BuildCommand { // Read the contents of the lune interpreter as our starting point println!( - "{} standalone binary using {}", - style("Compile").green().bold(), - style(input_path_displayed).underlined() + "Compiling standalone binary from {}", + style(input_path_displayed).green() ); let patched_bin = Metadata::create_env_patched_bin(base_exe_path, source_code.clone()) .await @@ -74,9 +68,8 @@ impl BuildCommand { // And finally write the patched binary to the output file println!( - " {} standalone binary to {}", - style("Write").blue().bold(), - style(output_path.display()).underlined() + "Writing standalone binary to {}", + style(output_path.display()).blue() ); write_executable_file_to(output_path, patched_bin).await?; // Read & execute for all, write for owner @@ -84,7 +77,10 @@ impl BuildCommand { } } -async fn write_executable_file_to(path: impl AsRef, bytes: impl AsRef<[u8]>) -> Result<()> { +async fn write_executable_file_to( + path: impl AsRef, + bytes: impl AsRef<[u8]>, +) -> Result<(), std::io::Error> { let mut options = fs::OpenOptions::new(); options.write(true).create(true).truncate(true); @@ -99,27 +95,30 @@ async fn write_executable_file_to(path: impl AsRef, bytes: impl AsRef<[u8] Ok(()) } -/// Possible ways in which the discovery and/or download of a base binary's path can error +/// Errors that may occur when building a standalone binary #[derive(Debug, Error)] -pub enum BasePathDiscoveryError { - /// An error in the decompression of the precompiled target - #[error("decompression error")] - Decompression(#[from] async_zip::error::ZipError), - #[error("precompiled base for target not found for {target}")] - TargetNotFound { target: String }, - /// An error in the precompiled target download process - #[error("failed to download precompiled binary base, reason: {0}")] - DownloadError(#[from] reqwest::Error), - /// An IO related error - #[error("a generic error related to an io operation occurred, details: {0}")] - IoError(#[from] anyhow::Error), +pub enum BuildError { + #[error("failed to find lune target '{0}' in GitHub release")] + ReleaseTargetNotFound(String), + #[error("failed to find lune binary '{0}' in downloaded zip file")] + ZippedBinaryNotFound(String), + #[error("failed to download lune binary: {0}")] + Download(#[from] reqwest::Error), + #[error("failed to unzip lune binary: {0}")] + Unzip(#[from] zip_next::result::ZipError), + #[error("panicked while unzipping lune binary: {0}")] + UnzipJoin(#[from] tokio::task::JoinError), + #[error("io error: {0}")] + IoError(#[from] std::io::Error), } +pub type BuildResult = std::result::Result; + /// Discovers the path to the base executable to use for cross-compilation async fn get_base_exe_path( target: Option, output_path: PathBuf, -) -> Result<(PathBuf, PathBuf), BasePathDiscoveryError> { +) -> BuildResult<(PathBuf, PathBuf)> { if let Some(target_inner) = target { let current_target = format!("{}-{}", consts::OS, consts::ARCH); @@ -140,16 +139,13 @@ async fn get_base_exe_path( // Create the target base directory in the lune home if it doesn't already exist if !TARGET_BASE_DIR.exists() { - fs::create_dir_all(TARGET_BASE_DIR.to_path_buf()) - .await - .map_err(anyhow::Error::from) - .map_err(BasePathDiscoveryError::IoError)?; + fs::create_dir_all(TARGET_BASE_DIR.to_path_buf()).await?; } // If a cached target base executable doesn't exist, attempt to download it if !path.exists() { - println!("Requested target hasn't been downloaded yet, attempting to download"); - cache_target(target_inner, target_exe_extension, &path).await?; + println!("Requested target does not exist in cache and must be downloaded"); + download_target_to_cache(target_inner, target_exe_extension, &path).await?; } Ok((path, output_path.with_extension(target_exe_extension))) @@ -162,89 +158,67 @@ async fn get_base_exe_path( } } -async fn cache_target( +/// Downloads the target base executable to the cache directory +async fn download_target_to_cache( target: String, target_exe_extension: &str, path: &PathBuf, -) -> Result<(), BasePathDiscoveryError> { +) -> BuildResult<()> { + let version = env!("CARGO_PKG_VERSION"); + let target_triple = format!("lune-{version}-{target}"); + let release_url = format!( - "https://github.com/lune-org/lune/releases/download/v{ver}/lune-{ver}-{target}.zip", - ver = env!("CARGO_PKG_VERSION"), - target = target + "{base_url}/v{version}/{target_triple}.zip", + base_url = "https://github.com/lune-org/lune/releases/download", ); + println!("Downloading {target_triple}"); - let target_full_display = release_url - .split('/') - .last() - .unwrap_or("lune-UNKNOWN-UNKNOWN") - .replace(".zip", format!(".{target_exe_extension}").as_str()); - - println!( - "{} target {}", - style("Download").green().bold(), - target_full_display - ); - - let resp = reqwest::get(release_url).await.map_err(|err| { - eprintln!( - " {} Unable to download base binary found for target `{}`", - style("Download").red().bold(), - target, - ); - - BasePathDiscoveryError::DownloadError(err) - })?; - - let resp_status = resp.status(); - - if resp_status != 200 && !resp_status.is_redirection() { - eprintln!( - " {} No precompiled base binary found for target `{}`", - style("Download").red().bold(), - target - ); - - return Err(BasePathDiscoveryError::TargetNotFound { target }); + // Try to request to download the zip file from the target url, + // making sure transient errors are handled gracefully and + // with a different error message than "not found" + let response = reqwest::get(release_url).await?; + if !response.status().is_success() { + if response.status().as_u16() == 404 { + return Err(BuildError::ReleaseTargetNotFound(target)); + } + return Err(BuildError::Download( + response.error_for_status().unwrap_err(), + )); } - // Wrap the request response in bytes so that we can decompress it, since `async_zip` - // requires the underlying reader to implement `AsyncRead` and `Seek`, which `Bytes` - // doesn't implement - let compressed_data = Cursor::new( - resp.bytes() - .await - .map_err(anyhow::Error::from) - .map_err(BasePathDiscoveryError::IoError)? - .to_vec(), + // Receive the full zip file + let zip_bytes = response.bytes().await?.to_vec(); + let zip_file = Cursor::new(zip_bytes); + + // Look for and extract the binary file from the zip file + let binary_file_name = format!( + "lune{}{target_exe_extension}", + if target_exe_extension.is_empty() { + "" + } else { + "." + } ); - // Construct a decoder and decompress the ZIP file using deflate - let mut decoder = ZipFileReader::new(compressed_data.compat()) - .await - .map_err(BasePathDiscoveryError::Decompression)?; + // NOTE: We use spawn_blocking here since reading a + // zip archive is a somewhat slow / blocking operation + let binary_file_handle = spawn_blocking(move || { + let mut archive = zip_next::ZipArchive::new(zip_file)?; - let mut decompressed = vec![]; + let mut binary = Vec::new(); + archive + .by_name(&binary_file_name) + .or(Err(BuildError::ZippedBinaryNotFound(binary_file_name)))? + .read_to_end(&mut binary)?; - decoder - .reader_without_entry(0) - .await - .map_err(BasePathDiscoveryError::Decompression)? - .compat() - .read_to_end(&mut decompressed) - .await - .map_err(anyhow::Error::from) - .map_err(BasePathDiscoveryError::IoError)?; + Ok::<_, BuildError>(binary) + }); + let binary_file_contents = binary_file_handle.await??; // Finally write the decompressed data to the target base directory - write_executable_file_to(&path, decompressed) - .await - .map_err(BasePathDiscoveryError::IoError)?; + write_executable_file_to(&path, binary_file_contents).await?; - println!( - " {} {}", - style("Downloaded").blue(), - style(target_full_display).underlined() - ); + println!("Downloaded {target_triple} successfully"); Ok(()) }