Implement cross-compilation of standalone binaries (#162)

Cargo.lock

@@ -150,6 +150,7 @@ dependencies = [
  "brotli",
  "flate2",
  "futures-core",
+ "futures-io",
  "memchr",
  "pin-project-lite",
  "tokio",
@@ -196,6 +197,21 @@ dependencies = [
  "syn 2.0.59",
 ]
 
+[[package]]
+name = "async_zip"
+version = "0.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "527207465fb6dcafbf661b0d4a51d0d2306c9d0c2975423079a6caa807930daf"
+dependencies = [
+ "async-compression",
+ "crc32fast",
+ "futures-lite",
+ "pin-project",
+ "thiserror",
+ "tokio",
+ "tokio-util",
+]
+
 [[package]]
 name = "atomic-waker"
 version = "1.1.2"
@@ -1349,6 +1365,7 @@ dependencies = [
  "anyhow",
  "async-compression",
  "async-trait",
+ "async_zip",
  "blocking",
  "chrono",
  "chrono_lc",
@@ -1391,6 +1408,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "tokio-tungstenite",
+ "tokio-util",
  "toml",
  "tracing",
  "tracing-subscriber",
@@ -2770,6 +2788,7 @@ checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15"
 dependencies = [
  "bytes",
  "futures-core",
+ "futures-io",
  "futures-sink",
  "pin-project-lite",
  "tokio",

Cargo.toml

@@ -26,6 +26,8 @@ cli = [
     "dep:include_dir",
     "dep:regex",
     "dep:rustyline",
+    "dep:async_zip",
+    "dep:tokio-util",
 ]
 roblox = [
     "dep:glam",
@@ -149,3 +151,10 @@ rbx_dom_weak = { optional = true, version = "2.6.0" }
 rbx_reflection = { optional = true, version = "4.4.0" }
 rbx_reflection_database = { optional = true, version = "0.2.9" }
 rbx_xml = { optional = true, version = "0.13.2" }
+
+### CROSS COMPILATION
+async_zip = { optional = true, version = "0.0.16", features = [
+    "tokio",
+    "deflate",
+] }
+tokio-util = { optional = true, version = "0.7", features = ["io-util"] }

src/cli/build.rs

@@ -1,17 +1,36 @@
 use std::{
-    env::consts::EXE_EXTENSION,
+    env::consts,
+    io::Cursor,
     path::{Path, PathBuf},
     process::ExitCode,
 };
 
 use anyhow::{Context, Result};
+use async_zip::base::read::seek::ZipFileReader;
 use clap::Parser;
 use console::style;
-use tokio::{fs, io::AsyncWriteExt as _};
+use directories::BaseDirs;
+use once_cell::sync::Lazy;
+use thiserror::Error;
+use tokio::{
+    fs,
+    io::{AsyncReadExt, AsyncWriteExt},
+};
+use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt};
 
-use crate::standalone::metadata::Metadata;
+use crate::standalone::metadata::{Metadata, CURRENT_EXE};
 
-/// Build a standalone executable
+const TARGET_BASE_DIR: Lazy<PathBuf> = Lazy::new(|| {
+    BaseDirs::new()
+        .unwrap()
+        .home_dir()
+        .to_path_buf()
+        .join(".lune")
+        .join("target")
+        .join(env!("CARGO_PKG_VERSION"))
+});
+
+// Build a standalone executable
 #[derive(Debug, Clone, Parser)]
 pub struct BuildCommand {
     /// The path to the input file
@@ -21,37 +40,45 @@ pub struct BuildCommand {
     /// input file path with an executable extension
     #[clap(short, long)]
     pub output: Option<PathBuf>,
+
+    /// The target to compile for - defaults to the host triple
+    #[clap(short, long)]
+    pub target: Option<String>,
 }
 
 impl BuildCommand {
     pub async fn run(self) -> Result<ExitCode> {
         let output_path = self
             .output
-            .unwrap_or_else(|| self.input.with_extension(EXE_EXTENSION));
+            .unwrap_or_else(|| self.input.with_extension(consts::EXE_EXTENSION));
 
         let input_path_displayed = self.input.display();
-        let output_path_displayed = output_path.display();
 
         // Try to read the input file
         let source_code = fs::read(&self.input)
             .await
             .context("failed to read input file")?;
 
+        // Dynamically derive the base executable path based on the CLI arguments provided
+        let (base_exe_path, output_path) = get_base_exe_path(self.target, output_path).await?;
+
         // Read the contents of the lune interpreter as our starting point
         println!(
-            "Creating standalone binary using {}",
+            "{} standalone binary using {}",
-            style(input_path_displayed).green()
+            style("Compile").green().bold(),
+            style(input_path_displayed).underlined()
         );
-        let patched_bin = Metadata::create_env_patched_bin(source_code.clone())
+        let patched_bin = Metadata::create_env_patched_bin(base_exe_path, source_code.clone())
             .await
             .context("failed to create patched binary")?;
 
         // And finally write the patched binary to the output file
         println!(
-            "Writing standalone binary to {}",
+            "   {} standalone binary to {}",
-            style(output_path_displayed).blue()
+            style("Write").blue().bold(),
+            style(output_path.display()).underlined()
         );
-        write_executable_file_to(output_path, patched_bin).await?;
+        write_executable_file_to(output_path, patched_bin).await?; // Read & execute for all, write for owner
 
         Ok(ExitCode::SUCCESS)
     }
@@ -71,3 +98,153 @@ async fn write_executable_file_to(path: impl AsRef<Path>, bytes: impl AsRef<[u8]
 
     Ok(())
 }
+
+/// Possible ways in which the discovery and/or download of a base binary's path can error
+#[derive(Debug, Error)]
+pub enum BasePathDiscoveryError {
+    /// An error in the decompression of the precompiled target
+    #[error("decompression error")]
+    Decompression(#[from] async_zip::error::ZipError),
+    #[error("precompiled base for target not found for {target}")]
+    TargetNotFound { target: String },
+    /// An error in the precompiled target download process
+    #[error("failed to download precompiled binary base, reason: {0}")]
+    DownloadError(#[from] reqwest::Error),
+    /// An IO related error
+    #[error("a generic error related to an io operation occurred, details: {0}")]
+    IoError(#[from] anyhow::Error),
+}
+
+/// Discovers the path to the base executable to use for cross-compilation
+async fn get_base_exe_path(
+    target: Option<String>,
+    output_path: PathBuf,
+) -> Result<(PathBuf, PathBuf), BasePathDiscoveryError> {
+    if let Some(target_inner) = target {
+        let current_target = format!("{}-{}", consts::OS, consts::ARCH);
+
+        let target_exe_extension = match target_inner.as_str() {
+            "windows-x86_64" => "exe",
+            _ => "",
+        };
+
+        if target_inner == current_target {
+            // If the target is the host target, just use the current executable
+            return Ok((
+                CURRENT_EXE.to_path_buf(),
+                output_path.with_extension(consts::EXE_EXTENSION),
+            ));
+        }
+
+        let path = TARGET_BASE_DIR.join(format!("lune-{target_inner}.{target_exe_extension}"));
+
+        // Create the target base directory in the lune home if it doesn't already exist
+        if !TARGET_BASE_DIR.exists() {
+            fs::create_dir_all(TARGET_BASE_DIR.to_path_buf())
+                .await
+                .map_err(anyhow::Error::from)
+                .map_err(BasePathDiscoveryError::IoError)?;
+        }
+
+        // If a cached target base executable doesn't exist, attempt to download it
+        if !path.exists() {
+            println!("Requested target hasn't been downloaded yet, attempting to download");
+            cache_target(target_inner, target_exe_extension, &path).await?;
+        }
+
+        Ok((path, output_path.with_extension(target_exe_extension)))
+    } else {
+        // If the target flag was not specified, just use the current executable
+        Ok((
+            CURRENT_EXE.to_path_buf(),
+            output_path.with_extension(consts::EXE_EXTENSION),
+        ))
+    }
+}
+
+async fn cache_target(
+    target: String,
+    target_exe_extension: &str,
+    path: &PathBuf,
+) -> Result<(), BasePathDiscoveryError> {
+    let release_url = format!(
+        "https://github.com/lune-org/lune/releases/download/v{ver}/lune-{ver}-{target}.zip",
+        ver = env!("CARGO_PKG_VERSION"),
+        target = target
+    );
+
+    let target_full_display = release_url
+        .split('/')
+        .last()
+        .unwrap_or("lune-UNKNOWN-UNKNOWN")
+        .replace(".zip", format!(".{target_exe_extension}").as_str());
+
+    println!(
+        "{} target {}",
+        style("Download").green().bold(),
+        target_full_display
+    );
+
+    let resp = reqwest::get(release_url).await.map_err(|err| {
+        eprintln!(
+            "   {} Unable to download base binary found for target `{}`",
+            style("Download").red().bold(),
+            target,
+        );
+
+        BasePathDiscoveryError::DownloadError(err)
+    })?;
+
+    let resp_status = resp.status();
+
+    if resp_status != 200 && !resp_status.is_redirection() {
+        eprintln!(
+            "   {} No precompiled base binary found for target `{}`",
+            style("Download").red().bold(),
+            target
+        );
+
+        return Err(BasePathDiscoveryError::TargetNotFound { target });
+    }
+
+    // Wrap the request response in bytes so that we can decompress it, since `async_zip`
+    // requires the underlying reader to implement `AsyncRead` and `Seek`, which `Bytes`
+    // doesn't implement
+    let compressed_data = Cursor::new(
+        resp.bytes()
+            .await
+            .map_err(anyhow::Error::from)
+            .map_err(BasePathDiscoveryError::IoError)?
+            .to_vec(),
+    );
+
+    // Construct a decoder and decompress the ZIP file using deflate
+    let mut decoder = ZipFileReader::new(compressed_data.compat())
+        .await
+        .map_err(BasePathDiscoveryError::Decompression)?;
+
+    let mut decompressed = vec![];
+
+    decoder
+        .reader_without_entry(0)
+        .await
+        .map_err(BasePathDiscoveryError::Decompression)?
+        .compat()
+        .read_to_end(&mut decompressed)
+        .await
+        .map_err(anyhow::Error::from)
+        .map_err(BasePathDiscoveryError::IoError)?;
+
+    // Finally write the decompressed data to the target base directory
+    write_executable_file_to(&path, decompressed)
+        .await
+        .map_err(BasePathDiscoveryError::IoError)?;
+
+    println!(
+        "  {} {}",
+        style("Downloaded").blue(),
+        style(target_full_display).underlined()
+    );
+
+    Ok(())
+}

src/main.rs

@@ -5,7 +5,9 @@
     clippy::match_bool,
     clippy::module_name_repetitions,
     clippy::multiple_crate_versions,
-    clippy::needless_pass_by_value
+    clippy::needless_pass_by_value,
+    clippy::declare_interior_mutable_const,
+    clippy::borrow_interior_mutable_const
 )]
 
 use std::process::ExitCode;

src/standalone/metadata.rs

@@ -5,10 +5,9 @@ use mlua::Compiler as LuaCompiler;
 use once_cell::sync::Lazy;
 use tokio::fs;
 
-const MAGIC: &[u8; 8] = b"cr3sc3nt";
+pub const CURRENT_EXE: Lazy<PathBuf> =
-
-static CURRENT_EXE: Lazy<PathBuf> =
     Lazy::new(|| env::current_exe().expect("failed to get current exe"));
+const MAGIC: &[u8; 8] = b"cr3sc3nt";
 
 /*
     TODO: Right now all we do is append the bytecode to the end
@@ -49,15 +48,19 @@ impl Metadata {
     /**
         Creates a patched standalone binary from the given script contents.
     */
-    pub async fn create_env_patched_bin(script_contents: impl Into<Vec<u8>>) -> Result<Vec<u8>> {
+    pub async fn create_env_patched_bin(
-        let mut patched_bin = fs::read(CURRENT_EXE.to_path_buf()).await?;
+        base_exe_path: PathBuf,
-
+        script_contents: impl Into<Vec<u8>>,
-        // Compile luau input into bytecode
+    ) -> Result<Vec<u8>> {
-        let bytecode = LuaCompiler::new()
+        let compiler = LuaCompiler::new()
             .set_optimization_level(2)
             .set_coverage_level(0)
-            .set_debug_level(1)
+            .set_debug_level(1);
-            .compile(script_contents.into());
+
+        let mut patched_bin = fs::read(base_exe_path).await?;
+
+        // Compile luau input into bytecode
+        let bytecode = compiler.compile(script_contents.into());
 
         // Append the bytecode / metadata to the end
         let meta = Self { bytecode };