From bfb89dec01fa73734bdf953ff7c0d3cfaba6efaf Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 20:15:10 +0200 Subject: [PATCH 01/27] Fix table indentation and newline issues with new value formatter --- crates/lune-utils/src/fmt/value/recursive.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/lune-utils/src/fmt/value/recursive.rs b/crates/lune-utils/src/fmt/value/recursive.rs index 7dfbef7..7f0e9c2 100644 --- a/crates/lune-utils/src/fmt/value/recursive.rs +++ b/crates/lune-utils/src/fmt/value/recursive.rs @@ -52,6 +52,7 @@ pub(crate) fn format_value_recursive( } else { writeln!(buffer, "{}", STYLE_DIM.apply_to("{"))?; + let mut is_empty = true; for res in t.clone().pairs::() { let (key, value) = res.expect("conversion to LuaValue should never fail"); let formatted = if let Some(plain_key) = lua_value_as_plain_string_key(&key) { @@ -75,10 +76,18 @@ pub(crate) fn format_value_recursive( ) }; buffer.push_str(&formatted); + buffer.push('\n'); + is_empty = false; } visited.remove(&LuaValueId::from(t)); - write!(buffer, "\n{}", STYLE_DIM.apply_to("}"))?; + write!( + buffer, + "{}{}{}", + INDENT.repeat(depth), + if is_empty { " " } else { "" }, + STYLE_DIM.apply_to("}") + )?; } } else { let prefer_plain = depth == 0; From f2c40a4bd5b85f19528e7e28e26f1726cd29f35d Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 20:24:48 +0200 Subject: [PATCH 02/27] Improve formatting for empty tables --- crates/lune-utils/src/fmt/value/recursive.rs | 26 ++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/crates/lune-utils/src/fmt/value/recursive.rs b/crates/lune-utils/src/fmt/value/recursive.rs index 7f0e9c2..1014996 100644 --- a/crates/lune-utils/src/fmt/value/recursive.rs +++ b/crates/lune-utils/src/fmt/value/recursive.rs @@ -50,9 +50,10 @@ pub(crate) fn format_value_recursive( } else if !visited.insert(LuaValueId::from(t)) { write!(buffer, "{}", STYLE_DIM.apply_to("{ recursive }"))?; } else { - writeln!(buffer, "{}", STYLE_DIM.apply_to("{"))?; + write!(buffer, "{}", STYLE_DIM.apply_to("{"))?; let mut is_empty = true; + let mut table_lines = Vec::new(); for res in t.clone().pairs::() { let (key, value) = res.expect("conversion to LuaValue should never fail"); let formatted = if let Some(plain_key) = lua_value_as_plain_string_key(&key) { @@ -75,19 +76,24 @@ pub(crate) fn format_value_recursive( STYLE_DIM.apply_to(","), ) }; - buffer.push_str(&formatted); - buffer.push('\n'); + table_lines.push(formatted); is_empty = false; } visited.remove(&LuaValueId::from(t)); - write!( - buffer, - "{}{}{}", - INDENT.repeat(depth), - if is_empty { " " } else { "" }, - STYLE_DIM.apply_to("}") - )?; + + if is_empty { + write!(buffer, " {}", STYLE_DIM.apply_to("}"))?; + } else { + write!( + buffer, + "\n{}\n{}{}{}", + table_lines.join("\n"), + INDENT.repeat(depth), + if is_empty { " " } else { "" }, + STYLE_DIM.apply_to("}") + )?; + } } } else { let prefer_plain = depth == 0; From 0850f416175661b253e21c45589d4c009a971edc Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 21:01:51 +0200 Subject: [PATCH 03/27] Improve pretty formatting for arrays --- crates/lune-utils/src/fmt/value/recursive.rs | 101 +++++++++++++------ 1 file changed, 71 insertions(+), 30 deletions(-) diff --git a/crates/lune-utils/src/fmt/value/recursive.rs b/crates/lune-utils/src/fmt/value/recursive.rs index 1014996..db897a6 100644 --- a/crates/lune-utils/src/fmt/value/recursive.rs +++ b/crates/lune-utils/src/fmt/value/recursive.rs @@ -52,33 +52,23 @@ pub(crate) fn format_value_recursive( } else { write!(buffer, "{}", STYLE_DIM.apply_to("{"))?; - let mut is_empty = true; - let mut table_lines = Vec::new(); - for res in t.clone().pairs::() { - let (key, value) = res.expect("conversion to LuaValue should never fail"); - let formatted = if let Some(plain_key) = lua_value_as_plain_string_key(&key) { - format!( - "{}{plain_key} {} {}{}", - INDENT.repeat(1 + depth), - STYLE_DIM.apply_to("="), - format_value_recursive(&value, config, visited, depth + 1)?, - STYLE_DIM.apply_to(","), - ) - } else { - format!( - "{}{}{}{} {} {}{}", - INDENT.repeat(1 + depth), - STYLE_DIM.apply_to("["), - format_value_recursive(&key, config, visited, depth + 1)?, - STYLE_DIM.apply_to("]"), - STYLE_DIM.apply_to("="), - format_value_recursive(&value, config, visited, depth + 1)?, - STYLE_DIM.apply_to(","), - ) - }; - table_lines.push(formatted); - is_empty = false; - } + let values = t + .clone() + .pairs::() + .map(|res| res.expect("conversion to LuaValue should never fail")) + .collect::>(); + + let is_empty = values.is_empty(); + let is_array = values + .iter() + .enumerate() + .all(|(i, (key, _))| key.as_integer().is_some_and(|x| x == (i as i32) + 1)); + + let formatted_values = if is_array { + format_array(values, config, visited, depth)? + } else { + format_table(values, config, visited, depth)? + }; visited.remove(&LuaValueId::from(t)); @@ -87,10 +77,9 @@ pub(crate) fn format_value_recursive( } else { write!( buffer, - "\n{}\n{}{}{}", - table_lines.join("\n"), + "\n{}\n{}{}", + formatted_values.join("\n"), INDENT.repeat(depth), - if is_empty { " " } else { "" }, STYLE_DIM.apply_to("}") )?; } @@ -102,3 +91,55 @@ pub(crate) fn format_value_recursive( Ok(buffer) } + +fn format_array( + values: Vec<(LuaValue, LuaValue)>, + config: &ValueFormatConfig, + visited: &mut HashSet, + depth: usize, +) -> Result, fmt::Error> { + values + .into_iter() + .map(|(_, value)| { + Ok(format!( + "{}{}{}", + INDENT.repeat(1 + depth), + format_value_recursive(&value, config, visited, depth + 1)?, + STYLE_DIM.apply_to(","), + )) + }) + .collect() +} + +fn format_table( + values: Vec<(LuaValue, LuaValue)>, + config: &ValueFormatConfig, + visited: &mut HashSet, + depth: usize, +) -> Result, fmt::Error> { + values + .into_iter() + .map(|(key, value)| { + if let Some(plain_key) = lua_value_as_plain_string_key(&key) { + Ok(format!( + "{}{plain_key} {} {}{}", + INDENT.repeat(1 + depth), + STYLE_DIM.apply_to("="), + format_value_recursive(&value, config, visited, depth + 1)?, + STYLE_DIM.apply_to(","), + )) + } else { + Ok(format!( + "{}{}{}{} {} {}{}", + INDENT.repeat(1 + depth), + STYLE_DIM.apply_to("["), + format_value_recursive(&key, config, visited, depth + 1)?, + STYLE_DIM.apply_to("]"), + STYLE_DIM.apply_to("="), + format_value_recursive(&value, config, visited, depth + 1)?, + STYLE_DIM.apply_to(","), + )) + } + }) + .collect() +} From 9498620e039bf96a5f90eda10e561e745254d0c9 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 21:08:47 +0200 Subject: [PATCH 04/27] Sort tables before formatting them --- crates/lune-utils/src/fmt/value/recursive.rs | 23 +++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/crates/lune-utils/src/fmt/value/recursive.rs b/crates/lune-utils/src/fmt/value/recursive.rs index db897a6..d8c7f4c 100644 --- a/crates/lune-utils/src/fmt/value/recursive.rs +++ b/crates/lune-utils/src/fmt/value/recursive.rs @@ -1,3 +1,4 @@ +use std::cmp::Ordering; use std::collections::HashSet; use std::fmt::{self, Write as _}; @@ -52,11 +53,12 @@ pub(crate) fn format_value_recursive( } else { write!(buffer, "{}", STYLE_DIM.apply_to("{"))?; - let values = t + let mut values = t .clone() .pairs::() .map(|res| res.expect("conversion to LuaValue should never fail")) .collect::>(); + sort_for_formatting(&mut values); let is_empty = values.is_empty(); let is_array = values @@ -92,6 +94,25 @@ pub(crate) fn format_value_recursive( Ok(buffer) } +fn sort_for_formatting(values: &mut [(LuaValue, LuaValue)]) { + values.sort_by(|(a, _), (b, _)| { + if a.type_name() == b.type_name() { + // If we have the same type, sort either numerically or alphabetically + match (a, b) { + (LuaValue::Integer(a), LuaValue::Integer(b)) => a.cmp(b), + (LuaValue::Number(a), LuaValue::Number(b)) => a.partial_cmp(b).unwrap(), + (LuaValue::String(a), LuaValue::String(b)) => a.to_str().ok().cmp(&b.to_str().ok()), + _ => Ordering::Equal, + } + } else { + // If we have different types, sort numbers first, then strings, then others + a.is_number() + .cmp(&b.is_number()) + .then_with(|| a.is_string().cmp(&b.is_string())) + } + }); +} + fn format_array( values: Vec<(LuaValue, LuaValue)>, config: &ValueFormatConfig, From 1fd17ca0b3fe8a314ef5416986ce972c6d4291db Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 21:17:29 +0200 Subject: [PATCH 05/27] Update changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb04ac7..0c8c5b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed + +- Improved table pretty formatting when using `print`, `warn`, and `stdio.format`: + + - Keys are sorted numerically / alphabetically when possible. + - Keys of different types are put in distinct sections for mixed tables. + - Tables that are arrays no longer display their keys. + - Empty tables are no longer spread across lines. + ## `0.8.4` - May 12th, 2024 ### Added From adc74f47c0da80b286dafa3f407730f10a70067e Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 21:24:45 +0200 Subject: [PATCH 06/27] Fix panic when spawning a program that does not exist --- crates/lune-std-process/src/lib.rs | 5 +---- crates/lune/src/tests.rs | 1 + tests/process/spawn/no_panic.luau | 7 +++++++ 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 tests/process/spawn/no_panic.luau diff --git a/crates/lune-std-process/src/lib.rs b/crates/lune-std-process/src/lib.rs index adc4eb8..d3fd502 100644 --- a/crates/lune-std-process/src/lib.rs +++ b/crates/lune-std-process/src/lib.rs @@ -145,10 +145,7 @@ async fn process_spawn( lua: &Lua, (program, args, options): (String, Option>, ProcessSpawnOptions), ) -> LuaResult { - let res = lua - .spawn(spawn_command(program, args, options)) - .await - .expect("Failed to receive result of spawned process"); + let res = lua.spawn(spawn_command(program, args, options)).await?; /* NOTE: If an exit code was not given by the child process, diff --git a/crates/lune/src/tests.rs b/crates/lune/src/tests.rs index ae599da..de726ce 100644 --- a/crates/lune/src/tests.rs +++ b/crates/lune/src/tests.rs @@ -140,6 +140,7 @@ create_tests! { process_spawn_async: "process/spawn/async", process_spawn_basic: "process/spawn/basic", process_spawn_cwd: "process/spawn/cwd", + process_spawn_no_panic: "process/spawn/no_panic", process_spawn_shell: "process/spawn/shell", process_spawn_stdin: "process/spawn/stdin", process_spawn_stdio: "process/spawn/stdio", diff --git a/tests/process/spawn/no_panic.luau b/tests/process/spawn/no_panic.luau new file mode 100644 index 0000000..3a57a9b --- /dev/null +++ b/tests/process/spawn/no_panic.luau @@ -0,0 +1,7 @@ +local process = require("@lune/process") + +-- Spawning a child process for a non-existent +-- program should not panic, but should error + +local success = pcall(process.spawn, "someProgramThatDoesNotExist") +assert(not success, "Spawned a non-existent program") From 636d0bf277025b31763427db39d092b018d78320 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 21:26:15 +0200 Subject: [PATCH 07/27] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c8c5b6..b0b5739 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Tables that are arrays no longer display their keys. - Empty tables are no longer spread across lines. +## Fixed + +- Fixed formatted values in tables not being separated by newlines +- Fixed panicking (crashing) when using `process.spawn` with a program that does not exist. + ## `0.8.4` - May 12th, 2024 ### Added From 4f6f1835d293c35c92d83f9b77b4d6caee14f8e9 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 21:38:14 +0200 Subject: [PATCH 08/27] Fix erroring when setting nil attributes on instances in roblox lib --- CHANGELOG.md | 3 ++- crates/lune-roblox/src/instance/base.rs | 17 +++++++++++------ crates/lune-roblox/src/instance/mod.rs | 23 +++++++++++++++++++++++ tests/roblox/instance/attributes.luau | 5 +++++ 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0b5739..e6f2925 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Fixed -- Fixed formatted values in tables not being separated by newlines +- Fixed formatted values in tables not being separated by newlines. - Fixed panicking (crashing) when using `process.spawn` with a program that does not exist. +- Fixed `instance:SetAttribute("name", nil)` throwing an error and not removing the attribute. ## `0.8.4` - May 12th, 2024 diff --git a/crates/lune-roblox/src/instance/base.rs b/crates/lune-roblox/src/instance/base.rs index cc35373..58a2ae7 100644 --- a/crates/lune-roblox/src/instance/base.rs +++ b/crates/lune-roblox/src/instance/base.rs @@ -155,13 +155,18 @@ pub fn add_methods<'lua, M: LuaUserDataMethods<'lua, Instance>>(m: &mut M) { |lua, this, (attribute_name, lua_value): (String, LuaValue)| { ensure_not_destroyed(this)?; ensure_valid_attribute_name(&attribute_name)?; - match lua_value.lua_to_dom_value(lua, None) { - Ok(dom_value) => { - ensure_valid_attribute_value(&dom_value)?; - this.set_attribute(attribute_name, dom_value); - Ok(()) + if lua_value.is_nil() || lua_value.is_null() { + this.remove_attribute(attribute_name); + Ok(()) + } else { + match lua_value.lua_to_dom_value(lua, None) { + Ok(dom_value) => { + ensure_valid_attribute_value(&dom_value)?; + this.set_attribute(attribute_name, dom_value); + Ok(()) + } + Err(e) => Err(e.into()), } - Err(e) => Err(e.into()), } }, ); diff --git a/crates/lune-roblox/src/instance/mod.rs b/crates/lune-roblox/src/instance/mod.rs index bc75f2f..da3ccb4 100644 --- a/crates/lune-roblox/src/instance/mod.rs +++ b/crates/lune-roblox/src/instance/mod.rs @@ -442,6 +442,29 @@ impl Instance { } } + /** + Removes an attribute from the instance. + + Note that this does not have an equivalent in the Roblox engine API, + but separating this from `set_attribute` lets `set_attribute` be more + ergonomic and not require an `Option` for the value argument. + The equivalent in the Roblox engine API would be `instance:SetAttribute(name, nil)`. + */ + pub fn remove_attribute(&self, name: impl AsRef) { + let mut dom = INTERNAL_DOM.lock().expect("Failed to lock document"); + let inst = dom + .get_by_ref_mut(self.dom_ref) + .expect("Failed to find instance in document"); + if let Some(DomValue::Attributes(attributes)) = + inst.properties.get_mut(PROPERTY_NAME_ATTRIBUTES) + { + attributes.remove(name.as_ref()); + if attributes.is_empty() { + inst.properties.remove(PROPERTY_NAME_ATTRIBUTES); + } + } + } + /** Adds a tag to the instance. diff --git a/tests/roblox/instance/attributes.luau b/tests/roblox/instance/attributes.luau index a75ac87..4c0a62f 100644 --- a/tests/roblox/instance/attributes.luau +++ b/tests/roblox/instance/attributes.luau @@ -101,6 +101,11 @@ local folder = Instance.new("Folder") folder:SetAttribute("Foo", "Bar") assert(folder:GetAttribute("Foo") == "Bar") +-- Setting attributes to nil should work + +folder:SetAttribute("Foo", nil) +assert(folder:GetAttribute("Foo") == nil) + -- Writing files with modified attributes should work local game = Instance.new("DataModel") From 23456ae0415d2b824ca916c3802bbf1708b5ea8f Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 21:42:36 +0200 Subject: [PATCH 09/27] Bump versions in all changed packages --- Cargo.lock | 2 +- crates/lune-roblox/Cargo.toml | 2 +- crates/lune-std-process/Cargo.toml | 2 +- crates/lune-std-roblox/Cargo.toml | 2 +- crates/lune-std/Cargo.toml | 6 +++--- crates/lune-utils/Cargo.toml | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cf0e82d..63091ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1470,7 +1470,7 @@ dependencies = [ [[package]] name = "lune-roblox" -version = "0.1.0" +version = "0.1.1" dependencies = [ "glam", "lune-utils", diff --git a/crates/lune-roblox/Cargo.toml b/crates/lune-roblox/Cargo.toml index b20bd50..a122b68 100644 --- a/crates/lune-roblox/Cargo.toml +++ b/crates/lune-roblox/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lune-roblox" -version = "0.1.0" +version = "0.1.1" edition = "2021" license = "MPL-2.0" repository = "https://github.com/lune-org/lune" diff --git a/crates/lune-std-process/Cargo.toml b/crates/lune-std-process/Cargo.toml index b3d3f0c..a8d3fc5 100644 --- a/crates/lune-std-process/Cargo.toml +++ b/crates/lune-std-process/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lune-std-process" -version = "0.1.0" +version = "0.1.1" edition = "2021" license = "MPL-2.0" repository = "https://github.com/lune-org/lune" diff --git a/crates/lune-std-roblox/Cargo.toml b/crates/lune-std-roblox/Cargo.toml index 269aad9..c3efe94 100644 --- a/crates/lune-std-roblox/Cargo.toml +++ b/crates/lune-std-roblox/Cargo.toml @@ -20,4 +20,4 @@ once_cell = "1.17" rbx_cookie = { version = "0.1.4", default-features = false } lune-utils = { version = "0.1.0", path = "../lune-utils" } -lune-roblox = { version = "0.1.0", path = "../lune-roblox" } +lune-roblox = { version = "0.1.1", path = "../lune-roblox" } diff --git a/crates/lune-std/Cargo.toml b/crates/lune-std/Cargo.toml index 7ca38e4..6529a86 100644 --- a/crates/lune-std/Cargo.toml +++ b/crates/lune-std/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lune-std" -version = "0.1.1" +version = "0.1.2" edition = "2021" license = "MPL-2.0" repository = "https://github.com/lune-org/lune" @@ -51,9 +51,9 @@ lune-std-datetime = { optional = true, version = "0.1.1", path = "../lune-std-da lune-std-fs = { optional = true, version = "0.1.0", path = "../lune-std-fs" } lune-std-luau = { optional = true, version = "0.1.0", path = "../lune-std-luau" } lune-std-net = { optional = true, version = "0.1.0", path = "../lune-std-net" } -lune-std-process = { optional = true, version = "0.1.0", path = "../lune-std-process" } +lune-std-process = { optional = true, version = "0.1.1", path = "../lune-std-process" } lune-std-regex = { optional = true, version = "0.1.0", path = "../lune-std-regex" } -lune-std-roblox = { optional = true, version = "0.1.0", path = "../lune-std-roblox" } +lune-std-roblox = { optional = true, version = "0.1.1", path = "../lune-std-roblox" } lune-std-serde = { optional = true, version = "0.1.0", path = "../lune-std-serde" } lune-std-stdio = { optional = true, version = "0.1.0", path = "../lune-std-stdio" } lune-std-task = { optional = true, version = "0.1.0", path = "../lune-std-task" } diff --git a/crates/lune-utils/Cargo.toml b/crates/lune-utils/Cargo.toml index f07b737..29658d0 100644 --- a/crates/lune-utils/Cargo.toml +++ b/crates/lune-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lune-utils" -version = "0.1.0" +version = "0.1.1" edition = "2021" license = "MPL-2.0" repository = "https://github.com/lune-org/lune" From 3030158159a4d75fac03b67c0a00c34a87d9833b Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 21:44:55 +0200 Subject: [PATCH 10/27] Bump std roblox version too --- Cargo.lock | 8 ++++---- crates/lune-std-roblox/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 63091ef..8b81e2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1487,7 +1487,7 @@ dependencies = [ [[package]] name = "lune-std" -version = "0.1.1" +version = "0.1.2" dependencies = [ "lune-std-datetime", "lune-std-fs", @@ -1560,7 +1560,7 @@ dependencies = [ [[package]] name = "lune-std-process" -version = "0.1.0" +version = "0.1.1" dependencies = [ "directories", "lune-utils", @@ -1583,7 +1583,7 @@ dependencies = [ [[package]] name = "lune-std-roblox" -version = "0.1.0" +version = "0.1.1" dependencies = [ "lune-roblox", "lune-utils", @@ -1632,7 +1632,7 @@ dependencies = [ [[package]] name = "lune-utils" -version = "0.1.0" +version = "0.1.1" dependencies = [ "console", "dunce", diff --git a/crates/lune-std-roblox/Cargo.toml b/crates/lune-std-roblox/Cargo.toml index c3efe94..924e61c 100644 --- a/crates/lune-std-roblox/Cargo.toml +++ b/crates/lune-std-roblox/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lune-std-roblox" -version = "0.1.0" +version = "0.1.1" edition = "2021" license = "MPL-2.0" repository = "https://github.com/lune-org/lune" From 649bdc4c312d67f59e86422f6b22af4d0d52b777 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 21:46:49 +0200 Subject: [PATCH 11/27] Bump lune-utils dependency in lune-std --- crates/lune-std/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/lune-std/Cargo.toml b/crates/lune-std/Cargo.toml index 6529a86..da644c7 100644 --- a/crates/lune-std/Cargo.toml +++ b/crates/lune-std/Cargo.toml @@ -45,7 +45,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version = "1", default-features = false, features = ["fs", "sync"] } -lune-utils = { version = "0.1.0", path = "../lune-utils" } +lune-utils = { version = "0.1.1", path = "../lune-utils" } lune-std-datetime = { optional = true, version = "0.1.1", path = "../lune-std-datetime" } lune-std-fs = { optional = true, version = "0.1.0", path = "../lune-std-fs" } From b628601cc886e395be0e1d220a9801375178bc0d Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 21:49:16 +0200 Subject: [PATCH 12/27] Version 0.8.5 --- CHANGELOG.md | 2 +- Cargo.lock | 2 +- crates/lune/Cargo.toml | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6f2925..a1c0536 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## `0.8.5` - June 1st, 2024 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 8b81e2c..8fd35b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1442,7 +1442,7 @@ dependencies = [ [[package]] name = "lune" -version = "0.8.4" +version = "0.8.5" dependencies = [ "anyhow", "clap", diff --git a/crates/lune/Cargo.toml b/crates/lune/Cargo.toml index 93ac966..51fe1a4 100644 --- a/crates/lune/Cargo.toml +++ b/crates/lune/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lune" -version = "0.8.4" +version = "0.8.5" edition = "2021" license = "MPL-2.0" repository = "https://github.com/lune-org/lune" @@ -70,9 +70,9 @@ reqwest = { version = "0.11", default-features = false, features = [ "rustls-tls", ] } -lune-std = { optional = true, version = "0.1.1", path = "../lune-std" } -lune-roblox = { optional = true, version = "0.1.0", path = "../lune-roblox" } -lune-utils = { version = "0.1.0", path = "../lune-utils" } +lune-std = { optional = true, version = "0.1.2", path = "../lune-std" } +lune-roblox = { optional = true, version = "0.1.1", path = "../lune-roblox" } +lune-utils = { version = "0.1.1", path = "../lune-utils" } ### CLI From cf513c672408b6368f93a5798ded6b0c8fc16f6d Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Sat, 1 Jun 2024 21:50:33 +0200 Subject: [PATCH 13/27] Update lockfile --- Cargo.lock | 321 ++++++++++++++++++++++++++++------------------------- 1 file changed, 170 insertions(+), 151 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fd35b5..e21c206 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -118,9 +118,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "arbitrary" @@ -151,22 +151,21 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "async-channel" -version = "2.2.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d4d23bcc79e27423727b36823d86233aad06dfea531837b038394d11e9928" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" dependencies = [ "concurrent-queue", - "event-listener 5.3.0", - "event-listener-strategy 0.5.2", + "event-listener-strategy", "futures-core", "pin-project-lite", ] [[package]] name = "async-compression" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c90a406b4495d129f00461241616194cb8a032c8d1c53c657f0961d5f8e0498" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" dependencies = [ "brotli", "flate2", @@ -178,9 +177,9 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b10202063978b3351199d68f8b22c4e47e4b1b822f8d43fd862d5ea8c006b29a" +checksum = "c8828ec6e544c02b0d6691d21ed9f9218d0384a82542855073c2a3f58304aaf0" dependencies = [ "async-task", "concurrent-queue", @@ -189,17 +188,6 @@ dependencies = [ "slab", ] -[[package]] -name = "async-lock" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" -dependencies = [ - "event-listener 4.0.3", - "event-listener-strategy 0.4.0", - "pin-project-lite", -] - [[package]] name = "async-task" version = "4.7.1" @@ -220,9 +208,9 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" dependencies = [ "addr2line", "cc", @@ -298,12 +286,11 @@ dependencies = [ [[package]] name = "blocking" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "495f7104e962b7356f0aeb34247aca1fe7d2e783b346582db7f2904cb5717e88" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" dependencies = [ "async-channel", - "async-lock", "async-task", "futures-io", "futures-lite", @@ -323,9 +310,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -389,9 +376,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" dependencies = [ "jobserver", "libc", @@ -426,9 +413,9 @@ dependencies = [ [[package]] name = "chrono_lc" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa1812634894df89eb9d5075eba0b97d42b4affe477bde7d0db3d2cf8454a800" +checksum = "568e485d6ad62f607516ebb0820ae8cc46361b0870aeb46eb8232440a00b2eb6" dependencies = [ "chrono", "lazy_static", @@ -480,7 +467,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -611,18 +598,18 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crypto-common" @@ -663,7 +650,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -749,7 +736,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -787,11 +774,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "erased-serde" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b73807008a3c7f171cc40312f37d95ef0396e048b5848d775f54b1a4dd4a0d3" +checksum = "24e2389d65ab4fab27dc2a5de7b191e1f6617d1f1c8855c0dc569c94a4cbb18d" dependencies = [ "serde", + "typeid", ] [[package]] @@ -823,32 +811,22 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.3.0" +version = "5.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" dependencies = [ "concurrent-queue", "parking", "pin-project-lite", ] -[[package]] -name = "event-listener-strategy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" -dependencies = [ - "event-listener 4.0.3", - "pin-project-lite", -] - [[package]] name = "event-listener-strategy" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" dependencies = [ - "event-listener 5.3.0", + "event-listener 5.3.1", "pin-project-lite", ] @@ -937,7 +915,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -1001,9 +979,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "glam" @@ -1038,15 +1016,15 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", "http 1.1.0", "indexmap", "slab", @@ -1192,7 +1170,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.4", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -1235,9 +1213,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ "bytes", "futures-channel", @@ -1369,9 +1347,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.154" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libloading" @@ -1411,9 +1389,9 @@ checksum = "dd1bc4d24ad230d21fb898d1116b1801d7adfc449d42026475862ab48b11e70e" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" @@ -1425,6 +1403,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.21" @@ -1433,9 +1417,9 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "luau0-src" -version = "0.8.6+luau622" +version = "0.9.1+luau625" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07758c1f5908f7f9dd9109efaf8c66907cc38acf312db03287e7ad2a64b5de1c" +checksum = "39df4ee5bd067cb2363b9f9e5eed0f1cf326fc80508af8ac037eed5b15459c4a" dependencies = [ "cc", ] @@ -1696,9 +1680,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] @@ -1716,9 +1700,9 @@ dependencies = [ [[package]] name = "mlua" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9bed6bce296397a9d6a86f995dd10a547a4e6949825d45225906bdcbfe7367" +checksum = "e340c022072f3208a4105458286f4985ba5355bfe243c3073afe45cbe9ecf491" dependencies = [ "bstr", "erased-serde", @@ -1751,9 +1735,9 @@ dependencies = [ [[package]] name = "mlua-sys" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16a9ba1dd2c6ac971b204262d434c24d65067038598f0638b64e5dca28d52b8" +checksum = "5552e7e4e22ada0463dfdeee6caf6dc057a189fdc83136408a8f950a5e5c5540" dependencies = [ "cc", "cfg-if", @@ -1827,10 +1811,31 @@ dependencies = [ ] [[package]] -name = "object" -version = "0.32.2" +name = "num_enum" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "object" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" dependencies = [ "memchr", ] @@ -1879,9 +1884,9 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -1951,7 +1956,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -1968,9 +1973,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "piper" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +checksum = "ae1d5c74c9876f070d3e8fd503d748c7d974c3e48da8f41350fa5222ef9b4391" dependencies = [ "atomic-waker", "fastrand", @@ -2009,6 +2014,15 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "proc-macro-crate" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +dependencies = [ + "toml_edit 0.21.1", +] + [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -2017,9 +2031,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.82" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" +checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" dependencies = [ "unicode-ident", ] @@ -2040,7 +2054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8021cf59c8ec9c432cfc2526ac6b8aa508ecaf29cd415f271b8406c1b851c3fd" dependencies = [ "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2428,7 +2442,7 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.3", + "rustls-webpki 0.102.4", "subtle", "zeroize", ] @@ -2460,9 +2474,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.3" +version = "0.102.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ "ring", "rustls-pki-types", @@ -2551,9 +2565,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.201" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780f1cebed1629e4753a1a38a3c72d30b97ec044f0aef68cb26650a3c5cf363c" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] @@ -2570,13 +2584,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.201" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2593,9 +2607,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ "serde", ] @@ -2795,9 +2809,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.63" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf5be731623ca1a1fb7d8be6f261a3be6d3e2337b8a1f97be944d020c8fcb704" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -2845,22 +2859,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2959,9 +2973,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -2978,13 +2992,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -3039,37 +3053,48 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.12" +version = "0.8.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +checksum = "a4e43f8cc456c9704c851ae29c67e17ef65d2c30017c17a9765b89c382dc8bba" dependencies = [ "indexmap", "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.22.13", ] [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.12" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow 0.5.40", +] + +[[package]] +name = "toml_edit" +version = "0.22.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c127785850e8c20836d49732ae6abfa47616e60bf9d9f57c43c250361a9db96c" dependencies = [ "indexmap", "serde", "serde_spanned", "toml_datetime", - "winnow", + "winnow 0.6.9", ] [[package]] @@ -3085,7 +3110,6 @@ dependencies = [ "tokio", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -3106,7 +3130,6 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3120,7 +3143,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -3190,10 +3213,10 @@ dependencies = [ ] [[package]] -name = "typed-arena" -version = "2.0.2" +name = "typeid" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +checksum = "059d83cc991e7a42fc37bd50941885db0888e34209f8cfd9aab07ddec03bc9cf" [[package]] name = "typenum" @@ -3339,7 +3362,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", "wasm-bindgen-shared", ] @@ -3373,7 +3396,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3590,9 +3613,18 @@ checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winnow" -version = "0.6.8" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c52e9c97a68071b23e836c9380edae937f17b9c4667bd021973efc689f618d" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86c949fede1d13936a99f14fafd3e76fd642b556dd2ce96287fbe2e0151bfac6" dependencies = [ "memchr", ] @@ -3624,29 +3656,15 @@ checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.63", -] +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zip" -version = "1.2.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700ea425e148de30c29c580c1f9508b93ca57ad31c9f4e96b83c194c37a7a8f" +checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" dependencies = [ "aes", "arbitrary", @@ -3660,12 +3678,11 @@ dependencies = [ "hmac", "indexmap", "lzma-rs", + "num_enum", "pbkdf2", - "rand", "sha1 0.10.6", "thiserror", "time 0.3.36", - "zeroize", "zopfli", "zstd", ] @@ -3681,14 +3698,16 @@ dependencies = [ [[package]] name = "zopfli" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1f48f3508a3a3f2faee01629564400bc12260f6214a056d06a3aaaa6ef0736" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" dependencies = [ + "bumpalo", "crc32fast", + "lockfree-object-pool", "log", + "once_cell", "simd-adler32", - "typed-arena", ] [[package]] From 5a292aabc513f0bd92b9c1a2878722a5cdf43d5f Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 5 Jun 2024 07:30:50 -0700 Subject: [PATCH 14/27] Implement hashing algorithms + HMac support (#193) --- CHANGELOG.md | 18 +++ Cargo.lock | 48 ++++++ crates/lune-std-serde/Cargo.toml | 10 ++ crates/lune-std-serde/src/hash.rs | 234 ++++++++++++++++++++++++++++++ crates/lune-std-serde/src/lib.rs | 12 ++ crates/lune/src/tests.rs | 2 + tests/serde/hashing/hash.luau | 48 ++++++ tests/serde/hashing/hmac.luau | 60 ++++++++ types/serde.luau | 25 ++++ 9 files changed, 457 insertions(+) create mode 100644 crates/lune-std-serde/src/hash.rs create mode 100644 tests/serde/hashing/hash.luau create mode 100644 tests/serde/hashing/hmac.luau diff --git a/CHANGELOG.md b/CHANGELOG.md index a1c0536..e7d222c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +- Added a builtin API for hashing and calculating HMACs as part of the `serde` library + + Basic usage: + + ```lua + local serde = require("@lune/serde") + local hash = serde.hash("sha256", "a message to hash") + local hmac = serde.hmac("sha256", "a message to hash", "a secret string") + + print(hash) + print(hmac) + ``` + + The returned hashes are sequences of lowercase hexadecimal digits. The following algorithms are supported: + `md5`, `sha1`, `sha224`, `sha256`, `sha384`, `sha512`, `sha3-224`, `sha3-256`, `sha3-384`, `sha3-512`, `blake3` + ## `0.8.5` - June 1st, 2024 ### Changed diff --git a/Cargo.lock b/Cargo.lock index e21c206..13ae8cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -273,6 +273,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq 0.3.0", + "digest", ] [[package]] @@ -1339,6 +1340,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1582,13 +1592,20 @@ name = "lune-std-serde" version = "0.1.0" dependencies = [ "async-compression", + "blake3", "bstr", + "digest", + "hmac", "lune-utils", "lz4", + "md-5", "mlua", "serde", "serde_json", "serde_yaml", + "sha1 0.10.6", + "sha2", + "sha3", "tokio", "toml", ] @@ -1666,6 +1683,16 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.2" @@ -2665,6 +2692,27 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + [[package]] name = "sharded-slab" version = "0.1.7" diff --git a/crates/lune-std-serde/Cargo.toml b/crates/lune-std-serde/Cargo.toml index 91786ff..ab7bec0 100644 --- a/crates/lune-std-serde/Cargo.toml +++ b/crates/lune-std-serde/Cargo.toml @@ -29,6 +29,16 @@ serde_json = { version = "1.0", features = ["preserve_order"] } serde_yaml = "0.9" toml = { version = "0.8", features = ["preserve_order"] } +digest = "0.10.7" +hmac = "0.12.1" +md-5 = "0.10.6" +sha1 = "0.10.6" +sha2 = "0.10.8" +sha3 = "0.10.8" +# This feature MIGHT break due to the unstable nature of the digest crate. +# Check before updating it. +blake3 = { version = "1.5.0", features = ["traits-preview"] } + tokio = { version = "1", default-features = false, features = [ "rt", "io-util", diff --git a/crates/lune-std-serde/src/hash.rs b/crates/lune-std-serde/src/hash.rs new file mode 100644 index 0000000..cf0d3c6 --- /dev/null +++ b/crates/lune-std-serde/src/hash.rs @@ -0,0 +1,234 @@ +use std::fmt::Write; + +use bstr::BString; +use md5::Md5; +use mlua::prelude::*; + +use blake3::Hasher as Blake3; +use sha1::Sha1; +use sha2::{Sha224, Sha256, Sha384, Sha512}; +use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512}; + +pub struct HashOptions { + algorithm: HashAlgorithm, + message: BString, + secret: Option, + // seed: Option, +} + +#[derive(Debug, Clone, Copy)] +enum HashAlgorithm { + Md5, + Sha1, + // SHA-2 variants + Sha2_224, + Sha2_256, + Sha2_384, + Sha2_512, + // SHA-3 variants + Sha3_224, + Sha3_256, + Sha3_384, + Sha3_512, + // Blake3 + Blake3, +} + +impl HashAlgorithm { + pub fn list_all_as_string() -> String { + [ + "md5", "sha1", "sha224", "sha256", "sha384", "sha512", "sha3-224", "sha3-256", + "sha3-384", "sha3-512", "blake3", + ] + .join(", ") + } +} + +impl HashOptions { + /** + Computes the hash for the `message` using whatever `algorithm` is + contained within this struct and returns it as a string of hex digits. + */ + #[inline] + #[must_use = "hashing a message is useless without using the resulting hash"] + pub fn hash(self) -> String { + use digest::Digest; + + let message = self.message; + let bytes = match self.algorithm { + HashAlgorithm::Md5 => Md5::digest(message).to_vec(), + HashAlgorithm::Sha1 => Sha1::digest(message).to_vec(), + HashAlgorithm::Sha2_224 => Sha224::digest(message).to_vec(), + HashAlgorithm::Sha2_256 => Sha256::digest(message).to_vec(), + HashAlgorithm::Sha2_384 => Sha384::digest(message).to_vec(), + HashAlgorithm::Sha2_512 => Sha512::digest(message).to_vec(), + + HashAlgorithm::Sha3_224 => Sha3_224::digest(message).to_vec(), + HashAlgorithm::Sha3_256 => Sha3_256::digest(message).to_vec(), + HashAlgorithm::Sha3_384 => Sha3_384::digest(message).to_vec(), + HashAlgorithm::Sha3_512 => Sha3_512::digest(message).to_vec(), + + HashAlgorithm::Blake3 => Blake3::digest(message).to_vec(), + }; + + // We don't want to return raw binary data generally, since that's not + // what most people want a hash for. So we have to make a hex string. + bytes + .iter() + .fold(String::with_capacity(bytes.len() * 2), |mut output, b| { + let _ = write!(output, "{b:02x}"); + output + }) + } + + /** + Computes the HMAC for the `message` using whatever `algorithm` and + `secret` are contained within this struct. The computed value is + returned as a string of hex digits. + + # Errors + + If the `secret` is not provided or is otherwise invalid. + */ + #[inline] + pub fn hmac(self) -> LuaResult { + use hmac::{Hmac, Mac, SimpleHmac}; + + let secret = self + .secret + .ok_or_else(|| LuaError::FromLuaConversionError { + from: "nil", + to: "string or buffer", + message: Some("Argument #3 missing or nil".to_string()), + })?; + + /* + These macros exist to remove what would ultimately be dozens of + repeating lines. Essentially, there's several step to processing + HMacs, which expands into the 3 lines you see below. However, + the Hmac struct is specialized towards eager block-based processes. + In order to support anything else, like blake3, there's a second + type named `SimpleHmac`. This results in duplicate macros like + there are below. + */ + macro_rules! hmac { + ($Type:ty) => {{ + let mut mac: Hmac<$Type> = Hmac::new_from_slice(&secret).into_lua_err()?; + mac.update(&self.message); + mac.finalize().into_bytes().to_vec() + }}; + } + macro_rules! hmac_no_blocks { + ($Type:ty) => {{ + let mut mac: SimpleHmac<$Type> = + SimpleHmac::new_from_slice(&secret).into_lua_err()?; + mac.update(&self.message); + mac.finalize().into_bytes().to_vec() + }}; + } + + let bytes = match self.algorithm { + HashAlgorithm::Md5 => hmac!(Md5), + HashAlgorithm::Sha1 => hmac!(Sha1), + + HashAlgorithm::Sha2_224 => hmac!(Sha224), + HashAlgorithm::Sha2_256 => hmac!(Sha256), + HashAlgorithm::Sha2_384 => hmac!(Sha384), + HashAlgorithm::Sha2_512 => hmac!(Sha512), + + HashAlgorithm::Sha3_224 => hmac!(Sha3_224), + HashAlgorithm::Sha3_256 => hmac!(Sha3_256), + HashAlgorithm::Sha3_384 => hmac!(Sha3_384), + HashAlgorithm::Sha3_512 => hmac!(Sha3_512), + + HashAlgorithm::Blake3 => hmac_no_blocks!(Blake3), + }; + Ok(bytes + .iter() + .fold(String::with_capacity(bytes.len() * 2), |mut output, b| { + let _ = write!(output, "{b:02x}"); + output + })) + } +} + +impl<'lua> FromLua<'lua> for HashAlgorithm { + fn from_lua(value: LuaValue<'lua>, _lua: &'lua Lua) -> LuaResult { + if let LuaValue::String(str) = value { + /* + Casing tends to vary for algorithms, so rather than force + people to remember it we'll just accept any casing. + */ + let str = str.to_str()?.to_ascii_lowercase(); + match str.as_str() { + "md5" => Ok(Self::Md5), + "sha1" => Ok(Self::Sha1), + + "sha224" => Ok(Self::Sha2_224), + "sha256" => Ok(Self::Sha2_256), + "sha384" => Ok(Self::Sha2_384), + "sha512" => Ok(Self::Sha2_512), + + "sha3-224" => Ok(Self::Sha3_224), + "sha3-256" => Ok(Self::Sha3_256), + "sha3-384" => Ok(Self::Sha3_384), + "sha3-512" => Ok(Self::Sha3_512), + + "blake3" => Ok(Self::Blake3), + + _ => Err(LuaError::FromLuaConversionError { + from: "string", + to: "HashAlgorithm", + message: Some(format!( + "Invalid hashing algorithm '{str}', valid kinds are:\n{}", + HashAlgorithm::list_all_as_string() + )), + }), + } + } else { + Err(LuaError::FromLuaConversionError { + from: value.type_name(), + to: "HashAlgorithm", + message: None, + }) + } + } +} + +impl<'lua> FromLuaMulti<'lua> for HashOptions { + fn from_lua_multi(mut values: LuaMultiValue<'lua>, lua: &'lua Lua) -> LuaResult { + let algorithm = values + .pop_front() + .map(|value| HashAlgorithm::from_lua(value, lua)) + .transpose()? + .ok_or_else(|| LuaError::FromLuaConversionError { + from: "nil", + to: "HashAlgorithm", + message: Some("Argument #1 missing or nil".to_string()), + })?; + let message = values + .pop_front() + .map(|value| BString::from_lua(value, lua)) + .transpose()? + .ok_or_else(|| LuaError::FromLuaConversionError { + from: "nil", + to: "string or buffer", + message: Some("Argument #2 missing or nil".to_string()), + })?; + let secret = values + .pop_front() + .map(|value| BString::from_lua(value, lua)) + .transpose()?; + // let seed = values + // .pop_front() + // .map(|value| BString::from_lua(value, lua)) + // .transpose()?; + + Ok(HashOptions { + algorithm, + message, + secret, + // seed, + }) + } +} diff --git a/crates/lune-std-serde/src/lib.rs b/crates/lune-std-serde/src/lib.rs index 4514a75..4a66adf 100644 --- a/crates/lune-std-serde/src/lib.rs +++ b/crates/lune-std-serde/src/lib.rs @@ -7,9 +7,11 @@ use lune_utils::TableBuilder; mod compress_decompress; mod encode_decode; +mod hash; pub use self::compress_decompress::{compress, decompress, CompressDecompressFormat}; pub use self::encode_decode::{decode, encode, EncodeDecodeConfig, EncodeDecodeFormat}; +pub use self::hash::HashOptions; /** Creates the `serde` standard library module. @@ -24,6 +26,8 @@ pub fn module(lua: &Lua) -> LuaResult { .with_function("decode", serde_decode)? .with_async_function("compress", serde_compress)? .with_async_function("decompress", serde_decompress)? + .with_function("hash", hash_message)? + .with_function("hmac", hmac_message)? .build_readonly() } @@ -55,3 +59,11 @@ async fn serde_decompress( let bytes = decompress(bs, format).await?; lua.create_string(bytes) } + +fn hash_message(lua: &Lua, options: HashOptions) -> LuaResult { + lua.create_string(options.hash()) +} + +fn hmac_message(lua: &Lua, options: HashOptions) -> LuaResult { + lua.create_string(options.hmac()?) +} diff --git a/crates/lune/src/tests.rs b/crates/lune/src/tests.rs index de726ce..0306b29 100644 --- a/crates/lune/src/tests.rs +++ b/crates/lune/src/tests.rs @@ -230,6 +230,8 @@ create_tests! { serde_json_encode: "serde/json/encode", serde_toml_decode: "serde/toml/decode", serde_toml_encode: "serde/toml/encode", + serde_hashing_hash: "serde/hashing/hash", + serde_hashing_hmac: "serde/hashing/hmac", } #[cfg(feature = "std-stdio")] diff --git a/tests/serde/hashing/hash.luau b/tests/serde/hashing/hash.luau new file mode 100644 index 0000000..c0d5a27 --- /dev/null +++ b/tests/serde/hashing/hash.luau @@ -0,0 +1,48 @@ +local serde = require("@lune/serde") + +local TEST_INPUT = + "Luau is a fast, small, safe, gradually typed embeddable scripting language derived from Lua." + +local function test_case_hash(algorithm: serde.HashAlgorithm, expected: string) + assert( + serde.hash(algorithm, TEST_INPUT) == expected, + `hashing algorithm '{algorithm}' did not hash test string correctly` + ) + assert( + serde.hash(algorithm, buffer.fromstring(TEST_INPUT)) == expected, + `hashing algorithm '{algorithm}' did not hash test buffer correctly` + ) +end + +test_case_hash("blake3", "eccfe3a6696b2a1861c64cc78663cff51301058e5dc22bb6249e7e1e0173d7fe") +test_case_hash("md5", "2aed9e020b49d219dc383884c5bd7acd") +test_case_hash("sha1", "9dce74190857f36e6d3f5e8eb7fe704a74060726") +test_case_hash("sha224", "f7ccd8a5f2697df8470b66f03824e073075292a1fab40d3a2ddc2e83") +test_case_hash("sha256", "f1d149bfd1ea38833ae6abf2a6fece1531532283820d719272e9cf3d9344efea") +test_case_hash( + "sha384", + "f6da4b47846c6016a9b32f01b861e45195cf1fa6fc5c9dd2257f7dc1c14092f11001839ec1223c30ab7adb7370812863" +) +test_case_hash( + "sha512", + "49fd834fdf3d4eaf4d4aff289acfc24d649f81cee7a5a7940e5c86854e04816f0a97c53f2ca4908969a512ec5ad1dc466422e3928f5ce3da9913959315df807c" +) +test_case_hash("sha3-224", "56a4dd1ff1bd9baff7f8bbe380dbf2c75b073161693f94ebf91aeee5") +test_case_hash("sha3-256", "ee01be10e0dc133cd702999e854b396f40b039d5ba6ddec9d04bf8623ba04dd7") +test_case_hash( + "sha3-384", + "e992f31e638b47802f33a4327c0a951823e32491ddcef5af9ce18cff84475c98ced23928d47ef51a8a4299dfe2ece361" +) +test_case_hash( + "sha3-512", + "08bd02aca3052b7740de80b8e8b9969dc9059a4bfae197095430e0aa204fbd3afb11731b127559b90c2f7e295835ea844ddbb29baf2fdb1d823046052c120fc9" +) + +local failed = pcall(serde.hash, "a random string" :: any, "input that shouldn't be hashed") +assert(failed == false, "serde.hash shouldn't allow invalid algorithms passed to it!") + +assert( + serde.hash("sha256", "\0oh no invalid utf-8\127\0\255") + == "c18ed3188f9e93f9ecd3582d7398c45120b0b30a0e26243809206228ab711b78", + "serde.hash should hash invalid UTF-8 just fine" +) diff --git a/tests/serde/hashing/hmac.luau b/tests/serde/hashing/hmac.luau new file mode 100644 index 0000000..0af7c23 --- /dev/null +++ b/tests/serde/hashing/hmac.luau @@ -0,0 +1,60 @@ +local serde = require("@lune/serde") + +local INPUT_STRING = "important data to verify the integrity of" + +-- if you read this string, you're obligated to keep it a secret! :-) +local SECRET_STRING = "don't read this we operate on the honor system" + +local function test_case_hmac(algorithm: serde.HashAlgorithm, expected: string) + assert( + serde.hmac(algorithm, INPUT_STRING, SECRET_STRING) == expected, + `HMAC test for algorithm '{algorithm}' was not correct with string input and string secret` + ) + assert( + serde.hmac(algorithm, INPUT_STRING, buffer.fromstring(SECRET_STRING)) == expected, + `HMAC test for algorithm '{algorithm}' was not correct with string input and buffer secret` + ) + assert( + serde.hmac(algorithm, buffer.fromstring(INPUT_STRING), SECRET_STRING) == expected, + `HMAC test for algorithm '{algorithm}' was not correct with buffer input and string secret` + ) + assert( + serde.hmac(algorithm, buffer.fromstring(INPUT_STRING), buffer.fromstring(SECRET_STRING)) + == expected, + `HMAC test for algorithm '{algorithm}' was not correct with buffer input and buffer secret` + ) +end + +test_case_hmac("blake3", "1d9c1b9405567fc565c2c3c6d6c0e170be72a2623d29911f43cb2ce42a373c01") +test_case_hmac("md5", "525379669c93ab5f59d2201024145b79") +test_case_hmac("sha1", "75227c11ed65133788feab0ce7eb8efc8c1f0517") +test_case_hmac("sha224", "47a4857d7d7e1070f47f76558323e03471a918facaf3667037519c29") +test_case_hmac("sha256", "4a4816ab8d4b780a8cf131e34a3df25e4c7bc4eba453cd86e50271aab4e95f45") +test_case_hmac( + "sha384", + "6b24aeae78d0f84ec8a4669b24bda1131205535233c344f4262c1f90f29af04c5537612c269bbab8aaca9d8293f4a280" +) +test_case_hmac( + "sha512", + "9fffa071241e2f361f8a47a97d251c1d4aae37498efbc49745bf9916d8431f1f361080d350067ed65744d3da42956da33ec57b04901a5fd63a891381a1485ef7" +) +test_case_hmac("sha3-224", "ea102dfaa74aa285555bdba29a04429dfd4e997fa40322459094929f") +test_case_hmac("sha3-256", "17bde287e4692e5b7f281e444efefe92e00696a089570bd6814fd0e03d7763d2") +test_case_hmac( + "sha3-384", + "24f68401653d25f36e7ee8635831215f8b46710d4e133c9d1e091e5972c69b0f1d0cb80f5507522fa174d5c4746963c1" +) +test_case_hmac( + "sha3-512", + "d2566d156c254ced0101159f97187dbf48d900b8361fa5ebdd7e81409856b1b6a21d93a1fb6e8f700e75620d244ab9e894454030da12d158e9362ffe090d2669" +) + +local failed = + pcall(serde.hmac, "a random string" :: any, "input that shouldn't be hashed", "not a secret") +assert(failed == false, "serde.hmac shouldn't allow invalid algorithms passed to it!") + +assert( + serde.hmac("sha256", "\0oh no invalid utf-8\127\0\255", SECRET_STRING) + == "1f0d7f65016e9e4c340e3ba23da2483a7dc101ce8a9405f834c23f2e19232c3d", + "serde.hmac should hash invalid UTF-8 just fine" +) diff --git a/types/serde.luau b/types/serde.luau index c4a21d8..ff12714 100644 --- a/types/serde.luau +++ b/types/serde.luau @@ -2,6 +2,19 @@ export type EncodeDecodeFormat = "json" | "yaml" | "toml" export type CompressDecompressFormat = "brotli" | "gzip" | "lz4" | "zlib" +export type HashAlgorithm = + "md5" + | "sha1" + | "sha224" + | "sha256" + | "sha384" + | "sha512" + | "sha3-224" + | "sha3-256" + | "sha3-384" + | "sha3-512" + | "blake3" + --[=[ @class Serde @@ -120,4 +133,16 @@ function serde.decompress(format: CompressDecompressFormat, s: buffer | string): return nil :: any end +function serde.hash(algorithm: HashAlgorithm, message: string | buffer): string + return nil :: any +end + +function serde.hmac( + algorithm: HashAlgorithm, + message: string | buffer, + secret: string | buffer +): string + return nil :: any +end + return serde From 2a855324480139f95a76c6cd806d02cb6020b0bb Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 16:45:53 +0200 Subject: [PATCH 15/27] Move mlua-luau-scheduler into this repository --- Cargo.lock | 163 +++++- Cargo.toml | 1 + crates/lune-std-net/Cargo.toml | 2 +- crates/lune-std-process/Cargo.toml | 2 +- crates/lune-std-roblox/Cargo.toml | 2 +- crates/lune-std-stdio/Cargo.toml | 2 +- crates/lune-std-task/Cargo.toml | 2 +- crates/lune-std/Cargo.toml | 2 +- crates/lune/Cargo.toml | 2 +- crates/mlua-luau-scheduler/Cargo.toml | 67 +++ crates/mlua-luau-scheduler/README.md | 78 +++ .../examples/basic_sleep.rs | 45 ++ .../examples/basic_spawn.rs | 64 +++ .../mlua-luau-scheduler/examples/callbacks.rs | 48 ++ .../mlua-luau-scheduler/examples/exit_code.rs | 43 ++ .../examples/lots_of_threads.rs | 51 ++ .../examples/lua/basic_sleep.luau | 13 + .../examples/lua/basic_spawn.luau | 17 + .../examples/lua/callbacks.luau | 4 + .../examples/lua/exit_code.luau | 8 + .../examples/lua/lots_of_threads.luau | 29 ++ .../examples/lua/scheduler_ordering.luau | 34 ++ .../examples/scheduler_ordering.rs | 56 ++ crates/mlua-luau-scheduler/examples/tracy.rs | 61 +++ .../mlua-luau-scheduler/src/error_callback.rs | 45 ++ crates/mlua-luau-scheduler/src/exit.rs | 31 ++ crates/mlua-luau-scheduler/src/functions.rs | 283 ++++++++++ crates/mlua-luau-scheduler/src/lib.rs | 18 + crates/mlua-luau-scheduler/src/queue.rs | 139 +++++ crates/mlua-luau-scheduler/src/result_map.rs | 64 +++ crates/mlua-luau-scheduler/src/scheduler.rs | 484 ++++++++++++++++++ crates/mlua-luau-scheduler/src/status.rs | 31 ++ crates/mlua-luau-scheduler/src/thread_id.rs | 30 ++ crates/mlua-luau-scheduler/src/traits.rs | 378 ++++++++++++++ crates/mlua-luau-scheduler/src/util.rs | 147 ++++++ 35 files changed, 2436 insertions(+), 10 deletions(-) create mode 100644 crates/mlua-luau-scheduler/Cargo.toml create mode 100644 crates/mlua-luau-scheduler/README.md create mode 100644 crates/mlua-luau-scheduler/examples/basic_sleep.rs create mode 100644 crates/mlua-luau-scheduler/examples/basic_spawn.rs create mode 100644 crates/mlua-luau-scheduler/examples/callbacks.rs create mode 100644 crates/mlua-luau-scheduler/examples/exit_code.rs create mode 100644 crates/mlua-luau-scheduler/examples/lots_of_threads.rs create mode 100644 crates/mlua-luau-scheduler/examples/lua/basic_sleep.luau create mode 100644 crates/mlua-luau-scheduler/examples/lua/basic_spawn.luau create mode 100644 crates/mlua-luau-scheduler/examples/lua/callbacks.luau create mode 100644 crates/mlua-luau-scheduler/examples/lua/exit_code.luau create mode 100644 crates/mlua-luau-scheduler/examples/lua/lots_of_threads.luau create mode 100644 crates/mlua-luau-scheduler/examples/lua/scheduler_ordering.luau create mode 100644 crates/mlua-luau-scheduler/examples/scheduler_ordering.rs create mode 100644 crates/mlua-luau-scheduler/examples/tracy.rs create mode 100644 crates/mlua-luau-scheduler/src/error_callback.rs create mode 100644 crates/mlua-luau-scheduler/src/exit.rs create mode 100644 crates/mlua-luau-scheduler/src/functions.rs create mode 100644 crates/mlua-luau-scheduler/src/lib.rs create mode 100644 crates/mlua-luau-scheduler/src/queue.rs create mode 100644 crates/mlua-luau-scheduler/src/result_map.rs create mode 100644 crates/mlua-luau-scheduler/src/scheduler.rs create mode 100644 crates/mlua-luau-scheduler/src/status.rs create mode 100644 crates/mlua-luau-scheduler/src/thread_id.rs create mode 100644 crates/mlua-luau-scheduler/src/traits.rs create mode 100644 crates/mlua-luau-scheduler/src/util.rs diff --git a/Cargo.lock b/Cargo.lock index 13ae8cd..ca25bc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -188,6 +188,47 @@ dependencies = [ "slab", ] +[[package]] +name = "async-fs" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcd09b382f40fcd159c2d695175b2ae620ffa5f3bd6f664131efff4e8b9e04a" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-io" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d6baa8f0178795da0e71bc42c9e5d13261aac7ee549853162e66a241ba17964" +dependencies = [ + "async-lock", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener 5.3.1", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-task" version = "4.7.1" @@ -946,6 +987,20 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186014d53bc231d0090ef8d6f03e0920c54d85a5ed22f4f2f74315ec56cf83fb" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1243,7 +1298,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -1425,6 +1480,19 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "luau0-src" version = "0.9.1+luau625" @@ -1746,10 +1814,10 @@ dependencies = [ [[package]] name = "mlua-luau-scheduler" version = "0.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13eabdbc57fa38cf0b604d98ce3431573c79a964aac56e09c16c240d36cb1bf" dependencies = [ "async-executor", + "async-fs", + "async-io", "blocking", "concurrent-queue", "derive_more", @@ -1758,6 +1826,8 @@ dependencies = [ "mlua", "rustc-hash", "tracing", + "tracing-subscriber", + "tracing-tracy", ] [[package]] @@ -2029,6 +2099,21 @@ dependencies = [ "time 0.3.36", ] +[[package]] +name = "polling" +version = "3.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6a007746f34ed64099e88783b0ae369eaa3da6392868ba262e2af9b8fbaea1" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -2510,6 +2595,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + [[package]] name = "rustyline" version = "14.0.0" @@ -2547,6 +2638,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -3233,6 +3330,37 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tracing-tracy" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6024d04f84a69fd0d1dc1eee3a2b070bd246530a0582f9982ae487cb6c703614" +dependencies = [ + "tracing-core", + "tracing-subscriber", + "tracy-client", +] + +[[package]] +name = "tracy-client" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59fb931a64ff88984f86d3e9bcd1ae8843aa7fe44dd0f8097527bc172351741d" +dependencies = [ + "loom", + "once_cell", + "tracy-client-sys", +] + +[[package]] +name = "tracy-client-sys" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d104d610dfa9dd154535102cc9c6164ae1fa37842bc2d9e83f9ac82b0ae0882" +dependencies = [ + "cc", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -3511,6 +3639,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.5", +] + [[package]] name = "windows-core" version = "0.52.0" @@ -3520,6 +3658,25 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result", + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-result" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "749f0da9cc72d82e600d8d2e44cadd0b9eedb9038f71a1c58556ac1c5791813b" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 904de54..221a0fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "crates/lune-std-stdio", "crates/lune-std-task", "crates/lune-utils", + "crates/mlua-luau-scheduler", ] # Profile for building the release binary, with the following options set: diff --git a/crates/lune-std-net/Cargo.toml b/crates/lune-std-net/Cargo.toml index 2cf086e..fa25858 100644 --- a/crates/lune-std-net/Cargo.toml +++ b/crates/lune-std-net/Cargo.toml @@ -14,7 +14,7 @@ workspace = true [dependencies] mlua = { version = "0.9.7", features = ["luau"] } -mlua-luau-scheduler = "0.0.2" +mlua-luau-scheduler = { version = "0.0.2", path = "../mlua-luau-scheduler" } bstr = "1.9" futures-util = "0.3" diff --git a/crates/lune-std-process/Cargo.toml b/crates/lune-std-process/Cargo.toml index a8d3fc5..83a792f 100644 --- a/crates/lune-std-process/Cargo.toml +++ b/crates/lune-std-process/Cargo.toml @@ -14,7 +14,7 @@ workspace = true [dependencies] mlua = { version = "0.9.7", features = ["luau"] } -mlua-luau-scheduler = "0.0.2" +mlua-luau-scheduler = { version = "0.0.2", path = "../mlua-luau-scheduler" } directories = "5.0" pin-project = "1.0" diff --git a/crates/lune-std-roblox/Cargo.toml b/crates/lune-std-roblox/Cargo.toml index 924e61c..a2cc387 100644 --- a/crates/lune-std-roblox/Cargo.toml +++ b/crates/lune-std-roblox/Cargo.toml @@ -14,7 +14,7 @@ workspace = true [dependencies] mlua = { version = "0.9.7", features = ["luau"] } -mlua-luau-scheduler = "0.0.2" +mlua-luau-scheduler = { version = "0.0.2", path = "../mlua-luau-scheduler" } once_cell = "1.17" rbx_cookie = { version = "0.1.4", default-features = false } diff --git a/crates/lune-std-stdio/Cargo.toml b/crates/lune-std-stdio/Cargo.toml index 7d3909e..2e26e98 100644 --- a/crates/lune-std-stdio/Cargo.toml +++ b/crates/lune-std-stdio/Cargo.toml @@ -15,7 +15,7 @@ workspace = true [dependencies] dialoguer = "0.11" mlua = { version = "0.9.7", features = ["luau"] } -mlua-luau-scheduler = "0.0.2" +mlua-luau-scheduler = { version = "0.0.2", path = "../mlua-luau-scheduler" } tokio = { version = "1", default-features = false, features = [ "io-std", diff --git a/crates/lune-std-task/Cargo.toml b/crates/lune-std-task/Cargo.toml index edc6e5a..b18fc7b 100644 --- a/crates/lune-std-task/Cargo.toml +++ b/crates/lune-std-task/Cargo.toml @@ -14,7 +14,7 @@ workspace = true [dependencies] mlua = { version = "0.9.7", features = ["luau"] } -mlua-luau-scheduler = "0.0.2" +mlua-luau-scheduler = { version = "0.0.2", path = "../mlua-luau-scheduler" } tokio = { version = "1", default-features = false, features = ["time"] } diff --git a/crates/lune-std/Cargo.toml b/crates/lune-std/Cargo.toml index da644c7..83ccab7 100644 --- a/crates/lune-std/Cargo.toml +++ b/crates/lune-std/Cargo.toml @@ -39,7 +39,7 @@ task = ["dep:lune-std-task"] [dependencies] mlua = { version = "0.9.7", features = ["luau"] } -mlua-luau-scheduler = "0.0.2" +mlua-luau-scheduler = { version = "0.0.2", path = "../mlua-luau-scheduler" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/lune/Cargo.toml b/crates/lune/Cargo.toml index 51fe1a4..ca5094f 100644 --- a/crates/lune/Cargo.toml +++ b/crates/lune/Cargo.toml @@ -51,7 +51,7 @@ workspace = true [dependencies] mlua = { version = "0.9.7", features = ["luau"] } -mlua-luau-scheduler = "0.0.2" +mlua-luau-scheduler = { version = "0.0.2", path = "../mlua-luau-scheduler" } anyhow = "1.0" console = "0.15" diff --git a/crates/mlua-luau-scheduler/Cargo.toml b/crates/mlua-luau-scheduler/Cargo.toml new file mode 100644 index 0000000..589d36d --- /dev/null +++ b/crates/mlua-luau-scheduler/Cargo.toml @@ -0,0 +1,67 @@ +[package] +name = "mlua-luau-scheduler" +version = "0.0.2" +edition = "2021" +license = "MPL-2.0" +repository = "https://github.com/lune-org/lune" +description = "Luau-based async scheduler, using mlua and async-executor" +readme = "README.md" +keywords = ["async", "luau", "scheduler"] +categories = ["async"] + +[lib] +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +async-executor = "1.8" +blocking = "1.5" +concurrent-queue = "2.4" +derive_more = "0.99" +event-listener = "4.0" +futures-lite = "2.2" +rustc-hash = "1.1" +tracing = "0.1" + +mlua = { version = "0.9.6", features = [ + "luau", + "luau-jit", + "async", + "serialize", +] } + +[dev-dependencies] +async-fs = "2.1" +async-io = "2.3" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-tracy = "0.11" + +[[example]] +name = "basic_sleep" +test = true + +[[example]] +name = "basic_spawn" +test = true + +[[example]] +name = "callbacks" +test = true + +[[example]] +name = "exit_code" +test = true + +[[example]] +name = "lots_of_threads" +test = true + +[[example]] +name = "scheduler_ordering" +test = true + +[[example]] +name = "tracy" +test = false diff --git a/crates/mlua-luau-scheduler/README.md b/crates/mlua-luau-scheduler/README.md new file mode 100644 index 0000000..e18ed3c --- /dev/null +++ b/crates/mlua-luau-scheduler/README.md @@ -0,0 +1,78 @@ + + + +# `mlua-luau-scheduler` + +An async scheduler for Luau, using [`mlua`][mlua] and built on top of [`async-executor`][async-executor]. + +This crate is runtime-agnostic and is compatible with any async runtime, including [Tokio][tokio], [smol][smol], [async-std][async-std], and others.
+However, since many dependencies are shared with [smol][smol], depending on it over other runtimes may be preferred. + +[async-executor]: https://crates.io/crates/async-executor +[async-std]: https://async.rs +[mlua]: https://crates.io/crates/mlua +[smol]: https://github.com/smol-rs/smol +[tokio]: https://tokio.rs + +## Example Usage + +### 1. Import dependencies + +```rs +use std::time::{Duration, Instant}; +use std::io::ErrorKind; + +use async_io::{block_on, Timer}; +use async_fs::read_to_string; + +use mlua::prelude::*; +use mlua_luau_scheduler::*; +``` + +### 2. Set up Lua environment + +```rs +let lua = Lua::new(); + +lua.globals().set( + "sleep", + lua.create_async_function(|_, duration: f64| async move { + let before = Instant::now(); + let after = Timer::after(Duration::from_secs_f64(duration)).await; + Ok((after - before).as_secs_f64()) + })?, +)?; + +lua.globals().set( + "readFile", + lua.create_async_function(|lua, path: String| async move { + // Spawn background task that does not take up resources on the lua thread + // Normally, futures in mlua can not be shared across threads, but this can + let task = lua.spawn(async move { + match read_to_string(path).await { + Ok(s) => Ok(Some(s)), + Err(e) if e.kind() == ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + } + }); + task.await.into_lua_err() + })?, +)?; +``` + +### 3. Set up scheduler, run threads + +```rs +let sched = Scheduler::new(&lua)?; + +// We can create multiple lua threads ... +let sleepThread = lua.load("sleep(0.1)"); +let fileThread = lua.load("readFile(\"Cargo.toml\")"); + +// ... spawn them both onto the scheduler ... +sched.push_thread_front(sleepThread, ()); +sched.push_thread_front(fileThread, ()); + +// ... and run until they finish +block_on(sched.run()); +``` diff --git a/crates/mlua-luau-scheduler/examples/basic_sleep.rs b/crates/mlua-luau-scheduler/examples/basic_sleep.rs new file mode 100644 index 0000000..228591d --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/basic_sleep.rs @@ -0,0 +1,45 @@ +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::cargo_common_metadata)] + +use std::time::{Duration, Instant}; + +use async_io::{block_on, Timer}; + +use mlua::prelude::*; +use mlua_luau_scheduler::Scheduler; + +const MAIN_SCRIPT: &str = include_str!("./lua/basic_sleep.luau"); + +pub fn main() -> LuaResult<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_target(false) + .without_time() + .init(); + + // Set up persistent Lua environment + let lua = Lua::new(); + lua.globals().set( + "sleep", + lua.create_async_function(|_, duration: f64| async move { + let before = Instant::now(); + let after = Timer::after(Duration::from_secs_f64(duration)).await; + Ok((after - before).as_secs_f64()) + })?, + )?; + + // Load the main script into a scheduler + let sched = Scheduler::new(&lua); + let main = lua.load(MAIN_SCRIPT); + sched.push_thread_front(main, ())?; + + // Run until completion + block_on(sched.run()); + + Ok(()) +} + +#[test] +fn test_basic_sleep() -> LuaResult<()> { + main() +} diff --git a/crates/mlua-luau-scheduler/examples/basic_spawn.rs b/crates/mlua-luau-scheduler/examples/basic_spawn.rs new file mode 100644 index 0000000..8e65a1a --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/basic_spawn.rs @@ -0,0 +1,64 @@ +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::cargo_common_metadata)] + +use std::io::ErrorKind; + +use async_fs::read_to_string; +use async_io::block_on; + +use mlua::prelude::*; +use mlua_luau_scheduler::{LuaSpawnExt, Scheduler}; + +const MAIN_SCRIPT: &str = include_str!("./lua/basic_spawn.luau"); + +pub fn main() -> LuaResult<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_target(false) + .without_time() + .init(); + + // Set up persistent Lua environment + let lua = Lua::new(); + lua.globals().set( + "readFile", + lua.create_async_function(|lua, path: String| async move { + // Spawn background task that does not take up resources on the Lua thread + let task = lua.spawn(async move { + match read_to_string(path).await { + Ok(s) => Ok(Some(s)), + Err(e) if e.kind() == ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + } + }); + + // Wait for it to complete + let result = task.await.into_lua_err(); + + // We can also spawn local tasks that do take up resources + // on the Lua thread, but that do not have the Send bound + if result.is_ok() { + lua.spawn_local(async move { + println!("File read successfully!"); + }); + } + + result + })?, + )?; + + // Load the main script into a scheduler + let sched = Scheduler::new(&lua); + let main = lua.load(MAIN_SCRIPT); + sched.push_thread_front(main, ())?; + + // Run until completion + block_on(sched.run()); + + Ok(()) +} + +#[test] +fn test_basic_spawn() -> LuaResult<()> { + main() +} diff --git a/crates/mlua-luau-scheduler/examples/callbacks.rs b/crates/mlua-luau-scheduler/examples/callbacks.rs new file mode 100644 index 0000000..44a28fe --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/callbacks.rs @@ -0,0 +1,48 @@ +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::missing_panics_doc)] +#![allow(clippy::cargo_common_metadata)] + +use mlua::prelude::*; +use mlua_luau_scheduler::Scheduler; + +use async_io::block_on; + +const MAIN_SCRIPT: &str = include_str!("./lua/callbacks.luau"); + +pub fn main() -> LuaResult<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_target(false) + .without_time() + .init(); + + // Set up persistent Lua environment + let lua = Lua::new(); + + // Create a new scheduler with custom callbacks + let sched = Scheduler::new(&lua); + sched.set_error_callback(|e| { + println!( + "Captured error from Lua!\n{}\n{e}\n{}", + "-".repeat(15), + "-".repeat(15) + ); + }); + + // Load the main script into the scheduler, and keep track of the thread we spawn + let main = lua.load(MAIN_SCRIPT); + let id = sched.push_thread_front(main, ())?; + + // Run until completion + block_on(sched.run()); + + // We should have gotten the error back from our script + assert!(sched.get_thread_result(id).unwrap().is_err()); + + Ok(()) +} + +#[test] +fn test_callbacks() -> LuaResult<()> { + main() +} diff --git a/crates/mlua-luau-scheduler/examples/exit_code.rs b/crates/mlua-luau-scheduler/examples/exit_code.rs new file mode 100644 index 0000000..ee4a9a4 --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/exit_code.rs @@ -0,0 +1,43 @@ +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::missing_panics_doc)] +#![allow(clippy::cargo_common_metadata)] + +use async_io::block_on; + +use mlua::prelude::*; +use mlua_luau_scheduler::{Functions, Scheduler}; + +const MAIN_SCRIPT: &str = include_str!("./lua/exit_code.luau"); + +pub fn main() -> LuaResult<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_target(false) + .without_time() + .init(); + + // Set up persistent Lua environment + let lua = Lua::new(); + let sched = Scheduler::new(&lua); + let fns = Functions::new(&lua)?; + + lua.globals().set("exit", fns.exit)?; + + // Load the main script into the scheduler + let main = lua.load(MAIN_SCRIPT); + sched.push_thread_front(main, ())?; + + // Run until completion + block_on(sched.run()); + + // Verify that we got a correct exit code + let code = sched.get_exit_code().unwrap_or_default(); + assert!(format!("{code:?}").contains("(1)")); + + Ok(()) +} + +#[test] +fn test_exit_code() -> LuaResult<()> { + main() +} diff --git a/crates/mlua-luau-scheduler/examples/lots_of_threads.rs b/crates/mlua-luau-scheduler/examples/lots_of_threads.rs new file mode 100644 index 0000000..33451aa --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/lots_of_threads.rs @@ -0,0 +1,51 @@ +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::cargo_common_metadata)] + +use std::time::Duration; + +use async_io::{block_on, Timer}; + +use mlua::prelude::*; +use mlua_luau_scheduler::{Functions, Scheduler}; + +const MAIN_SCRIPT: &str = include_str!("./lua/lots_of_threads.luau"); + +const ONE_NANOSECOND: Duration = Duration::from_nanos(1); + +pub fn main() -> LuaResult<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_target(false) + .without_time() + .init(); + + // Set up persistent Lua environment + let lua = Lua::new(); + let sched = Scheduler::new(&lua); + let fns = Functions::new(&lua)?; + + lua.globals().set("spawn", fns.spawn)?; + lua.globals().set( + "sleep", + lua.create_async_function(|_, ()| async move { + // Obviously we can't sleep for a single nanosecond since + // this uses OS scheduling under the hood, but we can try + Timer::after(ONE_NANOSECOND).await; + Ok(()) + })?, + )?; + + // Load the main script into the scheduler + let main = lua.load(MAIN_SCRIPT); + sched.push_thread_front(main, ())?; + + // Run until completion + block_on(sched.run()); + + Ok(()) +} + +#[test] +fn test_lots_of_threads() -> LuaResult<()> { + main() +} diff --git a/crates/mlua-luau-scheduler/examples/lua/basic_sleep.luau b/crates/mlua-luau-scheduler/examples/lua/basic_sleep.luau new file mode 100644 index 0000000..74418d0 --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/lua/basic_sleep.luau @@ -0,0 +1,13 @@ +--!nocheck +--!nolint UnknownGlobal + +print("Sleeping for 3 seconds...") + +sleep(1) +print("1 second passed") + +sleep(1) +print("2 seconds passed") + +sleep(1) +print("3 seconds passed") diff --git a/crates/mlua-luau-scheduler/examples/lua/basic_spawn.luau b/crates/mlua-luau-scheduler/examples/lua/basic_spawn.luau new file mode 100644 index 0000000..b8cce6b --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/lua/basic_spawn.luau @@ -0,0 +1,17 @@ +--!nocheck +--!nolint UnknownGlobal + +local _, err = pcall(function() + local file = readFile("Cargo.toml") + if file ~= nil then + print("Cargo.toml found!") + print("Contents:") + print(file) + else + print("Cargo.toml not found!") + end +end) + +if err ~= nil then + print("Error while reading file: " .. err) +end diff --git a/crates/mlua-luau-scheduler/examples/lua/callbacks.luau b/crates/mlua-luau-scheduler/examples/lua/callbacks.luau new file mode 100644 index 0000000..77e249e --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/lua/callbacks.luau @@ -0,0 +1,4 @@ +--!nocheck +--!nolint UnknownGlobal + +error("Oh no! Something went very very wrong!") diff --git a/crates/mlua-luau-scheduler/examples/lua/exit_code.luau b/crates/mlua-luau-scheduler/examples/lua/exit_code.luau new file mode 100644 index 0000000..0c627dd --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/lua/exit_code.luau @@ -0,0 +1,8 @@ +--!nocheck +--!nolint UnknownGlobal + +print("Setting exit code manually") + +exit(1) + +error("unreachable") diff --git a/crates/mlua-luau-scheduler/examples/lua/lots_of_threads.luau b/crates/mlua-luau-scheduler/examples/lua/lots_of_threads.luau new file mode 100644 index 0000000..d25bd25 --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/lua/lots_of_threads.luau @@ -0,0 +1,29 @@ +--!nocheck +--!nolint UnknownGlobal + +local NUM_BATCHES = 10 +local NUM_THREADS = 100_000 + +print(`Spawning {NUM_BATCHES * NUM_THREADS} threads split into {NUM_BATCHES} batches\n`) + +local before = os.clock() +for i = 1, NUM_BATCHES do + print(`Batch {i} of {NUM_BATCHES}`) + local thread = coroutine.running() + + local counter = 0 + for j = 1, NUM_THREADS do + spawn(function() + sleep(0.1) + counter += 1 + if counter == NUM_THREADS then + spawn(thread) + end + end) + end + + coroutine.yield() +end +local after = os.clock() + +print(`\nSpawned {NUM_BATCHES * NUM_THREADS} sleeping threads in {after - before}s`) diff --git a/crates/mlua-luau-scheduler/examples/lua/scheduler_ordering.luau b/crates/mlua-luau-scheduler/examples/lua/scheduler_ordering.luau new file mode 100644 index 0000000..b8aed74 --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/lua/scheduler_ordering.luau @@ -0,0 +1,34 @@ +--!nocheck +--!nolint UnknownGlobal + +local nums = {} +local function insert(n: number) + table.insert(nums, n) + print(n) +end + +insert(1) + +-- Defer will run at the end of the resumption cycle, but without yielding +defer(function() + insert(5) +end) + +-- Spawn will instantly run up until the first yield, and must then be resumed manually ... +spawn(function() + insert(2) + coroutine.yield() + error("unreachable code") +end) + +-- ... unless calling functions created using `lua.create_async_function(...)`, +-- which will resume their calling thread with their result automatically +spawn(function() + insert(3) + sleep(1) + insert(6) +end) + +insert(4) + +return nums diff --git a/crates/mlua-luau-scheduler/examples/scheduler_ordering.rs b/crates/mlua-luau-scheduler/examples/scheduler_ordering.rs new file mode 100644 index 0000000..2fd4181 --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/scheduler_ordering.rs @@ -0,0 +1,56 @@ +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::missing_panics_doc)] +#![allow(clippy::cargo_common_metadata)] + +use std::time::{Duration, Instant}; + +use async_io::{block_on, Timer}; + +use mlua::prelude::*; +use mlua_luau_scheduler::{Functions, Scheduler}; + +const MAIN_SCRIPT: &str = include_str!("./lua/scheduler_ordering.luau"); + +pub fn main() -> LuaResult<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_target(false) + .without_time() + .init(); + + // Set up persistent Lua environment + let lua = Lua::new(); + let sched = Scheduler::new(&lua); + let fns = Functions::new(&lua)?; + + lua.globals().set("spawn", fns.spawn)?; + lua.globals().set("defer", fns.defer)?; + lua.globals().set( + "sleep", + lua.create_async_function(|_, duration: Option| async move { + let duration = duration.unwrap_or_default().max(1.0 / 250.0); + let before = Instant::now(); + let after = Timer::after(Duration::from_secs_f64(duration)).await; + Ok((after - before).as_secs_f64()) + })?, + )?; + + // Load the main script into the scheduler, and keep track of the thread we spawn + let main = lua.load(MAIN_SCRIPT); + let id = sched.push_thread_front(main, ())?; + + // Run until completion + block_on(sched.run()); + + // We should have gotten proper values back from our script + let res = sched.get_thread_result(id).unwrap().unwrap(); + let nums = Vec::::from_lua_multi(res, &lua)?; + assert_eq!(nums, vec![1, 2, 3, 4, 5, 6]); + + Ok(()) +} + +#[test] +fn test_scheduler_ordering() -> LuaResult<()> { + main() +} diff --git a/crates/mlua-luau-scheduler/examples/tracy.rs b/crates/mlua-luau-scheduler/examples/tracy.rs new file mode 100644 index 0000000..01732c3 --- /dev/null +++ b/crates/mlua-luau-scheduler/examples/tracy.rs @@ -0,0 +1,61 @@ +/* + NOTE: This example is the same as "lots_of_threads", but with tracy set up for performance profiling. + + How to run: + + 1. Install tracy + - Follow the instructions at https://github.com/wolfpld/tracy + - Or install via something like homebrew: `brew install tracy` + 2. Run the server (`tracy`) in a terminal + 3. Run the example in another terminal + - `export RUST_LOG=trace` + - `cargo run --example tracy` +*/ + +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::cargo_common_metadata)] + +use std::time::Duration; + +use async_io::{block_on, Timer}; +use tracing_subscriber::layer::SubscriberExt; +use tracing_tracy::{client::Client as TracyClient, TracyLayer}; + +use mlua::prelude::*; +use mlua_luau_scheduler::{Functions, Scheduler}; + +const MAIN_SCRIPT: &str = include_str!("./lua/lots_of_threads.luau"); + +const ONE_NANOSECOND: Duration = Duration::from_nanos(1); + +pub fn main() -> LuaResult<()> { + let _client = TracyClient::start(); + let _ = tracing::subscriber::set_global_default( + tracing_subscriber::registry().with(TracyLayer::default()), + ); + + // Set up persistent Lua environment + let lua = Lua::new(); + let sched = Scheduler::new(&lua); + let fns = Functions::new(&lua)?; + + lua.globals().set("spawn", fns.spawn)?; + lua.globals().set( + "sleep", + lua.create_async_function(|_, ()| async move { + // Obviously we can't sleep for a single nanosecond since + // this uses OS scheduling under the hood, but we can try + Timer::after(ONE_NANOSECOND).await; + Ok(()) + })?, + )?; + + // Load the main script into the scheduler + let main = lua.load(MAIN_SCRIPT); + sched.push_thread_front(main, ())?; + + // Run until completion + block_on(sched.run()); + + Ok(()) +} diff --git a/crates/mlua-luau-scheduler/src/error_callback.rs b/crates/mlua-luau-scheduler/src/error_callback.rs new file mode 100644 index 0000000..9d8e0a2 --- /dev/null +++ b/crates/mlua-luau-scheduler/src/error_callback.rs @@ -0,0 +1,45 @@ +use std::{cell::RefCell, rc::Rc}; + +use mlua::prelude::*; + +type ErrorCallback = Box; + +#[derive(Clone)] +pub(crate) struct ThreadErrorCallback { + inner: Rc>>, +} + +impl ThreadErrorCallback { + pub fn new() -> Self { + Self { + inner: Rc::new(RefCell::new(None)), + } + } + + pub fn replace(&self, callback: impl Fn(LuaError) + Send + 'static) { + self.inner.borrow_mut().replace(Box::new(callback)); + } + + pub fn clear(&self) { + self.inner.borrow_mut().take(); + } + + pub fn call(&self, error: &LuaError) { + if let Some(cb) = &*self.inner.borrow() { + cb(error.clone()); + } + } +} + +#[allow(clippy::needless_pass_by_value)] +fn default_error_callback(e: LuaError) { + eprintln!("{e}"); +} + +impl Default for ThreadErrorCallback { + fn default() -> Self { + let this = Self::new(); + this.replace(default_error_callback); + this + } +} diff --git a/crates/mlua-luau-scheduler/src/exit.rs b/crates/mlua-luau-scheduler/src/exit.rs new file mode 100644 index 0000000..a2794dd --- /dev/null +++ b/crates/mlua-luau-scheduler/src/exit.rs @@ -0,0 +1,31 @@ +use std::{cell::Cell, process::ExitCode, rc::Rc}; + +use event_listener::Event; + +#[derive(Debug, Clone)] +pub(crate) struct Exit { + code: Rc>>, + event: Rc, +} + +impl Exit { + pub fn new() -> Self { + Self { + code: Rc::new(Cell::new(None)), + event: Rc::new(Event::new()), + } + } + + pub fn set(&self, code: ExitCode) { + self.code.set(Some(code)); + self.event.notify(usize::MAX); + } + + pub fn get(&self) -> Option { + self.code.get() + } + + pub async fn listen(&self) { + self.event.listen().await; + } +} diff --git a/crates/mlua-luau-scheduler/src/functions.rs b/crates/mlua-luau-scheduler/src/functions.rs new file mode 100644 index 0000000..7230b99 --- /dev/null +++ b/crates/mlua-luau-scheduler/src/functions.rs @@ -0,0 +1,283 @@ +#![allow(unused_imports)] +#![allow(clippy::too_many_lines)] + +use std::process::ExitCode; + +use mlua::prelude::*; + +use crate::{ + error_callback::ThreadErrorCallback, + queue::{DeferredThreadQueue, SpawnedThreadQueue}, + result_map::ThreadResultMap, + scheduler::Scheduler, + thread_id::ThreadId, + traits::LuaSchedulerExt, + util::{is_poll_pending, LuaThreadOrFunction, ThreadResult}, +}; + +const ERR_METADATA_NOT_ATTACHED: &str = "\ +Lua state does not have scheduler metadata attached!\ +\nThis is most likely caused by creating functions outside of a scheduler.\ +\nScheduler functions must always be created from within an active scheduler.\ +"; + +const EXIT_IMPL_LUA: &str = r" +exit(...) +yield() +"; + +const WRAP_IMPL_LUA: &str = r" +local t = create(...) +return function(...) + local r = { resume(t, ...) } + if r[1] then + return select(2, unpack(r)) + else + error(r[2], 2) + end +end +"; + +/** + A collection of lua functions that may be called to interact with a [`Scheduler`]. + + Note that these may all be implemented using [`LuaSchedulerExt`], however, this struct + is implemented using internal (non-public) APIs, and generally has better performance. +*/ +pub struct Functions<'lua> { + /** + Implementation of `coroutine.resume` that handles async polling properly. + + Defers onto the scheduler queue if the thread calls an async function. + */ + pub resume: LuaFunction<'lua>, + /** + Implementation of `coroutine.wrap` that handles async polling properly. + + Defers onto the scheduler queue if the thread calls an async function. + */ + pub wrap: LuaFunction<'lua>, + /** + Resumes a function / thread once instantly, and runs until first yield. + + Spawns onto the scheduler queue if not completed. + */ + pub spawn: LuaFunction<'lua>, + /** + Defers a function / thread onto the scheduler queue. + + Does not resume instantly, only adds to the queue. + */ + pub defer: LuaFunction<'lua>, + /** + Cancels a function / thread, removing it from the queue. + */ + pub cancel: LuaFunction<'lua>, + /** + Exits the scheduler, stopping all other threads and closing the scheduler. + + Yields the calling thread to ensure that it does not continue. + */ + pub exit: LuaFunction<'lua>, +} + +impl<'lua> Functions<'lua> { + /** + Creates a new collection of Lua functions that may be called to interact with a [`Scheduler`]. + + # Errors + + Errors when out of memory, or if default Lua globals are missing. + + # Panics + + Panics when the given [`Lua`] instance does not have an attached [`Scheduler`]. + */ + pub fn new(lua: &'lua Lua) -> LuaResult { + let spawn_queue = lua + .app_data_ref::() + .expect(ERR_METADATA_NOT_ATTACHED) + .clone(); + let defer_queue = lua + .app_data_ref::() + .expect(ERR_METADATA_NOT_ATTACHED) + .clone(); + let error_callback = lua + .app_data_ref::() + .expect(ERR_METADATA_NOT_ATTACHED) + .clone(); + let result_map = lua + .app_data_ref::() + .expect(ERR_METADATA_NOT_ATTACHED) + .clone(); + + let resume_queue = defer_queue.clone(); + let resume_map = result_map.clone(); + let resume = + lua.create_function(move |lua, (thread, args): (LuaThread, LuaMultiValue)| { + let _span = tracing::trace_span!("Scheduler::fn_resume").entered(); + match thread.resume::<_, LuaMultiValue>(args.clone()) { + Ok(v) => { + if v.get(0).is_some_and(is_poll_pending) { + // Pending, defer to scheduler and return nil + resume_queue.push_item(lua, &thread, args)?; + (true, LuaValue::Nil).into_lua_multi(lua) + } else { + // Not pending, store the value if thread is done + if thread.status() != LuaThreadStatus::Resumable { + let id = ThreadId::from(&thread); + if resume_map.is_tracked(id) { + let res = ThreadResult::new(Ok(v.clone()), lua); + resume_map.insert(id, res); + } + } + (true, v).into_lua_multi(lua) + } + } + Err(e) => { + // Not pending, store the error + let id = ThreadId::from(&thread); + if resume_map.is_tracked(id) { + let res = ThreadResult::new(Err(e.clone()), lua); + resume_map.insert(id, res); + } + (false, e.to_string()).into_lua_multi(lua) + } + } + })?; + + let wrap_env = lua.create_table_from(vec![ + ("resume", resume.clone()), + ("error", lua.globals().get::<_, LuaFunction>("error")?), + ("select", lua.globals().get::<_, LuaFunction>("select")?), + ("unpack", lua.globals().get::<_, LuaFunction>("unpack")?), + ( + "create", + lua.globals() + .get::<_, LuaTable>("coroutine")? + .get::<_, LuaFunction>("create")?, + ), + ])?; + let wrap = lua + .load(WRAP_IMPL_LUA) + .set_name("=__scheduler_wrap") + .set_environment(wrap_env) + .into_function()?; + + let spawn_map = result_map.clone(); + let spawn = lua.create_function( + move |lua, (tof, args): (LuaThreadOrFunction, LuaMultiValue)| { + let _span = tracing::trace_span!("Scheduler::fn_spawn").entered(); + let thread = tof.into_thread(lua)?; + if thread.status() == LuaThreadStatus::Resumable { + // NOTE: We need to resume the thread once instantly for correct behavior, + // and only if we get the pending value back we can spawn to async executor + match thread.resume::<_, LuaMultiValue>(args.clone()) { + Ok(v) => { + if v.get(0).is_some_and(is_poll_pending) { + spawn_queue.push_item(lua, &thread, args)?; + } else { + // Not pending, store the value if thread is done + if thread.status() != LuaThreadStatus::Resumable { + let id = ThreadId::from(&thread); + if spawn_map.is_tracked(id) { + let res = ThreadResult::new(Ok(v), lua); + spawn_map.insert(id, res); + } + } + } + } + Err(e) => { + error_callback.call(&e); + // Not pending, store the error + let id = ThreadId::from(&thread); + if spawn_map.is_tracked(id) { + let res = ThreadResult::new(Err(e), lua); + spawn_map.insert(id, res); + } + } + }; + } + Ok(thread) + }, + )?; + + let defer = lua.create_function( + move |lua, (tof, args): (LuaThreadOrFunction, LuaMultiValue)| { + let _span = tracing::trace_span!("Scheduler::fn_defer").entered(); + let thread = tof.into_thread(lua)?; + if thread.status() == LuaThreadStatus::Resumable { + defer_queue.push_item(lua, &thread, args)?; + } + Ok(thread) + }, + )?; + + let close = lua + .globals() + .get::<_, LuaTable>("coroutine")? + .get::<_, LuaFunction>("close")?; + let close_key = lua.create_registry_value(close)?; + let cancel = lua.create_function(move |lua, thread: LuaThread| { + let _span = tracing::trace_span!("Scheduler::fn_cancel").entered(); + let close: LuaFunction = lua.registry_value(&close_key)?; + match close.call(thread) { + Err(LuaError::CoroutineInactive) | Ok(()) => Ok(()), + Err(e) => Err(e), + } + })?; + + let exit_env = lua.create_table_from(vec![ + ( + "exit", + lua.create_function(|lua, code: Option| { + let _span = tracing::trace_span!("Scheduler::fn_exit").entered(); + let code = code.map(ExitCode::from).unwrap_or_default(); + lua.set_exit_code(code); + Ok(()) + })?, + ), + ( + "yield", + lua.globals() + .get::<_, LuaTable>("coroutine")? + .get::<_, LuaFunction>("yield")?, + ), + ])?; + let exit = lua + .load(EXIT_IMPL_LUA) + .set_name("=__scheduler_exit") + .set_environment(exit_env) + .into_function()?; + + Ok(Self { + resume, + wrap, + spawn, + defer, + cancel, + exit, + }) + } +} + +impl Functions<'_> { + /** + Injects [`Scheduler`]-compatible functions into the given [`Lua`] instance. + + This will overwrite the following functions: + + - `coroutine.resume` + - `coroutine.wrap` + + # Errors + + Errors when out of memory, or if default Lua globals are missing. + */ + pub fn inject_compat(&self, lua: &Lua) -> LuaResult<()> { + let co: LuaTable = lua.globals().get("coroutine")?; + co.set("resume", self.resume.clone())?; + co.set("wrap", self.wrap.clone())?; + Ok(()) + } +} diff --git a/crates/mlua-luau-scheduler/src/lib.rs b/crates/mlua-luau-scheduler/src/lib.rs new file mode 100644 index 0000000..7b82595 --- /dev/null +++ b/crates/mlua-luau-scheduler/src/lib.rs @@ -0,0 +1,18 @@ +#![allow(clippy::cargo_common_metadata)] + +mod error_callback; +mod exit; +mod functions; +mod queue; +mod result_map; +mod scheduler; +mod status; +mod thread_id; +mod traits; +mod util; + +pub use functions::Functions; +pub use scheduler::Scheduler; +pub use status::Status; +pub use thread_id::ThreadId; +pub use traits::{IntoLuaThread, LuaSchedulerExt, LuaSpawnExt}; diff --git a/crates/mlua-luau-scheduler/src/queue.rs b/crates/mlua-luau-scheduler/src/queue.rs new file mode 100644 index 0000000..aabb259 --- /dev/null +++ b/crates/mlua-luau-scheduler/src/queue.rs @@ -0,0 +1,139 @@ +use std::{pin::Pin, rc::Rc}; + +use concurrent_queue::ConcurrentQueue; +use derive_more::{Deref, DerefMut}; +use event_listener::Event; +use futures_lite::{Future, FutureExt}; +use mlua::prelude::*; + +use crate::{traits::IntoLuaThread, util::ThreadWithArgs, ThreadId}; + +/** + Queue for storing [`LuaThread`]s with associated arguments. + + Provides methods for pushing and draining the queue, as + well as listening for new items being pushed to the queue. +*/ +#[derive(Debug, Clone)] +pub(crate) struct ThreadQueue { + queue: Rc>, + event: Rc, +} + +impl ThreadQueue { + pub fn new() -> Self { + let queue = Rc::new(ConcurrentQueue::unbounded()); + let event = Rc::new(Event::new()); + Self { queue, event } + } + + pub fn push_item<'lua>( + &self, + lua: &'lua Lua, + thread: impl IntoLuaThread<'lua>, + args: impl IntoLuaMulti<'lua>, + ) -> LuaResult { + let thread = thread.into_lua_thread(lua)?; + let args = args.into_lua_multi(lua)?; + + tracing::trace!("pushing item to queue with {} args", args.len()); + let id = ThreadId::from(&thread); + let stored = ThreadWithArgs::new(lua, thread, args)?; + + self.queue.push(stored).into_lua_err()?; + self.event.notify(usize::MAX); + + Ok(id) + } + + #[inline] + pub fn drain_items<'outer, 'lua>( + &'outer self, + lua: &'lua Lua, + ) -> impl Iterator, LuaMultiValue<'lua>)> + 'outer + where + 'lua: 'outer, + { + self.queue.try_iter().map(|stored| stored.into_inner(lua)) + } + + #[inline] + pub async fn wait_for_item(&self) { + if self.queue.is_empty() { + let listener = self.event.listen(); + // NOTE: Need to check again, we could have gotten + // new queued items while creating our listener + if self.queue.is_empty() { + listener.await; + } + } + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.queue.is_empty() + } +} + +/** + Alias for [`ThreadQueue`], providing a newtype to store in Lua app data. +*/ +#[derive(Debug, Clone, Deref, DerefMut)] +pub(crate) struct SpawnedThreadQueue(ThreadQueue); + +impl SpawnedThreadQueue { + pub fn new() -> Self { + Self(ThreadQueue::new()) + } +} + +/** + Alias for [`ThreadQueue`], providing a newtype to store in Lua app data. +*/ +#[derive(Debug, Clone, Deref, DerefMut)] +pub(crate) struct DeferredThreadQueue(ThreadQueue); + +impl DeferredThreadQueue { + pub fn new() -> Self { + Self(ThreadQueue::new()) + } +} + +pub type LocalBoxFuture<'fut> = Pin + 'fut>>; + +/** + Queue for storing local futures. + + Provides methods for pushing and draining the queue, as + well as listening for new items being pushed to the queue. +*/ +#[derive(Debug, Clone)] +pub(crate) struct FuturesQueue<'fut> { + queue: Rc>>, + event: Rc, +} + +impl<'fut> FuturesQueue<'fut> { + pub fn new() -> Self { + let queue = Rc::new(ConcurrentQueue::unbounded()); + let event = Rc::new(Event::new()); + Self { queue, event } + } + + pub fn push_item(&self, fut: impl Future + 'fut) { + let _ = self.queue.push(fut.boxed_local()); + self.event.notify(usize::MAX); + } + + pub fn drain_items<'outer>( + &'outer self, + ) -> impl Iterator> + 'outer { + self.queue.try_iter() + } + + pub async fn wait_for_item(&self) { + if self.queue.is_empty() { + self.event.listen().await; + } + } +} diff --git a/crates/mlua-luau-scheduler/src/result_map.rs b/crates/mlua-luau-scheduler/src/result_map.rs new file mode 100644 index 0000000..fe08a5f --- /dev/null +++ b/crates/mlua-luau-scheduler/src/result_map.rs @@ -0,0 +1,64 @@ +#![allow(clippy::inline_always)] + +use std::{cell::RefCell, rc::Rc}; + +use event_listener::Event; +// NOTE: This is the hash algorithm that mlua also uses, so we +// are not adding any additional dependencies / bloat by using it. +use rustc_hash::{FxHashMap, FxHashSet}; + +use crate::{thread_id::ThreadId, util::ThreadResult}; + +#[derive(Clone)] +pub(crate) struct ThreadResultMap { + tracked: Rc>>, + results: Rc>>, + events: Rc>>>, +} + +impl ThreadResultMap { + pub fn new() -> Self { + Self { + tracked: Rc::new(RefCell::new(FxHashSet::default())), + results: Rc::new(RefCell::new(FxHashMap::default())), + events: Rc::new(RefCell::new(FxHashMap::default())), + } + } + + #[inline(always)] + pub fn track(&self, id: ThreadId) { + self.tracked.borrow_mut().insert(id); + } + + #[inline(always)] + pub fn is_tracked(&self, id: ThreadId) -> bool { + self.tracked.borrow().contains(&id) + } + + pub fn insert(&self, id: ThreadId, result: ThreadResult) { + debug_assert!(self.is_tracked(id), "Thread must be tracked"); + self.results.borrow_mut().insert(id, result); + if let Some(event) = self.events.borrow_mut().remove(&id) { + event.notify(usize::MAX); + } + } + + pub async fn listen(&self, id: ThreadId) { + debug_assert!(self.is_tracked(id), "Thread must be tracked"); + if !self.results.borrow().contains_key(&id) { + let listener = { + let mut events = self.events.borrow_mut(); + let event = events.entry(id).or_insert_with(|| Rc::new(Event::new())); + event.listen() + }; + listener.await; + } + } + + pub fn remove(&self, id: ThreadId) -> Option { + let res = self.results.borrow_mut().remove(&id)?; + self.tracked.borrow_mut().remove(&id); + self.events.borrow_mut().remove(&id); + Some(res) + } +} diff --git a/crates/mlua-luau-scheduler/src/scheduler.rs b/crates/mlua-luau-scheduler/src/scheduler.rs new file mode 100644 index 0000000..31f699e --- /dev/null +++ b/crates/mlua-luau-scheduler/src/scheduler.rs @@ -0,0 +1,484 @@ +#![allow(clippy::module_name_repetitions)] + +use std::{ + cell::Cell, + process::ExitCode, + rc::{Rc, Weak as WeakRc}, + sync::{Arc, Weak as WeakArc}, + thread::panicking, +}; + +use futures_lite::prelude::*; +use mlua::prelude::*; + +use async_executor::{Executor, LocalExecutor}; +use tracing::{debug, instrument, trace, trace_span, Instrument}; + +use crate::{ + error_callback::ThreadErrorCallback, + exit::Exit, + queue::{DeferredThreadQueue, FuturesQueue, SpawnedThreadQueue}, + result_map::ThreadResultMap, + status::Status, + thread_id::ThreadId, + traits::IntoLuaThread, + util::{run_until_yield, ThreadResult}, +}; + +const ERR_METADATA_ALREADY_ATTACHED: &str = "\ +Lua state already has scheduler metadata attached!\ +\nThis may be caused by running multiple schedulers on the same Lua state, or a call to Scheduler::run being cancelled.\ +\nOnly one scheduler can be used per Lua state at once, and schedulers must always run until completion.\ +"; + +const ERR_METADATA_REMOVED: &str = "\ +Lua state scheduler metadata was unexpectedly removed!\ +\nThis should never happen, and is likely a bug in the scheduler.\ +"; + +const ERR_SET_CALLBACK_WHEN_RUNNING: &str = "\ +Cannot set error callback when scheduler is running!\ +"; + +/** + A scheduler for running Lua threads and async tasks. +*/ +#[derive(Clone)] +pub struct Scheduler<'lua> { + lua: &'lua Lua, + queue_spawn: SpawnedThreadQueue, + queue_defer: DeferredThreadQueue, + error_callback: ThreadErrorCallback, + result_map: ThreadResultMap, + status: Rc>, + exit: Exit, +} + +impl<'lua> Scheduler<'lua> { + /** + Creates a new scheduler for the given Lua state. + + This scheduler will have a default error callback that prints errors to stderr. + + # Panics + + Panics if the given Lua state already has a scheduler attached to it. + */ + #[must_use] + pub fn new(lua: &'lua Lua) -> Scheduler<'lua> { + let queue_spawn = SpawnedThreadQueue::new(); + let queue_defer = DeferredThreadQueue::new(); + let error_callback = ThreadErrorCallback::default(); + let result_map = ThreadResultMap::new(); + let exit = Exit::new(); + + assert!( + lua.app_data_ref::().is_none(), + "{ERR_METADATA_ALREADY_ATTACHED}" + ); + assert!( + lua.app_data_ref::().is_none(), + "{ERR_METADATA_ALREADY_ATTACHED}" + ); + assert!( + lua.app_data_ref::().is_none(), + "{ERR_METADATA_ALREADY_ATTACHED}" + ); + assert!( + lua.app_data_ref::().is_none(), + "{ERR_METADATA_ALREADY_ATTACHED}" + ); + assert!( + lua.app_data_ref::().is_none(), + "{ERR_METADATA_ALREADY_ATTACHED}" + ); + + lua.set_app_data(queue_spawn.clone()); + lua.set_app_data(queue_defer.clone()); + lua.set_app_data(error_callback.clone()); + lua.set_app_data(result_map.clone()); + lua.set_app_data(exit.clone()); + + let status = Rc::new(Cell::new(Status::NotStarted)); + + Scheduler { + lua, + queue_spawn, + queue_defer, + error_callback, + result_map, + status, + exit, + } + } + + /** + Sets the current status of this scheduler and emits relevant tracing events. + */ + fn set_status(&self, status: Status) { + debug!(status = ?status, "status"); + self.status.set(status); + } + + /** + Returns the current status of this scheduler. + */ + #[must_use] + pub fn status(&self) -> Status { + self.status.get() + } + + /** + Sets the error callback for this scheduler. + + This callback will be called whenever a Lua thread errors. + + Overwrites any previous error callback. + + # Panics + + Panics if the scheduler is currently running. + */ + pub fn set_error_callback(&self, callback: impl Fn(LuaError) + Send + 'static) { + assert!( + !self.status().is_running(), + "{ERR_SET_CALLBACK_WHEN_RUNNING}" + ); + self.error_callback.replace(callback); + } + + /** + Clears the error callback for this scheduler. + + This will remove any current error callback, including default(s). + + # Panics + + Panics if the scheduler is currently running. + */ + pub fn remove_error_callback(&self) { + assert!( + !self.status().is_running(), + "{ERR_SET_CALLBACK_WHEN_RUNNING}" + ); + self.error_callback.clear(); + } + + /** + Gets the exit code for this scheduler, if one has been set. + */ + #[must_use] + pub fn get_exit_code(&self) -> Option { + self.exit.get() + } + + /** + Sets the exit code for this scheduler. + + This will cause [`Scheduler::run`] to exit immediately. + */ + pub fn set_exit_code(&self, code: ExitCode) { + self.exit.set(code); + } + + /** + Spawns a chunk / function / thread onto the scheduler queue. + + Threads are guaranteed to be resumed in the order that they were pushed to the queue. + + # Returns + + Returns a [`ThreadId`] that can be used to retrieve the result of the thread. + + Note that the result may not be available until [`Scheduler::run`] completes. + + # Errors + + Errors when out of memory. + */ + pub fn push_thread_front( + &self, + thread: impl IntoLuaThread<'lua>, + args: impl IntoLuaMulti<'lua>, + ) -> LuaResult { + let id = self.queue_spawn.push_item(self.lua, thread, args)?; + self.result_map.track(id); + Ok(id) + } + + /** + Defers a chunk / function / thread onto the scheduler queue. + + Deferred threads are guaranteed to run after all spawned threads either yield or complete. + + Threads are guaranteed to be resumed in the order that they were pushed to the queue. + + # Returns + + Returns a [`ThreadId`] that can be used to retrieve the result of the thread. + + Note that the result may not be available until [`Scheduler::run`] completes. + + # Errors + + Errors when out of memory. + */ + pub fn push_thread_back( + &self, + thread: impl IntoLuaThread<'lua>, + args: impl IntoLuaMulti<'lua>, + ) -> LuaResult { + let id = self.queue_defer.push_item(self.lua, thread, args)?; + self.result_map.track(id); + Ok(id) + } + + /** + Gets the tracked result for the [`LuaThread`] with the given [`ThreadId`]. + + Depending on the current [`Scheduler::status`], this method will return: + + - [`Status::NotStarted`]: returns `None`. + - [`Status::Running`]: may return `Some(Ok(v))` or `Some(Err(e))`, but it is not guaranteed. + - [`Status::Completed`]: returns `Some(Ok(v))` or `Some(Err(e))`. + + Note that this method also takes the value out of the scheduler and + stops tracking the given thread, so it may only be called once. + + Any subsequent calls after this method returns `Some` will return `None`. + */ + #[must_use] + pub fn get_thread_result(&self, id: ThreadId) -> Option>> { + self.result_map.remove(id).map(|r| r.value(self.lua)) + } + + /** + Waits for the [`LuaThread`] with the given [`ThreadId`] to complete. + + This will return instantly if the thread has already completed. + */ + pub async fn wait_for_thread(&self, id: ThreadId) { + self.result_map.listen(id).await; + } + + /** + Runs the scheduler until all Lua threads have completed. + + Note that the given Lua state must be the same one that was + used to create this scheduler, otherwise this method will panic. + + # Panics + + Panics if the given Lua state already has a scheduler attached to it. + */ + #[allow(clippy::too_many_lines)] + #[instrument(level = "debug", name = "Scheduler::run", skip(self))] + pub async fn run(&self) { + /* + Create new executors to use - note that we do not need create multiple executors + for work stealing, the user may do that themselves if they want to and it will work + just fine, as long as anything async is .await-ed from within a Lua async function. + + The main purpose of the two executors here is just to have one with + the Send bound, and another (local) one without it, for Lua scheduling. + + We also use the main executor to drive the main loop below forward, + saving a tiny bit of processing from going on the Lua executor itself. + */ + let local_exec = LocalExecutor::new(); + let main_exec = Arc::new(Executor::new()); + let fut_queue = Rc::new(FuturesQueue::new()); + + /* + Store the main executor and queue in Lua, so that they may be used with LuaSchedulerExt. + + Also ensure we do not already have an executor or queues - these are definite user errors + and may happen if the user tries to run multiple schedulers on the same Lua state at once. + */ + assert!( + self.lua.app_data_ref::>().is_none(), + "{ERR_METADATA_ALREADY_ATTACHED}" + ); + assert!( + self.lua.app_data_ref::>().is_none(), + "{ERR_METADATA_ALREADY_ATTACHED}" + ); + + self.lua.set_app_data(Arc::downgrade(&main_exec)); + self.lua.set_app_data(Rc::downgrade(&fut_queue.clone())); + + /* + Manually tick the Lua executor, while running under the main executor. + Each tick we wait for the next action to perform, in prioritized order: + + 1. The exit event is triggered by setting an exit code + 2. A Lua thread is available to run on the spawned queue + 3. A Lua thread is available to run on the deferred queue + 4. A new thread-local future is available to run on the local executor + 5. Task(s) scheduled on the Lua executor have made progress and should be polled again + + This ordering is vital to ensure that we don't accidentally exit the main loop + when there are new Lua threads to enqueue and potentially more work to be done. + */ + let fut = async { + let result_map = self.result_map.clone(); + let process_thread = |thread: LuaThread<'lua>, args| { + // NOTE: Thread may have been cancelled from Lua + // before we got here, so we need to check it again + if thread.status() == LuaThreadStatus::Resumable { + // Check if we should be tracking this thread + let id = ThreadId::from(&thread); + let id_tracked = result_map.is_tracked(id); + let result_map_inner = if id_tracked { + Some(result_map.clone()) + } else { + None + }; + // Create our future which will run the thread and store its final result + let fut = async move { + if id_tracked { + // Run until yield and check if we got a final result + if let Some(res) = run_until_yield(thread.clone(), args).await { + if let Err(e) = res.as_ref() { + self.error_callback.call(e); + } + if thread.status() != LuaThreadStatus::Resumable { + let thread_res = ThreadResult::new(res, self.lua); + result_map_inner.unwrap().insert(id, thread_res); + } + } + } else { + // Just run until yield + if let Some(res) = run_until_yield(thread, args).await { + if let Err(e) = res.as_ref() { + self.error_callback.call(e); + } + } + } + }; + // Spawn it on the executor + local_exec.spawn(fut).detach(); + } + }; + + loop { + let fut_exit = self.exit.listen(); // 1 + let fut_spawn = self.queue_spawn.wait_for_item(); // 2 + let fut_defer = self.queue_defer.wait_for_item(); // 3 + let fut_futs = fut_queue.wait_for_item(); // 4 + + // 5 + let mut num_processed = 0; + let span_tick = trace_span!("Scheduler::tick"); + let fut_tick = async { + local_exec.tick().await; + // NOTE: Try to do as much work as possible instead of just a single tick() + num_processed += 1; + while local_exec.try_tick() { + num_processed += 1; + } + }; + + // 1 + 2 + 3 + 4 + 5 + fut_exit + .or(fut_spawn) + .or(fut_defer) + .or(fut_futs) + .or(fut_tick.instrument(span_tick.or_current())) + .await; + + // Check if we should exit + if self.exit.get().is_some() { + debug!("exit signal received"); + break; + } + + // Process spawned threads first, then deferred threads, then futures + let mut num_spawned = 0; + let mut num_deferred = 0; + let mut num_futures = 0; + { + let _span = trace_span!("Scheduler::drain_spawned").entered(); + for (thread, args) in self.queue_spawn.drain_items(self.lua) { + process_thread(thread, args); + num_spawned += 1; + } + } + { + let _span = trace_span!("Scheduler::drain_deferred").entered(); + for (thread, args) in self.queue_defer.drain_items(self.lua) { + process_thread(thread, args); + num_deferred += 1; + } + } + { + let _span = trace_span!("Scheduler::drain_futures").entered(); + for fut in fut_queue.drain_items() { + local_exec.spawn(fut).detach(); + num_futures += 1; + } + } + + // Empty executor = we didn't spawn any new Lua tasks + // above, and there are no remaining tasks to run later + let completed = local_exec.is_empty() + && self.queue_spawn.is_empty() + && self.queue_defer.is_empty(); + trace!( + futures_spawned = num_futures, + futures_processed = num_processed, + lua_threads_spawned = num_spawned, + lua_threads_deferred = num_deferred, + "loop" + ); + if completed { + break; + } + } + }; + + // Run the executor inside a span until all lua threads complete + self.set_status(Status::Running); + main_exec.run(fut).await; + self.set_status(Status::Completed); + + // Clean up + self.lua + .remove_app_data::>() + .expect(ERR_METADATA_REMOVED); + self.lua + .remove_app_data::>() + .expect(ERR_METADATA_REMOVED); + } +} + +impl Drop for Scheduler<'_> { + fn drop(&mut self) { + if panicking() { + // Do not cause further panics if already panicking, as + // this may abort the program instead of safely unwinding + self.lua.remove_app_data::(); + self.lua.remove_app_data::(); + self.lua.remove_app_data::(); + self.lua.remove_app_data::(); + self.lua.remove_app_data::(); + } else { + // In any other case we panic if metadata was removed incorrectly + self.lua + .remove_app_data::() + .expect(ERR_METADATA_REMOVED); + self.lua + .remove_app_data::() + .expect(ERR_METADATA_REMOVED); + self.lua + .remove_app_data::() + .expect(ERR_METADATA_REMOVED); + self.lua + .remove_app_data::() + .expect(ERR_METADATA_REMOVED); + self.lua + .remove_app_data::() + .expect(ERR_METADATA_REMOVED); + } + } +} diff --git a/crates/mlua-luau-scheduler/src/status.rs b/crates/mlua-luau-scheduler/src/status.rs new file mode 100644 index 0000000..e9c139b --- /dev/null +++ b/crates/mlua-luau-scheduler/src/status.rs @@ -0,0 +1,31 @@ +#![allow(clippy::module_name_repetitions)] + +/** + The current status of a scheduler. +*/ +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Status { + /// The scheduler has not yet started running. + NotStarted, + /// The scheduler is currently running. + Running, + /// The scheduler has completed. + Completed, +} + +impl Status { + #[must_use] + pub const fn is_not_started(self) -> bool { + matches!(self, Self::NotStarted) + } + + #[must_use] + pub const fn is_running(self) -> bool { + matches!(self, Self::Running) + } + + #[must_use] + pub const fn is_completed(self) -> bool { + matches!(self, Self::Completed) + } +} diff --git a/crates/mlua-luau-scheduler/src/thread_id.rs b/crates/mlua-luau-scheduler/src/thread_id.rs new file mode 100644 index 0000000..e2efcaa --- /dev/null +++ b/crates/mlua-luau-scheduler/src/thread_id.rs @@ -0,0 +1,30 @@ +use std::hash::{Hash, Hasher}; + +use mlua::prelude::*; + +/** + Opaque and unique ID representing a [`LuaThread`]. + + Typically used for associating metadata with a thread in a structure such as a `HashMap`. + + Note that holding a `ThreadId` does not prevent the thread from being garbage collected. + The actual thread may or may not still exist and be active at any given point in time. +*/ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ThreadId { + inner: usize, +} + +impl From<&LuaThread<'_>> for ThreadId { + fn from(thread: &LuaThread) -> Self { + Self { + inner: thread.to_pointer() as usize, + } + } +} + +impl Hash for ThreadId { + fn hash(&self, state: &mut H) { + self.inner.hash(state); + } +} diff --git a/crates/mlua-luau-scheduler/src/traits.rs b/crates/mlua-luau-scheduler/src/traits.rs new file mode 100644 index 0000000..cbe2e6e --- /dev/null +++ b/crates/mlua-luau-scheduler/src/traits.rs @@ -0,0 +1,378 @@ +#![allow(unused_imports)] +#![allow(clippy::missing_errors_doc)] + +use std::{ + cell::Cell, future::Future, process::ExitCode, rc::Weak as WeakRc, sync::Weak as WeakArc, +}; + +use async_executor::{Executor, Task}; +use mlua::prelude::*; +use tracing::trace; + +use crate::{ + exit::Exit, + queue::{DeferredThreadQueue, FuturesQueue, SpawnedThreadQueue}, + result_map::ThreadResultMap, + scheduler::Scheduler, + thread_id::ThreadId, +}; + +/** + Trait for any struct that can be turned into an [`LuaThread`] + and passed to the scheduler, implemented for the following types: + + - Lua threads ([`LuaThread`]) + - Lua functions ([`LuaFunction`]) + - Lua chunks ([`LuaChunk`]) +*/ +pub trait IntoLuaThread<'lua> { + /** + Converts the value into a Lua thread. + + # Errors + + Errors when out of memory. + */ + fn into_lua_thread(self, lua: &'lua Lua) -> LuaResult>; +} + +impl<'lua> IntoLuaThread<'lua> for LuaThread<'lua> { + fn into_lua_thread(self, _: &'lua Lua) -> LuaResult> { + Ok(self) + } +} + +impl<'lua> IntoLuaThread<'lua> for LuaFunction<'lua> { + fn into_lua_thread(self, lua: &'lua Lua) -> LuaResult> { + lua.create_thread(self) + } +} + +impl<'lua> IntoLuaThread<'lua> for LuaChunk<'lua, '_> { + fn into_lua_thread(self, lua: &'lua Lua) -> LuaResult> { + lua.create_thread(self.into_function()?) + } +} + +impl<'lua, T> IntoLuaThread<'lua> for &T +where + T: IntoLuaThread<'lua> + Clone, +{ + fn into_lua_thread(self, lua: &'lua Lua) -> LuaResult> { + self.clone().into_lua_thread(lua) + } +} + +/** + Trait for interacting with the current [`Scheduler`]. + + Provides extra methods on the [`Lua`] struct for: + + - Setting the exit code and forcibly stopping the scheduler + - Pushing (spawning) and deferring (pushing to the back) lua threads + - Tracking and getting the result of lua threads +*/ +pub trait LuaSchedulerExt<'lua> { + /** + Sets the exit code of the current scheduler. + + See [`Scheduler::set_exit_code`] for more information. + + # Panics + + Panics if called outside of a running [`Scheduler`]. + */ + fn set_exit_code(&self, code: ExitCode); + + /** + Pushes (spawns) a lua thread to the **front** of the current scheduler. + + See [`Scheduler::push_thread_front`] for more information. + + # Panics + + Panics if called outside of a running [`Scheduler`]. + */ + fn push_thread_front( + &'lua self, + thread: impl IntoLuaThread<'lua>, + args: impl IntoLuaMulti<'lua>, + ) -> LuaResult; + + /** + Pushes (defers) a lua thread to the **back** of the current scheduler. + + See [`Scheduler::push_thread_back`] for more information. + + # Panics + + Panics if called outside of a running [`Scheduler`]. + */ + fn push_thread_back( + &'lua self, + thread: impl IntoLuaThread<'lua>, + args: impl IntoLuaMulti<'lua>, + ) -> LuaResult; + + /** + Registers the given thread to be tracked within the current scheduler. + + Must be called before waiting for a thread to complete or getting its result. + */ + fn track_thread(&'lua self, id: ThreadId); + + /** + Gets the result of the given thread. + + See [`Scheduler::get_thread_result`] for more information. + + # Panics + + Panics if called outside of a running [`Scheduler`]. + */ + fn get_thread_result(&'lua self, id: ThreadId) -> Option>>; + + /** + Waits for the given thread to complete. + + See [`Scheduler::wait_for_thread`] for more information. + + # Panics + + Panics if called outside of a running [`Scheduler`]. + */ + fn wait_for_thread(&'lua self, id: ThreadId) -> impl Future; +} + +/** + Trait for interacting with the [`Executor`] for the current [`Scheduler`]. + + Provides extra methods on the [`Lua`] struct for: + + - Spawning thread-local (`!Send`) futures on the current executor + - Spawning background (`Send`) futures on the current executor + - Spawning blocking tasks on a separate thread pool +*/ +pub trait LuaSpawnExt<'lua> { + /** + Spawns the given future on the current executor and returns its [`Task`]. + + # Panics + + Panics if called outside of a running [`Scheduler`]. + + # Example usage + + ```rust + use async_io::block_on; + + use mlua::prelude::*; + use mlua_luau_scheduler::*; + + fn main() -> LuaResult<()> { + let lua = Lua::new(); + + lua.globals().set( + "spawnBackgroundTask", + lua.create_async_function(|lua, ()| async move { + lua.spawn(async move { + println!("Hello from background task!"); + }).await; + Ok(()) + })? + )?; + + let sched = Scheduler::new(&lua); + sched.push_thread_front(lua.load("spawnBackgroundTask()"), ()); + block_on(sched.run()); + + Ok(()) + } + ``` + */ + fn spawn(&self, fut: F) -> Task + where + F: Future + Send + 'static, + T: Send + 'static; + + /** + Spawns the given thread-local future on the current executor. + + Note that this future will run detached and always to completion, + preventing the [`Scheduler`] was spawned on from completing until done. + + # Panics + + Panics if called outside of a running [`Scheduler`]. + + # Example usage + + ```rust + use async_io::block_on; + + use mlua::prelude::*; + use mlua_luau_scheduler::*; + + fn main() -> LuaResult<()> { + let lua = Lua::new(); + + lua.globals().set( + "spawnLocalTask", + lua.create_async_function(|lua, ()| async move { + lua.spawn_local(async move { + println!("Hello from local task!"); + }); + Ok(()) + })? + )?; + + let sched = Scheduler::new(&lua); + sched.push_thread_front(lua.load("spawnLocalTask()"), ()); + block_on(sched.run()); + + Ok(()) + } + ``` + */ + fn spawn_local(&self, fut: F) + where + F: Future + 'static; + + /** + Spawns the given blocking function and returns its [`Task`]. + + This function will run on a separate thread pool and not block the current executor. + + # Panics + + Panics if called outside of a running [`Scheduler`]. + + # Example usage + + ```rust + use async_io::block_on; + + use mlua::prelude::*; + use mlua_luau_scheduler::*; + + fn main() -> LuaResult<()> { + let lua = Lua::new(); + + lua.globals().set( + "spawnBlockingTask", + lua.create_async_function(|lua, ()| async move { + lua.spawn_blocking(|| { + println!("Hello from blocking task!"); + }).await; + Ok(()) + })? + )?; + + let sched = Scheduler::new(&lua); + sched.push_thread_front(lua.load("spawnBlockingTask()"), ()); + block_on(sched.run()); + + Ok(()) + } + ``` + */ + fn spawn_blocking(&self, f: F) -> Task + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static; +} + +impl<'lua> LuaSchedulerExt<'lua> for Lua { + fn set_exit_code(&self, code: ExitCode) { + let exit = self + .app_data_ref::() + .expect("exit code can only be set from within an active scheduler"); + exit.set(code); + } + + fn push_thread_front( + &'lua self, + thread: impl IntoLuaThread<'lua>, + args: impl IntoLuaMulti<'lua>, + ) -> LuaResult { + let queue = self + .app_data_ref::() + .expect("lua threads can only be pushed from within an active scheduler"); + queue.push_item(self, thread, args) + } + + fn push_thread_back( + &'lua self, + thread: impl IntoLuaThread<'lua>, + args: impl IntoLuaMulti<'lua>, + ) -> LuaResult { + let queue = self + .app_data_ref::() + .expect("lua threads can only be pushed from within an active scheduler"); + queue.push_item(self, thread, args) + } + + fn track_thread(&'lua self, id: ThreadId) { + let map = self + .app_data_ref::() + .expect("lua threads can only be tracked from within an active scheduler"); + map.track(id); + } + + fn get_thread_result(&'lua self, id: ThreadId) -> Option>> { + let map = self + .app_data_ref::() + .expect("lua threads results can only be retrieved from within an active scheduler"); + map.remove(id).map(|r| r.value(self)) + } + + fn wait_for_thread(&'lua self, id: ThreadId) -> impl Future { + let map = self + .app_data_ref::() + .expect("lua threads results can only be retrieved from within an active scheduler"); + async move { map.listen(id).await } + } +} + +impl<'lua> LuaSpawnExt<'lua> for Lua { + fn spawn(&self, fut: F) -> Task + where + F: Future + Send + 'static, + T: Send + 'static, + { + let exec = self + .app_data_ref::>() + .expect("tasks can only be spawned within an active scheduler") + .upgrade() + .expect("executor was dropped"); + trace!("spawning future on executor"); + exec.spawn(fut) + } + + fn spawn_local(&self, fut: F) + where + F: Future + 'static, + { + let queue = self + .app_data_ref::>() + .expect("tasks can only be spawned within an active scheduler") + .upgrade() + .expect("executor was dropped"); + trace!("spawning local task on executor"); + queue.push_item(fut); + } + + fn spawn_blocking(&self, f: F) -> Task + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, + { + let exec = self + .app_data_ref::>() + .expect("tasks can only be spawned within an active scheduler") + .upgrade() + .expect("executor was dropped"); + trace!("spawning blocking task on executor"); + exec.spawn(blocking::unblock(f)) + } +} diff --git a/crates/mlua-luau-scheduler/src/util.rs b/crates/mlua-luau-scheduler/src/util.rs new file mode 100644 index 0000000..2fe537b --- /dev/null +++ b/crates/mlua-luau-scheduler/src/util.rs @@ -0,0 +1,147 @@ +use futures_lite::StreamExt; +use mlua::prelude::*; +use tracing::instrument; + +/** + Runs a Lua thread until it manually yields (using coroutine.yield), errors, or completes. + + May return `None` if the thread was cancelled. + + Otherwise returns the values yielded by the thread, or the error that caused it to stop. +*/ +#[instrument(level = "trace", name = "Scheduler::run_until_yield", skip_all)] +pub(crate) async fn run_until_yield<'lua>( + thread: LuaThread<'lua>, + args: LuaMultiValue<'lua>, +) -> Option>> { + let mut stream = thread.into_async(args); + /* + NOTE: It is very important that we drop the thread/stream as + soon as we are done, it takes up valuable Lua registry space + and detached tasks will not drop until the executor does + + https://github.com/smol-rs/smol/issues/294 + + We also do not unwrap here since returning `None` is expected behavior for cancellation. + + Even though we are converting into a stream, and then immediately running it, + the future may still be cancelled before it is polled, which gives us None. + */ + stream.next().await +} + +/** + Checks if the given [`LuaValue`] is the async `POLL_PENDING` constant. +*/ +#[inline] +pub(crate) fn is_poll_pending(value: &LuaValue) -> bool { + value + .as_light_userdata() + .is_some_and(|l| l == Lua::poll_pending()) +} + +/** + Representation of a [`LuaResult`] with an associated [`LuaMultiValue`] currently stored in the Lua registry. +*/ +#[derive(Debug)] +pub(crate) struct ThreadResult { + inner: LuaResult, +} + +impl ThreadResult { + pub fn new(result: LuaResult, lua: &Lua) -> Self { + Self { + inner: match result { + Ok(v) => Ok({ + let vec = v.into_vec(); + lua.create_registry_value(vec).expect("out of memory") + }), + Err(e) => Err(e), + }, + } + } + + pub fn value(self, lua: &Lua) -> LuaResult { + match self.inner { + Ok(key) => { + let vec = lua.registry_value(&key).unwrap(); + lua.remove_registry_value(key).unwrap(); + Ok(LuaMultiValue::from_vec(vec)) + } + Err(e) => Err(e.clone()), + } + } +} + +/** + Representation of a [`LuaThread`] with its associated arguments currently stored in the Lua registry. +*/ +#[derive(Debug)] +pub(crate) struct ThreadWithArgs { + key_thread: LuaRegistryKey, + key_args: LuaRegistryKey, +} + +impl ThreadWithArgs { + pub fn new<'lua>( + lua: &'lua Lua, + thread: LuaThread<'lua>, + args: LuaMultiValue<'lua>, + ) -> LuaResult { + let argsv = args.into_vec(); + + let key_thread = lua.create_registry_value(thread)?; + let key_args = lua.create_registry_value(argsv)?; + + Ok(Self { + key_thread, + key_args, + }) + } + + pub fn into_inner(self, lua: &Lua) -> (LuaThread<'_>, LuaMultiValue<'_>) { + let thread = lua.registry_value(&self.key_thread).unwrap(); + let argsv = lua.registry_value(&self.key_args).unwrap(); + + let args = LuaMultiValue::from_vec(argsv); + + lua.remove_registry_value(self.key_thread).unwrap(); + lua.remove_registry_value(self.key_args).unwrap(); + + (thread, args) + } +} + +/** + Wrapper struct to accept either a Lua thread or a Lua function as function argument. + + [`LuaThreadOrFunction::into_thread`] may be used to convert the value into a Lua thread. +*/ +#[derive(Clone)] +pub(crate) enum LuaThreadOrFunction<'lua> { + Thread(LuaThread<'lua>), + Function(LuaFunction<'lua>), +} + +impl<'lua> LuaThreadOrFunction<'lua> { + pub(super) fn into_thread(self, lua: &'lua Lua) -> LuaResult> { + match self { + Self::Thread(t) => Ok(t), + Self::Function(f) => lua.create_thread(f), + } + } +} + +impl<'lua> FromLua<'lua> for LuaThreadOrFunction<'lua> { + fn from_lua(value: LuaValue<'lua>, _: &'lua Lua) -> LuaResult { + match value { + LuaValue::Thread(t) => Ok(Self::Thread(t)), + LuaValue::Function(f) => Ok(Self::Function(f)), + value => Err(LuaError::FromLuaConversionError { + from: value.type_name(), + to: "LuaThreadOrFunction", + message: Some("Expected thread or function".to_string()), + }), + } + } +} From 91ac6b00c1d1491ef156f601ace588a7994572f7 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 16:53:19 +0200 Subject: [PATCH 16/27] Make sure build, lint, test workflow runs for entire workspace --- .github/workflows/ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c5ebff1..6da758d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -88,17 +88,20 @@ jobs: - name: Build run: | cargo build \ + --workspace --locked --all-features \ --target ${{ matrix.cargo-target }} - name: Lint run: | cargo clippy \ + --workspace --locked --all-features \ --target ${{ matrix.cargo-target }} - name: Test run: | cargo test \ + --lib --workspace --locked --all-features \ --target ${{ matrix.cargo-target }} From 9d9f1685d8942938389c46363ca274fd305b0e5c Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 16:53:36 +0200 Subject: [PATCH 17/27] Update tooling --- aftman.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aftman.toml b/aftman.toml index 3b4fcc9..4441174 100644 --- a/aftman.toml +++ b/aftman.toml @@ -1,4 +1,4 @@ [tools] -luau-lsp = "JohnnyMorganz/luau-lsp@1.29.0" +luau-lsp = "JohnnyMorganz/luau-lsp@1.29.1" selene = "Kampfkarren/selene@0.27.1" stylua = "JohnnyMorganz/StyLua@0.20.0" From 8cb7b8a13a2b011037db3e06b436bdb70bb01adf Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 17:38:19 +0200 Subject: [PATCH 18/27] Fix CI --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6da758d..088884a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -88,20 +88,20 @@ jobs: - name: Build run: | cargo build \ - --workspace + --workspace \ --locked --all-features \ --target ${{ matrix.cargo-target }} - name: Lint run: | cargo clippy \ - --workspace + --workspace \ --locked --all-features \ --target ${{ matrix.cargo-target }} - name: Test run: | cargo test \ - --lib --workspace + --lib --workspace \ --locked --all-features \ --target ${{ matrix.cargo-target }} From 63493e78de43a525a6531a572a653ac7b678b5bd Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 18:33:21 +0200 Subject: [PATCH 19/27] Get sandbox working --- crates/lune-std-task/src/lib.rs | 6 ----- crates/lune/src/rt/runtime.rs | 47 ++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/crates/lune-std-task/src/lib.rs b/crates/lune-std-task/src/lib.rs index 47a78d5..dce0873 100644 --- a/crates/lune-std-task/src/lib.rs +++ b/crates/lune-std-task/src/lib.rs @@ -33,12 +33,6 @@ pub fn module(lua: &Lua) -> LuaResult { .set_environment(task_delay_env) .into_function()?; - // Overwrite resume & wrap functions on the coroutine global - // with ones that are compatible with our scheduler - let co = lua.globals().get::<_, LuaTable>("coroutine")?; - co.set("resume", fns.resume.clone())?; - co.set("wrap", fns.wrap.clone())?; - TableBuilder::new(lua)? .with_value("cancel", fns.cancel)? .with_value("defer", fns.defer)? diff --git a/crates/lune/src/rt/runtime.rs b/crates/lune/src/rt/runtime.rs index a5501c6..a302f53 100644 --- a/crates/lune/src/rt/runtime.rs +++ b/crates/lune/src/rt/runtime.rs @@ -9,8 +9,8 @@ use std::{ }, }; -use mlua::Lua; -use mlua_luau_scheduler::Scheduler; +use mlua::prelude::*; +use mlua_luau_scheduler::{Functions, Scheduler}; use super::{RuntimeError, RuntimeResult}; @@ -94,6 +94,12 @@ impl Runtime { eprintln!("{}", RuntimeError::from(e)); }); + // Overwrite resume & wrap functions on the coroutine global + // with ones that are compatible with our scheduler + // We also sandbox the VM, preventing further modifications + // to the global environment, and enabling optimizations + inject_scheduler_functions_and_sandbox(&self.lua)?; + // Load our "main" thread let main = self .lua @@ -105,12 +111,45 @@ impl Runtime { sched.run().await; // Return the exit code - default to FAILURE if we got any errors - Ok(sched.get_exit_code().unwrap_or({ + let exit_code = sched.get_exit_code().unwrap_or({ if got_any_error.load(Ordering::SeqCst) { ExitCode::FAILURE } else { ExitCode::SUCCESS } - })) + }); + + Ok(exit_code) } } + +fn inject_scheduler_functions_and_sandbox(lua: &Lua) -> LuaResult<()> { + let fns = Functions::new(lua)?; + + let co = lua.globals().get::<_, LuaTable>("coroutine")?; + co.set("resume", fns.resume.clone())?; + co.set("wrap", fns.wrap.clone())?; + + lua.sandbox(true)?; + + // NOTE: We need to create the _G table after + // sandboxing, otherwise it will be read-only + #[cfg(any( + feature = "std-datetime", + feature = "std-fs", + feature = "std-luau", + feature = "std-net", + feature = "std-process", + feature = "std-regex", + feature = "std-roblox", + feature = "std-serde", + feature = "std-stdio", + feature = "std-task", + ))] + { + let g_global = lune_std::LuneStandardGlobal::GTable; + lua.globals().set(g_global.name(), g_global.create(lua)?)?; + } + + Ok(()) +} From a94c9d6d54ff4f416b32aee62c37131c5bc8b2d2 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 18:50:23 +0200 Subject: [PATCH 20/27] Create inner runtime struct to preserve scheduler, globals, and sandboxing across runs --- Cargo.lock | 1 + crates/lune/Cargo.toml | 1 + crates/lune/src/rt/runtime.rs | 158 ++++++++++++++++++++-------------- 3 files changed, 93 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca25bc5..e6c68e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1521,6 +1521,7 @@ dependencies = [ "once_cell", "reqwest", "rustyline", + "self_cell", "serde", "serde_json", "thiserror", diff --git a/crates/lune/Cargo.toml b/crates/lune/Cargo.toml index ca5094f..ea5dac7 100644 --- a/crates/lune/Cargo.toml +++ b/crates/lune/Cargo.toml @@ -59,6 +59,7 @@ dialoguer = "0.11" directories = "5.0" futures-util = "0.3" once_cell = "1.17" +self_cell = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" diff --git a/crates/lune/src/rt/runtime.rs b/crates/lune/src/rt/runtime.rs index a302f53..5d515ca 100644 --- a/crates/lune/src/rt/runtime.rs +++ b/crates/lune/src/rt/runtime.rs @@ -11,12 +11,95 @@ use std::{ use mlua::prelude::*; use mlua_luau_scheduler::{Functions, Scheduler}; +use self_cell::self_cell; use super::{RuntimeError, RuntimeResult}; -#[derive(Debug)] +// NOTE: We need to use self_cell to create a self-referential +// struct storing both the Lua VM and the scheduler. The scheduler +// needs to be created at the same time so that we can also create +// and inject the scheduler functions which will be used across runs. +self_cell! { + struct RuntimeInner { + owner: Rc, + #[covariant] + dependent: Scheduler, + } +} + +impl RuntimeInner { + fn create() -> LuaResult { + let lua = Rc::new(Lua::new()); + + lua.set_app_data(Rc::downgrade(&lua)); + lua.set_app_data(Vec::::new()); + + Self::try_new(lua, |lua| { + let sched = Scheduler::new(lua); + let fns = Functions::new(lua)?; + + // Overwrite some globals that are not compatible with our scheduler + let co = lua.globals().get::<_, LuaTable>("coroutine")?; + co.set("resume", fns.resume.clone())?; + co.set("wrap", fns.wrap.clone())?; + + // Inject all the globals that are enabled + #[cfg(any( + feature = "std-datetime", + feature = "std-fs", + feature = "std-luau", + feature = "std-net", + feature = "std-process", + feature = "std-regex", + feature = "std-roblox", + feature = "std-serde", + feature = "std-stdio", + feature = "std-task", + ))] + { + lune_std::inject_globals(lua)?; + } + + // Sandbox the Luau VM and make it go zooooooooom + lua.sandbox(true)?; + + // _G table needs to be injected again after sandboxing, + // otherwise it will be read-only and completely unusable + #[cfg(any( + feature = "std-datetime", + feature = "std-fs", + feature = "std-luau", + feature = "std-net", + feature = "std-process", + feature = "std-regex", + feature = "std-roblox", + feature = "std-serde", + feature = "std-stdio", + feature = "std-task", + ))] + { + let g_table = lune_std::LuneStandardGlobal::GTable; + lua.globals().set(g_table.name(), g_table.create(lua)?)?; + } + + Ok(sched) + }) + } + + fn lua(&self) -> &Lua { + self.borrow_owner() + } + + fn scheduler(&self) -> &Scheduler { + self.borrow_dependent() + } +} + +/** + A Lune runtime. +*/ pub struct Runtime { - lua: Rc, + inner: RuntimeInner, args: Vec, } @@ -29,29 +112,8 @@ impl Runtime { #[must_use] #[allow(clippy::new_without_default)] pub fn new() -> Self { - let lua = Rc::new(Lua::new()); - - lua.set_app_data(Rc::downgrade(&lua)); - lua.set_app_data(Vec::::new()); - - #[cfg(any( - feature = "std-datetime", - feature = "std-fs", - feature = "std-luau", - feature = "std-net", - feature = "std-process", - feature = "std-regex", - feature = "std-roblox", - feature = "std-serde", - feature = "std-stdio", - feature = "std-task", - ))] - { - lune_std::inject_globals(&lua).expect("Failed to inject globals"); - } - Self { - lua, + inner: RuntimeInner::create().expect("Failed to create runtime"), args: Vec::new(), } } @@ -65,7 +127,7 @@ impl Runtime { V: Into>, { self.args = args.into(); - self.lua.set_app_data(self.args.clone()); + self.inner.lua().set_app_data(self.args.clone()); self } @@ -83,26 +145,19 @@ impl Runtime { script_name: impl AsRef, script_contents: impl AsRef<[u8]>, ) -> RuntimeResult { - // Create a new scheduler for this run - let sched = Scheduler::new(&self.lua); + let lua = self.inner.lua(); + let sched = self.inner.scheduler(); // Add error callback to format errors nicely + store status let got_any_error = Arc::new(AtomicBool::new(false)); let got_any_inner = Arc::clone(&got_any_error); - sched.set_error_callback(move |e| { + self.inner.scheduler().set_error_callback(move |e| { got_any_inner.store(true, Ordering::SeqCst); eprintln!("{}", RuntimeError::from(e)); }); - // Overwrite resume & wrap functions on the coroutine global - // with ones that are compatible with our scheduler - // We also sandbox the VM, preventing further modifications - // to the global environment, and enabling optimizations - inject_scheduler_functions_and_sandbox(&self.lua)?; - // Load our "main" thread - let main = self - .lua + let main = lua .load(script_contents.as_ref()) .set_name(script_name.as_ref()); @@ -122,34 +177,3 @@ impl Runtime { Ok(exit_code) } } - -fn inject_scheduler_functions_and_sandbox(lua: &Lua) -> LuaResult<()> { - let fns = Functions::new(lua)?; - - let co = lua.globals().get::<_, LuaTable>("coroutine")?; - co.set("resume", fns.resume.clone())?; - co.set("wrap", fns.wrap.clone())?; - - lua.sandbox(true)?; - - // NOTE: We need to create the _G table after - // sandboxing, otherwise it will be read-only - #[cfg(any( - feature = "std-datetime", - feature = "std-fs", - feature = "std-luau", - feature = "std-net", - feature = "std-process", - feature = "std-regex", - feature = "std-roblox", - feature = "std-serde", - feature = "std-stdio", - feature = "std-task", - ))] - { - let g_global = lune_std::LuneStandardGlobal::GTable; - lua.globals().set(g_global.name(), g_global.create(lua)?)?; - } - - Ok(()) -} From a3f0f279a8eca75974bcaf244432b05f411d6ec6 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 18:51:37 +0200 Subject: [PATCH 21/27] Remove unused runtime args field --- crates/lune/src/rt/runtime.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/lune/src/rt/runtime.rs b/crates/lune/src/rt/runtime.rs index 5d515ca..53a5970 100644 --- a/crates/lune/src/rt/runtime.rs +++ b/crates/lune/src/rt/runtime.rs @@ -100,7 +100,6 @@ impl RuntimeInner { */ pub struct Runtime { inner: RuntimeInner, - args: Vec, } impl Runtime { @@ -114,7 +113,6 @@ impl Runtime { pub fn new() -> Self { Self { inner: RuntimeInner::create().expect("Failed to create runtime"), - args: Vec::new(), } } @@ -122,12 +120,11 @@ impl Runtime { Sets arguments to give in `process.args` for Lune scripts. */ #[must_use] - pub fn with_args(mut self, args: V) -> Self + pub fn with_args(self, args: V) -> Self where V: Into>, { - self.args = args.into(); - self.inner.lua().set_app_data(self.args.clone()); + self.inner.lua().set_app_data(args.into()); self } From 3cf2be51bc192942a7f94a123e6bb93797d657a7 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 18:52:38 +0200 Subject: [PATCH 22/27] Make with_args more permissive --- crates/lune/src/rt/runtime.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/lune/src/rt/runtime.rs b/crates/lune/src/rt/runtime.rs index 53a5970..08a601c 100644 --- a/crates/lune/src/rt/runtime.rs +++ b/crates/lune/src/rt/runtime.rs @@ -120,11 +120,13 @@ impl Runtime { Sets arguments to give in `process.args` for Lune scripts. */ #[must_use] - pub fn with_args(self, args: V) -> Self + pub fn with_args(self, args: A) -> Self where - V: Into>, + A: IntoIterator, + S: Into, { - self.inner.lua().set_app_data(args.into()); + let args = args.into_iter().map(Into::into).collect::>(); + self.inner.lua().set_app_data(args); self } From d3b9a4b9e8ed489fab5af32d2fab83f9bf46db23 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 19:02:48 +0200 Subject: [PATCH 23/27] Add new options for global injection and codegen to luau.load --- crates/lune-std-luau/src/lib.rs | 41 +++++++++---- crates/lune-std-luau/src/options.rs | 18 ++++-- crates/lune/src/tests.rs | 1 + tests/luau/load.luau | 94 +++++++++++++++++++++++------ tests/luau/safeenv.luau | 64 ++++++++++++++++++++ types/luau.luau | 6 +- 6 files changed, 188 insertions(+), 36 deletions(-) create mode 100644 tests/luau/safeenv.luau diff --git a/crates/lune-std-luau/src/lib.rs b/crates/lune-std-luau/src/lib.rs index e41eed5..21eb912 100644 --- a/crates/lune-std-luau/src/lib.rs +++ b/crates/lune-std-luau/src/lib.rs @@ -44,26 +44,41 @@ fn load_source<'lua>( (source, options): (LuaString<'lua>, LuauLoadOptions), ) -> LuaResult> { let mut chunk = lua.load(source.as_bytes()).set_name(options.debug_name); + let env_changed = options.environment.is_some(); - if let Some(environment) = options.environment { - let environment_with_globals = lua.create_table()?; + if let Some(custom_environment) = options.environment { + let environment = lua.create_table()?; - if let Some(meta) = environment.get_metatable() { - environment_with_globals.set_metatable(Some(meta)); + // Inject all globals into the environment + if options.inject_globals { + for pair in lua.globals().pairs() { + let (key, value): (LuaValue, LuaValue) = pair?; + environment.set(key, value)?; + } + + if let Some(global_metatable) = lua.globals().get_metatable() { + environment.set_metatable(Some(global_metatable)); + } + } else if let Some(custom_metatable) = custom_environment.get_metatable() { + // Since we don't need to set the global metatable, + // we can just set a custom metatable if it exists + environment.set_metatable(Some(custom_metatable)); } - for pair in lua.globals().pairs() { + // Inject the custom environment + for pair in custom_environment.pairs() { let (key, value): (LuaValue, LuaValue) = pair?; - environment_with_globals.set(key, value)?; + environment.set(key, value)?; } - for pair in environment.pairs() { - let (key, value): (LuaValue, LuaValue) = pair?; - environment_with_globals.set(key, value)?; - } - - chunk = chunk.set_environment(environment_with_globals); + chunk = chunk.set_environment(environment); } - chunk.into_function() + // Enable JIT if codegen is enabled and the environment hasn't + // changed, otherwise disable JIT since it'll fall back anyways + lua.enable_jit(options.codegen_enabled && !env_changed); + let function = chunk.into_function()?; + lua.enable_jit(true); + + Ok(function) } diff --git a/crates/lune-std-luau/src/options.rs b/crates/lune-std-luau/src/options.rs index a2040ec..81b8ac0 100644 --- a/crates/lune-std-luau/src/options.rs +++ b/crates/lune-std-luau/src/options.rs @@ -79,13 +79,11 @@ impl<'lua> FromLua<'lua> for LuauCompileOptions { } } -/** - Options for loading Lua source code. -*/ -#[derive(Debug, Clone)] pub struct LuauLoadOptions<'lua> { pub(crate) debug_name: String, pub(crate) environment: Option>, + pub(crate) inject_globals: bool, + pub(crate) codegen_enabled: bool, } impl Default for LuauLoadOptions<'_> { @@ -93,6 +91,8 @@ impl Default for LuauLoadOptions<'_> { Self { debug_name: DEFAULT_DEBUG_NAME.to_string(), environment: None, + inject_globals: true, + codegen_enabled: false, } } } @@ -112,11 +112,21 @@ impl<'lua> FromLua<'lua> for LuauLoadOptions<'lua> { options.environment = Some(environment); } + if let Some(inject_globals) = t.get("injectGlobals")? { + options.inject_globals = inject_globals; + } + + if let Some(codegen_enabled) = t.get("codegenEnabled")? { + options.codegen_enabled = codegen_enabled; + } + options } LuaValue::String(s) => Self { debug_name: s.to_string_lossy().to_string(), environment: None, + inject_globals: true, + codegen_enabled: false, }, _ => { return Err(LuaError::FromLuaConversionError { diff --git a/crates/lune/src/tests.rs b/crates/lune/src/tests.rs index 0306b29..2e866dc 100644 --- a/crates/lune/src/tests.rs +++ b/crates/lune/src/tests.rs @@ -113,6 +113,7 @@ create_tests! { luau_compile: "luau/compile", luau_load: "luau/load", luau_options: "luau/options", + luau_safeenv: "luau/safeenv", } #[cfg(feature = "std-net")] diff --git a/tests/luau/load.luau b/tests/luau/load.luau index 7701270..1cff3f8 100644 --- a/tests/luau/load.luau +++ b/tests/luau/load.luau @@ -26,11 +26,11 @@ assert( "expected source block name for 'luau.load' to return a custom debug name" ) -local success = pcall(function() +local loadSuccess = pcall(function() luau.load(luau.compile(RETURN_LUAU_CODE_BLOCK)) end) -assert(success, "expected `luau.load` to be able to process the result of `luau.compile`") +assert(loadSuccess, "expected `luau.load` to be able to process the result of `luau.compile`") local CUSTOM_SOURCE_WITH_FOO_FN = "return foo()" @@ -48,34 +48,92 @@ local fooFn = luau.load(CUSTOM_SOURCE_WITH_FOO_FN, { local fooFnRet = fooFn() assert(fooFnRet == fooValue, "expected `luau.load` with custom environment to return proper values") -local CUSTOM_SOURCE_WITH_PRINT_FN = "return print()" - --- NOTE: Same as what we did above, new userdata to guarantee unique-ness -local overriddenValue = newproxy(false) -local overriddenFn = luau.load(CUSTOM_SOURCE_WITH_PRINT_FN, { +local fooValue2 = newproxy(false) +local fooFn2 = luau.load(CUSTOM_SOURCE_WITH_FOO_FN, { environment = { - print = function() - return overriddenValue + foo = function() + return fooValue2 end, }, + enableGlobals = false, }) -local overriddenFnRet = overriddenFn() +local fooFn2Ret = fooFn2() assert( - overriddenFnRet == overriddenValue, + fooFn2Ret == fooValue2, + "expected `luau.load` with custom environment and no default globals to still return proper values" +) + +local CUSTOM_SOURCE_WITH_PRINT_FN = "return print()" + +-- NOTE: Testing overriding the print function +local overriddenPrintValue1 = newproxy(false) +local overriddenPrintFn1 = luau.load(CUSTOM_SOURCE_WITH_PRINT_FN, { + environment = { + print = function() + return overriddenPrintValue1 + end, + }, + enableGlobals = true, +}) + +local overriddenPrintFnRet1 = overriddenPrintFn1() +assert( + overriddenPrintFnRet1 == overriddenPrintValue1, "expected `luau.load` with overridden environment to return proper values" ) -local CUSTOM_SOURCE_WITH_DEFAULT_FN = "return string.lower(...)" - -local overriddenFn2 = luau.load(CUSTOM_SOURCE_WITH_DEFAULT_FN, { +local overriddenPrintValue2 = newproxy(false) +local overriddenPrintFn2 = luau.load(CUSTOM_SOURCE_WITH_PRINT_FN, { environment = { - hello = "world", + print = function() + return overriddenPrintValue2 + end, }, + enableGlobals = false, }) -local overriddenFn2Ret = overriddenFn2("LOWERCASE") +local overriddenPrintFnRet2 = overriddenPrintFn2() assert( - overriddenFn2Ret == "lowercase", - "expected `luau.load` with overridden environment to contain default globals" + overriddenPrintFnRet2 == overriddenPrintValue2, + "expected `luau.load` with overridden environment and disabled default globals to return proper values" +) + +-- NOTE: Testing whether injectGlobals works +local CUSTOM_SOURCE_WITH_DEFAULT_FN = "return string.lower(...)" + +local lowerFn1 = luau.load(CUSTOM_SOURCE_WITH_DEFAULT_FN, { + environment = {}, + injectGlobals = false, +}) + +local lowerFn1Success = pcall(lowerFn1, "LOWERCASE") + +assert( + not lowerFn1Success, + "expected `luau.load` with injectGlobals = false and empty custom environment to not contain default globals" +) + +local lowerFn2 = luau.load(CUSTOM_SOURCE_WITH_DEFAULT_FN, { + environment = { string = string }, + injectGlobals = false, +}) + +local lowerFn2Success, lowerFn2Result = pcall(lowerFn2, "LOWERCASE") + +assert( + lowerFn2Success and lowerFn2Result == "lowercase", + "expected `luau.load` with injectGlobals = false and valid custom environment to return proper values" +) + +local lowerFn3 = luau.load(CUSTOM_SOURCE_WITH_DEFAULT_FN, { + environment = {}, + injectGlobals = true, +}) + +local lowerFn3Success, lowerFn3Result = pcall(lowerFn3, "LOWERCASE") + +assert( + lowerFn3Success and lowerFn3Result == "lowercase", + "expected `luau.load` with injectGlobals = true and empty custom environment to return proper values" ) diff --git a/tests/luau/safeenv.luau b/tests/luau/safeenv.luau new file mode 100644 index 0000000..f1a9d06 --- /dev/null +++ b/tests/luau/safeenv.luau @@ -0,0 +1,64 @@ +local luau = require("@lune/luau") + +local TEST_SCRIPT = [[ + local start = os.clock() + local x + for i = 1, 1e6 do + x = math.sqrt(i) + end + local finish = os.clock() + + return finish - start +]] + +local TEST_BYTECODE = luau.compile(TEST_SCRIPT, { + optimizationLevel = 2, + coverageLevel = 0, + debugLevel = 0, +}) + +-- Load the bytecode with different configurations +local safeCodegenFunction = luau.load(TEST_BYTECODE, { + debugName = "safeCodegenFunction", + codegenEnabled = true, +}) +local unsafeCodegenFunction = luau.load(TEST_BYTECODE, { + debugName = "unsafeCodegenFunction", + codegenEnabled = true, + environment = {}, + injectGlobals = true, +}) +local safeFunction = luau.load(TEST_BYTECODE, { + debugName = "safeFunction", + codegenEnabled = false, +}) +local unsafeFunction = luau.load(TEST_BYTECODE, { + debugName = "unsafeFunction", + codegenEnabled = false, + environment = {}, + injectGlobals = true, +}) + +-- Run the functions to get the timings +local safeCodegenTime = safeCodegenFunction() +local unsafeCodegenTime = unsafeCodegenFunction() +local safeTime = safeFunction() +local unsafeTime = unsafeFunction() + +-- Assert that safeCodegenTime is always twice as fast as both unsafe functions +local safeCodegenUpperBound = safeCodegenTime * 2 +assert( + unsafeCodegenTime > safeCodegenUpperBound and unsafeTime > safeCodegenUpperBound, + "expected luau.load with codegenEnabled = true and no custom environment to use codegen" +) + +-- Assert that safeTime is always atleast twice as fast as both unsafe functions +local safeUpperBound = safeTime * 2 +assert( + unsafeCodegenTime > safeUpperBound and unsafeTime > safeUpperBound, + "expected luau.load with codegenEnabled = false and no custom environment to have safeenv enabled" +) + +-- Normally we'd also want to check whether codegen is actually being enabled by +-- comparing timings of safe_codegen_fn and safe_fn but since we don't have a way of +-- checking whether the current device even supports codegen, we can't safely test this. diff --git a/types/luau.luau b/types/luau.luau index a810ee5..379c07d 100644 --- a/types/luau.luau +++ b/types/luau.luau @@ -27,11 +27,15 @@ export type CompileOptions = { This is a dictionary that may contain one or more of the following values: * `debugName` - The debug name of the closure. Defaults to `luau.load(...)`. - * `environment` - Environment values to set and/or override. Includes default globals unless overwritten. + * `environment` - A custom environment to load the chunk in. Setting a custom environment will deoptimize the chunk and forcefully disable codegen. Defaults to the global environment. + * `injectGlobals` - Whether or not to inject globals in the custom environment. Has no effect if no custom environment is provided. Defaults to `true`. + * `codegenEnabled` - Whether or not to enable codegen. Defaults to `true`. ]=] export type LoadOptions = { debugName: string?, environment: { [string]: any }?, + injectGlobals: boolean?, + codegenEnabled: boolean?, } --[=[ From c94ab0cde1c3a8e9475df25c9b7f6f8a4eaee9f9 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 19:21:30 +0200 Subject: [PATCH 24/27] Update changelog, some other minor fixes --- CHANGELOG.md | 15 +++++++++++++++ tests/luau/safeenv.luau | 4 ++-- types/luau.luau | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7d222c..7f159e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + - Added a builtin API for hashing and calculating HMACs as part of the `serde` library Basic usage: @@ -26,6 +28,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 The returned hashes are sequences of lowercase hexadecimal digits. The following algorithms are supported: `md5`, `sha1`, `sha224`, `sha256`, `sha384`, `sha512`, `sha3-224`, `sha3-256`, `sha3-384`, `sha3-512`, `blake3` +- Added two new options to `luau.load`: + + - `codegenEnabled` - whether or not codegen should be enabled for the loaded chunk. + - `injectGlobals` - whether or not to inject globals into a passed `environment`. + + By default, globals are injected and codegen is disabled. + Check the documentation for the `luau` standard library for more information. + +### Changed + +- Sandboxing and codegen in the Luau VM is now fully enabled, resulting in up to 2x or faster code execution. + This should not result in any behavior differences in Lune, but if it does, please open an issue. + ## `0.8.5` - June 1st, 2024 ### Changed diff --git a/tests/luau/safeenv.luau b/tests/luau/safeenv.luau index f1a9d06..be653a7 100644 --- a/tests/luau/safeenv.luau +++ b/tests/luau/safeenv.luau @@ -24,9 +24,9 @@ local safeCodegenFunction = luau.load(TEST_BYTECODE, { }) local unsafeCodegenFunction = luau.load(TEST_BYTECODE, { debugName = "unsafeCodegenFunction", - codegenEnabled = true, environment = {}, injectGlobals = true, + codegenEnabled = true, }) local safeFunction = luau.load(TEST_BYTECODE, { debugName = "safeFunction", @@ -34,9 +34,9 @@ local safeFunction = luau.load(TEST_BYTECODE, { }) local unsafeFunction = luau.load(TEST_BYTECODE, { debugName = "unsafeFunction", - codegenEnabled = false, environment = {}, injectGlobals = true, + codegenEnabled = false, }) -- Run the functions to get the timings diff --git a/types/luau.luau b/types/luau.luau index 379c07d..ab40c4f 100644 --- a/types/luau.luau +++ b/types/luau.luau @@ -29,7 +29,7 @@ export type CompileOptions = { * `debugName` - The debug name of the closure. Defaults to `luau.load(...)`. * `environment` - A custom environment to load the chunk in. Setting a custom environment will deoptimize the chunk and forcefully disable codegen. Defaults to the global environment. * `injectGlobals` - Whether or not to inject globals in the custom environment. Has no effect if no custom environment is provided. Defaults to `true`. - * `codegenEnabled` - Whether or not to enable codegen. Defaults to `true`. + * `codegenEnabled` - Whether or not to enable codegen. Defaults to `false`. ]=] export type LoadOptions = { debugName: string?, From 0efc2c565b7bc0a77e8a335965ab45bbd6557866 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 19:38:19 +0200 Subject: [PATCH 25/27] Expand test suite for stdio.format --- tests/stdio/format.luau | 72 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/tests/stdio/format.luau b/tests/stdio/format.luau index c4052f3..56d673f 100644 --- a/tests/stdio/format.luau +++ b/tests/stdio/format.luau @@ -1,13 +1,71 @@ +local process = require("@lune/process") +local roblox = require("@lune/roblox") local stdio = require("@lune/stdio") -assert( - stdio.format("Hello", "world", "!") == "Hello world !", - "Format should add a single space between arguments" +local function assertFormatting(errorMessage: string, formatted: string, expected: string) + if formatted ~= expected then + stdio.ewrite(string.format("%s\nExpected: %s\nGot: %s", errorMessage, expected, formatted)) + process.exit(1) + end +end + +assertFormatting( + "Should add a single space between arguments", + stdio.format("Hello", "world", "!"), + "Hello world !" ) -assert( - stdio.format({ Hello = "World" }) == '{\n Hello = "World",\n}', - "Format should print out proper tables" +assertFormatting( + "Should format tables in a sorted manner", + stdio.format({ A = "A", B = "B", C = "C" }), + '{\n A = "A",\n B = "B",\n C = "C",\n}' +) + +assertFormatting( + "Should format tables properly with single values", + stdio.format({ Hello = "World" }), + '{\n Hello = "World",\n}' +) + +assertFormatting( + "Should format tables properly with multiple values", + stdio.format({ Hello = "World", Hello2 = "Value" }), + '{\n Hello = "World",\n Hello2 = "Value",\n}' +) + +assertFormatting( + "Should simplify array-like tables and not format keys", + stdio.format({ "Hello", "World" }), + '{\n "Hello",\n "World",\n}' +) + +assertFormatting( + "Should still format numeric keys for mixed tables", + stdio.format({ "Hello", "World", Hello = "World" }), + '{\n [1] = "Hello",\n [2] = "World",\n Hello = "World",\n}' +) + +local userdatas = { + Foo = newproxy(false), + Bar = (roblox :: any).Vector3.new(), +} + +assertFormatting( + "Should format userdatas as their type (unknown userdata)", + stdio.format(userdatas.Foo), + "" +) + +assertFormatting( + "Should format userdatas as their type (known userdata)", + stdio.format(userdatas.Bar), + "" +) + +assertFormatting( + "Should format userdatas as their type in tables", + stdio.format(userdatas), + "{\n Foo = ,\n Bar = ,\n}" ) local nested = { @@ -24,5 +82,5 @@ local nested = { assert( string.find(stdio.format(nested), "Nesting = { ... }", 1, true) ~= nil, - "Format should print 4 levels of nested tables before cutting off" + "Should print 4 levels of nested tables before cutting off" ) From 1fb1d3e7b5f4a84810d6212daeb0bda149c56722 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 5 Jun 2024 20:18:23 +0200 Subject: [PATCH 26/27] Improve formatting / printing of userdata and tables with __type and / or __tostring metamethods --- crates/lune-std-regex/src/captures.rs | 2 +- crates/lune-std-regex/src/matches.rs | 2 +- crates/lune-std-regex/src/regex.rs | 2 +- crates/lune-utils/src/fmt/value/basic.rs | 49 ++++++++++++---- .../lune-utils/src/fmt/value/metamethods.rs | 52 ++++++++++------- tests/regex/metamethods.luau | 6 +- tests/stdio/format.luau | 58 +++++++++++++++---- 7 files changed, 121 insertions(+), 50 deletions(-) diff --git a/crates/lune-std-regex/src/captures.rs b/crates/lune-std-regex/src/captures.rs index 5dbea74..fcfde93 100644 --- a/crates/lune-std-regex/src/captures.rs +++ b/crates/lune-std-regex/src/captures.rs @@ -81,7 +81,7 @@ impl LuaUserData for LuaCaptures { methods.add_meta_method(LuaMetaMethod::Len, |_, this, ()| Ok(this.num_captures())); methods.add_meta_method(LuaMetaMethod::ToString, |_, this, ()| { - Ok(format!("RegexCaptures({})", this.num_captures())) + Ok(format!("{}", this.num_captures())) }); } diff --git a/crates/lune-std-regex/src/matches.rs b/crates/lune-std-regex/src/matches.rs index bc109f8..ad21491 100644 --- a/crates/lune-std-regex/src/matches.rs +++ b/crates/lune-std-regex/src/matches.rs @@ -47,7 +47,7 @@ impl LuaUserData for LuaMatch { fn add_methods<'lua, M: LuaUserDataMethods<'lua, Self>>(methods: &mut M) { methods.add_meta_method(LuaMetaMethod::Len, |_, this, ()| Ok(this.range().len())); methods.add_meta_method(LuaMetaMethod::ToString, |_, this, ()| { - Ok(format!("RegexMatch({})", this.slice())) + Ok(this.slice().to_string()) }); } } diff --git a/crates/lune-std-regex/src/regex.rs b/crates/lune-std-regex/src/regex.rs index 9b83544..2ae26d9 100644 --- a/crates/lune-std-regex/src/regex.rs +++ b/crates/lune-std-regex/src/regex.rs @@ -66,7 +66,7 @@ impl LuaUserData for LuaRegex { ); methods.add_meta_method(LuaMetaMethod::ToString, |_, this, ()| { - Ok(format!("Regex({})", this.inner.as_str())) + Ok(this.inner.as_str().to_string()) }); } diff --git a/crates/lune-utils/src/fmt/value/basic.rs b/crates/lune-utils/src/fmt/value/basic.rs index cc4f9fb..ff1f421 100644 --- a/crates/lune-utils/src/fmt/value/basic.rs +++ b/crates/lune-utils/src/fmt/value/basic.rs @@ -1,7 +1,12 @@ use mlua::prelude::*; +use crate::fmt::ErrorComponents; + use super::{ - metamethods::{call_table_tostring_metamethod, call_userdata_tostring_metamethod}, + metamethods::{ + call_table_tostring_metamethod, call_userdata_tostring_metamethod, + get_table_type_metavalue, get_userdata_type_metavalue, + }, style::{COLOR_CYAN, COLOR_GREEN, COLOR_MAGENTA, COLOR_YELLOW}, }; @@ -56,19 +61,39 @@ pub(crate) fn format_value_styled(value: &LuaValue, prefer_plain: bool) -> Strin LuaValue::Function(_) => COLOR_MAGENTA.apply_to("").to_string(), LuaValue::LightUserData(_) => COLOR_MAGENTA.apply_to("").to_string(), LuaValue::UserData(u) => { - if let Some(s) = call_userdata_tostring_metamethod(u) { - s - } else { - COLOR_MAGENTA.apply_to("").to_string() - } + let formatted = format_typename_and_tostringed( + "userdata", + get_userdata_type_metavalue(u), + call_userdata_tostring_metamethod(u), + ); + COLOR_MAGENTA.apply_to(formatted).to_string() } LuaValue::Table(t) => { - if let Some(s) = call_table_tostring_metamethod(t) { - s - } else { - COLOR_MAGENTA.apply_to("").to_string() - } + let formatted = format_typename_and_tostringed( + "table", + get_table_type_metavalue(t), + call_table_tostring_metamethod(t), + ); + COLOR_MAGENTA.apply_to(formatted).to_string() } - _ => COLOR_MAGENTA.apply_to("").to_string(), + LuaValue::Error(e) => COLOR_MAGENTA + .apply_to(format!( + "", + ErrorComponents::from(e.clone()) + )) + .to_string(), + } +} + +fn format_typename_and_tostringed( + fallback: &'static str, + typename: Option, + tostringed: Option, +) -> String { + match (typename, tostringed) { + (Some(typename), Some(tostringed)) => format!("<{typename}({tostringed})>"), + (Some(typename), None) => format!("<{typename}>"), + (None, Some(tostringed)) => format!("<{tostringed}>"), + (None, None) => format!("<{fallback}>"), } } diff --git a/crates/lune-utils/src/fmt/value/metamethods.rs b/crates/lune-utils/src/fmt/value/metamethods.rs index 8b00b1a..c553262 100644 --- a/crates/lune-utils/src/fmt/value/metamethods.rs +++ b/crates/lune-utils/src/fmt/value/metamethods.rs @@ -1,29 +1,37 @@ use mlua::prelude::*; +pub fn get_table_type_metavalue<'a>(tab: &'a LuaTable<'a>) -> Option { + let s = tab + .get_metatable()? + .get::<_, LuaString>(LuaMetaMethod::Type.name()) + .ok()?; + let s = s.to_str().ok()?; + Some(s.to_string()) +} + +pub fn get_userdata_type_metavalue<'a>(tab: &'a LuaAnyUserData<'a>) -> Option { + let s = tab + .get_metatable() + .ok()? + .get::(LuaMetaMethod::Type.name()) + .ok()?; + let s = s.to_str().ok()?; + Some(s.to_string()) +} + pub fn call_table_tostring_metamethod<'a>(tab: &'a LuaTable<'a>) -> Option { - let f = match tab.get_metatable() { - None => None, - Some(meta) => match meta.get::<_, LuaFunction>(LuaMetaMethod::ToString.name()) { - Ok(method) => Some(method), - Err(_) => None, - }, - }?; - match f.call::<_, String>(()) { - Ok(res) => Some(res), - Err(_) => None, - } + tab.get_metatable()? + .get::<_, LuaFunction>(LuaMetaMethod::ToString.name()) + .ok()? + .call(tab) + .ok() } pub fn call_userdata_tostring_metamethod<'a>(tab: &'a LuaAnyUserData<'a>) -> Option { - let f = match tab.get_metatable() { - Err(_) => None, - Ok(meta) => match meta.get::(LuaMetaMethod::ToString.name()) { - Ok(method) => Some(method), - Err(_) => None, - }, - }?; - match f.call::<_, String>(()) { - Ok(res) => Some(res), - Err(_) => None, - } + tab.get_metatable() + .ok()? + .get::(LuaMetaMethod::ToString.name()) + .ok()? + .call(tab) + .ok() } diff --git a/tests/regex/metamethods.luau b/tests/regex/metamethods.luau index f14231c..2a4f304 100644 --- a/tests/regex/metamethods.luau +++ b/tests/regex/metamethods.luau @@ -3,14 +3,14 @@ local regex = require("@lune/regex") local re = regex.new("[0-9]+") -assert(tostring(re) == "Regex([0-9]+)") +assert(tostring(re) == "[0-9]+") assert(typeof(re) == "Regex") local mtch = re:find("1337 wow") -assert(tostring(mtch) == "RegexMatch(1337)") +assert(tostring(mtch) == "1337") assert(typeof(mtch) == "RegexMatch") local re2 = regex.new("([0-9]+) ([0-9]+) wow! ([0-9]+) ([0-9]+)") local captures = re2:captures("1337 125600 wow! 1984 0") -assert(tostring(captures) == "RegexCaptures(4)") +assert(tostring(captures) == "4") assert(typeof(captures) == "RegexCaptures") diff --git a/tests/stdio/format.luau b/tests/stdio/format.luau index 56d673f..7ade5f5 100644 --- a/tests/stdio/format.luau +++ b/tests/stdio/format.luau @@ -1,4 +1,5 @@ local process = require("@lune/process") +local regex = require("@lune/regex") local roblox = require("@lune/roblox") local stdio = require("@lune/stdio") @@ -9,6 +10,13 @@ local function assertFormatting(errorMessage: string, formatted: string, expecte end end +local function assertContains(errorMessage: string, haystack: string, needle: string) + if string.find(haystack, needle) == nil then + stdio.ewrite(string.format("%s\nHaystack: %s\nNeedle: %s", errorMessage, needle, haystack)) + process.exit(1) + end +end + assertFormatting( "Should add a single space between arguments", stdio.format("Hello", "world", "!"), @@ -47,25 +55,38 @@ assertFormatting( local userdatas = { Foo = newproxy(false), - Bar = (roblox :: any).Vector3.new(), + Bar = regex.new("TEST"), + Baz = (roblox :: any).Vector3.new(1, 2, 3), } assertFormatting( - "Should format userdatas as their type (unknown userdata)", + "Should format userdatas as generic 'userdata' if unknown", stdio.format(userdatas.Foo), "" ) -assertFormatting( - "Should format userdatas as their type (known userdata)", +assertContains( + "Should format userdatas with their type if they have a __type metafield", stdio.format(userdatas.Bar), - "" + "Regex" +) + +assertContains( + "Should format userdatas with their type even if they have a __tostring metamethod", + stdio.format(userdatas.Baz), + "Vector3" +) + +assertContains( + "Should format userdatas with their tostringed value if they have a __tostring metamethod", + stdio.format(userdatas.Baz), + "1, 2, 3" ) assertFormatting( - "Should format userdatas as their type in tables", + "Should format userdatas properly in tables", stdio.format(userdatas), - "{\n Foo = ,\n Bar = ,\n}" + "{\n Bar = ,\n Baz = ,\n Foo = ,\n}" ) local nested = { @@ -80,7 +101,24 @@ local nested = { }, } -assert( - string.find(stdio.format(nested), "Nesting = { ... }", 1, true) ~= nil, - "Should print 4 levels of nested tables before cutting off" +assertContains( + "Should print 4 levels of nested tables before cutting off", + stdio.format(nested), + "Nesting = { ... }" ) + +local _, errorMessage = pcall(function() + local function innerInnerFn() + process.spawn("PROGRAM_THAT_DOES_NOT_EXIST") + end + local function innerFn() + innerInnerFn() + end + innerFn() +end) + +stdio.ewrite(typeof(errorMessage)) + +assertContains("Should format errors similarly to userdata", stdio.format(errorMessage), " Date: Wed, 5 Jun 2024 20:23:01 +0200 Subject: [PATCH 27/27] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f159e9..083569f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Sandboxing and codegen in the Luau VM is now fully enabled, resulting in up to 2x or faster code execution. This should not result in any behavior differences in Lune, but if it does, please open an issue. +- Improved formatting of custom error objects (such as when `fs.readFile` returns an error) when printed or formatted using `stdio.format`. + +### Fixed + +- Fixed `__type` and `__tostring` metamethods on userdatas and tables not being respected when printed or formatted using `stdio.format`. ## `0.8.5` - June 1st, 2024