Bundle type definition files in executable

CHANGELOG.md

@@ -12,6 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Type definitions are now bundled as part of the Lune executable, meaning they do not need to be downloaded, and are instead generated.
+  - `lune --generate-selene-types` will generate the Selene type definitions file, replacing `lune --download-selene-types`
+  - `lune --generate-luau-types` will generate the Luau type definitions file, replacing `lune --download-luau-types`
 - Improve error handling and messages for `net.serve`
 - Improve error handling and messages for `stdio.prompt`
 

Cargo.lock

@@ -814,6 +814,7 @@ dependencies = [
  "reqwest",
  "serde",
  "serde_json",
+ "serde_yaml",
  "tokio",
 ]
 
@@ -1289,6 +1290,19 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_yaml"
+version = "0.9.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fb06d4b6cdaef0e0c51fa881acb721bed3c924cfaa71d9c94a3b771dfdf6567"
+dependencies = [
+ "indexmap",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "sha1"
 version = "0.10.5"
@@ -1582,6 +1596,12 @@ version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
 
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc7ed8ba44ca06be78ea1ad2c3682a43349126c8818054231ee6f4748012aed2"
+
 [[package]]
 name = "untrusted"
 version = "0.7.1"

packages/cli/Cargo.toml

@@ -27,6 +27,7 @@ reqwest.workspace = true
 
 anyhow = "1.0.68"
 regex = "1.7.1"
+serde_yaml = "0.9.17"
 
 clap = { version = "4.1.1", features = ["derive"] }
 full_moon = { version = "0.17.0", features = ["roblox"] }

packages/cli/src/cli.rs

@@ -10,88 +10,83 @@ use crate::{
     gen::generate_docs_json_from_definitions,
     utils::{
         files::find_parse_file_path,
-        github::Client as GithubClient,
         listing::{find_lune_scripts, print_lune_scripts, sort_lune_scripts},
     },
 };
 
-pub(crate) const LUNE_SELENE_FILE_NAME: &str = "lune.yml";
-pub(crate) const LUNE_LUAU_FILE_NAME: &str = "luneTypes.d.luau";
-pub(crate) const LUNE_DOCS_FILE_NAME: &str = "luneDocs.json";
+pub(crate) const FILE_NAME_SELENE_TYPES: &str = "lune.yml";
+pub(crate) const FILE_NAME_LUAU_TYPES: &str = "luneTypes.d.luau";
+pub(crate) const FILE_NAME_DOCS: &str = "luneDocs.json";
 
-/// Lune CLI
-#[derive(Parser, Debug, Default)]
-#[command(author, version, about, long_about = None)]
+pub(crate) const FILE_CONTENTS_SELENE_TYPES: &str = include_str!("../../../lune.yml");
+pub(crate) const FILE_CONTENTS_LUAU_TYPES: &str = include_str!("../../../luneTypes.d.luau");
+
+/// A Luau script runner
+#[derive(Parser, Debug, Default, Clone)]
+#[command(version, long_about = None)]
 #[allow(clippy::struct_excessive_bools)]
 pub struct Cli {
-    /// Path to the file to run, or the name
-    /// of a luau file in a lune directory
-    ///
-    /// Can be omitted when downloading type definitions
+    /// Script name or full path to the file to run
     script_path: Option<String>,
-    /// Arguments to pass to the file as vararg (...)
+    /// Arguments to pass to the script, stored in process.args
     script_args: Vec<String>,
-    /// Pass this flag to list scripts inside of
-    /// nearby `lune` and / or `.lune` directories
+    /// List scripts found inside of a nearby `lune` directory
     #[clap(long, short = 'l')]
     list: bool,
-    /// Pass this flag to download the Selene type
-    /// definitions file to the current directory
+    /// Generate a Selene type definitions file in the current dir
     #[clap(long)]
-    download_selene_types: bool,
-    /// Pass this flag to download the Luau type
-    /// definitions file to the current directory
+    generate_selene_types: bool,
+    /// Generate a Luau type definitions file in the current dir
     #[clap(long)]
-    download_luau_types: bool,
-    /// Pass this flag to generate the Lune documentation file
-    /// from a luau type definitions file in the current directory
+    generate_luau_types: bool,
+    /// Generate a Lune documentation file for Luau LSP
     #[clap(long)]
     generate_docs_file: bool,
+    /// Generate the full Lune wiki directory
+    #[clap(long, hide = true)]
+    generate_wiki_dir: bool,
 }
 
 #[allow(dead_code)]
 impl Cli {
-    pub fn from_path<S>(path: S) -> Self
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn with_path<S>(mut self, path: S) -> Self
     where
         S: Into<String>,
     {
-        Self {
-            script_path: Some(path.into()),
-            ..Default::default()
-        }
+        self.script_path = Some(path.into());
+        self
     }
 
-    pub fn from_path_with_args<S, A>(path: S, args: A) -> Self
+    pub fn with_args<A>(mut self, args: A) -> Self
     where
-        S: Into<String>,
         A: Into<Vec<String>>,
     {
-        Self {
-            script_path: Some(path.into()),
-            script_args: args.into(),
-            ..Default::default()
-        }
+        self.script_args = args.into();
+        self
     }
 
-    pub fn download_selene_types() -> Self {
-        Self {
-            download_selene_types: true,
-            ..Default::default()
-        }
+    pub fn generate_selene_types(mut self) -> Self {
+        self.generate_selene_types = true;
+        self
     }
 
-    pub fn download_luau_types() -> Self {
-        Self {
-            download_luau_types: true,
-            ..Default::default()
-        }
+    pub fn generate_luau_types(mut self) -> Self {
+        self.generate_luau_types = true;
+        self
     }
 
-    pub fn list() -> Self {
-        Self {
-            list: true,
-            ..Default::default()
+    pub fn generate_docs_file(mut self) -> Self {
+        self.generate_docs_file = true;
+        self
     }
+
+    pub fn list(mut self) -> Self {
+        self.list = true;
+        self
     }
 
     pub async fn run(self) -> Result<ExitCode> {
@@ -115,36 +110,37 @@ impl Cli {
                 }
             }
         }
-        // Download definition files, if wanted
-        let download_types_requested = self.download_selene_types || self.download_luau_types;
-        if download_types_requested {
-            let client = GithubClient::new();
-            let release = client.fetch_release_for_this_version().await?;
-            if self.download_selene_types {
-                println!("Downloading Selene type definitions...");
-                client
-                    .fetch_release_asset(&release, LUNE_SELENE_FILE_NAME)
+        // Generate (save) definition files, if wanted
+        let generate_file_requested =
+            self.generate_selene_types || self.generate_luau_types || self.generate_docs_file;
+        if generate_file_requested {
+            if self.generate_selene_types {
+                generate_and_save_file(FILE_NAME_SELENE_TYPES, "Selene type definitions", || {
+                    Ok(FILE_CONTENTS_SELENE_TYPES.to_string())
+                })
                 .await?;
             }
-            if self.download_luau_types {
-                println!("Downloading Luau type definitions...");
-                client
-                    .fetch_release_asset(&release, LUNE_LUAU_FILE_NAME)
+            if self.generate_luau_types {
+                generate_and_save_file(FILE_NAME_LUAU_TYPES, "Luau type definitions", || {
+                    Ok(FILE_CONTENTS_LUAU_TYPES.to_string())
+                })
                 .await?;
             }
-        }
-        // Generate docs file, if wanted
             if self.generate_docs_file {
-            let defs_contents = read_to_string(LUNE_LUAU_FILE_NAME).await?;
-            let docs_root = generate_docs_json_from_definitions(&defs_contents, "roblox/global")?;
-            let docs_contents = serde_json::to_string_pretty(&docs_root)?;
-            write(LUNE_DOCS_FILE_NAME, &docs_contents).await?;
+                generate_and_save_file(FILE_NAME_DOCS, "Luau LSP documentation", || {
+                    let docs = &generate_docs_json_from_definitions(
+                        FILE_CONTENTS_LUAU_TYPES,
+                        "roblox/global",
+                    )?;
+                    Ok(serde_json::to_string_pretty(docs)?)
+                })
+                .await?;
+            }
         }
         if self.script_path.is_none() {
-            // Only downloading types without running a script is completely
+            // Only generating typedefs without running a script is completely
             // fine, and we should just exit the program normally afterwards
-            // Same thing goes for generating the docs file
-            if download_types_requested || self.generate_docs_file {
+            if generate_file_requested {
                 return Ok(ExitCode::SUCCESS);
             }
             // HACK: We know that we didn't get any arguments here but since
@@ -171,3 +167,34 @@ impl Cli {
         })
     }
 }
+
+async fn generate_and_save_file(
+    file_path: &str,
+    display_name: &str,
+    f: impl Fn() -> Result<String>,
+) -> Result<()> {
+    #[cfg(test)]
+    use crate::tests::fmt_path_relative_to_workspace_root;
+    match f() {
+        Ok(file_contents) => {
+            write(file_path, file_contents).await?;
+            #[cfg(not(test))]
+            println!("Generated {display_name} file at '{file_path}'");
+            #[cfg(test)]
+            println!(
+                "Generated {display_name} file at '{}'",
+                fmt_path_relative_to_workspace_root(file_path)
+            );
+        }
+        Err(e) => {
+            #[cfg(not(test))]
+            println!("Failed to generate {display_name} file at '{file_path}'\n{e}");
+            #[cfg(test)]
+            println!(
+                "Failed to generate {display_name} file at '{}'\n{e}",
+                fmt_path_relative_to_workspace_root(file_path)
+            );
+        }
+    }
+    Ok(())
+}

packages/cli/src/gen/mod.rs

@@ -12,7 +12,7 @@ mod visitor;
 
 use self::{doc::DocsFunctionParamLink, visitor::DocumentationVisitor};
 
-fn parse_definitions(contents: &str) -> Result<DocumentationVisitor> {
+pub fn parse_definitions(contents: &str) -> Result<DocumentationVisitor> {
     // TODO: Properly handle the "declare class" syntax, for now we just skip it
     let mut no_declares = contents.to_string();
     while let Some(dec) = no_declares.find("\ndeclare class") {

packages/cli/src/tests.rs

@@ -1,57 +0,0 @@
-use std::env::{current_dir, set_current_dir};
-
-use anyhow::{bail, Context, Result};
-use serde_json::Value;
-use tokio::fs::{create_dir_all, read_to_string, remove_file};
-
-use crate::cli::{Cli, LUNE_LUAU_FILE_NAME, LUNE_SELENE_FILE_NAME};
-
-async fn run_cli(cli: Cli) -> Result<()> {
-    let path = current_dir()
-        .context("Failed to get current dir")?
-        .join("bin");
-    create_dir_all(&path)
-        .await
-        .context("Failed to create bin dir")?;
-    set_current_dir(&path).context("Failed to set current dir")?;
-    cli.run().await?;
-    Ok(())
-}
-
-async fn ensure_file_exists_and_is_not_json(file_name: &str) -> Result<()> {
-    match read_to_string(file_name)
-        .await
-        .context("Failed to read definitions file")
-    {
-        Ok(file_contents) => match serde_json::from_str::<Value>(&file_contents) {
-            Err(_) => {
-                remove_file(file_name)
-                    .await
-                    .context("Failed to remove definitions file")?;
-                Ok(())
-            }
-            Ok(_) => bail!("Downloading selene definitions returned json, expected luau"),
-        },
-        Err(e) => bail!("Failed to download selene definitions!\n{e}"),
-    }
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn list() -> Result<()> {
-    Cli::list().run().await?;
-    Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn download_selene_types() -> Result<()> {
-    run_cli(Cli::download_selene_types()).await?;
-    ensure_file_exists_and_is_not_json(LUNE_SELENE_FILE_NAME).await?;
-    Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn download_luau_types() -> Result<()> {
-    run_cli(Cli::download_luau_types()).await?;
-    ensure_file_exists_and_is_not_json(LUNE_LUAU_FILE_NAME).await?;
-    Ok(())
-}

packages/cli/src/tests/bin_dir.rs

@@ -0,0 +1,20 @@
+use std::{env::set_current_dir, path::PathBuf};
+
+use anyhow::{Context, Result};
+use tokio::fs::create_dir_all;
+
+pub async fn enter_bin_dir() -> Result<()> {
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../bin");
+    if !path.exists() {
+        create_dir_all(&path)
+            .await
+            .context("Failed to enter bin dir")?;
+        set_current_dir(&path).context("Failed to set current dir")?;
+    }
+    Ok(())
+}
+
+pub fn leave_bin_dir() -> Result<()> {
+    set_current_dir(env!("CARGO_MANIFEST_DIR")).context("Failed to leave bin dir")?;
+    Ok(())
+}

packages/cli/src/tests/file_checks.rs

@@ -0,0 +1,57 @@
+use std::path::PathBuf;
+
+use anyhow::{bail, Context, Result};
+use tokio::fs::{read_to_string, remove_file};
+
+use super::bin_dir::{enter_bin_dir, leave_bin_dir};
+use super::file_type::FileType;
+
+pub fn fmt_path_relative_to_workspace_root(value: &str) -> String {
+    let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("../../")
+        .canonicalize()
+        .unwrap();
+    match PathBuf::from(value).strip_prefix(root) {
+        Err(_) => format!("{:#?}", PathBuf::from(value).display()),
+        Ok(inner) => format!("{:#?}", inner.display()),
+    }
+}
+
+async fn inner(file_name: &str, desired_type: FileType) -> Result<()> {
+    match read_to_string(file_name).await.with_context(|| {
+        format!(
+            "Failed to read definitions file at '{}'",
+            fmt_path_relative_to_workspace_root(file_name)
+        )
+    }) {
+        Ok(file_contents) => {
+            remove_file(file_name).await.with_context(|| {
+                format!(
+                    "Failed to remove definitions file at '{}'",
+                    fmt_path_relative_to_workspace_root(file_name)
+                )
+            })?;
+            let parsed_type = FileType::from_contents(&file_contents);
+            if parsed_type != Some(desired_type) {
+                bail!(
+                    "Generating definitions file at '{}' created '{}', expected '{}'",
+                    fmt_path_relative_to_workspace_root(file_name),
+                    parsed_type.map_or("unknown", |t| t.name()),
+                    desired_type.name()
+                )
+            }
+            Ok(())
+        }
+        Err(e) => bail!(
+            "Failed to generate definitions file at '{}'\n{e}",
+            fmt_path_relative_to_workspace_root(file_name)
+        ),
+    }
+}
+
+pub async fn ensure_file_exists_and_is(file_name: &str, desired_type: FileType) -> Result<()> {
+    enter_bin_dir().await?;
+    let res = inner(file_name, desired_type).await;
+    leave_bin_dir()?;
+    res
+}

packages/cli/src/tests/file_type.rs

@@ -0,0 +1,32 @@
+use serde_json::Value;
+
+use crate::gen::parse_definitions;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FileType {
+    Json,
+    Yaml,
+    Luau,
+}
+
+impl FileType {
+    pub fn from_contents(contents: &str) -> Option<Self> {
+        if serde_json::from_str::<Value>(contents).is_ok() {
+            Some(Self::Json)
+        } else if serde_yaml::from_str::<Value>(contents).is_ok() {
+            Some(Self::Yaml)
+        } else if parse_definitions(contents).is_ok() {
+            Some(Self::Luau)
+        } else {
+            None
+        }
+    }
+
+    pub fn name(self) -> &'static str {
+        match self {
+            FileType::Json => "json",
+            FileType::Yaml => "yaml",
+            FileType::Luau => "luau",
+        }
+    }
+}

packages/cli/src/tests/mod.rs

@@ -0,0 +1,40 @@
+use anyhow::Result;
+
+use crate::cli::{Cli, FILE_NAME_DOCS, FILE_NAME_LUAU_TYPES, FILE_NAME_SELENE_TYPES};
+
+mod bin_dir;
+mod file_checks;
+mod file_type;
+mod run_cli;
+
+pub(crate) use file_checks::*;
+pub(crate) use file_type::*;
+pub(crate) use run_cli::*;
+
+#[tokio::test]
+async fn list() -> Result<()> {
+    Cli::new().list().run().await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn generate_selene_types() -> Result<()> {
+    run_cli(Cli::new().generate_selene_types()).await?;
+    ensure_file_exists_and_is(FILE_NAME_SELENE_TYPES, FileType::Yaml).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn generate_luau_types() -> Result<()> {
+    run_cli(Cli::new().generate_luau_types()).await?;
+    ensure_file_exists_and_is(FILE_NAME_LUAU_TYPES, FileType::Luau).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn generate_docs_file() -> Result<()> {
+    run_cli(Cli::new().generate_luau_types()).await?;
+    run_cli(Cli::new().generate_docs_file()).await?;
+    ensure_file_exists_and_is(FILE_NAME_DOCS, FileType::Json).await?;
+    Ok(())
+}

packages/cli/src/tests/run_cli.rs

@@ -0,0 +1,12 @@
+use anyhow::Result;
+
+use crate::cli::Cli;
+
+use super::bin_dir::{enter_bin_dir, leave_bin_dir};
+
+pub async fn run_cli(cli: Cli) -> Result<()> {
+    enter_bin_dir().await?;
+    cli.run().await?;
+    leave_bin_dir()?;
+    Ok(())
+}

packages/cli/src/utils/github.rs

@@ -1,97 +0,0 @@
-use std::env::current_dir;
-
-use anyhow::{bail, Context, Result};
-use serde::{Deserialize, Serialize};
-
-use super::net::{get_github_owner_and_repo, get_request_user_agent_header};
-
-#[derive(Clone, Deserialize, Serialize)]
-pub struct ReleaseAsset {
-    id: u64,
-    url: String,
-    name: Option<String>,
-    label: Option<String>,
-    content_type: String,
-    size: u64,
-}
-
-#[derive(Clone, Deserialize, Serialize)]
-pub struct Release {
-    id: u64,
-    url: String,
-    tag_name: String,
-    name: Option<String>,
-    body: Option<String>,
-    draft: bool,
-    prerelease: bool,
-    assets: Vec<ReleaseAsset>,
-}
-
-pub struct Client {
-    github_owner: String,
-    github_repo: String,
-}
-
-impl Client {
-    pub fn new() -> Self {
-        let (github_owner, github_repo) = get_github_owner_and_repo();
-        Self {
-            github_owner,
-            github_repo,
-        }
-    }
-
-    async fn get(&self, url: &str, accept: Option<&str>) -> Result<Vec<u8>> {
-        let request = reqwest::ClientBuilder::new()
-            .build()?
-            .request(reqwest::Method::GET, url)
-            .header("User-Agent", &get_request_user_agent_header())
-            .header("Accept", accept.unwrap_or("application/vnd.github+json"))
-            .header("X-GitHub-Api-Version", "2022-11-28");
-        Ok(request.send().await?.bytes().await?.to_vec())
-    }
-
-    pub async fn fetch_releases(&self) -> Result<Vec<Release>> {
-        let release_api_url = format!(
-            "https://api.github.com/repos/{}/{}/releases",
-            &self.github_owner, &self.github_repo
-        );
-        let response_bytes = self.get(&release_api_url, None).await?;
-        Ok(serde_json::from_slice(&response_bytes)?)
-    }
-
-    pub async fn fetch_release_for_this_version(&self) -> Result<Release> {
-        let release_version_tag = format!("v{}", env!("CARGO_PKG_VERSION"));
-        let all_releases = self.fetch_releases().await?;
-        all_releases
-            .iter()
-            .find(|release| release.tag_name == release_version_tag)
-            .map(ToOwned::to_owned)
-            .with_context(|| format!("Failed to find release for version {release_version_tag}"))
-    }
-
-    pub async fn fetch_release_asset(&self, release: &Release, asset_name: &str) -> Result<()> {
-        if let Some(asset) = release
-            .assets
-            .iter()
-            .find(|asset| matches!(&asset.name, Some(name) if name == asset_name))
-        {
-            let file_path = current_dir()?.join(asset_name);
-            let file_bytes = self
-                .get(&asset.url, Some("application/octet-stream"))
-                .await?;
-            tokio::fs::write(&file_path, &file_bytes)
-                .await
-                .with_context(|| {
-                    format!("Failed to write file at path '{}'", &file_path.display())
-                })?;
-        } else {
-            bail!(
-                "Failed to find release asset '{}' for release '{}'",
-                asset_name,
-                &release.tag_name
-            )
-        }
-        Ok(())
-    }
-}

packages/cli/src/utils/mod.rs

@@ -1,4 +1,2 @@
 pub mod files;
-pub mod github;
 pub mod listing;
-pub mod net;

packages/cli/src/utils/net.rs

@@ -1,13 +0,0 @@
-pub fn get_github_owner_and_repo() -> (String, String) {
-    let (github_owner, github_repo) = env!("CARGO_PKG_REPOSITORY")
-        .strip_prefix("https://github.com/")
-        .unwrap()
-        .split_once('/')
-        .unwrap();
-    (github_owner.to_owned(), github_repo.to_owned())
-}
-
-pub fn get_request_user_agent_header() -> String {
-    let (github_owner, github_repo) = get_github_owner_and_repo();
-    format!("{github_owner}-{github_repo}-cli")
-}