Generate basic API reference markdown pages for wiki

.gitignore

@@ -1,5 +1,6 @@
 /bin
 /target
+/wiki
 
 .DS_Store
 */.DS_Store

packages/cli/src/cli.rs

@@ -7,7 +7,7 @@ use lune::Lune;
 use tokio::fs::{read_to_string, write};
 
 use crate::{
-    gen::generate_docs_json_from_definitions,
+    gen::{generate_docs_json_from_definitions, generate_wiki_dir_from_definitions},
     utils::{
         files::find_parse_file_path,
         listing::{find_lune_scripts, print_lune_scripts, sort_lune_scripts},
@@ -111,8 +111,10 @@ impl Cli {
             }
         }
         // Generate (save) definition files, if wanted
-        let generate_file_requested =
+        let generate_file_requested = self.generate_selene_types
-            self.generate_selene_types || self.generate_luau_types || self.generate_docs_file;
+            || self.generate_luau_types
+            || self.generate_docs_file
+            || self.generate_wiki_dir;
         if generate_file_requested {
             if self.generate_selene_types {
                 generate_and_save_file(FILE_NAME_SELENE_TYPES, "Selene type definitions", || {
@@ -128,14 +130,13 @@ impl Cli {
             }
             if self.generate_docs_file {
                 generate_and_save_file(FILE_NAME_DOCS, "Luau LSP documentation", || {
-                    let docs = &generate_docs_json_from_definitions(
+                    generate_docs_json_from_definitions(FILE_CONTENTS_LUAU_TYPES, "roblox/global")
-                        FILE_CONTENTS_LUAU_TYPES,
-                        "roblox/global",
-                    )?;
-                    Ok(serde_json::to_string_pretty(docs)?)
                 })
                 .await?;
             }
+            if self.generate_wiki_dir {
+                generate_wiki_dir_from_definitions(FILE_CONTENTS_LUAU_TYPES).await?;
+            }
         }
         if self.script_path.is_none() {
             // Only generating typedefs without running a script is completely

packages/cli/src/gen/mod.rs

@@ -1,15 +1,18 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, fmt::Write, path::PathBuf};
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use regex::Regex;
 use serde_json::{Map, Value};
 
 use full_moon::{parse as parse_luau_ast, visitors::Visitor};
+use tokio::fs::{create_dir_all, write};
 
 mod doc;
 mod tag;
 mod visitor;
 
+const GENERATED_COMMENT_TAG: &str = "@generated with lune-cli";
+
 use self::{doc::DocsFunctionParamLink, visitor::DocumentationVisitor};
 
 pub fn parse_definitions(contents: &str) -> Result<DocumentationVisitor> {
@@ -31,7 +34,7 @@ pub fn parse_definitions(contents: &str) -> Result<DocumentationVisitor> {
     Ok(visitor)
 }
 
-pub fn generate_docs_json_from_definitions(contents: &str, namespace: &str) -> Result<Value> {
+pub fn generate_docs_json_from_definitions(contents: &str, namespace: &str) -> Result<String> {
     let visitor = parse_definitions(contents)?;
     /*
         Extract globals, functions, params, returns from the visitor
@@ -87,5 +90,53 @@ pub fn generate_docs_json_from_definitions(contents: &str, namespace: &str) -> R
             serde_json::to_value(doc)?,
         );
     }
-    Ok(Value::Object(map))
+    serde_json::to_string_pretty(&Value::Object(map)).context("Failed to encode docs as json")
+}
+
+pub async fn generate_wiki_dir_from_definitions(contents: &str) -> Result<()> {
+    // Create the wiki dir at the repo root
+    let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("../../")
+        .canonicalize()
+        .unwrap();
+    create_dir_all(&root.join("wiki"))
+        .await
+        .context("Failed to create wiki dir")?;
+    let visitor = parse_definitions(contents)?;
+    for global in &visitor.globals {
+        // Create the dir for this global
+        let global_dir_path = root.join("wiki").join("api-reference").join(&global.0);
+        create_dir_all(&global_dir_path)
+            .await
+            .context("Failed to create doc dir for global")?;
+        // Create the markdown docs file for this global
+        let mut contents = String::new();
+        writeln!(contents, "<!-- {GENERATED_COMMENT_TAG} -->\n")?;
+        writeln!(contents, "# **{}**\n", global.0)?;
+        writeln!(contents, "{}\n", global.1.documentation)?;
+        if !global.1.code_sample.is_empty() {
+            writeln!(contents, "{}", global.1.code_sample)?;
+        }
+        let funcs = visitor
+            .functions
+            .iter()
+            .filter(|f| f.1.global_name == global.0)
+            .collect::<Vec<_>>();
+        if !funcs.is_empty() {
+            writeln!(contents, "## Functions\n")?;
+            for func in funcs {
+                writeln!(contents, "### {}\n", func.0)?;
+                writeln!(contents, "{}\n", func.1.documentation)?;
+                if !func.1.code_sample.is_empty() {
+                    writeln!(contents, "{}", func.1.code_sample)?;
+                }
+            }
+        }
+        // Write the file in the dir, with the same
+        // name as the dir to create an "index" page
+        write(&global_dir_path.join(format!("{}.md", &global.0)), contents)
+            .await
+            .context("Failed to create doc file for global")?;
+    }
+    Ok(())
 }

packages/cli/src/gen/tag.rs

@@ -6,6 +6,7 @@ pub enum DocsTagKind {
     Within,
     Param,
     Return,
+    Modifier,
 }
 
 impl DocsTagKind {
@@ -15,6 +16,7 @@ impl DocsTagKind {
             "within" => Ok(Self::Within),
             "param" => Ok(Self::Param),
             "return" => Ok(Self::Return),
+            "must_use" | "read_only" | "new_fields" => Ok(Self::Modifier),
             s => bail!("Unknown docs tag: '{}'", s),
         }
     }

packages/cli/src/gen/visitor.rs

@@ -21,7 +21,7 @@ pub struct DocumentationVisitor {
 
 impl DocumentationVisitor {
     pub fn new() -> Self {
-        let tag_regex = Regex::new(r#"^@(\S+)\s+(\S+)(.*)$"#).unwrap();
+        let tag_regex = Regex::new(r#"^@(\S+)\s*(.*)$"#).unwrap();
         Self {
             globals: vec![],
             functions: vec![],
@@ -35,12 +35,18 @@ impl DocumentationVisitor {
         if self.tag_regex.is_match(line) {
             let captures = self.tag_regex.captures(line).unwrap();
             let tag_kind = captures.get(1).unwrap().as_str();
-            let tag_name = captures.get(2).unwrap().as_str();
+            let tag_rest = captures.get(2).unwrap().as_str();
-            let tag_contents = captures.get(3).unwrap().as_str();
+            let mut tag_words = tag_rest.split_whitespace().collect::<Vec<_>>();
+            let tag_name = if tag_words.is_empty() {
+                String::new()
+            } else {
+                tag_words.remove(0).to_string()
+            };
+            let tag_contents = tag_words.join(" ");
             Some(DocsTag {
                 kind: DocsTagKind::parse(tag_kind).unwrap(),
-                name: tag_name.to_string(),
+                name: tag_name,
-                contents: tag_contents.to_string(),
+                contents: tag_contents,
             })
         } else {
             None

packages/cli/src/tests/mod.rs

@@ -33,7 +33,6 @@ async fn generate_luau_types() -> Result<()> {
 
 #[tokio::test]
 async fn generate_docs_file() -> Result<()> {
-    run_cli(Cli::new().generate_luau_types()).await?;
     run_cli(Cli::new().generate_docs_file()).await?;
     ensure_file_exists_and_is(FILE_NAME_DOCS, FileType::Json).await?;
     Ok(())