Finish new docs parser

docs/luneTypes.d.luau

@@ -431,7 +431,7 @@ declare stdio: {
 		Prompts for user input using the wanted kind of prompt:
 
 		* `"text"` - Prompts for a plain text string from the user
-		* `"confirm"` - Prompts the user to confirm with y / n
+		* `"confirm"` - Prompts the user to confirm with y / n (yes / no)
 		* `"select"` - Prompts the user to select *one* value from a list
 		* `"multiselect"` - Prompts the user to select *one or more* values from a list
 		* `nil` - Equivalent to `"text"` with no extra arguments
@@ -504,9 +504,28 @@ declare task: {
 	wait: (duration: number?) -> number,
 }
 
--- TODO: Write docs for these and include them in docs gen
+--[=[
+	Prints given value(s) to stdout.
 
+	This will format and prettify values such as tables, numbers, booleans, and more.
+]=]
 declare print: <T...>(T...) -> ()
+
+--[=[
+	Prints given value(s) to stdout with a leading "[INFO]" tag.
+
+	This will format and prettify values such as tables, numbers, booleans, and more.
+]=]
 declare info: <T...>(T...) -> ()
+
+--[=[
+	Prints given value(s) to stdout with a leading "[WARN]" tag.
+
+	This will format and prettify values such as tables, numbers, booleans, and more.
+]=]
 declare warn: <T...>(T...) -> ()
+
+--[=[
+	Throws an error and prints a formatted version of it with a leading "[ERROR]" tag.
+]=]
 declare error: <T>(message: T, level: number?) -> ()

packages/cli/src/gen/doc2/builder.rs

@@ -6,6 +6,7 @@ use super::{item::DocItem, kind::DocItemKind};
 pub struct DocItemBuilder {
     kind: Option<DocItemKind>,
     name: Option<String>,
+    meta: Option<String>,
     value: Option<String>,
     children: Vec<DocItem>,
 }
@@ -28,6 +29,11 @@ impl DocItemBuilder {
         self
     }
 
+    pub fn with_meta<S: AsRef<str>>(mut self, meta: S) -> Self {
+        self.meta = Some(meta.as_ref().to_string());
+        self
+    }
+
     pub fn with_value<S: AsRef<str>>(mut self, value: S) -> Self {
         self.value = Some(value.as_ref().to_string());
         self
@@ -45,18 +51,15 @@ impl DocItemBuilder {
 
     pub fn build(self) -> Result<DocItem> {
         if let Some(kind) = self.kind {
-            if let Some(name) = self.name {
+            let mut children = self.children;
-                let mut children = self.children;
+            children.sort();
-                children.sort();
+            Ok(DocItem {
-                Ok(DocItem {
+                kind,
-                    kind,
+                name: self.name,
-                    name,
+                meta: self.meta,
-                    value: self.value,
+                value: self.value,
-                    children,
+                children,
-                })
+            })
-            } else {
-                bail!("Missing doc item name")
-            }
         } else {
             bail!("Missing doc item kind")
         }

packages/cli/src/gen/doc2/item.rs

@@ -8,8 +8,13 @@ use super::kind::DocItemKind;
 #[serde(rename_all = "camelCase")]
 pub struct DocItem {
     pub(super) kind: DocItemKind,
-    pub(super) name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(super) name: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(super) meta: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub(super) value: Option<String>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
     pub(super) children: Vec<DocItem>,
 }
 
@@ -66,8 +71,12 @@ impl DocItem {
         self.kind.is_tag()
     }
 
-    pub fn get_name(&self) -> &str {
+    pub fn get_name(&self) -> Option<&str> {
-        &self.name
+        self.name.as_deref()
+    }
+
+    pub fn get_meta(&self) -> Option<&str> {
+        self.meta.as_deref()
     }
 
     pub fn get_value(&self) -> Option<&str> {

packages/cli/src/gen/doc2/kind.rs

@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
 #[serde(rename_all = "PascalCase")]
 pub enum DocItemKind {
     Root,
-    Global,
+    Table,
     Property,
     Function,
     Description,
@@ -19,8 +19,8 @@ impl DocItemKind {
         self == DocItemKind::Root
     }
 
-    pub fn is_global(self) -> bool {
+    pub fn is_table(self) -> bool {
-        self == DocItemKind::Global
+        self == DocItemKind::Table
     }
 
     pub fn is_property(self) -> bool {
@@ -47,7 +47,7 @@ impl fmt::Display for DocItemKind {
             "{}",
             match self {
                 Self::Root => "Root",
-                Self::Global => "Global",
+                Self::Table => "Table",
                 Self::Property => "Property",
                 Self::Function => "Function",
                 Self::Description => "Description",

packages/cli/src/gen/doc2/mod.rs

@@ -1,9 +1,11 @@
 mod builder;
 mod item;
 mod kind;
+mod parser;
+mod tag;
 mod tree;
-mod visitor;
 
 pub use item::DocItem;
 pub use kind::DocItemKind;
+pub use tag::DocsItemTag;
 pub use tree::DocTree;

packages/cli/src/gen/doc2/parser.rs

@@ -0,0 +1,190 @@
+use anyhow::{Context, Result};
+use full_moon::{
+    ast::{
+        types::{TypeFieldKey, TypeInfo},
+        Stmt,
+    },
+    tokenizer::{TokenReference, TokenType},
+};
+use regex::Regex;
+
+use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind};
+
+struct DocVisitorItem {
+    name: String,
+    comment: Option<String>,
+    exported: bool,
+    type_info: TypeInfo,
+}
+
+impl From<DocVisitorItem> for DocItem {
+    fn from(value: DocVisitorItem) -> Self {
+        let mut builder = DocItemBuilder::new()
+            .with_kind(match value.type_info {
+                TypeInfo::Array { .. } | TypeInfo::Table { .. } => DocItemKind::Table,
+                TypeInfo::Callback { .. } => DocItemKind::Function,
+                _ => unimplemented!("Support for globals that are not properties or functions is not yet implemented")
+            })
+            .with_name(value.name);
+        if let Some(comment) = value.comment {
+            builder = builder.with_children(&parse_moonwave_style_comment(&comment));
+        }
+        if let TypeInfo::Table { fields, .. } = value.type_info {
+            for field in fields.iter() {
+                if let TypeFieldKey::Name(name) = field.key() {
+                    let children = find_token_moonwave_comment(name)
+                        .as_deref()
+                        .map(parse_moonwave_style_comment)
+                        .unwrap_or_default();
+                    builder = builder.with_child(
+                        DocItemBuilder::new()
+                            .with_kind(match field.value() {
+                                TypeInfo::Callback { .. } => DocItemKind::Function,
+                                _ => DocItemKind::Property,
+                            })
+                            .with_name(name.token().to_string())
+                            .with_children(&children)
+                            .build()
+                            .unwrap(),
+                    );
+                }
+            }
+        }
+        builder.build().unwrap()
+    }
+}
+
+pub fn parse_type_definitions_into_doc_items<S>(contents: S) -> Result<Vec<DocItem>>
+where
+    S: AsRef<str>,
+{
+    let mut found_top_level_items = Vec::new();
+    let ast = full_moon::parse(&cleanup_type_definitions(contents.as_ref()))
+        .context("Failed to parse type definitions")?;
+    for stmt in ast.nodes().stmts() {
+        if let Some((declaration, token_reference)) = match stmt {
+            Stmt::ExportedTypeDeclaration(exp) => {
+                Some((exp.type_declaration(), exp.export_token()))
+            }
+            Stmt::TypeDeclaration(typ) => Some((typ, typ.type_token())),
+            _ => None,
+        } {
+            found_top_level_items.push(DocVisitorItem {
+                name: declaration.type_name().token().to_string(),
+                comment: find_token_moonwave_comment(token_reference),
+                exported: matches!(stmt, Stmt::ExportedTypeDeclaration(_)),
+                type_info: declaration.type_definition().clone(),
+            });
+        }
+    }
+    Ok(found_top_level_items
+        .drain(..)
+        .filter(|item| item.exported) // NOTE: Should we include items that are not exported? Probably not ..
+        .map(DocItem::from)
+        .collect())
+}
+
+fn should_separate_tag_meta(tag_kind: &str) -> bool {
+    matches!(tag_kind.trim().to_ascii_lowercase().as_ref(), "param")
+}
+
+fn parse_moonwave_style_tag(line: &str) -> Option<DocItem> {
+    let tag_regex = Regex::new(r#"^@(\S+)\s*(.*)$"#).unwrap();
+    if tag_regex.is_match(line) {
+        let captures = tag_regex.captures(line).unwrap();
+        let tag_kind = captures.get(1).unwrap().as_str();
+        let tag_rest = captures.get(2).unwrap().as_str();
+        let mut tag_words = tag_rest.split_whitespace().collect::<Vec<_>>();
+        let tag_name = if !tag_words.is_empty() && should_separate_tag_meta(tag_kind) {
+            tag_words.remove(0).to_string()
+        } else {
+            String::new()
+        };
+        let tag_contents = tag_words.join(" ");
+        if tag_kind.is_empty() {
+            None
+        } else {
+            let mut builder = DocItemBuilder::new()
+                .with_kind(DocItemKind::Tag)
+                .with_name(tag_kind);
+            if !tag_name.is_empty() {
+                builder = builder.with_meta(tag_name);
+            }
+            if !tag_contents.is_empty() {
+                builder = builder.with_value(tag_contents);
+            }
+            Some(builder.build().unwrap())
+        }
+    } else {
+        None
+    }
+}
+
+fn parse_moonwave_style_comment(comment: &str) -> Vec<DocItem> {
+    let lines = comment.lines().map(str::trim).collect::<Vec<_>>();
+    let indent_len = lines.iter().fold(usize::MAX, |acc, line| {
+        let first = line.chars().enumerate().find_map(|(idx, ch)| {
+            if ch.is_alphanumeric() {
+                Some(idx)
+            } else {
+                None
+            }
+        });
+        if let Some(first_alphanumeric) = first {
+            if first_alphanumeric > 0 {
+                acc.min(first_alphanumeric - 1)
+            } else {
+                0
+            }
+        } else {
+            acc
+        }
+    });
+    let unindented_lines = lines.iter().map(|line| &line[indent_len..]);
+    let mut doc_items = Vec::new();
+    let mut doc_lines = Vec::new();
+    for line in unindented_lines {
+        if let Some(tag) = parse_moonwave_style_tag(line) {
+            doc_items.push(tag);
+        } else {
+            doc_lines.push(line);
+        }
+    }
+    if !doc_lines.is_empty() {
+        doc_items.push(
+            DocItemBuilder::new()
+                .with_kind(DocItemKind::Description)
+                .with_value(doc_lines.join("\n").trim())
+                .build()
+                .unwrap(),
+        );
+    }
+    doc_items
+}
+
+fn find_token_moonwave_comment(token: &TokenReference) -> Option<String> {
+    token
+        .leading_trivia()
+        .filter_map(|trivia| match trivia.token_type() {
+            TokenType::MultiLineComment { blocks, comment } if blocks == &1 => Some(comment),
+            _ => None,
+        })
+        .last()
+        .map(|comment| comment.trim().to_string())
+}
+
+fn cleanup_type_definitions(contents: &str) -> String {
+    // TODO: Properly handle the "declare class" syntax, for now we just skip it
+    let mut no_declares = contents.to_string();
+    while let Some(dec) = no_declares.find("\ndeclare class") {
+        let end = no_declares.find("\nend").unwrap();
+        let before = &no_declares[0..dec];
+        let after = &no_declares[end + 4..];
+        no_declares = format!("{before}{after}");
+    }
+    let (regex, replacement) = (
+        Regex::new(r#"declare (?P<n>\w+): "#).unwrap(),
+        r#"export type $n = "#,
+    );
+    regex.replace_all(&no_declares, replacement).to_string()
+}

packages/cli/src/gen/doc2/tag.rs

@@ -0,0 +1,60 @@
+use anyhow::{bail, Context, Result};
+use serde::{Deserialize, Serialize};
+
+use super::item::DocItem;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub enum DocsItemTag {
+    Class(String),
+    Within(String),
+    Param((String, String)),
+    Return(String),
+    MustUse,
+    ReadOnly,
+    NewFields,
+}
+
+impl TryFrom<DocItem> for DocsItemTag {
+    type Error = anyhow::Error;
+    fn try_from(value: DocItem) -> Result<Self> {
+        if let Some(name) = value.get_name() {
+            Ok(match name.trim().to_ascii_lowercase().as_ref() {
+                "class" => Self::Class(
+                    value
+                        .get_value()
+                        .context("Missing class name for class tag")?
+                        .to_string(),
+                ),
+                "within" => Self::Within(
+                    value
+                        .get_value()
+                        .context("Missing class name for within tag")?
+                        .to_string(),
+                ),
+                "param" => Self::Param((
+                    value
+                        .get_meta()
+                        .context("Missing param name for param tag")?
+                        .to_string(),
+                    value
+                        .get_value()
+                        .context("Missing param value for param tag")?
+                        .to_string(),
+                )),
+                "return" => Self::Return(
+                    value
+                        .get_value()
+                        .context("Missing description for return tag")?
+                        .to_string(),
+                ),
+                "must_use" => Self::MustUse,
+                "read_only" => Self::ReadOnly,
+                "new_fields" => Self::NewFields,
+                s => bail!("Unknown docs tag: '{}'", s),
+            })
+        } else {
+            bail!("Doc item has no name")
+        }
+    }
+}

packages/cli/src/gen/doc2/tree.rs

@@ -1,14 +1,20 @@
 use std::ops::{Deref, DerefMut};
 
 use anyhow::{Context, Result};
+use serde::{Deserialize, Serialize};
 
-use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind, visitor::DocVisitor};
+use super::{
+    builder::DocItemBuilder, item::DocItem, kind::DocItemKind,
+    parser::parse_type_definitions_into_doc_items,
+};
 
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct DocTree(DocItem);
 
+#[allow(dead_code)]
 impl DocTree {
     pub fn from_type_definitions<S: AsRef<str>>(type_definitions_contents: S) -> Result<Self> {
-        let top_level_items = DocVisitor::visit_type_definitions_str(type_definitions_contents)
+        let top_level_items = parse_type_definitions_into_doc_items(type_definitions_contents)
             .context("Failed to visit type definitions AST")?;
         let root = DocItemBuilder::new()
             .with_kind(DocItemKind::Root)
@@ -18,10 +24,14 @@ impl DocTree {
         Ok(Self(root))
     }
 
-    #[allow(dead_code, clippy::unused_self)]
+    #[allow(clippy::unused_self)]
     pub fn is_root(&self) -> bool {
         true
     }
+
+    pub fn into_inner(self) -> DocItem {
+        self.0
+    }
 }
 
 impl Deref for DocTree {

packages/cli/src/gen/doc2/visitor.rs

@@ -1,103 +0,0 @@
-use anyhow::{Context, Result};
-use full_moon::{
-    ast::{types::TypeInfo, Ast, Stmt},
-    tokenizer::TokenKind,
-    visitors::Visitor,
-};
-use regex::Regex;
-
-use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind};
-
-struct DocVisitorItem {
-    name: String,
-    comment: Option<String>,
-    exported: bool,
-    ast: TypeInfo,
-}
-
-impl From<DocVisitorItem> for DocItem {
-    fn from(value: DocVisitorItem) -> Self {
-        let mut builder = DocItemBuilder::new()
-            .with_kind(DocItemKind::Global)
-            .with_name(value.name);
-        if let Some(comment) = value.comment {
-            builder = builder.with_child(
-                DocItemBuilder::new()
-                    .with_kind(DocItemKind::Description)
-                    .with_name("Description")
-                    .with_value(comment)
-                    .build()
-                    .unwrap(),
-            );
-        }
-        builder.build().unwrap()
-    }
-}
-
-pub struct DocVisitor {
-    pending_visitor_items: Vec<DocVisitorItem>,
-}
-
-impl DocVisitor {
-    pub fn visit_type_definitions_str<S>(contents: S) -> Result<Vec<DocItem>>
-    where
-        S: AsRef<str>,
-    {
-        let mut this = Self {
-            pending_visitor_items: Vec::new(),
-        };
-        this.visit_ast(
-            &full_moon::parse(&cleanup_type_definitions(contents.as_ref()))
-                .context("Failed to parse type definitions")?,
-        );
-        Ok(this
-            .pending_visitor_items
-            .drain(..)
-            .filter(|item| item.exported) // NOTE: Should we include items that are not exported? Probably not ..
-            .map(DocItem::from)
-            .collect())
-    }
-}
-
-impl Visitor for DocVisitor {
-    fn visit_ast(&mut self, ast: &Ast)
-    where
-        Self: Sized,
-    {
-        for stmt in ast.nodes().stmts() {
-            if let Some((declaration, leading_trivia)) = match stmt {
-                Stmt::ExportedTypeDeclaration(exp) => {
-                    Some((exp.type_declaration(), exp.export_token().leading_trivia()))
-                }
-                Stmt::TypeDeclaration(typ) => Some((typ, typ.type_token().leading_trivia())),
-                _ => None,
-            } {
-                self.pending_visitor_items.push(DocVisitorItem {
-                    name: declaration.type_name().to_string(),
-                    comment: leading_trivia
-                        .filter(|trivia| matches!(trivia.token_kind(), TokenKind::MultiLineComment))
-                        .last()
-                        .map(ToString::to_string),
-                    exported: matches!(stmt, Stmt::ExportedTypeDeclaration(_)),
-                    ast: declaration.type_definition().clone(),
-                });
-            }
-        }
-    }
-}
-
-fn cleanup_type_definitions(contents: &str) -> String {
-    // TODO: Properly handle the "declare class" syntax, for now we just skip it
-    let mut no_declares = contents.to_string();
-    while let Some(dec) = no_declares.find("\ndeclare class") {
-        let end = no_declares.find("\nend").unwrap();
-        let before = &no_declares[0..dec];
-        let after = &no_declares[end + 4..];
-        no_declares = format!("{before}{after}");
-    }
-    let (regex, replacement) = (
-        Regex::new(r#"declare (?P<n>\w+): "#).unwrap(),
-        r#"export type $n = "#,
-    );
-    regex.replace_all(&no_declares, replacement).to_string()
-}

packages/cli/src/gen/mod.rs

@@ -1,9 +1,10 @@
 mod doc;
-mod doc2;
 mod docs_file;
 mod selene_defs;
 mod wiki_dir;
 
+pub mod doc2;
+
 pub use docs_file::generate_from_type_definitions as generate_docs_json_from_definitions;
 pub use selene_defs::generate_from_type_definitions as generate_selene_defs_from_definitions;
 pub use wiki_dir::generate_from_type_definitions as generate_wiki_dir_from_definitions;