From c252db45985051e5777fbf4c781129cb3e4d2ca9 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Tue, 21 Feb 2023 21:06:11 +0100 Subject: [PATCH] Finish new docs parser --- docs/luneTypes.d.luau | 23 +++- packages/cli/src/gen/doc2/builder.rs | 27 ++-- packages/cli/src/gen/doc2/item.rs | 15 ++- packages/cli/src/gen/doc2/kind.rs | 8 +- packages/cli/src/gen/doc2/mod.rs | 4 +- packages/cli/src/gen/doc2/parser.rs | 190 +++++++++++++++++++++++++++ packages/cli/src/gen/doc2/tag.rs | 60 +++++++++ packages/cli/src/gen/doc2/tree.rs | 16 ++- packages/cli/src/gen/doc2/visitor.rs | 103 --------------- packages/cli/src/gen/mod.rs | 3 +- 10 files changed, 320 insertions(+), 129 deletions(-) create mode 100644 packages/cli/src/gen/doc2/parser.rs create mode 100644 packages/cli/src/gen/doc2/tag.rs delete mode 100644 packages/cli/src/gen/doc2/visitor.rs diff --git a/docs/luneTypes.d.luau b/docs/luneTypes.d.luau index d04bec4..9d23776 100644 --- a/docs/luneTypes.d.luau +++ b/docs/luneTypes.d.luau @@ -431,7 +431,7 @@ declare stdio: { Prompts for user input using the wanted kind of prompt: * `"text"` - Prompts for a plain text string from the user - * `"confirm"` - Prompts the user to confirm with y / n + * `"confirm"` - Prompts the user to confirm with y / n (yes / no) * `"select"` - Prompts the user to select *one* value from a list * `"multiselect"` - Prompts the user to select *one or more* values from a list * `nil` - Equivalent to `"text"` with no extra arguments @@ -504,9 +504,28 @@ declare task: { wait: (duration: number?) -> number, } --- TODO: Write docs for these and include them in docs gen +--[=[ + Prints given value(s) to stdout. + This will format and prettify values such as tables, numbers, booleans, and more. +]=] declare print: (T...) -> () + +--[=[ + Prints given value(s) to stdout with a leading "[INFO]" tag. + + This will format and prettify values such as tables, numbers, booleans, and more. +]=] declare info: (T...) -> () + +--[=[ + Prints given value(s) to stdout with a leading "[WARN]" tag. + + This will format and prettify values such as tables, numbers, booleans, and more. +]=] declare warn: (T...) -> () + +--[=[ + Throws an error and prints a formatted version of it with a leading "[ERROR]" tag. +]=] declare error: (message: T, level: number?) -> () diff --git a/packages/cli/src/gen/doc2/builder.rs b/packages/cli/src/gen/doc2/builder.rs index 5b229f2..816c55b 100644 --- a/packages/cli/src/gen/doc2/builder.rs +++ b/packages/cli/src/gen/doc2/builder.rs @@ -6,6 +6,7 @@ use super::{item::DocItem, kind::DocItemKind}; pub struct DocItemBuilder { kind: Option, name: Option, + meta: Option, value: Option, children: Vec, } @@ -28,6 +29,11 @@ impl DocItemBuilder { self } + pub fn with_meta>(mut self, meta: S) -> Self { + self.meta = Some(meta.as_ref().to_string()); + self + } + pub fn with_value>(mut self, value: S) -> Self { self.value = Some(value.as_ref().to_string()); self @@ -45,18 +51,15 @@ impl DocItemBuilder { pub fn build(self) -> Result { if let Some(kind) = self.kind { - if let Some(name) = self.name { - let mut children = self.children; - children.sort(); - Ok(DocItem { - kind, - name, - value: self.value, - children, - }) - } else { - bail!("Missing doc item name") - } + let mut children = self.children; + children.sort(); + Ok(DocItem { + kind, + name: self.name, + meta: self.meta, + value: self.value, + children, + }) } else { bail!("Missing doc item kind") } diff --git a/packages/cli/src/gen/doc2/item.rs b/packages/cli/src/gen/doc2/item.rs index 20368b7..df64cb1 100644 --- a/packages/cli/src/gen/doc2/item.rs +++ b/packages/cli/src/gen/doc2/item.rs @@ -8,8 +8,13 @@ use super::kind::DocItemKind; #[serde(rename_all = "camelCase")] pub struct DocItem { pub(super) kind: DocItemKind, - pub(super) name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) meta: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub(super) value: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] pub(super) children: Vec, } @@ -66,8 +71,12 @@ impl DocItem { self.kind.is_tag() } - pub fn get_name(&self) -> &str { - &self.name + pub fn get_name(&self) -> Option<&str> { + self.name.as_deref() + } + + pub fn get_meta(&self) -> Option<&str> { + self.meta.as_deref() } pub fn get_value(&self) -> Option<&str> { diff --git a/packages/cli/src/gen/doc2/kind.rs b/packages/cli/src/gen/doc2/kind.rs index 0b469af..59a690e 100644 --- a/packages/cli/src/gen/doc2/kind.rs +++ b/packages/cli/src/gen/doc2/kind.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; #[serde(rename_all = "PascalCase")] pub enum DocItemKind { Root, - Global, + Table, Property, Function, Description, @@ -19,8 +19,8 @@ impl DocItemKind { self == DocItemKind::Root } - pub fn is_global(self) -> bool { - self == DocItemKind::Global + pub fn is_table(self) -> bool { + self == DocItemKind::Table } pub fn is_property(self) -> bool { @@ -47,7 +47,7 @@ impl fmt::Display for DocItemKind { "{}", match self { Self::Root => "Root", - Self::Global => "Global", + Self::Table => "Table", Self::Property => "Property", Self::Function => "Function", Self::Description => "Description", diff --git a/packages/cli/src/gen/doc2/mod.rs b/packages/cli/src/gen/doc2/mod.rs index dbea269..326e35c 100644 --- a/packages/cli/src/gen/doc2/mod.rs +++ b/packages/cli/src/gen/doc2/mod.rs @@ -1,9 +1,11 @@ mod builder; mod item; mod kind; +mod parser; +mod tag; mod tree; -mod visitor; pub use item::DocItem; pub use kind::DocItemKind; +pub use tag::DocsItemTag; pub use tree::DocTree; diff --git a/packages/cli/src/gen/doc2/parser.rs b/packages/cli/src/gen/doc2/parser.rs new file mode 100644 index 0000000..5f6cd85 --- /dev/null +++ b/packages/cli/src/gen/doc2/parser.rs @@ -0,0 +1,190 @@ +use anyhow::{Context, Result}; +use full_moon::{ + ast::{ + types::{TypeFieldKey, TypeInfo}, + Stmt, + }, + tokenizer::{TokenReference, TokenType}, +}; +use regex::Regex; + +use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind}; + +struct DocVisitorItem { + name: String, + comment: Option, + exported: bool, + type_info: TypeInfo, +} + +impl From for DocItem { + fn from(value: DocVisitorItem) -> Self { + let mut builder = DocItemBuilder::new() + .with_kind(match value.type_info { + TypeInfo::Array { .. } | TypeInfo::Table { .. } => DocItemKind::Table, + TypeInfo::Callback { .. } => DocItemKind::Function, + _ => unimplemented!("Support for globals that are not properties or functions is not yet implemented") + }) + .with_name(value.name); + if let Some(comment) = value.comment { + builder = builder.with_children(&parse_moonwave_style_comment(&comment)); + } + if let TypeInfo::Table { fields, .. } = value.type_info { + for field in fields.iter() { + if let TypeFieldKey::Name(name) = field.key() { + let children = find_token_moonwave_comment(name) + .as_deref() + .map(parse_moonwave_style_comment) + .unwrap_or_default(); + builder = builder.with_child( + DocItemBuilder::new() + .with_kind(match field.value() { + TypeInfo::Callback { .. } => DocItemKind::Function, + _ => DocItemKind::Property, + }) + .with_name(name.token().to_string()) + .with_children(&children) + .build() + .unwrap(), + ); + } + } + } + builder.build().unwrap() + } +} + +pub fn parse_type_definitions_into_doc_items(contents: S) -> Result> +where + S: AsRef, +{ + let mut found_top_level_items = Vec::new(); + let ast = full_moon::parse(&cleanup_type_definitions(contents.as_ref())) + .context("Failed to parse type definitions")?; + for stmt in ast.nodes().stmts() { + if let Some((declaration, token_reference)) = match stmt { + Stmt::ExportedTypeDeclaration(exp) => { + Some((exp.type_declaration(), exp.export_token())) + } + Stmt::TypeDeclaration(typ) => Some((typ, typ.type_token())), + _ => None, + } { + found_top_level_items.push(DocVisitorItem { + name: declaration.type_name().token().to_string(), + comment: find_token_moonwave_comment(token_reference), + exported: matches!(stmt, Stmt::ExportedTypeDeclaration(_)), + type_info: declaration.type_definition().clone(), + }); + } + } + Ok(found_top_level_items + .drain(..) + .filter(|item| item.exported) // NOTE: Should we include items that are not exported? Probably not .. + .map(DocItem::from) + .collect()) +} + +fn should_separate_tag_meta(tag_kind: &str) -> bool { + matches!(tag_kind.trim().to_ascii_lowercase().as_ref(), "param") +} + +fn parse_moonwave_style_tag(line: &str) -> Option { + let tag_regex = Regex::new(r#"^@(\S+)\s*(.*)$"#).unwrap(); + if tag_regex.is_match(line) { + let captures = tag_regex.captures(line).unwrap(); + let tag_kind = captures.get(1).unwrap().as_str(); + let tag_rest = captures.get(2).unwrap().as_str(); + let mut tag_words = tag_rest.split_whitespace().collect::>(); + let tag_name = if !tag_words.is_empty() && should_separate_tag_meta(tag_kind) { + tag_words.remove(0).to_string() + } else { + String::new() + }; + let tag_contents = tag_words.join(" "); + if tag_kind.is_empty() { + None + } else { + let mut builder = DocItemBuilder::new() + .with_kind(DocItemKind::Tag) + .with_name(tag_kind); + if !tag_name.is_empty() { + builder = builder.with_meta(tag_name); + } + if !tag_contents.is_empty() { + builder = builder.with_value(tag_contents); + } + Some(builder.build().unwrap()) + } + } else { + None + } +} + +fn parse_moonwave_style_comment(comment: &str) -> Vec { + let lines = comment.lines().map(str::trim).collect::>(); + let indent_len = lines.iter().fold(usize::MAX, |acc, line| { + let first = line.chars().enumerate().find_map(|(idx, ch)| { + if ch.is_alphanumeric() { + Some(idx) + } else { + None + } + }); + if let Some(first_alphanumeric) = first { + if first_alphanumeric > 0 { + acc.min(first_alphanumeric - 1) + } else { + 0 + } + } else { + acc + } + }); + let unindented_lines = lines.iter().map(|line| &line[indent_len..]); + let mut doc_items = Vec::new(); + let mut doc_lines = Vec::new(); + for line in unindented_lines { + if let Some(tag) = parse_moonwave_style_tag(line) { + doc_items.push(tag); + } else { + doc_lines.push(line); + } + } + if !doc_lines.is_empty() { + doc_items.push( + DocItemBuilder::new() + .with_kind(DocItemKind::Description) + .with_value(doc_lines.join("\n").trim()) + .build() + .unwrap(), + ); + } + doc_items +} + +fn find_token_moonwave_comment(token: &TokenReference) -> Option { + token + .leading_trivia() + .filter_map(|trivia| match trivia.token_type() { + TokenType::MultiLineComment { blocks, comment } if blocks == &1 => Some(comment), + _ => None, + }) + .last() + .map(|comment| comment.trim().to_string()) +} + +fn cleanup_type_definitions(contents: &str) -> String { + // TODO: Properly handle the "declare class" syntax, for now we just skip it + let mut no_declares = contents.to_string(); + while let Some(dec) = no_declares.find("\ndeclare class") { + let end = no_declares.find("\nend").unwrap(); + let before = &no_declares[0..dec]; + let after = &no_declares[end + 4..]; + no_declares = format!("{before}{after}"); + } + let (regex, replacement) = ( + Regex::new(r#"declare (?P\w+): "#).unwrap(), + r#"export type $n = "#, + ); + regex.replace_all(&no_declares, replacement).to_string() +} diff --git a/packages/cli/src/gen/doc2/tag.rs b/packages/cli/src/gen/doc2/tag.rs new file mode 100644 index 0000000..741ab48 --- /dev/null +++ b/packages/cli/src/gen/doc2/tag.rs @@ -0,0 +1,60 @@ +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; + +use super::item::DocItem; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub enum DocsItemTag { + Class(String), + Within(String), + Param((String, String)), + Return(String), + MustUse, + ReadOnly, + NewFields, +} + +impl TryFrom for DocsItemTag { + type Error = anyhow::Error; + fn try_from(value: DocItem) -> Result { + if let Some(name) = value.get_name() { + Ok(match name.trim().to_ascii_lowercase().as_ref() { + "class" => Self::Class( + value + .get_value() + .context("Missing class name for class tag")? + .to_string(), + ), + "within" => Self::Within( + value + .get_value() + .context("Missing class name for within tag")? + .to_string(), + ), + "param" => Self::Param(( + value + .get_meta() + .context("Missing param name for param tag")? + .to_string(), + value + .get_value() + .context("Missing param value for param tag")? + .to_string(), + )), + "return" => Self::Return( + value + .get_value() + .context("Missing description for return tag")? + .to_string(), + ), + "must_use" => Self::MustUse, + "read_only" => Self::ReadOnly, + "new_fields" => Self::NewFields, + s => bail!("Unknown docs tag: '{}'", s), + }) + } else { + bail!("Doc item has no name") + } + } +} diff --git a/packages/cli/src/gen/doc2/tree.rs b/packages/cli/src/gen/doc2/tree.rs index 21286c9..982f6cf 100644 --- a/packages/cli/src/gen/doc2/tree.rs +++ b/packages/cli/src/gen/doc2/tree.rs @@ -1,14 +1,20 @@ use std::ops::{Deref, DerefMut}; use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; -use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind, visitor::DocVisitor}; +use super::{ + builder::DocItemBuilder, item::DocItem, kind::DocItemKind, + parser::parse_type_definitions_into_doc_items, +}; +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DocTree(DocItem); +#[allow(dead_code)] impl DocTree { pub fn from_type_definitions>(type_definitions_contents: S) -> Result { - let top_level_items = DocVisitor::visit_type_definitions_str(type_definitions_contents) + let top_level_items = parse_type_definitions_into_doc_items(type_definitions_contents) .context("Failed to visit type definitions AST")?; let root = DocItemBuilder::new() .with_kind(DocItemKind::Root) @@ -18,10 +24,14 @@ impl DocTree { Ok(Self(root)) } - #[allow(dead_code, clippy::unused_self)] + #[allow(clippy::unused_self)] pub fn is_root(&self) -> bool { true } + + pub fn into_inner(self) -> DocItem { + self.0 + } } impl Deref for DocTree { diff --git a/packages/cli/src/gen/doc2/visitor.rs b/packages/cli/src/gen/doc2/visitor.rs deleted file mode 100644 index 7263c83..0000000 --- a/packages/cli/src/gen/doc2/visitor.rs +++ /dev/null @@ -1,103 +0,0 @@ -use anyhow::{Context, Result}; -use full_moon::{ - ast::{types::TypeInfo, Ast, Stmt}, - tokenizer::TokenKind, - visitors::Visitor, -}; -use regex::Regex; - -use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind}; - -struct DocVisitorItem { - name: String, - comment: Option, - exported: bool, - ast: TypeInfo, -} - -impl From for DocItem { - fn from(value: DocVisitorItem) -> Self { - let mut builder = DocItemBuilder::new() - .with_kind(DocItemKind::Global) - .with_name(value.name); - if let Some(comment) = value.comment { - builder = builder.with_child( - DocItemBuilder::new() - .with_kind(DocItemKind::Description) - .with_name("Description") - .with_value(comment) - .build() - .unwrap(), - ); - } - builder.build().unwrap() - } -} - -pub struct DocVisitor { - pending_visitor_items: Vec, -} - -impl DocVisitor { - pub fn visit_type_definitions_str(contents: S) -> Result> - where - S: AsRef, - { - let mut this = Self { - pending_visitor_items: Vec::new(), - }; - this.visit_ast( - &full_moon::parse(&cleanup_type_definitions(contents.as_ref())) - .context("Failed to parse type definitions")?, - ); - Ok(this - .pending_visitor_items - .drain(..) - .filter(|item| item.exported) // NOTE: Should we include items that are not exported? Probably not .. - .map(DocItem::from) - .collect()) - } -} - -impl Visitor for DocVisitor { - fn visit_ast(&mut self, ast: &Ast) - where - Self: Sized, - { - for stmt in ast.nodes().stmts() { - if let Some((declaration, leading_trivia)) = match stmt { - Stmt::ExportedTypeDeclaration(exp) => { - Some((exp.type_declaration(), exp.export_token().leading_trivia())) - } - Stmt::TypeDeclaration(typ) => Some((typ, typ.type_token().leading_trivia())), - _ => None, - } { - self.pending_visitor_items.push(DocVisitorItem { - name: declaration.type_name().to_string(), - comment: leading_trivia - .filter(|trivia| matches!(trivia.token_kind(), TokenKind::MultiLineComment)) - .last() - .map(ToString::to_string), - exported: matches!(stmt, Stmt::ExportedTypeDeclaration(_)), - ast: declaration.type_definition().clone(), - }); - } - } - } -} - -fn cleanup_type_definitions(contents: &str) -> String { - // TODO: Properly handle the "declare class" syntax, for now we just skip it - let mut no_declares = contents.to_string(); - while let Some(dec) = no_declares.find("\ndeclare class") { - let end = no_declares.find("\nend").unwrap(); - let before = &no_declares[0..dec]; - let after = &no_declares[end + 4..]; - no_declares = format!("{before}{after}"); - } - let (regex, replacement) = ( - Regex::new(r#"declare (?P\w+): "#).unwrap(), - r#"export type $n = "#, - ); - regex.replace_all(&no_declares, replacement).to_string() -} diff --git a/packages/cli/src/gen/mod.rs b/packages/cli/src/gen/mod.rs index a568afe..2ddaa33 100644 --- a/packages/cli/src/gen/mod.rs +++ b/packages/cli/src/gen/mod.rs @@ -1,9 +1,10 @@ mod doc; -mod doc2; mod docs_file; mod selene_defs; mod wiki_dir; +pub mod doc2; + pub use docs_file::generate_from_type_definitions as generate_docs_json_from_definitions; pub use selene_defs::generate_from_type_definitions as generate_selene_defs_from_definitions; pub use wiki_dir::generate_from_type_definitions as generate_wiki_dir_from_definitions;