From 67f9d4eb9f5284d9853a13436d9d959e65e1d1f5 Mon Sep 17 00:00:00 2001 From: Filip Tibell Date: Wed, 22 Feb 2023 10:11:36 +0100 Subject: [PATCH] Implement full selene typedefs generator --- packages/cli/src/gen/doc2/builder.rs | 14 ++ packages/cli/src/gen/doc2/item.rs | 6 + packages/cli/src/gen/doc2/mod.rs | 2 + packages/cli/src/gen/doc2/moonwave.rs | 81 ++++++++ packages/cli/src/gen/doc2/parser.rs | 280 ++++++++++++++++---------- packages/cli/src/gen/selene_defs.rs | 41 +++- 6 files changed, 320 insertions(+), 104 deletions(-) create mode 100644 packages/cli/src/gen/doc2/moonwave.rs diff --git a/packages/cli/src/gen/doc2/builder.rs b/packages/cli/src/gen/doc2/builder.rs index 816c55b..69ae329 100644 --- a/packages/cli/src/gen/doc2/builder.rs +++ b/packages/cli/src/gen/doc2/builder.rs @@ -9,6 +9,7 @@ pub struct DocItemBuilder { meta: Option, value: Option, children: Vec, + arg_types: Vec, } #[allow(dead_code)] @@ -49,6 +50,18 @@ impl DocItemBuilder { self } + pub fn with_arg_type>(mut self, arg_type: S) -> Self { + self.arg_types.push(arg_type.as_ref().to_string()); + self + } + + pub fn with_arg_types>(mut self, arg_types: &[S]) -> Self { + for arg_type in arg_types { + self.arg_types.push(arg_type.as_ref().to_string()); + } + self + } + pub fn build(self) -> Result { if let Some(kind) = self.kind { let mut children = self.children; @@ -59,6 +72,7 @@ impl DocItemBuilder { meta: self.meta, value: self.value, children, + arg_types: self.arg_types, }) } else { bail!("Missing doc item kind") diff --git a/packages/cli/src/gen/doc2/item.rs b/packages/cli/src/gen/doc2/item.rs index dc9c014..97c02b7 100644 --- a/packages/cli/src/gen/doc2/item.rs +++ b/packages/cli/src/gen/doc2/item.rs @@ -16,6 +16,8 @@ pub struct DocItem { pub(super) value: Option, #[serde(skip_serializing_if = "Vec::is_empty")] pub(super) children: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub(super) arg_types: Vec, } impl PartialOrd for DocItem { @@ -90,4 +92,8 @@ impl DocItem { pub fn children(&self) -> &[DocItem] { &self.children } + + pub fn arg_types(&self) -> Vec<&str> { + self.arg_types.iter().map(String::as_str).collect() + } } diff --git a/packages/cli/src/gen/doc2/mod.rs b/packages/cli/src/gen/doc2/mod.rs index 326e35c..2d443bf 100644 --- a/packages/cli/src/gen/doc2/mod.rs +++ b/packages/cli/src/gen/doc2/mod.rs @@ -1,11 +1,13 @@ mod builder; mod item; mod kind; +mod moonwave; mod parser; mod tag; mod tree; pub use item::DocItem; pub use kind::DocItemKind; +pub use parser::PIPE_SEPARATOR; pub use tag::DocsItemTag; pub use tree::DocTree; diff --git a/packages/cli/src/gen/doc2/moonwave.rs b/packages/cli/src/gen/doc2/moonwave.rs new file mode 100644 index 0000000..69e8d8a --- /dev/null +++ b/packages/cli/src/gen/doc2/moonwave.rs @@ -0,0 +1,81 @@ +use regex::Regex; + +use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind}; + +fn should_separate_tag_meta(tag_kind: &str) -> bool { + matches!(tag_kind.trim().to_ascii_lowercase().as_ref(), "param") +} + +fn parse_moonwave_style_tag(line: &str) -> Option { + let tag_regex = Regex::new(r#"^@(\S+)\s*(.*)$"#).unwrap(); + if tag_regex.is_match(line) { + let captures = tag_regex.captures(line).unwrap(); + let tag_kind = captures.get(1).unwrap().as_str(); + let tag_rest = captures.get(2).unwrap().as_str(); + let mut tag_words = tag_rest.split_whitespace().collect::>(); + let tag_name = if !tag_words.is_empty() && should_separate_tag_meta(tag_kind) { + tag_words.remove(0).to_string() + } else { + String::new() + }; + let tag_contents = tag_words.join(" "); + if tag_kind.is_empty() { + None + } else { + let mut builder = DocItemBuilder::new() + .with_kind(DocItemKind::Tag) + .with_name(tag_kind); + if !tag_name.is_empty() { + builder = builder.with_meta(tag_name); + } + if !tag_contents.is_empty() { + builder = builder.with_value(tag_contents); + } + Some(builder.build().unwrap()) + } + } else { + None + } +} + +pub(super) fn parse_moonwave_style_comment(comment: &str) -> Vec { + let lines = comment.lines().map(str::trim).collect::>(); + let indent_len = lines.iter().fold(usize::MAX, |acc, line| { + let first = line.chars().enumerate().find_map(|(idx, ch)| { + if ch.is_alphanumeric() { + Some(idx) + } else { + None + } + }); + if let Some(first_alphanumeric) = first { + if first_alphanumeric > 0 { + acc.min(first_alphanumeric - 1) + } else { + 0 + } + } else { + acc + } + }); + let unindented_lines = lines.iter().map(|line| &line[indent_len..]); + let mut doc_items = Vec::new(); + let mut doc_lines = Vec::new(); + for line in unindented_lines { + if let Some(tag) = parse_moonwave_style_tag(line) { + doc_items.push(tag); + } else { + doc_lines.push(line); + } + } + if !doc_lines.is_empty() { + doc_items.push( + DocItemBuilder::new() + .with_kind(DocItemKind::Description) + .with_value(doc_lines.join("\n").trim()) + .build() + .unwrap(), + ); + } + doc_items +} diff --git a/packages/cli/src/gen/doc2/parser.rs b/packages/cli/src/gen/doc2/parser.rs index f082e80..3cf113c 100644 --- a/packages/cli/src/gen/doc2/parser.rs +++ b/packages/cli/src/gen/doc2/parser.rs @@ -1,53 +1,47 @@ use anyhow::{Context, Result}; use full_moon::{ ast::{ - types::{TypeFieldKey, TypeInfo}, + types::{TypeArgument, TypeFieldKey, TypeInfo}, Stmt, }, tokenizer::{TokenReference, TokenType}, }; use regex::Regex; -use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind}; +use super::{ + builder::DocItemBuilder, item::DocItem, kind::DocItemKind, + moonwave::parse_moonwave_style_comment, +}; + +pub const PIPE_SEPARATOR: &str = " | "; #[derive(Debug, Clone)] struct DocVisitorItem { name: String, comment: Option, - exported: bool, type_info: TypeInfo, } impl From for DocItem { fn from(value: DocVisitorItem) -> Self { let mut builder = DocItemBuilder::new() - .with_kind(match &value.type_info { - TypeInfo::Array { .. } | TypeInfo::Table { .. } => DocItemKind::Table, - typ if type_info_is_fn(typ) => DocItemKind::Function, - _ => unimplemented!("Support for globals that are not properties or functions is not yet implemented") - }) - .with_name(value.name); + .with_kind(DocItemKind::from(&value.type_info)) + .with_name(&value.name); if let Some(comment) = value.comment { builder = builder.with_children(&parse_moonwave_style_comment(&comment)); } + if let Some(args) = try_extract_normalized_function_args(&value.type_info) { + println!("{} > {args:?}", value.name); + builder = builder.with_arg_types(&args); + } if let TypeInfo::Table { fields, .. } = value.type_info { for field in fields.iter() { if let TypeFieldKey::Name(name) = field.key() { - let children = find_token_moonwave_comment(name) - .as_deref() - .map(parse_moonwave_style_comment) - .unwrap_or_default(); - builder = builder.with_child( - DocItemBuilder::new() - .with_kind(match field.value() { - typ if type_info_is_fn(typ) => DocItemKind::Function, - _ => DocItemKind::Property, - }) - .with_name(name.token().to_string()) - .with_children(&children) - .build() - .unwrap(), - ); + builder = builder.with_child(DocItem::from(DocVisitorItem { + name: name.token().to_string(), + comment: find_token_moonwave_comment(name), + type_info: field.value().clone(), + })); } } } @@ -55,6 +49,43 @@ impl From for DocItem { } } +impl From<&TypeInfo> for DocItemKind { + fn from(value: &TypeInfo) -> Self { + match value { + TypeInfo::Array { .. } | TypeInfo::Table { .. } => DocItemKind::Table, + TypeInfo::Basic(_) | TypeInfo::String(_) => DocItemKind::Property, + TypeInfo::Optional { base, .. } => DocItemKind::from(base.as_ref()), + TypeInfo::Tuple { types, .. } => { + let mut kinds = types.iter().map(DocItemKind::from).collect::>(); + let kinds_all_the_same = kinds.windows(2).all(|w| w[0] == w[1]); + if kinds_all_the_same && !kinds.is_empty() { + kinds.pop().unwrap() + } else { + unimplemented!( + "Missing support for tuple with differing types in type definitions parser", + ) + } + } + TypeInfo::Union { left, right, .. } | TypeInfo::Intersection { left, right, .. } => { + let kind_left = DocItemKind::from(left.as_ref()); + let kind_right = DocItemKind::from(right.as_ref()); + if kind_left == kind_right { + kind_left + } else { + unimplemented!( + "Missing support for union/intersection with differing types in type definitions parser", + ) + } + } + typ if type_info_is_fn(typ) => DocItemKind::Function, + typ => unimplemented!( + "Missing support for TypeInfo in type definitions parser:\n{}", + typ.to_string() + ), + } + } +} + pub fn parse_type_definitions_into_doc_items(contents: S) -> Result> where S: AsRef, @@ -73,108 +104,153 @@ where found_top_level_items.push(DocVisitorItem { name: declaration.type_name().token().to_string(), comment: find_token_moonwave_comment(token_reference), - exported: matches!(stmt, Stmt::ExportedTypeDeclaration(_)), type_info: declaration.type_definition().clone(), }); } } - Ok(found_top_level_items - .drain(..) - .filter(|item| item.exported) // NOTE: Should we include items that are not exported? Probably not .. - .map(DocItem::from) - .collect()) + Ok(found_top_level_items.drain(..).map(DocItem::from).collect()) +} + +fn simple_stringify_type_info(typ: &TypeInfo) -> String { + match typ { + TypeInfo::Array { type_info, .. } => { + format!("{{ {} }}", simple_stringify_type_info(type_info)) + } + TypeInfo::Basic(tok) => { + if tok.token().to_string() == "T" { + "any".to_string() // HACK: Assume that any literal type "T" is a generic that accepts any value + } else { + tok.token().to_string() + } + } + TypeInfo::String(str) => str.token().to_string(), + TypeInfo::Boolean(_) => "boolean".to_string(), + TypeInfo::Callback { .. } => "function".to_string(), + TypeInfo::Optional { base, .. } => format!("{}?", simple_stringify_type_info(base)), + TypeInfo::Table { .. } => "table".to_string(), + TypeInfo::Union { left, right, .. } => { + format!( + "{}{PIPE_SEPARATOR}{}", + simple_stringify_type_info(left), + simple_stringify_type_info(right) + ) + } + // TODO: Stringify custom table types properly, these show up as basic tokens + // and we should be able to look up the real type using found top level items + _ => "...".to_string(), + } } fn type_info_is_fn(typ: &TypeInfo) -> bool { match typ { TypeInfo::Callback { .. } => true, TypeInfo::Tuple { types, .. } => types.iter().all(type_info_is_fn), - TypeInfo::Union { left, right, .. } => type_info_is_fn(left) && type_info_is_fn(right), - TypeInfo::Intersection { left, right, .. } => { - type_info_is_fn(left) && type_info_is_fn(right) + TypeInfo::Union { left, right, .. } | TypeInfo::Intersection { left, right, .. } => { + type_info_is_fn(left) || type_info_is_fn(right) } _ => false, } } -fn should_separate_tag_meta(tag_kind: &str) -> bool { - matches!(tag_kind.trim().to_ascii_lowercase().as_ref(), "param") +fn type_info_extract_args<'a>( + typ: &'a TypeInfo, + base: Vec>, +) -> Vec> { + match typ { + TypeInfo::Callback { arguments, .. } => { + let mut result = base.clone(); + result.push(arguments.iter().collect::>()); + result + } + TypeInfo::Tuple { types, .. } => type_info_extract_args( + types.iter().next().expect("Function tuple type was empty"), + base.clone(), + ), + TypeInfo::Union { left, right, .. } | TypeInfo::Intersection { left, right, .. } => { + let mut result = base.clone(); + result = type_info_extract_args(left, result.clone()); + result = type_info_extract_args(right, result.clone()); + result + } + _ => base, + } } -fn parse_moonwave_style_tag(line: &str) -> Option { - let tag_regex = Regex::new(r#"^@(\S+)\s*(.*)$"#).unwrap(); - if tag_regex.is_match(line) { - let captures = tag_regex.captures(line).unwrap(); - let tag_kind = captures.get(1).unwrap().as_str(); - let tag_rest = captures.get(2).unwrap().as_str(); - let mut tag_words = tag_rest.split_whitespace().collect::>(); - let tag_name = if !tag_words.is_empty() && should_separate_tag_meta(tag_kind) { - tag_words.remove(0).to_string() - } else { - String::new() - }; - let tag_contents = tag_words.join(" "); - if tag_kind.is_empty() { - None - } else { - let mut builder = DocItemBuilder::new() - .with_kind(DocItemKind::Tag) - .with_name(tag_kind); - if !tag_name.is_empty() { - builder = builder.with_meta(tag_name); +fn try_extract_normalized_function_args(typ: &TypeInfo) -> Option> { + if type_info_is_fn(typ) { + let mut type_args_multi = type_info_extract_args(typ, Vec::new()); + match type_args_multi.len() { + 0 => None, + 1 => Some( + // We got a normal function with some known list of args, and we will + // stringify the arg types into simple ones such as "function", "table", .. + type_args_multi + .pop() + .unwrap() + .iter() + .map(|type_arg| simple_stringify_type_info(type_arg.type_info())) + .collect(), + ), + _ => { + // We got a union or intersection function, meaning it has + // several different overloads that accept different args + let mut unified_args = Vec::new(); + for index in 0..type_args_multi + .iter() + .fold(0, |acc, type_args| acc.max(type_args.len())) + { + // Gather function arg type strings for all + // of the different variants of this function + let mut type_arg_strings = type_args_multi + .iter() + .filter_map(|type_args| type_args.get(index)) + .map(|type_arg| simple_stringify_type_info(type_arg.type_info())) + .collect::>(); + if type_arg_strings.len() < type_args_multi.len() { + for _ in type_arg_strings.len()..type_args_multi.len() { + type_arg_strings.push("nil".to_string()); + } + } + // Type arg strings may themselves be stringified to something like number | string so we + // will split that out to be able to handle it better with the following unification process + let mut type_arg_strings_sep = Vec::new(); + for type_arg_string in type_arg_strings.drain(..) { + for typ_arg_string_inner in type_arg_string.split(PIPE_SEPARATOR) { + type_arg_strings_sep.push(typ_arg_string_inner.to_string()); + } + } + // Find out if we have any nillable type, to know if we + // should make the entire arg type union nillable or not + let has_any_optional = type_arg_strings_sep + .iter() + .any(|s| s == "nil" || s.ends_with('?')); + // Filter out any nils or optional markers (?), + // we will add this back at the end if necessary + let mut type_arg_strings_non_nil = type_arg_strings_sep + .iter() + .filter(|s| *s != "nil") + .map(|s| s.trim_end_matches('?').to_string()) + .collect::>(); + type_arg_strings_non_nil.sort(); // Need to sort for dedup + type_arg_strings_non_nil.dedup(); // Dedup to get rid of redundant types such as string | string + unified_args.push(if has_any_optional { + if type_arg_strings_non_nil.len() == 1 { + format!("{}?", type_arg_strings_non_nil.pop().unwrap()) + } else { + format!("({})?", type_arg_strings_non_nil.join(PIPE_SEPARATOR)) + } + } else { + type_arg_strings_non_nil.join(PIPE_SEPARATOR) + }); + } + Some(unified_args) } - if !tag_contents.is_empty() { - builder = builder.with_value(tag_contents); - } - Some(builder.build().unwrap()) } } else { None } } -fn parse_moonwave_style_comment(comment: &str) -> Vec { - let lines = comment.lines().map(str::trim).collect::>(); - let indent_len = lines.iter().fold(usize::MAX, |acc, line| { - let first = line.chars().enumerate().find_map(|(idx, ch)| { - if ch.is_alphanumeric() { - Some(idx) - } else { - None - } - }); - if let Some(first_alphanumeric) = first { - if first_alphanumeric > 0 { - acc.min(first_alphanumeric - 1) - } else { - 0 - } - } else { - acc - } - }); - let unindented_lines = lines.iter().map(|line| &line[indent_len..]); - let mut doc_items = Vec::new(); - let mut doc_lines = Vec::new(); - for line in unindented_lines { - if let Some(tag) = parse_moonwave_style_tag(line) { - doc_items.push(tag); - } else { - doc_lines.push(line); - } - } - if !doc_lines.is_empty() { - doc_items.push( - DocItemBuilder::new() - .with_kind(DocItemKind::Description) - .with_value(doc_lines.join("\n").trim()) - .build() - .unwrap(), - ); - } - doc_items -} - fn find_token_moonwave_comment(token: &TokenReference) -> Option { token .leading_trivia() diff --git a/packages/cli/src/gen/selene_defs.rs b/packages/cli/src/gen/selene_defs.rs index 32d846d..3cfdb58 100644 --- a/packages/cli/src/gen/selene_defs.rs +++ b/packages/cli/src/gen/selene_defs.rs @@ -3,7 +3,7 @@ use serde_yaml::{Mapping as YamlMapping, Sequence as YamlSequence, Value as Yaml use crate::gen::doc2::DocsItemTag; -use super::doc2::{DocItem, DocItemKind, DocTree}; +use super::doc2::{DocItem, DocItemKind, DocTree, PIPE_SEPARATOR}; pub fn generate_from_type_definitions(contents: &str) -> Result { let tree = DocTree::from_type_definitions(contents)?; @@ -110,10 +110,47 @@ fn doc_item_to_selene_yaml_mapping(item: &DocItem) -> Result { YamlValue::Bool(true), ); } + let mut args = YamlSequence::new(); + for arg_type in item.arg_types() { + let mut arg_mapping = YamlMapping::new(); + let (type_str, type_opt) = match arg_type.strip_suffix('?') { + Some(stripped) => (stripped, true), + None => (arg_type, false), + }; + if type_opt { + arg_mapping.insert( + YamlValue::String("required".to_string()), + YamlValue::Bool(false), + ); + } + arg_mapping.insert( + YamlValue::String("type".to_string()), + YamlValue::String(simplify_type_str_into_primitives( + type_str.trim_start_matches('(').trim_end_matches(')'), + )), + ); + args.push(YamlValue::Mapping(arg_mapping)); + } mapping.insert( YamlValue::String("args".to_string()), - YamlValue::Mapping(YamlMapping::new()), + YamlValue::Sequence(args), ); } Ok(mapping) } + +fn simplify_type_str_into_primitives(type_str: &str) -> String { + let mut primitives = Vec::new(); + for type_inner in type_str.split(PIPE_SEPARATOR) { + if type_inner.starts_with('{') && type_inner.ends_with('}') { + primitives.push("table"); + } else if type_inner.starts_with('"') && type_inner.ends_with('"') { + primitives.push("string"); + } else { + primitives.push(type_inner); + } + } + primitives.sort_unstable(); + primitives.dedup(); + primitives.join(PIPE_SEPARATOR) +}