Implement full selene typedefs generator

packages/cli/src/gen/doc2/builder.rs

@@ -9,6 +9,7 @@ pub struct DocItemBuilder {
     meta: Option<String>,
     value: Option<String>,
     children: Vec<DocItem>,
+    arg_types: Vec<String>,
 }
 
 #[allow(dead_code)]
@@ -49,6 +50,18 @@ impl DocItemBuilder {
         self
     }
 
+    pub fn with_arg_type<S: AsRef<str>>(mut self, arg_type: S) -> Self {
+        self.arg_types.push(arg_type.as_ref().to_string());
+        self
+    }
+
+    pub fn with_arg_types<S: AsRef<str>>(mut self, arg_types: &[S]) -> Self {
+        for arg_type in arg_types {
+            self.arg_types.push(arg_type.as_ref().to_string());
+        }
+        self
+    }
+
     pub fn build(self) -> Result<DocItem> {
         if let Some(kind) = self.kind {
             let mut children = self.children;
@@ -59,6 +72,7 @@ impl DocItemBuilder {
                 meta: self.meta,
                 value: self.value,
                 children,
+                arg_types: self.arg_types,
             })
         } else {
             bail!("Missing doc item kind")

packages/cli/src/gen/doc2/item.rs

@@ -16,6 +16,8 @@ pub struct DocItem {
     pub(super) value: Option<String>,
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub(super) children: Vec<DocItem>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub(super) arg_types: Vec<String>,
 }
 
 impl PartialOrd for DocItem {
@@ -90,4 +92,8 @@ impl DocItem {
     pub fn children(&self) -> &[DocItem] {
         &self.children
     }
+
+    pub fn arg_types(&self) -> Vec<&str> {
+        self.arg_types.iter().map(String::as_str).collect()
+    }
 }

packages/cli/src/gen/doc2/mod.rs

@@ -1,11 +1,13 @@
 mod builder;
 mod item;
 mod kind;
+mod moonwave;
 mod parser;
 mod tag;
 mod tree;
 
 pub use item::DocItem;
 pub use kind::DocItemKind;
+pub use parser::PIPE_SEPARATOR;
 pub use tag::DocsItemTag;
 pub use tree::DocTree;

packages/cli/src/gen/doc2/moonwave.rs

@@ -0,0 +1,81 @@
+use regex::Regex;
+
+use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind};
+
+fn should_separate_tag_meta(tag_kind: &str) -> bool {
+    matches!(tag_kind.trim().to_ascii_lowercase().as_ref(), "param")
+}
+
+fn parse_moonwave_style_tag(line: &str) -> Option<DocItem> {
+    let tag_regex = Regex::new(r#"^@(\S+)\s*(.*)$"#).unwrap();
+    if tag_regex.is_match(line) {
+        let captures = tag_regex.captures(line).unwrap();
+        let tag_kind = captures.get(1).unwrap().as_str();
+        let tag_rest = captures.get(2).unwrap().as_str();
+        let mut tag_words = tag_rest.split_whitespace().collect::<Vec<_>>();
+        let tag_name = if !tag_words.is_empty() && should_separate_tag_meta(tag_kind) {
+            tag_words.remove(0).to_string()
+        } else {
+            String::new()
+        };
+        let tag_contents = tag_words.join(" ");
+        if tag_kind.is_empty() {
+            None
+        } else {
+            let mut builder = DocItemBuilder::new()
+                .with_kind(DocItemKind::Tag)
+                .with_name(tag_kind);
+            if !tag_name.is_empty() {
+                builder = builder.with_meta(tag_name);
+            }
+            if !tag_contents.is_empty() {
+                builder = builder.with_value(tag_contents);
+            }
+            Some(builder.build().unwrap())
+        }
+    } else {
+        None
+    }
+}
+
+pub(super) fn parse_moonwave_style_comment(comment: &str) -> Vec<DocItem> {
+    let lines = comment.lines().map(str::trim).collect::<Vec<_>>();
+    let indent_len = lines.iter().fold(usize::MAX, |acc, line| {
+        let first = line.chars().enumerate().find_map(|(idx, ch)| {
+            if ch.is_alphanumeric() {
+                Some(idx)
+            } else {
+                None
+            }
+        });
+        if let Some(first_alphanumeric) = first {
+            if first_alphanumeric > 0 {
+                acc.min(first_alphanumeric - 1)
+            } else {
+                0
+            }
+        } else {
+            acc
+        }
+    });
+    let unindented_lines = lines.iter().map(|line| &line[indent_len..]);
+    let mut doc_items = Vec::new();
+    let mut doc_lines = Vec::new();
+    for line in unindented_lines {
+        if let Some(tag) = parse_moonwave_style_tag(line) {
+            doc_items.push(tag);
+        } else {
+            doc_lines.push(line);
+        }
+    }
+    if !doc_lines.is_empty() {
+        doc_items.push(
+            DocItemBuilder::new()
+                .with_kind(DocItemKind::Description)
+                .with_value(doc_lines.join("\n").trim())
+                .build()
+                .unwrap(),
+        );
+    }
+    doc_items
+}

packages/cli/src/gen/doc2/parser.rs

@@ -1,53 +1,47 @@
 use anyhow::{Context, Result};
 use full_moon::{
     ast::{
-        types::{TypeFieldKey, TypeInfo},
+        types::{TypeArgument, TypeFieldKey, TypeInfo},
         Stmt,
     },
     tokenizer::{TokenReference, TokenType},
 };
 use regex::Regex;
 
-use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind};
+use super::{
+    builder::DocItemBuilder, item::DocItem, kind::DocItemKind,
+    moonwave::parse_moonwave_style_comment,
+};
+
+pub const PIPE_SEPARATOR: &str = " | ";
 
 #[derive(Debug, Clone)]
 struct DocVisitorItem {
     name: String,
     comment: Option<String>,
-    exported: bool,
     type_info: TypeInfo,
 }
 
 impl From<DocVisitorItem> for DocItem {
     fn from(value: DocVisitorItem) -> Self {
         let mut builder = DocItemBuilder::new()
-            .with_kind(match &value.type_info {
+            .with_kind(DocItemKind::from(&value.type_info))
-                TypeInfo::Array { .. } | TypeInfo::Table { .. } => DocItemKind::Table,
+            .with_name(&value.name);
-                typ if type_info_is_fn(typ) => DocItemKind::Function,
-                _ => unimplemented!("Support for globals that are not properties or functions is not yet implemented")
-            })
-            .with_name(value.name);
         if let Some(comment) = value.comment {
             builder = builder.with_children(&parse_moonwave_style_comment(&comment));
         }
+        if let Some(args) = try_extract_normalized_function_args(&value.type_info) {
+            println!("{} > {args:?}", value.name);
+            builder = builder.with_arg_types(&args);
+        }
         if let TypeInfo::Table { fields, .. } = value.type_info {
             for field in fields.iter() {
                 if let TypeFieldKey::Name(name) = field.key() {
-                    let children = find_token_moonwave_comment(name)
+                    builder = builder.with_child(DocItem::from(DocVisitorItem {
-                        .as_deref()
+                        name: name.token().to_string(),
-                        .map(parse_moonwave_style_comment)
+                        comment: find_token_moonwave_comment(name),
-                        .unwrap_or_default();
+                        type_info: field.value().clone(),
-                    builder = builder.with_child(
+                    }));
-                        DocItemBuilder::new()
-                            .with_kind(match field.value() {
-                                typ if type_info_is_fn(typ) => DocItemKind::Function,
-                                _ => DocItemKind::Property,
-                            })
-                            .with_name(name.token().to_string())
-                            .with_children(&children)
-                            .build()
-                            .unwrap(),
-                    );
                 }
             }
         }
@@ -55,6 +49,43 @@ impl From<DocVisitorItem> for DocItem {
     }
 }
 
+impl From<&TypeInfo> for DocItemKind {
+    fn from(value: &TypeInfo) -> Self {
+        match value {
+            TypeInfo::Array { .. } | TypeInfo::Table { .. } => DocItemKind::Table,
+            TypeInfo::Basic(_) | TypeInfo::String(_) => DocItemKind::Property,
+            TypeInfo::Optional { base, .. } => DocItemKind::from(base.as_ref()),
+            TypeInfo::Tuple { types, .. } => {
+                let mut kinds = types.iter().map(DocItemKind::from).collect::<Vec<_>>();
+                let kinds_all_the_same = kinds.windows(2).all(|w| w[0] == w[1]);
+                if kinds_all_the_same && !kinds.is_empty() {
+                    kinds.pop().unwrap()
+                } else {
+                    unimplemented!(
+                        "Missing support for tuple with differing types in type definitions parser",
+                    )
+                }
+            }
+            TypeInfo::Union { left, right, .. } | TypeInfo::Intersection { left, right, .. } => {
+                let kind_left = DocItemKind::from(left.as_ref());
+                let kind_right = DocItemKind::from(right.as_ref());
+                if kind_left == kind_right {
+                    kind_left
+                } else {
+                    unimplemented!(
+                        "Missing support for union/intersection with differing types in type definitions parser",
+                    )
+                }
+            }
+            typ if type_info_is_fn(typ) => DocItemKind::Function,
+            typ => unimplemented!(
+                "Missing support for TypeInfo in type definitions parser:\n{}",
+                typ.to_string()
+            ),
+        }
+    }
+}
+
 pub fn parse_type_definitions_into_doc_items<S>(contents: S) -> Result<Vec<DocItem>>
 where
     S: AsRef<str>,
@@ -73,106 +104,151 @@ where
             found_top_level_items.push(DocVisitorItem {
                 name: declaration.type_name().token().to_string(),
                 comment: find_token_moonwave_comment(token_reference),
-                exported: matches!(stmt, Stmt::ExportedTypeDeclaration(_)),
                 type_info: declaration.type_definition().clone(),
             });
         }
     }
-    Ok(found_top_level_items
+    Ok(found_top_level_items.drain(..).map(DocItem::from).collect())
-        .drain(..)
+}
-        .filter(|item| item.exported) // NOTE: Should we include items that are not exported? Probably not ..
+
-        .map(DocItem::from)
+fn simple_stringify_type_info(typ: &TypeInfo) -> String {
-        .collect())
+    match typ {
+        TypeInfo::Array { type_info, .. } => {
+            format!("{{ {} }}", simple_stringify_type_info(type_info))
+        }
+        TypeInfo::Basic(tok) => {
+            if tok.token().to_string() == "T" {
+                "any".to_string() // HACK: Assume that any literal type "T" is a generic that accepts any value
+            } else {
+                tok.token().to_string()
+            }
+        }
+        TypeInfo::String(str) => str.token().to_string(),
+        TypeInfo::Boolean(_) => "boolean".to_string(),
+        TypeInfo::Callback { .. } => "function".to_string(),
+        TypeInfo::Optional { base, .. } => format!("{}?", simple_stringify_type_info(base)),
+        TypeInfo::Table { .. } => "table".to_string(),
+        TypeInfo::Union { left, right, .. } => {
+            format!(
+                "{}{PIPE_SEPARATOR}{}",
+                simple_stringify_type_info(left),
+                simple_stringify_type_info(right)
+            )
+        }
+        // TODO: Stringify custom table types properly, these show up as basic tokens
+        // and we should be able to look up the real type using found top level items
+        _ => "...".to_string(),
+    }
 }
 
 fn type_info_is_fn(typ: &TypeInfo) -> bool {
     match typ {
         TypeInfo::Callback { .. } => true,
         TypeInfo::Tuple { types, .. } => types.iter().all(type_info_is_fn),
-        TypeInfo::Union { left, right, .. } => type_info_is_fn(left) && type_info_is_fn(right),
+        TypeInfo::Union { left, right, .. } | TypeInfo::Intersection { left, right, .. } => {
-        TypeInfo::Intersection { left, right, .. } => {
+            type_info_is_fn(left) || type_info_is_fn(right)
-            type_info_is_fn(left) && type_info_is_fn(right)
         }
         _ => false,
     }
 }
 
-fn should_separate_tag_meta(tag_kind: &str) -> bool {
+fn type_info_extract_args<'a>(
-    matches!(tag_kind.trim().to_ascii_lowercase().as_ref(), "param")
+    typ: &'a TypeInfo,
-}
+    base: Vec<Vec<&'a TypeArgument>>,
-
+) -> Vec<Vec<&'a TypeArgument>> {
-fn parse_moonwave_style_tag(line: &str) -> Option<DocItem> {
+    match typ {
-    let tag_regex = Regex::new(r#"^@(\S+)\s*(.*)$"#).unwrap();
+        TypeInfo::Callback { arguments, .. } => {
-    if tag_regex.is_match(line) {
+            let mut result = base.clone();
-        let captures = tag_regex.captures(line).unwrap();
+            result.push(arguments.iter().collect::<Vec<_>>());
-        let tag_kind = captures.get(1).unwrap().as_str();
+            result
-        let tag_rest = captures.get(2).unwrap().as_str();
-        let mut tag_words = tag_rest.split_whitespace().collect::<Vec<_>>();
-        let tag_name = if !tag_words.is_empty() && should_separate_tag_meta(tag_kind) {
-            tag_words.remove(0).to_string()
-        } else {
-            String::new()
-        };
-        let tag_contents = tag_words.join(" ");
-        if tag_kind.is_empty() {
-            None
-        } else {
-            let mut builder = DocItemBuilder::new()
-                .with_kind(DocItemKind::Tag)
-                .with_name(tag_kind);
-            if !tag_name.is_empty() {
-                builder = builder.with_meta(tag_name);
         }
-            if !tag_contents.is_empty() {
+        TypeInfo::Tuple { types, .. } => type_info_extract_args(
-                builder = builder.with_value(tag_contents);
+            types.iter().next().expect("Function tuple type was empty"),
+            base.clone(),
+        ),
+        TypeInfo::Union { left, right, .. } | TypeInfo::Intersection { left, right, .. } => {
+            let mut result = base.clone();
+            result = type_info_extract_args(left, result.clone());
+            result = type_info_extract_args(right, result.clone());
+            result
         }
-            Some(builder.build().unwrap())
+        _ => base,
-        }
-    } else {
-        None
     }
 }
 
-fn parse_moonwave_style_comment(comment: &str) -> Vec<DocItem> {
+fn try_extract_normalized_function_args(typ: &TypeInfo) -> Option<Vec<String>> {
-    let lines = comment.lines().map(str::trim).collect::<Vec<_>>();
+    if type_info_is_fn(typ) {
-    let indent_len = lines.iter().fold(usize::MAX, |acc, line| {
+        let mut type_args_multi = type_info_extract_args(typ, Vec::new());
-        let first = line.chars().enumerate().find_map(|(idx, ch)| {
+        match type_args_multi.len() {
-            if ch.is_alphanumeric() {
+            0 => None,
-                Some(idx)
+            1 => Some(
+                // We got a normal function with some known list of args, and we will
+                // stringify the arg types into simple ones such as "function", "table", ..
+                type_args_multi
+                    .pop()
+                    .unwrap()
+                    .iter()
+                    .map(|type_arg| simple_stringify_type_info(type_arg.type_info()))
+                    .collect(),
+            ),
+            _ => {
+                // We got a union or intersection function, meaning it has
+                // several different overloads that accept different args
+                let mut unified_args = Vec::new();
+                for index in 0..type_args_multi
+                    .iter()
+                    .fold(0, |acc, type_args| acc.max(type_args.len()))
+                {
+                    // Gather function arg type strings for all
+                    // of the different variants of this function
+                    let mut type_arg_strings = type_args_multi
+                        .iter()
+                        .filter_map(|type_args| type_args.get(index))
+                        .map(|type_arg| simple_stringify_type_info(type_arg.type_info()))
+                        .collect::<Vec<_>>();
+                    if type_arg_strings.len() < type_args_multi.len() {
+                        for _ in type_arg_strings.len()..type_args_multi.len() {
+                            type_arg_strings.push("nil".to_string());
+                        }
+                    }
+                    // Type arg strings may themselves be stringified to something like number | string so we
+                    // will split that out to be able to handle it better with the following unification process
+                    let mut type_arg_strings_sep = Vec::new();
+                    for type_arg_string in type_arg_strings.drain(..) {
+                        for typ_arg_string_inner in type_arg_string.split(PIPE_SEPARATOR) {
+                            type_arg_strings_sep.push(typ_arg_string_inner.to_string());
+                        }
+                    }
+                    // Find out if we have any nillable type, to know if we
+                    // should make the entire arg type union nillable or not
+                    let has_any_optional = type_arg_strings_sep
+                        .iter()
+                        .any(|s| s == "nil" || s.ends_with('?'));
+                    // Filter out any nils or optional markers (?),
+                    // we will add this back at the end if necessary
+                    let mut type_arg_strings_non_nil = type_arg_strings_sep
+                        .iter()
+                        .filter(|s| *s != "nil")
+                        .map(|s| s.trim_end_matches('?').to_string())
+                        .collect::<Vec<_>>();
+                    type_arg_strings_non_nil.sort(); // Need to sort for dedup
+                    type_arg_strings_non_nil.dedup(); // Dedup to get rid of redundant types such as string | string
+                    unified_args.push(if has_any_optional {
+                        if type_arg_strings_non_nil.len() == 1 {
+                            format!("{}?", type_arg_strings_non_nil.pop().unwrap())
+                        } else {
+                            format!("({})?", type_arg_strings_non_nil.join(PIPE_SEPARATOR))
+                        }
+                    } else {
+                        type_arg_strings_non_nil.join(PIPE_SEPARATOR)
+                    });
+                }
+                Some(unified_args)
+            }
+        }
     } else {
         None
     }
-        });
-        if let Some(first_alphanumeric) = first {
-            if first_alphanumeric > 0 {
-                acc.min(first_alphanumeric - 1)
-            } else {
-                0
-            }
-        } else {
-            acc
-        }
-    });
-    let unindented_lines = lines.iter().map(|line| &line[indent_len..]);
-    let mut doc_items = Vec::new();
-    let mut doc_lines = Vec::new();
-    for line in unindented_lines {
-        if let Some(tag) = parse_moonwave_style_tag(line) {
-            doc_items.push(tag);
-        } else {
-            doc_lines.push(line);
-        }
-    }
-    if !doc_lines.is_empty() {
-        doc_items.push(
-            DocItemBuilder::new()
-                .with_kind(DocItemKind::Description)
-                .with_value(doc_lines.join("\n").trim())
-                .build()
-                .unwrap(),
-        );
-    }
-    doc_items
 }
 
 fn find_token_moonwave_comment(token: &TokenReference) -> Option<String> {

packages/cli/src/gen/selene_defs.rs

@@ -3,7 +3,7 @@ use serde_yaml::{Mapping as YamlMapping, Sequence as YamlSequence, Value as Yaml
 
 use crate::gen::doc2::DocsItemTag;
 
-use super::doc2::{DocItem, DocItemKind, DocTree};
+use super::doc2::{DocItem, DocItemKind, DocTree, PIPE_SEPARATOR};
 
 pub fn generate_from_type_definitions(contents: &str) -> Result<String> {
     let tree = DocTree::from_type_definitions(contents)?;
@@ -110,10 +110,47 @@ fn doc_item_to_selene_yaml_mapping(item: &DocItem) -> Result<YamlMapping> {
                 YamlValue::Bool(true),
             );
         }
+        let mut args = YamlSequence::new();
+        for arg_type in item.arg_types() {
+            let mut arg_mapping = YamlMapping::new();
+            let (type_str, type_opt) = match arg_type.strip_suffix('?') {
+                Some(stripped) => (stripped, true),
+                None => (arg_type, false),
+            };
+            if type_opt {
+                arg_mapping.insert(
+                    YamlValue::String("required".to_string()),
+                    YamlValue::Bool(false),
+                );
+            }
+            arg_mapping.insert(
+                YamlValue::String("type".to_string()),
+                YamlValue::String(simplify_type_str_into_primitives(
+                    type_str.trim_start_matches('(').trim_end_matches(')'),
+                )),
+            );
+            args.push(YamlValue::Mapping(arg_mapping));
+        }
         mapping.insert(
             YamlValue::String("args".to_string()),
-            YamlValue::Mapping(YamlMapping::new()),
+            YamlValue::Sequence(args),
         );
     }
     Ok(mapping)
 }
+
+fn simplify_type_str_into_primitives(type_str: &str) -> String {
+    let mut primitives = Vec::new();
+    for type_inner in type_str.split(PIPE_SEPARATOR) {
+        if type_inner.starts_with('{') && type_inner.ends_with('}') {
+            primitives.push("table");
+        } else if type_inner.starts_with('"') && type_inner.ends_with('"') {
+            primitives.push("string");
+        } else {
+            primitives.push(type_inner);
+        }
+    }
+    primitives.sort_unstable();
+    primitives.dedup();
+    primitives.join(PIPE_SEPARATOR)
+}