Implement most of the new docs parser

packages/cli/src/gen/doc2/builder.rs

@@ -0,0 +1,64 @@
+use anyhow::{bail, Result};
+
+use super::{item::DocItem, kind::DocItemKind};
+
+#[derive(Debug, Default, Clone)]
+pub struct DocItemBuilder {
+    kind: Option<DocItemKind>,
+    name: Option<String>,
+    value: Option<String>,
+    children: Vec<DocItem>,
+}
+
+#[allow(dead_code)]
+impl DocItemBuilder {
+    pub fn new() -> Self {
+        Self {
+            ..Default::default()
+        }
+    }
+
+    pub fn with_kind(mut self, kind: DocItemKind) -> Self {
+        self.kind = Some(kind);
+        self
+    }
+
+    pub fn with_name<S: AsRef<str>>(mut self, name: S) -> Self {
+        self.name = Some(name.as_ref().to_string());
+        self
+    }
+
+    pub fn with_value<S: AsRef<str>>(mut self, value: S) -> Self {
+        self.value = Some(value.as_ref().to_string());
+        self
+    }
+
+    pub fn with_child(mut self, child: DocItem) -> Self {
+        self.children.push(child);
+        self
+    }
+
+    pub fn with_children(mut self, children: &[DocItem]) -> Self {
+        self.children.extend_from_slice(children);
+        self
+    }
+
+    pub fn build(self) -> Result<DocItem> {
+        if let Some(kind) = self.kind {
+            if let Some(name) = self.name {
+                let mut children = self.children;
+                children.sort();
+                Ok(DocItem {
+                    kind,
+                    name,
+                    value: self.value,
+                    children,
+                })
+            } else {
+                bail!("Missing doc item name")
+            }
+        } else {
+            bail!("Missing doc item kind")
+        }
+    }
+}

packages/cli/src/gen/doc2/item.rs

@@ -0,0 +1,84 @@
+use std::cmp::Ordering;
+
+use serde::{Deserialize, Serialize};
+
+use super::kind::DocItemKind;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DocItem {
+    pub(super) kind: DocItemKind,
+    pub(super) name: String,
+    pub(super) value: Option<String>,
+    pub(super) children: Vec<DocItem>,
+}
+
+impl PartialOrd for DocItem {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        match self.kind.partial_cmp(&other.kind).unwrap() {
+            Ordering::Equal => {}
+            ord => return Some(ord),
+        }
+        match self.name.partial_cmp(&other.name).unwrap() {
+            Ordering::Equal => {}
+            ord => return Some(ord),
+        }
+        match (&self.value, &other.value) {
+            (Some(value_self), Some(value_other)) => {
+                match value_self.partial_cmp(value_other).unwrap() {
+                    Ordering::Equal => {}
+                    ord => return Some(ord),
+                }
+            }
+            (Some(_), None) => return Some(Ordering::Less),
+            (None, Some(_)) => return Some(Ordering::Greater),
+            (None, None) => {}
+        }
+        Some(Ordering::Equal)
+    }
+}
+
+impl Ord for DocItem {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.partial_cmp(other).unwrap()
+    }
+}
+
+#[allow(dead_code)]
+impl DocItem {
+    pub fn is_root(&self) -> bool {
+        self.kind.is_root()
+    }
+
+    pub fn is_property(&self) -> bool {
+        self.kind.is_property()
+    }
+
+    pub fn is_function(&self) -> bool {
+        self.kind.is_function()
+    }
+
+    pub fn is_description(&self) -> bool {
+        self.kind.is_description()
+    }
+
+    pub fn is_tag(&self) -> bool {
+        self.kind.is_tag()
+    }
+
+    pub fn get_name(&self) -> &str {
+        &self.name
+    }
+
+    pub fn get_value(&self) -> Option<&str> {
+        self.value.as_deref()
+    }
+}
+
+impl IntoIterator for DocItem {
+    type Item = DocItem;
+    type IntoIter = std::vec::IntoIter<Self::Item>;
+    fn into_iter(self) -> Self::IntoIter {
+        self.children.into_iter()
+    }
+}

packages/cli/src/gen/doc2/kind.rs

@@ -0,0 +1,58 @@
+use std::fmt;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub enum DocItemKind {
+    Root,
+    Global,
+    Property,
+    Function,
+    Description,
+    Tag,
+}
+
+#[allow(dead_code)]
+impl DocItemKind {
+    pub fn is_root(self) -> bool {
+        self == DocItemKind::Root
+    }
+
+    pub fn is_global(self) -> bool {
+        self == DocItemKind::Global
+    }
+
+    pub fn is_property(self) -> bool {
+        self == DocItemKind::Property
+    }
+
+    pub fn is_function(self) -> bool {
+        self == DocItemKind::Function
+    }
+
+    pub fn is_description(self) -> bool {
+        self == DocItemKind::Description
+    }
+
+    pub fn is_tag(self) -> bool {
+        self == DocItemKind::Tag
+    }
+}
+
+impl fmt::Display for DocItemKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "{}",
+            match self {
+                Self::Root => "Root",
+                Self::Global => "Global",
+                Self::Property => "Property",
+                Self::Function => "Function",
+                Self::Description => "Description",
+                Self::Tag => "Tag",
+            }
+        )
+    }
+}

packages/cli/src/gen/doc2/mod.rs

@@ -0,0 +1,9 @@
+mod builder;
+mod item;
+mod kind;
+mod tree;
+mod visitor;
+
+pub use item::DocItem;
+pub use kind::DocItemKind;
+pub use tree::DocTree;

packages/cli/src/gen/doc2/tree.rs

@@ -0,0 +1,46 @@
+use std::ops::{Deref, DerefMut};
+
+use anyhow::{Context, Result};
+
+use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind, visitor::DocVisitor};
+
+pub struct DocTree(DocItem);
+
+impl DocTree {
+    pub fn from_type_definitions<S: AsRef<str>>(type_definitions_contents: S) -> Result<Self> {
+        let top_level_items = DocVisitor::visit_type_definitions_str(type_definitions_contents)
+            .context("Failed to visit type definitions AST")?;
+        let root = DocItemBuilder::new()
+            .with_kind(DocItemKind::Root)
+            .with_name("<<<ROOT>>>")
+            .with_children(&top_level_items)
+            .build()?;
+        Ok(Self(root))
+    }
+
+    #[allow(dead_code, clippy::unused_self)]
+    pub fn is_root(&self) -> bool {
+        true
+    }
+}
+
+impl Deref for DocTree {
+    type Target = DocItem;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl DerefMut for DocTree {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl IntoIterator for DocTree {
+    type Item = DocItem;
+    type IntoIter = std::vec::IntoIter<Self::Item>;
+    fn into_iter(self) -> Self::IntoIter {
+        self.0.into_iter()
+    }
+}

packages/cli/src/gen/doc2/visitor.rs

@@ -0,0 +1,103 @@
+use anyhow::{Context, Result};
+use full_moon::{
+    ast::{types::TypeInfo, Ast, Stmt},
+    tokenizer::TokenKind,
+    visitors::Visitor,
+};
+use regex::Regex;
+
+use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind};
+
+struct DocVisitorItem {
+    name: String,
+    comment: Option<String>,
+    exported: bool,
+    ast: TypeInfo,
+}
+
+impl From<DocVisitorItem> for DocItem {
+    fn from(value: DocVisitorItem) -> Self {
+        let mut builder = DocItemBuilder::new()
+            .with_kind(DocItemKind::Global)
+            .with_name(value.name);
+        if let Some(comment) = value.comment {
+            builder = builder.with_child(
+                DocItemBuilder::new()
+                    .with_kind(DocItemKind::Description)
+                    .with_name("Description")
+                    .with_value(comment)
+                    .build()
+                    .unwrap(),
+            );
+        }
+        builder.build().unwrap()
+    }
+}
+
+pub struct DocVisitor {
+    pending_visitor_items: Vec<DocVisitorItem>,
+}
+
+impl DocVisitor {
+    pub fn visit_type_definitions_str<S>(contents: S) -> Result<Vec<DocItem>>
+    where
+        S: AsRef<str>,
+    {
+        let mut this = Self {
+            pending_visitor_items: Vec::new(),
+        };
+        this.visit_ast(
+            &full_moon::parse(&cleanup_type_definitions(contents.as_ref()))
+                .context("Failed to parse type definitions")?,
+        );
+        Ok(this
+            .pending_visitor_items
+            .drain(..)
+            .filter(|item| item.exported) // NOTE: Should we include items that are not exported? Probably not ..
+            .map(DocItem::from)
+            .collect())
+    }
+}
+
+impl Visitor for DocVisitor {
+    fn visit_ast(&mut self, ast: &Ast)
+    where
+        Self: Sized,
+    {
+        for stmt in ast.nodes().stmts() {
+            if let Some((declaration, leading_trivia)) = match stmt {
+                Stmt::ExportedTypeDeclaration(exp) => {
+                    Some((exp.type_declaration(), exp.export_token().leading_trivia()))
+                }
+                Stmt::TypeDeclaration(typ) => Some((typ, typ.type_token().leading_trivia())),
+                _ => None,
+            } {
+                self.pending_visitor_items.push(DocVisitorItem {
+                    name: declaration.type_name().to_string(),
+                    comment: leading_trivia
+                        .filter(|trivia| matches!(trivia.token_kind(), TokenKind::MultiLineComment))
+                        .last()
+                        .map(ToString::to_string),
+                    exported: matches!(stmt, Stmt::ExportedTypeDeclaration(_)),
+                    ast: declaration.type_definition().clone(),
+                });
+            }
+        }
+    }
+}
+
+fn cleanup_type_definitions(contents: &str) -> String {
+    // TODO: Properly handle the "declare class" syntax, for now we just skip it
+    let mut no_declares = contents.to_string();
+    while let Some(dec) = no_declares.find("\ndeclare class") {
+        let end = no_declares.find("\nend").unwrap();
+        let before = &no_declares[0..dec];
+        let after = &no_declares[end + 4..];
+        no_declares = format!("{before}{after}");
+    }
+    let (regex, replacement) = (
+        Regex::new(r#"declare (?P<n>\w+): "#).unwrap(),
+        r#"export type $n = "#,
+    );
+    regex.replace_all(&no_declares, replacement).to_string()
+}

packages/cli/src/gen/mod.rs

@@ -1,4 +1,5 @@
 mod doc;
+mod doc2;
 mod docs_file;
 mod selene_defs;
 mod wiki_dir;