Use new typedefs parser to generate docs file

This commit is contained in:
Filip Tibell 2023-02-22 14:25:50 +01:00
parent d6d0b74ce0
commit 8e894c7ac9
No known key found for this signature in database
10 changed files with 239 additions and 436 deletions

View file

@ -1,5 +1,3 @@
-- Lune v0.4.0
--[=[
@class fs
@ -527,5 +525,8 @@ declare warn: <T...>(T...) -> ()
--[=[
Throws an error and prints a formatted version of it with a leading "[ERROR]" tag.
@param message The error message to throw
@param level The stack level to throw the error at, defaults to 0
]=]
declare error: <T>(message: T, level: number?) -> ()

View file

@ -8,8 +8,8 @@ use tokio::fs::{read_to_string, write};
use crate::{
gen::{
generate_docs_json_from_definitions, generate_selene_defs_from_definitions,
generate_wiki_dir_from_definitions,
generate_docs_json_from_definitions, generate_luau_defs_from_definitions,
generate_selene_defs_from_definitions, generate_wiki_dir_from_definitions,
},
utils::{
files::find_parse_file_path,
@ -120,7 +120,7 @@ impl Cli {
if generate_file_requested {
if self.generate_luau_types {
generate_and_save_file(FILE_NAME_LUAU_TYPES, "Luau type definitions", || {
Ok(FILE_CONTENTS_LUAU_TYPES.to_string())
generate_luau_defs_from_definitions(FILE_CONTENTS_LUAU_TYPES)
})
.await?;
}

View file

@ -1,52 +0,0 @@
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
mod tag;
mod visitor;
pub use tag::*;
pub use visitor::*;
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
pub struct DocsGlobal {
pub documentation: String,
pub keys: HashMap<String, String>,
pub learn_more_link: String,
pub code_sample: String,
}
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
pub struct DocsFunctionParamLink {
pub name: String,
pub documentation: String,
}
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
pub struct DocsFunction {
#[serde(skip)]
pub global_name: String,
pub documentation: String,
pub params: Vec<DocsFunctionParamLink>,
pub returns: Vec<String>,
pub learn_more_link: String,
pub code_sample: String,
}
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
pub struct DocsParam {
#[serde(skip)]
pub global_name: String,
#[serde(skip)]
pub function_name: String,
pub documentation: String,
}
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
pub struct DocsReturn {
#[serde(skip)]
pub global_name: String,
#[serde(skip)]
pub function_name: String,
pub documentation: String,
}

View file

@ -1,66 +0,0 @@
use anyhow::{bail, Result};
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum DocsTagKind {
Class,
Within,
Param,
Return,
Modifier,
}
impl DocsTagKind {
pub fn parse(s: &str) -> Result<Self> {
match s.trim().to_ascii_lowercase().as_ref() {
"class" => Ok(Self::Class),
"within" => Ok(Self::Within),
"param" => Ok(Self::Param),
"return" => Ok(Self::Return),
"must_use" | "read_only" | "new_fields" => Ok(Self::Modifier),
s => bail!("Unknown docs tag: '{}'", s),
}
}
}
#[derive(Clone, Debug)]
pub struct DocsTag {
pub kind: DocsTagKind,
pub name: String,
pub contents: String,
}
#[derive(Clone, Debug)]
pub struct DocsTagList {
tags: Vec<DocsTag>,
}
impl DocsTagList {
pub fn new() -> Self {
Self { tags: vec![] }
}
pub fn push(&mut self, tag: DocsTag) {
self.tags.push(tag);
}
pub fn contains(&mut self, kind: DocsTagKind) -> bool {
self.tags.iter().any(|tag| tag.kind == kind)
}
pub fn find(&mut self, kind: DocsTagKind) -> Option<&DocsTag> {
self.tags.iter().find(|tag| tag.kind == kind)
}
pub fn is_empty(&self) -> bool {
self.tags.is_empty()
}
}
impl IntoIterator for DocsTagList {
type Item = DocsTag;
type IntoIter = std::vec::IntoIter<Self::Item>;
fn into_iter(self) -> Self::IntoIter {
self.tags.into_iter()
}
}

View file

@ -1,216 +0,0 @@
use anyhow::Result;
use full_moon::{
ast::types::{ExportedTypeDeclaration, TypeField, TypeFieldKey},
parse as parse_luau_ast,
tokenizer::{Token, TokenType},
visitors::Visitor,
};
use regex::Regex;
use super::{
{DocsFunction, DocsFunctionParamLink, DocsGlobal, DocsParam, DocsReturn},
{DocsTag, DocsTagKind, DocsTagList},
};
#[derive(Debug, Clone)]
pub struct DocumentationVisitor {
pub globals: Vec<(String, DocsGlobal)>,
pub functions: Vec<(String, DocsFunction)>,
pub params: Vec<(String, DocsParam)>,
pub returns: Vec<(String, DocsReturn)>,
tag_regex: Regex,
}
impl DocumentationVisitor {
pub fn new() -> Self {
let tag_regex = Regex::new(r#"^@(\S+)\s*(.*)$"#).unwrap();
Self {
globals: vec![],
functions: vec![],
params: vec![],
returns: vec![],
tag_regex,
}
}
pub fn from_definitions(definitions_file_contents: &str) -> Result<Self> {
// TODO: Properly handle the "declare class" syntax, for now we just skip it
let mut no_declares = definitions_file_contents.to_string();
while let Some(dec) = no_declares.find("\ndeclare class") {
let end = no_declares.find("\nend").unwrap();
let before = &no_declares[0..dec];
let after = &no_declares[end + 4..];
no_declares = format!("{before}{after}");
}
let (regex, replacement) = (
Regex::new(r#"declare (?P<n>\w+): "#).unwrap(),
r#"export type $n = "#,
);
let defs_ast = parse_luau_ast(&regex.replace_all(&no_declares, replacement))?;
let mut visitor = DocumentationVisitor::new();
visitor.visit_ast(&defs_ast);
Ok(visitor)
}
pub fn parse_moonwave_style_tag(&self, line: &str) -> Option<DocsTag> {
if self.tag_regex.is_match(line) {
let captures = self.tag_regex.captures(line).unwrap();
let tag_kind = captures.get(1).unwrap().as_str();
let tag_rest = captures.get(2).unwrap().as_str();
let mut tag_words = tag_rest.split_whitespace().collect::<Vec<_>>();
let tag_name = if tag_words.is_empty() {
String::new()
} else {
tag_words.remove(0).to_string()
};
let tag_contents = tag_words.join(" ");
Some(DocsTag {
kind: DocsTagKind::parse(tag_kind).unwrap(),
name: tag_name,
contents: tag_contents,
})
} else {
None
}
}
pub fn parse_moonwave_style_comment(&self, comment: &str) -> (String, DocsTagList) {
let lines = comment.lines().map(str::trim).collect::<Vec<_>>();
let indent_len = lines.iter().fold(usize::MAX, |acc, line| {
let first = line.chars().enumerate().find_map(|(idx, ch)| {
if ch.is_alphanumeric() {
Some(idx)
} else {
None
}
});
if let Some(first_alphanumeric) = first {
if first_alphanumeric > 0 {
acc.min(first_alphanumeric - 1)
} else {
0
}
} else {
acc
}
});
let unindented_lines = lines.iter().map(|line| &line[indent_len..]);
let mut doc_lines = Vec::new();
let mut doc_tags = DocsTagList::new();
for line in unindented_lines {
if let Some(tag) = self.parse_moonwave_style_tag(line) {
doc_tags.push(tag);
} else {
doc_lines.push(line);
}
}
(doc_lines.join("\n").trim().to_owned(), doc_tags)
}
fn extract_moonwave_comment(&mut self, token: &Token) -> Option<(String, DocsTagList)> {
if let TokenType::MultiLineComment { comment, .. } = token.token_type() {
let (doc, tags) = self.parse_moonwave_style_comment(comment);
if doc.is_empty() && tags.is_empty() {
None
} else {
Some((doc, tags))
}
} else {
None
}
}
}
impl Visitor for DocumentationVisitor {
fn visit_exported_type_declaration(&mut self, node: &ExportedTypeDeclaration) {
for token in node.export_token().leading_trivia() {
if let Some((doc, mut tags)) = self.extract_moonwave_comment(token) {
if tags.contains(DocsTagKind::Class) {
self.globals.push((
node.type_declaration().type_name().token().to_string(),
DocsGlobal {
documentation: doc,
..Default::default()
},
));
break;
}
}
}
}
fn visit_type_field(&mut self, node: &TypeField) {
// Parse out names, moonwave comments from the ast
let mut parsed_data = Vec::new();
if let TypeFieldKey::Name(name) = node.key() {
for token in name.leading_trivia() {
if let Some((doc, mut tags)) = self.extract_moonwave_comment(token) {
if let Some(within) = tags.find(DocsTagKind::Within).map(ToOwned::to_owned) {
parsed_data.push((within.name, name, doc, tags));
}
}
}
}
for (global_name, name, doc, tags) in parsed_data {
// Find the global definition, which is guaranteed to
// be visited and parsed before its inner members, and
// add a ref to the found function / member to it
let name = name.token().to_string();
for (name, global) in &mut self.globals {
if name == &global_name {
global.keys.insert(name.clone(), name.clone());
}
}
// Look through tags to find and create doc params and returns
let mut param_links = Vec::new();
let mut return_links = Vec::new();
for tag in tags {
match tag.kind {
DocsTagKind::Param => {
let idx_string = param_links.len().to_string();
self.params.push((
idx_string.clone(),
DocsParam {
global_name: global_name.clone(),
function_name: name.clone(),
documentation: tag.contents.trim().to_owned(),
},
));
param_links.push(DocsFunctionParamLink {
name: tag.name.clone(),
documentation: idx_string.clone(),
});
}
DocsTagKind::Return => {
// NOTE: Returns don't have names but we still parse
// them as such, so we should concat name & contents
let doc = format!("{} {}", tag.name.trim(), tag.contents.trim());
let idx_string = return_links.len().to_string();
self.returns.push((
idx_string.clone(),
DocsReturn {
global_name: global_name.clone(),
function_name: name.clone(),
documentation: doc,
},
));
return_links.push(idx_string.clone());
}
_ => {}
}
}
// Finally, add our complete doc
// function with links into the list
self.functions.push((
name,
DocsFunction {
global_name,
documentation: doc,
params: param_links,
returns: return_links,
..Default::default()
},
));
}
}
}

View file

@ -1,65 +1,228 @@
use std::collections::HashMap;
use anyhow::{Context, Result};
use lazy_static::lazy_static;
use serde_json::{Map as JsonMap, Value as JsonValue};
use super::doc::{DocsFunctionParamLink, DocumentationVisitor};
use super::definitions::{DefinitionsItem, DefinitionsItemTag, DefinitionsTree};
lazy_static! {
static ref KEY_DOCUMENTATION: String = "documentation".to_string();
static ref KEY_KEYS: String = "keys".to_string();
static ref KEY_NAME: String = "name".to_string();
static ref KEY_CODE_SAMPLE: String = "code_sample".to_string();
static ref KEY_LEARN_MORE_LINK: String = "learn_more_link".to_string();
static ref VALUE_EMPTY: String = String::new();
}
pub fn generate_from_type_definitions(contents: &str, namespace: &str) -> Result<String> {
let visitor = DocumentationVisitor::from_definitions(contents)?;
let tree = DefinitionsTree::from_type_definitions(contents)?;
/*
Extract globals, functions, params, returns from the visitor
Extract globals, functions, params, returns from the type definitions tree
Here we will also convert the plain names into proper namespaced names according to the spec at
https://raw.githubusercontent.com/MaximumADHD/Roblox-Client-Tracker/roblox/api-docs/en-us.json
*/
let mut map = JsonMap::new();
for (name, mut doc) in visitor.globals {
doc.keys = doc
.keys
.iter()
.map(|(key, value)| (key.clone(), format!("@{namespace}/{name}.{value}")))
.collect::<HashMap<String, String>>();
map.insert(format!("@{namespace}/{name}"), serde_json::to_value(doc)?);
}
for (name, mut doc) in visitor.functions {
doc.params = doc
.params
.iter()
.map(|param| DocsFunctionParamLink {
name: param.name.clone(),
documentation: format!(
"@{namespace}/{}.{name}/param/{}",
doc.global_name, param.documentation
),
})
.collect::<Vec<_>>();
doc.returns = doc
.returns
.iter()
.map(|ret| format!("@{namespace}/{}.{name}/return/{ret}", doc.global_name))
.collect::<Vec<_>>();
map.insert(
format!("@{namespace}/{}.{name}", doc.global_name),
serde_json::to_value(doc)?,
);
}
for (name, doc) in visitor.params {
map.insert(
format!(
"@{namespace}/{}.{}/param/{name}",
doc.global_name, doc.function_name
),
serde_json::to_value(doc)?,
);
}
for (name, doc) in visitor.returns {
map.insert(
format!(
"@{namespace}/{}.{}/return/{name}",
doc.global_name, doc.function_name
),
serde_json::to_value(doc)?,
);
// Go over all the exported classes first (globals)
let exported_items = tree.children().iter().filter(|item| {
item.is_exported()
&& (item.is_function()
|| item.children().iter().any(|item_child| {
item_child.is_tag() && item_child.get_name().unwrap() == "class"
}))
});
for item in exported_items {
parse_and_insert(&mut map, item, namespace, None)?;
}
// Go over the rest, these will be global types
// that exported items are referencing somewhere
serde_json::to_string_pretty(&JsonValue::Object(map)).context("Failed to encode docs as json")
}
#[allow(clippy::too_many_lines)]
fn parse_and_insert(
map: &mut JsonMap<String, JsonValue>,
item: &DefinitionsItem,
namespace: &str,
parent: Option<&DefinitionsItem>,
) -> Result<()> {
let mut item_map = JsonMap::new();
let item_name = item
.get_name()
.with_context(|| format!("Missing name for doc item: {item:#?}"))?;
// Include parent name in full name, unless there is no parent (top-level global)
let item_name_full = match parent {
Some(parent) => format!(
"{}.{item_name}",
parent
.get_name()
.with_context(|| format!("Missing parent name for doc item: {item:#?}"))?
),
None => item_name.to_string(),
};
// Try to parse params & returns to use later
let mut params = Vec::new();
let mut returns = Vec::new();
if item.is_function() {
// Map and separate found tags into params & returns
let mut tags = item
.children()
.iter()
.filter_map(|child| {
if let Ok(tag) = DefinitionsItemTag::try_from(child) {
Some(tag)
} else {
None
}
})
.collect::<Vec<_>>();
for tag in tags.drain(..) {
if tag.is_param() {
params.push(tag);
} else if tag.is_return() {
returns.push(tag);
}
}
}
// Try to parse the description for this typedef item, if it has one,
// insert description + code sample + learn more link if they exist
if let Some(description) = item.children().iter().find(|child| child.is_description()) {
let (description, code_sample, learn_more_link) = try_parse_description_for_docs(
description
.get_value()
.context("Missing description value for doc item")?
.to_string(),
);
item_map.insert(KEY_DOCUMENTATION.clone(), JsonValue::String(description));
if let Some(code_sample) = code_sample {
item_map.insert(KEY_CODE_SAMPLE.clone(), JsonValue::String(code_sample));
} else {
item_map.insert(
KEY_CODE_SAMPLE.clone(),
JsonValue::String(VALUE_EMPTY.clone()),
);
}
if let Some(learn_more_link) = learn_more_link {
item_map.insert(
KEY_LEARN_MORE_LINK.clone(),
JsonValue::String(learn_more_link),
);
} else {
item_map.insert(
KEY_LEARN_MORE_LINK.clone(),
JsonValue::String(VALUE_EMPTY.clone()),
);
}
}
/*
If the typedef item is a table, we should include keys
which are references from this global to its members,
then we should parse its members and add them in
If it is a function, we should parse its params and args,
make links to them in this object, and then add them in as
separate items into the globals map, with their documentation
*/
if item.is_table() {
let mut keys = item
.children()
.iter()
.filter_map(|child| {
if child.is_property() || child.is_table() || child.is_function() {
Some(child.get_name().expect("Missing name for doc item child"))
} else {
None
}
})
.collect::<Vec<_>>();
if keys.is_empty() {
item_map.insert("keys".to_string(), JsonValue::Object(JsonMap::new()));
} else {
let mut keys_map = JsonMap::new();
for key in keys.drain(..) {
keys_map.insert(
key.to_string(),
JsonValue::String(format!("@{namespace}/{item_name_full}.{key}")),
);
}
item_map.insert("keys".to_string(), JsonValue::Object(keys_map));
}
} else if item.is_function() {
// Add links to params
if params.is_empty() {
item_map.insert("params".to_string(), JsonValue::Array(vec![]));
} else {
let mut params_vec = Vec::new();
for (index, param) in params.iter().enumerate() {
let mut param_map = JsonMap::new();
if let DefinitionsItemTag::Param((name, _)) = param {
param_map.insert(KEY_NAME.clone(), JsonValue::String(name.to_string()));
param_map.insert(
KEY_DOCUMENTATION.clone(),
JsonValue::String(format!("@{namespace}/{item_name_full}/param/{index}")),
);
}
params_vec.push(JsonValue::Object(param_map));
}
item_map.insert("params".to_string(), JsonValue::Array(params_vec));
}
// Add links to returns
if returns.is_empty() {
item_map.insert("returns".to_string(), JsonValue::Array(vec![]));
} else {
let mut returns_vec = Vec::new();
for (index, _) in returns.iter().enumerate() {
returns_vec.push(JsonValue::String(format!(
"@{namespace}/{item_name_full}/return/{index}"
)));
}
item_map.insert("returns".to_string(), JsonValue::Array(returns_vec));
}
}
map.insert(
format!("@{namespace}/{item_name_full}"),
JsonValue::Object(item_map),
);
if item.is_table() {
for child in item
.children()
.iter()
.filter(|child| !child.is_description() && !child.is_tag())
{
parse_and_insert(map, child, namespace, Some(item))?;
}
} else if item.is_function() {
// FIXME: It seems the order of params and returns here is not
// deterministic, they can be unordered which leads to confusing docs
for (index, param) in params.iter().enumerate() {
let mut param_map = JsonMap::new();
if let DefinitionsItemTag::Param((_, doc)) = param {
param_map.insert(
KEY_DOCUMENTATION.clone(),
JsonValue::String(format!("{doc}\n\n---\n")),
);
}
map.insert(
format!("@{namespace}/{item_name_full}/param/{index}"),
JsonValue::Object(param_map),
);
}
for (index, ret) in returns.iter().enumerate() {
let mut return_map = JsonMap::new();
if let DefinitionsItemTag::Return(doc) = ret {
return_map.insert(
KEY_DOCUMENTATION.clone(),
JsonValue::String(doc.to_string()),
);
}
map.insert(
format!("@{namespace}/{item_name_full}/return/{index}"),
JsonValue::Object(return_map),
);
}
}
Ok(())
}
fn try_parse_description_for_docs(description: String) -> (String, Option<String>, Option<String>) {
// TODO: Implement this
(description, None, None)
}

View file

@ -0,0 +1,10 @@
use anyhow::Result;
#[allow(clippy::unnecessary_wraps)]
pub fn generate_from_type_definitions(contents: &str) -> Result<String> {
Ok(format!(
"--> Lune v{}\n\n{}",
env!("CARGO_PKG_VERSION"),
contents
))
}

View file

@ -1,14 +1,11 @@
mod doc;
mod docs_file;
mod luau_defs;
mod selene_defs;
mod wiki_dir;
pub mod definitions;
pub use docs_file::generate_from_type_definitions as generate_docs_json_from_definitions;
pub use luau_defs::generate_from_type_definitions as generate_luau_defs_from_definitions;
pub use selene_defs::generate_from_type_definitions as generate_selene_defs_from_definitions;
pub use wiki_dir::generate_from_type_definitions as generate_wiki_dir_from_definitions;
pub use self::doc::DocumentationVisitor;
pub const GENERATED_COMMENT_TAG: &str = "@generated with lune-cli";

View file

@ -4,11 +4,12 @@ use anyhow::{Context, Result};
use tokio::fs::{create_dir_all, write};
use super::doc::DocumentationVisitor;
use super::GENERATED_COMMENT_TAG;
use super::definitions::DefinitionsTree;
pub const GENERATED_COMMENT_TAG: &str = "@generated with lune-cli";
pub async fn generate_from_type_definitions(contents: &str) -> Result<()> {
let visitor = DocumentationVisitor::from_definitions(contents)?;
let tree = DefinitionsTree::from_type_definitions(contents)?;
// Create the wiki dir at the repo root
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../../")
@ -17,40 +18,5 @@ pub async fn generate_from_type_definitions(contents: &str) -> Result<()> {
create_dir_all(&root.join("wiki"))
.await
.context("Failed to create wiki dir")?;
for global in &visitor.globals {
// Create the dir for this global
let global_dir_path = root.join("wiki").join("api-reference").join(&global.0);
create_dir_all(&global_dir_path)
.await
.context("Failed to create doc dir for global")?;
// Create the markdown docs file for this global
let mut contents = String::new();
writeln!(contents, "<!-- {GENERATED_COMMENT_TAG} -->\n")?;
writeln!(contents, "# **{}**\n", global.0)?;
writeln!(contents, "{}\n", global.1.documentation)?;
if !global.1.code_sample.is_empty() {
writeln!(contents, "{}", global.1.code_sample)?;
}
let funcs = visitor
.functions
.iter()
.filter(|f| f.1.global_name == global.0)
.collect::<Vec<_>>();
if !funcs.is_empty() {
writeln!(contents, "## Functions\n")?;
for func in funcs {
writeln!(contents, "### {}\n", func.0)?;
writeln!(contents, "{}\n", func.1.documentation)?;
if !func.1.code_sample.is_empty() {
writeln!(contents, "{}", func.1.code_sample)?;
}
}
}
// Write the file in the dir, with the same
// name as the dir to create an "index" page
write(&global_dir_path.join(format!("{}.md", &global.0)), contents)
.await
.context("Failed to create doc file for global")?;
}
Ok(())
}

View file

@ -1,7 +1,7 @@
use serde_json::Value as JsonValue;
use serde_yaml::Value as YamlValue;
use crate::gen::DocumentationVisitor;
use crate::gen::definitions::DefinitionsTree;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FileType {
@ -16,7 +16,7 @@ impl FileType {
Some(Self::Json)
} else if serde_yaml::from_str::<YamlValue>(contents).is_ok() {
Some(Self::Yaml)
} else if DocumentationVisitor::from_definitions(contents).is_ok() {
} else if DefinitionsTree::from_type_definitions(contents).is_ok() {
Some(Self::Luau)
} else {
None