lune-org/lune
1
1
Fork 0
mirror of https://github.com/lune-org/lune.git synced 2024-12-12 13:00:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement full selene typedefs generator

Browse source
This commit is contained in:
Filip Tibell 2023-02-22 10:11:36 +01:00
parent f4c55ff2cf
commit 67f9d4eb9f
No known key found for this signature in database
6 changed files with 320 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
packages/cli/src/gen/doc2/builder.rs
View file

@ -9,6 +9,7 @@ pub struct DocItemBuilder {
meta: Option<String>,
value: Option<String>,
children: Vec<DocItem>,
arg_types: Vec<String>,
}
#[allow(dead_code)]
@ -49,6 +50,18 @@ impl DocItemBuilder {
self
}
pub fn with_arg_type<S: AsRef<str>>(mut self, arg_type: S) -> Self {
self.arg_types.push(arg_type.as_ref().to_string());
self
}
pub fn with_arg_types<S: AsRef<str>>(mut self, arg_types: &[S]) -> Self {
for arg_type in arg_types {
self.arg_types.push(arg_type.as_ref().to_string());
}
self
}
pub fn build(self) -> Result<DocItem> {
if let Some(kind) = self.kind {
let mut children = self.children;
@ -59,6 +72,7 @@ impl DocItemBuilder {
meta: self.meta,
value: self.value,
children,
arg_types: self.arg_types,
})
} else {
bail!("Missing doc item kind")

6
packages/cli/src/gen/doc2/item.rs
View file

@ -16,6 +16,8 @@ pub struct DocItem {
pub(super) value: Option<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub(super) children: Vec<DocItem>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub(super) arg_types: Vec<String>,
}
impl PartialOrd for DocItem {
@ -90,4 +92,8 @@ impl DocItem {
pub fn children(&self) -> &[DocItem] {
&self.children
}
pub fn arg_types(&self) -> Vec<&str> {
self.arg_types.iter().map(String::as_str).collect()
}
}

2
packages/cli/src/gen/doc2/mod.rs
View file

@ -1,11 +1,13 @@
mod builder;
mod item;
mod kind;
mod moonwave;
mod parser;
mod tag;
mod tree;
pub use item::DocItem;
pub use kind::DocItemKind;
pub use parser::PIPE_SEPARATOR;
pub use tag::DocsItemTag;
pub use tree::DocTree;

81
packages/cli/src/gen/doc2/moonwave.rs Normal file
View file

@ -0,0 +1,81 @@
use regex::Regex;
use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind};
fn should_separate_tag_meta(tag_kind: &str) -> bool {
matches!(tag_kind.trim().to_ascii_lowercase().as_ref(), "param")
}
fn parse_moonwave_style_tag(line: &str) -> Option<DocItem> {
let tag_regex = Regex::new(r#"^@(\S+)\s*(.*)$"#).unwrap();
if tag_regex.is_match(line) {
let captures = tag_regex.captures(line).unwrap();
let tag_kind = captures.get(1).unwrap().as_str();
let tag_rest = captures.get(2).unwrap().as_str();
let mut tag_words = tag_rest.split_whitespace().collect::<Vec<_>>();
let tag_name = if !tag_words.is_empty() && should_separate_tag_meta(tag_kind) {
tag_words.remove(0).to_string()
} else {
String::new()
};
let tag_contents = tag_words.join(" ");
if tag_kind.is_empty() {
None
} else {
let mut builder = DocItemBuilder::new()
.with_kind(DocItemKind::Tag)
.with_name(tag_kind);
if !tag_name.is_empty() {
builder = builder.with_meta(tag_name);
}
if !tag_contents.is_empty() {
builder = builder.with_value(tag_contents);
}
Some(builder.build().unwrap())
}
} else {
None
}
}
pub(super) fn parse_moonwave_style_comment(comment: &str) -> Vec<DocItem> {
let lines = comment.lines().map(str::trim).collect::<Vec<_>>();
let indent_len = lines.iter().fold(usize::MAX, |acc, line| {
let first = line.chars().enumerate().find_map(|(idx, ch)| {
if ch.is_alphanumeric() {
Some(idx)
} else {
None
}
});
if let Some(first_alphanumeric) = first {
if first_alphanumeric > 0 {
acc.min(first_alphanumeric - 1)
} else {
0
}
} else {
acc
}
});
let unindented_lines = lines.iter().map(|line| &line[indent_len..]);
let mut doc_items = Vec::new();
let mut doc_lines = Vec::new();
for line in unindented_lines {
if let Some(tag) = parse_moonwave_style_tag(line) {
doc_items.push(tag);
} else {
doc_lines.push(line);
}
}
if !doc_lines.is_empty() {
doc_items.push(
DocItemBuilder::new()
.with_kind(DocItemKind::Description)
.with_value(doc_lines.join("\n").trim())
.build()
.unwrap(),
);
}
doc_items
}

278
packages/cli/src/gen/doc2/parser.rs
View file

@ -1,53 +1,47 @@
use anyhow::{Context, Result};
use full_moon::{
ast::{
types::{TypeFieldKey, TypeInfo},
types::{TypeArgument, TypeFieldKey, TypeInfo},
Stmt,
},
tokenizer::{TokenReference, TokenType},
};
use regex::Regex;
use super::{builder::DocItemBuilder, item::DocItem, kind::DocItemKind};
use super::{
builder::DocItemBuilder, item::DocItem, kind::DocItemKind,
moonwave::parse_moonwave_style_comment,
};
pub const PIPE_SEPARATOR: &str = " | ";
#[derive(Debug, Clone)]
struct DocVisitorItem {
name: String,
comment: Option<String>,
exported: bool,
type_info: TypeInfo,
}
impl From<DocVisitorItem> for DocItem {
fn from(value: DocVisitorItem) -> Self {
let mut builder = DocItemBuilder::new()
.with_kind(match &value.type_info {
TypeInfo::Array { .. } | TypeInfo::Table { .. } => DocItemKind::Table,
typ if type_info_is_fn(typ) => DocItemKind::Function,
_ => unimplemented!("Support for globals that are not properties or functions is not yet implemented")
})
.with_name(value.name);
.with_kind(DocItemKind::from(&value.type_info))
.with_name(&value.name);
if let Some(comment) = value.comment {
builder = builder.with_children(&parse_moonwave_style_comment(&comment));
}
if let Some(args) = try_extract_normalized_function_args(&value.type_info) {
println!("{} > {args:?}", value.name);
builder = builder.with_arg_types(&args);
}
if let TypeInfo::Table { fields, .. } = value.type_info {
for field in fields.iter() {
if let TypeFieldKey::Name(name) = field.key() {
let children = find_token_moonwave_comment(name)
.as_deref()
.map(parse_moonwave_style_comment)
.unwrap_or_default();
builder = builder.with_child(
DocItemBuilder::new()
.with_kind(match field.value() {
typ if type_info_is_fn(typ) => DocItemKind::Function,
_ => DocItemKind::Property,
})
.with_name(name.token().to_string())
.with_children(&children)
.build()
.unwrap(),
);
builder = builder.with_child(DocItem::from(DocVisitorItem {
name: name.token().to_string(),
comment: find_token_moonwave_comment(name),
type_info: field.value().clone(),
}));
}
}
}
@ -55,6 +49,43 @@ impl From<DocVisitorItem> for DocItem {
}
}
impl From<&TypeInfo> for DocItemKind {
fn from(value: &TypeInfo) -> Self {
match value {
TypeInfo::Array { .. } | TypeInfo::Table { .. } => DocItemKind::Table,
TypeInfo::Basic(_) | TypeInfo::String(_) => DocItemKind::Property,
TypeInfo::Optional { base, .. } => DocItemKind::from(base.as_ref()),
TypeInfo::Tuple { types, .. } => {
let mut kinds = types.iter().map(DocItemKind::from).collect::<Vec<_>>();
let kinds_all_the_same = kinds.windows(2).all(|w| w[0] == w[1]);
if kinds_all_the_same && !kinds.is_empty() {
kinds.pop().unwrap()
} else {
unimplemented!(
"Missing support for tuple with differing types in type definitions parser",
)
}
}
TypeInfo::Union { left, right, .. } | TypeInfo::Intersection { left, right, .. } => {
let kind_left = DocItemKind::from(left.as_ref());
let kind_right = DocItemKind::from(right.as_ref());
if kind_left == kind_right {
kind_left
} else {
unimplemented!(
"Missing support for union/intersection with differing types in type definitions parser",
)
}
}
typ if type_info_is_fn(typ) => DocItemKind::Function,
typ => unimplemented!(
"Missing support for TypeInfo in type definitions parser:\n{}",
typ.to_string()
),
}
}
}
pub fn parse_type_definitions_into_doc_items<S>(contents: S) -> Result<Vec<DocItem>>
where
S: AsRef<str>,
@ -73,106 +104,151 @@ where
found_top_level_items.push(DocVisitorItem {
name: declaration.type_name().token().to_string(),
comment: find_token_moonwave_comment(token_reference),
exported: matches!(stmt, Stmt::ExportedTypeDeclaration(_)),
type_info: declaration.type_definition().clone(),
});
}
}
Ok(found_top_level_items
.drain(..)
.filter(|item| item.exported) // NOTE: Should we include items that are not exported? Probably not ..
.map(DocItem::from)
.collect())
Ok(found_top_level_items.drain(..).map(DocItem::from).collect())
}
fn simple_stringify_type_info(typ: &TypeInfo) -> String {
match typ {
TypeInfo::Array { type_info, .. } => {
format!("{{ {} }}", simple_stringify_type_info(type_info))
}
TypeInfo::Basic(tok) => {
if tok.token().to_string() == "T" {
"any".to_string() // HACK: Assume that any literal type "T" is a generic that accepts any value
} else {
tok.token().to_string()
}
}
TypeInfo::String(str) => str.token().to_string(),
TypeInfo::Boolean(_) => "boolean".to_string(),
TypeInfo::Callback { .. } => "function".to_string(),
TypeInfo::Optional { base, .. } => format!("{}?", simple_stringify_type_info(base)),
TypeInfo::Table { .. } => "table".to_string(),
TypeInfo::Union { left, right, .. } => {
format!(
"{}{PIPE_SEPARATOR}{}",
simple_stringify_type_info(left),
simple_stringify_type_info(right)
)
}
// TODO: Stringify custom table types properly, these show up as basic tokens
// and we should be able to look up the real type using found top level items
_ => "...".to_string(),
}
}
fn type_info_is_fn(typ: &TypeInfo) -> bool {
match typ {
TypeInfo::Callback { .. } => true,
TypeInfo::Tuple { types, .. } => types.iter().all(type_info_is_fn),
TypeInfo::Union { left, right, .. } => type_info_is_fn(left) && type_info_is_fn(right),
TypeInfo::Intersection { left, right, .. } => {
type_info_is_fn(left) && type_info_is_fn(right)
TypeInfo::Union { left, right, .. } | TypeInfo::Intersection { left, right, .. } => {
type_info_is_fn(left) || type_info_is_fn(right)
}
_ => false,
}
}
fn should_separate_tag_meta(tag_kind: &str) -> bool {
matches!(tag_kind.trim().to_ascii_lowercase().as_ref(), "param")
fn type_info_extract_args<'a>(
typ: &'a TypeInfo,
base: Vec<Vec<&'a TypeArgument>>,
) -> Vec<Vec<&'a TypeArgument>> {
match typ {
TypeInfo::Callback { arguments, .. } => {
let mut result = base.clone();
result.push(arguments.iter().collect::<Vec<_>>());
result
}
fn parse_moonwave_style_tag(line: &str) -> Option<DocItem> {
let tag_regex = Regex::new(r#"^@(\S+)\s*(.*)$"#).unwrap();
if tag_regex.is_match(line) {
let captures = tag_regex.captures(line).unwrap();
let tag_kind = captures.get(1).unwrap().as_str();
let tag_rest = captures.get(2).unwrap().as_str();
let mut tag_words = tag_rest.split_whitespace().collect::<Vec<_>>();
let tag_name = if !tag_words.is_empty() && should_separate_tag_meta(tag_kind) {
tag_words.remove(0).to_string()
} else {
String::new()
};
let tag_contents = tag_words.join(" ");
if tag_kind.is_empty() {
None
} else {
let mut builder = DocItemBuilder::new()
.with_kind(DocItemKind::Tag)
.with_name(tag_kind);
if !tag_name.is_empty() {
builder = builder.with_meta(tag_name);
TypeInfo::Tuple { types, .. } => type_info_extract_args(
types.iter().next().expect("Function tuple type was empty"),
base.clone(),
),
TypeInfo::Union { left, right, .. } | TypeInfo::Intersection { left, right, .. } => {
let mut result = base.clone();
result = type_info_extract_args(left, result.clone());
result = type_info_extract_args(right, result.clone());
result
}
if !tag_contents.is_empty() {
builder = builder.with_value(tag_contents);
}
Some(builder.build().unwrap())
}
} else {
None
_ => base,
}
}
fn parse_moonwave_style_comment(comment: &str) -> Vec<DocItem> {
let lines = comment.lines().map(str::trim).collect::<Vec<_>>();
let indent_len = lines.iter().fold(usize::MAX, |acc, line| {
let first = line.chars().enumerate().find_map(|(idx, ch)| {
if ch.is_alphanumeric() {
Some(idx)
fn try_extract_normalized_function_args(typ: &TypeInfo) -> Option<Vec<String>> {
if type_info_is_fn(typ) {
let mut type_args_multi = type_info_extract_args(typ, Vec::new());
match type_args_multi.len() {
0 => None,
1 => Some(
// We got a normal function with some known list of args, and we will
// stringify the arg types into simple ones such as "function", "table", ..
type_args_multi
.pop()
.unwrap()
.iter()
.map(|type_arg| simple_stringify_type_info(type_arg.type_info()))
.collect(),
),
_ => {
// We got a union or intersection function, meaning it has
// several different overloads that accept different args
let mut unified_args = Vec::new();
for index in 0..type_args_multi
.iter()
.fold(0, |acc, type_args| acc.max(type_args.len()))
{
// Gather function arg type strings for all
// of the different variants of this function
let mut type_arg_strings = type_args_multi
.iter()
.filter_map(|type_args| type_args.get(index))
.map(|type_arg| simple_stringify_type_info(type_arg.type_info()))
.collect::<Vec<_>>();
if type_arg_strings.len() < type_args_multi.len() {
for _ in type_arg_strings.len()..type_args_multi.len() {
type_arg_strings.push("nil".to_string());
}
}
// Type arg strings may themselves be stringified to something like number | string so we
// will split that out to be able to handle it better with the following unification process
let mut type_arg_strings_sep = Vec::new();
for type_arg_string in type_arg_strings.drain(..) {
for typ_arg_string_inner in type_arg_string.split(PIPE_SEPARATOR) {
type_arg_strings_sep.push(typ_arg_string_inner.to_string());
}
}
// Find out if we have any nillable type, to know if we
// should make the entire arg type union nillable or not
let has_any_optional = type_arg_strings_sep
.iter()
.any(|s| s == "nil" || s.ends_with('?'));
// Filter out any nils or optional markers (?),
// we will add this back at the end if necessary
let mut type_arg_strings_non_nil = type_arg_strings_sep
.iter()
.filter(|s| *s != "nil")
.map(|s| s.trim_end_matches('?').to_string())
.collect::<Vec<_>>();
type_arg_strings_non_nil.sort(); // Need to sort for dedup
type_arg_strings_non_nil.dedup(); // Dedup to get rid of redundant types such as string | string
unified_args.push(if has_any_optional {
if type_arg_strings_non_nil.len() == 1 {
format!("{}?", type_arg_strings_non_nil.pop().unwrap())
} else {
format!("({})?", type_arg_strings_non_nil.join(PIPE_SEPARATOR))
}
} else {
type_arg_strings_non_nil.join(PIPE_SEPARATOR)
});
}
Some(unified_args)
}
}
} else {
None
}
});
if let Some(first_alphanumeric) = first {
if first_alphanumeric > 0 {
acc.min(first_alphanumeric - 1)
} else {
0
}
} else {
acc
}
});
let unindented_lines = lines.iter().map(|line| &line[indent_len..]);
let mut doc_items = Vec::new();
let mut doc_lines = Vec::new();
for line in unindented_lines {
if let Some(tag) = parse_moonwave_style_tag(line) {
doc_items.push(tag);
} else {
doc_lines.push(line);
}
}
if !doc_lines.is_empty() {
doc_items.push(
DocItemBuilder::new()
.with_kind(DocItemKind::Description)
.with_value(doc_lines.join("\n").trim())
.build()
.unwrap(),
);
}
doc_items
}
fn find_token_moonwave_comment(token: &TokenReference) -> Option<String> {

41
packages/cli/src/gen/selene_defs.rs
View file

@ -3,7 +3,7 @@ use serde_yaml::{Mapping as YamlMapping, Sequence as YamlSequence, Value as Yaml
use crate::gen::doc2::DocsItemTag;
use super::doc2::{DocItem, DocItemKind, DocTree};
use super::doc2::{DocItem, DocItemKind, DocTree, PIPE_SEPARATOR};
pub fn generate_from_type_definitions(contents: &str) -> Result<String> {
let tree = DocTree::from_type_definitions(contents)?;
@ -110,10 +110,47 @@ fn doc_item_to_selene_yaml_mapping(item: &DocItem) -> Result<YamlMapping> {
YamlValue::Bool(true),
);
}
let mut args = YamlSequence::new();
for arg_type in item.arg_types() {
let mut arg_mapping = YamlMapping::new();
let (type_str, type_opt) = match arg_type.strip_suffix('?') {
Some(stripped) => (stripped, true),
None => (arg_type, false),
};
if type_opt {
arg_mapping.insert(
YamlValue::String("required".to_string()),
YamlValue::Bool(false),
);
}
arg_mapping.insert(
YamlValue::String("type".to_string()),
YamlValue::String(simplify_type_str_into_primitives(
type_str.trim_start_matches('(').trim_end_matches(')'),
)),
);
args.push(YamlValue::Mapping(arg_mapping));
}
mapping.insert(
YamlValue::String("args".to_string()),
YamlValue::Mapping(YamlMapping::new()),
YamlValue::Sequence(args),
);
}
Ok(mapping)
}
fn simplify_type_str_into_primitives(type_str: &str) -> String {
let mut primitives = Vec::new();
for type_inner in type_str.split(PIPE_SEPARATOR) {
if type_inner.starts_with('{') && type_inner.ends_with('}') {
primitives.push("table");
} else if type_inner.starts_with('"') && type_inner.ends_with('"') {
primitives.push("string");
} else {
primitives.push(type_inner);
}
}
primitives.sort_unstable();
primitives.dedup();
primitives.join(PIPE_SEPARATOR)
}