From 7717439274f8f202f3fe08c73ca1c0c6aae98b1b Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Wed, 31 Aug 2011 12:09:25 -0500 Subject: [PATCH] Experimenting with a different AST structure and ideas. Ideas: * For literate output, format like Docco, tables like Doc | Code -----------|------------ | docblock | codeblock | | docblock | codeblock | | docblock | codeblock | * For javadoc output, maybe create a running 'pure source' object containing just the code lines. Then run an sup-parser for the language the code is written in and build a seperate AST for the code. This code AST then gets tagged onto the overall AST and is used in the generation phase for code comprehension. Would still need a way to map doc blocks to code blocks. I could probably use line numbers. In that case I would need to map the original source line from the commented input to the 'pure source' while processing the 'pure source' and store the original line number in the code AST. That would give me a reliable way to lookup the closest code structure to a doc block. * The code AST would need to made of generic pieces if I want to have language-agnostic generator code. What may be better is to allow the language parser to create it's code AST however is wants and just have some pluggable bit of the generator for each language. Would increase generator code complexity though. --- doc/grammar.rst | 38 ++-- resources/main/test.groovy | 2 + .../com/jdblabs/jlp/MarkdownGenerator.groovy | 3 +- .../jlp/experimental/JLPPegParser.java | 195 ++++++++++++++++++ .../jlp/experimental/ast/ASTNode.groovy | 10 + .../jdblabs/jlp/experimental/ast/Block.groovy | 12 ++ .../jlp/experimental/ast/CodeBlock.groovy | 16 ++ .../jlp/experimental/ast/Directive.groovy | 24 +++ .../jlp/experimental/ast/DocBlock.groovy | 15 ++ .../jlp/experimental/ast/DocText.groovy | 10 + .../jlp/experimental/ast/SourceFile.groovy | 6 + 11 files changed, 311 insertions(+), 20 deletions(-) create mode 100644 src/main/com/jdblabs/jlp/experimental/JLPPegParser.java create mode 100644 src/main/com/jdblabs/jlp/experimental/ast/ASTNode.groovy create mode 100644 src/main/com/jdblabs/jlp/experimental/ast/Block.groovy create mode 100644 src/main/com/jdblabs/jlp/experimental/ast/CodeBlock.groovy create mode 100644 src/main/com/jdblabs/jlp/experimental/ast/Directive.groovy create mode 100644 src/main/com/jdblabs/jlp/experimental/ast/DocBlock.groovy create mode 100644 src/main/com/jdblabs/jlp/experimental/ast/DocText.groovy create mode 100644 src/main/com/jdblabs/jlp/experimental/ast/SourceFile.groovy diff --git a/doc/grammar.rst b/doc/grammar.rst index 4a96a9b..4634cc0 100644 --- a/doc/grammar.rst +++ b/doc/grammar.rst @@ -1,29 +1,29 @@ -SourceFile -> (DocBlock / CodeBlock)* +SourceFile -> + (Block / DocBlock / CodeBlock)+ -DocBlock -> (DirectiveBlock / MarkdownBlock)+ +Block -> + DocBlock CodeBlock -Code Block -> ((!DOC_START RemainingLine) / EmptyLine)+ +DocBlock -> + (Directive / DocText)+ -DirectiveBlock -> DOC_START DIRECTIVE_START (LongDirective / LineDirective) - -MarkdownBlock -> MarkdownLine+ +Directive -> + DocLineStart AT (LongDirective / ShortDirective) LongDirective -> - (AUTHOR_DIR / DOC_DIR / EXAMPLE_DIR) RemainingLine MarkdownBlock? + ("author" / "doc" / "example") RemainingLine DocText? -LineDirective -> ORG_DIR RemainingLine +ShortDirective -> + ("org" / "copyright") RemainingLine -MarkdownLine -> DOC_START !DIRECTIVE_START RemainingLine +DocText -> + (DocLineStart !AT RemainingLine)+ -RemainingLine -> (!EOL)+ (EOL / EOI) - -EmptyLine -> EOL - -Tokens ------- - -DOC_START -> "%% " -EOL -> "\n" -DIRECTIVE_START -> "@" +DocLineStart -> + Space* DOC_LINE_START Space? +CodeBlock -> + (!DocLineStart RemainingLine)+ +RemainingLine -> + ((!EOL)* EOL) / ((!EOL)+ EOI) diff --git a/resources/main/test.groovy b/resources/main/test.groovy index 6838edb..628bb1a 100644 --- a/resources/main/test.groovy +++ b/resources/main/test.groovy @@ -43,4 +43,6 @@ vbsTest = { println "----------------------------------" (new File('vbs_result.html')).withWriter { out -> out.println vbsResult } + + return [vbsParsed, vbsResult] } diff --git a/src/main/com/jdblabs/jlp/MarkdownGenerator.groovy b/src/main/com/jdblabs/jlp/MarkdownGenerator.groovy index 862f1f4..9ba8d21 100644 --- a/src/main/com/jdblabs/jlp/MarkdownGenerator.groovy +++ b/src/main/com/jdblabs/jlp/MarkdownGenerator.groovy @@ -14,7 +14,8 @@ public class MarkdownGenerator extends JLPBaseGenerator { protected MarkdownGenerator() { super() - pegdown = new PegDownProcessor(Extensions.TABLES) } + pegdown = new PegDownProcessor( + Extensions.TABLES & Extensions.DEFINITIONS) } protected static Map generateDocuments( Map> sources) { diff --git a/src/main/com/jdblabs/jlp/experimental/JLPPegParser.java b/src/main/com/jdblabs/jlp/experimental/JLPPegParser.java new file mode 100644 index 0000000..5f75a43 --- /dev/null +++ b/src/main/com/jdblabs/jlp/experimental/JLPPegParser.java @@ -0,0 +1,195 @@ +package com.jdblabs.jlp.experimental; + +import com.jdblabs.jlp.experimental.ast.*; +import java.util.ArrayList; +import java.util.List; +import org.parboiled.Action; +import org.parboiled.BaseParser; +import org.parboiled.Context; +import org.parboiled.Rule; +import org.parboiled.annotations.*; + +@BuildParseTree +public class JLPPegParser extends BaseParser { + + int curLineNum = 1; + + /** + * Parses the rule: + * SourceFile = (Block / DocBlock / CodeBlock)+ + * + * Pushes a SourceFile node on the stack. + */ + public Rule SourceFile() { + return Sequence( + clearLineCount(), + push(new SourceFile()), + + OneOrMore(Sequence( + FirstOf(Block(), DocBlock(), CodeBlock()), + + addBlock((ASTNode) pop())))); } + + /** + * Parses the rule: + * Block = DocBlock CodeBlock + * + * Pushes a Block onto the stack + */ + Rule Block() { + return Sequence( + push(curLineNum), + DocBlock(), CodeBlock(), + + push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); } + + /** + * Parses the rule: + * DocBlock = (Directive / DocText)+ + * + * Pushes a DocBlock object onto the stack + */ + Rule DocBlock() { + return Sequence( + push(new DocBlock(curLineNum)), + OneOrMore(Sequence( + FirstOf(Directive(), DocText()), + addToDocBlock((ASTNode) pop())))); } + + /** + * Parses the rule: + * CodeBlock = (!DocLineStart RemainingLine)+ + * + * Pushes a CodeBlock onto the stack. + */ + Rule CodeBlock() { + return Sequence( + push(new CodeBlock(curLineNum)), + OneOrMore(Sequence( + TestNot(DocLineStart()), RemainingLine(), + addToCodeBlock(match())))); } + + /** + * Parses the rule: + * Directive = DocLineStart AT (LongDirective / ShortFirective) + * + * Pushes a Directive node on the stack. + */ + Rule Directive() { + return Sequence( + DocLineStart(), AT, FirstOf(LongDirective(), ShortDirective())); } + + /** + * Parses the rule: + * LongDirective = + * (AUTHOR_DIR / DOC_DIR / EXAMPLE_DIR) RemainingLine DocText? + * + * Pushes a Directive node onto the stack. + */ + Rule LongDirective() { + return Sequence( + push(curLineNum), + + FirstOf(AUTHOR_DIR, DOC_DIR, EXAMPLE_DIR), push(match()), + RemainingLine(), push(match()), + + Optional(Sequence( + DocText(), + swap(), + push(popAsString() + ((DocText) pop()).value))), + + push(new Directive(popAsString(), popAsString(), popAsInt()))); } + + /** + * Parses the rule: + * ShortDirective = (ORG_DIR / COPYRIGHT_DIR) RemainingLine + * + * Pushes a Directive node onto the stack. + */ + Rule ShortDirective() { + return Sequence( + push(curLineNum), + FirstOf(ORG_DIR, COPYRIGHT_DIR), push(match()), + RemainingLine(), + + push(new Directive(match().trim(), popAsString(), popAsInt()))); } + + /** + * Parses the rule: + * DocText = (DocLineStart !AT RemainingLine)+ + * + * Pushes a DocText node onto the stack. + */ + Rule DocText() { + return Sequence( + push(new DocText(curLineNum)), + OneOrMore(Sequence( + DocLineStart(), TestNot(AT), RemainingLine(), + addToDocText(match())))); } + + @SuppressSubnodes + Rule DocLineStart() { + return Sequence( + ZeroOrMore(SPACE), DOC_LINE_START, Optional(SPACE)); } + + @SuppressSubnodes + Rule RemainingLine() { + return FirstOf( + Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()), + Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); } + + Rule AT = Ch('@'); + Rule EOL = OneOrMore(AnyOf("\r\n")); + Rule NOT_EOL = Sequence(TestNot(EOL), ANY); + Rule SPACE = AnyOf(" \t"); + Rule DOC_LINE_START = String("%%"); + + // directive terminals + Rule AUTHOR_DIR = IgnoreCase("author"); + Rule COPYRIGHT_DIR = IgnoreCase("copyright"); + Rule DOC_DIR = IgnoreCase("doc"); + Rule EXAMPLE_DIR = IgnoreCase("example"); + Rule ORG_DIR = IgnoreCase("org"); + + String popAsString() { return (String) pop(); } + + Integer popAsInt() { return (Integer) pop(); } + + boolean clearLineCount() { curLineNum = 1; return true; } + + boolean incLineCount() { curLineNum++; return true; } + + boolean addBlock(ASTNode block) { + SourceFile sourceFile = (SourceFile) pop(); + sourceFile.blocks.add(block); + return push(sourceFile); } + + /** + * Pop off a DocBlock, add the given Directive or DocText and push the + * DocBlock back onto the stack. + */ + boolean addToDocBlock(ASTNode an) { + DocBlock docBlock = (DocBlock) pop(); + if (an instanceof Directive) { + docBlock.directives.add((Directive) an); } + else if (an instanceof DocText) { + docBlock.docTexts.add((DocText) an); } + else { throw new IllegalStateException(); } + return push(docBlock); } + + boolean addToCodeBlock(String line) { + CodeBlock codeBlock = (CodeBlock) pop(); + codeBlock.lines.put(curLineNum - 1, line); + return push(codeBlock); } + + boolean addToDocText(String line) { + DocText docText = (DocText) pop(); + docText.value += line; + return push(docText); } + + boolean printValueStack() { + for (int i = 0; i < getContext().getValueStack().size(); i++) { + System.out.println(i + ": " + peek(i)); } + return true; } + +} diff --git a/src/main/com/jdblabs/jlp/experimental/ast/ASTNode.groovy b/src/main/com/jdblabs/jlp/experimental/ast/ASTNode.groovy new file mode 100644 index 0000000..f04e66c --- /dev/null +++ b/src/main/com/jdblabs/jlp/experimental/ast/ASTNode.groovy @@ -0,0 +1,10 @@ +package com.jdblabs.jlp.experimental.ast + +public class ASTNode { + + protected int lineNumber + + public ASTNode(int lineNum) { this.lineNumber = lineNum } + + public int getLineNumber() { lineNumber } +} diff --git a/src/main/com/jdblabs/jlp/experimental/ast/Block.groovy b/src/main/com/jdblabs/jlp/experimental/ast/Block.groovy new file mode 100644 index 0000000..3bc4311 --- /dev/null +++ b/src/main/com/jdblabs/jlp/experimental/ast/Block.groovy @@ -0,0 +1,12 @@ +package com.jdblabs.jlp.experimental.ast + +public class Block extends ASTNode { + public final DocBlock docBlock + public final CodeBlock codeBlock + + public Block(CodeBlock cb, DocBlock db, int lineNum) { + super(lineNum); docBlock = db; codeBlock = cb } + + public String toString() { + "[${lineNumber}:Block: ${docBlock}, ${codeBlock}]" } +} diff --git a/src/main/com/jdblabs/jlp/experimental/ast/CodeBlock.groovy b/src/main/com/jdblabs/jlp/experimental/ast/CodeBlock.groovy new file mode 100644 index 0000000..0bbfd0d --- /dev/null +++ b/src/main/com/jdblabs/jlp/experimental/ast/CodeBlock.groovy @@ -0,0 +1,16 @@ +package com.jdblabs.jlp.experimental.ast + +import java.util.Map + +public class CodeBlock extends ASTNode { + + public Map lines = [:] + + public CodeBlock(int lineNumber) { super(lineNumber) } + + public String toString() { + def linesVal = "" + lines.each { lineNum, value -> linesVal += "${lineNum}:${value}" } + + return "[${lineNumber}:CodeBlock: ${linesVal}]" } +} diff --git a/src/main/com/jdblabs/jlp/experimental/ast/Directive.groovy b/src/main/com/jdblabs/jlp/experimental/ast/Directive.groovy new file mode 100644 index 0000000..758c315 --- /dev/null +++ b/src/main/com/jdblabs/jlp/experimental/ast/Directive.groovy @@ -0,0 +1,24 @@ +package com.jdblabs.jlp.experimental.ast + +public class Directive extends ASTNode { + + public static enum DirectiveType { + Author, + Copyright, + Doc, + Example, + Org; + + public static DirectiveType parse(String typeString) { + valueOf(typeString.toLowerCase().capitalize()) } } + + public final DirectiveType type; + public final String value; + + public Directive(String value, String typeString, int lineNumber) { + super(lineNumber) + this.value = value + this.type = DirectiveType.parse(typeString) } + + public String toString() { "[${lineNumber}:Directive: ${type}, ${value}]" } +} diff --git a/src/main/com/jdblabs/jlp/experimental/ast/DocBlock.groovy b/src/main/com/jdblabs/jlp/experimental/ast/DocBlock.groovy new file mode 100644 index 0000000..d43c405 --- /dev/null +++ b/src/main/com/jdblabs/jlp/experimental/ast/DocBlock.groovy @@ -0,0 +1,15 @@ +package com.jdblabs.jlp.experimental.ast + +import java.util.ArrayList +import java.util.List + +public class DocBlock extends ASTNode { + + public List directives = new ArrayList() + public List docTexts = new ArrayList() + + public DocBlock(int lineNumber) { super(lineNumber) } + + public String toString() { + "[${lineNumber}:DocBlock: Directives ${directives}, DocTexts ${docTexts}]" } +} diff --git a/src/main/com/jdblabs/jlp/experimental/ast/DocText.groovy b/src/main/com/jdblabs/jlp/experimental/ast/DocText.groovy new file mode 100644 index 0000000..f9f9c52 --- /dev/null +++ b/src/main/com/jdblabs/jlp/experimental/ast/DocText.groovy @@ -0,0 +1,10 @@ +package com.jdblabs.jlp.experimental.ast + +public class DocText extends ASTNode { + + public String value + + public DocText(int lineNumber) { super(lineNumber) } + + public String toString() { value } +} diff --git a/src/main/com/jdblabs/jlp/experimental/ast/SourceFile.groovy b/src/main/com/jdblabs/jlp/experimental/ast/SourceFile.groovy new file mode 100644 index 0000000..4f2d0ba --- /dev/null +++ b/src/main/com/jdblabs/jlp/experimental/ast/SourceFile.groovy @@ -0,0 +1,6 @@ +package com.jdblabs.jlp.experimental.ast + +public class SourceFile { + public List blocks = [] + public def codeAST +}