Experimenting with a different AST structure and ideas.

Ideas:

* For literate output, format like Docco, tables like

    Doc        | Code
    -----------|------------
    | docblock | codeblock |
    | docblock | codeblock |
    | docblock | codeblock |

* For javadoc output, maybe create a running 'pure source' object containing
  just the code lines. Then run an sup-parser for the language the code is
  written in and build a seperate AST for the code. This code AST then gets
  tagged onto the overall AST and is used in the generation phase for code
  comprehension. Would still need a way to map doc blocks to code blocks. I
  could probably use line numbers. In that case I would need to map the original
  source line from the commented input to the 'pure source' while processing the
  'pure source' and store the original line number in the code AST. That would
  give me a reliable way to lookup the closest code structure to a doc block.
* The code AST would need to made of generic pieces if I want to have
  language-agnostic generator code. What may be better is to allow the language
  parser to create it's code AST however is wants and just have some pluggable
  bit of the generator for each language. Would increase generator code
  complexity though.

doc/grammar.rst

@@ -1,29 +1,29 @@
-SourceFile -> (DocBlock / CodeBlock)*
+SourceFile ->
+    (Block / DocBlock / CodeBlock)+
 
-DocBlock -> (DirectiveBlock / MarkdownBlock)+
+Block ->
+    DocBlock CodeBlock
 
-Code Block -> ((!DOC_START RemainingLine) / EmptyLine)+
+DocBlock ->
+    (Directive / DocText)+
 
-DirectiveBlock -> DOC_START DIRECTIVE_START (LongDirective / LineDirective)
-
-MarkdownBlock -> MarkdownLine+
+Directive ->
+    DocLineStart AT (LongDirective / ShortDirective)
 
 LongDirective ->
-    (AUTHOR_DIR / DOC_DIR / EXAMPLE_DIR) RemainingLine MarkdownBlock?
+    ("author" / "doc" / "example") RemainingLine DocText?
 
-LineDirective -> ORG_DIR RemainingLine
+ShortDirective ->
+    ("org" / "copyright") RemainingLine
 
-MarkdownLine -> DOC_START !DIRECTIVE_START RemainingLine
+DocText ->
+    (DocLineStart !AT RemainingLine)+
 
-RemainingLine -> (!EOL)+ (EOL / EOI)
-
-EmptyLine -> EOL
-
-Tokens
-------
-
-DOC_START           -> "%% "
-EOL                 -> "\n"
-DIRECTIVE_START     -> "@"
+DocLineStart ->
+    Space* DOC_LINE_START Space?
 
+CodeBlock ->
+    (!DocLineStart RemainingLine)+
 
+RemainingLine ->
+   ((!EOL)* EOL) / ((!EOL)+ EOI)

resources/main/test.groovy

@@ -43,4 +43,6 @@ vbsTest = {
     println "----------------------------------"
 
     (new File('vbs_result.html')).withWriter { out -> out.println vbsResult }
+
+    return [vbsParsed, vbsResult]
 }

src/main/com/jdblabs/jlp/MarkdownGenerator.groovy

@@ -14,7 +14,8 @@ public class MarkdownGenerator extends JLPBaseGenerator {
     protected MarkdownGenerator() {
         super()
 
-        pegdown = new PegDownProcessor(Extensions.TABLES) }
+        pegdown = new PegDownProcessor(
+            Extensions.TABLES & Extensions.DEFINITIONS) }
 
     protected static Map<String, String> generateDocuments(
     Map<String, List<ASTNode>> sources) {

src/main/com/jdblabs/jlp/experimental/JLPPegParser.java

@@ -0,0 +1,195 @@
+package com.jdblabs.jlp.experimental;
+
+import com.jdblabs.jlp.experimental.ast.*;
+import java.util.ArrayList;
+import java.util.List;
+import org.parboiled.Action;
+import org.parboiled.BaseParser;
+import org.parboiled.Context;
+import org.parboiled.Rule;
+import org.parboiled.annotations.*;
+
+@BuildParseTree
+public class JLPPegParser extends BaseParser<Object> {
+
+    int curLineNum = 1;
+
+    /**
+     * Parses the rule:
+     *  SourceFile = (Block / DocBlock / CodeBlock)+
+     *
+     * Pushes a SourceFile node on the stack.
+     */
+    public Rule SourceFile() {
+        return Sequence(
+            clearLineCount(),
+            push(new SourceFile()),
+
+            OneOrMore(Sequence(
+                FirstOf(Block(), DocBlock(), CodeBlock()),
+
+                addBlock((ASTNode) pop())))); }
+
+    /**
+     * Parses the rule:
+     *   Block = DocBlock CodeBlock
+     *
+     * Pushes a Block onto the stack
+     */
+    Rule Block() {
+        return Sequence(
+            push(curLineNum),
+            DocBlock(), CodeBlock(),
+
+            push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); }
+
+    /**
+     * Parses the rule:
+     *  DocBlock = (Directive / DocText)+
+     *
+     * Pushes a DocBlock object onto the stack
+     */
+    Rule DocBlock() {
+        return Sequence(
+            push(new DocBlock(curLineNum)),
+            OneOrMore(Sequence(
+                FirstOf(Directive(), DocText()),
+                addToDocBlock((ASTNode) pop())))); }
+
+    /**
+     * Parses the rule:
+     *  CodeBlock = (!DocLineStart RemainingLine)+
+     *
+     * Pushes a CodeBlock onto the stack.
+     */
+    Rule CodeBlock() {
+        return Sequence(
+            push(new CodeBlock(curLineNum)),
+            OneOrMore(Sequence(
+                TestNot(DocLineStart()), RemainingLine(),
+                addToCodeBlock(match())))); }
+
+    /**
+     * Parses the rule:
+     *  Directive = DocLineStart AT (LongDirective / ShortFirective)
+     *
+     * Pushes a Directive node on the stack.
+     */
+    Rule Directive() {
+        return Sequence(
+            DocLineStart(), AT, FirstOf(LongDirective(), ShortDirective())); }
+
+    /**
+     * Parses the rule:
+     *  LongDirective =
+     *      (AUTHOR_DIR / DOC_DIR / EXAMPLE_DIR) RemainingLine DocText?
+     *
+     * Pushes a Directive node onto the stack.
+     */
+    Rule LongDirective() {
+        return Sequence(
+            push(curLineNum),
+
+            FirstOf(AUTHOR_DIR, DOC_DIR, EXAMPLE_DIR),  push(match()),
+            RemainingLine(),                            push(match()),
+
+            Optional(Sequence(
+                DocText(),
+                swap(),
+                push(popAsString() + ((DocText) pop()).value))),
+                
+            push(new Directive(popAsString(), popAsString(), popAsInt()))); }
+
+    /**
+     * Parses the rule:
+     *  ShortDirective = (ORG_DIR / COPYRIGHT_DIR) RemainingLine
+     *
+     * Pushes a Directive node onto the stack.
+     */
+    Rule ShortDirective() {
+        return Sequence(
+            push(curLineNum),
+            FirstOf(ORG_DIR, COPYRIGHT_DIR), push(match()),
+            RemainingLine(),
+            
+            push(new Directive(match().trim(), popAsString(), popAsInt()))); }
+
+    /**
+     * Parses the rule:
+     *  DocText = (DocLineStart !AT RemainingLine)+
+     *
+     * Pushes a DocText node onto the stack.
+     */
+    Rule DocText() {
+        return Sequence(
+            push(new DocText(curLineNum)),
+            OneOrMore(Sequence(
+                DocLineStart(), TestNot(AT), RemainingLine(),
+                addToDocText(match())))); }
+
+    @SuppressSubnodes
+    Rule DocLineStart() {
+        return Sequence(
+            ZeroOrMore(SPACE), DOC_LINE_START, Optional(SPACE)); }
+
+    @SuppressSubnodes
+    Rule RemainingLine() {
+        return FirstOf(
+            Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()),
+            Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); }
+
+    Rule AT         = Ch('@');
+    Rule EOL        = OneOrMore(AnyOf("\r\n"));
+    Rule NOT_EOL    = Sequence(TestNot(EOL), ANY);
+    Rule SPACE      = AnyOf(" \t");
+    Rule DOC_LINE_START = String("%%");
+
+    // directive terminals
+    Rule AUTHOR_DIR     = IgnoreCase("author");
+    Rule COPYRIGHT_DIR  = IgnoreCase("copyright");
+    Rule DOC_DIR        = IgnoreCase("doc");
+    Rule EXAMPLE_DIR    = IgnoreCase("example");
+    Rule ORG_DIR        = IgnoreCase("org");
+
+    String popAsString() { return (String) pop(); }
+
+    Integer popAsInt() { return (Integer) pop(); }
+
+    boolean clearLineCount() { curLineNum = 1; return true; }
+
+    boolean incLineCount() { curLineNum++; return true; }
+
+    boolean addBlock(ASTNode block) {
+        SourceFile sourceFile = (SourceFile) pop();
+        sourceFile.blocks.add(block);
+        return push(sourceFile); }
+
+    /**
+     * Pop off a DocBlock, add the given Directive or DocText and push the
+     * DocBlock back onto the stack.
+     */
+    boolean addToDocBlock(ASTNode an) {
+        DocBlock docBlock = (DocBlock) pop();
+        if (an instanceof Directive) {
+            docBlock.directives.add((Directive) an); }
+        else if (an instanceof DocText) {
+            docBlock.docTexts.add((DocText) an); }
+        else { throw new IllegalStateException(); }
+        return push(docBlock); }
+        
+    boolean addToCodeBlock(String line) {
+        CodeBlock codeBlock = (CodeBlock) pop();
+        codeBlock.lines.put(curLineNum - 1, line);
+        return push(codeBlock); }
+
+    boolean addToDocText(String line) {
+        DocText docText = (DocText) pop();
+        docText.value += line;
+        return push(docText); }
+
+    boolean printValueStack() {
+        for (int i = 0; i < getContext().getValueStack().size(); i++) {
+            System.out.println(i + ": " + peek(i)); }
+        return true; }
+
+}

src/main/com/jdblabs/jlp/experimental/ast/ASTNode.groovy

@@ -0,0 +1,10 @@
+package com.jdblabs.jlp.experimental.ast
+
+public class ASTNode {
+
+    protected int lineNumber
+
+    public ASTNode(int lineNum) { this.lineNumber = lineNum }
+
+    public int getLineNumber() { lineNumber }
+}

src/main/com/jdblabs/jlp/experimental/ast/Block.groovy

@@ -0,0 +1,12 @@
+package com.jdblabs.jlp.experimental.ast
+
+public class Block extends ASTNode {
+    public final DocBlock docBlock
+    public final CodeBlock codeBlock
+
+    public Block(CodeBlock cb, DocBlock db, int lineNum) {
+        super(lineNum); docBlock = db; codeBlock = cb }
+
+    public String toString() {
+        "[${lineNumber}:Block: ${docBlock}, ${codeBlock}]" }
+}

src/main/com/jdblabs/jlp/experimental/ast/CodeBlock.groovy

@@ -0,0 +1,16 @@
+package com.jdblabs.jlp.experimental.ast
+
+import java.util.Map
+
+public class CodeBlock extends ASTNode {
+
+    public Map<Integer, String> lines = [:]
+
+    public CodeBlock(int lineNumber) { super(lineNumber) }
+
+    public String toString() {
+        def linesVal = ""
+        lines.each { lineNum, value -> linesVal += "${lineNum}:${value}" }
+            
+        return "[${lineNumber}:CodeBlock: ${linesVal}]" }
+}

src/main/com/jdblabs/jlp/experimental/ast/Directive.groovy

@@ -0,0 +1,24 @@
+package com.jdblabs.jlp.experimental.ast
+
+public class Directive extends ASTNode {
+
+    public static enum DirectiveType {
+        Author,
+        Copyright,
+        Doc,
+        Example,
+        Org;
+        
+        public static DirectiveType parse(String typeString) {
+            valueOf(typeString.toLowerCase().capitalize()) } }
+
+    public final DirectiveType type;
+    public final String value;
+
+    public Directive(String value, String typeString, int lineNumber) {
+        super(lineNumber)
+        this.value = value
+        this.type = DirectiveType.parse(typeString) }
+        
+    public String toString() { "[${lineNumber}:Directive: ${type}, ${value}]" }
+}

src/main/com/jdblabs/jlp/experimental/ast/DocBlock.groovy

@@ -0,0 +1,15 @@
+package com.jdblabs.jlp.experimental.ast
+
+import java.util.ArrayList
+import java.util.List
+
+public class DocBlock extends ASTNode {
+    
+    public List<Directive> directives = new ArrayList<Directive>()
+    public List<DocText> docTexts = new ArrayList<DocText>()
+
+    public DocBlock(int lineNumber) { super(lineNumber) }
+
+    public String toString() {
+        "[${lineNumber}:DocBlock: Directives ${directives}, DocTexts ${docTexts}]" }
+}

src/main/com/jdblabs/jlp/experimental/ast/DocText.groovy

@@ -0,0 +1,10 @@
+package com.jdblabs.jlp.experimental.ast
+
+public class DocText extends ASTNode {
+
+    public String value
+
+    public DocText(int lineNumber) { super(lineNumber) }
+
+    public String toString() { value }
+}

src/main/com/jdblabs/jlp/experimental/ast/SourceFile.groovy

@@ -0,0 +1,6 @@
+package com.jdblabs.jlp.experimental.ast
+
+public class SourceFile {
+    public List<ASTNode> blocks = []
+    public def codeAST
+}