Promoting the experimental code to replace the exising main code.

resources/main/test.groovy

@@ -1,26 +1,16 @@
 import com.jdblabs.jlp.*
-import com.jdblabs.jlp.experimental.LiterateMarkdownGenerator
 import org.parboiled.Parboiled
 import org.parboiled.parserunners.ReportingParseRunner
 import org.parboiled.parserunners.RecoveringParseRunner
 
-makeParser = {
+"Making the standard parser."
-    println "Making the standard parser."
+"---------------------------"
-    println "---------------------------"
 
-    parser = Parboiled.createParser(JLPPegParser.class)
+parser = Parboiled.createParser(JLPPegParser.class)
-    parseRunner = new ReportingParseRunner(parser.SourceFile())
+parseRunner = new ReportingParseRunner(parser.SourceFile())
-}
-
-makeExperimentalParser = {
-    println "Making the experimental parser."
-    println "-------------------------------"
-
-    parser = Parboiled.createParser(com.jdblabs.jlp.experimental.JLPPegParser.class)
-    parseRunner = new ReportingParseRunner(parser.SourceFile())
-}
 
 simpleTest = {
+
     println "Parsing the simple test into 'result'."
     println "--------------------------------------"
 
@@ -46,33 +36,12 @@ vbsTest = {
 
     vbsParsed = parseRunner.run(vbsTestInput)
 
-    vbsResult = MarkdownGenerator.generateDocuments([vbs: vbsParsed.resultValue]).vbs
+    vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl"
 
-    println "Writing to file 'vbs_result.html'."
+    println "Writing to file 'vbs_db_records.html'."
-    println "----------------------------------"
+    println "--------------------------------------"
 
-    (new File('vbs_result.html')).withWriter { out -> out.println vbsResult }
+    (new File('vbs_db_records.html')).withWriter { out -> out.println vbsResult }
 
     return [vbsParsed, vbsResult]
 }
-
-experimentalTest = {
-    makeExperimentalParser()
-
-    println "Parsing vbs_db_records.hrl into 'vbsResult'."
-    println "--------------------------------------------"
-
-    vbsTestFile = new File('vbs_db_records.hrl')
-    println "vbsTestFile is ${vbsTestFile.exists() ? 'present' : 'absent'}."
-    vbsTestInput = vbsTestFile.text
-
-    vbsParsed = parseRunner.run(vbsTestInput)
-
-    vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl"
-
-    println "Writing to file 'vbs_result.html'."
-    println "----------------------------------"
-
-    (new File('vbs_result.html')).withWriter { out -> out.println vbsResult }
-
-}

src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy

@@ -1,7 +1,6 @@
 package com.jdblabs.jlp
 
 import com.jdblabs.jlp.ast.*
-import com.jdblabs.jlp.ast.Directive.DirectiveType
 import java.util.List
 import java.util.Map
 
@@ -10,54 +9,54 @@ public abstract class JLPBaseGenerator {
     protected Map docState
 
     protected JLPBaseGenerator() {
-        docState = [orgs:           [:],
+        docState = [orgs:           [:],        // stores `@org` references
-                    currentDocId:   false ] }
+                    codeTrees:      [:],        // stores code ASTs for
+                    currentDocId:   false ] }   // store the current docid
 
-    protected Map<String, String> generate(Map<String, List<ASTNode>> sources) {
+    protected Map<String, String> generate(Map<String, SourceFile> sources) {
         Map result = [:]
-        sources.each { sourceId, sourceNodes ->
+
+        // run the parse phase
+        sources.each { sourceId, sourceAST ->
+
+            // set up the current generator state for this source
             docState.currentDocId = sourceId
-            result[sourceId] = emitDocument(sourceNodes) }
+            docState.codeTrees[sourceId] = sourceAST.codeAST
+
+            parse(sourceAST) }
+
+        // run the emit phase
+        sources.each { sourceId, sourceAST ->
+
+            // set up the current generator state for this source
+            docState.currentDocId = sourceId
+
+            // generate the doc for this source
+            result[sourceId] = emit(sourceAST) }
+
+        // return our results
         return result }
 
-    protected String emitDocument(List<ASTNode> sourceNodes) {
+    protected void parse(SourceFile sourceFile) {
-        StringBuilder result =
+        sourceFile.blocks.each { block -> parse(block) } }
-            sourceNodes.inject(new StringBuilder()) { sb, node ->
-                sb.append(emit(node))
-                return sb }
 
-        return result.toString() }
+    protected void parse(Block block) {
+        parse(block.docBlock)
+        parse(block.codeBlock) }
 
-    protected String emit(DocBlock docBlock) {
+    protected void parse(DocBlock docBlock) {
-        List printQueue 
+        docBlock.directives.each { directive -> parse(directive) }
-        StringBuilder result
+        docBlock.docTexts.each { docText -> parse(docText) } }
 
-        printQueue = docBlock.directives.collect { directive ->
+    protected abstract void parse(Directive directive)
-            def queueItem = [line: directive.lineNumber, value: directive]
+    protected abstract void parse(CodeBlock codeBlock)
-            switch (directive.type) {
+    protected abstract void parse(DocText docText)
-                case DirectiveType.Author:  queueItem.priority = 50; break
-                case DirectiveType.Doc:     queueItem.priority = 50; break
-                case DirectiveType.Example: queueItem.priority = 50; break
-                case DirectiveType.Org:     queueItem.priority = 10; break }
 
-            return queueItem }
+    protected abstract String emit(SourceFile sourceFile)
-
+    protected abstract String emit(Block block)
-        printQueue.addAll(docBlock.textBlocks.collect { textBlock ->
+    protected abstract String emit(DocBlock docBlock)
-            [ priority: 50, line: textBlock.lineNumber, value: textBlock ] })
+    protected abstract String emit(CodeBlock codeBlock)
-
+    protected abstract String emit(DocText docText)
-        // sort by priority, then by line number
-        printQueue.sort(
-            {i1, i2 -> i1.priority != i2.priority ?
-                        i1.priority - i2.priority :
-                        i1.line - i2.line} as Comparator)
-        
-
-        result = printQueue.inject(new StringBuilder()) { sb, printable ->
-            sb.append(emit(printable.value))
-            return sb }
-
-        return result.toString() } 
-
-    protected abstract String emit(TextBlock textBlock)
     protected abstract String emit(Directive directive)
+
 }

src/main/com/jdblabs/jlp/JLPPegParser.java

@@ -9,189 +9,242 @@ import org.parboiled.Context;
 import org.parboiled.Rule;
 import org.parboiled.annotations.*;
 
-import static com.jdblabs.jlp.ast.TextBlock.makeCodeBlock;
-import static com.jdblabs.jlp.ast.TextBlock.makeTextBlock;
-
 @BuildParseTree
 public class JLPPegParser extends BaseParser<Object> {
 
     int curLineNum = 1;
 
+    /**
+     * Parses the rule:
+     *  SourceFile = (Block / DocBlock / CodeBlock)+
+     *
+     * Pushes a SourceFile node on the stack.
+     */
     public Rule SourceFile() {
         return Sequence(
+            // At the start of processing a new SourceFile, we need to set up
+            // our internal state. 
+
+            // Clear the line count.
             clearLineCount(),
-            push(new ArrayList<ASTNode>()),
+
-            ZeroOrMore(Sequence(
+            // Add the top-level SourceFile AST node.
+            push(new SourceFile()),
+
+            // A SourceFile is made up of one or more Blocks
+            OneOrMore(Sequence(
+                // All of these options result in one new Block pushed onto the
+                // stack.
                 FirstOf(
-                    DocBlock(),
+
-                    CodeBlock()),
+                    // Match a whole Block. This pushes a Block on the stack.
-                push(addToList(pop(), (List<Object>)pop()))))); }
+                    Block(),    
+
+                    // A standalone DocBlock. We will create an empty CodeBlock
+                    // to pair with it, then push a new Block onto the stack
+                    // made from the DocBlock and the empty CodeBlock
+                    Sequence(
+                        // 1. We need to remember the line number to create the
+                        // Block
+                        push(curLineNum),
+
+                        // 2. Match the DocBlock.
+                        DocBlock(),
+
+                        // 3. Create the empty CodeBlock.
+                        push(new CodeBlock(curLineNum)),
+
+                        // 4. Create the Block and push it onto the stack.
+                        push(new Block((CodeBlock) pop(), (DocBlock) pop(),
+                            popAsInt()))),
+
+                    // A standalone CodeBlock. Similar to the standalone
+                    // DocBlock, we will create an empty DocBlock to pair with
+                    // the CodeBlock to make a Block, which is pushed onto the
+                    // stack:
+                    //
+                    // *Note: With the way the parser is currently written,
+                    //        this will only match a CodeBlock that occurs
+                    //        before any DocBlock.*
+                    Sequence(
+                        // 1. Remember the line number for the Block.
+                        push(curLineNum),
+
+                        // 2. Create the empty DocBlock.
+                        push(new DocBlock(curLineNum)),
+
+                        // 3. Match the CodeBlock
+                        CodeBlock(),
+
+                        // 4. Create the Block and push it onto the stack
+                        push(new Block((CodeBlock) pop(), (DocBlock) pop(),
+                            popAsInt())))),
+
+                // pop the Block created by one of the above options and add it
+                // to the SourceFile
+                addBlockToSourceFile((Block) pop())))); }
 
     /**
      * Parses the rule:
-     *  DocBlock = (DirectiveBlock / DocTextBlock)+
+     *   Block = DocBlock CodeBlock
      *
-     * Pushes a DocBlock object onto the stack.
+     * Pushes a Block onto the stack
+     */
+    Rule Block() {
+        return Sequence(
+            push(curLineNum),
+            DocBlock(), CodeBlock(),
+
+            push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); }
+
+    /**
+     * Parses the rule:
+     *  DocBlock = (Directive / DocText)+
+     *
+     * Pushes a DocBlock object onto the stack
      */
     Rule DocBlock() {
         return Sequence(
             push(new DocBlock(curLineNum)),
-            OneOrMore(
+            OneOrMore(Sequence(
-                FirstOf(
+                FirstOf(Directive(), DocText()),
-                    Sequence(DirectiveBlock(),
+                addToDocBlock((ASTNode) pop())))); }
-                        push(addDirectiveBlock((Directive) pop(), (DocBlock) pop()))),
-                    Sequence(DocTextBlock(),
-                        push(addTextBlock((TextBlock) pop(), (DocBlock) pop())))))); }
 
     /**
      * Parses the rule:
-     *  CodeBlock = !DocStart RemainingLine
+     *  CodeBlock = (!DocLineStart RemainingLine)+
      *
      * Pushes a CodeBlock onto the stack.
      */
     Rule CodeBlock() {
         return Sequence(
-            push(curLineNum),
+            push(new CodeBlock(curLineNum)),
-            push(""),
             OneOrMore(Sequence(
-                TestNot(DocStart()), RemainingLine(),
+                TestNot(DocLineStart()), RemainingLine(),
-                push(popAsString() + match()))),
+                addToCodeBlock(match())))); }
 
-            push(makeCodeBlock(popAsString(),popAsInt()))); }
-                
     /**
      * Parses the rule:
-     *  DocStart = SPACE* DOC_START
+     *  Directive = DocLineStart AT (LongDirective / ShortDirective)
-     */
-     Rule DocStart() {
-        return Sequence(ZeroOrMore(SPACE), DOC_START); }
-            
-    /**
-     * Parses the rule:
-     *  DirectiveBlock =
-     *      DocStart DIRECTIVE_START (LongDirective / LineDirective)
      *
-     * Pushes a Directive onto the stack.
+     * Pushes a Directive node on the stack.
      */
-    Rule DirectiveBlock() {
+    Rule Directive() {
         return Sequence(
-            DocStart(), DIRECTIVE_START,
+            DocLineStart(), AT, FirstOf(LongDirective(), ShortDirective())); }
-            FirstOf(LongDirective(), LineDirective())); }
 
     /**
      * Parses the rule:
      *  LongDirective =
-     *      (AUTHOR_DIR / DOC_DIR / EXAMPLE_DIR) RemainingLine DocTextBlock?
+     *      (API_DIR / EXAMPLE_DIR) RemainingLine DocText?
-     * 
+     *
-     * Pushes a Directive object onto the value stack.
+     * Pushes a Directive node onto the stack.
      */
     Rule LongDirective() {
         return Sequence(
             push(curLineNum),
-            FirstOf(AUTHOR_DIR, DOC_DIR, EXAMPLE_DIR),  push(match()),
+
+            FirstOf(API_DIR, EXAMPLE_DIR),  push(match()),
             RemainingLine(),                            push(match()),
+
             Optional(Sequence(
-                DocTextBlock(),                        // pushes block
+                DocText(),
                 swap(),
-                push(popAsString() + ((TextBlock) pop()).value))),
+                push(popAsString() + ((DocText) pop()).value))),
-            
+                
-            // pull off the value, type and create the directive
             push(new Directive(popAsString(), popAsString(), popAsInt()))); }
 
     /**
      * Parses the rule:
-     *  LineDirective = 
+     *  ShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingLine
-     *      ORG_DIR RemainingLine
      *
-     * Pushes a Directive object onto the value stack.
+     * Pushes a Directive node onto the stack.
      */
-    Rule LineDirective() {
+    Rule ShortDirective() {
         return Sequence(
             push(curLineNum),
-            ORG_DIR,            push(match()),
+            FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()),
             RemainingLine(),
-
+            
-            // pull off the value, type and create the directive
             push(new Directive(match().trim(), popAsString(), popAsInt()))); }
 
     /**
      * Parses the rule:
-     *  DocTextBlock =  DocTextLine+
+     *  DocText = (DocLineStart !AT RemainingLine)+
      *
-     * Pushes a DocTextBlock onto the stack as a string.
+     * Pushes a DocText node onto the stack.
      */
-    Rule DocTextBlock() {
+    Rule DocText() {
         return Sequence(
-            push(curLineNum),
+            push(new DocText(curLineNum)),
-            DocTextLine(), // pushes the value onto the stack
+            OneOrMore(Sequence(
-            ZeroOrMore(Sequence(
+                DocLineStart(), TestNot(AT), RemainingLine(),
-                DocTextLine(),
+                addToDocText(match())))); }
-                swap(),
-                push(popAsString() + popAsString()))),
-            
-            push(makeTextBlock(popAsString(), popAsInt()))); }
 
-    /**
+    Rule DocLineStart() {
-     * Parses the rule:
-     *  DocTextLine =
-     *      DocStart !DIRECTIVE_START RemainingLine
-     *
-     * Pushes the line value (not including the DocStart) onto the stack.
-     */
-    Rule DocTextLine() {
         return Sequence(
-            DocStart(), TestNot(DIRECTIVE_START),
+            ZeroOrMore(SPACE), DOC_LINE_START, Optional(SPACE)); }
-            RemainingLine(), push(match())); }
+
+    Rule NonEmptyLine() {
+        return Sequence(OneOrMore(NOT_EOL), FirstOf(EOL, EOI)); }
 
-    /**
-     * Parses the rule:
-     *  RemainingLine = (!EOL)+ EOL
-     */
-    @SuppressSubnodes
     Rule RemainingLine() {
         return FirstOf(
             Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()),
+            Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); }
 
-            // allow EOI as a line delimiter only if the line is not empty,
+    Rule AT         = Ch('@').label("AT");
-            // otherwise it will match infinitely if RemainingLine is used in a
+    Rule EOL        = FirstOf(String("\r\n"), Ch('\n'), Ch('\r')).label("EOL");
-            // OneOrMore context.
+    Rule NOT_EOL    = Sequence(TestNot(EOL), ANY).label("NOT_EOL");
-            Sequence(OneOrMore(NOT_EOL), EOI)); }    
+    Rule SPACE      = AnyOf(" \t").label("SPACE");
-
+    Rule DOC_LINE_START = String("%%").label("DOC_LINE_START");
-    Rule DOC_START      = String("%% ");
-    Rule EOL            = Ch('\n');
-    Rule NOT_EOL        = Sequence(TestNot(EOL), ANY);
-    Rule DIRECTIVE_START= Ch('@');
-    Rule SLASH          = Ch('/');
-    Rule SPACE          = AnyOf(" \t");
 
     // directive terminals
-    Rule AUTHOR_DIR   = IgnoreCase("author");
+    Rule AUTHOR_DIR     = IgnoreCase("author");
-    Rule DOC_DIR      = IgnoreCase("doc");
+    Rule COPYRIGHT_DIR  = IgnoreCase("copyright");
-    Rule EXAMPLE_DIR  = IgnoreCase("example");
+    Rule API_DIR        = IgnoreCase("api");
-    Rule ORG_DIR      = IgnoreCase("org");
+    Rule EXAMPLE_DIR    = IgnoreCase("example");
+    Rule ORG_DIR        = IgnoreCase("org");
 
     String popAsString() { return (String) pop(); }
 
     Integer popAsInt() { return (Integer) pop(); }
 
-    static <T> List<T> addToList(T value, List<T> list) {
+    boolean clearLineCount() { curLineNum = 1; return true; }
-        list.add(value);
+
-        return list; }
+    boolean incLineCount() { curLineNum++; return true; }
+
+    boolean addBlockToSourceFile(Block block) {
+        SourceFile sourceFile = (SourceFile) pop();
+        sourceFile.blocks.add(block);
+        return push(sourceFile); }
+
+    /**
+     * Pop off a DocBlock, add the given Directive or DocText and push the
+     * DocBlock back onto the stack.
+     */
+    boolean addToDocBlock(ASTNode an) {
+        DocBlock docBlock = (DocBlock) pop();
+        if (an instanceof Directive) {
+            docBlock.directives.add((Directive) an); }
+        else if (an instanceof DocText) {
+            docBlock.docTexts.add((DocText) an); }
+        else { throw new IllegalStateException(); }
+        return push(docBlock); }
+        
+    boolean addToCodeBlock(String line) {
+        CodeBlock codeBlock = (CodeBlock) pop();
+        codeBlock.lines.put(curLineNum - 1, line);
+        return push(codeBlock); }
+
+    boolean addToDocText(String line) {
+        DocText docText = (DocText) pop();
+        docText.value += line;
+        return push(docText); }
 
     boolean printValueStack() {
         for (int i = 0; i < getContext().getValueStack().size(); i++) {
             System.out.println(i + ": " + peek(i)); }
         return true; }
 
-    boolean clearLineCount() { curLineNum = 1; return true; }
-
-    boolean incLineCount() { curLineNum++; return true; }
-
-    boolean echo(String msg) { System.out.println(msg); return true; }
-
-    static DocBlock addDirectiveBlock(Directive dir, DocBlock docBlock) {
-        docBlock.directives.add(dir); return docBlock; }
-
-    static DocBlock addTextBlock(TextBlock tb, DocBlock docBlock) {
-        docBlock.textBlocks.add(tb); return docBlock; }
 }

src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy

@@ -1,7 +1,7 @@
-package com.jdblabs.jlp.experimental
+package com.jdblabs.jlp
 
-import com.jdblabs.jlp.experimental.ast.*
+import com.jdblabs.jlp.ast.*
-import com.jdblabs.jlp.experimental.ast.Directive.DirectiveType
+import com.jdblabs.jlp.ast.Directive.DirectiveType
 
 import org.pegdown.Extensions
 import org.pegdown.PegDownProcessor

src/main/com/jdblabs/jlp/MarkdownGenerator.groovy

@@ -1,65 +0,0 @@
-package com.jdblabs.jlp
-
-import com.jdblabs.jlp.ast.*
-import com.jdblabs.jlp.ast.Directive.DirectiveType
-import com.jdblabs.jlp.ast.TextBlock.TextBlockType
-
-import org.pegdown.PegDownProcessor
-import org.pegdown.Extensions
-
-public class MarkdownGenerator extends JLPBaseGenerator {
-
-    protected PegDownProcessor pegdown
-
-    protected MarkdownGenerator() {
-        super()
-
-        pegdown = new PegDownProcessor(
-            Extensions.TABLES & Extensions.DEFINITIONS) }
-
-    protected static Map<String, String> generateDocuments(
-    Map<String, List<ASTNode>> sources) {
-        MarkdownGenerator inst = new MarkdownGenerator()
-        return inst.generate(sources) }
-
-    protected String emit(TextBlock textBlock) {
-        switch (textBlock.type) {
-
-            // text block, just convert to markdown
-            case TextBlockType.TextBlock:
-                return formatText(textBlock.value)
-
-            // code block, we want to emit as a code snippet
-            case TextBlockType.CodeBlock:
-                // so prepend all lines with four spaces to tell markdown that
-                // this is code
-                String value = textBlock.value.replaceAll(/(^|\n)/, /$1    /)
-                // then convert to markdown
-                return pegdown.markdownToHtml(value) } }
-
-    protected String emit(Directive d) {
-        switch (d.type) {
-            case DirectiveType.Author:
-                return "<span class='author'>Author: ${formatText(d.value)}</span>"
-            case DirectiveType.Doc:
-                return formatText(d.value)
-            case DirectiveType.Example:
-                return formatText(d.value)
-            case DirectiveType.Org:
-                docState.orgs[d.value] = [line: d.lineNumber,
-                                          file: docState.currentDocId]
-                return "<a name='${d.value}'/>" }
-    }
-
-    protected String formatText(String text) {
-
-        // convert to HTML from Markdown
-        String md = pegdown.markdownToHtml(text)
-
-        // replace internal `jlp://` links with actual links based on`@org`
-        // references
-        md = md.replaceAll(/jlp:\/\/([^\s"])/, /#$1/)
-
-        return md;
-    }
-}

src/main/com/jdblabs/jlp/TransparentGenerator.groovy

@@ -1,24 +0,0 @@
-package com.jdblabs.jlp
-
-import com.jdblabs.jlp.ast.*
-import com.jdblabs.jlp.ast.Directive.DirectiveType
-import java.util.List
-import java.util.Map
-
-public class TransparentGenerator extends JLPBaseGenerator {
-
-    protected TransparentGenerator() {}
-
-    public static Map<String, String> generateDocuments(Map<String,
-    List<ASTNode>> sources) {
-        TransparentGenerator inst = new TransparentGenerator()
-        return inst.generate(sources) }
-
-    protected String emit(TextBlock textBlock) { textBlock.value }
-    protected String emit(Directive directive) {
-        switch (directive.type) {
-            case DirectiveType.Author:  return "Author: ${directive.value}\n"
-            case DirectiveType.Doc:     return directive.value
-            case DirectiveType.Example: return "Example: ${directive.value}"
-            case DirectiveType.Org:     return "" } }
-}

src/main/com/jdblabs/jlp/ast/ASTNode.groovy

@@ -1,5 +1,10 @@
 package com.jdblabs.jlp.ast
 
-public interface ASTNode {
+public class ASTNode {
-    public int getLineNumber()
+
+    protected int lineNumber
+
+    public ASTNode(int lineNum) { this.lineNumber = lineNum }
+
+    public int getLineNumber() { lineNumber }
 }

src/main/com/jdblabs/jlp/ast/Block.groovy

@@ -1,4 +1,4 @@
-package com.jdblabs.jlp.experimental.ast
+package com.jdblabs.jlp.ast
 
 public class Block extends ASTNode {
     public final DocBlock docBlock

src/main/com/jdblabs/jlp/ast/CodeBlock.groovy

@@ -1,4 +1,4 @@
-package com.jdblabs.jlp.experimental.ast
+package com.jdblabs.jlp.ast
 
 import java.util.Map
 

src/main/com/jdblabs/jlp/ast/Directive.groovy

@@ -1,10 +1,11 @@
 package com.jdblabs.jlp.ast
 
-public class Directive implements ASTNode {
+public class Directive extends ASTNode {
 
     public static enum DirectiveType {
+        Api,
         Author,
-        Doc,
+        Copyright,
         Example,
         Org;
         
@@ -13,14 +14,11 @@ public class Directive implements ASTNode {
 
     public final DirectiveType type;
     public final String value;
-    public final int lineNumber;
 
     public Directive(String value, String typeString, int lineNumber) {
+        super(lineNumber)
         this.value = value
-        this.type = DirectiveType.parse(typeString)
+        this.type = DirectiveType.parse(typeString) }
-        this.lineNumber = lineNumber }
         
-    public int getLineNumber() { return lineNumber }
+    public String toString() { "[${lineNumber}:Directive: ${type}, ${value}]" }
-
-    public String toString() { return "[Directive(${lineNumber}): ${type}, ${value}]" }
 }

src/main/com/jdblabs/jlp/ast/DocBlock.groovy

@@ -3,16 +3,13 @@ package com.jdblabs.jlp.ast
 import java.util.ArrayList
 import java.util.List
 
-public class DocBlock implements ASTNode {
+public class DocBlock extends ASTNode {
     
-    public final int lineNumber
     public List<Directive> directives = new ArrayList<Directive>()
-    public List<TextBlock> textBlocks = new ArrayList<TextBlock>()
+    public List<DocText> docTexts = new ArrayList<DocText>()
 
-    public DocBlock(int lineNumber) { this.lineNumber = lineNumber }
+    public DocBlock(int lineNumber) { super(lineNumber) }
-
-    public int getLineNumber() { lineNumber }
 
     public String toString() {
-        "[DocBlock: Directives ${directives}, TextBlocks ${textBlocks}]" }
+        "[${lineNumber}:DocBlock: Directives ${directives}, DocTexts ${docTexts}]" }
 }

src/main/com/jdblabs/jlp/ast/DocText.groovy

@@ -1,4 +1,4 @@
-package com.jdblabs.jlp.experimental.ast
+package com.jdblabs.jlp.ast
 
 public class DocText extends ASTNode {
 

src/main/com/jdblabs/jlp/ast/SourceFile.groovy

@@ -1,4 +1,4 @@
-package com.jdblabs.jlp.experimental.ast
+package com.jdblabs.jlp.ast
 
 public class SourceFile {
     public List<ASTNode> blocks = []

src/main/com/jdblabs/jlp/ast/TextBlock.groovy

@@ -1,25 +0,0 @@
-package com.jdblabs.jlp.ast
-
-public class TextBlock implements ASTNode {
-
-    public static enum TextBlockType { TextBlock, CodeBlock }
-
-    public final TextBlockType type
-    public final String value
-    public final int lineNumber
-
-    public TextBlock(TextBlockType type, String value, int lineNumber) {
-        this.type = type
-        this.value = value
-        this.lineNumber = lineNumber }
-
-    public int getLineNumber() { return lineNumber }
-
-    public String toString() { return "[${type}(${lineNumber}): ${value}]" }
-
-    public static TextBlock makeTextBlock(String value, int lineNumber) {
-        return new TextBlock(TextBlockType.TextBlock, value, lineNumber) }
-
-    public static TextBlock makeCodeBlock(String value, int lineNumber) {
-        return new TextBlock(TextBlockType.CodeBlock, value, lineNumber) }
-}

src/main/com/jdblabs/jlp/experimental/DeadSimple.java

@@ -1,23 +0,0 @@
-package com.jdblabs.jlp.experimental;
-
-import org.parboiled.Action;
-import org.parboiled.BaseParser;
-import org.parboiled.Context;
-import org.parboiled.Rule;
-import org.parboiled.annotations.*;
-
-@BuildParseTree
-public class DeadSimple extends BaseParser {
-
-    public Rule S() {
-        return OneOrMore(FirstOf(Line(), EmptyLine()).skipNode()); }
-
-    public Rule Line() {
-        return Sequence(OneOrMore(NOT_EOL).skipNode(), EOL); }
-
-    public Rule EmptyLine() {
-        return EOL; }
-
-    public Rule EOL = Ch('\n');
-    public Rule NOT_EOL = Sequence(TestNot(EOL), ANY);
-}

src/main/com/jdblabs/jlp/experimental/JLPBaseGenerator.groovy

@@ -1,62 +0,0 @@
-package com.jdblabs.jlp.experimental
-
-import com.jdblabs.jlp.experimental.ast.*
-import java.util.List
-import java.util.Map
-
-public abstract class JLPBaseGenerator {
-
-    protected Map docState
-
-    protected JLPBaseGenerator() {
-        docState = [orgs:           [:],        // stores `@org` references
-                    codeTrees:      [:],        // stores code ASTs for
-                    currentDocId:   false ] }   // store the current docid
-
-    protected Map<String, String> generate(Map<String, SourceFile> sources) {
-        Map result = [:]
-
-        // run the parse phase
-        sources.each { sourceId, sourceAST ->
-
-            // set up the current generator state for this source
-            docState.currentDocId = sourceId
-            docState.codeTrees[sourceId] = sourceAST.codeAST
-
-            parse(sourceAST) }
-
-        // run the emit phase
-        sources.each { sourceId, sourceAST ->
-
-            // set up the current generator state for this source
-            docState.currentDocId = sourceId
-
-            // generate the doc for this source
-            result[sourceId] = emit(sourceAST) }
-
-        // return our results
-        return result }
-
-    protected void parse(SourceFile sourceFile) {
-        sourceFile.blocks.each { block -> parse(block) } }
-
-    protected void parse(Block block) {
-        parse(block.docBlock)
-        parse(block.codeBlock) }
-
-    protected void parse(DocBlock docBlock) {
-        docBlock.directives.each { directive -> parse(directive) }
-        docBlock.docTexts.each { docText -> parse(docText) } }
-
-    protected abstract void parse(Directive directive)
-    protected abstract void parse(CodeBlock codeBlock)
-    protected abstract void parse(DocText docText)
-
-    protected abstract String emit(SourceFile sourceFile)
-    protected abstract String emit(Block block)
-    protected abstract String emit(DocBlock docBlock)
-    protected abstract String emit(CodeBlock codeBlock)
-    protected abstract String emit(DocText docText)
-    protected abstract String emit(Directive directive)
-
-}

src/main/com/jdblabs/jlp/experimental/JLPPegParser.java

@@ -1,250 +0,0 @@
-package com.jdblabs.jlp.experimental;
-
-import com.jdblabs.jlp.experimental.ast.*;
-import java.util.ArrayList;
-import java.util.List;
-import org.parboiled.Action;
-import org.parboiled.BaseParser;
-import org.parboiled.Context;
-import org.parboiled.Rule;
-import org.parboiled.annotations.*;
-
-@BuildParseTree
-public class JLPPegParser extends BaseParser<Object> {
-
-    int curLineNum = 1;
-
-    /**
-     * Parses the rule:
-     *  SourceFile = (Block / DocBlock / CodeBlock)+
-     *
-     * Pushes a SourceFile node on the stack.
-     */
-    public Rule SourceFile() {
-        return Sequence(
-            // At the start of processing a new SourceFile, we need to set up
-            // our internal state. 
-
-            // Clear the line count.
-            clearLineCount(),
-
-            // Add the top-level SourceFile AST node.
-            push(new SourceFile()),
-
-            // A SourceFile is made up of one or more Blocks
-            OneOrMore(Sequence(
-                // All of these options result in one new Block pushed onto the
-                // stack.
-                FirstOf(
-
-                    // Match a whole Block. This pushes a Block on the stack.
-                    Block(),    
-
-                    // A standalone DocBlock. We will create an empty CodeBlock
-                    // to pair with it, then push a new Block onto the stack
-                    // made from the DocBlock and the empty CodeBlock
-                    Sequence(
-                        // 1. We need to remember the line number to create the
-                        // Block
-                        push(curLineNum),
-
-                        // 2. Match the DocBlock.
-                        DocBlock(),
-
-                        // 3. Create the empty CodeBlock.
-                        push(new CodeBlock(curLineNum)),
-
-                        // 4. Create the Block and push it onto the stack.
-                        push(new Block((CodeBlock) pop(), (DocBlock) pop(),
-                            popAsInt()))),
-
-                    // A standalone CodeBlock. Similar to the standalone
-                    // DocBlock, we will create an empty DocBlock to pair with
-                    // the CodeBlock to make a Block, which is pushed onto the
-                    // stack:
-                    //
-                    // *Note: With the way the parser is currently written,
-                    //        this will only match a CodeBlock that occurs
-                    //        before any DocBlock.*
-                    Sequence(
-                        // 1. Remember the line number for the Block.
-                        push(curLineNum),
-
-                        // 2. Create the empty DocBlock.
-                        push(new DocBlock(curLineNum)),
-
-                        // 3. Match the CodeBlock
-                        CodeBlock(),
-
-                        // 4. Create the Block and push it onto the stack
-                        push(new Block((CodeBlock) pop(), (DocBlock) pop(),
-                            popAsInt())))),
-
-                // pop the Block created by one of the above options and add it
-                // to the SourceFile
-                addBlockToSourceFile((Block) pop())))); }
-
-    /**
-     * Parses the rule:
-     *   Block = DocBlock CodeBlock
-     *
-     * Pushes a Block onto the stack
-     */
-    Rule Block() {
-        return Sequence(
-            push(curLineNum),
-            DocBlock(), CodeBlock(),
-
-            push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); }
-
-    /**
-     * Parses the rule:
-     *  DocBlock = (Directive / DocText)+
-     *
-     * Pushes a DocBlock object onto the stack
-     */
-    Rule DocBlock() {
-        return Sequence(
-            push(new DocBlock(curLineNum)),
-            OneOrMore(Sequence(
-                FirstOf(Directive(), DocText()),
-                addToDocBlock((ASTNode) pop())))); }
-
-    /**
-     * Parses the rule:
-     *  CodeBlock = (!DocLineStart RemainingLine)+
-     *
-     * Pushes a CodeBlock onto the stack.
-     */
-    Rule CodeBlock() {
-        return Sequence(
-            push(new CodeBlock(curLineNum)),
-            OneOrMore(Sequence(
-                TestNot(DocLineStart()), RemainingLine(),
-                addToCodeBlock(match())))); }
-
-    /**
-     * Parses the rule:
-     *  Directive = DocLineStart AT (LongDirective / ShortDirective)
-     *
-     * Pushes a Directive node on the stack.
-     */
-    Rule Directive() {
-        return Sequence(
-            DocLineStart(), AT, FirstOf(LongDirective(), ShortDirective())); }
-
-    /**
-     * Parses the rule:
-     *  LongDirective =
-     *      (API_DIR / EXAMPLE_DIR) RemainingLine DocText?
-     *
-     * Pushes a Directive node onto the stack.
-     */
-    Rule LongDirective() {
-        return Sequence(
-            push(curLineNum),
-
-            FirstOf(API_DIR, EXAMPLE_DIR),  push(match()),
-            RemainingLine(),                            push(match()),
-
-            Optional(Sequence(
-                DocText(),
-                swap(),
-                push(popAsString() + ((DocText) pop()).value))),
-                
-            push(new Directive(popAsString(), popAsString(), popAsInt()))); }
-
-    /**
-     * Parses the rule:
-     *  ShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingLine
-     *
-     * Pushes a Directive node onto the stack.
-     */
-    Rule ShortDirective() {
-        return Sequence(
-            push(curLineNum),
-            FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()),
-            RemainingLine(),
-            
-            push(new Directive(match().trim(), popAsString(), popAsInt()))); }
-
-    /**
-     * Parses the rule:
-     *  DocText = (DocLineStart !AT RemainingLine)+
-     *
-     * Pushes a DocText node onto the stack.
-     */
-    Rule DocText() {
-        return Sequence(
-            push(new DocText(curLineNum)),
-            OneOrMore(Sequence(
-                DocLineStart(), TestNot(AT), RemainingLine(),
-                addToDocText(match())))); }
-
-    Rule DocLineStart() {
-        return Sequence(
-            ZeroOrMore(SPACE), DOC_LINE_START, Optional(SPACE)); }
-
-    Rule NonEmptyLine() {
-        return Sequence(OneOrMore(NOT_EOL), FirstOf(EOL, EOI)); }
-
-    Rule RemainingLine() {
-        return FirstOf(
-            Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()),
-            Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); }
-
-    Rule AT         = Ch('@').label("AT");
-    Rule EOL        = FirstOf(String("\r\n"), Ch('\n'), Ch('\r')).label("EOL");
-    Rule NOT_EOL    = Sequence(TestNot(EOL), ANY).label("NOT_EOL");
-    Rule SPACE      = AnyOf(" \t").label("SPACE");
-    Rule DOC_LINE_START = String("%%").label("DOC_LINE_START");
-
-    // directive terminals
-    Rule AUTHOR_DIR     = IgnoreCase("author");
-    Rule COPYRIGHT_DIR  = IgnoreCase("copyright");
-    Rule API_DIR        = IgnoreCase("api");
-    Rule EXAMPLE_DIR    = IgnoreCase("example");
-    Rule ORG_DIR        = IgnoreCase("org");
-
-    String popAsString() { return (String) pop(); }
-
-    Integer popAsInt() { return (Integer) pop(); }
-
-    boolean clearLineCount() { curLineNum = 1; return true; }
-
-    boolean incLineCount() { curLineNum++; return true; }
-
-    boolean addBlockToSourceFile(Block block) {
-        SourceFile sourceFile = (SourceFile) pop();
-        sourceFile.blocks.add(block);
-        return push(sourceFile); }
-
-    /**
-     * Pop off a DocBlock, add the given Directive or DocText and push the
-     * DocBlock back onto the stack.
-     */
-    boolean addToDocBlock(ASTNode an) {
-        DocBlock docBlock = (DocBlock) pop();
-        if (an instanceof Directive) {
-            docBlock.directives.add((Directive) an); }
-        else if (an instanceof DocText) {
-            docBlock.docTexts.add((DocText) an); }
-        else { throw new IllegalStateException(); }
-        return push(docBlock); }
-        
-    boolean addToCodeBlock(String line) {
-        CodeBlock codeBlock = (CodeBlock) pop();
-        codeBlock.lines.put(curLineNum - 1, line);
-        return push(codeBlock); }
-
-    boolean addToDocText(String line) {
-        DocText docText = (DocText) pop();
-        docText.value += line;
-        return push(docText); }
-
-    boolean printValueStack() {
-        for (int i = 0; i < getContext().getValueStack().size(); i++) {
-            System.out.println(i + ": " + peek(i)); }
-        return true; }
-
-}

src/main/com/jdblabs/jlp/experimental/ast/ASTNode.groovy

@@ -1,10 +0,0 @@
-package com.jdblabs.jlp.experimental.ast
-
-public class ASTNode {
-
-    protected int lineNumber
-
-    public ASTNode(int lineNum) { this.lineNumber = lineNum }
-
-    public int getLineNumber() { lineNumber }
-}

src/main/com/jdblabs/jlp/experimental/ast/Directive.groovy

@@ -1,24 +0,0 @@
-package com.jdblabs.jlp.experimental.ast
-
-public class Directive extends ASTNode {
-
-    public static enum DirectiveType {
-        Api,
-        Author,
-        Copyright,
-        Example,
-        Org;
-        
-        public static DirectiveType parse(String typeString) {
-            valueOf(typeString.toLowerCase().capitalize()) } }
-
-    public final DirectiveType type;
-    public final String value;
-
-    public Directive(String value, String typeString, int lineNumber) {
-        super(lineNumber)
-        this.value = value
-        this.type = DirectiveType.parse(typeString) }
-        
-    public String toString() { "[${lineNumber}:Directive: ${type}, ${value}]" }
-}

src/main/com/jdblabs/jlp/experimental/ast/DocBlock.groovy

@@ -1,15 +0,0 @@
-package com.jdblabs.jlp.experimental.ast
-
-import java.util.ArrayList
-import java.util.List
-
-public class DocBlock extends ASTNode {
-    
-    public List<Directive> directives = new ArrayList<Directive>()
-    public List<DocText> docTexts = new ArrayList<DocText>()
-
-    public DocBlock(int lineNumber) { super(lineNumber) }
-
-    public String toString() {
-        "[${lineNumber}:DocBlock: Directives ${directives}, DocTexts ${docTexts}]" }
-}