From bab19431206c7f50fa2b616ff4be6edd82db64f1 Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Tue, 6 Sep 2011 16:26:36 -0500 Subject: [PATCH] Promoting the experimental code to replace the exising main code. --- resources/main/test.groovy | 49 +--- .../com/jdblabs/jlp/JLPBaseGenerator.groovy | 81 +++--- src/main/com/jdblabs/jlp/JLPPegParser.java | 265 +++++++++++------- .../LiterateMarkdownGenerator.groovy | 6 +- .../com/jdblabs/jlp/MarkdownGenerator.groovy | 65 ----- .../jdblabs/jlp/TransparentGenerator.groovy | 24 -- src/main/com/jdblabs/jlp/ast/ASTNode.groovy | 9 +- .../jlp/{experimental => }/ast/Block.groovy | 2 +- .../{experimental => }/ast/CodeBlock.groovy | 2 +- src/main/com/jdblabs/jlp/ast/Directive.groovy | 14 +- src/main/com/jdblabs/jlp/ast/DocBlock.groovy | 11 +- .../jlp/{experimental => }/ast/DocText.groovy | 2 +- .../{experimental => }/ast/SourceFile.groovy | 2 +- src/main/com/jdblabs/jlp/ast/TextBlock.groovy | 25 -- .../jdblabs/jlp/experimental/DeadSimple.java | 23 -- .../jlp/experimental/JLPBaseGenerator.groovy | 62 ---- .../jlp/experimental/JLPPegParser.java | 250 ----------------- .../jlp/experimental/ast/ASTNode.groovy | 10 - .../jlp/experimental/ast/Directive.groovy | 24 -- .../jlp/experimental/ast/DocBlock.groovy | 15 - 20 files changed, 232 insertions(+), 709 deletions(-) rename src/main/com/jdblabs/jlp/{experimental => }/LiterateMarkdownGenerator.groovy (97%) delete mode 100644 src/main/com/jdblabs/jlp/MarkdownGenerator.groovy delete mode 100644 src/main/com/jdblabs/jlp/TransparentGenerator.groovy rename src/main/com/jdblabs/jlp/{experimental => }/ast/Block.groovy (88%) rename src/main/com/jdblabs/jlp/{experimental => }/ast/CodeBlock.groovy (90%) rename src/main/com/jdblabs/jlp/{experimental => }/ast/DocText.groovy (82%) rename src/main/com/jdblabs/jlp/{experimental => }/ast/SourceFile.groovy (73%) delete mode 100644 src/main/com/jdblabs/jlp/ast/TextBlock.groovy delete mode 100644 src/main/com/jdblabs/jlp/experimental/DeadSimple.java delete mode 100644 src/main/com/jdblabs/jlp/experimental/JLPBaseGenerator.groovy delete mode 100644 src/main/com/jdblabs/jlp/experimental/JLPPegParser.java delete mode 100644 src/main/com/jdblabs/jlp/experimental/ast/ASTNode.groovy delete mode 100644 src/main/com/jdblabs/jlp/experimental/ast/Directive.groovy delete mode 100644 src/main/com/jdblabs/jlp/experimental/ast/DocBlock.groovy diff --git a/resources/main/test.groovy b/resources/main/test.groovy index 56497c5..98e11d1 100644 --- a/resources/main/test.groovy +++ b/resources/main/test.groovy @@ -1,26 +1,16 @@ import com.jdblabs.jlp.* -import com.jdblabs.jlp.experimental.LiterateMarkdownGenerator import org.parboiled.Parboiled import org.parboiled.parserunners.ReportingParseRunner import org.parboiled.parserunners.RecoveringParseRunner -makeParser = { - println "Making the standard parser." - println "---------------------------" +"Making the standard parser." +"---------------------------" - parser = Parboiled.createParser(JLPPegParser.class) - parseRunner = new ReportingParseRunner(parser.SourceFile()) -} - -makeExperimentalParser = { - println "Making the experimental parser." - println "-------------------------------" - - parser = Parboiled.createParser(com.jdblabs.jlp.experimental.JLPPegParser.class) - parseRunner = new ReportingParseRunner(parser.SourceFile()) -} +parser = Parboiled.createParser(JLPPegParser.class) +parseRunner = new ReportingParseRunner(parser.SourceFile()) simpleTest = { + println "Parsing the simple test into 'result'." println "--------------------------------------" @@ -46,33 +36,12 @@ vbsTest = { vbsParsed = parseRunner.run(vbsTestInput) - vbsResult = MarkdownGenerator.generateDocuments([vbs: vbsParsed.resultValue]).vbs + vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl" - println "Writing to file 'vbs_result.html'." - println "----------------------------------" + println "Writing to file 'vbs_db_records.html'." + println "--------------------------------------" - (new File('vbs_result.html')).withWriter { out -> out.println vbsResult } + (new File('vbs_db_records.html')).withWriter { out -> out.println vbsResult } return [vbsParsed, vbsResult] } - -experimentalTest = { - makeExperimentalParser() - - println "Parsing vbs_db_records.hrl into 'vbsResult'." - println "--------------------------------------------" - - vbsTestFile = new File('vbs_db_records.hrl') - println "vbsTestFile is ${vbsTestFile.exists() ? 'present' : 'absent'}." - vbsTestInput = vbsTestFile.text - - vbsParsed = parseRunner.run(vbsTestInput) - - vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl" - - println "Writing to file 'vbs_result.html'." - println "----------------------------------" - - (new File('vbs_result.html')).withWriter { out -> out.println vbsResult } - -} diff --git a/src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy b/src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy index 131143a..897dd35 100644 --- a/src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy +++ b/src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy @@ -1,7 +1,6 @@ package com.jdblabs.jlp import com.jdblabs.jlp.ast.* -import com.jdblabs.jlp.ast.Directive.DirectiveType import java.util.List import java.util.Map @@ -10,54 +9,54 @@ public abstract class JLPBaseGenerator { protected Map docState protected JLPBaseGenerator() { - docState = [orgs: [:], - currentDocId: false ] } + docState = [orgs: [:], // stores `@org` references + codeTrees: [:], // stores code ASTs for + currentDocId: false ] } // store the current docid - protected Map generate(Map> sources) { + protected Map generate(Map sources) { Map result = [:] - sources.each { sourceId, sourceNodes -> + + // run the parse phase + sources.each { sourceId, sourceAST -> + + // set up the current generator state for this source docState.currentDocId = sourceId - result[sourceId] = emitDocument(sourceNodes) } + docState.codeTrees[sourceId] = sourceAST.codeAST + + parse(sourceAST) } + + // run the emit phase + sources.each { sourceId, sourceAST -> + + // set up the current generator state for this source + docState.currentDocId = sourceId + + // generate the doc for this source + result[sourceId] = emit(sourceAST) } + + // return our results return result } - protected String emitDocument(List sourceNodes) { - StringBuilder result = - sourceNodes.inject(new StringBuilder()) { sb, node -> - sb.append(emit(node)) - return sb } + protected void parse(SourceFile sourceFile) { + sourceFile.blocks.each { block -> parse(block) } } - return result.toString() } + protected void parse(Block block) { + parse(block.docBlock) + parse(block.codeBlock) } - protected String emit(DocBlock docBlock) { - List printQueue - StringBuilder result + protected void parse(DocBlock docBlock) { + docBlock.directives.each { directive -> parse(directive) } + docBlock.docTexts.each { docText -> parse(docText) } } - printQueue = docBlock.directives.collect { directive -> - def queueItem = [line: directive.lineNumber, value: directive] - switch (directive.type) { - case DirectiveType.Author: queueItem.priority = 50; break - case DirectiveType.Doc: queueItem.priority = 50; break - case DirectiveType.Example: queueItem.priority = 50; break - case DirectiveType.Org: queueItem.priority = 10; break } + protected abstract void parse(Directive directive) + protected abstract void parse(CodeBlock codeBlock) + protected abstract void parse(DocText docText) - return queueItem } - - printQueue.addAll(docBlock.textBlocks.collect { textBlock -> - [ priority: 50, line: textBlock.lineNumber, value: textBlock ] }) - - // sort by priority, then by line number - printQueue.sort( - {i1, i2 -> i1.priority != i2.priority ? - i1.priority - i2.priority : - i1.line - i2.line} as Comparator) - - - result = printQueue.inject(new StringBuilder()) { sb, printable -> - sb.append(emit(printable.value)) - return sb } - - return result.toString() } - - protected abstract String emit(TextBlock textBlock) + protected abstract String emit(SourceFile sourceFile) + protected abstract String emit(Block block) + protected abstract String emit(DocBlock docBlock) + protected abstract String emit(CodeBlock codeBlock) + protected abstract String emit(DocText docText) protected abstract String emit(Directive directive) + } diff --git a/src/main/com/jdblabs/jlp/JLPPegParser.java b/src/main/com/jdblabs/jlp/JLPPegParser.java index 630105f..088016e 100644 --- a/src/main/com/jdblabs/jlp/JLPPegParser.java +++ b/src/main/com/jdblabs/jlp/JLPPegParser.java @@ -9,189 +9,242 @@ import org.parboiled.Context; import org.parboiled.Rule; import org.parboiled.annotations.*; -import static com.jdblabs.jlp.ast.TextBlock.makeCodeBlock; -import static com.jdblabs.jlp.ast.TextBlock.makeTextBlock; - @BuildParseTree public class JLPPegParser extends BaseParser { int curLineNum = 1; + /** + * Parses the rule: + * SourceFile = (Block / DocBlock / CodeBlock)+ + * + * Pushes a SourceFile node on the stack. + */ public Rule SourceFile() { return Sequence( + // At the start of processing a new SourceFile, we need to set up + // our internal state. + + // Clear the line count. clearLineCount(), - push(new ArrayList()), - ZeroOrMore(Sequence( + + // Add the top-level SourceFile AST node. + push(new SourceFile()), + + // A SourceFile is made up of one or more Blocks + OneOrMore(Sequence( + // All of these options result in one new Block pushed onto the + // stack. FirstOf( - DocBlock(), - CodeBlock()), - push(addToList(pop(), (List)pop()))))); } + + // Match a whole Block. This pushes a Block on the stack. + Block(), + + // A standalone DocBlock. We will create an empty CodeBlock + // to pair with it, then push a new Block onto the stack + // made from the DocBlock and the empty CodeBlock + Sequence( + // 1. We need to remember the line number to create the + // Block + push(curLineNum), + + // 2. Match the DocBlock. + DocBlock(), + + // 3. Create the empty CodeBlock. + push(new CodeBlock(curLineNum)), + + // 4. Create the Block and push it onto the stack. + push(new Block((CodeBlock) pop(), (DocBlock) pop(), + popAsInt()))), + + // A standalone CodeBlock. Similar to the standalone + // DocBlock, we will create an empty DocBlock to pair with + // the CodeBlock to make a Block, which is pushed onto the + // stack: + // + // *Note: With the way the parser is currently written, + // this will only match a CodeBlock that occurs + // before any DocBlock.* + Sequence( + // 1. Remember the line number for the Block. + push(curLineNum), + + // 2. Create the empty DocBlock. + push(new DocBlock(curLineNum)), + + // 3. Match the CodeBlock + CodeBlock(), + + // 4. Create the Block and push it onto the stack + push(new Block((CodeBlock) pop(), (DocBlock) pop(), + popAsInt())))), + + // pop the Block created by one of the above options and add it + // to the SourceFile + addBlockToSourceFile((Block) pop())))); } /** * Parses the rule: - * DocBlock = (DirectiveBlock / DocTextBlock)+ + * Block = DocBlock CodeBlock * - * Pushes a DocBlock object onto the stack. + * Pushes a Block onto the stack + */ + Rule Block() { + return Sequence( + push(curLineNum), + DocBlock(), CodeBlock(), + + push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); } + + /** + * Parses the rule: + * DocBlock = (Directive / DocText)+ + * + * Pushes a DocBlock object onto the stack */ Rule DocBlock() { return Sequence( push(new DocBlock(curLineNum)), - OneOrMore( - FirstOf( - Sequence(DirectiveBlock(), - push(addDirectiveBlock((Directive) pop(), (DocBlock) pop()))), - Sequence(DocTextBlock(), - push(addTextBlock((TextBlock) pop(), (DocBlock) pop())))))); } + OneOrMore(Sequence( + FirstOf(Directive(), DocText()), + addToDocBlock((ASTNode) pop())))); } /** * Parses the rule: - * CodeBlock = !DocStart RemainingLine + * CodeBlock = (!DocLineStart RemainingLine)+ * * Pushes a CodeBlock onto the stack. */ Rule CodeBlock() { return Sequence( - push(curLineNum), - push(""), + push(new CodeBlock(curLineNum)), OneOrMore(Sequence( - TestNot(DocStart()), RemainingLine(), - push(popAsString() + match()))), + TestNot(DocLineStart()), RemainingLine(), + addToCodeBlock(match())))); } - push(makeCodeBlock(popAsString(),popAsInt()))); } - /** * Parses the rule: - * DocStart = SPACE* DOC_START - */ - Rule DocStart() { - return Sequence(ZeroOrMore(SPACE), DOC_START); } - - /** - * Parses the rule: - * DirectiveBlock = - * DocStart DIRECTIVE_START (LongDirective / LineDirective) + * Directive = DocLineStart AT (LongDirective / ShortDirective) * - * Pushes a Directive onto the stack. + * Pushes a Directive node on the stack. */ - Rule DirectiveBlock() { + Rule Directive() { return Sequence( - DocStart(), DIRECTIVE_START, - FirstOf(LongDirective(), LineDirective())); } + DocLineStart(), AT, FirstOf(LongDirective(), ShortDirective())); } /** * Parses the rule: * LongDirective = - * (AUTHOR_DIR / DOC_DIR / EXAMPLE_DIR) RemainingLine DocTextBlock? - * - * Pushes a Directive object onto the value stack. + * (API_DIR / EXAMPLE_DIR) RemainingLine DocText? + * + * Pushes a Directive node onto the stack. */ Rule LongDirective() { return Sequence( push(curLineNum), - FirstOf(AUTHOR_DIR, DOC_DIR, EXAMPLE_DIR), push(match()), + + FirstOf(API_DIR, EXAMPLE_DIR), push(match()), RemainingLine(), push(match()), + Optional(Sequence( - DocTextBlock(), // pushes block + DocText(), swap(), - push(popAsString() + ((TextBlock) pop()).value))), - - // pull off the value, type and create the directive + push(popAsString() + ((DocText) pop()).value))), + push(new Directive(popAsString(), popAsString(), popAsInt()))); } /** * Parses the rule: - * LineDirective = - * ORG_DIR RemainingLine + * ShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingLine * - * Pushes a Directive object onto the value stack. + * Pushes a Directive node onto the stack. */ - Rule LineDirective() { + Rule ShortDirective() { return Sequence( push(curLineNum), - ORG_DIR, push(match()), + FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()), RemainingLine(), - - // pull off the value, type and create the directive + push(new Directive(match().trim(), popAsString(), popAsInt()))); } /** * Parses the rule: - * DocTextBlock = DocTextLine+ + * DocText = (DocLineStart !AT RemainingLine)+ * - * Pushes a DocTextBlock onto the stack as a string. + * Pushes a DocText node onto the stack. */ - Rule DocTextBlock() { + Rule DocText() { return Sequence( - push(curLineNum), - DocTextLine(), // pushes the value onto the stack - ZeroOrMore(Sequence( - DocTextLine(), - swap(), - push(popAsString() + popAsString()))), - - push(makeTextBlock(popAsString(), popAsInt()))); } + push(new DocText(curLineNum)), + OneOrMore(Sequence( + DocLineStart(), TestNot(AT), RemainingLine(), + addToDocText(match())))); } - /** - * Parses the rule: - * DocTextLine = - * DocStart !DIRECTIVE_START RemainingLine - * - * Pushes the line value (not including the DocStart) onto the stack. - */ - Rule DocTextLine() { + Rule DocLineStart() { return Sequence( - DocStart(), TestNot(DIRECTIVE_START), - RemainingLine(), push(match())); } + ZeroOrMore(SPACE), DOC_LINE_START, Optional(SPACE)); } + + Rule NonEmptyLine() { + return Sequence(OneOrMore(NOT_EOL), FirstOf(EOL, EOI)); } - /** - * Parses the rule: - * RemainingLine = (!EOL)+ EOL - */ - @SuppressSubnodes Rule RemainingLine() { return FirstOf( Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()), + Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); } - // allow EOI as a line delimiter only if the line is not empty, - // otherwise it will match infinitely if RemainingLine is used in a - // OneOrMore context. - Sequence(OneOrMore(NOT_EOL), EOI)); } - - Rule DOC_START = String("%% "); - Rule EOL = Ch('\n'); - Rule NOT_EOL = Sequence(TestNot(EOL), ANY); - Rule DIRECTIVE_START= Ch('@'); - Rule SLASH = Ch('/'); - Rule SPACE = AnyOf(" \t"); + Rule AT = Ch('@').label("AT"); + Rule EOL = FirstOf(String("\r\n"), Ch('\n'), Ch('\r')).label("EOL"); + Rule NOT_EOL = Sequence(TestNot(EOL), ANY).label("NOT_EOL"); + Rule SPACE = AnyOf(" \t").label("SPACE"); + Rule DOC_LINE_START = String("%%").label("DOC_LINE_START"); // directive terminals - Rule AUTHOR_DIR = IgnoreCase("author"); - Rule DOC_DIR = IgnoreCase("doc"); - Rule EXAMPLE_DIR = IgnoreCase("example"); - Rule ORG_DIR = IgnoreCase("org"); + Rule AUTHOR_DIR = IgnoreCase("author"); + Rule COPYRIGHT_DIR = IgnoreCase("copyright"); + Rule API_DIR = IgnoreCase("api"); + Rule EXAMPLE_DIR = IgnoreCase("example"); + Rule ORG_DIR = IgnoreCase("org"); String popAsString() { return (String) pop(); } Integer popAsInt() { return (Integer) pop(); } - static List addToList(T value, List list) { - list.add(value); - return list; } + boolean clearLineCount() { curLineNum = 1; return true; } + + boolean incLineCount() { curLineNum++; return true; } + + boolean addBlockToSourceFile(Block block) { + SourceFile sourceFile = (SourceFile) pop(); + sourceFile.blocks.add(block); + return push(sourceFile); } + + /** + * Pop off a DocBlock, add the given Directive or DocText and push the + * DocBlock back onto the stack. + */ + boolean addToDocBlock(ASTNode an) { + DocBlock docBlock = (DocBlock) pop(); + if (an instanceof Directive) { + docBlock.directives.add((Directive) an); } + else if (an instanceof DocText) { + docBlock.docTexts.add((DocText) an); } + else { throw new IllegalStateException(); } + return push(docBlock); } + + boolean addToCodeBlock(String line) { + CodeBlock codeBlock = (CodeBlock) pop(); + codeBlock.lines.put(curLineNum - 1, line); + return push(codeBlock); } + + boolean addToDocText(String line) { + DocText docText = (DocText) pop(); + docText.value += line; + return push(docText); } boolean printValueStack() { for (int i = 0; i < getContext().getValueStack().size(); i++) { System.out.println(i + ": " + peek(i)); } return true; } - boolean clearLineCount() { curLineNum = 1; return true; } - - boolean incLineCount() { curLineNum++; return true; } - - boolean echo(String msg) { System.out.println(msg); return true; } - - static DocBlock addDirectiveBlock(Directive dir, DocBlock docBlock) { - docBlock.directives.add(dir); return docBlock; } - - static DocBlock addTextBlock(TextBlock tb, DocBlock docBlock) { - docBlock.textBlocks.add(tb); return docBlock; } } diff --git a/src/main/com/jdblabs/jlp/experimental/LiterateMarkdownGenerator.groovy b/src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy similarity index 97% rename from src/main/com/jdblabs/jlp/experimental/LiterateMarkdownGenerator.groovy rename to src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy index c877f69..0422f86 100644 --- a/src/main/com/jdblabs/jlp/experimental/LiterateMarkdownGenerator.groovy +++ b/src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy @@ -1,7 +1,7 @@ -package com.jdblabs.jlp.experimental +package com.jdblabs.jlp -import com.jdblabs.jlp.experimental.ast.* -import com.jdblabs.jlp.experimental.ast.Directive.DirectiveType +import com.jdblabs.jlp.ast.* +import com.jdblabs.jlp.ast.Directive.DirectiveType import org.pegdown.Extensions import org.pegdown.PegDownProcessor diff --git a/src/main/com/jdblabs/jlp/MarkdownGenerator.groovy b/src/main/com/jdblabs/jlp/MarkdownGenerator.groovy deleted file mode 100644 index 9ba8d21..0000000 --- a/src/main/com/jdblabs/jlp/MarkdownGenerator.groovy +++ /dev/null @@ -1,65 +0,0 @@ -package com.jdblabs.jlp - -import com.jdblabs.jlp.ast.* -import com.jdblabs.jlp.ast.Directive.DirectiveType -import com.jdblabs.jlp.ast.TextBlock.TextBlockType - -import org.pegdown.PegDownProcessor -import org.pegdown.Extensions - -public class MarkdownGenerator extends JLPBaseGenerator { - - protected PegDownProcessor pegdown - - protected MarkdownGenerator() { - super() - - pegdown = new PegDownProcessor( - Extensions.TABLES & Extensions.DEFINITIONS) } - - protected static Map generateDocuments( - Map> sources) { - MarkdownGenerator inst = new MarkdownGenerator() - return inst.generate(sources) } - - protected String emit(TextBlock textBlock) { - switch (textBlock.type) { - - // text block, just convert to markdown - case TextBlockType.TextBlock: - return formatText(textBlock.value) - - // code block, we want to emit as a code snippet - case TextBlockType.CodeBlock: - // so prepend all lines with four spaces to tell markdown that - // this is code - String value = textBlock.value.replaceAll(/(^|\n)/, /$1 /) - // then convert to markdown - return pegdown.markdownToHtml(value) } } - - protected String emit(Directive d) { - switch (d.type) { - case DirectiveType.Author: - return "Author: ${formatText(d.value)}" - case DirectiveType.Doc: - return formatText(d.value) - case DirectiveType.Example: - return formatText(d.value) - case DirectiveType.Org: - docState.orgs[d.value] = [line: d.lineNumber, - file: docState.currentDocId] - return "" } - } - - protected String formatText(String text) { - - // convert to HTML from Markdown - String md = pegdown.markdownToHtml(text) - - // replace internal `jlp://` links with actual links based on`@org` - // references - md = md.replaceAll(/jlp:\/\/([^\s"])/, /#$1/) - - return md; - } -} diff --git a/src/main/com/jdblabs/jlp/TransparentGenerator.groovy b/src/main/com/jdblabs/jlp/TransparentGenerator.groovy deleted file mode 100644 index 802cda5..0000000 --- a/src/main/com/jdblabs/jlp/TransparentGenerator.groovy +++ /dev/null @@ -1,24 +0,0 @@ -package com.jdblabs.jlp - -import com.jdblabs.jlp.ast.* -import com.jdblabs.jlp.ast.Directive.DirectiveType -import java.util.List -import java.util.Map - -public class TransparentGenerator extends JLPBaseGenerator { - - protected TransparentGenerator() {} - - public static Map generateDocuments(Map> sources) { - TransparentGenerator inst = new TransparentGenerator() - return inst.generate(sources) } - - protected String emit(TextBlock textBlock) { textBlock.value } - protected String emit(Directive directive) { - switch (directive.type) { - case DirectiveType.Author: return "Author: ${directive.value}\n" - case DirectiveType.Doc: return directive.value - case DirectiveType.Example: return "Example: ${directive.value}" - case DirectiveType.Org: return "" } } -} diff --git a/src/main/com/jdblabs/jlp/ast/ASTNode.groovy b/src/main/com/jdblabs/jlp/ast/ASTNode.groovy index 6e66126..d0ace5b 100644 --- a/src/main/com/jdblabs/jlp/ast/ASTNode.groovy +++ b/src/main/com/jdblabs/jlp/ast/ASTNode.groovy @@ -1,5 +1,10 @@ package com.jdblabs.jlp.ast -public interface ASTNode { - public int getLineNumber() +public class ASTNode { + + protected int lineNumber + + public ASTNode(int lineNum) { this.lineNumber = lineNum } + + public int getLineNumber() { lineNumber } } diff --git a/src/main/com/jdblabs/jlp/experimental/ast/Block.groovy b/src/main/com/jdblabs/jlp/ast/Block.groovy similarity index 88% rename from src/main/com/jdblabs/jlp/experimental/ast/Block.groovy rename to src/main/com/jdblabs/jlp/ast/Block.groovy index 3bc4311..d242e7e 100644 --- a/src/main/com/jdblabs/jlp/experimental/ast/Block.groovy +++ b/src/main/com/jdblabs/jlp/ast/Block.groovy @@ -1,4 +1,4 @@ -package com.jdblabs.jlp.experimental.ast +package com.jdblabs.jlp.ast public class Block extends ASTNode { public final DocBlock docBlock diff --git a/src/main/com/jdblabs/jlp/experimental/ast/CodeBlock.groovy b/src/main/com/jdblabs/jlp/ast/CodeBlock.groovy similarity index 90% rename from src/main/com/jdblabs/jlp/experimental/ast/CodeBlock.groovy rename to src/main/com/jdblabs/jlp/ast/CodeBlock.groovy index 0bbfd0d..0f40058 100644 --- a/src/main/com/jdblabs/jlp/experimental/ast/CodeBlock.groovy +++ b/src/main/com/jdblabs/jlp/ast/CodeBlock.groovy @@ -1,4 +1,4 @@ -package com.jdblabs.jlp.experimental.ast +package com.jdblabs.jlp.ast import java.util.Map diff --git a/src/main/com/jdblabs/jlp/ast/Directive.groovy b/src/main/com/jdblabs/jlp/ast/Directive.groovy index 73e9dd6..3f17fbd 100644 --- a/src/main/com/jdblabs/jlp/ast/Directive.groovy +++ b/src/main/com/jdblabs/jlp/ast/Directive.groovy @@ -1,10 +1,11 @@ package com.jdblabs.jlp.ast -public class Directive implements ASTNode { +public class Directive extends ASTNode { public static enum DirectiveType { + Api, Author, - Doc, + Copyright, Example, Org; @@ -13,14 +14,11 @@ public class Directive implements ASTNode { public final DirectiveType type; public final String value; - public final int lineNumber; public Directive(String value, String typeString, int lineNumber) { + super(lineNumber) this.value = value - this.type = DirectiveType.parse(typeString) - this.lineNumber = lineNumber } + this.type = DirectiveType.parse(typeString) } - public int getLineNumber() { return lineNumber } - - public String toString() { return "[Directive(${lineNumber}): ${type}, ${value}]" } + public String toString() { "[${lineNumber}:Directive: ${type}, ${value}]" } } diff --git a/src/main/com/jdblabs/jlp/ast/DocBlock.groovy b/src/main/com/jdblabs/jlp/ast/DocBlock.groovy index 43ad763..41640fb 100644 --- a/src/main/com/jdblabs/jlp/ast/DocBlock.groovy +++ b/src/main/com/jdblabs/jlp/ast/DocBlock.groovy @@ -3,16 +3,13 @@ package com.jdblabs.jlp.ast import java.util.ArrayList import java.util.List -public class DocBlock implements ASTNode { +public class DocBlock extends ASTNode { - public final int lineNumber public List directives = new ArrayList() - public List textBlocks = new ArrayList() + public List docTexts = new ArrayList() - public DocBlock(int lineNumber) { this.lineNumber = lineNumber } - - public int getLineNumber() { lineNumber } + public DocBlock(int lineNumber) { super(lineNumber) } public String toString() { - "[DocBlock: Directives ${directives}, TextBlocks ${textBlocks}]" } + "[${lineNumber}:DocBlock: Directives ${directives}, DocTexts ${docTexts}]" } } diff --git a/src/main/com/jdblabs/jlp/experimental/ast/DocText.groovy b/src/main/com/jdblabs/jlp/ast/DocText.groovy similarity index 82% rename from src/main/com/jdblabs/jlp/experimental/ast/DocText.groovy rename to src/main/com/jdblabs/jlp/ast/DocText.groovy index 8fb75a9..e76f418 100644 --- a/src/main/com/jdblabs/jlp/experimental/ast/DocText.groovy +++ b/src/main/com/jdblabs/jlp/ast/DocText.groovy @@ -1,4 +1,4 @@ -package com.jdblabs.jlp.experimental.ast +package com.jdblabs.jlp.ast public class DocText extends ASTNode { diff --git a/src/main/com/jdblabs/jlp/experimental/ast/SourceFile.groovy b/src/main/com/jdblabs/jlp/ast/SourceFile.groovy similarity index 73% rename from src/main/com/jdblabs/jlp/experimental/ast/SourceFile.groovy rename to src/main/com/jdblabs/jlp/ast/SourceFile.groovy index b77517b..d1ceb45 100644 --- a/src/main/com/jdblabs/jlp/experimental/ast/SourceFile.groovy +++ b/src/main/com/jdblabs/jlp/ast/SourceFile.groovy @@ -1,4 +1,4 @@ -package com.jdblabs.jlp.experimental.ast +package com.jdblabs.jlp.ast public class SourceFile { public List blocks = [] diff --git a/src/main/com/jdblabs/jlp/ast/TextBlock.groovy b/src/main/com/jdblabs/jlp/ast/TextBlock.groovy deleted file mode 100644 index d06ed19..0000000 --- a/src/main/com/jdblabs/jlp/ast/TextBlock.groovy +++ /dev/null @@ -1,25 +0,0 @@ -package com.jdblabs.jlp.ast - -public class TextBlock implements ASTNode { - - public static enum TextBlockType { TextBlock, CodeBlock } - - public final TextBlockType type - public final String value - public final int lineNumber - - public TextBlock(TextBlockType type, String value, int lineNumber) { - this.type = type - this.value = value - this.lineNumber = lineNumber } - - public int getLineNumber() { return lineNumber } - - public String toString() { return "[${type}(${lineNumber}): ${value}]" } - - public static TextBlock makeTextBlock(String value, int lineNumber) { - return new TextBlock(TextBlockType.TextBlock, value, lineNumber) } - - public static TextBlock makeCodeBlock(String value, int lineNumber) { - return new TextBlock(TextBlockType.CodeBlock, value, lineNumber) } -} diff --git a/src/main/com/jdblabs/jlp/experimental/DeadSimple.java b/src/main/com/jdblabs/jlp/experimental/DeadSimple.java deleted file mode 100644 index e763d94..0000000 --- a/src/main/com/jdblabs/jlp/experimental/DeadSimple.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.jdblabs.jlp.experimental; - -import org.parboiled.Action; -import org.parboiled.BaseParser; -import org.parboiled.Context; -import org.parboiled.Rule; -import org.parboiled.annotations.*; - -@BuildParseTree -public class DeadSimple extends BaseParser { - - public Rule S() { - return OneOrMore(FirstOf(Line(), EmptyLine()).skipNode()); } - - public Rule Line() { - return Sequence(OneOrMore(NOT_EOL).skipNode(), EOL); } - - public Rule EmptyLine() { - return EOL; } - - public Rule EOL = Ch('\n'); - public Rule NOT_EOL = Sequence(TestNot(EOL), ANY); -} diff --git a/src/main/com/jdblabs/jlp/experimental/JLPBaseGenerator.groovy b/src/main/com/jdblabs/jlp/experimental/JLPBaseGenerator.groovy deleted file mode 100644 index 8f84dcb..0000000 --- a/src/main/com/jdblabs/jlp/experimental/JLPBaseGenerator.groovy +++ /dev/null @@ -1,62 +0,0 @@ -package com.jdblabs.jlp.experimental - -import com.jdblabs.jlp.experimental.ast.* -import java.util.List -import java.util.Map - -public abstract class JLPBaseGenerator { - - protected Map docState - - protected JLPBaseGenerator() { - docState = [orgs: [:], // stores `@org` references - codeTrees: [:], // stores code ASTs for - currentDocId: false ] } // store the current docid - - protected Map generate(Map sources) { - Map result = [:] - - // run the parse phase - sources.each { sourceId, sourceAST -> - - // set up the current generator state for this source - docState.currentDocId = sourceId - docState.codeTrees[sourceId] = sourceAST.codeAST - - parse(sourceAST) } - - // run the emit phase - sources.each { sourceId, sourceAST -> - - // set up the current generator state for this source - docState.currentDocId = sourceId - - // generate the doc for this source - result[sourceId] = emit(sourceAST) } - - // return our results - return result } - - protected void parse(SourceFile sourceFile) { - sourceFile.blocks.each { block -> parse(block) } } - - protected void parse(Block block) { - parse(block.docBlock) - parse(block.codeBlock) } - - protected void parse(DocBlock docBlock) { - docBlock.directives.each { directive -> parse(directive) } - docBlock.docTexts.each { docText -> parse(docText) } } - - protected abstract void parse(Directive directive) - protected abstract void parse(CodeBlock codeBlock) - protected abstract void parse(DocText docText) - - protected abstract String emit(SourceFile sourceFile) - protected abstract String emit(Block block) - protected abstract String emit(DocBlock docBlock) - protected abstract String emit(CodeBlock codeBlock) - protected abstract String emit(DocText docText) - protected abstract String emit(Directive directive) - -} diff --git a/src/main/com/jdblabs/jlp/experimental/JLPPegParser.java b/src/main/com/jdblabs/jlp/experimental/JLPPegParser.java deleted file mode 100644 index 5016878..0000000 --- a/src/main/com/jdblabs/jlp/experimental/JLPPegParser.java +++ /dev/null @@ -1,250 +0,0 @@ -package com.jdblabs.jlp.experimental; - -import com.jdblabs.jlp.experimental.ast.*; -import java.util.ArrayList; -import java.util.List; -import org.parboiled.Action; -import org.parboiled.BaseParser; -import org.parboiled.Context; -import org.parboiled.Rule; -import org.parboiled.annotations.*; - -@BuildParseTree -public class JLPPegParser extends BaseParser { - - int curLineNum = 1; - - /** - * Parses the rule: - * SourceFile = (Block / DocBlock / CodeBlock)+ - * - * Pushes a SourceFile node on the stack. - */ - public Rule SourceFile() { - return Sequence( - // At the start of processing a new SourceFile, we need to set up - // our internal state. - - // Clear the line count. - clearLineCount(), - - // Add the top-level SourceFile AST node. - push(new SourceFile()), - - // A SourceFile is made up of one or more Blocks - OneOrMore(Sequence( - // All of these options result in one new Block pushed onto the - // stack. - FirstOf( - - // Match a whole Block. This pushes a Block on the stack. - Block(), - - // A standalone DocBlock. We will create an empty CodeBlock - // to pair with it, then push a new Block onto the stack - // made from the DocBlock and the empty CodeBlock - Sequence( - // 1. We need to remember the line number to create the - // Block - push(curLineNum), - - // 2. Match the DocBlock. - DocBlock(), - - // 3. Create the empty CodeBlock. - push(new CodeBlock(curLineNum)), - - // 4. Create the Block and push it onto the stack. - push(new Block((CodeBlock) pop(), (DocBlock) pop(), - popAsInt()))), - - // A standalone CodeBlock. Similar to the standalone - // DocBlock, we will create an empty DocBlock to pair with - // the CodeBlock to make a Block, which is pushed onto the - // stack: - // - // *Note: With the way the parser is currently written, - // this will only match a CodeBlock that occurs - // before any DocBlock.* - Sequence( - // 1. Remember the line number for the Block. - push(curLineNum), - - // 2. Create the empty DocBlock. - push(new DocBlock(curLineNum)), - - // 3. Match the CodeBlock - CodeBlock(), - - // 4. Create the Block and push it onto the stack - push(new Block((CodeBlock) pop(), (DocBlock) pop(), - popAsInt())))), - - // pop the Block created by one of the above options and add it - // to the SourceFile - addBlockToSourceFile((Block) pop())))); } - - /** - * Parses the rule: - * Block = DocBlock CodeBlock - * - * Pushes a Block onto the stack - */ - Rule Block() { - return Sequence( - push(curLineNum), - DocBlock(), CodeBlock(), - - push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); } - - /** - * Parses the rule: - * DocBlock = (Directive / DocText)+ - * - * Pushes a DocBlock object onto the stack - */ - Rule DocBlock() { - return Sequence( - push(new DocBlock(curLineNum)), - OneOrMore(Sequence( - FirstOf(Directive(), DocText()), - addToDocBlock((ASTNode) pop())))); } - - /** - * Parses the rule: - * CodeBlock = (!DocLineStart RemainingLine)+ - * - * Pushes a CodeBlock onto the stack. - */ - Rule CodeBlock() { - return Sequence( - push(new CodeBlock(curLineNum)), - OneOrMore(Sequence( - TestNot(DocLineStart()), RemainingLine(), - addToCodeBlock(match())))); } - - /** - * Parses the rule: - * Directive = DocLineStart AT (LongDirective / ShortDirective) - * - * Pushes a Directive node on the stack. - */ - Rule Directive() { - return Sequence( - DocLineStart(), AT, FirstOf(LongDirective(), ShortDirective())); } - - /** - * Parses the rule: - * LongDirective = - * (API_DIR / EXAMPLE_DIR) RemainingLine DocText? - * - * Pushes a Directive node onto the stack. - */ - Rule LongDirective() { - return Sequence( - push(curLineNum), - - FirstOf(API_DIR, EXAMPLE_DIR), push(match()), - RemainingLine(), push(match()), - - Optional(Sequence( - DocText(), - swap(), - push(popAsString() + ((DocText) pop()).value))), - - push(new Directive(popAsString(), popAsString(), popAsInt()))); } - - /** - * Parses the rule: - * ShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingLine - * - * Pushes a Directive node onto the stack. - */ - Rule ShortDirective() { - return Sequence( - push(curLineNum), - FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()), - RemainingLine(), - - push(new Directive(match().trim(), popAsString(), popAsInt()))); } - - /** - * Parses the rule: - * DocText = (DocLineStart !AT RemainingLine)+ - * - * Pushes a DocText node onto the stack. - */ - Rule DocText() { - return Sequence( - push(new DocText(curLineNum)), - OneOrMore(Sequence( - DocLineStart(), TestNot(AT), RemainingLine(), - addToDocText(match())))); } - - Rule DocLineStart() { - return Sequence( - ZeroOrMore(SPACE), DOC_LINE_START, Optional(SPACE)); } - - Rule NonEmptyLine() { - return Sequence(OneOrMore(NOT_EOL), FirstOf(EOL, EOI)); } - - Rule RemainingLine() { - return FirstOf( - Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()), - Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); } - - Rule AT = Ch('@').label("AT"); - Rule EOL = FirstOf(String("\r\n"), Ch('\n'), Ch('\r')).label("EOL"); - Rule NOT_EOL = Sequence(TestNot(EOL), ANY).label("NOT_EOL"); - Rule SPACE = AnyOf(" \t").label("SPACE"); - Rule DOC_LINE_START = String("%%").label("DOC_LINE_START"); - - // directive terminals - Rule AUTHOR_DIR = IgnoreCase("author"); - Rule COPYRIGHT_DIR = IgnoreCase("copyright"); - Rule API_DIR = IgnoreCase("api"); - Rule EXAMPLE_DIR = IgnoreCase("example"); - Rule ORG_DIR = IgnoreCase("org"); - - String popAsString() { return (String) pop(); } - - Integer popAsInt() { return (Integer) pop(); } - - boolean clearLineCount() { curLineNum = 1; return true; } - - boolean incLineCount() { curLineNum++; return true; } - - boolean addBlockToSourceFile(Block block) { - SourceFile sourceFile = (SourceFile) pop(); - sourceFile.blocks.add(block); - return push(sourceFile); } - - /** - * Pop off a DocBlock, add the given Directive or DocText and push the - * DocBlock back onto the stack. - */ - boolean addToDocBlock(ASTNode an) { - DocBlock docBlock = (DocBlock) pop(); - if (an instanceof Directive) { - docBlock.directives.add((Directive) an); } - else if (an instanceof DocText) { - docBlock.docTexts.add((DocText) an); } - else { throw new IllegalStateException(); } - return push(docBlock); } - - boolean addToCodeBlock(String line) { - CodeBlock codeBlock = (CodeBlock) pop(); - codeBlock.lines.put(curLineNum - 1, line); - return push(codeBlock); } - - boolean addToDocText(String line) { - DocText docText = (DocText) pop(); - docText.value += line; - return push(docText); } - - boolean printValueStack() { - for (int i = 0; i < getContext().getValueStack().size(); i++) { - System.out.println(i + ": " + peek(i)); } - return true; } - -} diff --git a/src/main/com/jdblabs/jlp/experimental/ast/ASTNode.groovy b/src/main/com/jdblabs/jlp/experimental/ast/ASTNode.groovy deleted file mode 100644 index f04e66c..0000000 --- a/src/main/com/jdblabs/jlp/experimental/ast/ASTNode.groovy +++ /dev/null @@ -1,10 +0,0 @@ -package com.jdblabs.jlp.experimental.ast - -public class ASTNode { - - protected int lineNumber - - public ASTNode(int lineNum) { this.lineNumber = lineNum } - - public int getLineNumber() { lineNumber } -} diff --git a/src/main/com/jdblabs/jlp/experimental/ast/Directive.groovy b/src/main/com/jdblabs/jlp/experimental/ast/Directive.groovy deleted file mode 100644 index 4de3c86..0000000 --- a/src/main/com/jdblabs/jlp/experimental/ast/Directive.groovy +++ /dev/null @@ -1,24 +0,0 @@ -package com.jdblabs.jlp.experimental.ast - -public class Directive extends ASTNode { - - public static enum DirectiveType { - Api, - Author, - Copyright, - Example, - Org; - - public static DirectiveType parse(String typeString) { - valueOf(typeString.toLowerCase().capitalize()) } } - - public final DirectiveType type; - public final String value; - - public Directive(String value, String typeString, int lineNumber) { - super(lineNumber) - this.value = value - this.type = DirectiveType.parse(typeString) } - - public String toString() { "[${lineNumber}:Directive: ${type}, ${value}]" } -} diff --git a/src/main/com/jdblabs/jlp/experimental/ast/DocBlock.groovy b/src/main/com/jdblabs/jlp/experimental/ast/DocBlock.groovy deleted file mode 100644 index d43c405..0000000 --- a/src/main/com/jdblabs/jlp/experimental/ast/DocBlock.groovy +++ /dev/null @@ -1,15 +0,0 @@ -package com.jdblabs.jlp.experimental.ast - -import java.util.ArrayList -import java.util.List - -public class DocBlock extends ASTNode { - - public List directives = new ArrayList() - public List docTexts = new ArrayList() - - public DocBlock(int lineNumber) { super(lineNumber) } - - public String toString() { - "[${lineNumber}:DocBlock: Directives ${directives}, DocTexts ${docTexts}]" } -}