jdb/jlp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Promoting the experimental code to replace the exising main code.

Browse Source
This commit is contained in:
Jonathan Bernard
2011-09-06 16:26:36 -05:00
parent 7a5870dc09
commit bab1943120
20 changed files with 232 additions and 709 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
resources/main/test.groovy
View File

@@ -1,26 +1,16 @@
import com.jdblabs.jlp.*
import com.jdblabs.jlp.experimental.LiterateMarkdownGenerator
import org.parboiled.Parboiled
import org.parboiled.parserunners.ReportingParseRunner
import org.parboiled.parserunners.RecoveringParseRunner
makeParser = {
println "Making the standard parser."
println "---------------------------"
"Making the standard parser."
"---------------------------"
parser = Parboiled.createParser(JLPPegParser.class)
parseRunner = new ReportingParseRunner(parser.SourceFile())
}
makeExperimentalParser = {
println "Making the experimental parser."
println "-------------------------------"
parser = Parboiled.createParser(com.jdblabs.jlp.experimental.JLPPegParser.class)
parseRunner = new ReportingParseRunner(parser.SourceFile())
}
simpleTest = {
println "Parsing the simple test into 'result'."
println "--------------------------------------"
@@ -46,33 +36,12 @@ vbsTest = {
vbsParsed = parseRunner.run(vbsTestInput)
vbsResult = MarkdownGenerator.generateDocuments([vbs: vbsParsed.resultValue]).vbs
vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl"
println "Writing to file 'vbs_result.html'."
println "----------------------------------"
println "Writing to file 'vbs_db_records.html'."
println "--------------------------------------"
(new File('vbs_result.html')).withWriter { out -> out.println vbsResult }
(new File('vbs_db_records.html')).withWriter { out -> out.println vbsResult }
return [vbsParsed, vbsResult]
}
experimentalTest = {
makeExperimentalParser()
println "Parsing vbs_db_records.hrl into 'vbsResult'."
println "--------------------------------------------"
vbsTestFile = new File('vbs_db_records.hrl')
println "vbsTestFile is ${vbsTestFile.exists() ? 'present' : 'absent'}."
vbsTestInput = vbsTestFile.text
vbsParsed = parseRunner.run(vbsTestInput)
vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl"
println "Writing to file 'vbs_result.html'."
println "----------------------------------"
(new File('vbs_result.html')).withWriter { out -> out.println vbsResult }
}

81
src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy
View File

@@ -1,7 +1,6 @@
package com.jdblabs.jlp
import com.jdblabs.jlp.ast.*
import com.jdblabs.jlp.ast.Directive.DirectiveType
import java.util.List
import java.util.Map
@@ -10,54 +9,54 @@ public abstract class JLPBaseGenerator {
protected Map docState
protected JLPBaseGenerator() {
docState = [orgs: [:],
currentDocId: false ] }
docState = [orgs: [:], // stores `@org` references
codeTrees: [:], // stores code ASTs for
currentDocId: false ] } // store the current docid
protected Map<String, String> generate(Map<String, List<ASTNode>> sources) {
protected Map<String, String> generate(Map<String, SourceFile> sources) {
Map result = [:]
sources.each { sourceId, sourceNodes ->
// run the parse phase
sources.each { sourceId, sourceAST ->
// set up the current generator state for this source
docState.currentDocId = sourceId
result[sourceId] = emitDocument(sourceNodes) }
docState.codeTrees[sourceId] = sourceAST.codeAST
parse(sourceAST) }
// run the emit phase
sources.each { sourceId, sourceAST ->
// set up the current generator state for this source
docState.currentDocId = sourceId
// generate the doc for this source
result[sourceId] = emit(sourceAST) }
// return our results
return result }
protected String emitDocument(List<ASTNode> sourceNodes) {
StringBuilder result =
sourceNodes.inject(new StringBuilder()) { sb, node ->
sb.append(emit(node))
return sb }
protected void parse(SourceFile sourceFile) {
sourceFile.blocks.each { block -> parse(block) } }
return result.toString() }
protected void parse(Block block) {
parse(block.docBlock)
parse(block.codeBlock) }
protected String emit(DocBlock docBlock) {
List printQueue
StringBuilder result
protected void parse(DocBlock docBlock) {
docBlock.directives.each { directive -> parse(directive) }
docBlock.docTexts.each { docText -> parse(docText) } }
printQueue = docBlock.directives.collect { directive ->
def queueItem = [line: directive.lineNumber, value: directive]
switch (directive.type) {
case DirectiveType.Author: queueItem.priority = 50; break
case DirectiveType.Doc: queueItem.priority = 50; break
case DirectiveType.Example: queueItem.priority = 50; break
case DirectiveType.Org: queueItem.priority = 10; break }
protected abstract void parse(Directive directive)
protected abstract void parse(CodeBlock codeBlock)
protected abstract void parse(DocText docText)
return queueItem }
printQueue.addAll(docBlock.textBlocks.collect { textBlock ->
[ priority: 50, line: textBlock.lineNumber, value: textBlock ] })
// sort by priority, then by line number
printQueue.sort(
{i1, i2 -> i1.priority != i2.priority ?
i1.priority - i2.priority :
i1.line - i2.line} as Comparator)
result = printQueue.inject(new StringBuilder()) { sb, printable ->
sb.append(emit(printable.value))
return sb }
return result.toString() }
protected abstract String emit(TextBlock textBlock)
protected abstract String emit(SourceFile sourceFile)
protected abstract String emit(Block block)
protected abstract String emit(DocBlock docBlock)
protected abstract String emit(CodeBlock codeBlock)
protected abstract String emit(DocText docText)
protected abstract String emit(Directive directive)
}

257
src/main/com/jdblabs/jlp/JLPPegParser.java
View File

@@ -9,164 +9,200 @@ import org.parboiled.Context;
import org.parboiled.Rule;
import org.parboiled.annotations.*;
import static com.jdblabs.jlp.ast.TextBlock.makeCodeBlock;
import static com.jdblabs.jlp.ast.TextBlock.makeTextBlock;
@BuildParseTree
public class JLPPegParser extends BaseParser<Object> {
int curLineNum = 1;
/**
* Parses the rule:
* SourceFile = (Block / DocBlock / CodeBlock)+
*
* Pushes a SourceFile node on the stack.
*/
public Rule SourceFile() {
return Sequence(
// At the start of processing a new SourceFile, we need to set up
// our internal state.
// Clear the line count.
clearLineCount(),
push(new ArrayList<ASTNode>()),
ZeroOrMore(Sequence(
// Add the top-level SourceFile AST node.
push(new SourceFile()),
// A SourceFile is made up of one or more Blocks
OneOrMore(Sequence(
// All of these options result in one new Block pushed onto the
// stack.
FirstOf(
// Match a whole Block. This pushes a Block on the stack.
Block(),
// A standalone DocBlock. We will create an empty CodeBlock
// to pair with it, then push a new Block onto the stack
// made from the DocBlock and the empty CodeBlock
Sequence(
// 1. We need to remember the line number to create the
// Block
push(curLineNum),
// 2. Match the DocBlock.
DocBlock(),
CodeBlock()),
push(addToList(pop(), (List<Object>)pop()))))); }
// 3. Create the empty CodeBlock.
push(new CodeBlock(curLineNum)),
// 4. Create the Block and push it onto the stack.
push(new Block((CodeBlock) pop(), (DocBlock) pop(),
popAsInt()))),
// A standalone CodeBlock. Similar to the standalone
// DocBlock, we will create an empty DocBlock to pair with
// the CodeBlock to make a Block, which is pushed onto the
// stack:
//
// *Note: With the way the parser is currently written,
// this will only match a CodeBlock that occurs
// before any DocBlock.*
Sequence(
// 1. Remember the line number for the Block.
push(curLineNum),
// 2. Create the empty DocBlock.
push(new DocBlock(curLineNum)),
// 3. Match the CodeBlock
CodeBlock(),
// 4. Create the Block and push it onto the stack
push(new Block((CodeBlock) pop(), (DocBlock) pop(),
popAsInt())))),
// pop the Block created by one of the above options and add it
// to the SourceFile
addBlockToSourceFile((Block) pop())))); }
/**
* Parses the rule:
* DocBlock = (DirectiveBlock / DocTextBlock)+
* Block = DocBlock CodeBlock
*
* Pushes a DocBlock object onto the stack.
* Pushes a Block onto the stack
*/
Rule Block() {
return Sequence(
push(curLineNum),
DocBlock(), CodeBlock(),
push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); }
/**
* Parses the rule:
* DocBlock = (Directive / DocText)+
*
* Pushes a DocBlock object onto the stack
*/
Rule DocBlock() {
return Sequence(
push(new DocBlock(curLineNum)),
OneOrMore(
FirstOf(
Sequence(DirectiveBlock(),
push(addDirectiveBlock((Directive) pop(), (DocBlock) pop()))),
Sequence(DocTextBlock(),
push(addTextBlock((TextBlock) pop(), (DocBlock) pop())))))); }
OneOrMore(Sequence(
FirstOf(Directive(), DocText()),
addToDocBlock((ASTNode) pop())))); }
/**
* Parses the rule:
* CodeBlock = !DocStart RemainingLine
* CodeBlock = (!DocLineStart RemainingLine)+
*
* Pushes a CodeBlock onto the stack.
*/
Rule CodeBlock() {
return Sequence(
push(curLineNum),
push(""),
push(new CodeBlock(curLineNum)),
OneOrMore(Sequence(
TestNot(DocStart()), RemainingLine(),
push(popAsString() + match()))),
push(makeCodeBlock(popAsString(),popAsInt()))); }
TestNot(DocLineStart()), RemainingLine(),
addToCodeBlock(match())))); }
/**
* Parses the rule:
* DocStart = SPACE* DOC_START
*/
Rule DocStart() {
return Sequence(ZeroOrMore(SPACE), DOC_START); }
/**
* Parses the rule:
* DirectiveBlock =
* DocStart DIRECTIVE_START (LongDirective / LineDirective)
* Directive = DocLineStart AT (LongDirective / ShortDirective)
*
* Pushes a Directive onto the stack.
* Pushes a Directive node on the stack.
*/
Rule DirectiveBlock() {
Rule Directive() {
return Sequence(
DocStart(), DIRECTIVE_START,
FirstOf(LongDirective(), LineDirective())); }
DocLineStart(), AT, FirstOf(LongDirective(), ShortDirective())); }
/**
* Parses the rule:
* LongDirective =
* (AUTHOR_DIR / DOC_DIR / EXAMPLE_DIR) RemainingLine DocTextBlock?
* (API_DIR / EXAMPLE_DIR) RemainingLine DocText?
*
* Pushes a Directive object onto the value stack.
* Pushes a Directive node onto the stack.
*/
Rule LongDirective() {
return Sequence(
push(curLineNum),
FirstOf(AUTHOR_DIR, DOC_DIR, EXAMPLE_DIR), push(match()),
RemainingLine(), push(match()),
Optional(Sequence(
DocTextBlock(), // pushes block
swap(),
push(popAsString() + ((TextBlock) pop()).value))),
// pull off the value, type and create the directive
FirstOf(API_DIR, EXAMPLE_DIR), push(match()),
RemainingLine(), push(match()),
Optional(Sequence(
DocText(),
swap(),
push(popAsString() + ((DocText) pop()).value))),
push(new Directive(popAsString(), popAsString(), popAsInt()))); }
/**
* Parses the rule:
* LineDirective =
* ORG_DIR RemainingLine
* ShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingLine
*
* Pushes a Directive object onto the value stack.
* Pushes a Directive node onto the stack.
*/
Rule LineDirective() {
Rule ShortDirective() {
return Sequence(
push(curLineNum),
ORG_DIR, push(match()),
FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()),
RemainingLine(),
// pull off the value, type and create the directive
push(new Directive(match().trim(), popAsString(), popAsInt()))); }
/**
* Parses the rule:
* DocTextBlock = DocTextLine+
* DocText = (DocLineStart !AT RemainingLine)+
*
* Pushes a DocTextBlock onto the stack as a string.
* Pushes a DocText node onto the stack.
*/
Rule DocTextBlock() {
Rule DocText() {
return Sequence(
push(curLineNum),
DocTextLine(), // pushes the value onto the stack
ZeroOrMore(Sequence(
DocTextLine(),
swap(),
push(popAsString() + popAsString()))),
push(new DocText(curLineNum)),
OneOrMore(Sequence(
DocLineStart(), TestNot(AT), RemainingLine(),
addToDocText(match())))); }
push(makeTextBlock(popAsString(), popAsInt()))); }
/**
* Parses the rule:
* DocTextLine =
* DocStart !DIRECTIVE_START RemainingLine
*
* Pushes the line value (not including the DocStart) onto the stack.
*/
Rule DocTextLine() {
Rule DocLineStart() {
return Sequence(
DocStart(), TestNot(DIRECTIVE_START),
RemainingLine(), push(match())); }
ZeroOrMore(SPACE), DOC_LINE_START, Optional(SPACE)); }
Rule NonEmptyLine() {
return Sequence(OneOrMore(NOT_EOL), FirstOf(EOL, EOI)); }
/**
* Parses the rule:
* RemainingLine = (!EOL)+ EOL
*/
@SuppressSubnodes
Rule RemainingLine() {
return FirstOf(
Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()),
Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); }
// allow EOI as a line delimiter only if the line is not empty,
// otherwise it will match infinitely if RemainingLine is used in a
// OneOrMore context.
Sequence(OneOrMore(NOT_EOL), EOI)); }
Rule DOC_START = String("%% ");
Rule EOL = Ch('\n');
Rule NOT_EOL = Sequence(TestNot(EOL), ANY);
Rule DIRECTIVE_START= Ch('@');
Rule SLASH = Ch('/');
Rule SPACE = AnyOf(" \t");
Rule AT = Ch('@').label("AT");
Rule EOL = FirstOf(String("\r\n"), Ch('\n'), Ch('\r')).label("EOL");
Rule NOT_EOL = Sequence(TestNot(EOL), ANY).label("NOT_EOL");
Rule SPACE = AnyOf(" \t").label("SPACE");
Rule DOC_LINE_START = String("%%").label("DOC_LINE_START");
// directive terminals
Rule AUTHOR_DIR = IgnoreCase("author");
Rule DOC_DIR = IgnoreCase("doc");
Rule COPYRIGHT_DIR = IgnoreCase("copyright");
Rule API_DIR = IgnoreCase("api");
Rule EXAMPLE_DIR = IgnoreCase("example");
Rule ORG_DIR = IgnoreCase("org");
@@ -174,24 +210,41 @@ public class JLPPegParser extends BaseParser<Object> {
Integer popAsInt() { return (Integer) pop(); }
static <T> List<T> addToList(T value, List<T> list) {
list.add(value);
return list; }
boolean clearLineCount() { curLineNum = 1; return true; }
boolean incLineCount() { curLineNum++; return true; }
boolean addBlockToSourceFile(Block block) {
SourceFile sourceFile = (SourceFile) pop();
sourceFile.blocks.add(block);
return push(sourceFile); }
/**
* Pop off a DocBlock, add the given Directive or DocText and push the
* DocBlock back onto the stack.
*/
boolean addToDocBlock(ASTNode an) {
DocBlock docBlock = (DocBlock) pop();
if (an instanceof Directive) {
docBlock.directives.add((Directive) an); }
else if (an instanceof DocText) {
docBlock.docTexts.add((DocText) an); }
else { throw new IllegalStateException(); }
return push(docBlock); }
boolean addToCodeBlock(String line) {
CodeBlock codeBlock = (CodeBlock) pop();
codeBlock.lines.put(curLineNum - 1, line);
return push(codeBlock); }
boolean addToDocText(String line) {
DocText docText = (DocText) pop();
docText.value += line;
return push(docText); }
boolean printValueStack() {
for (int i = 0; i < getContext().getValueStack().size(); i++) {
System.out.println(i + ": " + peek(i)); }
return true; }
boolean clearLineCount() { curLineNum = 1; return true; }
boolean incLineCount() { curLineNum++; return true; }
boolean echo(String msg) { System.out.println(msg); return true; }
static DocBlock addDirectiveBlock(Directive dir, DocBlock docBlock) {
docBlock.directives.add(dir); return docBlock; }
static DocBlock addTextBlock(TextBlock tb, DocBlock docBlock) {
docBlock.textBlocks.add(tb); return docBlock; }
}

6
src/main/com/jdblabs/jlp/experimental/LiterateMarkdownGenerator.groovy → src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy
View File

@@ -1,7 +1,7 @@
package com.jdblabs.jlp.experimental
package com.jdblabs.jlp
import com.jdblabs.jlp.experimental.ast.*
import com.jdblabs.jlp.experimental.ast.Directive.DirectiveType
import com.jdblabs.jlp.ast.*
import com.jdblabs.jlp.ast.Directive.DirectiveType
import org.pegdown.Extensions
import org.pegdown.PegDownProcessor

65
src/main/com/jdblabs/jlp/MarkdownGenerator.groovy
View File

@@ -1,65 +0,0 @@
package com.jdblabs.jlp
import com.jdblabs.jlp.ast.*
import com.jdblabs.jlp.ast.Directive.DirectiveType
import com.jdblabs.jlp.ast.TextBlock.TextBlockType
import org.pegdown.PegDownProcessor
import org.pegdown.Extensions
public class MarkdownGenerator extends JLPBaseGenerator {
protected PegDownProcessor pegdown
protected MarkdownGenerator() {
super()
pegdown = new PegDownProcessor(
Extensions.TABLES & Extensions.DEFINITIONS) }
protected static Map<String, String> generateDocuments(
Map<String, List<ASTNode>> sources) {
MarkdownGenerator inst = new MarkdownGenerator()
return inst.generate(sources) }
protected String emit(TextBlock textBlock) {
switch (textBlock.type) {
// text block, just convert to markdown
case TextBlockType.TextBlock:
return formatText(textBlock.value)
// code block, we want to emit as a code snippet
case TextBlockType.CodeBlock:
// so prepend all lines with four spaces to tell markdown that
// this is code
String value = textBlock.value.replaceAll(/(^|\n)/, /$1 /)
// then convert to markdown
return pegdown.markdownToHtml(value) } }
protected String emit(Directive d) {
switch (d.type) {
case DirectiveType.Author:
return "<span class='author'>Author: ${formatText(d.value)}</span>"
case DirectiveType.Doc:
return formatText(d.value)
case DirectiveType.Example:
return formatText(d.value)
case DirectiveType.Org:
docState.orgs[d.value] = [line: d.lineNumber,
file: docState.currentDocId]
return "<a name='${d.value}'/>" }
}
protected String formatText(String text) {
// convert to HTML from Markdown
String md = pegdown.markdownToHtml(text)
// replace internal `jlp://` links with actual links based on`@org`
// references
md = md.replaceAll(/jlp:\/\/([^\s"])/, /#$1/)
return md;
}
}

24
src/main/com/jdblabs/jlp/TransparentGenerator.groovy
View File

@@ -1,24 +0,0 @@
package com.jdblabs.jlp
import com.jdblabs.jlp.ast.*
import com.jdblabs.jlp.ast.Directive.DirectiveType
import java.util.List
import java.util.Map
public class TransparentGenerator extends JLPBaseGenerator {
protected TransparentGenerator() {}
public static Map<String, String> generateDocuments(Map<String,
List<ASTNode>> sources) {
TransparentGenerator inst = new TransparentGenerator()
return inst.generate(sources) }
protected String emit(TextBlock textBlock) { textBlock.value }
protected String emit(Directive directive) {
switch (directive.type) {
case DirectiveType.Author: return "Author: ${directive.value}\n"
case DirectiveType.Doc: return directive.value
case DirectiveType.Example: return "Example: ${directive.value}"
case DirectiveType.Org: return "" } }
}

9
src/main/com/jdblabs/jlp/ast/ASTNode.groovy
View File

@@ -1,5 +1,10 @@
package com.jdblabs.jlp.ast
public interface ASTNode {
public int getLineNumber()
public class ASTNode {
protected int lineNumber
public ASTNode(int lineNum) { this.lineNumber = lineNum }
public int getLineNumber() { lineNumber }
}

2
src/main/com/jdblabs/jlp/experimental/ast/Block.groovy → src/main/com/jdblabs/jlp/ast/Block.groovy
View File

@@ -1,4 +1,4 @@
package com.jdblabs.jlp.experimental.ast
package com.jdblabs.jlp.ast
public class Block extends ASTNode {
public final DocBlock docBlock

2
src/main/com/jdblabs/jlp/experimental/ast/CodeBlock.groovy → src/main/com/jdblabs/jlp/ast/CodeBlock.groovy
View File

@@ -1,4 +1,4 @@
package com.jdblabs.jlp.experimental.ast
package com.jdblabs.jlp.ast
import java.util.Map

14
src/main/com/jdblabs/jlp/ast/Directive.groovy
View File

@@ -1,10 +1,11 @@
package com.jdblabs.jlp.ast
public class Directive implements ASTNode {
public class Directive extends ASTNode {
public static enum DirectiveType {
Api,
Author,
Doc,
Copyright,
Example,
Org;
@@ -13,14 +14,11 @@ public class Directive implements ASTNode {
public final DirectiveType type;
public final String value;
public final int lineNumber;
public Directive(String value, String typeString, int lineNumber) {
super(lineNumber)
this.value = value
this.type = DirectiveType.parse(typeString)
this.lineNumber = lineNumber }
this.type = DirectiveType.parse(typeString) }
public int getLineNumber() { return lineNumber }
public String toString() { return "[Directive(${lineNumber}): ${type}, ${value}]" }
public String toString() { "[${lineNumber}:Directive: ${type}, ${value}]" }
}

11
src/main/com/jdblabs/jlp/ast/DocBlock.groovy
View File

@@ -3,16 +3,13 @@ package com.jdblabs.jlp.ast
import java.util.ArrayList
import java.util.List
public class DocBlock implements ASTNode {
public class DocBlock extends ASTNode {
public final int lineNumber
public List<Directive> directives = new ArrayList<Directive>()
public List<TextBlock> textBlocks = new ArrayList<TextBlock>()
public List<DocText> docTexts = new ArrayList<DocText>()
public DocBlock(int lineNumber) { this.lineNumber = lineNumber }
public int getLineNumber() { lineNumber }
public DocBlock(int lineNumber) { super(lineNumber) }
public String toString() {
"[DocBlock: Directives ${directives}, TextBlocks ${textBlocks}]" }
"[${lineNumber}:DocBlock: Directives ${directives}, DocTexts ${docTexts}]" }
}

2
src/main/com/jdblabs/jlp/experimental/ast/DocText.groovy → src/main/com/jdblabs/jlp/ast/DocText.groovy
View File

@@ -1,4 +1,4 @@
package com.jdblabs.jlp.experimental.ast
package com.jdblabs.jlp.ast
public class DocText extends ASTNode {

2
src/main/com/jdblabs/jlp/experimental/ast/SourceFile.groovy → src/main/com/jdblabs/jlp/ast/SourceFile.groovy
View File

@@ -1,4 +1,4 @@
package com.jdblabs.jlp.experimental.ast
package com.jdblabs.jlp.ast
public class SourceFile {
public List<ASTNode> blocks = []

25
src/main/com/jdblabs/jlp/ast/TextBlock.groovy
View File

@@ -1,25 +0,0 @@
package com.jdblabs.jlp.ast
public class TextBlock implements ASTNode {
public static enum TextBlockType { TextBlock, CodeBlock }
public final TextBlockType type
public final String value
public final int lineNumber
public TextBlock(TextBlockType type, String value, int lineNumber) {
this.type = type
this.value = value
this.lineNumber = lineNumber }
public int getLineNumber() { return lineNumber }
public String toString() { return "[${type}(${lineNumber}): ${value}]" }
public static TextBlock makeTextBlock(String value, int lineNumber) {
return new TextBlock(TextBlockType.TextBlock, value, lineNumber) }
public static TextBlock makeCodeBlock(String value, int lineNumber) {
return new TextBlock(TextBlockType.CodeBlock, value, lineNumber) }
}

23
src/main/com/jdblabs/jlp/experimental/DeadSimple.java
View File

@@ -1,23 +0,0 @@
package com.jdblabs.jlp.experimental;
import org.parboiled.Action;
import org.parboiled.BaseParser;
import org.parboiled.Context;
import org.parboiled.Rule;
import org.parboiled.annotations.*;
@BuildParseTree
public class DeadSimple extends BaseParser {
public Rule S() {
return OneOrMore(FirstOf(Line(), EmptyLine()).skipNode()); }
public Rule Line() {
return Sequence(OneOrMore(NOT_EOL).skipNode(), EOL); }
public Rule EmptyLine() {
return EOL; }
public Rule EOL = Ch('\n');
public Rule NOT_EOL = Sequence(TestNot(EOL), ANY);
}

62
src/main/com/jdblabs/jlp/experimental/JLPBaseGenerator.groovy
View File

@@ -1,62 +0,0 @@
package com.jdblabs.jlp.experimental
import com.jdblabs.jlp.experimental.ast.*
import java.util.List
import java.util.Map
public abstract class JLPBaseGenerator {
protected Map docState
protected JLPBaseGenerator() {
docState = [orgs: [:], // stores `@org` references
codeTrees: [:], // stores code ASTs for
currentDocId: false ] } // store the current docid
protected Map<String, String> generate(Map<String, SourceFile> sources) {
Map result = [:]
// run the parse phase
sources.each { sourceId, sourceAST ->
// set up the current generator state for this source
docState.currentDocId = sourceId
docState.codeTrees[sourceId] = sourceAST.codeAST
parse(sourceAST) }
// run the emit phase
sources.each { sourceId, sourceAST ->
// set up the current generator state for this source
docState.currentDocId = sourceId
// generate the doc for this source
result[sourceId] = emit(sourceAST) }
// return our results
return result }
protected void parse(SourceFile sourceFile) {
sourceFile.blocks.each { block -> parse(block) } }
protected void parse(Block block) {
parse(block.docBlock)
parse(block.codeBlock) }
protected void parse(DocBlock docBlock) {
docBlock.directives.each { directive -> parse(directive) }
docBlock.docTexts.each { docText -> parse(docText) } }
protected abstract void parse(Directive directive)
protected abstract void parse(CodeBlock codeBlock)
protected abstract void parse(DocText docText)
protected abstract String emit(SourceFile sourceFile)
protected abstract String emit(Block block)
protected abstract String emit(DocBlock docBlock)
protected abstract String emit(CodeBlock codeBlock)
protected abstract String emit(DocText docText)
protected abstract String emit(Directive directive)
}

250
src/main/com/jdblabs/jlp/experimental/JLPPegParser.java
View File

@@ -1,250 +0,0 @@
package com.jdblabs.jlp.experimental;
import com.jdblabs.jlp.experimental.ast.*;
import java.util.ArrayList;
import java.util.List;
import org.parboiled.Action;
import org.parboiled.BaseParser;
import org.parboiled.Context;
import org.parboiled.Rule;
import org.parboiled.annotations.*;
@BuildParseTree
public class JLPPegParser extends BaseParser<Object> {
int curLineNum = 1;
/**
* Parses the rule:
* SourceFile = (Block / DocBlock / CodeBlock)+
*
* Pushes a SourceFile node on the stack.
*/
public Rule SourceFile() {
return Sequence(
// At the start of processing a new SourceFile, we need to set up
// our internal state.
// Clear the line count.
clearLineCount(),
// Add the top-level SourceFile AST node.
push(new SourceFile()),
// A SourceFile is made up of one or more Blocks
OneOrMore(Sequence(
// All of these options result in one new Block pushed onto the
// stack.
FirstOf(
// Match a whole Block. This pushes a Block on the stack.
Block(),
// A standalone DocBlock. We will create an empty CodeBlock
// to pair with it, then push a new Block onto the stack
// made from the DocBlock and the empty CodeBlock
Sequence(
// 1. We need to remember the line number to create the
// Block
push(curLineNum),
// 2. Match the DocBlock.
DocBlock(),
// 3. Create the empty CodeBlock.
push(new CodeBlock(curLineNum)),
// 4. Create the Block and push it onto the stack.
push(new Block((CodeBlock) pop(), (DocBlock) pop(),
popAsInt()))),
// A standalone CodeBlock. Similar to the standalone
// DocBlock, we will create an empty DocBlock to pair with
// the CodeBlock to make a Block, which is pushed onto the
// stack:
//
// *Note: With the way the parser is currently written,
// this will only match a CodeBlock that occurs
// before any DocBlock.*
Sequence(
// 1. Remember the line number for the Block.
push(curLineNum),
// 2. Create the empty DocBlock.
push(new DocBlock(curLineNum)),
// 3. Match the CodeBlock
CodeBlock(),
// 4. Create the Block and push it onto the stack
push(new Block((CodeBlock) pop(), (DocBlock) pop(),
popAsInt())))),
// pop the Block created by one of the above options and add it
// to the SourceFile
addBlockToSourceFile((Block) pop())))); }
/**
* Parses the rule:
* Block = DocBlock CodeBlock
*
* Pushes a Block onto the stack
*/
Rule Block() {
return Sequence(
push(curLineNum),
DocBlock(), CodeBlock(),
push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); }
/**
* Parses the rule:
* DocBlock = (Directive / DocText)+
*
* Pushes a DocBlock object onto the stack
*/
Rule DocBlock() {
return Sequence(
push(new DocBlock(curLineNum)),
OneOrMore(Sequence(
FirstOf(Directive(), DocText()),
addToDocBlock((ASTNode) pop())))); }
/**
* Parses the rule:
* CodeBlock = (!DocLineStart RemainingLine)+
*
* Pushes a CodeBlock onto the stack.
*/
Rule CodeBlock() {
return Sequence(
push(new CodeBlock(curLineNum)),
OneOrMore(Sequence(
TestNot(DocLineStart()), RemainingLine(),
addToCodeBlock(match())))); }
/**
* Parses the rule:
* Directive = DocLineStart AT (LongDirective / ShortDirective)
*
* Pushes a Directive node on the stack.
*/
Rule Directive() {
return Sequence(
DocLineStart(), AT, FirstOf(LongDirective(), ShortDirective())); }
/**
* Parses the rule:
* LongDirective =
* (API_DIR / EXAMPLE_DIR) RemainingLine DocText?
*
* Pushes a Directive node onto the stack.
*/
Rule LongDirective() {
return Sequence(
push(curLineNum),
FirstOf(API_DIR, EXAMPLE_DIR), push(match()),
RemainingLine(), push(match()),
Optional(Sequence(
DocText(),
swap(),
push(popAsString() + ((DocText) pop()).value))),
push(new Directive(popAsString(), popAsString(), popAsInt()))); }
/**
* Parses the rule:
* ShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingLine
*
* Pushes a Directive node onto the stack.
*/
Rule ShortDirective() {
return Sequence(
push(curLineNum),
FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()),
RemainingLine(),
push(new Directive(match().trim(), popAsString(), popAsInt()))); }
/**
* Parses the rule:
* DocText = (DocLineStart !AT RemainingLine)+
*
* Pushes a DocText node onto the stack.
*/
Rule DocText() {
return Sequence(
push(new DocText(curLineNum)),
OneOrMore(Sequence(
DocLineStart(), TestNot(AT), RemainingLine(),
addToDocText(match())))); }
Rule DocLineStart() {
return Sequence(
ZeroOrMore(SPACE), DOC_LINE_START, Optional(SPACE)); }
Rule NonEmptyLine() {
return Sequence(OneOrMore(NOT_EOL), FirstOf(EOL, EOI)); }
Rule RemainingLine() {
return FirstOf(
Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()),
Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); }
Rule AT = Ch('@').label("AT");
Rule EOL = FirstOf(String("\r\n"), Ch('\n'), Ch('\r')).label("EOL");
Rule NOT_EOL = Sequence(TestNot(EOL), ANY).label("NOT_EOL");
Rule SPACE = AnyOf(" \t").label("SPACE");
Rule DOC_LINE_START = String("%%").label("DOC_LINE_START");
// directive terminals
Rule AUTHOR_DIR = IgnoreCase("author");
Rule COPYRIGHT_DIR = IgnoreCase("copyright");
Rule API_DIR = IgnoreCase("api");
Rule EXAMPLE_DIR = IgnoreCase("example");
Rule ORG_DIR = IgnoreCase("org");
String popAsString() { return (String) pop(); }
Integer popAsInt() { return (Integer) pop(); }
boolean clearLineCount() { curLineNum = 1; return true; }
boolean incLineCount() { curLineNum++; return true; }
boolean addBlockToSourceFile(Block block) {
SourceFile sourceFile = (SourceFile) pop();
sourceFile.blocks.add(block);
return push(sourceFile); }
/**
* Pop off a DocBlock, add the given Directive or DocText and push the
* DocBlock back onto the stack.
*/
boolean addToDocBlock(ASTNode an) {
DocBlock docBlock = (DocBlock) pop();
if (an instanceof Directive) {
docBlock.directives.add((Directive) an); }
else if (an instanceof DocText) {
docBlock.docTexts.add((DocText) an); }
else { throw new IllegalStateException(); }
return push(docBlock); }
boolean addToCodeBlock(String line) {
CodeBlock codeBlock = (CodeBlock) pop();
codeBlock.lines.put(curLineNum - 1, line);
return push(codeBlock); }
boolean addToDocText(String line) {
DocText docText = (DocText) pop();
docText.value += line;
return push(docText); }
boolean printValueStack() {
for (int i = 0; i < getContext().getValueStack().size(); i++) {
System.out.println(i + ": " + peek(i)); }
return true; }
}

10
src/main/com/jdblabs/jlp/experimental/ast/ASTNode.groovy
View File

@@ -1,10 +0,0 @@
package com.jdblabs.jlp.experimental.ast
public class ASTNode {
protected int lineNumber
public ASTNode(int lineNum) { this.lineNumber = lineNum }
public int getLineNumber() { lineNumber }
}

24
src/main/com/jdblabs/jlp/experimental/ast/Directive.groovy
View File

@@ -1,24 +0,0 @@
package com.jdblabs.jlp.experimental.ast
public class Directive extends ASTNode {
public static enum DirectiveType {
Api,
Author,
Copyright,
Example,
Org;
public static DirectiveType parse(String typeString) {
valueOf(typeString.toLowerCase().capitalize()) } }
public final DirectiveType type;
public final String value;
public Directive(String value, String typeString, int lineNumber) {
super(lineNumber)
this.value = value
this.type = DirectiveType.parse(typeString) }
public String toString() { "[${lineNumber}:Directive: ${type}, ${value}]" }
}

15
src/main/com/jdblabs/jlp/experimental/ast/DocBlock.groovy
View File

@@ -1,15 +0,0 @@
package com.jdblabs.jlp.experimental.ast
import java.util.ArrayList
import java.util.List
public class DocBlock extends ASTNode {
public List<Directive> directives = new ArrayList<Directive>()
public List<DocText> docTexts = new ArrayList<DocText>()
public DocBlock(int lineNumber) { super(lineNumber) }
public String toString() {
"[${lineNumber}:DocBlock: Directives ${directives}, DocTexts ${docTexts}]" }
}