jdb/jlp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Basic transformative functionality implemented.

Browse Source 
* Updated test data to include additional parsing edge cases.
* Updated `vbs_db_records.hrl` to use `@org` directives.
* Refactored Generator/Emitter dual-object phase concept into one object, the
  Generator. The emitter ended up needing basically full visibility into the
  generator anyways.
* Implemented `JLPBaseGenerator`, `MarkdownGenerator`, and
  `TransparentGenerator`
* Modified the way the parser handles remaining lines to allow it to safely
  handle empty lines.
This commit is contained in:
Jonathan Bernard
2011-08-31 09:46:25 -05:00
parent b8a47cac7e
commit 5081ebbd30
14 changed files with 231 additions and 307 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/com/jdblabs/jlp/EchoEmitter.groovy
View File

@ -1,13 +0,0 @@
package com.jdblabs.jlp
import com.jdblabs.jlp.ast.*
public class EchoEmitter extends JLPEmitter {
public String emitAuthor(String value) { "Author: $value" }
public String emitDoc(String value) { value }
public String emitExample(String value) { "Example:\n$value" }
public String emitOrg(String value) { "Org: $value" }
public String emitBlock(TextBlock textBlock) { textBlock.value }
}

7
src/main/com/jdblabs/jlp/Formatter.groovy
View File

@ -1,7 +0,0 @@
package com.jdblabs.jlp
public interface Formatter {
String formatText(String text)
String formatCode(String code)
String formatReference(String ref) }

46
src/main/com/jdblabs/jlp/FormattingEmitter.groovy
View File

@ -1,46 +0,0 @@
package com.jdblabs.jlp
import com.jdblabs.jlp.ast.*
import com.jdblabs.jlp.ast.Directive.DirectiveType
import org.slf4j.Logger
import org.slf4j.LoggerFactory
public class FormattingEmitter extends JLPBaseEmitter {
Formatter formatter
private Logger log = LoggerFactory.getLogger(this.getClass())
public FormattingEmitter(Formatter f, def generationState) {
super(generationState)
this.formatter = f }
protected String emit(TextBlock textBlock) {
return formatter.format(textBlock) }
protected String emit(Directive directive) {
switch (directive.type) {
case DirectiveType.Author:
case DirectiveType.Doc:
case DirectiveType.Example:
case DirectiveType.Org:
def orgValue = directive.value
if generationState.orgs.contains(orgValue) {
log.warn("Duplicate @org id: '${orgValue}'.")
def orgMatcher = (orgValue =~ /(.*)-(\d+)/
if (orgMatcher.matches()) {
orgValue = "${m[0][1]}-${(m[0][2] as int) + 1}" }
else { orgValue += "-1" } }
generationState.orgs << orgValue
formatter.formatReference(orgValue)
break } }
private formatText(String s) {
// fix links to internal targets
s = s.eachMatch(/jlp:\/\/([^\s]+)/, s)
// format with formatter
return formatter.formatText(s)
}
}

33
src/main/com/jdblabs/jlp/JLPBaseEmitter.groovy → src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy
View File

@ -2,18 +2,28 @@ package com.jdblabs.jlp
import com.jdblabs.jlp.ast.*
import com.jdblabs.jlp.ast.Directive.DirectiveType
import java.util.List
import java.util.Map
public abstract class JLPBaseEmitter {
public abstract class JLPBaseGenerator {
def generationState
protected Map docState
public JLPBaseEmitter(def generationState) {
this.generationState = generationState }
protected JLPBaseGenerator() {
docState = [orgs: [:],
currentDocId: false ] }
public String emitDocument(List<ASTNode> sourceNodes) {
protected Map<String, String> generate(Map<String, List<ASTNode>> sources) {
Map result = [:]
sources.each { sourceId, sourceNodes ->
docState.currentDocId = sourceId
result[sourceId] = emitDocument(sourceNodes) }
return result }
protected String emitDocument(List<ASTNode> sourceNodes) {
StringBuilder result =
sourceNodes.inject(new StringBuilder()) { sb, node ->
sb.append(emit(node, generationState))
sb.append(emit(node))
return sb }
return result.toString() }
@ -24,15 +34,16 @@ public abstract class JLPBaseEmitter {
printQueue = docBlock.directives.collect { directive ->
def queueItem = [line: directive.lineNumber, value: directive]
switch (direcive.type) {
case DirectiveType.Author: queueItem.priority = 90; break
switch (directive.type) {
case DirectiveType.Author: queueItem.priority = 50; break
case DirectiveType.Doc: queueItem.priority = 50; break
case DirectiveType.Example: queueItem.priority = 50; break
case DirectiveType.Org: queueItem.priority = 10; break
case DirectiveType.Org: queueItem.priority = 10; break }
return queueItem }
printQueue.addAll(docBlock.textBlocks.collect { textBlock ->
[ priority: 50, line: textBlock.lineNumber, value: textBlock ] }
[ priority: 50, line: textBlock.lineNumber, value: textBlock ] })
// sort by priority, then by line number
printQueue.sort(
@ -45,7 +56,7 @@ public abstract class JLPBaseEmitter {
sb.append(emit(printable.value))
return sb }
return result.toString() }
return result.toString() }
protected abstract String emit(TextBlock textBlock)
protected abstract String emit(Directive directive)

38
src/main/com/jdblabs/jlp/JLPPegParser.java
View File

@ -45,7 +45,7 @@ public class JLPPegParser extends BaseParser<Object> {
/**
* Parses the rule:
* CodeBlock = !DOC_START RemainingLine
* CodeBlock = !DocStart RemainingLine
*
* Pushes a CodeBlock onto the stack.
*/
@ -53,24 +53,29 @@ public class JLPPegParser extends BaseParser<Object> {
return Sequence(
push(curLineNum),
push(""),
OneOrMore(FirstOf(
Sequence(
TestNot(DOC_START), RemainingLine(),
push(popAsString() + match())),
Sequence(EmptyLine(),
push(popAsString() + match())))),
OneOrMore(Sequence(
TestNot(DocStart()), RemainingLine(),
push(popAsString() + match()))),
push(makeCodeBlock(popAsString(),popAsInt()))); }
/**
* Parses the rule:
* DocStart = SPACE* DOC_START
*/
Rule DocStart() {
return Sequence(ZeroOrMore(SPACE), DOC_START); }
/**
* Parses the rule:
* DirectiveBlock =
* DOC_START DIRECTIVE_START (LongDirective / LineDirective)
* DocStart DIRECTIVE_START (LongDirective / LineDirective)
*
* Pushes a Directive onto the stack.
*/
Rule DirectiveBlock() {
return Sequence(
DOC_START, DIRECTIVE_START,
DocStart(), DIRECTIVE_START,
FirstOf(LongDirective(), LineDirective())); }
/**
@ -129,13 +134,13 @@ public class JLPPegParser extends BaseParser<Object> {
/**
* Parses the rule:
* DocTextLine =
* DOC_START !DIRECTIVE_START RemainingLine
* DocStart !DIRECTIVE_START RemainingLine
*
* Pushes the line value (not including the DOC_START) onto the stack.
* Pushes the line value (not including the DocStart) onto the stack.
*/
Rule DocTextLine() {
return Sequence(
DOC_START, TestNot(DIRECTIVE_START),
DocStart(), TestNot(DIRECTIVE_START),
RemainingLine(), push(match())); }
/**
@ -144,10 +149,13 @@ public class JLPPegParser extends BaseParser<Object> {
*/
@SuppressSubnodes
Rule RemainingLine() {
return Sequence(OneOrMore(NOT_EOL), FirstOf(EOL, EOI), incLineCount()); }
return FirstOf(
Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()),
Rule EmptyLine() {
return Sequence(EOL, incLineCount()); }
// allow EOI as a line delimiter only if the line is not empty,
// otherwise it will match infinitely if RemainingLine is used in a
// OneOrMore context.
Sequence(OneOrMore(NOT_EOL), EOI)); }
Rule DOC_START = String("%% ");
Rule EOL = Ch('\n');

133
src/main/com/jdblabs/jlp/JLPPegParser.java.noactions
View File

@ -1,133 +0,0 @@
package com.jdblabs.jlp;
import com.jdblabs.jlp.ast.*;
import java.util.ArrayList;
import java.util.List;
import org.parboiled.Action;
import org.parboiled.BaseParser;
import org.parboiled.Context;
import org.parboiled.Rule;
import org.parboiled.annotations.*;
import static com.jdblabs.jlp.ast.TextBlock.makeCodeBlock;
import static com.jdblabs.jlp.ast.TextBlock.makeMarkdownBlock;
@BuildParseTree
public class JLPPegParser extends BaseParser<Object> {
public Rule CodePage() {
return ZeroOrMore(FirstOf(
DocBlock(),
CodeBlock())); }
/**
* Parses the rule:
* DocBlock = DirectiveBlock / MarkdownBlock
*
* Pushes a DocBlock object onto the stack.
*/
Rule DocBlock() {
return OneOrMore(FirstOf(
DirectiveBlock(),
MarkdownBlock())); }
/**
* Parses the rule:
* CodeBlock = !DOC_START RemainingLine
*
* Pushes a CodeBlock onto the stack.
*/
Rule CodeBlock() {
return Sequence(
TestNot(DOC_START), RemainingLine(),
ZeroOrMore(Sequence(
TestNot(DOC_START), RemainingLine()))); }
/**
* Parses the rule:
* DirectiveBlock =
* DOC_START DIRECTIVE_START (LongDirective / LineDirective)
*
* Pushes a Directive onto the stack.
*/
Rule DirectiveBlock() {
return Sequence(
DOC_START, DIRECTIVE_START,
FirstOf(LongDirective(), LineDirective())); }
/**
* Parses the rule:
* LongDirective =
* (AUTHOR_DIR / DOC_DIR / EXAMPLE_DIR) RemainingLine MarkdownBlock?
*
* Pushes a Directive object onto the value stack.
*/
Rule LongDirective() {
return Sequence(
FirstOf(AUTHOR_DIR, DOC_DIR, EXAMPLE_DIR),
RemainingLine(),
Optional(MarkdownBlock())); }
/**
* Parses the rule:
* LineDirective =
* ORG_DIR RemainingLine
*
* Pushes a Directive object onto the value stack.
*/
Rule LineDirective() {
return Sequence(
ORG_DIR,
RemainingLine()); }
/**
* Parses the rule:
* MarkdownBlock = MarkdownLine+
*
* Pushes a MarkdownBlock onto the stack as a string.
*/
Rule MarkdownBlock() { return OneOrMore(MarkdownLine()); }
/**
* Parses the rule:
* MarkdownLine =
* DOC_START !DIRECTIVE_START RemainingLine
*
* Pushes the line value (not including the DOC_START) onto the stack.
*/
Rule MarkdownLine() {
return Sequence(
DOC_START, TestNot(DIRECTIVE_START), RemainingLine()); }
/**
* Parses the rule:
* RemainingLine = (!EOL)+ EOL
*/
@SuppressSubnodes
Rule RemainingLine() {
return Sequence(OneOrMore(NOT_EOL), EOL); }
Rule DOC_START = String("%% ");
Rule EOL = FirstOf(Ch('\n'), EOI);
Rule NOT_EOL = Sequence(TestNot(EOL), ANY);
Rule DIRECTIVE_START= Ch('@');
Rule SLASH = Ch('/');
// directive terminals
Rule AUTHOR_DIR = IgnoreCase("author");
Rule DOC_DIR = IgnoreCase("doc");
Rule EXAMPLE_DIR = IgnoreCase("example");
Rule ORG_DIR = IgnoreCase("org");
String popAsString() {
return (String) pop(); }
List<ASTNode> addToList(ASTNode value, List<ASTNode> list) {
list.add(value);
return list; }
boolean printValueStack() {
for (int i = 0; i < getContext().getValueStack().size(); i++) {
System.out.println(i + ": " + peek(i)); }
return true; }
}

23
src/main/com/jdblabs/jlp/MarkdownEmitter.groovy
View File

@ -1,23 +0,0 @@
package com.jdblabs.jlp
import com.jdblabs.jlp.ast.*
import org.pegdown.PegDownParser
public class MarkdownEmitter extends JLPEmitter {
protected MarkdownEmitter() {}
def pegdown = new PegDownParser()
protected String emitAuthor(String value) {
'<span class="author">${value}</span>' }
protected String emitDoc(String value) { /* parse as MD */ }
protected String emitExample(String value) {/* parse as MD */ }
protected String emitOrg(String value) { }
protected String emitBlock(TextBlock textBlock) { "todo" }
}

16
src/main/com/jdblabs/jlp/MarkdownFormatter.groovy
View File

@ -1,16 +0,0 @@
package com.jdblabs.jlp
public class MarkdownFormatter implements Formatter {
private PegDownProcessor pegdown
public MarkdownFormatter() {
pegdown = new PegDownProcessor() }
public String formatText(String s) { pegdown.markdownToHtml(s) }
public String formatCode(String s) {
pegdown.markdownToHtml(s.replaceAll(/(^|\n)/, /$1 /)) }
public String formatReference(String s) { '<a name="${s}"/>' }
}

64
src/main/com/jdblabs/jlp/MarkdownGenerator.groovy Normal file
View File

@ -0,0 +1,64 @@
package com.jdblabs.jlp
import com.jdblabs.jlp.ast.*
import com.jdblabs.jlp.ast.Directive.DirectiveType
import com.jdblabs.jlp.ast.TextBlock.TextBlockType
import org.pegdown.PegDownProcessor
import org.pegdown.Extensions
public class MarkdownGenerator extends JLPBaseGenerator {
protected PegDownProcessor pegdown
protected MarkdownGenerator() {
super()
pegdown = new PegDownProcessor(Extensions.TABLES) }
protected static Map<String, String> generateDocuments(
Map<String, List<ASTNode>> sources) {
MarkdownGenerator inst = new MarkdownGenerator()
return inst.generate(sources) }
protected String emit(TextBlock textBlock) {
switch (textBlock.type) {
// text block, just convert to markdown
case TextBlockType.TextBlock:
return formatText(textBlock.value)
// code block, we want to emit as a code snippet
case TextBlockType.CodeBlock:
// so prepend all lines with four spaces to tell markdown that
// this is code
String value = textBlock.value.replaceAll(/(^|\n)/, /$1 /)
// then convert to markdown
return pegdown.markdownToHtml(value) } }
protected String emit(Directive d) {
switch (d.type) {
case DirectiveType.Author:
return "<span class='author'>Author: ${formatText(d.value)}</span>"
case DirectiveType.Doc:
return formatText(d.value)
case DirectiveType.Example:
return formatText(d.value)
case DirectiveType.Org:
docState.orgs[d.value] = [line: d.lineNumber,
file: docState.currentDocId]
return "<a name='${d.value}'/>" }
}
protected String formatText(String text) {
// convert to HTML from Markdown
String md = pegdown.markdownToHtml(text)
// replace internal `jlp://` links with actual links based on`@org`
// references
md = md.replaceAll(/jlp:\/\/([^\s"])/, /#$1/)
return md;
}
}

7
src/main/com/jdblabs/jlp/ParserActions.groovy
View File

@ -1,7 +0,0 @@
package com.jdblabs.jlp
import org.parboiled.Action
public class ParserActions {
}

7
src/main/com/jdblabs/jlp/TransparentFormatter.groovy
View File

@ -1,7 +0,0 @@
package com.jdblabs.jlp
public class TransparentFormatter implements Formatter {
public String formatText(String text) { return text }
public String formatCode(String code) { return code }
public String formatReference(String ref) { return "ref#${ref}" } }

24
src/main/com/jdblabs/jlp/TransparentGenerator.groovy Normal file
View File

@ -0,0 +1,24 @@
package com.jdblabs.jlp
import com.jdblabs.jlp.ast.*
import com.jdblabs.jlp.ast.Directive.DirectiveType
import java.util.List
import java.util.Map
public class TransparentGenerator extends JLPBaseGenerator {
protected TransparentGenerator() {}
public static Map<String, String> generateDocuments(Map<String,
List<ASTNode>> sources) {
TransparentGenerator inst = new TransparentGenerator()
return inst.generate(sources) }
protected String emit(TextBlock textBlock) { textBlock.value }
protected String emit(Directive directive) {
switch (directive.type) {
case DirectiveType.Author: return "Author: ${directive.value}\n"
case DirectiveType.Doc: return directive.value
case DirectiveType.Example: return "Example: ${directive.value}"
case DirectiveType.Org: return "" } }
}