jdb/jlp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support for multi=line comments, detects file type.

Browse Source 
* Added support for multi-line comments to the JLPPegParser grammar
  implementation.
* Added a Java sample file.
* Updated test script to add convenience functions for the java test file and
  for using a TracingParseRunner for parse runs.
* Added an option, `--css-file`, to allow the caller to specify their own css
  file.
* Added basic logic to the Processor class to detect source file types and build
  a parser and a generator for that source type. Support currently exists for
  the following languages: C (.c, .h), C++ (.cpp, .c++, .hpp, .h++), Erlang
  (.erl), Groovy (.groovy), Java (.java), JavaScript (.js).
This commit is contained in:
Jonathan Bernard 2011-12-25 22:07:48 -06:00
parent 3c34e080ef
commit 9eb80e91a6
8 changed files with 367 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
doc/grammar.rst
View File

@ -5,25 +5,53 @@ Block ->
DocBlock CodeBlock
DocBlock ->
(Directive / DocText)+
(SDocBlock / MDocBlock)
Directive ->
DocLineStart AT (LongDirective / ShortDirective)
SDocBlock ->
(SDirective / SDocText)+
LongDirective ->
("author" / "doc" / "example") RemainingLine DocText?
ShortDirective ->
("org" / "copyright") RemainingLine
DocText ->
(DocLineStart !AT RemainingLine)+
DocLineStart ->
Space* DOC_LINE_START Space?
MDocBlock ->
MDOC_START (!MDOC_END / MDirective / MDocText)* MDOC_END
CodeBlock ->
(!DocLineStart RemainingLine)+
(RemainingCodeLine)+
RemainingLine ->
((!EOL)* EOL) / ((!EOL)+ EOI)
SDirective ->
SDocLineStart AT (SLongDirective / SShortDirective)
MDirective ->
MDocLineStart? AT (MLongDirective / MShortDirective)
SLongDirective ->
("api" / "example") RemainingSDocLine SDocText?
MLongDirective ->
("api" / "example") RemainingMDocLine MDocText?
SShortDirective ->
("author" / "org" / "copyright") RemainingSDocLine
MShortDirective ->
("author" / "org" / "copyright") RemainingMDocLine
SDocText ->
(SDocLineStart !AT RemainingSDocLine)+
MDocText ->
(MDocLineStart? !AT RemainingMDocLine)+
SDocLineStart ->
SPACE* SDOC_START SPACE?
MDocLineStart ->
SPACE* !MDOC_END MDOC_LINE_START SPACE?
RemainingSDocLine ->
((!EOL)* EOL) / ((!EOL)+ EOI)
RemainingMDocLine ->
((!(EOL / MDOC_END))* EOL) / ((!MDOC_END)+)
RemainingCodeLine ->
((!(EOL / MDOC_START / SDocLineStart))* EOL) /
(!(MDOC_START / SDocLineStart))+

6
project.properties
View File

@ -1,6 +1,6 @@
#Mon, 12 Sep 2011 10:56:06 -0500
#Sun, 25 Dec 2011 21:56:17 -0600
name=jlp
version=0.3
build.number=8
version=1.0
build.number=0
lib.local=true
release.dir=release

26
resources/main/Test.java Normal file
View File

@ -0,0 +1,26 @@
package test;
import java.util.Array;
/** This is a test class. Mainly for testing my parser.
* It should work. And now we are just filing space.
* @author Jonathan Bernard
* @copyright JDB Labs 2011 */
public class Test {
/**
| @org test-ref
| This is an embedded comment.
| It spreads over at least 3 lines.
| And this is the third line.
*/
public static void main(String[] args) {
/** Yes, this is a hello world example. */
System.out.println("Hello World!");
}
/// This is a single-line comment block. */ /** Other comment
/// modifiers should not matter within this block.
/// @org last-doc
}

44
resources/main/test.groovy
View File

@ -1,21 +1,30 @@
import com.jdblabs.jlp.*
import org.parboiled.Parboiled
import org.parboiled.parserunners.ReportingParseRunner
import org.parboiled.parserunners.RecoveringParseRunner
import org.parboiled.parserunners.*
"Making the standard parser."
"---------------------------"
parser = Parboiled.createParser(JLPPegParser.class)
parseRunner = new ReportingParseRunner(parser.SourceFile())
jp = Parboiled.createParser(JLPPegParser.class)
ep = Parboiled.createParser(JLPPegParser, '%%')
jrpr = new ReportingParseRunner(jp.SourceFile())
jtpr = new TracingParseRunner(jp.SourceFile())
erpr = new ReportingParseRunner(ep.SourceFile())
etpr = new TracingParseRunner(ep.SourceFile())
simpleTest = {
vbsFile = new File('vbs_db_records.hrl')
javaFile = new File('Test.java')
docsDir = new File('jlp-docs')
docsDir.mkdirs()
simpleTest = { parseRunner ->
println "Parsing the simple test into 'result'."
println "--------------------------------------"
testLine = """%% This the first test line.
%% Second Line
Actual third line that screws stuff up.
%% Third Line \n\n Fifth line \n\n %% Seventh line \n\n
%% @author Eigth Line
%% @Example Ninth Line
@ -26,22 +35,35 @@ simpleTest = {
simpleResult = parseRunner.run(testLine)
}
vbsTest = {
vbsTest = { parseRunner ->
println "Parsing vbs_db_records.hrl into 'vbsResult'."
println "--------------------------------------------"
vbsTestFile = new File('vbs_db_records.hrl')
println "vbsTestFile is ${vbsTestFile.exists() ? 'present' : 'absent'}."
vbsTestInput = vbsTestFile.text
println "vbsFile is ${vbsFile.exists() ? 'present' : 'absent'}."
vbsTestInput = vbsFile.text
vbsParsed = parseRunner.run(vbsTestInput)
vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl"
/*vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl"
println "Writing to file 'vbs_db_records.html'."
println "--------------------------------------"
(new File('vbs_db_records.html')).withWriter { out -> out.println vbsResult }
return [vbsParsed, vbsResult]
return [vbsParsed, vbsResult]*/
return vbsParsed
}
javaTest = { parseRunner ->
println "Parsing Test.java into 'javaResult'."
println "------------------------------------"
println "javaFile is ${javaFile.exists() ? 'present' : 'absent'}."
javaTestInput = javaFile.text
javaParsed = parseRunner.run(javaTestInput)
javaSF = javaParsed.valueStack.peek()
return [javaParsed: javaParsed, javaSF: javaSF]
}

24
src/main/com/jdblabs/jlp/JLPMain.groovy
View File

@ -18,6 +18,8 @@ public class JLPMain {
cli.o("Output directory (defaults to 'jlp-docs').",
longOpt: 'output-dir', args: 1, argName: "output-dir",
required: false)
cli._('Use <css-file> for the documentation css.',
longOpt: 'css-file', args: 1, required: false, argName: 'css-file')
cli._(longOpt: 'relative-path-root', args: 1, required: false,
'Resolve all relative paths against this root.')
@ -47,8 +49,26 @@ public class JLPMain {
// create the output directory if it does not exist
if (!outputDir.exists()) outputDir.mkdirs()
// get the CSS theme to use
def css = JLPMain.class.getResourceAsStream("/jlp.css") // TODO: make an option
// get the CSS theme to use. We will start by assuming the default will
// be used.
def css = JLPMain.class.getResourceAsStream("/jlp.css")
// If the CSS file was specified on the command-line, let's look for it.
if (opts.'css-file') {
def cssFile = new File(opts.'css-file')
// resolve against our relative root
if (!cssFile.isAbsolute()) {
cssFile = new File(pathRoot, cssFile.path) }
// Finally, make sure the file actually exists.
if (cssFile.exists()) { css = cssFile }
else {
println "WARN: Could not fine the custom CSS file: '" +
"${cssFile.canonicalPath}'."
println " Using the default CSS." }}
// Extract the text from our css source (either an InputStream or a
// File)
css = css.text
// get files passed in

211
src/main/com/jdblabs/jlp/JLPPegParser.java
View File

@ -14,6 +14,23 @@ public class JLPPegParser extends BaseParser<Object> {
int curLineNum = 1;
public JLPPegParser(String mdocStart, String mdocEnd,
String mdocLineStart, String sdocStart) {
MDOC_START = String(mdocStart).label("MDOC_START");
MDOC_END = String(mdocEnd).label("MDOC_END");
SDOC_START = String(sdocStart).label("SDOC_START");
MDOC_LINE_START = AnyOf(mdocLineStart).label("MDOC_LINE_START"); }
public JLPPegParser(String sdocStart) {
MDOC_START = NOTHING;
MDOC_LINE_START = NOTHING;
MDOC_END = NOTHING;
SDOC_START = String(sdocStart).label("SDOC_START"); }
public JLPPegParser() {
this("/**", "*/", "!#$%^&*()_-=+|;:'\",<>?~`", "///"); }
/**
* Parses the rule:
* SourceFile = (Block / DocBlock / CodeBlock)+
@ -95,60 +112,99 @@ public class JLPPegParser extends BaseParser<Object> {
push(curLineNum),
DocBlock(), CodeBlock(),
// A DocBlock and a CodeBlock are pushed onto the stack by the
// above rules. Pop them off, along with the line number we pushed
// before that, and create a new Block node.
push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); }
/**
* Parses the rule:
* DocBlock = (Directive / DocText)+
* DocBlock = SDocBlock / MDocBlock
*
* Pushes a DocBlock onto the stack.
*/
Rule DocBlock() { return FirstOf(SDocBlock(), MDocBlock()); }
/**
* Parses the rule:
* SDocBlock = (SDirective / SDocText)+
*
* Pushes a DocBlock object onto the stack
*/
Rule DocBlock() {
Rule SDocBlock() {
return Sequence(
push(new DocBlock(curLineNum)),
OneOrMore(Sequence(
FirstOf(Directive(), DocText()),
FirstOf(SDirective(), SDocText()),
addToDocBlock((ASTNode) pop())))); }
/**
* Parses the rule:
* CodeBlock = (!DocLineStart RemainingLine)+
* MDocBlock = MDOC_START (MDirective / MDocText)+ MDOC_END
*
* Pushes a DocBlock object onto the stack
*/
Rule MDocBlock() {
return Sequence(
push(new DocBlock(curLineNum)),
MDOC_START,
ZeroOrMore(Sequence(
// We need to be careful to exclude MDOC_END here, as there can
// be some confusion otherwise between the start of a line with
// MDOC_LINE_START and MDOC_END depending on what values the
// user has chosen for them
TestNot(MDOC_END), FirstOf(MDirective(), MDocText()),
addToDocBlock((ASTNode) pop()))),
MDOC_END); }
/**
* Parses the rule:
* CodeBlock = (RemainingCodeLine)+
*
* Pushes a CodeBlock onto the stack.
*/
Rule CodeBlock() {
return Sequence(
push(new CodeBlock(curLineNum)),
OneOrMore(Sequence(
TestNot(DocLineStart()), RemainingLine(),
OneOrMore(Sequence(RemainingCodeLine(),
addToCodeBlock(match())))); }
/**
* Parses the rule:
* Directive = DocLineStart AT (LongDirective / ShortDirective)
* SDirective = SDocLineStart AT (SLongDirective / SShortDirective)
*
* Pushes a Directive node on the stack.
*/
Rule Directive() {
Rule SDirective() {
return Sequence(
DocLineStart(), AT, FirstOf(LongDirective(), ShortDirective())); }
SDocLineStart(), AT, FirstOf(SLongDirective(), SShortDirective())); }
/**
* Parses the rule:
* LongDirective =
* (API_DIR / EXAMPLE_DIR) RemainingLine DocText?
* MDirective = MDocLineStart? AT (MLongDirective / MShortDirective)
*
* Pushes a Directive node onto the stack.
*/
Rule LongDirective() {
Rule MDirective() {
return Sequence(
Optional(MDocLineStart()),
AT, FirstOf(MLongDirective(), MShortDirective())); }
/**
* Parses the rule:
* SLongDirective =
* (API_DIR / EXAMPLE_DIR) RemainingSDocLine SDocText?
*
* Pushes a Directive node onto the stack.
*/
Rule SLongDirective() {
return Sequence(
push(curLineNum),
FirstOf(API_DIR, EXAMPLE_DIR), push(match()),
RemainingLine(), push(match()),
RemainingSDocLine(), push(match()),
Optional(Sequence(
DocText(),
SDocText(),
swap(),
push(popAsString() + ((DocText) pop()).value))),
@ -156,48 +212,151 @@ public class JLPPegParser extends BaseParser<Object> {
/**
* Parses the rule:
* ShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingLine
* MLongDirective =
* (API_DIR / EXAMPLE_DIR) RemainingMDocLine MDocText?
*
* Pushes a Directive node onto the stack.
*/
Rule ShortDirective() {
Rule MLongDirective() {
return Sequence(
push(curLineNum),
FirstOf(API_DIR, EXAMPLE_DIR), push(match()),
RemainingMDocLine(), push(match()),
Optional(Sequence(
MDocText(),
swap(),
push(popAsString() + ((DocText) pop()).value))),
push(new Directive(popAsString(), popAsString(), popAsInt()))); }
/**
* Parses the rule:
* SShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingSDocLine
*
* Pushes a Directive node onto the stack.
*/
Rule SShortDirective() {
return Sequence(
push(curLineNum),
FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()),
RemainingLine(),
RemainingSDocLine(),
push(new Directive(match().trim(), popAsString(), popAsInt()))); }
/**
* Parses the rule:
* DocText = (DocLineStart !AT RemainingLine)+
* MShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingMDocLine
*
* Pushes a Directive node onto the stack.
*/
Rule MShortDirective() {
return Sequence(
push(curLineNum),
FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()),
RemainingMDocLine(),
push(new Directive(match().trim(), popAsString(), popAsInt()))); }
/**
* Parses the rule:
* SDocText = (SDocLineStart !AT RemainingSDocLine)+
*
* Pushes a DocText node onto the stack.
*/
Rule DocText() {
Rule SDocText() {
return Sequence(
push(new DocText(curLineNum)),
OneOrMore(Sequence(
DocLineStart(), TestNot(AT), RemainingLine(),
SDocLineStart(), TestNot(AT), RemainingSDocLine(),
addToDocText(match())))); }
Rule DocLineStart() {
/**
* Parses the rule:
* MDocText = (MDocLineStart? !AT RemainingMDocLine)+
*
* Pushes a DocText node onto the stack.
*/
Rule MDocText() {
return Sequence(
ZeroOrMore(SPACE), DOC_LINE_START, Optional(SPACE)); }
push(new DocText(curLineNum)),
OneOrMore(Sequence(
Optional(MDocLineStart()),
TestNot(AT), RemainingMDocLine(),
addToDocText(match())))); }
Rule NonEmptyLine() {
return Sequence(OneOrMore(NOT_EOL), FirstOf(EOL, EOI)); }
/**
* Parses the rule:
* SDocLineStart = SPACE* SDOC_START SPACE?
*/
Rule SDocLineStart() {
return Sequence(
ZeroOrMore(SPACE), SDOC_START, Optional(SPACE)); }
Rule RemainingLine() {
/**
* Parses the rule:
* MDocLineStart = SPACE* !MDOC_END MDOC_LINE_START SPACE?
*/
Rule MDocLineStart() {
return Sequence(
ZeroOrMore(SPACE), TestNot(MDOC_END), MDOC_LINE_START, Optional(SPACE)); }
/**
* Parses the rule:
* RemainingSDocLine = ((!EOL)* EOL) / ((!EOL)+ EOI)
*/
Rule RemainingSDocLine() {
return FirstOf(
Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()),
Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); }
/**
* Parses the rule:
* RemainingMDocLine =
* ((!(EOL / MDOC_END))* EOL) /
* ((!MDOC_END)+)
*/
Rule RemainingMDocLine() {
return FirstOf(
// End of line, still within the an M-style comment block
Sequence(
ZeroOrMore(Sequence(TestNot(FirstOf(EOL, MDOC_END)), ANY)),
EOL,
incLineCount()),
// End of M-style comment block
OneOrMore(Sequence(TestNot(MDOC_END), ANY))); }
/**
* Parses the rule:
* RemainingCodeLine =
* ((!(EOL / MDOC_START / SDocLineStart))* EOL) /
* (!(MDOC_START / SDocLineStart))+
*/
Rule RemainingCodeLine() {
return FirstOf(
// End of line, still within the code block.
Sequence(
ZeroOrMore(Sequence(
TestNot(FirstOf(EOL, MDOC_START, SDocLineStart())),
ANY)),
EOL,
incLineCount()),
// Found an MDOC_START or SDocLineStart
OneOrMore(Sequence(TestNot(FirstOf(MDOC_START, SDocLineStart())), ANY))); }
Rule AT = Ch('@').label("AT");
Rule EOL = FirstOf(String("\r\n"), Ch('\n'), Ch('\r')).label("EOL");
Rule NOT_EOL = Sequence(TestNot(EOL), ANY).label("NOT_EOL");
Rule SPACE = AnyOf(" \t").label("SPACE");
Rule DOC_LINE_START = String("%%").label("DOC_LINE_START");
// Configurable
Rule MDOC_START;
Rule MDOC_END;
Rule MDOC_LINE_START;
Rule SDOC_START;
// directive terminals
Rule AUTHOR_DIR = IgnoreCase("author");

69
src/main/com/jdblabs/jlp/Processor.groovy
View File

@ -22,8 +22,8 @@ public class Processor {
// shortcut for docs[currentDocId]
public TargetDoc currentDoc
protected Map<Class, BaseParser> parsers = [:]
protected Map<Class, JLPBaseGenerator> generators = [:]
protected Map<String, BaseParser> parsers = [:]
protected Map<String, JLPBaseGenerator> generators = [:]
public static void process(File outputDir, String css,
List<File> inputFiles) {
@ -60,9 +60,10 @@ public class Processor {
// TODO: add logic to configure or autodetect the correct parser for
// each file
def parser = getParser(JLPPegParser)
def parser = getParser(sourceTypeForFile(currentDoc.sourceFile))
def parseRunner = new ReportingParseRunner(parser.SourceFile())
// TODO: error detection
currentDoc.sourceAST = parseRunner.run(
currentDoc.sourceFile.text).resultValue }
@ -71,7 +72,8 @@ public class Processor {
// TODO: add logic to configure or autodetect the correct generator
// for each file
def generator = getGenerator(LiterateMarkdownGenerator)
def generator =
getGenerator(sourceTypeForFile(currentDoc.sourceFile))
generator.parse(currentDoc.sourceAST) }
@ -80,7 +82,8 @@ public class Processor {
// TODO: add logic to configure or autodetect the correct generator
// for each file
def generator = getGenerator(LiterateMarkdownGenerator)
def generator =
getGenerator(sourceTypeForFile(currentDoc.sourceFile))
currentDoc.output = generator.emit(currentDoc.sourceAST) }
// Write the output to the output directory
@ -94,8 +97,7 @@ public class Processor {
File outputDir = outputFile.parentFile
// create the directory if need be
if (!outputDir.exists()) {
outputDir.mkdirs() }
if (!outputDir.exists()) { outputDir.mkdirs() }
// write the css file if it does not exist
File cssFile = new File(outputDir, "jlp.css")
@ -159,17 +161,50 @@ public class Processor {
return new File(newPath.join('/')) }
protected getGenerator(Class generatorClass) {
if (generators[generatorClass] == null) {
def constructor = generatorClass.getConstructor(Processor)
generators[generatorClass] = constructor.newInstance(this)
}
public static sourceTypeForFile(File sourceFile) {
String extension
def nameParts = sourceFile.name.split(/\./)
return generators[generatorClass] }
if (nameParts.length == 1) { return 'binary' }
else { extension = nameParts[-1] }
protected getParser(Class parserClass) {
if (parsers[parserClass] == null) {
parsers[parserClass] = Parboiled.createParser(parserClass) }
switch (extension) {
case 'c': case 'h': return 'c';
case 'c++': case 'h++': case 'cpp': case 'hpp': return 'c++';
case 'erl': case 'hrl': return 'erlang';
case 'groovy': return 'groovy';
case 'java': return 'java';
case 'js': return 'javascript';
default: return 'unknown'; }}
return parsers[parserClass] }
protected getGenerator(String sourceType) {
if (generators[sourceType] == null) {
switch(sourceType) {
default:
generators[sourceType] =
new LiterateMarkdownGenerator(this) }}
return generators[sourceType] }
protected getParser(String sourceType) {
println "Looking for a ${sourceType} parser."
if (parsers[sourceType] == null) {
switch(sourceType) {
case 'erlang':
parsers[sourceType] = Parboiled.createParser(
JLPPegParser, '%%')
println "Built an erlang parser."
break
case 'c':
case 'c++':
case 'groovy':
case 'java':
case 'javascript':
default:
parsers[sourceType] = Parboiled.createParser(JLPPegParser,
'/**', '*/', '!#$%^&*()_-=+|;:\'",<>?~`', '///')
println "Built a java parser."
break }}
return parsers[sourceType] }
}

6
src/main/com/jdblabs/jlp/ast/Directive.groovy
View File

@ -3,11 +3,7 @@ package com.jdblabs.jlp.ast
public class Directive extends ASTNode {
public static enum DirectiveType {
Api,
Author,
Copyright,
Example,
Org;
Api, Author, Copyright, Example, Org;
public static DirectiveType parse(String typeString) {
valueOf(typeString.toLowerCase().capitalize()) } }