From 9eb80e91a65fbc1a2a7a92693fcc72753e109489 Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Sun, 25 Dec 2011 22:07:48 -0600 Subject: [PATCH] Support for multi=line comments, detects file type. * Added support for multi-line comments to the JLPPegParser grammar implementation. * Added a Java sample file. * Updated test script to add convenience functions for the java test file and for using a TracingParseRunner for parse runs. * Added an option, `--css-file`, to allow the caller to specify their own css file. * Added basic logic to the Processor class to detect source file types and build a parser and a generator for that source type. Support currently exists for the following languages: C (.c, .h), C++ (.cpp, .c++, .hpp, .h++), Erlang (.erl), Groovy (.groovy), Java (.java), JavaScript (.js). --- doc/grammar.rst | 62 +++-- project.properties | 6 +- resources/main/Test.java | 26 +++ resources/main/test.groovy | 44 +++- src/main/com/jdblabs/jlp/JLPMain.groovy | 24 +- src/main/com/jdblabs/jlp/JLPPegParser.java | 211 +++++++++++++++--- src/main/com/jdblabs/jlp/Processor.groovy | 69 ++++-- src/main/com/jdblabs/jlp/ast/Directive.groovy | 6 +- 8 files changed, 367 insertions(+), 81 deletions(-) create mode 100644 resources/main/Test.java diff --git a/doc/grammar.rst b/doc/grammar.rst index 4634cc0..f76c5c0 100644 --- a/doc/grammar.rst +++ b/doc/grammar.rst @@ -5,25 +5,53 @@ Block -> DocBlock CodeBlock DocBlock -> - (Directive / DocText)+ + (SDocBlock / MDocBlock) -Directive -> - DocLineStart AT (LongDirective / ShortDirective) +SDocBlock -> + (SDirective / SDocText)+ -LongDirective -> - ("author" / "doc" / "example") RemainingLine DocText? - -ShortDirective -> - ("org" / "copyright") RemainingLine - -DocText -> - (DocLineStart !AT RemainingLine)+ - -DocLineStart -> - Space* DOC_LINE_START Space? +MDocBlock -> + MDOC_START (!MDOC_END / MDirective / MDocText)* MDOC_END CodeBlock -> - (!DocLineStart RemainingLine)+ + (RemainingCodeLine)+ -RemainingLine -> - ((!EOL)* EOL) / ((!EOL)+ EOI) +SDirective -> + SDocLineStart AT (SLongDirective / SShortDirective) + +MDirective -> + MDocLineStart? AT (MLongDirective / MShortDirective) + +SLongDirective -> + ("api" / "example") RemainingSDocLine SDocText? + +MLongDirective -> + ("api" / "example") RemainingMDocLine MDocText? + +SShortDirective -> + ("author" / "org" / "copyright") RemainingSDocLine + +MShortDirective -> + ("author" / "org" / "copyright") RemainingMDocLine + +SDocText -> + (SDocLineStart !AT RemainingSDocLine)+ + +MDocText -> + (MDocLineStart? !AT RemainingMDocLine)+ + +SDocLineStart -> + SPACE* SDOC_START SPACE? + +MDocLineStart -> + SPACE* !MDOC_END MDOC_LINE_START SPACE? + +RemainingSDocLine -> + ((!EOL)* EOL) / ((!EOL)+ EOI) + +RemainingMDocLine -> + ((!(EOL / MDOC_END))* EOL) / ((!MDOC_END)+) + +RemainingCodeLine -> + ((!(EOL / MDOC_START / SDocLineStart))* EOL) / + (!(MDOC_START / SDocLineStart))+ diff --git a/project.properties b/project.properties index be87361..7b363f1 100644 --- a/project.properties +++ b/project.properties @@ -1,6 +1,6 @@ -#Mon, 12 Sep 2011 10:56:06 -0500 +#Sun, 25 Dec 2011 21:56:17 -0600 name=jlp -version=0.3 -build.number=8 +version=1.0 +build.number=0 lib.local=true release.dir=release diff --git a/resources/main/Test.java b/resources/main/Test.java new file mode 100644 index 0000000..a7f1c32 --- /dev/null +++ b/resources/main/Test.java @@ -0,0 +1,26 @@ +package test; + +import java.util.Array; + +/** This is a test class. Mainly for testing my parser. + * It should work. And now we are just filing space. + * @author Jonathan Bernard + * @copyright JDB Labs 2011 */ +public class Test { + + /** + | @org test-ref + | This is an embedded comment. + | It spreads over at least 3 lines. + | And this is the third line. + */ + + public static void main(String[] args) { + /** Yes, this is a hello world example. */ + System.out.println("Hello World!"); + } + + /// This is a single-line comment block. */ /** Other comment + /// modifiers should not matter within this block. + /// @org last-doc +} diff --git a/resources/main/test.groovy b/resources/main/test.groovy index 98e11d1..05d2184 100644 --- a/resources/main/test.groovy +++ b/resources/main/test.groovy @@ -1,21 +1,30 @@ import com.jdblabs.jlp.* import org.parboiled.Parboiled -import org.parboiled.parserunners.ReportingParseRunner -import org.parboiled.parserunners.RecoveringParseRunner +import org.parboiled.parserunners.* "Making the standard parser." "---------------------------" -parser = Parboiled.createParser(JLPPegParser.class) -parseRunner = new ReportingParseRunner(parser.SourceFile()) +jp = Parboiled.createParser(JLPPegParser.class) +ep = Parboiled.createParser(JLPPegParser, '%%') +jrpr = new ReportingParseRunner(jp.SourceFile()) +jtpr = new TracingParseRunner(jp.SourceFile()) +erpr = new ReportingParseRunner(ep.SourceFile()) +etpr = new TracingParseRunner(ep.SourceFile()) -simpleTest = { +vbsFile = new File('vbs_db_records.hrl') +javaFile = new File('Test.java') +docsDir = new File('jlp-docs') +docsDir.mkdirs() + +simpleTest = { parseRunner -> println "Parsing the simple test into 'result'." println "--------------------------------------" testLine = """%% This the first test line. %% Second Line + Actual third line that screws stuff up. %% Third Line \n\n Fifth line \n\n %% Seventh line \n\n %% @author Eigth Line %% @Example Ninth Line @@ -26,22 +35,35 @@ simpleTest = { simpleResult = parseRunner.run(testLine) } -vbsTest = { +vbsTest = { parseRunner -> println "Parsing vbs_db_records.hrl into 'vbsResult'." println "--------------------------------------------" - vbsTestFile = new File('vbs_db_records.hrl') - println "vbsTestFile is ${vbsTestFile.exists() ? 'present' : 'absent'}." - vbsTestInput = vbsTestFile.text + println "vbsFile is ${vbsFile.exists() ? 'present' : 'absent'}." + vbsTestInput = vbsFile.text vbsParsed = parseRunner.run(vbsTestInput) - vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl" + /*vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl" println "Writing to file 'vbs_db_records.html'." println "--------------------------------------" (new File('vbs_db_records.html')).withWriter { out -> out.println vbsResult } - return [vbsParsed, vbsResult] + return [vbsParsed, vbsResult]*/ + return vbsParsed +} + +javaTest = { parseRunner -> + println "Parsing Test.java into 'javaResult'." + println "------------------------------------" + + println "javaFile is ${javaFile.exists() ? 'present' : 'absent'}." + javaTestInput = javaFile.text + + javaParsed = parseRunner.run(javaTestInput) + javaSF = javaParsed.valueStack.peek() + + return [javaParsed: javaParsed, javaSF: javaSF] } diff --git a/src/main/com/jdblabs/jlp/JLPMain.groovy b/src/main/com/jdblabs/jlp/JLPMain.groovy index d2472a9..97742c7 100644 --- a/src/main/com/jdblabs/jlp/JLPMain.groovy +++ b/src/main/com/jdblabs/jlp/JLPMain.groovy @@ -18,6 +18,8 @@ public class JLPMain { cli.o("Output directory (defaults to 'jlp-docs').", longOpt: 'output-dir', args: 1, argName: "output-dir", required: false) + cli._('Use for the documentation css.', + longOpt: 'css-file', args: 1, required: false, argName: 'css-file') cli._(longOpt: 'relative-path-root', args: 1, required: false, 'Resolve all relative paths against this root.') @@ -47,8 +49,26 @@ public class JLPMain { // create the output directory if it does not exist if (!outputDir.exists()) outputDir.mkdirs() - // get the CSS theme to use - def css = JLPMain.class.getResourceAsStream("/jlp.css") // TODO: make an option + // get the CSS theme to use. We will start by assuming the default will + // be used. + def css = JLPMain.class.getResourceAsStream("/jlp.css") + + // If the CSS file was specified on the command-line, let's look for it. + if (opts.'css-file') { + def cssFile = new File(opts.'css-file') + // resolve against our relative root + if (!cssFile.isAbsolute()) { + cssFile = new File(pathRoot, cssFile.path) } + + // Finally, make sure the file actually exists. + if (cssFile.exists()) { css = cssFile } + else { + println "WARN: Could not fine the custom CSS file: '" + + "${cssFile.canonicalPath}'." + println " Using the default CSS." }} + + // Extract the text from our css source (either an InputStream or a + // File) css = css.text // get files passed in diff --git a/src/main/com/jdblabs/jlp/JLPPegParser.java b/src/main/com/jdblabs/jlp/JLPPegParser.java index 088016e..79d54a9 100644 --- a/src/main/com/jdblabs/jlp/JLPPegParser.java +++ b/src/main/com/jdblabs/jlp/JLPPegParser.java @@ -14,6 +14,23 @@ public class JLPPegParser extends BaseParser { int curLineNum = 1; + public JLPPegParser(String mdocStart, String mdocEnd, + String mdocLineStart, String sdocStart) { + + MDOC_START = String(mdocStart).label("MDOC_START"); + MDOC_END = String(mdocEnd).label("MDOC_END"); + SDOC_START = String(sdocStart).label("SDOC_START"); + MDOC_LINE_START = AnyOf(mdocLineStart).label("MDOC_LINE_START"); } + + public JLPPegParser(String sdocStart) { + MDOC_START = NOTHING; + MDOC_LINE_START = NOTHING; + MDOC_END = NOTHING; + SDOC_START = String(sdocStart).label("SDOC_START"); } + + public JLPPegParser() { + this("/**", "*/", "!#$%^&*()_-=+|;:'\",<>?~`", "///"); } + /** * Parses the rule: * SourceFile = (Block / DocBlock / CodeBlock)+ @@ -95,60 +112,99 @@ public class JLPPegParser extends BaseParser { push(curLineNum), DocBlock(), CodeBlock(), + // A DocBlock and a CodeBlock are pushed onto the stack by the + // above rules. Pop them off, along with the line number we pushed + // before that, and create a new Block node. push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); } /** * Parses the rule: - * DocBlock = (Directive / DocText)+ + * DocBlock = SDocBlock / MDocBlock + * + * Pushes a DocBlock onto the stack. + */ + Rule DocBlock() { return FirstOf(SDocBlock(), MDocBlock()); } + + /** + * Parses the rule: + * SDocBlock = (SDirective / SDocText)+ * * Pushes a DocBlock object onto the stack */ - Rule DocBlock() { + Rule SDocBlock() { return Sequence( push(new DocBlock(curLineNum)), OneOrMore(Sequence( - FirstOf(Directive(), DocText()), + FirstOf(SDirective(), SDocText()), addToDocBlock((ASTNode) pop())))); } /** * Parses the rule: - * CodeBlock = (!DocLineStart RemainingLine)+ + * MDocBlock = MDOC_START (MDirective / MDocText)+ MDOC_END + * + * Pushes a DocBlock object onto the stack + */ + Rule MDocBlock() { + return Sequence( + push(new DocBlock(curLineNum)), + MDOC_START, + ZeroOrMore(Sequence( + // We need to be careful to exclude MDOC_END here, as there can + // be some confusion otherwise between the start of a line with + // MDOC_LINE_START and MDOC_END depending on what values the + // user has chosen for them + TestNot(MDOC_END), FirstOf(MDirective(), MDocText()), + addToDocBlock((ASTNode) pop()))), + MDOC_END); } + /** + * Parses the rule: + * CodeBlock = (RemainingCodeLine)+ * * Pushes a CodeBlock onto the stack. */ Rule CodeBlock() { return Sequence( push(new CodeBlock(curLineNum)), - OneOrMore(Sequence( - TestNot(DocLineStart()), RemainingLine(), + OneOrMore(Sequence(RemainingCodeLine(), addToCodeBlock(match())))); } /** * Parses the rule: - * Directive = DocLineStart AT (LongDirective / ShortDirective) + * SDirective = SDocLineStart AT (SLongDirective / SShortDirective) * * Pushes a Directive node on the stack. */ - Rule Directive() { + Rule SDirective() { return Sequence( - DocLineStart(), AT, FirstOf(LongDirective(), ShortDirective())); } + SDocLineStart(), AT, FirstOf(SLongDirective(), SShortDirective())); } /** * Parses the rule: - * LongDirective = - * (API_DIR / EXAMPLE_DIR) RemainingLine DocText? + * MDirective = MDocLineStart? AT (MLongDirective / MShortDirective) * * Pushes a Directive node onto the stack. */ - Rule LongDirective() { + Rule MDirective() { + return Sequence( + Optional(MDocLineStart()), + AT, FirstOf(MLongDirective(), MShortDirective())); } + + /** + * Parses the rule: + * SLongDirective = + * (API_DIR / EXAMPLE_DIR) RemainingSDocLine SDocText? + * + * Pushes a Directive node onto the stack. + */ + Rule SLongDirective() { return Sequence( push(curLineNum), FirstOf(API_DIR, EXAMPLE_DIR), push(match()), - RemainingLine(), push(match()), + RemainingSDocLine(), push(match()), Optional(Sequence( - DocText(), + SDocText(), swap(), push(popAsString() + ((DocText) pop()).value))), @@ -156,48 +212,151 @@ public class JLPPegParser extends BaseParser { /** * Parses the rule: - * ShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingLine + * MLongDirective = + * (API_DIR / EXAMPLE_DIR) RemainingMDocLine MDocText? * * Pushes a Directive node onto the stack. */ - Rule ShortDirective() { + Rule MLongDirective() { + return Sequence( + push(curLineNum), + + FirstOf(API_DIR, EXAMPLE_DIR), push(match()), + RemainingMDocLine(), push(match()), + + Optional(Sequence( + MDocText(), + swap(), + push(popAsString() + ((DocText) pop()).value))), + + push(new Directive(popAsString(), popAsString(), popAsInt()))); } + + /** + * Parses the rule: + * SShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingSDocLine + * + * Pushes a Directive node onto the stack. + */ + Rule SShortDirective() { return Sequence( push(curLineNum), FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()), - RemainingLine(), + RemainingSDocLine(), push(new Directive(match().trim(), popAsString(), popAsInt()))); } /** * Parses the rule: - * DocText = (DocLineStart !AT RemainingLine)+ + * MShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingMDocLine + * + * Pushes a Directive node onto the stack. + */ + Rule MShortDirective() { + return Sequence( + push(curLineNum), + FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()), + RemainingMDocLine(), + + push(new Directive(match().trim(), popAsString(), popAsInt()))); } + + /** + * Parses the rule: + * SDocText = (SDocLineStart !AT RemainingSDocLine)+ * * Pushes a DocText node onto the stack. */ - Rule DocText() { + Rule SDocText() { return Sequence( push(new DocText(curLineNum)), OneOrMore(Sequence( - DocLineStart(), TestNot(AT), RemainingLine(), + SDocLineStart(), TestNot(AT), RemainingSDocLine(), addToDocText(match())))); } - Rule DocLineStart() { + /** + * Parses the rule: + * MDocText = (MDocLineStart? !AT RemainingMDocLine)+ + * + * Pushes a DocText node onto the stack. + */ + Rule MDocText() { return Sequence( - ZeroOrMore(SPACE), DOC_LINE_START, Optional(SPACE)); } + push(new DocText(curLineNum)), + OneOrMore(Sequence( + Optional(MDocLineStart()), + TestNot(AT), RemainingMDocLine(), + addToDocText(match())))); } - Rule NonEmptyLine() { - return Sequence(OneOrMore(NOT_EOL), FirstOf(EOL, EOI)); } + /** + * Parses the rule: + * SDocLineStart = SPACE* SDOC_START SPACE? + */ + Rule SDocLineStart() { + return Sequence( + ZeroOrMore(SPACE), SDOC_START, Optional(SPACE)); } - Rule RemainingLine() { + /** + * Parses the rule: + * MDocLineStart = SPACE* !MDOC_END MDOC_LINE_START SPACE? + */ + Rule MDocLineStart() { + return Sequence( + ZeroOrMore(SPACE), TestNot(MDOC_END), MDOC_LINE_START, Optional(SPACE)); } + + /** + * Parses the rule: + * RemainingSDocLine = ((!EOL)* EOL) / ((!EOL)+ EOI) + */ + Rule RemainingSDocLine() { return FirstOf( Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()), Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); } + /** + * Parses the rule: + * RemainingMDocLine = + * ((!(EOL / MDOC_END))* EOL) / + * ((!MDOC_END)+) + */ + Rule RemainingMDocLine() { + return FirstOf( + // End of line, still within the an M-style comment block + Sequence( + ZeroOrMore(Sequence(TestNot(FirstOf(EOL, MDOC_END)), ANY)), + EOL, + incLineCount()), + + // End of M-style comment block + OneOrMore(Sequence(TestNot(MDOC_END), ANY))); } + + /** + * Parses the rule: + * RemainingCodeLine = + * ((!(EOL / MDOC_START / SDocLineStart))* EOL) / + * (!(MDOC_START / SDocLineStart))+ + */ + Rule RemainingCodeLine() { + return FirstOf( + // End of line, still within the code block. + Sequence( + ZeroOrMore(Sequence( + TestNot(FirstOf(EOL, MDOC_START, SDocLineStart())), + ANY)), + EOL, + incLineCount()), + + // Found an MDOC_START or SDocLineStart + OneOrMore(Sequence(TestNot(FirstOf(MDOC_START, SDocLineStart())), ANY))); } + Rule AT = Ch('@').label("AT"); Rule EOL = FirstOf(String("\r\n"), Ch('\n'), Ch('\r')).label("EOL"); Rule NOT_EOL = Sequence(TestNot(EOL), ANY).label("NOT_EOL"); Rule SPACE = AnyOf(" \t").label("SPACE"); - Rule DOC_LINE_START = String("%%").label("DOC_LINE_START"); + + // Configurable + Rule MDOC_START; + Rule MDOC_END; + Rule MDOC_LINE_START; + Rule SDOC_START; // directive terminals Rule AUTHOR_DIR = IgnoreCase("author"); diff --git a/src/main/com/jdblabs/jlp/Processor.groovy b/src/main/com/jdblabs/jlp/Processor.groovy index 958a3e6..95c922a 100644 --- a/src/main/com/jdblabs/jlp/Processor.groovy +++ b/src/main/com/jdblabs/jlp/Processor.groovy @@ -22,8 +22,8 @@ public class Processor { // shortcut for docs[currentDocId] public TargetDoc currentDoc - protected Map parsers = [:] - protected Map generators = [:] + protected Map parsers = [:] + protected Map generators = [:] public static void process(File outputDir, String css, List inputFiles) { @@ -60,9 +60,10 @@ public class Processor { // TODO: add logic to configure or autodetect the correct parser for // each file - def parser = getParser(JLPPegParser) + def parser = getParser(sourceTypeForFile(currentDoc.sourceFile)) def parseRunner = new ReportingParseRunner(parser.SourceFile()) + // TODO: error detection currentDoc.sourceAST = parseRunner.run( currentDoc.sourceFile.text).resultValue } @@ -71,7 +72,8 @@ public class Processor { // TODO: add logic to configure or autodetect the correct generator // for each file - def generator = getGenerator(LiterateMarkdownGenerator) + def generator = + getGenerator(sourceTypeForFile(currentDoc.sourceFile)) generator.parse(currentDoc.sourceAST) } @@ -80,7 +82,8 @@ public class Processor { // TODO: add logic to configure or autodetect the correct generator // for each file - def generator = getGenerator(LiterateMarkdownGenerator) + def generator = + getGenerator(sourceTypeForFile(currentDoc.sourceFile)) currentDoc.output = generator.emit(currentDoc.sourceAST) } // Write the output to the output directory @@ -94,8 +97,7 @@ public class Processor { File outputDir = outputFile.parentFile // create the directory if need be - if (!outputDir.exists()) { - outputDir.mkdirs() } + if (!outputDir.exists()) { outputDir.mkdirs() } // write the css file if it does not exist File cssFile = new File(outputDir, "jlp.css") @@ -159,17 +161,50 @@ public class Processor { return new File(newPath.join('/')) } - protected getGenerator(Class generatorClass) { - if (generators[generatorClass] == null) { - def constructor = generatorClass.getConstructor(Processor) - generators[generatorClass] = constructor.newInstance(this) - } + public static sourceTypeForFile(File sourceFile) { + String extension + def nameParts = sourceFile.name.split(/\./) - return generators[generatorClass] } + if (nameParts.length == 1) { return 'binary' } + else { extension = nameParts[-1] } - protected getParser(Class parserClass) { - if (parsers[parserClass] == null) { - parsers[parserClass] = Parboiled.createParser(parserClass) } + switch (extension) { + case 'c': case 'h': return 'c'; + case 'c++': case 'h++': case 'cpp': case 'hpp': return 'c++'; + case 'erl': case 'hrl': return 'erlang'; + case 'groovy': return 'groovy'; + case 'java': return 'java'; + case 'js': return 'javascript'; + default: return 'unknown'; }} - return parsers[parserClass] } + protected getGenerator(String sourceType) { + if (generators[sourceType] == null) { + switch(sourceType) { + default: + generators[sourceType] = + new LiterateMarkdownGenerator(this) }} + + return generators[sourceType] } + + protected getParser(String sourceType) { + println "Looking for a ${sourceType} parser." + if (parsers[sourceType] == null) { + switch(sourceType) { + case 'erlang': + parsers[sourceType] = Parboiled.createParser( + JLPPegParser, '%%') + println "Built an erlang parser." + break + case 'c': + case 'c++': + case 'groovy': + case 'java': + case 'javascript': + default: + parsers[sourceType] = Parboiled.createParser(JLPPegParser, + '/**', '*/', '!#$%^&*()_-=+|;:\'",<>?~`', '///') + println "Built a java parser." + break }} + + return parsers[sourceType] } } diff --git a/src/main/com/jdblabs/jlp/ast/Directive.groovy b/src/main/com/jdblabs/jlp/ast/Directive.groovy index 3f17fbd..264d230 100644 --- a/src/main/com/jdblabs/jlp/ast/Directive.groovy +++ b/src/main/com/jdblabs/jlp/ast/Directive.groovy @@ -3,11 +3,7 @@ package com.jdblabs.jlp.ast public class Directive extends ASTNode { public static enum DirectiveType { - Api, - Author, - Copyright, - Example, - Org; + Api, Author, Copyright, Example, Org; public static DirectiveType parse(String typeString) { valueOf(typeString.toLowerCase().capitalize()) } }