Support for multi=line comments, detects file type.

* Added support for multi-line comments to the JLPPegParser grammar
  implementation.
* Added a Java sample file.
* Updated test script to add convenience functions for the java test file and
  for using a TracingParseRunner for parse runs.
* Added an option, `--css-file`, to allow the caller to specify their own css
  file.
* Added basic logic to the Processor class to detect source file types and build
  a parser and a generator for that source type. Support currently exists for
  the following languages: C (.c, .h), C++ (.cpp, .c++, .hpp, .h++), Erlang
  (.erl), Groovy (.groovy), Java (.java), JavaScript (.js).

doc/grammar.rst

@@ -5,25 +5,53 @@ Block ->
     DocBlock CodeBlock
 
 DocBlock ->
-    (Directive / DocText)+
+    (SDocBlock / MDocBlock)
 
-Directive ->
+SDocBlock ->
-    DocLineStart AT (LongDirective / ShortDirective)
+    (SDirective / SDocText)+
 
-LongDirective ->
+MDocBlock ->
-    ("author" / "doc" / "example") RemainingLine DocText?
+    MDOC_START (!MDOC_END / MDirective / MDocText)* MDOC_END
-
-ShortDirective ->
-    ("org" / "copyright") RemainingLine
-
-DocText ->
-    (DocLineStart !AT RemainingLine)+
-
-DocLineStart ->
-    Space* DOC_LINE_START Space?
 
 CodeBlock ->
-    (!DocLineStart RemainingLine)+
+    (RemainingCodeLine)+
 
-RemainingLine ->
+SDirective ->
-   ((!EOL)* EOL) / ((!EOL)+ EOI)
+    SDocLineStart AT (SLongDirective / SShortDirective)
+
+MDirective ->
+    MDocLineStart? AT (MLongDirective / MShortDirective)
+
+SLongDirective ->
+    ("api" / "example") RemainingSDocLine SDocText?
+
+MLongDirective ->
+    ("api" / "example") RemainingMDocLine MDocText?
+
+SShortDirective ->
+    ("author" / "org" / "copyright") RemainingSDocLine
+
+MShortDirective ->
+    ("author" / "org" / "copyright") RemainingMDocLine
+
+SDocText ->
+    (SDocLineStart !AT RemainingSDocLine)+
+
+MDocText ->
+    (MDocLineStart? !AT RemainingMDocLine)+
+
+SDocLineStart ->
+    SPACE* SDOC_START SPACE?
+
+MDocLineStart ->
+    SPACE* !MDOC_END MDOC_LINE_START SPACE?
+
+RemainingSDocLine ->
+    ((!EOL)* EOL) / ((!EOL)+ EOI)
+
+RemainingMDocLine ->
+    ((!(EOL / MDOC_END))* EOL) / ((!MDOC_END)+)
+
+RemainingCodeLine ->
+    ((!(EOL / MDOC_START / SDocLineStart))* EOL) /
+    (!(MDOC_START / SDocLineStart))+

project.properties

@@ -1,6 +1,6 @@
-#Mon, 12 Sep 2011 10:56:06 -0500
+#Sun, 25 Dec 2011 21:56:17 -0600
 name=jlp
-version=0.3
+version=1.0
-build.number=8
+build.number=0
 lib.local=true
 release.dir=release

resources/main/Test.java

@@ -0,0 +1,26 @@
+package test;
+
+import java.util.Array;
+
+/** This is a test class. Mainly for testing my parser.
+  * It should work. And now we are just filing space.
+  * @author Jonathan Bernard
+  * @copyright JDB Labs 2011 */
+public class Test {
+
+    /**
+      | @org test-ref
+      | This is an embedded comment.
+      | It spreads over at least 3 lines.
+      | And this is the third line.
+      */
+
+    public static void main(String[] args) {
+        /** Yes, this is a hello world example. */
+        System.out.println("Hello World!");
+    }
+
+    /// This is a single-line comment block. */ /** Other comment
+    /// modifiers should not matter within this block.
+    /// @org last-doc
+}

resources/main/test.groovy

@@ -1,21 +1,30 @@
 import com.jdblabs.jlp.*
 import org.parboiled.Parboiled
-import org.parboiled.parserunners.ReportingParseRunner
+import org.parboiled.parserunners.*
-import org.parboiled.parserunners.RecoveringParseRunner
 
 "Making the standard parser."
 "---------------------------"
 
-parser = Parboiled.createParser(JLPPegParser.class)
+jp = Parboiled.createParser(JLPPegParser.class)
-parseRunner = new ReportingParseRunner(parser.SourceFile())
+ep = Parboiled.createParser(JLPPegParser, '%%')
+jrpr = new ReportingParseRunner(jp.SourceFile())
+jtpr = new TracingParseRunner(jp.SourceFile())
+erpr = new ReportingParseRunner(ep.SourceFile())
+etpr = new TracingParseRunner(ep.SourceFile())
 
-simpleTest = {
+vbsFile = new File('vbs_db_records.hrl')
+javaFile = new File('Test.java')
+docsDir = new File('jlp-docs')
+docsDir.mkdirs()
+
+simpleTest = { parseRunner ->
 
     println "Parsing the simple test into 'result'."
     println "--------------------------------------"
 
     testLine = """%% This the first test line.
     %% Second Line
+    Actual third line that screws stuff up.
     %% Third Line \n\n Fifth line \n\n %% Seventh line \n\n
     %% @author Eigth Line
     %% @Example Ninth Line
@@ -26,22 +35,35 @@ simpleTest = {
     simpleResult = parseRunner.run(testLine)
 }
 
-vbsTest = {
+vbsTest = { parseRunner ->
     println "Parsing vbs_db_records.hrl into 'vbsResult'."
     println "--------------------------------------------"
 
-    vbsTestFile = new File('vbs_db_records.hrl')
+    println "vbsFile is ${vbsFile.exists() ? 'present' : 'absent'}."
-    println "vbsTestFile is ${vbsTestFile.exists() ? 'present' : 'absent'}."
+    vbsTestInput = vbsFile.text
-    vbsTestInput = vbsTestFile.text
 
     vbsParsed = parseRunner.run(vbsTestInput)
 
-    vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl"
+    /*vbsResult = LiterateMarkdownGenerator.generateDocuments(["vbs_db_records.hrl": vbsParsed.resultValue])."vbs_db_records.hrl"
 
     println "Writing to file 'vbs_db_records.html'."
     println "--------------------------------------"
 
     (new File('vbs_db_records.html')).withWriter { out -> out.println vbsResult }
 
-    return [vbsParsed, vbsResult]
+    return [vbsParsed, vbsResult]*/
+    return vbsParsed
+}
+
+javaTest = { parseRunner ->
+    println "Parsing Test.java into 'javaResult'."
+    println "------------------------------------"
+
+    println "javaFile is ${javaFile.exists() ? 'present' : 'absent'}."
+    javaTestInput = javaFile.text
+
+    javaParsed = parseRunner.run(javaTestInput)
+    javaSF = javaParsed.valueStack.peek()
+
+    return [javaParsed: javaParsed, javaSF: javaSF]
 }

src/main/com/jdblabs/jlp/JLPMain.groovy

@@ -18,6 +18,8 @@ public class JLPMain {
         cli.o("Output directory (defaults to 'jlp-docs').",
             longOpt: 'output-dir', args: 1, argName: "output-dir",
             required: false)
+        cli._('Use <css-file> for the documentation css.',
+            longOpt: 'css-file', args: 1, required: false, argName: 'css-file')
         cli._(longOpt: 'relative-path-root', args: 1, required: false,
             'Resolve all relative paths against this root.')
 
@@ -47,8 +49,26 @@ public class JLPMain {
         // create the output directory if it does not exist
         if (!outputDir.exists()) outputDir.mkdirs()
 
-        // get the CSS theme to use
+        // get the CSS theme to use. We will start by assuming the default will
-        def css = JLPMain.class.getResourceAsStream("/jlp.css") // TODO: make an option
+        // be used.
+        def css = JLPMain.class.getResourceAsStream("/jlp.css")
+
+        // If the CSS file was specified on the command-line, let's look for it.
+        if (opts.'css-file') {
+            def cssFile = new File(opts.'css-file')
+            // resolve against our relative root
+            if (!cssFile.isAbsolute()) {
+                cssFile = new File(pathRoot, cssFile.path) }
+                
+            // Finally, make sure the file actually exists.
+            if (cssFile.exists()) { css = cssFile }
+            else {
+                println "WARN: Could not fine the custom CSS file: '" +
+                    "${cssFile.canonicalPath}'."
+                println "      Using the default CSS." }}
+
+        // Extract the text from our css source (either an InputStream or a
+        // File)
         css = css.text
 
         // get files passed in

src/main/com/jdblabs/jlp/JLPPegParser.java

@@ -14,6 +14,23 @@ public class JLPPegParser extends BaseParser<Object> {
 
     int curLineNum = 1;
 
+    public JLPPegParser(String mdocStart, String mdocEnd,
+        String mdocLineStart, String sdocStart) {
+
+        MDOC_START = String(mdocStart).label("MDOC_START");
+        MDOC_END = String(mdocEnd).label("MDOC_END");
+        SDOC_START = String(sdocStart).label("SDOC_START");
+        MDOC_LINE_START = AnyOf(mdocLineStart).label("MDOC_LINE_START"); }
+
+    public JLPPegParser(String sdocStart) {
+        MDOC_START = NOTHING;
+        MDOC_LINE_START = NOTHING;
+        MDOC_END = NOTHING;
+        SDOC_START = String(sdocStart).label("SDOC_START"); }
+
+    public JLPPegParser() {
+        this("/**", "*/", "!#$%^&*()_-=+|;:'\",<>?~`", "///"); }
+
     /**
      * Parses the rule:
      *  SourceFile = (Block / DocBlock / CodeBlock)+
@@ -95,60 +112,99 @@ public class JLPPegParser extends BaseParser<Object> {
             push(curLineNum),
             DocBlock(), CodeBlock(),
 
+            // A DocBlock and a CodeBlock are pushed onto the stack by the
+            // above rules. Pop them off, along with the line number we pushed
+            // before that, and create a new Block node.
             push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); }
 
     /**
      * Parses the rule:
-     *  DocBlock = (Directive / DocText)+
+     *   DocBlock = SDocBlock / MDocBlock
+     *
+     * Pushes a DocBlock onto the stack.
+     */
+    Rule DocBlock() { return FirstOf(SDocBlock(), MDocBlock()); }
+
+    /**
+     * Parses the rule:
+     *  SDocBlock = (SDirective / SDocText)+
      *
      * Pushes a DocBlock object onto the stack
      */
-    Rule DocBlock() {
+    Rule SDocBlock() {
         return Sequence(
             push(new DocBlock(curLineNum)),
             OneOrMore(Sequence(
-                FirstOf(Directive(), DocText()),
+                FirstOf(SDirective(), SDocText()),
                 addToDocBlock((ASTNode) pop())))); }
 
     /**
      * Parses the rule:
-     *  CodeBlock = (!DocLineStart RemainingLine)+
+     *  MDocBlock = MDOC_START (MDirective / MDocText)+ MDOC_END
+     *
+     * Pushes a DocBlock object onto the stack
+     */
+    Rule MDocBlock() {
+        return Sequence(
+            push(new DocBlock(curLineNum)),
+            MDOC_START,
+            ZeroOrMore(Sequence(
+                // We need to be careful to exclude MDOC_END here, as there can
+                // be some confusion otherwise between the start of a line with
+                // MDOC_LINE_START and MDOC_END depending on what values the
+                // user has chosen for them
+                TestNot(MDOC_END), FirstOf(MDirective(), MDocText()),
+                addToDocBlock((ASTNode) pop()))),
+            MDOC_END); }
+    /**
+     * Parses the rule:
+     *  CodeBlock = (RemainingCodeLine)+
      *
      * Pushes a CodeBlock onto the stack.
      */
     Rule CodeBlock() {
         return Sequence(
             push(new CodeBlock(curLineNum)),
-            OneOrMore(Sequence(
+            OneOrMore(Sequence(RemainingCodeLine(),
-                TestNot(DocLineStart()), RemainingLine(),
                 addToCodeBlock(match())))); }
 
     /**
      * Parses the rule:
-     *  Directive = DocLineStart AT (LongDirective / ShortDirective)
+     *  SDirective = SDocLineStart AT (SLongDirective / SShortDirective)
      *
      * Pushes a Directive node on the stack.
      */
-    Rule Directive() {
+    Rule SDirective() {
         return Sequence(
-            DocLineStart(), AT, FirstOf(LongDirective(), ShortDirective())); }
+            SDocLineStart(), AT, FirstOf(SLongDirective(), SShortDirective())); }
 
     /**
      * Parses the rule:
-     *  LongDirective =
+     *  MDirective = MDocLineStart? AT (MLongDirective / MShortDirective)
-     *      (API_DIR / EXAMPLE_DIR) RemainingLine DocText?
      *
      * Pushes a Directive node onto the stack.
      */
-    Rule LongDirective() {
+    Rule MDirective() {
+        return Sequence(
+            Optional(MDocLineStart()),
+            AT, FirstOf(MLongDirective(), MShortDirective())); }
+
+    /**
+     * Parses the rule:
+     *  SLongDirective =
+     *      (API_DIR / EXAMPLE_DIR) RemainingSDocLine SDocText?
+     *
+     * Pushes a Directive node onto the stack.
+     */
+    Rule SLongDirective() {
         return Sequence(
             push(curLineNum),
 
             FirstOf(API_DIR, EXAMPLE_DIR),  push(match()),
-            RemainingLine(),                            push(match()),
+            RemainingSDocLine(),            push(match()),
 
             Optional(Sequence(
-                DocText(),
+                SDocText(),
                 swap(),
                 push(popAsString() + ((DocText) pop()).value))),
                 
@@ -156,48 +212,151 @@ public class JLPPegParser extends BaseParser<Object> {
 
     /**
      * Parses the rule:
-     *  ShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingLine
+     *  MLongDirective = 
+     *      (API_DIR / EXAMPLE_DIR) RemainingMDocLine MDocText?
      *
      * Pushes a Directive node onto the stack.
      */
-    Rule ShortDirective() {
+    Rule MLongDirective() {
+        return Sequence(
+            push(curLineNum),
+
+            FirstOf(API_DIR, EXAMPLE_DIR), push(match()),
+            RemainingMDocLine(),           push(match()),
+
+            Optional(Sequence(
+                MDocText(),
+                swap(),
+                push(popAsString() + ((DocText) pop()).value))),
+
+            push(new Directive(popAsString(), popAsString(), popAsInt()))); }
+
+    /**
+     * Parses the rule:
+     *  SShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingSDocLine
+     *
+     * Pushes a Directive node onto the stack.
+     */
+    Rule SShortDirective() {
         return Sequence(
             push(curLineNum),
             FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()),
-            RemainingLine(),
+            RemainingSDocLine(),
             
             push(new Directive(match().trim(), popAsString(), popAsInt()))); }
 
     /**
      * Parses the rule:
-     *  DocText = (DocLineStart !AT RemainingLine)+
+     *  MShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingMDocLine
+     *
+     * Pushes a Directive node onto the stack.
+     */
+    Rule MShortDirective() {
+        return Sequence(
+            push(curLineNum),
+            FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()),
+            RemainingMDocLine(),
+
+            push(new Directive(match().trim(), popAsString(), popAsInt()))); }
+
+    /**
+     * Parses the rule:
+     *  SDocText = (SDocLineStart !AT RemainingSDocLine)+
      *
      * Pushes a DocText node onto the stack.
      */
-    Rule DocText() {
+    Rule SDocText() {
         return Sequence(
             push(new DocText(curLineNum)),
             OneOrMore(Sequence(
-                DocLineStart(), TestNot(AT), RemainingLine(),
+                SDocLineStart(), TestNot(AT), RemainingSDocLine(),
                 addToDocText(match())))); }
 
-    Rule DocLineStart() {
+    /**
+     * Parses the rule:
+     *  MDocText = (MDocLineStart? !AT RemainingMDocLine)+
+     *
+     * Pushes a DocText node onto the stack.
+     */
+    Rule MDocText() {
         return Sequence(
-            ZeroOrMore(SPACE), DOC_LINE_START, Optional(SPACE)); }
+            push(new DocText(curLineNum)),
+            OneOrMore(Sequence(
+                Optional(MDocLineStart()),
+                TestNot(AT), RemainingMDocLine(),
+                addToDocText(match())))); }
 
-    Rule NonEmptyLine() {
+    /**
-        return Sequence(OneOrMore(NOT_EOL), FirstOf(EOL, EOI)); }
+     * Parses the rule:
+     *  SDocLineStart = SPACE* SDOC_START SPACE?
+     */
+    Rule SDocLineStart() {
+        return Sequence(
+            ZeroOrMore(SPACE), SDOC_START, Optional(SPACE)); }
 
-    Rule RemainingLine() {
+    /**
+     * Parses the rule:
+     *  MDocLineStart = SPACE* !MDOC_END MDOC_LINE_START SPACE?
+     */
+    Rule MDocLineStart() {
+        return Sequence(
+            ZeroOrMore(SPACE), TestNot(MDOC_END), MDOC_LINE_START, Optional(SPACE)); }
+
+    /**
+     * Parses the rule:
+     *  RemainingSDocLine = ((!EOL)* EOL) / ((!EOL)+ EOI)
+     */
+    Rule RemainingSDocLine() {
         return FirstOf(
             Sequence(ZeroOrMore(NOT_EOL), EOL, incLineCount()),
             Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); }
 
+    /**
+     * Parses the rule:
+     *  RemainingMDocLine = 
+     *      ((!(EOL / MDOC_END))* EOL) /
+     *      ((!MDOC_END)+)
+     */
+    Rule RemainingMDocLine() {
+        return FirstOf(
+            // End of line, still within the an M-style comment block
+            Sequence(
+                ZeroOrMore(Sequence(TestNot(FirstOf(EOL, MDOC_END)), ANY)),
+                EOL,
+                incLineCount()),
+
+            // End of M-style comment block
+            OneOrMore(Sequence(TestNot(MDOC_END), ANY))); }
+
+    /**
+     * Parses the rule:
+     *  RemainingCodeLine = 
+     *      ((!(EOL / MDOC_START / SDocLineStart))* EOL) /
+     *      (!(MDOC_START / SDocLineStart))+
+     */
+    Rule RemainingCodeLine() {
+        return FirstOf(
+            // End of line, still within the code block.
+            Sequence(
+                ZeroOrMore(Sequence(
+                    TestNot(FirstOf(EOL, MDOC_START, SDocLineStart())),
+                    ANY)),
+                EOL,
+                incLineCount()),
+
+            // Found an MDOC_START or SDocLineStart
+            OneOrMore(Sequence(TestNot(FirstOf(MDOC_START, SDocLineStart())), ANY))); }
+
     Rule AT         = Ch('@').label("AT");
     Rule EOL        = FirstOf(String("\r\n"), Ch('\n'), Ch('\r')).label("EOL");
     Rule NOT_EOL    = Sequence(TestNot(EOL), ANY).label("NOT_EOL");
     Rule SPACE      = AnyOf(" \t").label("SPACE");
-    Rule DOC_LINE_START = String("%%").label("DOC_LINE_START");
+
+    // Configurable
+    Rule MDOC_START;
+    Rule MDOC_END;
+    Rule MDOC_LINE_START;
+    Rule SDOC_START;
 
     // directive terminals
     Rule AUTHOR_DIR     = IgnoreCase("author");

src/main/com/jdblabs/jlp/Processor.groovy

@@ -22,8 +22,8 @@ public class Processor {
     // shortcut for docs[currentDocId]
     public TargetDoc currentDoc
 
-    protected Map<Class, BaseParser> parsers = [:]
+    protected Map<String, BaseParser> parsers = [:]
-    protected Map<Class, JLPBaseGenerator> generators = [:]
+    protected Map<String, JLPBaseGenerator> generators = [:]
 
     public static void process(File outputDir, String css,
     List<File> inputFiles) {
@@ -60,9 +60,10 @@ public class Processor {
             
             // TODO: add logic to configure or autodetect the correct parser for
             // each file
-            def parser = getParser(JLPPegParser)
+            def parser = getParser(sourceTypeForFile(currentDoc.sourceFile))
             def parseRunner = new ReportingParseRunner(parser.SourceFile())
 
+            // TODO: error detection
             currentDoc.sourceAST = parseRunner.run(
                 currentDoc.sourceFile.text).resultValue }
 
@@ -71,7 +72,8 @@ public class Processor {
 
             // TODO: add logic to configure or autodetect the correct generator
             // for each file
-            def generator = getGenerator(LiterateMarkdownGenerator)
+            def generator =
+                getGenerator(sourceTypeForFile(currentDoc.sourceFile))
             generator.parse(currentDoc.sourceAST) }
 
 
@@ -80,7 +82,8 @@ public class Processor {
             
             // TODO: add logic to configure or autodetect the correct generator
             // for each file
-            def generator = getGenerator(LiterateMarkdownGenerator)
+            def generator =
+                getGenerator(sourceTypeForFile(currentDoc.sourceFile))
             currentDoc.output = generator.emit(currentDoc.sourceAST) }
 
         // Write the output to the output directory
@@ -94,8 +97,7 @@ public class Processor {
             File outputDir = outputFile.parentFile
 
             // create the directory if need be
-            if (!outputDir.exists()) {
+            if (!outputDir.exists()) { outputDir.mkdirs() }
-                outputDir.mkdirs() }
 
             // write the css file if it does not exist
             File cssFile = new File(outputDir, "jlp.css")
@@ -159,17 +161,50 @@ public class Processor {
 
             return new File(newPath.join('/')) }
 
-    protected getGenerator(Class generatorClass) {
+    public static sourceTypeForFile(File sourceFile) {
-        if (generators[generatorClass] == null) {
+        String extension
-            def constructor = generatorClass.getConstructor(Processor)
+        def nameParts = sourceFile.name.split(/\./)
-            generators[generatorClass] = constructor.newInstance(this)
-        }
 
-        return generators[generatorClass] }
+        if (nameParts.length == 1) { return 'binary' }
+        else { extension = nameParts[-1] }
 
-    protected getParser(Class parserClass) {
+        switch (extension) {
-        if (parsers[parserClass] == null) {
+            case 'c': case 'h': return 'c';
-            parsers[parserClass] = Parboiled.createParser(parserClass) }
+            case 'c++': case 'h++': case 'cpp': case 'hpp': return 'c++';
+            case 'erl': case 'hrl': return 'erlang';
+            case 'groovy': return 'groovy';
+            case 'java': return 'java';
+            case 'js': return 'javascript';
+            default: return 'unknown'; }}
 
-        return parsers[parserClass] }
+    protected getGenerator(String sourceType) {
+        if (generators[sourceType] == null) {
+            switch(sourceType) {
+                default:
+                    generators[sourceType] =
+                        new LiterateMarkdownGenerator(this) }}
+
+        return generators[sourceType] }
+
+    protected getParser(String sourceType) {
+        println "Looking for a ${sourceType} parser."
+        if (parsers[sourceType] == null) {
+            switch(sourceType) {
+                case 'erlang':
+                    parsers[sourceType] = Parboiled.createParser(
+                        JLPPegParser, '%%')
+                    println "Built an erlang parser."
+                    break
+                case 'c':
+                case 'c++':
+                case 'groovy':
+                case 'java':
+                case 'javascript':
+                default:
+                    parsers[sourceType] = Parboiled.createParser(JLPPegParser,
+                        '/**', '*/', '!#$%^&*()_-=+|;:\'",<>?~`', '///')
+                    println "Built a java parser."
+                    break }}
+
+        return parsers[sourceType] }
 }

src/main/com/jdblabs/jlp/ast/Directive.groovy

@@ -3,11 +3,7 @@ package com.jdblabs.jlp.ast
 public class Directive extends ASTNode {
 
     public static enum DirectiveType {
-        Api,
+        Api, Author, Copyright, Example, Org;
-        Author,
-        Copyright,
-        Example,
-        Org;
         
         public static DirectiveType parse(String typeString) {
             valueOf(typeString.toLowerCase().capitalize()) } }