Ground work for include directive, documentation.

* Further documentation.
* Decided to add an `include` directive. This will have reprecussions directly
  for the JLPPegParser, Directive, and LinkAnchor classes. Include needs more
  semantic meaning than we currently have in the process because the author
  needs some way to understand what is being included. If you include an org
  link defined earlier does it include the whole file? Just that doc block? Both
  may be the desired behavior in different situations, but I do not want to add
  a complex syntax for selecting, just name the link. Therefore there must be
  something about the link that determines how much in included. This means we
  need more information in LinkAnchor, some link `type` at a minimum. My current
  thought is:

    * @org defined links--this is the only type of LinkAnchor defined right
      now--always bring the whole DocBlock when they are included.
    * A new type of link anchor, call them source file links, bring the whole
      file when they are included. These types of anchors would be automatically
      created by the parser/generator (have not decided where it belongs yet).
      The whole SourceFile would need to be included, but we do not want to emit
      the normal header for the file so we may have to do somthing about the
      initial emit method.
    * Additional types of link anchors arise when we add language awareness to
      the process. In my current vision we will automatically add link anchors
      of different types when we parse code sections (function definition, class
      definition, etc.) and the include directive would bring in all the
      DocBlocks related to that code node.
    * Additional types of link anchors will arise when we implement the @api
      directive, we will think about that in the future.

* Updated the JLPPegParser to recognise include directives.
* Added the include directive to Directive.DirectiveTypes

doc/issues/0007fn5.rst

@@ -0,0 +1,12 @@
+`Include` Directive.
+====================
+
+Add a new directive: `include`. This directive allows the author to include
+the contents of a URL inline in the documentation (can be an internal JLP
+reference URL).
+
+----
+
+======== ===================
+Created: 2011-12-29T11:23:52
+======== ===================

project.properties

@@ -1,7 +1,7 @@
-#Wed, 28 Dec 2011 15:44:01 -0600
+#Thu, 29 Dec 2011 15:43:59 -0600
 name=jlp
-version=1.2
+version=1.3a
-build.number=10
+build.number=0
 lib.local=true
 release.dir=release
 main.class=com.jdblabs.jlp.JLPMain

src/main/com/jdblabs/jlp/JLPPegParser.java

@@ -19,6 +19,29 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
 
     int curLineNum = 1;
 
+    /// ### Constructors ###
+    /// --------------------
+    /// @org jlp.jdb-labs.com/JLPPegParser/constructors
+
+    /**
+     * Being a Parboiled parser, this class should not be instantiated by
+     * directly calling the constructor, but by calling
+     * `Parboiled.createParser()`.
+     * @example Here are some examples of how you would instantiate the
+     * JLPPegParser:
+     *
+     *     // Erlang-style documentation
+     *     Parboiled.createParser(JLPPegParser.class, "%%");
+     *
+     *     // C++-style documentation
+     *     Parboiled.createParser(JLPPegParser.class, "/**", "* /", "*", "///");
+     */
+
+    /**
+     * #### Full constructor
+     * This allows the caller to specific all four of the comment delimiters
+     * recognized by the parser.
+     */
     public JLPPegParser(String mdocStart, String mdocEnd,
         String mdocLineStart, String sdocStart) {
 
@@ -27,12 +50,28 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
         SDOC_START = String(sdocStart).label("SDOC_START");
         MDOC_LINE_START = AnyOf(mdocLineStart).label("MDOC_LINE_START"); }
 
+    /**
+     * #### Single-line comments only constructor.
+     * This allows the caller to easily set up the parser to only recognize
+     * single-line comments with no defined syntax for multi-line comments. For
+     * example this is used to instantiate a parser for Erlang source files
+     * because Erlang has no defined multi-line comment syntax.
+     */
     public JLPPegParser(String sdocStart) {
         MDOC_START = NOTHING;
         MDOC_LINE_START = NOTHING;
         MDOC_END = NOTHING;
         SDOC_START = String(sdocStart).label("SDOC_START"); }
 
+    /**
+     * #### Default constructor.
+     * The default constructor creates a JLPPegParser configured to recognize
+     * comments used by languages like C++ and Java. It follows the convention
+     * of using `/**` to start multiline documentation comments and `///` to
+     * start a single-line comment. This allows the author to choose which
+     * comments remain with the code and which are considered part of the formal
+     * documentation.
+     */
     public JLPPegParser() {
         this("/**", "*/", "!#$%^&*()_-=+|;:'\",<>?~`", "///"); }
 
@@ -40,7 +79,11 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
         ReportingParseRunner rpr = new ReportingParseRunner(this.SourceFile());
         return (SourceFile) rpr.run(input).resultValue; }
 
+    /// ### Parser Rules ###
+    /// --------------------
+
     /**
+     * #### SourceFile
      * Parses the rule:
      *
      *     SourceFile = (Block / DocBlock / CodeBlock)+
@@ -90,9 +133,9 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
                     /// the CodeBlock to make a Block, which is pushed onto the
                     /// stack:
                     ///
-                    /// *Note: With the way the parser is currently written,
+                    /// *Note: With the way the parser is currently written,*
-                    ///        this will only match a CodeBlock that occurs
+                    /// *this will only match a CodeBlock that occurs before*
-                    ///        before any DocBlock.*
+                    /// *any DocBlock.*
                     Sequence(
                         /// 1. Remember the line number for the Block.
                         push(curLineNum),
@@ -112,6 +155,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
                 addBlockToSourceFile((Block) pop())))); }
 
     /**
+     * #### Block
      * Parses the rule:
      *
      *     Block = DocBlock CodeBlock
@@ -129,6 +173,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
             push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); }
 
     /**
+     * #### DocBlock
      * Parses the rule:
      *
      *     DocBlock = SDocBlock / MDocBlock
@@ -138,6 +183,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
     Rule DocBlock() { return FirstOf(SDocBlock(), MDocBlock()); }
 
     /**
+     * #### SDocBlock 
      * Parses the rule:
      *
      *     SDocBlock = (SDirective / SDocText)+
@@ -152,6 +198,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
                 addToDocBlock((ASTNode) pop())))); }
 
     /**
+     * #### MDocBlock 
      * Parses the rule:
      *
      *     MDocBlock = MDOC_START (MDirective / MDocText)+ MDOC_END
@@ -163,14 +210,15 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
             push(new DocBlock(curLineNum)),
             MDOC_START,
             ZeroOrMore(Sequence(
-                /// We need to be careful to exclude MDOC_END here, as there can
+                /// We need to be careful to exclude `MDOC_END` here, as there can
                 /// be some confusion otherwise between the start of a line with
-                /// MDOC_LINE_START and MDOC_END depending on what values the
+                /// `MDOC_LINE_START` and `MDOC_END` depending on what values the
                 /// user has chosen for them
                 TestNot(MDOC_END), FirstOf(MDirective(), MDocText()),
                 addToDocBlock((ASTNode) pop()))),
             MDOC_END); }
     /**
+     * #### CodeBlock 
      * Parses the rule:
      *
      *     CodeBlock = (RemainingCodeLine)+
@@ -184,6 +232,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
                 addToCodeBlock(match())))); }
 
     /**
+     * #### SDirective 
      * Parses the rule:
      *
      *     SDirective = SDocLineStart AT (SLongDirective / SShortDirective)
@@ -195,6 +244,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
             SDocLineStart(), AT, FirstOf(SLongDirective(), SShortDirective())); }
 
     /**
+     * #### MDirective 
      * Parses the rule:
      *
      *     MDirective = MDocLineStart? AT (MLongDirective / MShortDirective)
@@ -207,6 +257,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
             AT, FirstOf(MLongDirective(), MShortDirective())); }
 
     /**
+     * #### SLongDirective 
      * Parses the rule:
      *
      *     SLongDirective =
@@ -229,6 +280,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
             push(new Directive(popAsString(), popAsString(), popAsInt()))); }
 
     /**
+     * #### MLongDirective 
      * Parses the rule:
      *
      *     MLongDirective = 
@@ -251,36 +303,43 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
             push(new Directive(popAsString(), popAsString(), popAsInt()))); }
 
     /**
+     * #### SShortDirective 
      * Parses the rule:
      *
-     *     SShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingSDocLine
+     *     SShortDirective =
+     *       (AUTHOR_DIR / ORG_DIR / INCLUDE_DIR / COPYRIGHT_DIR)
+     *       RemainingSDocLine
      *
      * Pushes a Directive node onto the stack.
      */
     Rule SShortDirective() {
         return Sequence(
             push(curLineNum),
-            FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()),
+            FirstOf(AUTHOR_DIR, ORG_DIR, INCLUDE_DIR, COPYRIGHT_DIR), push(match()),
             RemainingSDocLine(),
             
             push(new Directive(match().trim(), popAsString(), popAsInt()))); }
 
     /**
+     * #### MShortDirective 
      * Parses the rule:
      *
-     *     MShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingMDocLine
+     *     MShortDirective =
+     *       (AUTHOR_DIR / ORG_DIR / INCLUDE_DIR / COPYRIGHT_DIR)
+     *       RemainingMDocLine
      *
      * Pushes a Directive node onto the stack.
      */
     Rule MShortDirective() {
         return Sequence(
             push(curLineNum),
-            FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()),
+            FirstOf(AUTHOR_DIR, ORG_DIR, INCLUDE_DIR, COPYRIGHT_DIR), push(match()),
             RemainingMDocLine(),
 
             push(new Directive(match().trim(), popAsString(), popAsInt()))); }
 
     /**
+     * #### SDocText 
      * Parses the rule:
      *
      *     SDocText = (SDocLineStart !AT RemainingSDocLine)+
@@ -295,6 +354,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
                 addToDocText(match())))); }
 
     /**
+     * #### MDocText 
      * Parses the rule:
      *
      *     MDocText = (MDocLineStart? !AT RemainingMDocLine)+
@@ -310,6 +370,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
                 addToDocText(match())))); }
 
     /**
+     * #### SDocLineStart 
      * Parses the rule:
      *
      *     SDocLineStart = SPACE* SDOC_START SPACE?
@@ -319,6 +380,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
             ZeroOrMore(SPACE), SDOC_START, Optional(SPACE)); }
 
     /**
+     * #### MDocLineStart 
      * Parses the rule:
      *
      *     MDocLineStart = SPACE* !MDOC_END MDOC_LINE_START SPACE?
@@ -328,6 +390,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
             ZeroOrMore(SPACE), TestNot(MDOC_END), MDOC_LINE_START, Optional(SPACE)); }
 
     /**
+     * #### RemainingSDocLine 
      * Parses the rule:
      *
      *     RemainingSDocLine = ((!EOL)* EOL) / ((!EOL)+ EOI)
@@ -338,6 +401,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
             Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); }
 
     /**
+     * #### RemainingMDocLine 
      * Parses the rule:
      *
      *     RemainingMDocLine = 
@@ -356,6 +420,7 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
             OneOrMore(Sequence(TestNot(MDOC_END), ANY))); }
 
     /**
+     * #### RemainingCodeLine 
      * Parses the rule:
      *
      *     RemainingCodeLine = 
@@ -375,45 +440,60 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
             /// Found an MDOC_START or SDocLineStart
             OneOrMore(Sequence(TestNot(FirstOf(MDOC_START, SDocLineStart())), ANY))); }
 
+    /// ### Terminal Rules ###
+    /// ----------------------
+
+    /// #### Directive terminals
+    Rule API_DIR        = IgnoreCase("api");
+    Rule AUTHOR_DIR     = IgnoreCase("author");
+    Rule COPYRIGHT_DIR  = IgnoreCase("copyright");
+    Rule EXAMPLE_DIR    = IgnoreCase("example");
+    Rule INCLUDE_DIR    = IgnoreCase("include");
+    Rule ORG_DIR        = IgnoreCase("org");
+
+    /// #### Hard-coded terminals.
     Rule AT         = Ch('@').label("AT");
     Rule EOL        = FirstOf(String("\r\n"), Ch('\n'), Ch('\r')).label("EOL");
     Rule NOT_EOL    = Sequence(TestNot(EOL), ANY).label("NOT_EOL");
     Rule SPACE      = AnyOf(" \t").label("SPACE");
 
-    /// Configurable
+    /// #### Configurable terminals
     Rule MDOC_START;
     Rule MDOC_END;
     Rule MDOC_LINE_START;
     Rule SDOC_START;
 
-    /// directive terminals
+    /// ### Utility/Helper Functions. ###
-    Rule AUTHOR_DIR     = IgnoreCase("author");
+    /// ---------------------------------
-    Rule COPYRIGHT_DIR  = IgnoreCase("copyright");
-    Rule API_DIR        = IgnoreCase("api");
-    Rule EXAMPLE_DIR    = IgnoreCase("example");
-    Rule ORG_DIR        = IgnoreCase("org");
 
+    /// The `popAs` functions exist primarily to make the parser rules more readable
+    /// by providing shortcuts for common casts.
     String popAsString() { return (String) pop(); }
-
     Integer popAsInt() { return (Integer) pop(); }
 
+    /// Line number management functions.
     boolean clearLineCount() { curLineNum = 1; return true; }
-
     boolean incLineCount() { curLineNum++; return true; }
 
+    /**
+     * #### addToDocBlock
+     * Add the given block to the SourceFile node expected to be at the top of
+     * the parser value stack.
+     */
     boolean addBlockToSourceFile(Block block) {
-        SourceFile sourceFile = (SourceFile) pop();
+        ((SourceFile) peek()).blocks.add(block);
-        sourceFile.blocks.add(block);
+        return true; }
-        return push(sourceFile); }
 
     /**
-     * Pop off a DocBlock, add the given Directive or DocText and push the
+     * #### addToDocBlock
-     * DocBlock back onto the stack.
+     * Add the given Directive or DocText to the DocBlock expected to be at the
+     * top of the parser value stack.
      */
     boolean addToDocBlock(ASTNode an) {
         DocBlock docBlock = (DocBlock) pop();
         if (an instanceof Directive) {
-            docBlock.directives.add((Directive) an); }
+            docBlock.directives.add((Directive) an);
+            ((Directive) an).parentBlock = docBlock; }
         else if (an instanceof DocText) {
             docBlock.docTexts.add((DocText) an); }
         else { throw new IllegalStateException(); }
@@ -429,6 +509,12 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
         docText.value += line;
         return push(docText); }
 
+    /**
+     * #### printValueStack
+     * A method to help with debugging. It can be called during the parser
+     * rules. When called it prints out the current contents of the parser value
+     * stack.
+     */
     boolean printValueStack() {
         for (int i = 0; i < getContext().getValueStack().size(); i++) {
             System.out.println(i + ": " + peek(i)); }

src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy

@@ -145,6 +145,7 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator {
                 case DirectiveType.Author:      queueItem.priority = 10; break
                 case DirectiveType.Copyright:   queueItem.priority = 11; break
                 case DirectiveType.Example:     queueItem.priority = 50; break
+                case DirectiveType.Include:     queueItem.priority = 50; break
                 case DirectiveType.Org:         queueItem.priority =  0; break }
             
             return queueItem }
@@ -211,6 +212,9 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator {
             case DirectiveType.Example:
                 return directive.value
 
+            // TODO:
+            // case DirectiveType.Include:
+
             /// An `@org` directive is ignored here. We already emitted the id
             /// when we started the block.
             case DirectiveType.Org: return "" } }

src/main/com/jdblabs/jlp/ast/Directive.groovy

@@ -36,6 +36,13 @@ public class Directive extends ASTNode {
      *     following the directive will be included inline as an example,
      *     typically typeset in a monospace font.
      *
+     * Include
+     * :   Include the contents of a url inline in the documentation where this
+     *     directive occurs. This is primarily intended to allow the author to
+     *     include other parts of the documentation inline via the `jlp`
+     *     protocol and link anchors, but it is not restriced to the `jlp`
+     *     protocol. The include directive is followed by a URL to the resource
+     *     to include. This can be any valid URL.
      * Org
      * :   Used to create a link anchor in the documentation. The `jlp` protocol
      *     in a URL allows the author to link back to a link anchor. Refer to
@@ -43,11 +50,12 @@ public class Directive extends ASTNode {
      *     more information about link anchors.
      */
     public static enum DirectiveType {
-        Api, Author, Copyright, Example, Org;
+        Api, Author, Copyright, Example, Include, Org;
         
         public static DirectiveType parse(String typeString) {
             valueOf(typeString.toLowerCase().capitalize()) } }
 
+    public final DocBlock parentBlock;
     public final DirectiveType type;
     public final String value;