From aac5915df7ffbbf6d2df8d8ec66247a606650cb5 Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Thu, 29 Dec 2011 15:45:21 -0600 Subject: [PATCH] Ground work for include directive, documentation. * Further documentation. * Decided to add an `include` directive. This will have reprecussions directly for the JLPPegParser, Directive, and LinkAnchor classes. Include needs more semantic meaning than we currently have in the process because the author needs some way to understand what is being included. If you include an org link defined earlier does it include the whole file? Just that doc block? Both may be the desired behavior in different situations, but I do not want to add a complex syntax for selecting, just name the link. Therefore there must be something about the link that determines how much in included. This means we need more information in LinkAnchor, some link `type` at a minimum. My current thought is: * @org defined links--this is the only type of LinkAnchor defined right now--always bring the whole DocBlock when they are included. * A new type of link anchor, call them source file links, bring the whole file when they are included. These types of anchors would be automatically created by the parser/generator (have not decided where it belongs yet). The whole SourceFile would need to be included, but we do not want to emit the normal header for the file so we may have to do somthing about the initial emit method. * Additional types of link anchors arise when we add language awareness to the process. In my current vision we will automatically add link anchors of different types when we parse code sections (function definition, class definition, etc.) and the include directive would bring in all the DocBlocks related to that code node. * Additional types of link anchors will arise when we implement the @api directive, we will think about that in the future. * Updated the JLPPegParser to recognise include directives. * Added the include directive to Directive.DirectiveTypes --- doc/issues/0007fn5.rst | 12 ++ project.properties | 6 +- src/main/com/jdblabs/jlp/JLPPegParser.java | 134 ++++++++++++++---- .../jlp/LiterateMarkdownGenerator.groovy | 4 + src/main/com/jdblabs/jlp/ast/Directive.groovy | 10 +- 5 files changed, 138 insertions(+), 28 deletions(-) create mode 100644 doc/issues/0007fn5.rst diff --git a/doc/issues/0007fn5.rst b/doc/issues/0007fn5.rst new file mode 100644 index 0000000..9d837de --- /dev/null +++ b/doc/issues/0007fn5.rst @@ -0,0 +1,12 @@ +`Include` Directive. +==================== + +Add a new directive: `include`. This directive allows the author to include +the contents of a URL inline in the documentation (can be an internal JLP +reference URL). + +---- + +======== =================== +Created: 2011-12-29T11:23:52 +======== =================== diff --git a/project.properties b/project.properties index 99ea335..4b0c514 100644 --- a/project.properties +++ b/project.properties @@ -1,7 +1,7 @@ -#Wed, 28 Dec 2011 15:44:01 -0600 +#Thu, 29 Dec 2011 15:43:59 -0600 name=jlp -version=1.2 -build.number=10 +version=1.3a +build.number=0 lib.local=true release.dir=release main.class=com.jdblabs.jlp.JLPMain diff --git a/src/main/com/jdblabs/jlp/JLPPegParser.java b/src/main/com/jdblabs/jlp/JLPPegParser.java index beb93dd..4e6bc73 100644 --- a/src/main/com/jdblabs/jlp/JLPPegParser.java +++ b/src/main/com/jdblabs/jlp/JLPPegParser.java @@ -19,6 +19,29 @@ public class JLPPegParser extends BaseParser implements JLPParser { int curLineNum = 1; + /// ### Constructors ### + /// -------------------- + /// @org jlp.jdb-labs.com/JLPPegParser/constructors + + /** + * Being a Parboiled parser, this class should not be instantiated by + * directly calling the constructor, but by calling + * `Parboiled.createParser()`. + * @example Here are some examples of how you would instantiate the + * JLPPegParser: + * + * // Erlang-style documentation + * Parboiled.createParser(JLPPegParser.class, "%%"); + * + * // C++-style documentation + * Parboiled.createParser(JLPPegParser.class, "/**", "* /", "*", "///"); + */ + + /** + * #### Full constructor + * This allows the caller to specific all four of the comment delimiters + * recognized by the parser. + */ public JLPPegParser(String mdocStart, String mdocEnd, String mdocLineStart, String sdocStart) { @@ -27,12 +50,28 @@ public class JLPPegParser extends BaseParser implements JLPParser { SDOC_START = String(sdocStart).label("SDOC_START"); MDOC_LINE_START = AnyOf(mdocLineStart).label("MDOC_LINE_START"); } + /** + * #### Single-line comments only constructor. + * This allows the caller to easily set up the parser to only recognize + * single-line comments with no defined syntax for multi-line comments. For + * example this is used to instantiate a parser for Erlang source files + * because Erlang has no defined multi-line comment syntax. + */ public JLPPegParser(String sdocStart) { MDOC_START = NOTHING; MDOC_LINE_START = NOTHING; MDOC_END = NOTHING; SDOC_START = String(sdocStart).label("SDOC_START"); } + /** + * #### Default constructor. + * The default constructor creates a JLPPegParser configured to recognize + * comments used by languages like C++ and Java. It follows the convention + * of using `/**` to start multiline documentation comments and `///` to + * start a single-line comment. This allows the author to choose which + * comments remain with the code and which are considered part of the formal + * documentation. + */ public JLPPegParser() { this("/**", "*/", "!#$%^&*()_-=+|;:'\",<>?~`", "///"); } @@ -40,7 +79,11 @@ public class JLPPegParser extends BaseParser implements JLPParser { ReportingParseRunner rpr = new ReportingParseRunner(this.SourceFile()); return (SourceFile) rpr.run(input).resultValue; } + /// ### Parser Rules ### + /// -------------------- + /** + * #### SourceFile * Parses the rule: * * SourceFile = (Block / DocBlock / CodeBlock)+ @@ -90,9 +133,9 @@ public class JLPPegParser extends BaseParser implements JLPParser { /// the CodeBlock to make a Block, which is pushed onto the /// stack: /// - /// *Note: With the way the parser is currently written, - /// this will only match a CodeBlock that occurs - /// before any DocBlock.* + /// *Note: With the way the parser is currently written,* + /// *this will only match a CodeBlock that occurs before* + /// *any DocBlock.* Sequence( /// 1. Remember the line number for the Block. push(curLineNum), @@ -112,6 +155,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { addBlockToSourceFile((Block) pop())))); } /** + * #### Block * Parses the rule: * * Block = DocBlock CodeBlock @@ -129,6 +173,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { push(new Block((CodeBlock) pop(), (DocBlock) pop(), popAsInt()))); } /** + * #### DocBlock * Parses the rule: * * DocBlock = SDocBlock / MDocBlock @@ -138,6 +183,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { Rule DocBlock() { return FirstOf(SDocBlock(), MDocBlock()); } /** + * #### SDocBlock * Parses the rule: * * SDocBlock = (SDirective / SDocText)+ @@ -152,6 +198,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { addToDocBlock((ASTNode) pop())))); } /** + * #### MDocBlock * Parses the rule: * * MDocBlock = MDOC_START (MDirective / MDocText)+ MDOC_END @@ -163,14 +210,15 @@ public class JLPPegParser extends BaseParser implements JLPParser { push(new DocBlock(curLineNum)), MDOC_START, ZeroOrMore(Sequence( - /// We need to be careful to exclude MDOC_END here, as there can + /// We need to be careful to exclude `MDOC_END` here, as there can /// be some confusion otherwise between the start of a line with - /// MDOC_LINE_START and MDOC_END depending on what values the + /// `MDOC_LINE_START` and `MDOC_END` depending on what values the /// user has chosen for them TestNot(MDOC_END), FirstOf(MDirective(), MDocText()), addToDocBlock((ASTNode) pop()))), MDOC_END); } /** + * #### CodeBlock * Parses the rule: * * CodeBlock = (RemainingCodeLine)+ @@ -184,6 +232,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { addToCodeBlock(match())))); } /** + * #### SDirective * Parses the rule: * * SDirective = SDocLineStart AT (SLongDirective / SShortDirective) @@ -195,6 +244,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { SDocLineStart(), AT, FirstOf(SLongDirective(), SShortDirective())); } /** + * #### MDirective * Parses the rule: * * MDirective = MDocLineStart? AT (MLongDirective / MShortDirective) @@ -207,6 +257,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { AT, FirstOf(MLongDirective(), MShortDirective())); } /** + * #### SLongDirective * Parses the rule: * * SLongDirective = @@ -229,6 +280,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { push(new Directive(popAsString(), popAsString(), popAsInt()))); } /** + * #### MLongDirective * Parses the rule: * * MLongDirective = @@ -251,36 +303,43 @@ public class JLPPegParser extends BaseParser implements JLPParser { push(new Directive(popAsString(), popAsString(), popAsInt()))); } /** + * #### SShortDirective * Parses the rule: * - * SShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingSDocLine + * SShortDirective = + * (AUTHOR_DIR / ORG_DIR / INCLUDE_DIR / COPYRIGHT_DIR) + * RemainingSDocLine * * Pushes a Directive node onto the stack. */ Rule SShortDirective() { return Sequence( push(curLineNum), - FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()), + FirstOf(AUTHOR_DIR, ORG_DIR, INCLUDE_DIR, COPYRIGHT_DIR), push(match()), RemainingSDocLine(), push(new Directive(match().trim(), popAsString(), popAsInt()))); } /** + * #### MShortDirective * Parses the rule: * - * MShortDirective = (AUTHOR_DIR / ORG_DIR / COPYRIGHT_DIR) RemainingMDocLine + * MShortDirective = + * (AUTHOR_DIR / ORG_DIR / INCLUDE_DIR / COPYRIGHT_DIR) + * RemainingMDocLine * * Pushes a Directive node onto the stack. */ Rule MShortDirective() { return Sequence( push(curLineNum), - FirstOf(AUTHOR_DIR, ORG_DIR, COPYRIGHT_DIR), push(match()), + FirstOf(AUTHOR_DIR, ORG_DIR, INCLUDE_DIR, COPYRIGHT_DIR), push(match()), RemainingMDocLine(), push(new Directive(match().trim(), popAsString(), popAsInt()))); } /** + * #### SDocText * Parses the rule: * * SDocText = (SDocLineStart !AT RemainingSDocLine)+ @@ -295,6 +354,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { addToDocText(match())))); } /** + * #### MDocText * Parses the rule: * * MDocText = (MDocLineStart? !AT RemainingMDocLine)+ @@ -310,6 +370,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { addToDocText(match())))); } /** + * #### SDocLineStart * Parses the rule: * * SDocLineStart = SPACE* SDOC_START SPACE? @@ -319,6 +380,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { ZeroOrMore(SPACE), SDOC_START, Optional(SPACE)); } /** + * #### MDocLineStart * Parses the rule: * * MDocLineStart = SPACE* !MDOC_END MDOC_LINE_START SPACE? @@ -328,6 +390,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { ZeroOrMore(SPACE), TestNot(MDOC_END), MDOC_LINE_START, Optional(SPACE)); } /** + * #### RemainingSDocLine * Parses the rule: * * RemainingSDocLine = ((!EOL)* EOL) / ((!EOL)+ EOI) @@ -338,6 +401,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { Sequence(OneOrMore(NOT_EOL), EOI, incLineCount())); } /** + * #### RemainingMDocLine * Parses the rule: * * RemainingMDocLine = @@ -356,6 +420,7 @@ public class JLPPegParser extends BaseParser implements JLPParser { OneOrMore(Sequence(TestNot(MDOC_END), ANY))); } /** + * #### RemainingCodeLine * Parses the rule: * * RemainingCodeLine = @@ -375,45 +440,60 @@ public class JLPPegParser extends BaseParser implements JLPParser { /// Found an MDOC_START or SDocLineStart OneOrMore(Sequence(TestNot(FirstOf(MDOC_START, SDocLineStart())), ANY))); } + /// ### Terminal Rules ### + /// ---------------------- + + /// #### Directive terminals + Rule API_DIR = IgnoreCase("api"); + Rule AUTHOR_DIR = IgnoreCase("author"); + Rule COPYRIGHT_DIR = IgnoreCase("copyright"); + Rule EXAMPLE_DIR = IgnoreCase("example"); + Rule INCLUDE_DIR = IgnoreCase("include"); + Rule ORG_DIR = IgnoreCase("org"); + + /// #### Hard-coded terminals. Rule AT = Ch('@').label("AT"); Rule EOL = FirstOf(String("\r\n"), Ch('\n'), Ch('\r')).label("EOL"); Rule NOT_EOL = Sequence(TestNot(EOL), ANY).label("NOT_EOL"); Rule SPACE = AnyOf(" \t").label("SPACE"); - /// Configurable + /// #### Configurable terminals Rule MDOC_START; Rule MDOC_END; Rule MDOC_LINE_START; Rule SDOC_START; - /// directive terminals - Rule AUTHOR_DIR = IgnoreCase("author"); - Rule COPYRIGHT_DIR = IgnoreCase("copyright"); - Rule API_DIR = IgnoreCase("api"); - Rule EXAMPLE_DIR = IgnoreCase("example"); - Rule ORG_DIR = IgnoreCase("org"); + /// ### Utility/Helper Functions. ### + /// --------------------------------- + /// The `popAs` functions exist primarily to make the parser rules more readable + /// by providing shortcuts for common casts. String popAsString() { return (String) pop(); } - Integer popAsInt() { return (Integer) pop(); } + /// Line number management functions. boolean clearLineCount() { curLineNum = 1; return true; } - boolean incLineCount() { curLineNum++; return true; } + /** + * #### addToDocBlock + * Add the given block to the SourceFile node expected to be at the top of + * the parser value stack. + */ boolean addBlockToSourceFile(Block block) { - SourceFile sourceFile = (SourceFile) pop(); - sourceFile.blocks.add(block); - return push(sourceFile); } + ((SourceFile) peek()).blocks.add(block); + return true; } /** - * Pop off a DocBlock, add the given Directive or DocText and push the - * DocBlock back onto the stack. + * #### addToDocBlock + * Add the given Directive or DocText to the DocBlock expected to be at the + * top of the parser value stack. */ boolean addToDocBlock(ASTNode an) { DocBlock docBlock = (DocBlock) pop(); if (an instanceof Directive) { - docBlock.directives.add((Directive) an); } + docBlock.directives.add((Directive) an); + ((Directive) an).parentBlock = docBlock; } else if (an instanceof DocText) { docBlock.docTexts.add((DocText) an); } else { throw new IllegalStateException(); } @@ -429,6 +509,12 @@ public class JLPPegParser extends BaseParser implements JLPParser { docText.value += line; return push(docText); } + /** + * #### printValueStack + * A method to help with debugging. It can be called during the parser + * rules. When called it prints out the current contents of the parser value + * stack. + */ boolean printValueStack() { for (int i = 0; i < getContext().getValueStack().size(); i++) { System.out.println(i + ": " + peek(i)); } diff --git a/src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy b/src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy index b028fc2..6eac0a4 100644 --- a/src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy +++ b/src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy @@ -145,6 +145,7 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator { case DirectiveType.Author: queueItem.priority = 10; break case DirectiveType.Copyright: queueItem.priority = 11; break case DirectiveType.Example: queueItem.priority = 50; break + case DirectiveType.Include: queueItem.priority = 50; break case DirectiveType.Org: queueItem.priority = 0; break } return queueItem } @@ -211,6 +212,9 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator { case DirectiveType.Example: return directive.value + // TODO: + // case DirectiveType.Include: + /// An `@org` directive is ignored here. We already emitted the id /// when we started the block. case DirectiveType.Org: return "" } } diff --git a/src/main/com/jdblabs/jlp/ast/Directive.groovy b/src/main/com/jdblabs/jlp/ast/Directive.groovy index 35047e9..3ebece3 100644 --- a/src/main/com/jdblabs/jlp/ast/Directive.groovy +++ b/src/main/com/jdblabs/jlp/ast/Directive.groovy @@ -36,6 +36,13 @@ public class Directive extends ASTNode { * following the directive will be included inline as an example, * typically typeset in a monospace font. * + * Include + * : Include the contents of a url inline in the documentation where this + * directive occurs. This is primarily intended to allow the author to + * include other parts of the documentation inline via the `jlp` + * protocol and link anchors, but it is not restriced to the `jlp` + * protocol. The include directive is followed by a URL to the resource + * to include. This can be any valid URL. * Org * : Used to create a link anchor in the documentation. The `jlp` protocol * in a URL allows the author to link back to a link anchor. Refer to @@ -43,11 +50,12 @@ public class Directive extends ASTNode { * more information about link anchors. */ public static enum DirectiveType { - Api, Author, Copyright, Example, Org; + Api, Author, Copyright, Example, Include, Org; public static DirectiveType parse(String typeString) { valueOf(typeString.toLowerCase().capitalize()) } } + public final DocBlock parentBlock; public final DirectiveType type; public final String value;