Started documenting JLP with JLP.

.gitignore

@@ -1,2 +1,3 @@
+docs/jlp-docs/
 build/
 .*.sw?

README.md

@@ -1,6 +1,5 @@
-Jonathan's Literate Programming
+# Overview
-===============================
+*Jonathan's Literate Programming* is my take on literate programming.
-
 This project grew out of a desire for a documentation system that:
 
 * generates all documentation from source-code comments,
@@ -8,3 +7,75 @@ This project grew out of a desire for a documentation system that:
   literate programming style of documentation,
 * has pluggable formatting (default to Markdown),
 
+It is inspired by Donald Knuth's concept of literate programming, as well as
+the Docco system. I wanted something that provided the readability of Docco
+but was more full-featured. To that end, JLP currently features:
+
+* *Documentation alongside code, distinct from normal comments.*
+
+    JLP uses a javadoc-like extra delimiter to seperate normal comments from
+    JLP comments.
+
+* *Support for multiple languages out of the box.*
+
+    JLP allows you to define custom comment delimiters for any language that
+    supports single-line or multi-line comments. It comes configured with
+    default settings for several languages. Ultimately I hope to cover most
+    of the common programming languages.
+
+This project is in its infancy and some of the larger goals are still unmet:
+
+* *Syntax highligting.*
+
+    All code blocks will be highlighted according to the language they are
+    written in.
+
+* *Code awareness.*
+
+    JLP will understand the code it is processing. This will require building
+    a parser for each supported language. By doing so JLP will be able to
+    generate javadoc-style API documentation intelligently, and allow the
+    author to reference code features in a native way (think javadoc @link
+    but more generic).
+
+* *Documentation Directives*
+
+    Generally I want documentation to conform to code, not code to
+    documentation, but I think some processing directives to JLP (how to
+    combine several files into one, or split one in to many for example)
+    would be useful.
+
+    In the same line of thought, it would be usefull to be able to switch
+    the presentation layer of the documentation system depending on the type
+    of file being displayed. For example, interface definitions and core
+    pieces of the API may work better with a side-by-side layout whereas
+    implementation details may work better in an interleaved layout.
+    JLP processing directives would allow the author to specify which is
+    intended on a file (or block?) level.
+
+# Project Architecture
+
+## Control and Flow
+
+* [JLPMain](jlp://jlp.jdb-labs.com/JLPMain)
+
+    The entry point to the JLP executable. Parses the command line input and
+    sets up the processor.
+
+* [Processor](jlp://jlp.jdb-labs.com/Processor)
+
+    The Processor processes one batch of input files to create a set of output files.
+    It holds the intermediate state needed by the generators and coordinates the
+    work of the parsers and generators for each of the input files.
+
+## Parsing
+
+* [JLPParser](jlp://jlp.jdb-labs.com/JLPParser)
+
+    A very simple interface for parsing JLP input.
+
+## Abstract Syntax Tree
+
+* [SourceFile](jlp://jlp.jdb-labs.com/ast/SourceFile)
+
+    The top-level AST element. This represents a source file.

doc/issues/0004tn7.rst

@@ -7,7 +7,10 @@ line with the general nature of delimited comment blocks, which do not place
 any restrictions on what comes before the start delimiter or after the end
 delimiter.
 
-========= ==========
+
-Created:  2011-09-07
+----
-Resolved: YYYY-MM-DD
+
-========= ==========
+========= ===================
+Created : 2011-09-07         
+Resolved: 2011-12-25T23:26:07
+========= ===================

doc/issues/0004ts7.rst

@@ -0,0 +1,15 @@
+Fix delimited doc block behavior.
+=================================
+
+Delimited doc blocks require that the start token be the first non-space token
+on the line it is on and that the end token be on it's own line. This is not in
+line with the general nature of delimited comment blocks, which do not place
+any restrictions on what comes before the start delimiter or after the end
+delimiter.
+
+----
+
+========= ==========
+Created:  2011-09-07
+Resolved: YYYY-MM-DD
+========= ==========

doc/issues/0006bn5.rst

@@ -0,0 +1,12 @@
+Encode Documentation and Code Characters for HTML
+=================================================
+
+The text of the documentation and the code is not being HTML encoded,
+so some characters (most notably `<`) are causing wierd display issues
+in the resulting output.
+
+----
+
+======== ===================
+Created: 2011-12-26T00:43:44
+======== ===================

project.properties

@@ -1,7 +1,7 @@
-#Sun, 25 Dec 2011 23:07:02 -0600
+#Sun, 25 Dec 2011 23:23:16 -0600
 name=jlp
 version=1.1
-build.number=6
+build.number=7
 lib.local=true
 release.dir=release
 main.class=com.jdblabs.jlp.JLPMain

resources/main/jlp.css

@@ -28,7 +28,7 @@ table td {
     outline: 0; }
 
 td.docs, th.docs {
-    max-width: 450px;
+    max-width: 600px;
     min-width: 450px;
     min-height: 5pc;
     padding: 10px 25px 1px 50px;

src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy

@@ -1,9 +1,16 @@
+/**
+ * @author Jonathan Bernard (jdb@jdb-labs.com)
+ * @copyright JDB Labs 2010-2011
+ */
 package com.jdblabs.jlp
 
 import com.jdblabs.jlp.ast.*
 import java.util.List
 import java.util.Map
 
+/**
+ * @org jlp.jdb-labs.com/JLPBaseGenerator
+ */
 public abstract class JLPBaseGenerator {
 
     protected Processor processor

src/main/com/jdblabs/jlp/JLPMain.groovy

@@ -1,3 +1,7 @@
+/**
+ * @author Jonathan Bernard
+ * @copyright JDB Labs 2010-2011
+ */
 package com.jdblabs.jlp
 
 import com.jdblabs.jlp.ast.ASTNode
@@ -5,11 +9,18 @@ import com.jdblabs.jlp.ast.SourceFile
 import org.parboiled.Parboiled
 import org.parboiled.parserunners.ReportingParseRunner
 
+/**
+ * @api JLPMain is the entrypoint for the system. It is responsible for parsing
+ * the command-line options and invoking the Processor.
+ * @org jlp.jdb-labs.com/JLPMain
+ */
 public class JLPMain {
 
     public static void main(String[] args) {
 
-        // create command-line parser
+        /// #### Define command-line options.
+        /// We are using the Groovy wrapper around the Apache Commons CLI
+        /// library.
         CliBuilder cli = new CliBuilder(
             usage: 'jlp [options] <src-file> <src-file> ...')
 
@@ -23,78 +34,91 @@ public class JLPMain {
         cli._(longOpt: 'relative-path-root', args: 1, required: false,
             'Resolve all relative paths against this root.')
 
-        // parse options
+        /// #### Parse the options.
         def opts = cli.parse(args)
 
-        // display help if requested
+        /// Display help if requested.
         if (opts.h) {
             cli.usage()
             return }
 
-        // get the relative path root (or set to current directory if not given)
+        /// Get the relative path root (or set to current directory if it was
+        /// not given)
         def pathRoot = new File(opts."relative-path-root" ?: ".")
 
-        // fail if our root is non-existant
+        /// If our root is non-existant we will print an error and exit.. This
+        /// is possible if a relative path root was passed as an option.
         if (!pathRoot.exists() || !pathRoot.isDirectory()) {
             System.err.println "'${pathRoot.path}' is not a valid directory."
             System.exit(1) }
 
-        // get the output directory and create it if necessary
+        /// Get the output directory, either from the command line or by
+        /// default.
         def outputDir = opts.o ? new File(opts.o) : new File("jlp-docs")
 
-        // resolve the output directory against our relative root
+        /// Resolve the output directory against our relative root
         if (!outputDir.isAbsolute()) {
             outputDir = new File(pathRoot, outputDir.path) }
 
-        // create the output directory if it does not exist
+        /// Create the output directory if it does not exist.
         if (!outputDir.exists()) outputDir.mkdirs()
 
-        // get the CSS theme to use. We will start by assuming the default will
+        /// Get the CSS theme to use. We will start by assuming the default will
-        // be used.
+        /// be used.
         def css = JLPMain.class.getResourceAsStream("/jlp.css")
 
-        // If the CSS file was specified on the command-line, let's look for it.
+        /// If the CSS file was specified on the command-line, let's look for it.
         if (opts.'css-file') {
             def cssFile = new File(opts.'css-file')
-            // resolve against our relative root
+
+            /// Resolve the file against our relative root.
             if (!cssFile.isAbsolute()) {
                 cssFile = new File(pathRoot, cssFile.path) }
                 
-            // Finally, make sure the file actually exists.
+            /// Finally, make sure the CSS file actually exists.
             if (cssFile.exists()) { css = cssFile }
+
+            /// If it does not, we are going to warn the user and keep the
+            /// default.
             else {
                 println "WARN: Could not fine the custom CSS file: '" +
                     "${cssFile.canonicalPath}'."
                 println "      Using the default CSS." }}
 
-        // Extract the text from our css source (either an InputStream or a
+        /// Extract the text from our css source (either an InputStream or a
-        // File)
+        /// File)
         css = css.text
 
-        // get files passed in
+        /// #### Create the input file list.
+
+        /// We will start with the filenames passed as arguments on the command
+        /// line.
         def filenames = opts.getArgs()
         def inputFiles = []
 
         filenames.each { filename ->
-            // create a File object
-            File file = new File(filename)
 
-            // if this is a relative path, resolve it against our path root
+            /// For each filename we try to resolve it to an actual file
+            /// relative to our root.
+            File file = new File(filename)
             if (!file.isAbsolute()) { file = new File(pathRoot, filename) } 
 
-            // if this file does not exist, warn the user and skip it
+            /// If this file does not exist, warn the user and skip it.
             if (!file.exists()) {
                 System.err.println(
                     "'${file.canonicalPath}' does not exist: ignored.")
                 return }
                 
-            // if this file is a directory, add all the files in it (recurse
+            /// If this file is a directory, we want to add all the files in it
-            // into sub-directories and add their contents as well).
+            /// to our input list, recursing into all the subdirectories and
+            /// adding their files as well.
             if (file.isDirectory()) { file.eachFileRecurse {
                 if (it.isFile()) { inputFiles << it }}}
 
+            /// Not a directory, just add the file.
             else { inputFiles << file } }
 
+        /// #### Process the files.
         Processor.process(outputDir, css, inputFiles)
     }
 

src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy

@@ -1,3 +1,7 @@
+/**
+ * @author Jonathan Bernard (jdb@jdb-labs.com)
+ * @copyright JDB Labs 2010-2011
+ */
 package com.jdblabs.jlp
 
 import com.jdblabs.jlp.ast.*
@@ -8,6 +12,9 @@ import org.pegdown.PegDownProcessor
 
 import java.util.List
 
+/**
+ * @org jlp.jdb-labs.com/LiterateMarkdownGenerator
+ */
 public class LiterateMarkdownGenerator extends JLPBaseGenerator {
 
     protected PegDownProcessor pegdown

src/main/com/jdblabs/jlp/Processor.groovy

@@ -1,3 +1,7 @@
+/**
+ * @author Jonathan Bernard
+ * @copyright JDB Labs 2010-2011
+ */
 package com.jdblabs.jlp
 
 import org.parboiled.BaseParser
@@ -7,106 +11,142 @@ import org.parboiled.Parboiled
  * Processor processes one batch of input files to create a set of output files.
  * It holds the intermediate state needed by the generators and coordinates the
  * work of the parsers and generators for each of the input files.
+ * @org jlp.jdb-labs.com/Processor
  */
 public class Processor {
 
+    /// ### Public State
+    /// @org jlp.jdb-labs.com/Processor/public-state
+
+    /// A map of all the link anchors defined in the documents.
     public Map<String, LinkAnchor> linkAnchors = [:]
+
+    /// A map of all the documents being processed.
     public Map<String, TargetDoc> docs = [:]
+
+    /// The id of the document currently being processed.
     public String currentDocId = null
 
+    /// The root of the input path.
     public File inputRoot
+
+    /// The root of the output path.
     public File outputRoot
+
+    /// The CSS that will be used for the resulting HTML documents. Note that
+    /// this is the CSS file contents, not the name of a CSS file.
     public String css
 
-    // shortcut for docs[currentDocId]
+    /// A shortcut for `docs[currentDocId]`
     public TargetDoc currentDoc
 
+    /// ### Non-public State
+    /// @org jlp.jdb-labs.com/Processor/non-public-state
+
+    /// Maps of all the parsers and generators by input file type. Parsers and
+    /// generators are both safe for re-use within a single thread, so we cache
+    /// them here.
     protected Map<String, JLPParser> parsers = [:]
     protected Map<String, JLPBaseGenerator> generators = [:]
 
+    /// ### Public Methods.
+    /// @org jlp.jdb-labs.com/Processor/public-methods
+
+    /**
+     * #### Processor.process
+     * @org jlp.jdb-labs.com/Processor/process
+     * @api Process the input files given, writing the resulting documentation
+     * to the directory named in `outputDir`, using the CSS given in `css`
+     */
     public static void process(File outputDir, String css,
     List<File> inputFiles) {
 
-        // find the closest common parent folder to all of the files
+        /// Find the closest common parent folder to all of the files given.
+        /// This will be our input root for the parsing process.
         File inputDir = inputFiles.inject(inputFiles[0]) { commonRoot, file ->
             getCommonParent(commonRoot, file) }
 
-        // create our processor instance
+        /// Create an instance of this class with the options given.
         Processor inst = new Processor(
             inputRoot: inputDir,
             outputRoot: outputDir,
             css: css)
 
-        // run the process
+        /// Run the process.
-        inst.process(inputFiles)
+        inst.process(inputFiles) }
-    }
 
+    /// ### Non-Public implementation methods.
+    /// @org jlp.jdb-labs.com/Processor/non-public-methods
+
+    /**
+     * #### process
+     * @org jlp.jdb-labs.com/Processor/process2
+     */
     protected void process(inputFiles) {
 
-        // Remember that the data for the processing run was initialized by the
+        /// Remember that the data for the processing run was initialized by the
-        // constructor.
+        /// constructor.
 
+        /// * Create the processing context for each input file. We are using
+        ///   the relative path of the file as the document id.
         inputFiles.each { file ->
-
-            // set the current doc id
             def docId = getRelativeFilepath(inputRoot, file)
-
-            // create the processing context for this file
             docs[docId] = new TargetDoc(sourceFile: file) }
 
-        // Parse the input files.
+        /// * Run the parse phase on each of the files. For each file, we load
+        ///   the parser for that file type and parse the file into an abstract
+        ///   syntax tree (AST).
         processDocs {
-            
-            // TODO: add logic to configure or autodetect the correct parser for
-            // each file
             def parser = getParser(sourceTypeForFile(currentDoc.sourceFile))
-
             // TODO: error detection
             currentDoc.sourceAST = parser.parse(currentDoc.sourceFile.text) }
 
-        // run our generator parse phase (first pass over the ASTs)
+        /// * Run our generator parse phase (see
+        ///   jlp://com.jdb-labs.jlp.JLPBaseGenerator/phases for an explanation
+        ///   of the generator phases).
         processDocs {
-
+            def generator = getGenerator(sourceTypeForFile(currentDoc.sourceFile))
-            // TODO: add logic to configure or autodetect the correct generator
+            // TODO: error detection
-            // for each file
-            def generator =
-                getGenerator(sourceTypeForFile(currentDoc.sourceFile))
             generator.parse(currentDoc.sourceAST) }
 
 
-        // Second pass by the generators, create output.
+        /// * Second pass by the generators, the emit phase.
         processDocs {
-            
+            def generator = getGenerator(sourceTypeForFile(currentDoc.sourceFile))
-            // TODO: add logic to configure or autodetect the correct generator
-            // for each file
-            def generator =
-                getGenerator(sourceTypeForFile(currentDoc.sourceFile))
             currentDoc.output = generator.emit(currentDoc.sourceAST) }
 
-        // Write the output to the output directory
+        /// * Write the output to the output directory.
         processDocs {
             
-            // create the path to the output file
+            /// Create the path and file object for the output file
             String relativePath =
                 getRelativeFilepath(inputRoot, currentDoc.sourceFile)
 
             File outputFile = new File(outputRoot, relativePath + ".html")
             File outputDir = outputFile.parentFile
 
-            // create the directory if need be
+            /// Create the directory for this file if it does not exist.
             if (!outputDir.exists()) { outputDir.mkdirs() }
 
-            // write the css file if it does not exist
+            /// Write the CSS file if it does not exist.
             File cssFile = new File(outputDir, "jlp.css")
             if (!cssFile.exists()) { cssFile.withWriter{ it.println css } }
 
-            // Copy the source file over
+            /// Copy the source file over.
+            // TODO: make this behavior customizable.
             (new File(outputRoot, relativePath)).withWriter {
                 it.print currentDoc.sourceFile.text }
 
-            // Write the output to the file.
+            /// Write the output to the file.
             outputFile.withWriter { it.println currentDoc.output } } }
 
+    /**
+     * #### processDocs
+     * A helper method to walk over every document the processor is aware of,
+     * setting up the `currentDocId` and `currentDoc` variables before calling
+     * the given closure.
+     * @org jlp.jdb-labs.com/Processor/processDocs
+     */
     protected def processDocs(Closure c) {
         docs.each { docId, doc ->
             currentDocId = docId
@@ -115,33 +155,41 @@ public class Processor {
             return c() } }
 
     /**
+     * #### getRelativeFilepath
      * Assuming our current directory is `root`, get the relative path to 
      * `file`.
+     * @org jlp.jdb-labs.com/Processor/getRelativeFilepath
      */
     public static String getRelativeFilepath(File root, File file) {
-        // make sure our root is a directory
+        /// Make sure our root is a directory
         if (!root.isDirectory()) root= root.parentFile
 
+        /// Split both paths into their individual parts.
         def rootPath = root.canonicalPath.split('/')
         def filePath = file.canonicalPath.split('/')
 
         def relativePath = []
 
-        // find the point of divergence in the two paths
+        /// Find the point of divergence in the two paths by walking down their
+        /// parts until we find a pair that do not match.
         int i = 0
         while (i < Math.min(rootPath.length, filePath.length) &&
                rootPath[i] == filePath[i]) { i++ }
 
-        // backtrack from our root to our common parent directory
+        /// Backtrack from our root to our newly-found common parent directory.
         (i..<rootPath.length).each { relativePath << ".." }
 
-        // add the path from our common parent directory to our file
+        /// Add the remainder of the path from our common parent directory to
+        /// our file.
         (i..<filePath.length).each { j -> relativePath << filePath[j] }
 
+        /// Reconstitute the parts into one string.
         return relativePath.join('/') }
 
     /**
+     * #### getCommonParent
      * Find the common parent directory to the given files.
+     * @org jlp.jdb-labs.com/Processor/getCommonParent
      */ 
     public static File getCommonParent(File file1, File file2) {
             def path1 = file1.canonicalPath.split('/')
@@ -158,13 +206,23 @@ public class Processor {
 
             return new File(newPath.join('/')) }
 
+    /**
+     * #### sourceTypeForFile
+     * Lookup the source type for a given file. We do a lookup based on the file
+     * extension for file types we recognize.
+     * @org jlp.jdb-labs.com/Processor/sourceTypeForFile
+     */
     public static sourceTypeForFile(File sourceFile) {
+
+        /// First we need to find the file extension.
         String extension
         def nameParts = sourceFile.name.split(/\./)
 
+        /// If there is no extension, then this is a binary file.
         if (nameParts.length == 1) { return 'binary' }
         else { extension = nameParts[-1] }
 
+        /// Lookup the file type by extension
         switch (extension) {
             case 'c': case 'h': return 'c';
             case 'c++': case 'h++': case 'cpp': case 'hpp': return 'c++';
@@ -175,17 +233,33 @@ public class Processor {
             case 'md': return 'markdown';
             default: return 'unknown'; }}
 
+    /**
+     * #### getGenerator
+     * Get a generator for the given source file type.
+     * @org jlp.jdb-labs.com/Processor/getGenerator
+     */
     protected getGenerator(String sourceType) {
+        /// We lazily create the generators.
         if (generators[sourceType] == null) {
             switch(sourceType) {
+                /// So far, all languages are using the vanilla
+                ///[`LiterateMarkdownGenerator`]
+                ///(jlp://jlp.jdb-labs.com/LiterateMarkdownGenerator)
                 default:
                     generators[sourceType] =
                         new LiterateMarkdownGenerator(this) }}
 
         return generators[sourceType] }
 
+    /**
+     * #### getParser
+     * Get a parser for the given source file type.
+     * @org jlp.jdb-labs.com/Processor/getParser
+     */
     protected getParser(String sourceType) {
+        /// We are lazily loading the parsers also.
         if (parsers[sourceType] == null) {
+            /// We do have different parsers for different languages.
             switch(sourceType) {
                 case 'erlang':
                     parsers[sourceType] = Parboiled.createParser(

src/main/com/jdblabs/jlp/ast/SourceFile.groovy

@@ -1,5 +1,13 @@
+/**
+ * @author Jonathan Bernard
+ * @copyright JDB Labs 2010-2011
+ */
 package com.jdblabs.jlp.ast
 
+/**
+ * The top-level AST element. This represents a source file.
+ * @org jlp.jdb-labs.com/ast/SourceFile
+ */
 public class SourceFile {
     public List<ASTNode> blocks = []
     public def codeAST