Refactored the overall process, fixed #0005: link behaviour.

* Refactored the overall process flow. Instead of ``JLPMain`` handling the
  process, it now reads the command line options and defers to ``Processor`` to
  handle the actual process. The ``Processor`` instance is responsible for
  processing one batch of input files and holds all the state that is common to
  this process.
* ``JLPBaseGenerator`` and generators based on it are now only responsible for
  handling one file, generating output from a source AST. As a consequence
  state that is common to the overall process is no longer stored in the
  generator but is stored on the ``Processor`` instance, which is exposed to the
  generators.
* Generators can now be instantiated directly (instead of having just a public
  static method) and are no longer one-time use. Now the life of a generator is
  expected to be the same as the life of the ``Processor``.
* Fixed inter-doc link behaviour.
* Created some data classes to replace the ad-hoc maps used to store state in
  the generator (now in the ``Processor``)

doc/issues/0005ts3.rst

@@ -0,0 +1,25 @@
+Internal links are not smart enough about cross-file linking.
+=============================================================
+
+There are two main problems with internal linking as it works now:
+
+1. The links are resolved relative to the directory of the file being processed
+   when they should be resolved relative to the root output directory.
+   
+   For example, consider ``@org id1`` defined in ``dir/file1.src``, then
+   referenced in ``dir/file2.src`` in this manner: ``[link](jlp://id1``. The
+   url written in ``dir/file2.html`` is ``dir/file1.html#id1``. However, when
+   the page is viewed, this url is interpreted relative to the directory of the
+   current page. So if the docs live at ``file:///docs`` then the url will
+   resolve to ``file:///docs/dir/dir/file1.src#id1`` instead of
+   ``file:///docs/dir/file1.html#id1``.
+
+2. The links do substitute the ``html`` suffix to the file names in place of the
+   files' suffixes. In the above example note that the actual urls end in
+   ``.src`` like the original input files, but the expected url ends in
+   ``.html``.
+
+========= ==========
+Created:  2011-09-08
+Resolved: 2011-09-09
+========= ==========

project.properties

@@ -1,6 +1,6 @@
-#Thu, 08 Sep 2011 12:27:26 -0500
+#Fri, 09 Sep 2011 14:26:03 -0500
 name=jlp
-version=0.2
+version=0.3
 build.number=1
 lib.local=true
 release.dir=release

src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy

@@ -6,36 +6,18 @@ import java.util.Map
 
 public abstract class JLPBaseGenerator {
 
-    protected Map docState
+    protected Processor processor
 
-    protected JLPBaseGenerator() {
+    protected JLPBaseGenerator(Processor processor) {
-        docState = [orgs:           [:],        // stores `@org` references
+        this.processor = processor }
-                    codeTrees:      [:],        // stores code ASTs for
-                    currentDocId:   false ] }   // store the current docid
 
-    protected Map<String, String> generate(Map<String, SourceFile> sources) {
+    protected String generate(SourceFile source) {
-        Map result = [:]
 
         // run the parse phase
-        sources.each { sourceId, sourceAST ->
+        parse(source)
-
-            // set up the current generator state for this source
-            docState.currentDocId = sourceId
-            docState.codeTrees[sourceId] = sourceAST.codeAST
-
-            parse(sourceAST) }
 
         // run the emit phase
-        sources.each { sourceId, sourceAST ->
+        return emit(source) }
-
-            // set up the current generator state for this source
-            docState.currentDocId = sourceId
-
-            // generate the doc for this source
-            result[sourceId] = emit(sourceAST) }
-
-        // return our results
-        return result }
 
     protected void parse(SourceFile sourceFile) {
         sourceFile.blocks.each { block -> parse(block) } }

src/main/com/jdblabs/jlp/JLPMain.groovy

@@ -7,12 +7,8 @@ import org.parboiled.parserunners.ReportingParseRunner
 
 public class JLPMain {
 
-    private JLPPegParser parser
-
     public static void main(String[] args) {
 
-        JLPMain inst = new JLPMain()
-
         // create command-line parser
         CliBuilder cli = new CliBuilder(
             usage: 'jlp [options] <src-file> <src-file> ...')
@@ -20,7 +16,8 @@ public class JLPMain {
         // define options
         cli.h('Print this help information.', longOpt: 'help', required: false)
         cli.o("Output directory (defaults to 'jlp-docs').",
-            longOpt: 'output-dir', required: false)
+            longOpt: 'output-dir', args: 1, argName: "output-dir",
+            required: false)
         cli._(longOpt: 'relative-path-root', args: 1, required: false,
             'Resolve all relative paths against this root.')
 
@@ -56,67 +53,21 @@ public class JLPMain {
 
         // get files passed in
         def filenames = opts.getArgs()
-        
+        def inputFiles = (filenames.collect { filename ->
-        // parse input
+            // create a File object
-        Map parsedFiles = filenames.inject([:]) { acc, filename ->
-
-            // create the File object
             File file = new File(filename) 
             
-            // if this is a relative path, resolve it against our root path
+            // if this is a relative path, resolve it against our path root
             if (!file.isAbsolute()) { file = new File(pathRoot, filename) } 
             
-            // parse the file, store the result
+            // warn the user about files that do not exist
-            acc[filename] = inst.parse(file)
+            if (!file.exists())  {
-            return acc }
+                System.err.println
+                    "'${file.canonicalPath}' does not exist: ignored." }
 
-        // generate output
+            return file }).findAll { it.exists() }
-        Map htmlDocs = LiterateMarkdownGenerator.generateDocuments(parsedFiles)
 
-        // write output files
+        Processor.process(outputDir, css, inputFiles)
-        htmlDocs.each { filename, html ->
-
-            // split the path into parts
-            def fileParts = filename.split(/[\.\/]/)
-
-            // default the subdirectory to the output directory
-            File subDir = outputDir
-
-            // if the input file was in a subdirectory, we want to mirror that
-            // structure here.
-            if (fileParts.length > 2) {
-
-                // find the relative subdirectory of this file
-                subDir = new File(outputDir, fileParts[0..-3].join('/'))
-
-                // create that directory if needed
-                if (!subDir.exists()) subDir.mkdirs()
     }
 
-            // recreate the output filename
-            def outputFilename = fileParts[-2] + ".html"
-
-            // write the HTML to the file
-            new File(subDir, outputFilename).withWriter { fileOut ->
-
-                // write the CSS if it is not present
-                File cssFile = new File(subDir, "jlp.css")
-                if (!cssFile.exists()) cssFile.withWriter { cssOut ->
-                    cssOut.println css }
-
-
-                // write the file
-                fileOut.println html } }
-    }
-
-    public JLPMain() {
-        parser = Parboiled.createParser(JLPPegParser.class)
-    }
-
-    public SourceFile parse(File inputFile) {
-        def parseRunner = new ReportingParseRunner(parser.SourceFile())
-
-        // parse the file
-        return parseRunner.run(inputFile.text).resultValue
-    }
 }

src/main/com/jdblabs/jlp/LinkAnchor.groovy

@@ -0,0 +1,11 @@
+package com.jdblabs.jlp
+
+import com.jdblabs.jlp.ast.Directive
+
+public class LinkAnchor {
+
+    public String id
+    public Directive directive
+    public String sourceDocId
+
+}

src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy

@@ -12,25 +12,21 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator {
 
     protected PegDownProcessor pegdown
 
-    protected LiterateMarkdownGenerator() {
+    public LiterateMarkdownGenerator(Processor processor) {
-        super()
+        super(processor)
 
         pegdown = new PegDownProcessor(
             Extensions.TABLES | Extensions.DEFINITIONS) }
 
-    protected static Map<String, String> generateDocuments(
-    Map<String, SourceFile> sources) {
-        LiterateMarkdownGenerator inst = new LiterateMarkdownGenerator()
-        return inst.generate(sources) }
-
     protected void parse(Directive directive) {
         switch(directive.type) {
             case DirectiveType.Org:
-                def orgMap = [:]
+                LinkAnchor anchor = new LinkAnchor(
-                orgMap.id = directive.value
+                    id: directive.value,
-                orgMap.directive = directive
+                    directive: directive,
-                orgMap.sourceDocId = docState.currentDocId
+                    sourceDocId: processor.currentDocId)
-                docState.orgs[directive.value] = orgMap
+
+                processor.linkAnchors[anchor.id] = anchor
                 break;
             default:
                 break // do nothing
@@ -48,7 +44,7 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator {
 """<!DOCTYPE html>
 <html>
     <head>
-        <title>${docState.currentDocId}</title>
+        <title>${processor.currentDocId}</title>
         <meta http-equiv="content-type" content="text/html; charset=UTF-8">
         <link rel="stylesheet" media="all" href="jlp.css"/>
     </head>
@@ -56,7 +52,7 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator {
         <div id="container">
             <table cellpadding="0" cellspacing="0">
                 <thead><tr>
-                    <th class="docs"><h1>${docState.currentDocId}</h1></th>
+                    <th class="docs"><h1>${processor.currentDocId}</h1></th>
                     <th class="code"/>
                 </tr></thead>
                 <tbody>""")
@@ -187,14 +183,33 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator {
         // replace internal `jlp://` links with actual links based on`@org`
         // references
         html = html.replaceAll(/jlp:\/\/([^\s"]+)/) { wholeMatch, linkId ->
-            def link = docState.orgs[linkId]
+
+            // Get the org data stored for this org id.
+            def link = processor.linkAnchors[linkId]
             String newLink
+
             if (!link) {
+                // We do not have any reference to this id.
                 /* TODO: log error */
                 newLink = "broken_link(${linkId})" }
-            else if (docState.currentDocId == link.sourceDocId) {
+
+            // This link points to a location in this document.
+            else if (processor.currentDocId == link.sourceDocId) {
                 newLink = "#$linkId" }
-            else { newLink = "${link.sourceDocId}#${linkId}" }
+
+            // The link should point to a different document.
+            else {
+                thisDoc = processor.currentDoc
+                linkDoc = processor.docs[link.sourceDocId]
+
+                pathToLinkedDoc = processor.getRelativePath(
+                    thisDoc.sourceFile.parentFile,
+                    thatDoc.sourceFile)
+
+                // The target document may not be in the same directory
+                // as us, backtrack to the (relative) top of our directory
+                // structure.
+                newLink = pathToLinkedDoc + "/" + ".html#${linkId}" }
                 
             return newLink }
 

src/main/com/jdblabs/jlp/Processor.groovy

@@ -0,0 +1,166 @@
+package com.jdblabs.jlp
+
+import org.parboiled.BaseParser
+import org.parboiled.Parboiled
+import org.parboiled.parserunners.ReportingParseRunner
+
+/**
+ * Processor processes one batch of input files to create a set of output files.
+ * It holds the intermediate state needed by the generators and coordinates the
+ * work of the parsers and generators for each of the input files.
+ */
+public class Processor {
+
+    public Map<String, LinkAnchor> linkAnchors = [:]
+    public Map<String, TargetDoc> docs = [:]
+    public String currentDocId = null
+
+    public File inputRoot
+    public File outputRoot
+    public String css
+
+    // shortcut for docs[currentDocId]
+    public TargetDoc currentDoc
+
+    protected Map<Class, BaseParser> parsers = [:]
+    protected Map<Class, JLPBaseGenerator> generators = [:]
+
+    public static void process(File outputDir, String css,
+    List<File> inputFiles) {
+
+        // find the closest common parent folder to all of the files
+        File inputDir = inputFiles.inject(inputFiles[0]) { commonRoot, file ->
+            getCommonParent(commonRoot, file) }
+
+        // create our processor instance
+        Processor inst = new Processor(
+            inputRoot: inputDir,
+            outputRoot: outputDir,
+            css: css)
+
+        // run the process
+        inst.process(inputFiles)
+    }
+
+    protected void process(inputFiles) {
+
+        // Remember that the data for the processing run was initialized by the
+        // constructor.
+
+        inputFiles.each { file ->
+
+            // set the current doc id
+            def docId = getRelativeFilepath(inputRoot, file)
+
+            // create the processing context for this file
+            docs[docId] = new TargetDoc(sourceFile: file) }
+
+        // Parse the input files.
+        processDocs {
+            
+            // TODO: add logic to configure or autodetect the correct parser for
+            // each file
+            def parser = getParser(JLPPegParser)
+            def parseRunner = new ReportingParseRunner(parser.SourceFile())
+
+            currentDoc.sourceAST = parseRunner.run(
+                currentDoc.sourceFile.text).resultValue }
+
+        // generate output
+        processDocs {
+
+            // TODO: add logic to configure or autodetect the correct generator
+            // for each file
+            def generator = getGenerator(LiterateMarkdownGenerator)
+            currentDoc.output = generator.generate(currentDoc.sourceAST) }
+
+        // Write the output to the output directory
+        processDocs {
+            
+            // create the path to the output file
+            String relativePath =
+                getRelativeFilepath(inputRoot, currentDoc.sourceFile)
+
+            File outputFile = new File(outputRoot, relativePath + ".html")
+            File outputDir = outputFile.parentFile
+
+            // create the directory if need be
+            if (!outputDir.exists()) {
+                outputDir.mkdirs() }
+
+            // write the css file if it does not exist
+            File cssFile = new File(outputDir, "jlp.css")
+            if (!cssFile.exists()) { cssFile.withWriter{ it.println css } }
+
+            // Copy the source file over
+            (new File(outputRoot, relativePath)).withWriter {
+                it.print currentDoc.sourceFile.text }
+
+            // Write the output to the file.
+            outputFile.withWriter { it.println currentDoc.output } } }
+
+    protected def processDocs(Closure c) {
+        docs.each { docId, doc ->
+            currentDocId = docId
+            currentDoc = doc
+
+            return c() } }
+
+    /**
+     * Assuming our current directory is `root`, get the relative path to 
+     * `file`.
+     */
+    public static String getRelativeFilepath(File root, File file) {
+        // make sure our root is a directory
+        if (!root.isDirectory()) root= root.parentFile
+
+        def rootPath = root.canonicalPath.split('/')
+        def filePath = file.canonicalPath.split('/')
+
+        def relativePath = []
+
+        // find the point of divergence in the two paths
+        int i = 0
+        while (i < Math.min(rootPath.length, filePath.length) &&
+               rootPath[i] == filePath[i]) { i++ }
+
+        // backtrack from our root to our common parent directory
+        (i..<rootPath.length).each { relativePath << ".." }
+
+        // add the path from our common parent directory to our file
+        (i..<filePath.length).each { j -> relativePath << filePath[j] }
+
+        return relativePath.join('/') }
+
+    /**
+     * Find the common parent directory to the given files.
+     */ 
+    public static File getCommonParent(File file1, File file2) {
+            def path1 = file1.canonicalPath.split('/')
+            def path2 = file2.canonicalPath.split('/')
+            def newPath = []
+
+            // build new commonPath based on matching paths so far
+            int i = 0
+            while (i < Math.min(path1.length, path2.length) &&
+                   path1[i] == path2[i]) {
+                
+                newPath << path2[i]
+                i++ }
+
+            return new File(newPath.join('/')) }
+
+    protected getGenerator(Class generatorClass) {
+        if (generators[generatorClass] == null) {
+            def constructor = generatorClass.getConstructor(Processor)
+            generators[generatorClass] = constructor.newInstance(this)
+        }
+
+        return generators[generatorClass] }
+
+    protected getParser(Class parserClass) {
+        if (parsers[parserClass] == null) {
+            parsers[parserClass] = Parboiled.createParser(parserClass) }
+
+        return parsers[parserClass] }
+}

src/main/com/jdblabs/jlp/TargetDoc.groovy

@@ -0,0 +1,10 @@
+package com.jdblabs.jlp
+
+import com.jdblabs.jlp.ast.SourceFile
+
+public class TargetDoc {
+
+    public SourceFile sourceAST
+    public File sourceFile
+    public String output
+}