From d31d17d1e22d5ae92e787578164bc09be9310a50 Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Fri, 9 Sep 2011 14:28:48 -0500 Subject: [PATCH] Refactored the overall process, fixed #0005: link behaviour. * Refactored the overall process flow. Instead of ``JLPMain`` handling the process, it now reads the command line options and defers to ``Processor`` to handle the actual process. The ``Processor`` instance is responsible for processing one batch of input files and holds all the state that is common to this process. * ``JLPBaseGenerator`` and generators based on it are now only responsible for handling one file, generating output from a source AST. As a consequence state that is common to the overall process is no longer stored in the generator but is stored on the ``Processor`` instance, which is exposed to the generators. * Generators can now be instantiated directly (instead of having just a public static method) and are no longer one-time use. Now the life of a generator is expected to be the same as the life of the ``Processor``. * Fixed inter-doc link behaviour. * Created some data classes to replace the ad-hoc maps used to store state in the generator (now in the ``Processor``) --- doc/issues/0005ts3.rst | 25 +++ project.properties | 4 +- .../com/jdblabs/jlp/JLPBaseGenerator.groovy | 30 +--- src/main/com/jdblabs/jlp/JLPMain.groovy | 79 ++------- src/main/com/jdblabs/jlp/LinkAnchor.groovy | 11 ++ .../jlp/LiterateMarkdownGenerator.groovy | 49 ++++-- src/main/com/jdblabs/jlp/Processor.groovy | 166 ++++++++++++++++++ src/main/com/jdblabs/jlp/TargetDoc.groovy | 10 ++ 8 files changed, 267 insertions(+), 107 deletions(-) create mode 100644 doc/issues/0005ts3.rst create mode 100644 src/main/com/jdblabs/jlp/LinkAnchor.groovy create mode 100644 src/main/com/jdblabs/jlp/Processor.groovy create mode 100644 src/main/com/jdblabs/jlp/TargetDoc.groovy diff --git a/doc/issues/0005ts3.rst b/doc/issues/0005ts3.rst new file mode 100644 index 0000000..edde633 --- /dev/null +++ b/doc/issues/0005ts3.rst @@ -0,0 +1,25 @@ +Internal links are not smart enough about cross-file linking. +============================================================= + +There are two main problems with internal linking as it works now: + +1. The links are resolved relative to the directory of the file being processed + when they should be resolved relative to the root output directory. + + For example, consider ``@org id1`` defined in ``dir/file1.src``, then + referenced in ``dir/file2.src`` in this manner: ``[link](jlp://id1``. The + url written in ``dir/file2.html`` is ``dir/file1.html#id1``. However, when + the page is viewed, this url is interpreted relative to the directory of the + current page. So if the docs live at ``file:///docs`` then the url will + resolve to ``file:///docs/dir/dir/file1.src#id1`` instead of + ``file:///docs/dir/file1.html#id1``. + +2. The links do substitute the ``html`` suffix to the file names in place of the + files' suffixes. In the above example note that the actual urls end in + ``.src`` like the original input files, but the expected url ends in + ``.html``. + +========= ========== +Created: 2011-09-08 +Resolved: 2011-09-09 +========= ========== diff --git a/project.properties b/project.properties index 53e6b18..d4e378d 100644 --- a/project.properties +++ b/project.properties @@ -1,6 +1,6 @@ -#Thu, 08 Sep 2011 12:27:26 -0500 +#Fri, 09 Sep 2011 14:26:03 -0500 name=jlp -version=0.2 +version=0.3 build.number=1 lib.local=true release.dir=release diff --git a/src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy b/src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy index 897dd35..91f0697 100644 --- a/src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy +++ b/src/main/com/jdblabs/jlp/JLPBaseGenerator.groovy @@ -6,36 +6,18 @@ import java.util.Map public abstract class JLPBaseGenerator { - protected Map docState + protected Processor processor - protected JLPBaseGenerator() { - docState = [orgs: [:], // stores `@org` references - codeTrees: [:], // stores code ASTs for - currentDocId: false ] } // store the current docid + protected JLPBaseGenerator(Processor processor) { + this.processor = processor } - protected Map generate(Map sources) { - Map result = [:] + protected String generate(SourceFile source) { // run the parse phase - sources.each { sourceId, sourceAST -> - - // set up the current generator state for this source - docState.currentDocId = sourceId - docState.codeTrees[sourceId] = sourceAST.codeAST - - parse(sourceAST) } + parse(source) // run the emit phase - sources.each { sourceId, sourceAST -> - - // set up the current generator state for this source - docState.currentDocId = sourceId - - // generate the doc for this source - result[sourceId] = emit(sourceAST) } - - // return our results - return result } + return emit(source) } protected void parse(SourceFile sourceFile) { sourceFile.blocks.each { block -> parse(block) } } diff --git a/src/main/com/jdblabs/jlp/JLPMain.groovy b/src/main/com/jdblabs/jlp/JLPMain.groovy index 2e2626c..d2472a9 100644 --- a/src/main/com/jdblabs/jlp/JLPMain.groovy +++ b/src/main/com/jdblabs/jlp/JLPMain.groovy @@ -7,12 +7,8 @@ import org.parboiled.parserunners.ReportingParseRunner public class JLPMain { - private JLPPegParser parser - public static void main(String[] args) { - JLPMain inst = new JLPMain() - // create command-line parser CliBuilder cli = new CliBuilder( usage: 'jlp [options] ...') @@ -20,7 +16,8 @@ public class JLPMain { // define options cli.h('Print this help information.', longOpt: 'help', required: false) cli.o("Output directory (defaults to 'jlp-docs').", - longOpt: 'output-dir', required: false) + longOpt: 'output-dir', args: 1, argName: "output-dir", + required: false) cli._(longOpt: 'relative-path-root', args: 1, required: false, 'Resolve all relative paths against this root.') @@ -56,67 +53,21 @@ public class JLPMain { // get files passed in def filenames = opts.getArgs() - - // parse input - Map parsedFiles = filenames.inject([:]) { acc, filename -> + def inputFiles = (filenames.collect { filename -> + // create a File object + File file = new File(filename) + + // if this is a relative path, resolve it against our path root + if (!file.isAbsolute()) { file = new File(pathRoot, filename) } + + // warn the user about files that do not exist + if (!file.exists()) { + System.err.println + "'${file.canonicalPath}' does not exist: ignored." } - // create the File object - File file = new File(filename) + return file }).findAll { it.exists() } - // if this is a relative path, resolve it against our root path - if (!file.isAbsolute()) { file = new File(pathRoot, filename) } - - // parse the file, store the result - acc[filename] = inst.parse(file) - return acc } - - // generate output - Map htmlDocs = LiterateMarkdownGenerator.generateDocuments(parsedFiles) - - // write output files - htmlDocs.each { filename, html -> - - // split the path into parts - def fileParts = filename.split(/[\.\/]/) - - // default the subdirectory to the output directory - File subDir = outputDir - - // if the input file was in a subdirectory, we want to mirror that - // structure here. - if (fileParts.length > 2) { - - // find the relative subdirectory of this file - subDir = new File(outputDir, fileParts[0..-3].join('/')) - - // create that directory if needed - if (!subDir.exists()) subDir.mkdirs() - } - - // recreate the output filename - def outputFilename = fileParts[-2] + ".html" - - // write the HTML to the file - new File(subDir, outputFilename).withWriter { fileOut -> - - // write the CSS if it is not present - File cssFile = new File(subDir, "jlp.css") - if (!cssFile.exists()) cssFile.withWriter { cssOut -> - cssOut.println css } - - - // write the file - fileOut.println html } } + Processor.process(outputDir, css, inputFiles) } - public JLPMain() { - parser = Parboiled.createParser(JLPPegParser.class) - } - - public SourceFile parse(File inputFile) { - def parseRunner = new ReportingParseRunner(parser.SourceFile()) - - // parse the file - return parseRunner.run(inputFile.text).resultValue - } } diff --git a/src/main/com/jdblabs/jlp/LinkAnchor.groovy b/src/main/com/jdblabs/jlp/LinkAnchor.groovy new file mode 100644 index 0000000..c7cc865 --- /dev/null +++ b/src/main/com/jdblabs/jlp/LinkAnchor.groovy @@ -0,0 +1,11 @@ +package com.jdblabs.jlp + +import com.jdblabs.jlp.ast.Directive + +public class LinkAnchor { + + public String id + public Directive directive + public String sourceDocId + +} diff --git a/src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy b/src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy index db45fd8..ae4d2a2 100644 --- a/src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy +++ b/src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy @@ -12,25 +12,21 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator { protected PegDownProcessor pegdown - protected LiterateMarkdownGenerator() { - super() + public LiterateMarkdownGenerator(Processor processor) { + super(processor) pegdown = new PegDownProcessor( Extensions.TABLES | Extensions.DEFINITIONS) } - protected static Map generateDocuments( - Map sources) { - LiterateMarkdownGenerator inst = new LiterateMarkdownGenerator() - return inst.generate(sources) } - protected void parse(Directive directive) { switch(directive.type) { case DirectiveType.Org: - def orgMap = [:] - orgMap.id = directive.value - orgMap.directive = directive - orgMap.sourceDocId = docState.currentDocId - docState.orgs[directive.value] = orgMap + LinkAnchor anchor = new LinkAnchor( + id: directive.value, + directive: directive, + sourceDocId: processor.currentDocId) + + processor.linkAnchors[anchor.id] = anchor break; default: break // do nothing @@ -48,7 +44,7 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator { """ - ${docState.currentDocId} + ${processor.currentDocId} @@ -56,7 +52,7 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator {
- + """) @@ -187,14 +183,33 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator { // replace internal `jlp://` links with actual links based on`@org` // references html = html.replaceAll(/jlp:\/\/([^\s"]+)/) { wholeMatch, linkId -> - def link = docState.orgs[linkId] + + // Get the org data stored for this org id. + def link = processor.linkAnchors[linkId] String newLink + if (!link) { + // We do not have any reference to this id. /* TODO: log error */ newLink = "broken_link(${linkId})" } - else if (docState.currentDocId == link.sourceDocId) { + + // This link points to a location in this document. + else if (processor.currentDocId == link.sourceDocId) { newLink = "#$linkId" } - else { newLink = "${link.sourceDocId}#${linkId}" } + + // The link should point to a different document. + else { + thisDoc = processor.currentDoc + linkDoc = processor.docs[link.sourceDocId] + + pathToLinkedDoc = processor.getRelativePath( + thisDoc.sourceFile.parentFile, + thatDoc.sourceFile) + + // The target document may not be in the same directory + // as us, backtrack to the (relative) top of our directory + // structure. + newLink = pathToLinkedDoc + "/" + ".html#${linkId}" } return newLink } diff --git a/src/main/com/jdblabs/jlp/Processor.groovy b/src/main/com/jdblabs/jlp/Processor.groovy new file mode 100644 index 0000000..82ec9c5 --- /dev/null +++ b/src/main/com/jdblabs/jlp/Processor.groovy @@ -0,0 +1,166 @@ +package com.jdblabs.jlp + +import org.parboiled.BaseParser +import org.parboiled.Parboiled +import org.parboiled.parserunners.ReportingParseRunner + +/** + * Processor processes one batch of input files to create a set of output files. + * It holds the intermediate state needed by the generators and coordinates the + * work of the parsers and generators for each of the input files. + */ +public class Processor { + + public Map linkAnchors = [:] + public Map docs = [:] + public String currentDocId = null + + public File inputRoot + public File outputRoot + public String css + + // shortcut for docs[currentDocId] + public TargetDoc currentDoc + + protected Map parsers = [:] + protected Map generators = [:] + + public static void process(File outputDir, String css, + List inputFiles) { + + // find the closest common parent folder to all of the files + File inputDir = inputFiles.inject(inputFiles[0]) { commonRoot, file -> + getCommonParent(commonRoot, file) } + + // create our processor instance + Processor inst = new Processor( + inputRoot: inputDir, + outputRoot: outputDir, + css: css) + + // run the process + inst.process(inputFiles) + } + + protected void process(inputFiles) { + + // Remember that the data for the processing run was initialized by the + // constructor. + + inputFiles.each { file -> + + // set the current doc id + def docId = getRelativeFilepath(inputRoot, file) + + // create the processing context for this file + docs[docId] = new TargetDoc(sourceFile: file) } + + // Parse the input files. + processDocs { + + // TODO: add logic to configure or autodetect the correct parser for + // each file + def parser = getParser(JLPPegParser) + def parseRunner = new ReportingParseRunner(parser.SourceFile()) + + currentDoc.sourceAST = parseRunner.run( + currentDoc.sourceFile.text).resultValue } + + // generate output + processDocs { + + // TODO: add logic to configure or autodetect the correct generator + // for each file + def generator = getGenerator(LiterateMarkdownGenerator) + currentDoc.output = generator.generate(currentDoc.sourceAST) } + + // Write the output to the output directory + processDocs { + + // create the path to the output file + String relativePath = + getRelativeFilepath(inputRoot, currentDoc.sourceFile) + + File outputFile = new File(outputRoot, relativePath + ".html") + File outputDir = outputFile.parentFile + + // create the directory if need be + if (!outputDir.exists()) { + outputDir.mkdirs() } + + // write the css file if it does not exist + File cssFile = new File(outputDir, "jlp.css") + if (!cssFile.exists()) { cssFile.withWriter{ it.println css } } + + // Copy the source file over + (new File(outputRoot, relativePath)).withWriter { + it.print currentDoc.sourceFile.text } + + // Write the output to the file. + outputFile.withWriter { it.println currentDoc.output } } } + + protected def processDocs(Closure c) { + docs.each { docId, doc -> + currentDocId = docId + currentDoc = doc + + return c() } } + + /** + * Assuming our current directory is `root`, get the relative path to + * `file`. + */ + public static String getRelativeFilepath(File root, File file) { + // make sure our root is a directory + if (!root.isDirectory()) root= root.parentFile + + def rootPath = root.canonicalPath.split('/') + def filePath = file.canonicalPath.split('/') + + def relativePath = [] + + // find the point of divergence in the two paths + int i = 0 + while (i < Math.min(rootPath.length, filePath.length) && + rootPath[i] == filePath[i]) { i++ } + + // backtrack from our root to our common parent directory + (i.. relativePath << filePath[j] } + + return relativePath.join('/') } + + /** + * Find the common parent directory to the given files. + */ + public static File getCommonParent(File file1, File file2) { + def path1 = file1.canonicalPath.split('/') + def path2 = file2.canonicalPath.split('/') + def newPath = [] + + // build new commonPath based on matching paths so far + int i = 0 + while (i < Math.min(path1.length, path2.length) && + path1[i] == path2[i]) { + + newPath << path2[i] + i++ } + + return new File(newPath.join('/')) } + + protected getGenerator(Class generatorClass) { + if (generators[generatorClass] == null) { + def constructor = generatorClass.getConstructor(Processor) + generators[generatorClass] = constructor.newInstance(this) + } + + return generators[generatorClass] } + + protected getParser(Class parserClass) { + if (parsers[parserClass] == null) { + parsers[parserClass] = Parboiled.createParser(parserClass) } + + return parsers[parserClass] } +} diff --git a/src/main/com/jdblabs/jlp/TargetDoc.groovy b/src/main/com/jdblabs/jlp/TargetDoc.groovy new file mode 100644 index 0000000..273c88f --- /dev/null +++ b/src/main/com/jdblabs/jlp/TargetDoc.groovy @@ -0,0 +1,10 @@ +package com.jdblabs.jlp + +import com.jdblabs.jlp.ast.SourceFile + +public class TargetDoc { + + public SourceFile sourceAST + public File sourceFile + public String output +}

${docState.currentDocId}

${processor.currentDocId}