v1.7: added --no-source option, foxpro, SQL support.

* Added `--no-source` option. By default JLP copies the original source code
  into the output directory. THis option disables that behavior.
* Added basic error handling after parsing input files: input files that do not
  parse correctly are ignored. Beforehand they were causing null pointer
  exceptions in the second parse phase of the processor.
* Made the top-level support directories hidden in the output root (ie. `/.css`
  instead of `/css`.
* Added configuration to handle Visual FoxPro files (no syntax highlighter
  available) and SQL files.
* Expanded the list of binary file types. Binary and unknown file types are not
  parsed.

project.properties

@@ -1,7 +1,7 @@
-#Tue, 10 Jan 2012 10:00:20 -0600
+#Fri, 27 Jan 2012 18:21:58 -0600
 name=jlp
-version=1.6
+version=1.7
-build.number=5
+build.number=24
 lib.local=true
 release.dir=release
 main.class=com.jdblabs.jlp.JLPMain

src/main/com/jdblabs/jlp/JLPMain.groovy

@@ -19,7 +19,7 @@ import org.slf4j.LoggerFactory
  */
 public class JLPMain {
 
-    public static final String VERSION = "1.6"
+    public static final String VERSION = "1.7"
 
     private static Logger log = LoggerFactory.getLogger(JLPMain.class)
 
@@ -60,6 +60,12 @@ public class JLPMain {
         /// :   Display JLP versioning information.
         cli._(longOpt: 'version', 'Display the JLP version information.')
 
+        /// --no-source
+        /// :   Do not copy the source files into the output directory alongside
+        ///     the documentation.
+        cli._(longOpt: 'no-source', 'Do not copy the source files into the' +
+            ' output directory alongside the documentation.')
+
         /// #### Parse the options.
         def opts = cli.parse(args)
 
@@ -121,6 +127,9 @@ public class JLPMain {
                     "${cssFile.canonicalPath}'."
                 println "      Using the default CSS." }}
 
+        /// Look for our `--no-source` option.
+        def includeSource = !opts."no-source"
+
         /// #### Create the input file list.
 
         /// We will start with the filenames passed as arguments on the command
@@ -151,7 +160,8 @@ public class JLPMain {
             else { inputFiles << file } }
 
         /// #### Process the files.
-        Processor.process(outputDir, css, inputFiles)
+        log.trace("Starting JLP processor.")
+        Processor.process(outputDir, css, inputFiles, includeSource)
     }
 
 }

src/main/com/jdblabs/jlp/JLPPegParser.java

@@ -65,6 +65,19 @@ public class JLPPegParser extends BaseParser<Object> implements JLPParser {
         MDOC_END = NOTHING;
         SDOC_START = String(sdocStart).label("SDOC_START"); }
 
+    /**
+     * #### Single-line comments only constructor.
+     * This is the same as the previous constructor except it allows the caller
+     * to define several different syntax for single-line comments. This is
+     * useful for languages that support several different syntaxes for comment
+     * lines (e.g. bash, Visual FoxPro).
+     */
+    public JLPPegParser(List sdocStarts) {
+        MDOC_START = NOTHING;
+        MDOC_LINE_START = NOTHING;
+        MDOC_END = NOTHING;
+        SDOC_START = FirstOf(sdocStarts.toArray()).label("SDOC_START"); }
+
     /**
      * #### Default constructor.
      * The default constructor creates a JLPPegParser configured to recognize

src/main/com/jdblabs/jlp/LiterateMarkdownGenerator.groovy

@@ -142,15 +142,15 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator {
         <title>${escape(processor.currentDocId)}</title>
         <meta http-equiv="content-type" content="text/html; charset=UTF-8">
         <link type="text/css" rel="stylesheet" media="all"
-            href="${resolveLink('/css/jlp.css')}"></link>
+            href="${resolveLink('/.css/jlp.css')}"></link>
 
         <!-- syntax highlighting plugin -->
         <link type="text/css" rel="stylesheet" media="all"
-            href="${resolveLink('/sh/styles/shCoreDefault.css')}"></link>
+            href="${resolveLink('/.sh/styles/shCoreDefault.css')}"></link>
         <script type="text/javascript"
-            src="${resolveLink('/sh/scripts/XRegExp.js')}"></script>
+            src="${resolveLink('/.sh/scripts/XRegExp.js')}"></script>
         <script type="text/javascript"
-            src="${resolveLink('/sh/scripts/shCore.js')}"></script>""")
+            src="${resolveLink('/.sh/scripts/shCore.js')}"></script>""")
 
         /// If there is a language-specific brush, include it
         def shBrush = processor.shBrushForSourceType(
@@ -159,7 +159,7 @@ public class LiterateMarkdownGenerator extends JLPBaseGenerator {
         if (shBrush) { sb.append("""
 
         <script type="text/javascript"
-            src="${resolveLink('/sh/scripts/' + shBrush + '.js')}"></script>""") }
+            src="${resolveLink('/.sh/scripts/' + shBrush + '.js')}"></script>""") }
 
         /// Finish our header and begin the body.
         sb.append("""

src/main/com/jdblabs/jlp/Processor.groovy

@@ -46,6 +46,10 @@ public class Processor {
     /// A shortcut for `docs[currentDocId]`
     public TargetDoc currentDoc
 
+    /// Setting to control whether the source code is copied into the final
+    /// documentation directory or not.
+    public boolean includeSource
+
     /// ### Non-public State
     /// @org jlp.jdb-labs.com/Processor/non-public-state
 
@@ -67,7 +71,7 @@ public class Processor {
      * to the directory named in `outputDir`, using the CSS given in `css`
      */
     public static void process(File outputDir, def css,
-    List<File> inputFiles) {
+    List<File> inputFiles, boolean includeSource) {
 
         /// Find the closest common parent folder to all of the files given.
         /// This will be our input root for the parsing process.
@@ -78,7 +82,8 @@ public class Processor {
         Processor inst = new Processor(
             inputRoot: inputDir,
             outputRoot: outputDir,
-            css: css)
+            css: css,
+            includeSource: includeSource)
 
         /// Run the process.
         inst.process(inputFiles) }
@@ -96,17 +101,24 @@ public class Processor {
         /// constructor.
 
         /// * Write the CSS file to our output directory.
-        File cssFile = new File(outputRoot, "css/jlp.css")
+        File cssFile = new File(outputRoot, ".css/jlp.css")
         cssFile.parentFile.mkdirs()
         cssFile.text = css.text
 
         /// * Extract the syntax highlighter files to the output directory.
         File shFile = new File(outputRoot, "temp.jar")
+        File shDir = new File(outputRoot, "sh")
+        File metaDir = new File(outputRoot, "META-INF")
         getClass().getResourceAsStream("/syntax-highlighter.jar").withStream { is ->
             shFile.withOutputStream { os ->
                 while (is.available()) { os.write(is.read()) }}}
         JarUtils.extract(shFile, outputRoot)
+        shDir.renameTo(new File(outputRoot, '.sh'))
+
+        /// * Delete our temporary jar file and the META-INF directory extracted
+        /// from it.
         shFile.delete()
+        metaDir.deleteDir()
 
         /// * Create the processing context for each input file. We are using
         ///   the name of the file (including the extension) as the id. If there
@@ -118,6 +130,12 @@ public class Processor {
             def relPath = getRelativeFilepath(inputRoot, file)
             def pathParts = relPath.split('/') as List
 
+            // Get our file type.
+            def fileType = sourceTypeForFile(file)
+
+            // We will skip binary files and files we know nothing about.
+            if (fileType == 'binary' || fileType == 'unknown') { return; }
+
             // Start with just the file name.
             def docId = pathParts.pop()
 
@@ -137,11 +155,20 @@ public class Processor {
         /// * Run the parse phase on each of the files. For each file, we load
         ///   the parser for that file type and parse the file into an abstract
         ///   syntax tree (AST).
+        def badDocs = []
         processDocs {
             log.trace("Parsing '{}'.", currentDocId)
             def parser = getParser(currentDoc.sourceType)
-            // TODO: error detection
+
-            currentDoc.sourceAST = parser.parse(currentDoc.sourceFile.text) }
+            // TODO: better error detection and handling
+            currentDoc.sourceAST = parser.parse(currentDoc.sourceFile.text)
+            
+            if (currentDoc.sourceAST == null) {
+                log.warn("Unable to parse '{}'. Ignoring this document.", currentDocId)
+                badDocs << currentDocId }}
+
+        /// * Remove all the documents we could not parse from our doc list.
+        docs = docs.findAll { docId, doc -> !badDocs.contains(docId) }
 
         /// * Run our generator parse phase (see
         ///   [`JLPBaseGenerator`](jlp://com.jdb-labs.jlp.JLPBaseGenerator/phases)
@@ -176,9 +203,9 @@ public class Processor {
             if (!outputDir.exists()) { outputDir.mkdirs() }
 
             /// Copy the source file over.
-            // TODO: make this behavior customizable.
+            if (includeSource) {
-            (new File(outputRoot, relativePath)).withWriter {
+                (new File(outputRoot, relativePath)).withWriter {
-                it.print currentDoc.sourceFile.text }
+                    it.print currentDoc.sourceFile.text }}
 
             /// Write the output to the file.
             outputFile.withWriter { it.println currentDoc.output } } }
@@ -338,15 +365,22 @@ public class Processor {
 
         /// Lookup the file type by extension
         switch (extension) {
-            case 'c': case 'h': return 'c';
+            case 'c': case 'h': return 'c'
-            case 'c++': case 'h++': case 'cpp': case 'hpp': return 'cpp';
+            case 'c++': case 'h++': case 'cpp': case 'hpp': return 'cpp'
-            case 'erl': case 'hrl': return 'erlang';
+            case 'erl': case 'hrl': return 'erlang'
-            case 'groovy': return 'groovy';
+            case 'groovy': return 'groovy'
-            case 'java': return 'java';
+            case 'java': return 'java'
-            case 'js': return 'javascript';
+            case 'js': return 'javascript'
-            case 'md': return 'markdown';
+            case 'md': return 'markdown'
-            case 'html': return 'html';
+            case 'html': return 'html'
-            case 'xml': case 'xhtml': return 'xml';
+            case 'xml': case 'xhtml': return 'xml'
+            case 'prg': return 'foxpro'
+            case 'sql': return 'sql'
+
+            // binary file types
+            case 'bin': case 'com': case 'exe': case 'o':
+            case 'bz2': case 'tar': case 'tgz': case 'zip': case 'jar':
+                return 'binary'
             default: return 'unknown'; }}
 
     /**
@@ -362,6 +396,7 @@ public class Processor {
             case 'java': return 'shBrushJava'
             case 'javascript': return 'shBrushJScript'
             case 'html': case 'xml': return 'shBrushXml'
+            case 'sql': return 'shBrushSql'
             default: return null }}
 
     /**
@@ -396,6 +431,10 @@ public class Processor {
                     parsers[sourceType] = Parboiled.createParser(
                         JLPPegParser, '%%')
                     break
+                case 'foxpro':
+                    parsers[sourceType] = Parboiled.createParser(
+                        JLPPegParser, ['**', '&&&'])
+                    break
                 case 'markdown':
                     parsers[sourceType] = new MarkdownParser()
                     break
@@ -404,6 +443,10 @@ public class Processor {
                         JLPPegParser, '<!--', '-->',
                         '#$%^&*()_-+=|;:\'",<>?~`', '<<?')
                     break
+                case 'sql':
+                    parsers[sourceType] = Parboiled.createParser(JLPPegParser,
+                        '/**', '*/', '!#$%^&*()_-=+|;:\'",<>?~`', '---')
+                    break
                 case 'c':
                 case 'cpp':
                 case 'groovy':