Updates for Nim 2.0

.gitignore

@@ -2,3 +2,4 @@
 nimcache/
 build/
 .gradle/
+/treediff

build.gradle

@@ -3,17 +3,18 @@ apply plugin: "application"
 apply plugin: "maven"
 
 group = "com.jdblabs"
-version = "1.3"
+version = "1.4.4"
 mainClassName = "com.jdblabs.file.treediff.TreeDiff"
 
 repositories {
     mavenLocal()
-    mavenCentral() }
+    mavenCentral()
+    maven { url "http://mvn.jdb-labs.com/repo" } }
 
 dependencies {
-    compile 'org.codehaus.groovy:groovy-all:2.4.3'
-	compile 'com.jdbernard:jdb-util:3.8'
-	compile 'commons-codec:commons-codec:1.10'
+    compile localGroovy()
+    compile 'com.jdbernard:jdb-util:4.+'
+    compile 'commons-codec:commons-codec:1.10'
     compile 'com.fasterxml.jackson.core:jackson-databind:2.4.4'
 
     testCompile 'junit:junit:4.12'

src/main/groovy/com/jdblabs/file/treediff/TreeDiff.groovy

@@ -8,7 +8,7 @@ import org.apache.commons.codec.digest.DigestUtils
 
 public class TreeDiff {
 
-    public static final String VERSION = "1.3"
+    public static final String VERSION = "1.4.3"
 
     private ObjectMapper objectMapper = new ObjectMapper()
     private PrintStream stdout
@@ -143,7 +143,7 @@ public class TreeDiff {
 
             if (rootName.startsWith('/')) rightOut = new File(rootName + '.right')
             else rightOut = new File(relativeRoot, rootName + '.right')
-            
+
             objectMapper.writeValue(leftOut, left)
             objectMapper.writeValue(rightOut, right) }
      }
@@ -171,7 +171,7 @@ public class TreeDiff {
         frame(title: "TreeDif v${VERSION}", show: true) {
             boxLayout()
 
-            
+
         }
     }
 
@@ -182,7 +182,7 @@ public class TreeDiff {
     }
 
     public static Set<String> samePathDifferentContents(DirAnalysis left, DirAnalysis right) {
-        return left.allEntries.findAll { l -> 
+        return left.allEntries.findAll { l ->
             FileEntry match = right.byRelativePath[l.relativePath]
             return match != null && l.checksum != match.checksum }
             .collect { it.relativePath } }
@@ -300,15 +300,15 @@ where options are:
     -g, --gui       Launch the graphical interface (not yet implemented).
 
     -s, --same
-    
+
         Output information about files that are the same on both sides.
 
     -S, --exclude-same
-    
+
         Do not output information about files that are the same on both sides.
 
     -c, --content-mismatch
-    
+
         Output information about files that have the same relative path on both
         side but whose contents differ.
 
@@ -363,7 +363,7 @@ where options are:
         directories to scan for example).
 
     -i, --analysis-in   <left-dir-analysis> <right-dir-analysis>
-    
+
         Use pre-calculated directory analysis in place of reading local
         directories. This is useful if you wish to do diffs between two
         directory trees that are not on the same filesystem, or if you wish to

src/main/nim/cliconstants.nim

@@ -0,0 +1,63 @@
+const VERSION* = "2.0.1"
+
+const USAGE* = """
+Usage:
+  treediff <left> [<right>] [options]
+  treediff (-h | --help)
+  treediff (-V | --version)
+
+  <left> and <right> represent paths to directory roots to be compared. If one
+  of these paths points to a file instead of a directory, treediff assumes that
+  the file represents a saved directory analysis to be loaded in place of a
+  directory to compare. For example:
+
+      treediff /path/to/dir /path/to/output.json
+
+  will analyze the directory tree at '/path/to/dir' to create the left-side
+  analysis and load a pre-existing analysis from '/path/to/output.json' as the
+  right-side analysis.
+
+Options:
+  -h --help       Show this usage information.
+  -V --version    Show the program version.
+  -v --verbose    Enable verbose output.
+  -q --quiet      Suppress all output and error messages except for the
+                  progress indicator.
+  -Q --very-quiet Suppress all output and error messages includeing the
+                  progress indicator.
+
+  -1 --save-left <left_out>   Save the left analysis to <left_out> (will be
+                              formatted as JSON)
+  -2 --save-right <right_out> Save the right analysis to <right_out> (will be
+                              formatted as JSON)
+
+  -s --same
+  -S --exclude-same
+
+      Show or hide information about files which are the same in both trees.
+
+  -c --content-mismatch
+  -C --exclude-content-mismatch
+
+      Show or hide information about files whose relative paths are the same
+      in both trees but whose contents differ.
+
+  -p --path-mismatch
+  -P --exclude-path-mismatch
+
+      Show or hide information about files whose contents are the same in both
+      trees but whose relative paths differ.
+
+  -l --left-only
+  -L --exclude-left-only
+
+      Show or hide information about files which are found only in the left
+      tree.
+
+  -r --right-only
+  -R --exclude-right-only
+
+      Show or hide information about files which are found only in the right
+      tree.
+
+"""

src/main/nim/incremental_md5.nim

@@ -0,0 +1,40 @@
+import std/streams
+import checksums/md5
+
+proc fileToMD5*(filename: string) : string =
+
+  const blockSize: int = 8192 # read files in 8KB chunnks
+  var
+    c: MD5Context
+    d: MD5Digest
+    fs: FileStream
+    buffer: string
+
+  #read chunk of file, calling update until all bytes have been read
+  try:
+    fs = filename.open.newFileStream
+
+    md5Init(c)
+    buffer = fs.readStr(blockSize)
+
+    while buffer.len > 0:
+      md5Update(c, buffer.cstring, buffer.len)
+      buffer = fs.readStr(blockSize)
+
+    md5Final(c, d)
+
+  except IOError: echo("File not found.")
+  finally:
+    if fs != nil:
+      close(fs)
+
+  result = $d
+
+when isMainModule:
+
+  if paramCount() > 0:
+    let arguments = commandLineParams()
+    echo("MD5: ", fileToMD5(arguments[0]))
+  else:
+    echo("Must pass filename.")
+    quit(-1)

src/main/nim/treediff.nim

@@ -0,0 +1,284 @@
+## Tree Diff
+## =========
+##
+## Utility to compare the file contents of two directory trees.
+
+import std/[json, jsonutils, os, tables, sequtils, strutils]
+import docopt
+import incremental_md5, console_progress
+
+import ./cliconstants
+
+
+type
+  ProgressWrapper* = tuple[impl: Progress, verbosity: Verbosity]
+    ## Wrapper around a console_progress.Progress.
+
+  Verbosity* = enum ## Enum representing the level of output verbosity the tool will emit.
+    very_quiet, ## suppress all output including the progress indicator
+    quiet,      ## suppress all output except the progress indicator
+    normal      ## emit all output
+
+proc newProgressWrapper*(outFile = stdout, verbosity = normal): ProgressWrapper =
+  ## Create a new ProgressWrapper for the given verbosity.
+  if verbosity > very_quiet:
+    result = (impl: newProgress(0, outFile), verbosity: verbosity)
+  else: result = (impl: nil, verbosity: verbosity)
+
+proc init(p: ProgressWrapper, root: string, fileCount: int): void =
+  if p.verbosity == normal:
+      echo "-- ", root.expandFilename, "\L   ", fileCount, " files"
+  if p.verbosity > very_quiet: p.impl.setMax(fileCount)
+
+proc update(p: ProgressWrapper, count: int, file: string): void =
+  if p.verbosity > very_quiet:
+    p.impl.updateProgress(count, file[max(file.high - 15, 0)..file.high])
+
+proc finish(p: ProgressWrapper): void =
+  if p.verbosity > very_quiet:
+    p.impl.erase
+  if p.verbosity == normal: echo "  ", p.impl.getMax, " files.\L"
+
+
+
+proc countFiles(root: string): int =
+  for file in walkDirRec(root): result += 1
+
+proc getRelPath(ancestor, child: string): string =
+  ## Given a ancestor path and a child path, assuming the child path is
+  ## contained within the ancestor path, return the relative path from the
+  ## ancestor to the child.
+
+  let ancestorPath = ancestor.expandFilename.split({DirSep, AltSep})
+  let childPath = child.expandFilename.split({DirSep, AltSep})
+
+  # If the ancestor path is longer it cannot contain the child path and we
+  # cannot construct a relative path without backtracking.
+  if (ancestorPath.len > childPath.len): return ""
+
+  # Compare the ancestor and child path up until the end of the ancestor path.
+  var idx = 0
+  while idx < ancestorPath.len and ancestorPath[idx] == childPath[idx]: idx += 1
+
+  # If we stopped before reaching the end of the ancestor path it must be that
+  # the paths do not match. The ancestor cannot contain the child and we cannot
+  # build a relative path without backtracking.
+  if idx != ancestorPath.len: return ""
+  return foldl(@["."] & childPath[idx..childPath.high], joinPath(a, b))
+
+
+
+
+type
+  FileEntry* = ref tuple[relPath: string, checksum: string]
+    ## Data about one file that has been analyzed
+
+  DirAnalysis* = ## Analysis data about one directory tree.
+    tuple[allEntries: seq[FileEntry],
+          byRelPath: TableRef[string, FileEntry],
+          byChecksum: TableRef[string, seq[FileEntry]]]
+
+
+  DisplayOptions = tuple[left, right, same, content, path: bool]
+    ## Consolidated description of which types of results to display.
+
+func `$`(f: FileEntry): string = f.checksum & ": " & f.relPath
+
+proc getOrFail(n: JsonNode, key: string, objName: string = ""): JsonNode =
+  ## convenience method to get a key from a JObject or raise an exception
+  if not n.hasKey(key): raise newException(Exception, objName & " missing key '" & key & "'")
+  return n[key]
+
+proc getIfExists(n: JsonNode, key: string): JsonNode =
+  ## convenience method to get a key from a JObject or return null
+  result = if n.hasKey(key): n[key]
+           else: newJNull()
+
+func parseFileEntry(n: JsonNode): FileEntry =
+  result = new(FileEntry)
+  result.relPath = n.getOrFail("relPath").getStr
+  result.checksum = n.getOrFail("checksum").getStr
+
+func initDirAnalysis(): DirAnalysis =
+  (allEntries: @[],
+   byRelPath: newTable[string, FileEntry](),
+   byChecksum: newTable[string, seq[FileEntry]]())
+
+func indexEntries(da: var DirAnalysis) =
+  for e in da.allEntries:
+    da.byRelPath[e.relPath] = e
+    if not da.byChecksum.hasKey(e.checksum):
+      da.byChecksum[e.checksum] = newSeq[FileEntry]()
+    da.byChecksum[e.checksum].add(e)
+
+proc analyzeDir*(root: string, progress: ProgressWrapper): DirAnalysis =
+  ## Inspect a directory and analyze all files, noting their relative paths and
+  ## checksum of their contents.
+  let fileCount = countFiles(root)
+
+  progress.init(root, fileCount + 10)
+
+  result = initDirAnalysis()
+
+  var count = 0
+  for file in walkDirRec(root):
+    let md5sum = fileToMd5(file)
+
+    var fileEntry: FileEntry = new(FileEntry)
+    fileEntry[] = (relPath: getRelPath(root, file), checksum: md5sum)
+    result.allEntries.add(fileEntry)
+
+    progress.update(count, file)
+    count += 1
+
+  result.indexEntries
+  count += 10
+  progress.finish()
+
+proc loadAnalysis*(path: string): DirAnalysis =
+  ## Load a previously performed directory analysis.
+  let allEntriesJson = parseJson(readFile(path))
+  result = initDirAnalysis()
+  result.allEntries = toSeq(items(allEntriesJson)).map(parseFileEntry)
+  result.indexEntries
+
+proc saveAnalysis*(path: string, analysis: DirAnalysis): void =
+  ## Save a completed analysis.
+  writeFile(path, $(analysis.allEntries.toJson))
+
+proc intersection*(left, right: DirAnalysis): seq[FileEntry] =
+  ## Find all ``FileEntry`` that are the same on both sides: matching contents
+  ## and paths.
+  return left.allEntries.filter do (item: FileEntry) -> bool:
+    if not right.byRelPath.hasKey(item.relPath): return false
+    let match = right.byRelPath[item.relPath]
+    if match == nil: return false
+    return item.checksum == match.checksum
+
+proc difference*(left, right: DirAnalysis): seq[FileEntry] =
+  ## Find all ``FileEntry`` that are present in the left but not present in
+  ## the right.
+  return left.allEntries.filter do (item: FileEntry) -> bool:
+    return not right.byRelPath.hasKey(item.relPath) and
+           not right.byChecksum.hasKey(item.checksum)
+
+proc `*`*(left, right: DirAnalysis): seq[FileEntry] {.inline.} =
+  ## Alias for `intersection(left, right) <#intersection>`_
+  return intersection(left, right)
+
+proc `-`*(left, right: DirAnalysis): seq[FileEntry] {.inline.} =
+  ## Alias for `difference(left, right) <#difference>`_
+  return difference(left, right)
+
+proc samePathDifferentContents*(left, right: DirAnalysis): seq[string] =
+  ## Find all ``FileEntry`` that have the same paths in both trees but whose
+  ## contents differ.
+  let matchingEntries = left.allEntries.filter do (item: FileEntry) -> bool:
+    if not right.byRelPath.hasKey(item.relPath): return false
+    let match = right.byRelPath[item.relPath]
+    return item.checksum != match.checksum
+  return matchingEntries.map(proc(item: FileEntry): string = return item.relPath)
+
+proc sameContentsDifferentPaths*(left, right: DirAnalysis): seq[tuple[left, right: FileEntry]] =
+  ## Find all ``FileEntry`` whose contents are the same in both trees but
+  ## which are located at differenc paths.
+  result = @[]
+  for item in left.allEntries:
+    if not right.byChecksum.hasKey(item.checksum): continue
+    for match in right.byChecksum[item.checksum]:
+      if item.relPath != match.relPath: result.add((left: item, right:match))
+
+when isMainModule:
+
+  let quitWithError = proc (error: string): void =
+    stderr.writeLine("treediff: " & error)
+    quit(QuitFailure)
+
+  let args = docopt(USAGE, version = "treediff " & VERSION)
+
+  var verbosity = normal
+  if args["--quiet"]: verbosity = quiet
+  if args["--very-quiet"]: verbosity = very_quiet
+  let progressWrapper = newProgressWrapper(verbosity = verbosity)
+
+  # Load or perform analysis
+  if not args["<left>"]:
+    quitWithError("Missing <left> parameter.")
+
+  let leftPath: string = $args["<left>"]
+
+  let loadPath = proc (path: string): DirAnalysis =
+    if not path.fileExists and not path.dirExists:
+      quitWithError($path & ": no such file or directory.")
+
+    let fileInfo = path.getFileInfo
+    if fileInfo.kind == pcDir:
+      return analyzeDir(path, progressWrapper)
+    elif fileInfo.kind == pcFile:
+      result = loadAnalysis(path)
+    else:
+      quitWithError($path & ": is not a file or directory")
+
+  var leftAnalysis, rightAnalysis: DirAnalysis
+
+  leftAnalysis = loadPath(leftPath)
+
+  if not args["<right>"]:
+    rightAnalysis = (allEntries: @[],
+                 byRelPath: newTable[string, FileEntry](),
+                 byChecksum: newTable[string, seq[FileEntry]]())
+  else:
+    var rightPath: string = $args["<right>"]
+    rightAnalysis = loadPath(rightPath)
+
+  # Check for output options
+  if args["--save-left"]:
+    saveAnalysis($args["--save-left"], leftAnalysis)
+
+  if args["--save-right"] and rightAnalysis.allEntries.len > 0:
+    saveAnalysis($args["--save-right"], rightAnalysis)
+
+  # Parse filter options
+  var displayOptions: DisplayOptions = (
+    left: false, right: false, same: false, content: false, path: false)
+
+  # If none of the explicit selectors are given, assume all are expected.
+  if not (args["--left-only"] or args["--right-only"] or
+          args["--same"] or args["--content-mismatch"] or
+          args["--path-mismatch"] ):
+    displayOptions = (left: true, right: true, same: true,
+                      content: true, path: true)
+
+  if args["--same"]:                      displayOptions.same = true
+  if args["--exclude-same"]:              displayOptions.same = false
+  if args["--content-mismatch"]:          displayOptions.content = true
+  if args["--exclude-content-mismatch"]:  displayOptions.content = false
+  if args["--path-mismatch"]:             displayOptions.path = true
+  if args["--exclude-path-mismatch"]:     displayOptions.path = false
+  if args["--left-only"]:                 displayOptions.left = true
+  if args["--exclude-left-only"]:         displayOptions.left = false
+  if args["--right-only"]:                displayOptions.right = true
+  if args["--exclude-right-only"]:        displayOptions.right = false
+
+  # Display output results
+  if verbosity == normal:
+    if displayOptions.same:
+      let sameEntries = leftAnalysis * rightAnalysis
+      for fe in sameEntries: echo "same: ", fe.relPath
+
+    if displayOptions.content:
+      let contentsDiffer = samePathDifferentContents(leftAnalysis, rightAnalysis)
+      for path in contentsDiffer: echo "contents differ: ", path
+
+    if displayOptions.path:
+      let pathsDiffer = sameContentsDifferentPaths(leftAnalysis, rightAnalysis)
+      for pair in pathsDiffer:
+        echo "paths differ: ", pair.left.relPath, " ", pair.right.relPath
+
+    if displayOptions.left:
+      let leftOnly = leftAnalysis - rightAnalysis
+      for fe in leftOnly: echo "left only: ", fe.relPath
+
+    if displayOptions.right:
+      let rightOnly = rightAnalysis - leftAnalysis
+      for fe in rightOnly: echo "right only: ", fe.relPath

treediff.nim

@@ -1,39 +0,0 @@
-import os, docopt, tables, md5, iterutils, re
-
-proc studyDir(root: string, ignore: Iterable[string]): TableRef[string, string] =
-    result = newTable[string, string]()
-
-    for path in walkDirRec(root):
-        var relPath = substr(path, len(root))
-
-        if foldl(ignore, proc (acc: bool, it: string): bool = acc and match(relPath, re(it)), true): continue
-
-        var fileInfo = getFileInfo(path)
-
-        if fileInfo.kind == pcFile:
-            result.add(relPath, $(toMD5(readFile(path))))
-        elif fileInfo.kind == pcDir:
-            result.add(relPath, "directory")
-
-when isMainModule:
-
-    let doc = """
-treediff
-
-Usage:
-  treediff [-i <regex>]... [<path>]...
-  treediff (-h | --help)
-  treediff (-v | --version)
-
-Options:
-  -h --help     Show this usage information.
-  -v --version  Show the program version.
-"""
-
-    let args = docopt(doc, version = "treediff 0.1")
-
-    for root in @(args["<path>"]):
-        echo "Looking at ", root
-
-        echo studyDir(root, @(args["<regex>"]))
-        echo ""

treediff.nimble

@@ -1,11 +1,16 @@
-[Package]
-name          = "treeediff"
-version       = "0.1.0"
+# Package
+version       = "2.0.1"
 author        = "Jonathan Bernard (jdb@jdb-labs.com)"
-description   = "Tree Diff"
+description   = "Utility to generate diffs of full directory trees."
 license       = "BSD"
+bin           = @["treediff"]
+srcDir        = "src/main/nim"
 
-bin           = "treediff"
+# Dependencies
+requires: @["nim >= 2.0.0", "docopt == 0.7.1", "checksums"]
 
-[Deps]
-Requires: "nim >= 0.10.0, docopt >= 0.1.0, iterutils >= 0.1.0"
+# Dependencies from git.jdb-software.com/jdb/nim-packages
+requires: @["console_progress >= 1.2.2", "update_nim_package_version"]
+
+task updateVersion, "Update the version of this package.":
+  exec "update_nim_package_version treediff 'src/main/nim/cliconstants.nim'"

worklog.md

@@ -0,0 +1,6 @@
+To Do
+========================================
+
+* Rework the JSON output format so that
+  the Groovy and Nim implementations can
+  read each other's saved analysis.