Add truecolor and unicode support to terminal utilities.

Nim's `terminal` library provides ANSI terminal escape support, but
only when writing directly to a File handle. This commit adds the
`term{FG,BG}` procedures and `termReset` constant to help with embedding
ANSI escape codes in string data.

.gitignore

@@ -1,3 +1,2 @@
 nimcache/
-cliutils
 *.sw?

cliutils.nim

@@ -1,164 +1,32 @@
-import docopt, json, osproc, posix, nre, streams, strtabs, unicode
+import std/[strtabs, strutils, tables]
-import os except sleep
+
-import strutils except toUpper, toLower
+import ./cliutils/ansiterm
+import ./cliutils/config
+import ./cliutils/daemonize
+import ./cliutils/procutil
+import ./cliutils/queue_logger
+
+export ansiterm
+export config
+export daemonize
+export procutil
+export queue_logger
+
+proc doParseQueryParams[T](queryParams: T): seq[string] =
+  result = @[]
+
+  for k,v in queryParams:
+    # support ?arg1=val1&arg2=val2 -> cmd val1 val2
+    if k.startsWith("arg"): result.add(v)
+
+    else :
+      result.add("--" & k)
+      if v != "true": result.add(v) # support things like ?verbose=true -> cmd --verbose
 
 
-type
+proc queryParamsToCliArgs*(queryParams: Table[string, string]): seq[string] =
-  CombinedConfig* = object
+  doParseQueryParams(queryParams)
-    docopt*: Table[string, Value]
-    json*: JsonNode
-
-proc getVal*(cfg: CombinedConfig, key, default: string): string =
-  let argKey = "--" & key
-  let envKey = key.replace('-', '_').toUpper
-  let jsonKey = key.replace(re"(-\w)", proc (m: RegexMatch): string = ($m)[1..1].toUpper)
-
-  if cfg.docopt[argKey]: return $cfg.docopt[argKey]
-  elif existsEnv(envKey): return getEnv(envKey)
-  elif cfg.json.hasKey(jsonKey): return cfg.json[jsonKey].getStr
-  else: return default
 
 
-proc loadEnv*(): StringTableRef =
+proc queryParamsToCliArgs*(queryParams: StringTableRef): seq[string] =
-  result = newStringTable()
+  doParseQueryParams(queryParams)
-
-  for k, v in envPairs():
-    result[k] = v
-
-## Process execution
-type HandleProcMsgCB* = proc (outMsg: TaintedString,
-                              errMsg: TaintedString, cmd: string): void
-
-proc sendMsg*(h: HandleProcMsgCB, outMsg: TaintedString,
-              errMsg: TaintedString = nil, cmd: string = ""): void =
-  if h != nil: h(outMsg, errMsg, cmd)
-
-proc makeProcMsgHandler*(outSink, errSink: File, prefixCmd: bool = true): HandleProcMsgCB =
-  result = proc(outMsg, errMsg: TaintedString, cmd: string): void {.closure.} =
-    let prefix = if cmd == nil or not prefixCmd: "" else: cmd & ": "
-    if outMsg != nil: outSink.writeLine(prefix & outMsg)
-    if errMsg != nil: errSink.writeLine(prefix & errMsg)
-
-proc makeProcMsgHandler*(outSink, errSink: Stream, prefixCmd: bool = true): HandleProcMsgCB =
-  result = proc(outMsg, errMsg: TaintedString, cmd: string): void {.closure.} =
-    let prefix = if cmd == nil or not prefixCmd: "" else: cmd & ": "
-    if outMsg != nil: outSink.writeLine(prefix & outMsg)
-    if errMsg != nil: errSink.writeLine(prefix & errMsg)
-
-proc combineProcMsgHandlers*(a, b: HandleProcMsgCB): HandleProcMsgCB =
-  if a == nil: result = b
-  elif b == nil: result = a
-  else:
-    result = proc(cmd: string, outMsg, errMsg: TaintedString): void =
-      a(cmd, outMsg, errMsg)
-      b(cmd, outMsg, errMsg)
-
-proc waitFor*(p: Process, msgCB: HandleProcMsgCB, procCmd: string = ""): int =
-
-  var pout = outputStream(p)
-  var perr = errorStream(p)
-
-  var line = newStringOfCap(120).TaintedString
-  while true:
-    if pout.readLine(line):
-      msgCB.sendMsg(line, nil, procCmd)
-    elif perr.readLine(line):
-      msgCB.sendMsg(nil, line, procCmd)
-    else:
-      result = peekExitCode(p)
-      if result != -1: break
-  close(p)
-
-proc exec*(command: string, workingDir: string = "",
-           args: openArray[string] = [], env: StringTableRef = nil,
-           options: set[ProcessOption] = {poUsePath},
-           msgCB: HandleProcMsgCB = nil): int
-    {.tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
-
-  var p = startProcess(command, workingDir, args, env, options)
-  result = waitFor(p, msgCB, command)
-
-proc execWithOutput*(command: string, workingDir:string = "",
-           args: openArray[string] = [], env: StringTableRef = nil,
-           options: set[ProcessOption] = {poUsePath},
-           msgCB: HandleProcMsgCB = nil):
-    tuple[output: TaintedString, error: TaintedString, exitCode: int] =
-
-  result = (TaintedString"", TaintedString"", -1)
-  var outSeq, errSeq: seq[TaintedString]
-  outSeq = @[]; errSeq = @[]
-  var outputCollector = combineProcMsgHandlers(msgCB,
-    proc(outMsg, errMsg: TaintedString, cmd: string): void {.closure.} =
-      if outMsg != nil: outSeq.add(outMsg)
-      if errMsg != nil: errSeq.add(errMsg))
-
-  result[2] = exec(command, workingDir, args, env, options, outputCollector)
-  result[0] = outSeq.join("\n")
-  result[1] = errSeq.join("\n")
-
-## Daemonize
-
-var
-  pidFileInner: string
-  fi, fo, fe: File
-
-proc onStop(sig: cint) {.noconv.} =
-  close(fi)
-  close(fo)
-  close(fe)
-  removeFile(pidFileInner)
-
-  quit(QuitSuccess)
-
-proc daemonize*(pidfile, si, so, se: string, daemonMain: proc(): void): Pid =
-
-  if fileExists(pidfile):
-    raise newException(IOError, "pidfile " & pidfile & " already exists, daemon already running?")
-
-  let pid1 = fork()
-
-  # Are we the child process?
-  if pid1 == 0:
-
-    # Yes, so let's get ready to execute the main code given to us.
-    discard chdir("/")
-    discard setsid()
-    discard umask(0)
-
-    # Fork again... but I'm not sure why.
-    let pid2 = fork()
-
-    # We don't need the intermediate process, so if we are not the child this
-    # time, lets just quit
-    if pid2 > 0: quit(QuitSuccess)
-
-    # If we are the grandchild let's set up our environment.
-    flushFile(stdout)
-    flushFile(stderr)
-
-    if not si.isNil and si != "":
-      fi = open(si, fmRead)
-      discard dup2(getFileHandle(fi), getFileHandle(stdin))
-
-    if not so.isNil and so != "":
-      fo = open(so, fmAppend)
-      discard dup2(getFileHandle(fo), getFileHandle(stdout))
-
-    if not se.isNil and so != "":
-      fe = open(se, fmAppend)
-      discard dup2(getFileHandle(fe), getFileHandle(stderr))
-
-    pidFileInner = pidfile
-
-    # Add hooks to cleanup after ourselves when we're asked to die.
-    signal(SIGINT, onStop)
-    signal(SIGTERM, onStop)
-
-    # Find out what our actual PID is and save it
-    let childPid = getpid()
-    writeFile(pidfile, $childPid)
-
-    # Finally, execute our main
-    daemonMain()
-
-  return pid1

cliutils.nimble

@@ -1,11 +1,11 @@
 # Package
 
-version       = "0.3.4"
+version       = "0.11.0"
 author        = "Jonathan Bernard"
 description   = "Helper functions for writing command line interfaces."
 license       = "MIT"
 
+
 # Dependencies
 
-requires @["nim >= 0.18.0", "docopt"]
+requires @["nim >= 1.6.0", "docopt >= 0.6.8"]
-

cliutils/ansiterm.nim

@@ -0,0 +1,172 @@
+import std/[nre, sequtils, strutils, unicode]
+
+# See
+# - https://gist.github.com/ConnerWill/d4b6c776b509add763e17f9f113fd25b
+# - https://en.wikipedia.org/wiki/ANSI_escape_code
+
+const CSI = "\x1b["
+const RESET_FORMATTING* = "\x1b[0m"
+const ANSI_ESCAPE_CODE_ENDINGS*: seq[Rune] =
+  toRunes(toSeq('A'..'Z') & toSeq('a'..'z'))
+
+let FORMATTING_REGEX* = re("\x1b\\[([0-9;]*)([a-zA-Z])")
+
+type
+  TrueColor* = tuple[r, g, b: int]
+
+  CursorType* = enum
+    ctBlockBlink = 1, ctBlock, ctUnderlineBlink, ctUnderline, ctBarBlink, ctBar
+
+  EraseMode* = enum
+    emToEnd, emToStart, emAll
+
+  TerminalColors* = enum
+    cBlack = 0, cRed, cGreen, cYellow, cBlue, cMagenta, cCyan, cWhite,
+    cDefault = 9
+
+    cBrightBlack = 60, cBrightRed, cBrightGreen, cBrightYellow, cBrightBlue,
+    cBrightMagenta, cBrightCyan, cBrightWhite
+
+  TerminalStyle* = enum
+    tsBold = 1, tsDim, tsItalic, tsUnderline, tsBlink, tsInverse, tsHidden,
+    tsStrikethrough
+
+proc stripFormatting*(text: string): string =
+  text.replace(FORMATTING_REGEX, "")
+
+proc stripAnsi*(text: string): string = stripFormatting(text)
+
+func ansiAwareSubstring*(s: string, start, length: int): string =
+  var outputRunes = newSeq[Rune]()
+  var curAnsiSequences = newSeq[Rune]()
+  var runes = s.toRunes
+  var visibleRunesSeen = 0
+  var visibleOutputLen = 0
+  var i = 0
+
+  while i < runes.len and visibleOutputLen < length:
+
+    if runes.len > i + runeLen(CSI) and
+       $runes[i..<i + len(CSI)] == CSI:
+      # Recognize ANSI escape codes...
+
+      var j = i + runeLen(CSI)
+      while j < runes.len and runes[j] notin ANSI_ESCAPE_CODE_ENDINGS: j += 1
+
+      # and remember it if we're before the start of the substring
+      if visibleRunesSeen < start: curAnsiSequences.add(runes[i..j])
+
+      # or add it to the output without increasing our substring length
+      else: outputRunes.add(runes[i..j])
+
+      # either way we want to pick up after it
+      i = j
+
+    else:
+      if visibleRunesSeen >= start:
+        outputRunes.add(runes[i])
+        inc visibleOutputLen
+      inc visibleRunesSeen
+
+    inc i
+
+    result = $(curAnsiSequences & outputRunes)
+
+
+func ansiAwareSubstring*[T, U](s: string, r: HSlice[T,U]): string =
+  let idxStart =
+    when r.a is BackwardsIndex: s.runeLen - int(r.a)
+    else: int(r.a)
+
+  let idxEnd =
+    when r.b is BackwardsIndex: s.runeLen - int(r.b)
+    else: int(r.b)
+
+  ansiAwareSubstring(s, idxStart, idxEnd - idxStart + 1)
+
+
+func ansiEscSeq*(style: set[TerminalStyle]): string =
+  CSI & toSeq(style).mapIt($int(it)).join(";") & "m"
+
+func ansiEscSeq*(fg: int): string = CSI & "38;5;" & $fg & "m"
+func ansiEscSeq*(bg: int): string = CSI & "48;5;" & $bg & "m"
+
+
+func ansiEscSeq*(fg: TrueColor): string =
+  "$#38;2;$#;$#;$#m" % [CSI, $fg.r, $fg.g, $fg.b]
+
+func ansiEscSeq*(bg: TrueColor): string =
+  "$#48;2;$#;$#;$#m" % [CSI, $bg.r, $bg.g, $bg.b]
+
+
+func ansiEscSeq*(fg: TerminalColors): string = CSI & $(int(fg) + 30) & "m"
+func ansiEscSeq*(bg: TerminalColors): string = CSI & $(int(bg) + 40) & "m"
+
+
+func ansiEscSeq*(fg: TerminalColors, bg: TerminalColors, style: set[TerminalStyle]): string =
+  result = CSI
+  if style != {}: result &= toSeq(style).mapIt($int(it)).join(";") & ";"
+  result &= $(int(fg) + 30) & ";" & $(int(bg) + 40) & "m"
+
+
+func termFmt*(text: string, fg = cWhite, bg = cBlack, style: set[TerminalStyle] = {}): string =
+  return ansiEscSeq(fg, bg, style) & text & RESET_FORMATTING
+
+
+func termFmt*[C: int | TrueColor](
+    text: string,
+    fg, bg: C,
+    style: set[TerminalStyle] = {}): string =
+  return ansiEscSeq(style) & ansiEscSeq(fg = fg) & ansiEscSeq(bg = bg) &
+    text & RESET_FORMATTING
+
+
+func color*(text: string, fg = cDefault, bg = cDefault): string =
+  return termFmt(text, fg, bg, {})
+
+func invert*(text: string): string = return CSI & "7m" & text & RESET_FORMATTING
+func striketrough*(text: string): string = return CSI & "9m" & text & RESET_FORMATTING
+
+
+func eraseDisplay*(mode = emToEnd): string = return CSI & $int(mode) & "J"
+func eraseLine*(mode = emToEnd): string = return CSI & $int(mode) & "K"
+
+
+func blinkSlow*(text: string): string = return CSI & "5m" & text & RESET_FORMATTING
+func blinkFast*(text: string): string = return CSI & "6m" & text & RESET_FORMATTING
+
+
+func cursorUp*(n: int = 1): string = return CSI & $int(n) & "A"
+func cursorDown*(n: int = 1): string = return CSI & $int(n) & "B"
+func cursorForward*(n: int = 1): string = return CSI & $int(n) & "C"
+func cursorBackward*(n: int = 1): string = return CSI & $int(n) & "D"
+func cursorNextLine*(n: int = 1): string = return CSI & $int(n) & "E"
+func cursorPrevLine*(n: int = 1): string = return CSI & $int(n) & "F"
+func cursorHorizontalAbsolute*(col: int): string = return CSI & $int(col) & "G"
+
+
+func cursorPosition*(row, col: int): string =
+  return CSI & $int(row) & ";" & $int(col) & "H"
+
+
+proc cursorType*(ct: CursorType): string = return CSI & $int(ct) & " q"
+func saveCursorPosition*(): string = return CSI & "s"
+func restoreCursorPosition*(): string = return CSI & "u"
+
+
+func scrollUp*(n: int = 1): string = return CSI & $int(n) & "S"
+func scrollDown*(n: int = 1): string = return CSI & $int(n) & "T"
+
+
+proc write*(f: File, x, y: int, text: string) =
+  f.write(cursorPosition(y, x) & text)
+
+
+proc write*(x, y: int, text: string) =
+  stdout.write(cursorPosition(y, x) & text)
+
+
+proc setCursorType*(ct: CursorType) = stdout.write(cursorType(ct))
+proc setCursorPosition*(x, y: int) = stdout.write(cursorPosition(y, x))
+proc hideCursor*() = stdout.write(CSI & "?25l")
+proc showCursor*() = stdout.write(CSI & "?25l")

cliutils/config.nim

@@ -0,0 +1,121 @@
+import std/[json, nre, os, sequtils, strtabs, strutils]
+import docopt
+
+type
+  CombinedConfig* = object
+    docopt*: Table[string, Value]
+    json*: JsonNode
+
+proc keyNames(key: string): tuple[arg, env, json: string] {.raises: [KeyError].} =
+  try:
+    result = (
+      "--" & key,
+      key.replace('-', '_').toUpper,
+      key.replace(re"(-\w)", proc (m: RegexMatch): string = ($m)[1..1].toUpper))
+  except Exception:
+    raise newException(KeyError, "invalid config key: '" & key & "'")
+
+template walkFieldDefs*(t: NimNode, body: untyped) =
+  let tTypeImpl = t.getTypeImpl
+
+  var nodeToItr: NimNode
+  if tTypeImpl.typeKind == ntyObject: nodeToItr = tTypeImpl[2]
+  elif tTypeImpl.typeKind == ntyTypeDesc: nodeToItr = tTypeImpl.getType[1].getType[2]
+  else: error $t & " is not an object or type desc (it's a " & $tTypeImpl.typeKind & ")."
+
+  for fieldDef {.inject.} in nodeToItr.children:
+    # ignore AST nodes that are not field definitions
+    if fieldDef.kind == nnkIdentDefs:
+      let fieldIdent {.inject.} = fieldDef[0]
+      let fieldType {.inject.} = fieldDef[1]
+      body
+
+    elif fieldDef.kind == nnkSym:
+      let fieldIdent {.inject.} = fieldDef
+      let fieldType {.inject.} = fieldDef.getType
+      body
+
+
+# TODO: Generic config loader
+# macro loadConfig*(filePath: string, cfgType: typed, args: Table[string, docopt.Value): untyped =
+#   result = newNimNode(nnkObjConstr).(cfgType)
+#
+#   var idx = 0
+#   cfgType.walkFieldDefs:
+#     let valLookup = quote do: `getVal`(
+
+proc initCombinedConfig*(
+    filename: string,
+    docopt: Table[string, Value] = initTable[string, Value]()
+  ): CombinedConfig =
+
+  result = CombinedConfig(docopt: docopt, json: parseFile(filename))
+
+proc findConfigFile*(name: string, otherLocations: seq[string] = @[]): string =
+  let cfgLocations = @[
+    $getEnv(name.strip(chars = {'.'}).toUpper()),
+    $getEnv("HOME") / name,
+    "." / name ] & otherLocations
+
+  result = cfgLocations.foldl(if fileExists(b): b else: a, "")
+  if result == "" or not fileExists(result):
+    raise newException(ValueError, "could not find configuration file")
+
+proc getVal*(cfg: CombinedConfig, key: string): string =
+  let (argKey, envKey, jsonKey) = keyNames(key)
+
+  if cfg.docopt.contains(argKey) and cfg.docopt[argKey]: return $cfg.docopt[argKey]
+  elif existsEnv(envKey): return getEnv(envKey)
+  elif cfg.json.hasKey(jsonKey):
+    let node = cfg.json[jsonKey]
+    case node.kind
+    of JString: return node.getStr
+    of JInt: return $node.getInt
+    of JFloat: return $node.getFloat
+    of JBool: return $node.getBool
+    of JNull: return ""
+    of JObject: return $node
+    of JArray: return $node
+  else: raise newException(ValueError, "cannot find a configuration value for \"" & key & "\"")
+
+proc getVal*(cfg: CombinedConfig, key, default: string): string {.raises: [].} =
+  try: return getVal(cfg, key)
+  except CatchableError: return default
+
+proc getJson*(
+    cfg: CombinedConfig,
+    key: string
+  ): JsonNode {.raises: [KeyError, ValueError, IOError, OSError]} =
+
+  let (argKey, envKey, jsonKey) = keyNames(key)
+
+  try:
+    if cfg.docopt.contains(argKey) and cfg.docopt[argKey]: return parseJson($cfg.docopt[argKey])
+    elif existsEnv(envKey): return parseJson(getEnv(envKey))
+    elif cfg.json.hasKey(jsonKey): return cfg.json[jsonKey]
+    else: raise newException(ValueError, "cannot find a configuration value for \"" & key & "\"")
+  except Exception:
+    raise newException(ValueError, "cannot parse [" & getVal(cfg, key) & "] as JSON:" & getCurrentExceptionMsg())
+
+proc getJson*(
+    cfg: CombinedConfig,
+    key: string,
+    default: JsonNode
+  ) : JsonNode {.raises: []} =
+
+  try: return getJson(cfg, key)
+  except CatchableError: return default
+
+proc hasKey*(cfg: CombinedConfig, key: string): bool =
+  let (argKey, envKey, jsonKey) = keyNames(key)
+
+  return
+    (cfg.docopt.contains(argKey) and cfg.docopt[argKey]) or
+    existsEnv(envKey) or
+    cfg.json.hasKey(jsonKey)
+
+proc loadEnv*(): StringTableRef =
+  result = newStringTable()
+
+  for k, v in envPairs():
+    result[k] = v

cliutils/daemonize.nim

@@ -0,0 +1,94 @@
+import posix
+import os except sleep
+
+## Daemonize
+var
+  pidFileInner: string
+  fi, fo, fe: File
+
+proc onStop(sig: cint) {.noconv.} =
+  close(fi)
+  close(fo)
+  close(fe)
+  removeFile(pidFileInner)
+
+  quit(QuitSuccess)
+
+proc daemonize*(pidfile, si, so, se: string, daemonMain: proc(): void): Pid =
+
+  if fileExists(pidfile):
+    raise newException(IOError, "pidfile " & pidfile & " already exists, daemon already running?")
+
+  # We're about to do a little dance to end up with a process that is
+  # disconnected from the calling program's process group and is not a session
+  # leader (a daemon process).
+  #
+  # Remember, after a call to fork the code is now running in two processes,
+  # the parent process and the child process.  So the returned PID will be a
+  # positive value if we are still on the parent process, and represents the
+  # PID of the newly created process. If we are on the newly created process,
+  # the PID returned with be 0.
+
+  let pid1 = fork()
+
+  # Are we the child process?
+  if pid1 == 0:
+    # Yes, so let's get ready to execute the main code given to us. Note that
+    # this child process will not actually live long. As part of our setup,
+    # we're going to create a new process session, resulting in this child
+    # process being the process group owner and session leader. We don't our
+    # daemon process to be a session leader because that allows it to be
+    # connected to a terminal. So after we've setup the new process group we
+    # will fork again and actually run the logic we're daemonizing in the
+    # grandchild process.
+
+    # Ignore whatever the original working directory was.
+    discard chdir("/")
+
+    # Create a new session to decouple us from the original terminal.
+    discard setsid()
+
+    # Set the process file mode creation mask to 0000
+    discard umask(0)
+
+    # Fork again so the grandchild process will not be the session leader.
+    let pid2 = fork()
+
+    # If we're still on the child process, we're done.
+    if pid2 > 0: quit(QuitSuccess)
+
+    # Otherwise, if we are the grandchild let's finish setting up our
+    # environment.
+    flushFile(stdout)
+    flushFile(stderr)
+
+    # Setup our STDIN, STDERR, and STDOUT pipes (if given)
+    if si.len > 0:
+      fi = open(si, fmRead)
+      discard dup2(getFileHandle(fi), getFileHandle(stdin))
+
+    if so.len > 0:
+      fo = open(so, fmAppend)
+      discard dup2(getFileHandle(fo), getFileHandle(stdout))
+
+    if se.len > 0:
+      fe = open(se, fmAppend)
+      discard dup2(getFileHandle(fe), getFileHandle(stderr))
+
+    # Keep a record of the PID filename in our process memory so we can
+    # reference it in our cleanup hooks.
+    pidFileInner = pidfile
+
+    # Add hooks to cleanup after ourselves when we're asked to die.
+    signal(SIGINT, onStop)
+    signal(SIGTERM, onStop)
+
+    # Find out what our actual PID is and save it to the PID file.
+    let childPid = getpid()
+    writeFile(pidfile, $childPid)
+
+    # Finally, execute the logic given to us to daemonize
+    daemonMain()
+
+  # If we're not the child proces, return the new PID of the daemonized process
+  return pid1

cliutils/procutil.nim

@@ -0,0 +1,71 @@
+import osproc, streams, strtabs, strutils
+
+## Process execution
+type HandleProcMsgCB* = proc (outMsg: string, errMsg: string, cmd: string): void
+
+proc sendMsg*(h: HandleProcMsgCB, outMsg: string,
+              errMsg: string = "", cmd: string = ""): void =
+  if h != nil: h(outMsg, errMsg, cmd)
+
+proc makeProcMsgHandler*(outSink, errSink: File, prefixCmd: bool = true): HandleProcMsgCB =
+  result = proc(outMsg, errMsg: string, cmd: string): void {.closure.} =
+    let prefix = if cmd.len == 0 or not prefixCmd: "" else: cmd & ": "
+    if outMsg.len > 0: outSink.writeLine(prefix & outMsg)
+    if errMsg.len > 0: errSink.writeLine(prefix & errMsg)
+
+proc makeProcMsgHandler*(outSink, errSink: Stream, prefixCmd: bool = true): HandleProcMsgCB =
+  result = proc(outMsg, errMsg: string, cmd: string): void {.closure.} =
+    let prefix = if cmd.len == 0 or not prefixCmd: "" else: cmd & ": "
+    if outMsg.len > 0: outSink.writeLine(prefix & outMsg)
+    if errMsg.len > 0: errSink.writeLine(prefix & errMsg)
+
+proc combineProcMsgHandlers*(a, b: HandleProcMsgCB): HandleProcMsgCB =
+  if a == nil: result = b
+  elif b == nil: result = a
+  else:
+    result = proc(cmd: string, outMsg, errMsg: string): void =
+      a(cmd, outMsg, errMsg)
+      b(cmd, outMsg, errMsg)
+
+proc waitFor*(p: Process, msgCB: HandleProcMsgCB, procCmd: string = ""): int =
+
+  var pout = outputStream(p)
+  var perr = errorStream(p)
+
+  var line = newStringOfCap(120)
+  while true:
+    if pout.readLine(line):
+      msgCB.sendMsg(line, "", procCmd)
+    elif perr.readLine(line):
+      msgCB.sendMsg("", line, procCmd)
+    else:
+      result = peekExitCode(p)
+      if result != -1: break
+  close(p)
+
+proc exec*(command: string, workingDir: string = "",
+           args: openArray[string] = [], env: StringTableRef = nil,
+           options: set[ProcessOption] = {poUsePath},
+           msgCB: HandleProcMsgCB = nil): int
+    {.tags: [ExecIOEffect, ReadIOEffect, RootEffect].} =
+
+  var p = startProcess(command, workingDir, args, env, options)
+  result = waitFor(p, msgCB, command)
+
+proc execWithOutput*(command: string, workingDir:string = "",
+           args: openArray[string] = [], env: StringTableRef = nil,
+           options: set[ProcessOption] = {poUsePath},
+           msgCB: HandleProcMsgCB = nil):
+    tuple[output, error: string, exitCode: int] =
+
+  result = ("", "", -1)
+  var outSeq, errSeq: seq[string]
+  outSeq = @[]; errSeq = @[]
+  let outputCollector = combineProcMsgHandlers(msgCB,
+    proc(outMsg, errMsg: string, cmd: string): void {.closure.} =
+      if outMsg.len > 0: outSeq.add(outMsg)
+      if errMsg.len > 0: errSeq.add(errMsg))
+
+  result[2] = exec(command, workingDir, args, env, options, outputCollector)
+  result[0] = outSeq.join("\n")
+  result[1] = errSeq.join("\n")

cliutils/queue_logger.nim

@@ -0,0 +1,25 @@
+import std/[logging]
+
+type
+  QueuedLogMessage* = tuple[lvl: Level, msg: string]
+
+  QueueLogger* = ref object of Logger
+    queue: seq[QueuedLogMessage]
+
+proc newQueueLogger*(
+  levelThreshold = lvlAll,
+  fmtStr = logging.defaultFmtStr): QueueLogger =
+
+  new result
+  result.fmtStr = fmtStr
+  result.levelThreshold = levelThreshold
+  result.queue = newSeq[QueuedLogMessage]()
+
+method log*(logger: QueueLogger, level: Level, args: varargs[string, `$`]) =
+  if level >= logger.levelThreshold:
+    logger.queue.add((level, substituteLog(logger.fmtStr, level, args)))
+
+proc clearQueue*(logger: QueueLogger) =
+  logger.queue = newSeq[QueuedLogMessage]()
+
+func messages*(logger: QueueLogger): seq[QueuedLogMessage] = logger.queue