Add truecolor and unicode support to terminal utilities.

.gitignore

@@ -1,3 +1,2 @@
 nimcache/
-cliutils
 *.sw?

cliutils.nim

@@ -1,50 +1,32 @@
-import nre, terminal, strtabs, tables, unicode
-import strutils except toUpper, toLower
+import std/[strtabs, strutils, tables]
 
+import ./cliutils/ansiterm
 import ./cliutils/config
 import ./cliutils/daemonize
 import ./cliutils/procutil
+import ./cliutils/queue_logger
 
+export ansiterm
 export config
 export daemonize
 export procutil
+export queue_logger
 
-type
-  TermColor = ForegroundColor or BackgroundColor
+proc doParseQueryParams[T](queryParams: T): seq[string] =
+  result = @[]
 
-const termReset* = "\e[0;m"
+  for k,v in queryParams:
+    # support ?arg1=val1&arg2=val2 -> cmd val1 val2
+    if k.startsWith("arg"): result.add(v)
 
-proc termColor*(color: TermColor, bright, bold = false): string =
-  var colorVal = ord(color)
-  if bright: inc(colorVal, 60)
-  return "\e[" & $colorVal & (if bold: ";1" else: "") & "m"
+    else :
+      result.add("--" & k)
+      if v != "true": result.add(v) # support things like ?verbose=true -> cmd --verbose
 
-proc withColor*(str: string, color: TermColor, bright, bold = false): string =
-  return termColor(color, bright, bold) & str
-
-
-proc stripAnsi*(str: string): string =
-  let STRIP_ANSI_REGEX = re"\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]"
-  return str.replace(STRIP_ANSI_REGEX, "")
 
 proc queryParamsToCliArgs*(queryParams: Table[string, string]): seq[string] =
-  result = @[]
+  doParseQueryParams(queryParams)
 
-  for k,v in queryParams:
-    # support ?arg1=val1&arg2=val2 -> cmd val1 val2
-    if k.startsWith("arg"): result.add(v)
-
-    else :
-      result.add("--" & k)
-      if v != "true": result.add(v) # support things like ?verbose=true -> cmd --verbose
 
 proc queryParamsToCliArgs*(queryParams: StringTableRef): seq[string] =
-  result = @[]
-
-  for k,v in queryParams:
-    # support ?arg1=val1&arg2=val2 -> cmd val1 val2
-    if k.startsWith("arg"): result.add(v)
-
-    else :
-      result.add("--" & k)
-      if v != "true": result.add(v) # support things like ?verbose=true -> cmd --verbose
+  doParseQueryParams(queryParams)

cliutils.nimble

@@ -1,6 +1,6 @@
 # Package
 
-version       = "0.8.0"
+version       = "0.11.0"
 author        = "Jonathan Bernard"
 description   = "Helper functions for writing command line interfaces."
 license       = "MIT"

cliutils/ansiterm.nim

@@ -0,0 +1,172 @@
+import std/[nre, sequtils, strutils, unicode]
+
+# See
+# - https://gist.github.com/ConnerWill/d4b6c776b509add763e17f9f113fd25b
+# - https://en.wikipedia.org/wiki/ANSI_escape_code
+
+const CSI = "\x1b["
+const RESET_FORMATTING* = "\x1b[0m"
+const ANSI_ESCAPE_CODE_ENDINGS*: seq[Rune] =
+  toRunes(toSeq('A'..'Z') & toSeq('a'..'z'))
+
+let FORMATTING_REGEX* = re("\x1b\\[([0-9;]*)([a-zA-Z])")
+
+type
+  TrueColor* = tuple[r, g, b: int]
+
+  CursorType* = enum
+    ctBlockBlink = 1, ctBlock, ctUnderlineBlink, ctUnderline, ctBarBlink, ctBar
+
+  EraseMode* = enum
+    emToEnd, emToStart, emAll
+
+  TerminalColors* = enum
+    cBlack = 0, cRed, cGreen, cYellow, cBlue, cMagenta, cCyan, cWhite,
+    cDefault = 9
+
+    cBrightBlack = 60, cBrightRed, cBrightGreen, cBrightYellow, cBrightBlue,
+    cBrightMagenta, cBrightCyan, cBrightWhite
+
+  TerminalStyle* = enum
+    tsBold = 1, tsDim, tsItalic, tsUnderline, tsBlink, tsInverse, tsHidden,
+    tsStrikethrough
+
+proc stripFormatting*(text: string): string =
+  text.replace(FORMATTING_REGEX, "")
+
+proc stripAnsi*(text: string): string = stripFormatting(text)
+
+func ansiAwareSubstring*(s: string, start, length: int): string =
+  var outputRunes = newSeq[Rune]()
+  var curAnsiSequences = newSeq[Rune]()
+  var runes = s.toRunes
+  var visibleRunesSeen = 0
+  var visibleOutputLen = 0
+  var i = 0
+
+  while i < runes.len and visibleOutputLen < length:
+
+    if runes.len > i + runeLen(CSI) and
+       $runes[i..<i + len(CSI)] == CSI:
+      # Recognize ANSI escape codes...
+
+      var j = i + runeLen(CSI)
+      while j < runes.len and runes[j] notin ANSI_ESCAPE_CODE_ENDINGS: j += 1
+
+      # and remember it if we're before the start of the substring
+      if visibleRunesSeen < start: curAnsiSequences.add(runes[i..j])
+
+      # or add it to the output without increasing our substring length
+      else: outputRunes.add(runes[i..j])
+
+      # either way we want to pick up after it
+      i = j
+
+    else:
+      if visibleRunesSeen >= start:
+        outputRunes.add(runes[i])
+        inc visibleOutputLen
+      inc visibleRunesSeen
+
+    inc i
+
+    result = $(curAnsiSequences & outputRunes)
+
+
+func ansiAwareSubstring*[T, U](s: string, r: HSlice[T,U]): string =
+  let idxStart =
+    when r.a is BackwardsIndex: s.runeLen - int(r.a)
+    else: int(r.a)
+
+  let idxEnd =
+    when r.b is BackwardsIndex: s.runeLen - int(r.b)
+    else: int(r.b)
+
+  ansiAwareSubstring(s, idxStart, idxEnd - idxStart + 1)
+
+
+func ansiEscSeq*(style: set[TerminalStyle]): string =
+  CSI & toSeq(style).mapIt($int(it)).join(";") & "m"
+
+func ansiEscSeq*(fg: int): string = CSI & "38;5;" & $fg & "m"
+func ansiEscSeq*(bg: int): string = CSI & "48;5;" & $bg & "m"
+
+
+func ansiEscSeq*(fg: TrueColor): string =
+  "$#38;2;$#;$#;$#m" % [CSI, $fg.r, $fg.g, $fg.b]
+
+func ansiEscSeq*(bg: TrueColor): string =
+  "$#48;2;$#;$#;$#m" % [CSI, $bg.r, $bg.g, $bg.b]
+
+
+func ansiEscSeq*(fg: TerminalColors): string = CSI & $(int(fg) + 30) & "m"
+func ansiEscSeq*(bg: TerminalColors): string = CSI & $(int(bg) + 40) & "m"
+
+
+func ansiEscSeq*(fg: TerminalColors, bg: TerminalColors, style: set[TerminalStyle]): string =
+  result = CSI
+  if style != {}: result &= toSeq(style).mapIt($int(it)).join(";") & ";"
+  result &= $(int(fg) + 30) & ";" & $(int(bg) + 40) & "m"
+
+
+func termFmt*(text: string, fg = cWhite, bg = cBlack, style: set[TerminalStyle] = {}): string =
+  return ansiEscSeq(fg, bg, style) & text & RESET_FORMATTING
+
+
+func termFmt*[C: int | TrueColor](
+    text: string,
+    fg, bg: C,
+    style: set[TerminalStyle] = {}): string =
+  return ansiEscSeq(style) & ansiEscSeq(fg = fg) & ansiEscSeq(bg = bg) &
+    text & RESET_FORMATTING
+
+
+func color*(text: string, fg = cDefault, bg = cDefault): string =
+  return termFmt(text, fg, bg, {})
+
+func invert*(text: string): string = return CSI & "7m" & text & RESET_FORMATTING
+func striketrough*(text: string): string = return CSI & "9m" & text & RESET_FORMATTING
+
+
+func eraseDisplay*(mode = emToEnd): string = return CSI & $int(mode) & "J"
+func eraseLine*(mode = emToEnd): string = return CSI & $int(mode) & "K"
+
+
+func blinkSlow*(text: string): string = return CSI & "5m" & text & RESET_FORMATTING
+func blinkFast*(text: string): string = return CSI & "6m" & text & RESET_FORMATTING
+
+
+func cursorUp*(n: int = 1): string = return CSI & $int(n) & "A"
+func cursorDown*(n: int = 1): string = return CSI & $int(n) & "B"
+func cursorForward*(n: int = 1): string = return CSI & $int(n) & "C"
+func cursorBackward*(n: int = 1): string = return CSI & $int(n) & "D"
+func cursorNextLine*(n: int = 1): string = return CSI & $int(n) & "E"
+func cursorPrevLine*(n: int = 1): string = return CSI & $int(n) & "F"
+func cursorHorizontalAbsolute*(col: int): string = return CSI & $int(col) & "G"
+
+
+func cursorPosition*(row, col: int): string =
+  return CSI & $int(row) & ";" & $int(col) & "H"
+
+
+proc cursorType*(ct: CursorType): string = return CSI & $int(ct) & " q"
+func saveCursorPosition*(): string = return CSI & "s"
+func restoreCursorPosition*(): string = return CSI & "u"
+
+
+func scrollUp*(n: int = 1): string = return CSI & $int(n) & "S"
+func scrollDown*(n: int = 1): string = return CSI & $int(n) & "T"
+
+
+proc write*(f: File, x, y: int, text: string) =
+  f.write(cursorPosition(y, x) & text)
+
+
+proc write*(x, y: int, text: string) =
+  stdout.write(cursorPosition(y, x) & text)
+
+
+proc setCursorType*(ct: CursorType) = stdout.write(cursorType(ct))
+proc setCursorPosition*(x, y: int) = stdout.write(cursorPosition(y, x))
+proc hideCursor*() = stdout.write(CSI & "?25l")
+proc showCursor*() = stdout.write(CSI & "?25l")

cliutils/config.nim

@@ -1,10 +1,20 @@
-import docopt, json, nre, os, strtabs, strutils
+import std/[json, nre, os, sequtils, strtabs, strutils]
+import docopt
 
 type
   CombinedConfig* = object
     docopt*: Table[string, Value]
     json*: JsonNode
 
+proc keyNames(key: string): tuple[arg, env, json: string] {.raises: [KeyError].} =
+  try:
+    result = (
+      "--" & key,
+      key.replace('-', '_').toUpper,
+      key.replace(re"(-\w)", proc (m: RegexMatch): string = ($m)[1..1].toUpper))
+  except Exception:
+    raise newException(KeyError, "invalid config key: '" & key & "'")
+
 template walkFieldDefs*(t: NimNode, body: untyped) =
   let tTypeImpl = t.getTypeImpl
 
@@ -34,10 +44,25 @@ template walkFieldDefs*(t: NimNode, body: untyped) =
 #   cfgType.walkFieldDefs:
 #     let valLookup = quote do: `getVal`(
 
+proc initCombinedConfig*(
+    filename: string,
+    docopt: Table[string, Value] = initTable[string, Value]()
+  ): CombinedConfig =
+
+  result = CombinedConfig(docopt: docopt, json: parseFile(filename))
+
+proc findConfigFile*(name: string, otherLocations: seq[string] = @[]): string =
+  let cfgLocations = @[
+    $getEnv(name.strip(chars = {'.'}).toUpper()),
+    $getEnv("HOME") / name,
+    "." / name ] & otherLocations
+
+  result = cfgLocations.foldl(if fileExists(b): b else: a, "")
+  if result == "" or not fileExists(result):
+    raise newException(ValueError, "could not find configuration file")
+
 proc getVal*(cfg: CombinedConfig, key: string): string =
-  let argKey = "--" & key
-  let envKey = key.replace('-', '_').toUpper
-  let jsonKey = key.replace(re"(-\w)", proc (m: RegexMatch): string = ($m)[1..1].toUpper)
+  let (argKey, envKey, jsonKey) = keyNames(key)
 
   if cfg.docopt.contains(argKey) and cfg.docopt[argKey]: return $cfg.docopt[argKey]
   elif existsEnv(envKey): return getEnv(envKey)
@@ -53,23 +78,41 @@ proc getVal*(cfg: CombinedConfig, key: string): string =
     of JArray: return $node
   else: raise newException(ValueError, "cannot find a configuration value for \"" & key & "\"")
 
-proc getVal*(cfg: CombinedConfig, key, default: string): string =
+proc getVal*(cfg: CombinedConfig, key, default: string): string {.raises: [].} =
   try: return getVal(cfg, key)
-  except: return default
+  except CatchableError: return default
 
-proc getJson*(cfg: CombinedConfig, key: string): JsonNode =
-  let argKey = "--" & key
-  let envKey = key.replace('-', '_').toUpper
-  let jsonKey = key.replace(re"(-\w)", proc (m: RegexMatch): string = ($m)[1..1].toUpper)
+proc getJson*(
+    cfg: CombinedConfig,
+    key: string
+  ): JsonNode {.raises: [KeyError, ValueError, IOError, OSError]} =
 
+  let (argKey, envKey, jsonKey) = keyNames(key)
+
+  try:
     if cfg.docopt.contains(argKey) and cfg.docopt[argKey]: return parseJson($cfg.docopt[argKey])
     elif existsEnv(envKey): return parseJson(getEnv(envKey))
     elif cfg.json.hasKey(jsonKey): return cfg.json[jsonKey]
     else: raise newException(ValueError, "cannot find a configuration value for \"" & key & "\"")
+  except Exception:
+    raise newException(ValueError, "cannot parse [" & getVal(cfg, key) & "] as JSON:" & getCurrentExceptionMsg())
+
+proc getJson*(
+    cfg: CombinedConfig,
+    key: string,
+    default: JsonNode
+  ) : JsonNode {.raises: []} =
 
-proc getJson*(cfg: CombinedConfig, key: string, default: JsonNode): JsonNode =
   try: return getJson(cfg, key)
-  except: return default
+  except CatchableError: return default
+
+proc hasKey*(cfg: CombinedConfig, key: string): bool =
+  let (argKey, envKey, jsonKey) = keyNames(key)
+
+  return
+    (cfg.docopt.contains(argKey) and cfg.docopt[argKey]) or
+    existsEnv(envKey) or
+    cfg.json.hasKey(jsonKey)
 
 proc loadEnv*(): StringTableRef =
   result = newStringTable()

cliutils/daemonize.nim

@@ -19,27 +19,50 @@ proc daemonize*(pidfile, si, so, se: string, daemonMain: proc(): void): Pid =
   if fileExists(pidfile):
     raise newException(IOError, "pidfile " & pidfile & " already exists, daemon already running?")
 
+  # We're about to do a little dance to end up with a process that is
+  # disconnected from the calling program's process group and is not a session
+  # leader (a daemon process).
+  #
+  # Remember, after a call to fork the code is now running in two processes,
+  # the parent process and the child process.  So the returned PID will be a
+  # positive value if we are still on the parent process, and represents the
+  # PID of the newly created process. If we are on the newly created process,
+  # the PID returned with be 0.
+
   let pid1 = fork()
 
   # Are we the child process?
   if pid1 == 0:
+    # Yes, so let's get ready to execute the main code given to us. Note that
+    # this child process will not actually live long. As part of our setup,
+    # we're going to create a new process session, resulting in this child
+    # process being the process group owner and session leader. We don't our
+    # daemon process to be a session leader because that allows it to be
+    # connected to a terminal. So after we've setup the new process group we
+    # will fork again and actually run the logic we're daemonizing in the
+    # grandchild process.
 
-    # Yes, so let's get ready to execute the main code given to us.
+    # Ignore whatever the original working directory was.
     discard chdir("/")
+
+    # Create a new session to decouple us from the original terminal.
     discard setsid()
+
+    # Set the process file mode creation mask to 0000
     discard umask(0)
 
-    # Fork again... but I'm not sure why.
+    # Fork again so the grandchild process will not be the session leader.
     let pid2 = fork()
 
-    # We don't need the intermediate process, so if we are not the child this
-    # time, lets just quit
+    # If we're still on the child process, we're done.
     if pid2 > 0: quit(QuitSuccess)
 
-    # If we are the grandchild let's set up our environment.
+    # Otherwise, if we are the grandchild let's finish setting up our
+    # environment.
     flushFile(stdout)
     flushFile(stderr)
 
+    # Setup our STDIN, STDERR, and STDOUT pipes (if given)
     if si.len > 0:
       fi = open(si, fmRead)
       discard dup2(getFileHandle(fi), getFileHandle(stdin))
@@ -52,17 +75,20 @@ proc daemonize*(pidfile, si, so, se: string, daemonMain: proc(): void): Pid =
       fe = open(se, fmAppend)
       discard dup2(getFileHandle(fe), getFileHandle(stderr))
 
+    # Keep a record of the PID filename in our process memory so we can
+    # reference it in our cleanup hooks.
     pidFileInner = pidfile
 
     # Add hooks to cleanup after ourselves when we're asked to die.
     signal(SIGINT, onStop)
     signal(SIGTERM, onStop)
 
-    # Find out what our actual PID is and save it
+    # Find out what our actual PID is and save it to the PID file.
     let childPid = getpid()
     writeFile(pidfile, $childPid)
 
-    # Finally, execute our main
+    # Finally, execute the logic given to us to daemonize
     daemonMain()
 
+  # If we're not the child proces, return the new PID of the daemonized process
   return pid1

cliutils/procutil.nim

@@ -47,7 +47,7 @@ proc exec*(command: string, workingDir: string = "",
            args: openArray[string] = [], env: StringTableRef = nil,
            options: set[ProcessOption] = {poUsePath},
            msgCB: HandleProcMsgCB = nil): int
-    {.tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
+    {.tags: [ExecIOEffect, ReadIOEffect, RootEffect].} =
 
   var p = startProcess(command, workingDir, args, env, options)
   result = waitFor(p, msgCB, command)

cliutils/queue_logger.nim

@@ -0,0 +1,25 @@
+import std/[logging]
+
+type
+  QueuedLogMessage* = tuple[lvl: Level, msg: string]
+
+  QueueLogger* = ref object of Logger
+    queue: seq[QueuedLogMessage]
+
+proc newQueueLogger*(
+  levelThreshold = lvlAll,
+  fmtStr = logging.defaultFmtStr): QueueLogger =
+
+  new result
+  result.fmtStr = fmtStr
+  result.levelThreshold = levelThreshold
+  result.queue = newSeq[QueuedLogMessage]()
+
+method log*(logger: QueueLogger, level: Level, args: varargs[string, `$`]) =
+  if level >= logger.levelThreshold:
+    logger.queue.add((level, substituteLog(logger.fmtStr, level, args)))
+
+proc clearQueue*(logger: QueueLogger) =
+  logger.queue = newSeq[QueuedLogMessage]()
+
+func messages*(logger: QueueLogger): seq[QueuedLogMessage] = logger.queue