Remove {.gcsafe.} from procutils.exec.

.gitignore

@@ -1,3 +1,2 @@
 nimcache/
-cliutils
 *.sw?

cliutils.nim

@@ -1,182 +1,19 @@
-import docopt, json, osproc, posix, nre, streams, strtabs, tables, terminal, unicode
-import os except sleep
+import std/[nre, terminal, strtabs, tables, unicode]
 import strutils except toUpper, toLower
 
+import ./cliutils/config
+import ./cliutils/daemonize
+import ./cliutils/procutil
+import ./cliutils/queue_logger
+
+export config
+export daemonize
+export procutil
+export queue_logger
 
 type
-  CombinedConfig* = object
-    docopt*: Table[string, Value]
-    json*: JsonNode
-
   TermColor = ForegroundColor or BackgroundColor
 
-proc getVal*(cfg: CombinedConfig, key: string): string =
-  let argKey = "--" & key
-  let envKey = key.replace('-', '_').toUpper
-  let jsonKey = key.replace(re"(-\w)", proc (m: RegexMatch): string = ($m)[1..1].toUpper)
-
-  if cfg.docopt.contains(argKey) and cfg.docopt[argKey]: return $cfg.docopt[argKey]
-  elif existsEnv(envKey): return getEnv(envKey)
-  elif cfg.json.hasKey(jsonKey):
-    let node = cfg.json[jsonKey]
-    case node.kind
-    of JString: return node.getStr
-    of JInt: return $node.getInt
-    of JFloat: return $node.getFloat
-    of JBool: return $node.getBool
-    of JNull: return ""
-    of JObject: return $node
-    of JArray: return $node
-  else: raise newException(ValueError, "cannot find a configuration value for \"" & key & "\"")
-
-proc getVal*(cfg: CombinedConfig, key, default: string): string =
-  try: return getVal(cfg, key)
-  except: return default
-
-proc loadEnv*(): StringTableRef =
-  result = newStringTable()
-
-  for k, v in envPairs():
-    result[k] = v
-
-## Process execution
-type HandleProcMsgCB* = proc (outMsg: TaintedString,
-                              errMsg: TaintedString, cmd: string): void
-
-proc sendMsg*(h: HandleProcMsgCB, outMsg: TaintedString,
-              errMsg: TaintedString = "", cmd: string = ""): void =
-  if h != nil: h(outMsg, errMsg, cmd)
-
-proc makeProcMsgHandler*(outSink, errSink: File, prefixCmd: bool = true): HandleProcMsgCB =
-  result = proc(outMsg, errMsg: TaintedString, cmd: string): void {.closure.} =
-    let prefix = if cmd.len == 0 or not prefixCmd: "" else: cmd & ": "
-    if outMsg.len > 0: outSink.writeLine(prefix & outMsg)
-    if errMsg.len > 0: errSink.writeLine(prefix & errMsg)
-
-proc makeProcMsgHandler*(outSink, errSink: Stream, prefixCmd: bool = true): HandleProcMsgCB =
-  result = proc(outMsg, errMsg: TaintedString, cmd: string): void {.closure.} =
-    let prefix = if cmd.len == 0 or not prefixCmd: "" else: cmd & ": "
-    if outMsg.len > 0: outSink.writeLine(prefix & outMsg)
-    if errMsg.len > 0: errSink.writeLine(prefix & errMsg)
-
-proc combineProcMsgHandlers*(a, b: HandleProcMsgCB): HandleProcMsgCB =
-  if a == nil: result = b
-  elif b == nil: result = a
-  else:
-    result = proc(cmd: string, outMsg, errMsg: TaintedString): void =
-      a(cmd, outMsg, errMsg)
-      b(cmd, outMsg, errMsg)
-
-proc waitFor*(p: Process, msgCB: HandleProcMsgCB, procCmd: string = ""): int =
-
-  var pout = outputStream(p)
-  var perr = errorStream(p)
-
-  var line = newStringOfCap(120).TaintedString
-  while true:
-    if pout.readLine(line):
-      msgCB.sendMsg(line, "", procCmd)
-    elif perr.readLine(line):
-      msgCB.sendMsg("", line, procCmd)
-    else:
-      result = peekExitCode(p)
-      if result != -1: break
-  close(p)
-
-proc exec*(command: string, workingDir: string = "",
-           args: openArray[string] = [], env: StringTableRef = nil,
-           options: set[ProcessOption] = {poUsePath},
-           msgCB: HandleProcMsgCB = nil): int
-    {.tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
-
-  var p = startProcess(command, workingDir, args, env, options)
-  result = waitFor(p, msgCB, command)
-
-proc execWithOutput*(command: string, workingDir:string = "",
-           args: openArray[string] = [], env: StringTableRef = nil,
-           options: set[ProcessOption] = {poUsePath},
-           msgCB: HandleProcMsgCB = nil):
-    tuple[output, error: TaintedString, exitCode: int] =
-
-  result = (TaintedString"", TaintedString"", -1)
-  var outSeq, errSeq: seq[TaintedString]
-  outSeq = @[]; errSeq = @[]
-  var outputCollector = combineProcMsgHandlers(msgCB,
-    proc(outMsg, errMsg: TaintedString, cmd: string): void {.closure.} =
-      if outMsg.len > 0: outSeq.add(outMsg)
-      if errMsg.len > 0: errSeq.add(errMsg))
-
-  result[2] = exec(command, workingDir, args, env, options, outputCollector)
-  result[0] = outSeq.join("\n")
-  result[1] = errSeq.join("\n")
-
-## Daemonize
-
-var
-  pidFileInner: string
-  fi, fo, fe: File
-
-proc onStop(sig: cint) {.noconv.} =
-  close(fi)
-  close(fo)
-  close(fe)
-  removeFile(pidFileInner)
-
-  quit(QuitSuccess)
-
-proc daemonize*(pidfile, si, so, se: string, daemonMain: proc(): void): Pid =
-
-  if fileExists(pidfile):
-    raise newException(IOError, "pidfile " & pidfile & " already exists, daemon already running?")
-
-  let pid1 = fork()
-
-  # Are we the child process?
-  if pid1 == 0:
-
-    # Yes, so let's get ready to execute the main code given to us.
-    discard chdir("/")
-    discard setsid()
-    discard umask(0)
-
-    # Fork again... but I'm not sure why.
-    let pid2 = fork()
-
-    # We don't need the intermediate process, so if we are not the child this
-    # time, lets just quit
-    if pid2 > 0: quit(QuitSuccess)
-
-    # If we are the grandchild let's set up our environment.
-    flushFile(stdout)
-    flushFile(stderr)
-
-    if si.len > 0:
-      fi = open(si, fmRead)
-      discard dup2(getFileHandle(fi), getFileHandle(stdin))
-
-    if so.len > 0:
-      fo = open(so, fmAppend)
-      discard dup2(getFileHandle(fo), getFileHandle(stdout))
-
-    if se.len > 0:
-      fe = open(se, fmAppend)
-      discard dup2(getFileHandle(fe), getFileHandle(stderr))
-
-    pidFileInner = pidfile
-
-    # Add hooks to cleanup after ourselves when we're asked to die.
-    signal(SIGINT, onStop)
-    signal(SIGTERM, onStop)
-
-    # Find out what our actual PID is and save it
-    let childPid = getpid()
-    writeFile(pidfile, $childPid)
-
-    # Finally, execute our main
-    daemonMain()
-
-  return pid1
-
 const termReset* = "\e[0;m"
 
 proc termColor*(color: TermColor, bright, bold = false): string =
@@ -187,22 +24,12 @@ proc termColor*(color: TermColor, bright, bold = false): string =
 proc withColor*(str: string, color: TermColor, bright, bold = false): string =
   return termColor(color, bright, bold) & str
 
-let STRIP_ANSI_REGEX = re"\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]"
 
-proc stripAnsi*(str: string): string = return str.replace(STRIP_ANSI_REGEX, "")
+proc stripAnsi*(str: string): string =
+  let STRIP_ANSI_REGEX = re"\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]"
+  return str.replace(STRIP_ANSI_REGEX, "")
 
-proc queryParamsToCliArgs*(queryParams: Table[string, string]): seq[string] =
-  result = @[]
-
-  for k,v in queryParams:
-    # support ?arg1=val1&arg2=val2 -> cmd val1 val2
-    if k.startsWith("arg"): result.add(v)
-
-    else :
-      result[1].add("--" & k)
-      if v != "true": result[1].add(v) # support things like ?verbose=true -> cmd --verbose
-
-proc queryParamsToCliArgs*(queryParams: StringTableRef): seq[string] =
+proc doParseQueryParams[T](queryParams: T): seq[string] =
   result = @[]
 
   for k,v in queryParams:
@@ -212,3 +39,9 @@ proc queryParamsToCliArgs*(queryParams: StringTableRef): seq[string] =
     else :
       result.add("--" & k)
       if v != "true": result.add(v) # support things like ?verbose=true -> cmd --verbose
+
+proc queryParamsToCliArgs*(queryParams: Table[string, string]): seq[string] =
+  doParseQueryParams(queryParams)
+
+proc queryParamsToCliArgs*(queryParams: StringTableRef): seq[string] =
+  doParseQueryParams(queryParams)

cliutils.nimble

@@ -1,11 +1,11 @@
 # Package
 
-version       = "0.6.3"
+version       = "0.9.1"
 author        = "Jonathan Bernard"
 description   = "Helper functions for writing command line interfaces."
 license       = "MIT"
 
+
 # Dependencies
 
-requires @["nim >= 0.19.0", "docopt >= 0.6.8"]
-
+requires @["nim >= 1.6.0", "docopt >= 0.6.8"]

cliutils/config.nim

@@ -0,0 +1,121 @@
+import std/[json, nre, os, sequtils, strtabs, strutils]
+import docopt
+
+type
+  CombinedConfig* = object
+    docopt*: Table[string, Value]
+    json*: JsonNode
+
+proc keyNames(key: string): tuple[arg, env, json: string] {.raises: [KeyError].} =
+  try:
+    result = (
+      "--" & key,
+      key.replace('-', '_').toUpper,
+      key.replace(re"(-\w)", proc (m: RegexMatch): string = ($m)[1..1].toUpper))
+  except Exception:
+    raise newException(KeyError, "invalid config key: '" & key & "'")
+
+template walkFieldDefs*(t: NimNode, body: untyped) =
+  let tTypeImpl = t.getTypeImpl
+
+  var nodeToItr: NimNode
+  if tTypeImpl.typeKind == ntyObject: nodeToItr = tTypeImpl[2]
+  elif tTypeImpl.typeKind == ntyTypeDesc: nodeToItr = tTypeImpl.getType[1].getType[2]
+  else: error $t & " is not an object or type desc (it's a " & $tTypeImpl.typeKind & ")."
+
+  for fieldDef {.inject.} in nodeToItr.children:
+    # ignore AST nodes that are not field definitions
+    if fieldDef.kind == nnkIdentDefs:
+      let fieldIdent {.inject.} = fieldDef[0]
+      let fieldType {.inject.} = fieldDef[1]
+      body
+
+    elif fieldDef.kind == nnkSym:
+      let fieldIdent {.inject.} = fieldDef
+      let fieldType {.inject.} = fieldDef.getType
+      body
+
+
+# TODO: Generic config loader
+# macro loadConfig*(filePath: string, cfgType: typed, args: Table[string, docopt.Value): untyped =
+#   result = newNimNode(nnkObjConstr).(cfgType)
+#
+#   var idx = 0
+#   cfgType.walkFieldDefs:
+#     let valLookup = quote do: `getVal`(
+
+proc initCombinedConfig*(
+    filename: string,
+    docopt: Table[string, Value] = initTable[string, Value]()
+  ): CombinedConfig =
+
+  result = CombinedConfig(docopt: docopt, json: parseFile(filename))
+
+proc findConfigFile*(name: string, otherLocations: seq[string] = @[]): string =
+  let cfgLocations = @[
+    $getEnv(name.strip(chars = {'.'}).toUpper()),
+    $getEnv("HOME") / name,
+    "." / name ] & otherLocations
+
+  result = cfgLocations.foldl(if fileExists(b): b else: a, "")
+  if result == "" or not fileExists(result):
+    raise newException(ValueError, "could not find configuration file")
+
+proc getVal*(cfg: CombinedConfig, key: string): string =
+  let (argKey, envKey, jsonKey) = keyNames(key)
+
+  if cfg.docopt.contains(argKey) and cfg.docopt[argKey]: return $cfg.docopt[argKey]
+  elif existsEnv(envKey): return getEnv(envKey)
+  elif cfg.json.hasKey(jsonKey):
+    let node = cfg.json[jsonKey]
+    case node.kind
+    of JString: return node.getStr
+    of JInt: return $node.getInt
+    of JFloat: return $node.getFloat
+    of JBool: return $node.getBool
+    of JNull: return ""
+    of JObject: return $node
+    of JArray: return $node
+  else: raise newException(ValueError, "cannot find a configuration value for \"" & key & "\"")
+
+proc getVal*(cfg: CombinedConfig, key, default: string): string {.raises: [].} =
+  try: return getVal(cfg, key)
+  except CatchableError: return default
+
+proc getJson*(
+    cfg: CombinedConfig,
+    key: string
+  ): JsonNode {.raises: [KeyError, ValueError, IOError, OSError]} =
+
+  let (argKey, envKey, jsonKey) = keyNames(key)
+
+  try:
+    if cfg.docopt.contains(argKey) and cfg.docopt[argKey]: return parseJson($cfg.docopt[argKey])
+    elif existsEnv(envKey): return parseJson(getEnv(envKey))
+    elif cfg.json.hasKey(jsonKey): return cfg.json[jsonKey]
+    else: raise newException(ValueError, "cannot find a configuration value for \"" & key & "\"")
+  except Exception:
+    raise newException(ValueError, "cannot parse value as JSON:" & getCurrentExceptionMsg())
+
+proc getJson*(
+    cfg: CombinedConfig,
+    key: string,
+    default: JsonNode
+  ) : JsonNode {.raises: []} =
+
+  try: return getJson(cfg, key)
+  except CatchableError: return default
+
+proc hasKey*(cfg: CombinedConfig, key: string): bool =
+  let (argKey, envKey, jsonKey) = keyNames(key)
+
+  return
+    (cfg.docopt.contains(argKey) and cfg.docopt[argKey]) or
+    existsEnv(envKey) or
+    cfg.json.hasKey(jsonKey)
+
+proc loadEnv*(): StringTableRef =
+  result = newStringTable()
+
+  for k, v in envPairs():
+    result[k] = v

cliutils/daemonize.nim

@@ -0,0 +1,94 @@
+import posix
+import os except sleep
+
+## Daemonize
+var
+  pidFileInner: string
+  fi, fo, fe: File
+
+proc onStop(sig: cint) {.noconv.} =
+  close(fi)
+  close(fo)
+  close(fe)
+  removeFile(pidFileInner)
+
+  quit(QuitSuccess)
+
+proc daemonize*(pidfile, si, so, se: string, daemonMain: proc(): void): Pid =
+
+  if fileExists(pidfile):
+    raise newException(IOError, "pidfile " & pidfile & " already exists, daemon already running?")
+
+  # We're about to do a little dance to end up with a process that is
+  # disconnected from the calling program's process group and is not a session
+  # leader (a daemon process).
+  #
+  # Remember, after a call to fork the code is now running in two processes,
+  # the parent process and the child process.  So the returned PID will be a
+  # positive value if we are still on the parent process, and represents the
+  # PID of the newly created process. If we are on the newly created process,
+  # the PID returned with be 0.
+
+  let pid1 = fork()
+
+  # Are we the child process?
+  if pid1 == 0:
+    # Yes, so let's get ready to execute the main code given to us. Note that
+    # this child process will not actually live long. As part of our setup,
+    # we're going to create a new process session, resulting in this child
+    # process being the process group owner and session leader. We don't our
+    # daemon process to be a session leader because that allows it to be
+    # connected to a terminal. So after we've setup the new process group we
+    # will fork again and actually run the logic we're daemonizing in the
+    # grandchild process.
+
+    # Ignore whatever the original working directory was.
+    discard chdir("/")
+
+    # Create a new session to decouple us from the original terminal.
+    discard setsid()
+
+    # Set the process file mode creation mask to 0000
+    discard umask(0)
+
+    # Fork again so the grandchild process will not be the session leader.
+    let pid2 = fork()
+
+    # If we're still on the child process, we're done.
+    if pid2 > 0: quit(QuitSuccess)
+
+    # Otherwise, if we are the grandchild let's finish setting up our
+    # environment.
+    flushFile(stdout)
+    flushFile(stderr)
+
+    # Setup our STDIN, STDERR, and STDOUT pipes (if given)
+    if si.len > 0:
+      fi = open(si, fmRead)
+      discard dup2(getFileHandle(fi), getFileHandle(stdin))
+
+    if so.len > 0:
+      fo = open(so, fmAppend)
+      discard dup2(getFileHandle(fo), getFileHandle(stdout))
+
+    if se.len > 0:
+      fe = open(se, fmAppend)
+      discard dup2(getFileHandle(fe), getFileHandle(stderr))
+
+    # Keep a record of the PID filename in our process memory so we can
+    # reference it in our cleanup hooks.
+    pidFileInner = pidfile
+
+    # Add hooks to cleanup after ourselves when we're asked to die.
+    signal(SIGINT, onStop)
+    signal(SIGTERM, onStop)
+
+    # Find out what our actual PID is and save it to the PID file.
+    let childPid = getpid()
+    writeFile(pidfile, $childPid)
+
+    # Finally, execute the logic given to us to daemonize
+    daemonMain()
+
+  # If we're not the child proces, return the new PID of the daemonized process
+  return pid1

cliutils/procutil.nim

@@ -0,0 +1,71 @@
+import osproc, streams, strtabs, strutils
+
+## Process execution
+type HandleProcMsgCB* = proc (outMsg: string, errMsg: string, cmd: string): void
+
+proc sendMsg*(h: HandleProcMsgCB, outMsg: string,
+              errMsg: string = "", cmd: string = ""): void =
+  if h != nil: h(outMsg, errMsg, cmd)
+
+proc makeProcMsgHandler*(outSink, errSink: File, prefixCmd: bool = true): HandleProcMsgCB =
+  result = proc(outMsg, errMsg: string, cmd: string): void {.closure.} =
+    let prefix = if cmd.len == 0 or not prefixCmd: "" else: cmd & ": "
+    if outMsg.len > 0: outSink.writeLine(prefix & outMsg)
+    if errMsg.len > 0: errSink.writeLine(prefix & errMsg)
+
+proc makeProcMsgHandler*(outSink, errSink: Stream, prefixCmd: bool = true): HandleProcMsgCB =
+  result = proc(outMsg, errMsg: string, cmd: string): void {.closure.} =
+    let prefix = if cmd.len == 0 or not prefixCmd: "" else: cmd & ": "
+    if outMsg.len > 0: outSink.writeLine(prefix & outMsg)
+    if errMsg.len > 0: errSink.writeLine(prefix & errMsg)
+
+proc combineProcMsgHandlers*(a, b: HandleProcMsgCB): HandleProcMsgCB =
+  if a == nil: result = b
+  elif b == nil: result = a
+  else:
+    result = proc(cmd: string, outMsg, errMsg: string): void =
+      a(cmd, outMsg, errMsg)
+      b(cmd, outMsg, errMsg)
+
+proc waitFor*(p: Process, msgCB: HandleProcMsgCB, procCmd: string = ""): int =
+
+  var pout = outputStream(p)
+  var perr = errorStream(p)
+
+  var line = newStringOfCap(120)
+  while true:
+    if pout.readLine(line):
+      msgCB.sendMsg(line, "", procCmd)
+    elif perr.readLine(line):
+      msgCB.sendMsg("", line, procCmd)
+    else:
+      result = peekExitCode(p)
+      if result != -1: break
+  close(p)
+
+proc exec*(command: string, workingDir: string = "",
+           args: openArray[string] = [], env: StringTableRef = nil,
+           options: set[ProcessOption] = {poUsePath},
+           msgCB: HandleProcMsgCB = nil): int
+    {.tags: [ExecIOEffect, ReadIOEffect, RootEffect].} =
+
+  var p = startProcess(command, workingDir, args, env, options)
+  result = waitFor(p, msgCB, command)
+
+proc execWithOutput*(command: string, workingDir:string = "",
+           args: openArray[string] = [], env: StringTableRef = nil,
+           options: set[ProcessOption] = {poUsePath},
+           msgCB: HandleProcMsgCB = nil):
+    tuple[output, error: string, exitCode: int] =
+
+  result = ("", "", -1)
+  var outSeq, errSeq: seq[string]
+  outSeq = @[]; errSeq = @[]
+  let outputCollector = combineProcMsgHandlers(msgCB,
+    proc(outMsg, errMsg: string, cmd: string): void {.closure.} =
+      if outMsg.len > 0: outSeq.add(outMsg)
+      if errMsg.len > 0: errSeq.add(errMsg))
+
+  result[2] = exec(command, workingDir, args, env, options, outputCollector)
+  result[0] = outSeq.join("\n")
+  result[1] = errSeq.join("\n")

cliutils/queue_logger.nim

@@ -0,0 +1,25 @@
+import std/[logging]
+
+type
+  QueuedLogMessage* = tuple[lvl: Level, msg: string]
+
+  QueueLogger* = ref object of Logger
+    queue: seq[QueuedLogMessage]
+
+proc newQueueLogger*(
+  levelThreshold = lvlAll,
+  fmtStr = logging.defaultFmtStr): QueueLogger =
+
+  new result
+  result.fmtStr = fmtStr
+  result.levelThreshold = levelThreshold
+  result.queue = newSeq[QueuedLogMessage]()
+
+method log*(logger: QueueLogger, level: Level, args: varargs[string, `$`]) =
+  if level >= logger.levelThreshold:
+    logger.queue.add((level, substituteLog(logger.fmtStr, level, args)))
+
+proc clearQueue*(logger: QueueLogger) =
+  logger.queue = newSeq[QueuedLogMessage]()
+
+func messages*(logger: QueueLogger): seq[QueuedLogMessage] = logger.queue