Don't show sample highlighting for expected fields (which will never be highlighted).

It didn't really work when reading from /dev/tty anyways, as the program
upstream from slfmt is probably also reading from /dev/tty and has
priority.

slfmt.nimble

@@ -1,6 +1,6 @@
 # Package
 
-version       = "0.2.2"
+version       = "1.0.0"
 author        = "Jonathan Bernard"
 description   = "Small utility to pretty-print strucutured logs."
 license       = "MIT"
@@ -10,5 +10,5 @@ bin           = @["slfmt"]
 
 # Dependencies
 
-requires @["nim >= 2.2.0", "docopt"]
+requires @["nim >= 2.2.0", "docopt >= 0.7.1"]
-requires @["timeutils", "zero_functional"]
+requires @["cliutils >= 0.11.0", "timeutils", "zero_functional"]

src/slfmt.nim

@@ -1,25 +1,146 @@
-import std/[json, options, sequtils, streams, strutils, terminal, times]
+import std/[atomics, json, options, os, posix, sequtils, setutils, strutils,
-import docopt, timeutils, zero_functional
+            terminal, times, unicode]
+import cliutils, docopt, timeutils, zero_functional
 
 from std/logging import Level
-from std/sequtils import toSeq
+import std/nre except toSeq
 
-const VERSION = "0.2.2"
+const VERSION = "1.0.0"
 
 const USAGE = """Usage:
-   slfmt [options]
+    slfmt [options]
 
 Options:
 
-   -h, --help               Print this usage and help information
+    -h, --help               Print this usage and help information
-   -c, --compact            Compact output
+    -c, --compact            Compact output
-   -l, --log-level <lvl>    Only show log events at or above this level
+    -l, --log-level <lvl>    Only show log events at or above this level
-   -n, --namespace <ns>     Only show log events from this namespace
+    -n, --namespace <ns>     Only show log events from this namespace
+
+    -e, --expected <expected>
+
+        Add a filter for something expected to be present in the logs. It's
+        absence will be flagged. On the command-line, <expected> should be
+        formatted as:
+
+          field-name:regex-pattern:label
+
+        <label> is optional and when provided is used as the value displayed
+        when this expectation is flagged. If the log stream is not a structured
+        log, the <field-name> is ignored and the log text is matched directly
+        against the log line. Multiple expectations can be provided, separating
+        each with ';'.
+
+    -E, --unexpected <unexpected>
+
+        Add a filter for something that should not be present in the logs. It's
+        presence will be flagged. On the command-line, <unexpected> should be
+        formatted as:
+
+          field-name:regex-pattern:label
+
+        This option can be provided multiple times. <label> is optional and
+        when provided is used as the value displayed when this expectation is
+        flagged. If the log stream is not a structured log, the <field-name> is
+        ignored and the log text is matched directly against the log line.
+        Multiple expectations can be provided, separating each with ';'.
+
+    --config <cfgFile>
+
+        Read expectation data from a json config file formatted as follows:
+
+            [
+              {
+                "fieldName": "<field-name>",
+                "label": "<label value>", // optional
+                "pattern": "<regex pattern>",
+                "expected": false,
+                "style": {    // optional
+                  "fg": "brightwhite",
+                  "bg": "red",
+                  "style": ["bold"]
+                },
+                // ...
+            ]
+
+        The file should be a valid JSON array of expectation objects. Each
+        object must include fieldName, label, pattern, and expected fields. The
+        expected field determines whether this field should or should not be
+        found in the output. If set to true, this  expectation operates like
+        --expected. If set to false it operates like --unexpected.
+
+        If label is not present or is null, the fieldName:pattern combination
+        will be used, similar to the CLI arguments behavior.
+
+        If the style field is set, it must be an object with fg, bg, and style
+        keys. There are three types of values that can be passed to fg and bg:
+
+        1. A string value representing named terminal color. The list of
+           recognized colors is
+
+             black, red, green, yellow, blue, magenta, cyan, white, default
+
+          There are also bright versions of each (except default) for terminals
+          that support them. For example, brightred, brightblack, etc.
+
+          For example:
+
+              "fg": "red",
+
+        2. A number representing an 8-bit color value (1-255). For example:
+
+              "bg": 60,
+
+        3. An object with r, g, and b fields representing a 24-bit Truecolor
+           value. For example:
+
+              "fg": { "r": 22, "g": 57, "b": 105 }
+
+        If there is no style set, one of the default styles is used (like when
+        using the CLI args to define expectations).
 """
 
+type
+  CtrlCmd = enum ccQuit, ccReset
+
+  Expectation = ref object
+    fieldName: string
+    label: string
+    pattern: Regex
+    expected: bool
+    count: int
+    termStyleCode: string # ANSI escape code
+
+
 const fieldDisplayOrder = @[
   "scope", "level", "ts", "code", "sid", "sub", "msg", "err", "stack", "method", "args"]
 
+
+let FORMATTING_REGEX* = re("\x1b\\[([0-9;]*)([a-zA-Z])")
+
+
+const EXPECTATION_STYLES = [
+  (ansiEscSeq(fg = cBrightWhite) & ansiEscSeq(bg = (r: 105, g: 22, b:33))), # red
+  (ansiEscSeq(fg = cBrightWhite) & ansiEscSeq(bg = (r: 62, g: 100, b:15))), # green
+  (ansiEscSeq(fg = cBrightWhite) & ansiEscSeq(bg = (r: 22, g: 57, b:105))), # blue
+  (ansiEscSeq(fg = cBrightWhite) & ansiEscSeq(bg = (r: 79, g: 79, b:32))),  # yellow
+  (ansiEscSeq(fg = cBrightWhite) & ansiEscSeq(bg = (r: 73, g: 55, b:78))),  # magenta
+  (ansiEscSeq(fg = cBrightWhite) & ansiEscSeq(bg = (r: 41, g: 84, b:98)))   # cyan
+]
+
+var stopFlag: Atomic[bool]
+
+
+proc handleSignal(sig: cint) {.noconv.} = stopFlag.store(true)
+
+
+func getOrFail(n: JsonNode, key: string): JsonNode =
+  if not n.contains(key):
+    raise newException(ValueError, "missing " & key &
+      " field in the following JSON node:\n" & n.pretty)
+  n[key]
+
+
 func parseLogLevel(s: string): Level =
   case s.toUpper
   of "DEBUG": result = Level.lvlDebug
@@ -30,92 +151,254 @@ func parseLogLevel(s: string): Level =
   of "FATAL": result = Level.lvlFatal
   else: result = Level.lvlAll
 
-func decorate(
-    s: string,
-    fg = fgDefault,
-    style: set[Style] = {}): string =
 
-  result = ""
+proc processFormattedFieldExpectations(
+  expectations: seq[Expectation] = @[],
+  value: string): string =
 
-  if style != {}:
+  result = value
-    result &= toSeq(items(style)).mapIt(ansiStyleCode(it)).join("")
 
-  if fg != fgDefault: result &= ansiForegroundColorCode(fg)
+  for exp in expectations:
+    let match = find(stripAnsi(value), exp.pattern)
+    if match.isSome:
+      inc exp.count
 
-  result &= s & ansiResetCode
+      if not exp.expected:
+        let bounds = match.get.matchBounds
+        result =
+          ansiAwareSubstring(result, (0..<bounds.a)) &
+          exp.termStyleCode &
+          stripAnsi(ansiAwareSubstring(result, (bounds.a .. bounds.b))) &
+          RESET_FORMATTING &
+          ansiAwareSubstring(result, (bounds.b + 1 ..< ^1))
 
 
-proc fullFormatField(name: string, value: JsonNode): string =
+proc processFormattedFieldExpectations(
-  result = decorate(name, fgCyan) & ":" & " ".repeat(max(1, 10 - name.len))
+    name: string,
+    expectations: seq[Expectation],
+    value: string): string =
+
+  processFormattedFieldExpectations(
+    expectations.filterIt(it.fieldName == name),
+    value)
+
+proc fullFormatField(
+    name: string,
+    value: JsonNode,
+    expectations: seq[Expectation] = @[]): string =
+  result = color(name, cCyan) & ":" & " ".repeat(max(1, 10 - name.len))
 
   var strVal: string = ""
   case name:
     of "ts":
       let dt = parseIso8601(value.getStr)
-      strVal = decorate(dt.local.formatIso8601 & " (local)  ", fgBlue, {styleBright}) &
+      strVal =
+        color(dt.local.formatIso8601 & " (local)  ", cBrightBlue) &
         dt.utc.formatIso8601 & " (UTC)"
-    of "sid", "sub": strVal = decorate(value.getStr, fgGreen)
+    of "sid", "sub": strVal = color(value.getStr, cGreen)
-    of "err": strVal = decorate(value.getStr, fgRed)
+    of "err": strVal = color(value.getStr, cRed)
-    of "msg": strVal = decorate(value.getStr, fgYellow)
+    of "msg": strVal = color(value.getStr, cYellow)
-    of "stack": strVal = decorate(value.getStr, fgBlack, {styleBright})
+    of "stack": strVal = color(value.getStr, cBrightBlack)
     else:
-      if value.kind == JString: strVal = decorate(value.getStr)
+      if value.kind == JString: strVal = value.getStr
       else: strVal = pretty(value)
 
+  strVal = processFormattedFieldExpectations(name, expectations, strVal)
+
   let valLines = splitLines(strVal)
   if name.len + strVal.len + 6 > terminalWidth() or valLines.len > 1:
-    result &= "\n" & valLines.mapIt("  " & it).join("\n") & "\n"
+    result &= "\p" & valLines.mapIt("  " & it).join("\p") & "\p"
-  else: result &= strVal & "\n"
+  else: result &= strVal & "\p"
 
-proc fullFormat(logJson: JsonNode): string =
+proc fullFormat(logJson: JsonNode, expectations: seq[Expectation] = @[]): string =
-  result = '-'.repeat(terminalWidth()) & "\n"
+  result = '-'.repeat(terminalWidth()) & "\p"
 
   # Print the known fields in order first
   for f in fieldDisplayOrder:
     if logJson.hasKey(f):
-      result &= fullFormatField(f, logJson[f])
+      result &= fullFormatField(f, logJson[f], expectations)
       logJson.delete(f)
 
   # Print the rest of the fields
-  for (key, val) in pairs(logJson): result &= fullFormatField(key, val)
+  for (key, val) in pairs(logJson):
+    result &= fullFormatField(key, val, expectations)
 
-  result &= "\n"
+  result &= "\p"
+
+proc compactFormat(logJson: JsonNode, expectations: seq[Expectation] = @[]): string =
+  var ts = parseIso8601(logJson["ts"].getStr).local.formatIso8601
+  ts = color(alignLeft(ts[0..21], 23), cBrightBlue)
+  ts = processFormattedFieldExpectations("ts", expectations, ts)
 
-proc compactFormat(logJson: JsonNode): string =
-  let ts = parseIso8601(logJson["ts"].getStr).local.formatIso8601
   var level = logJson["level"].getStr
-  if level == "ERROR": level = decorate(alignLeft(level, 7), fgRed)
+  if level == "ERROR": level = color(alignLeft(level, 7), cRed)
-  elif level == "WARN": level = decorate(alignLeft(level, 7), fgYellow)
+  elif level == "WARN": level = color(alignLeft(level, 7), cYellow)
   else: level = alignLeft(level, 7)
+  level = processFormattedFieldExpectations("level", expectations, level)
 
-  result = "$1 $2 $3: $4" % [
+  let scope = processFormattedFieldExpectations("scope", expectations,
-    level,
+    color(logJson["scope"].getStr, cGreen))
-    decorate(alignLeft(ts[0..21], 23), fgBlue, {styleBright}),
+
-    decorate(logJson["scope"].getStr, fgGreen),
+  let msg = processFormattedFieldExpectations("msg", expectations,
-    decorate(logJson["msg"].getStr, fgYellow)]
+    color(logJson["msg"].getStr, cYellow))
+
+  result = "$1 $2 $3: $4" % [ level, ts, scope, msg]
 
   let restNodes = (toSeq(pairs(logJson))) -->
     filter(not ["level", "scope", "ts", "msg"].contains(it[0]))
 
   let restMsg = join(restNodes -->
-    map("$1: $2" % [decorate(it[0], fgCyan), it[1].getStr]), "  ")
+    map("$1: $2" % [
+      color(it[0], cCyan),
+      processFormattedFieldExpectations(it[0], expectations,
+        if it[1].kind == JString: it[1].getStr
+        else: pretty(it[1])),
+      "  "]))
 
-  if restMsg.len + result.len + 2 < terminalWidth():
+  if runeLen(stripAnsi(restMsg)) + runeLen(stripAnsi(result)) + 2 < terminalWidth():
     result &= "  " & restMsg
   else:
     var line = "  "
     for (key, val) in restNodes:
-      let field = "$1: $2" % [decorate(key, fgCyan), val.getStr]
+      let fieldVal = processFormattedFieldExpectations(key, expectations,
-      if line.len + field.len + 2 > terminalWidth():
+        if val.kind == JString: val.getStr
-        result &= "\n  " & line
+        else: pretty(val))
+      let field = "$1: $2" % [color(key, cCyan), fieldVal]
+      if runeLen(stripAnsi(line)) + runeLen(stripAnsi(field)) + 2 > terminalWidth():
+        result &= "\p  " & line
         line = "  "
       line &= field & "  "
-    result &= "\n  " & line
+    result &= "\p  " & line
+
+
+func formatExpectationLabels(expectations: seq[Expectation]): string =
+  let maxLabelLen = expectations.mapIt(runeLen(it.label)).max(system.cmp)
+  let maxCountLen = runeLen($expectations.mapIt(it.count).max(system.cmp))
+  let expectationLabels = expectations.map(proc (exp: Expectation): string =
+    let label = alignLeft(exp.label, maxLabelLen)
+    let count = alignLeft($exp.count, maxCountLen)
+
+    if exp.expected:
+      result = label & " - "
+    else:
+      result = exp.termStyleCode & label & RESET_FORMATTING & " - "
+
+    if (exp.expected and exp.count == 0) or
+       (not exp.expected and exp.count > 0):
+      result &= termFmt(count, fg = cBrightRed, bg = cDefault, style = {tsBold})
+    else:
+      result &= count
+  )
+
+  return "[ " & expectationLabels.join(" | ") & " ]"
 
 proc parseLogLine(logLine: string): JsonNode =
   result = parseJson(logLine)
 
+
+proc parseExpectations(args: Table[string, docopt.Value]): seq[Expectation] =
+  result = @[]
+
+  for isExpected in [false, true]:
+    let argName = if isExpected: "--expected" else: "--unexpected"
+    if args[argName]:
+      let exptArgs = split($args[argName], ';')
+      for ea in exptArgs:
+        let parts = split(ea, ':', 2)
+        result.add(Expectation(
+          fieldName: parts[0],
+          pattern: re(parts[1]),
+          termStyleCode: EXPECTATION_STYLES[result.len mod EXPECTATION_STYLES.len],
+          label:
+            if parts.len > 2: parts[2]
+            else: ea,
+          expected: isExpected,
+          count: 0))
+
+  if args["--config"]:
+    let filename = $args["--config"]
+    if not fileExists(filename):
+      stderr.writeLine("slfmt - WARN: " &
+        filename & " does not exist, ignoring")
+
+    try:
+      let cfg: JsonNode = parseFile(filename)
+      for expJson in cfg.getElems:
+        var exp = Expectation(
+          fieldName: expJson.getOrFail("fieldName").getStr,
+          pattern: re(expJson.getOrFail("pattern").getStr),
+          expected: expJson.getOrFail("expected").getBool,
+          termStyleCode: EXPECTATION_STYLES[result.len mod EXPECTATION_STYLES.len],
+          label:
+            if expJson.contains("label") and expJson["label"].kind == JString:
+              expJson["label"].getStr
+            else:
+              expJson.getOrFail("fieldName").getStr & ":" &
+              expJson.getOrFail("pattern").getStr)
+
+        if expJson.contains("style"):
+          let styleJson = expJson["style"]
+          let fmtStyles: set[TerminalStyle] = styleJson
+            .getOrFail("style")
+            .getElems
+            .mapIt(parseEnum[TerminalStyle]("ts" & it.getStr))
+            .toSet
+
+
+          let fgJson = styleJson.getOrFail("fg")
+          let bgJson = styleJson.getOrFail("bg")
+
+          if fgJson.kind == JString and bgJson.kind == JString:
+            # simple case, both fg and bg are TerminalColors
+            exp.termStyleCode = ansiEscSeq(
+              fg = parseEnum[TerminalColors]("c" & fgJson.getStr),
+              bg = parseEnum[TerminalColors]("c" & bgJson.getStr),
+              fmtStyles)
+
+          else:
+            exp.termStyleCode = ansiEscSeq(fmtStyles)
+
+            case fgJson.kind
+            of JString:
+              exp.termStyleCode &= ansiEscSeq(
+                fg = parseEnum[TerminalColors]("c" & fgJson.getStr))
+            of JInt: exp.termStyleCode &= ansiEscSeq(fg = fgJson.getInt)
+            of JObject:
+              exp.termStyleCode &= ansiEscSeq(fg = (
+                r: fgJson.getOrFail("r").getInt,
+                g: fgJson.getOrFail("g").getInt,
+                b: fgJson.getOrFail("b").getInt))
+            else:
+              raise newException(ValueError,
+                "The 'fg' field must be a string, number, or object.")
+
+            case bgJson.kind
+            of JString:
+              exp.termStyleCode &= ansiEscSeq(
+                bg = parseEnum[TerminalColors]("c" & bgJson.getStr))
+            of JInt: exp.termStyleCode &= ansiEscSeq(bg = bgJson.getInt)
+            of JObject:
+              exp.termStyleCode &= ansiEscSeq(bg = (
+                r: bgJson.getOrFail("r").getInt,
+                g: bgJson.getOrFail("g").getInt,
+                b: bgJson.getOrFail("b").getInt))
+            else:
+              raise newException(ValueError,
+                "The 'bg' field must be a string, number, or object.")
+
+        result.add(exp)
+
+    except:
+      stderr.writeLine("slfmt - WARN: unable to parse config file, ignoring.")
+      stderr.writeLine("slfmt - DEBUG: " & getCurrentExceptionMsg())
+
+
+proc eraseAndWriteLine(f: File, s: string) =
+  f.writeline(eraseLine(emAll) & s)
+
+
 when isMainModule:
+
   try:
     let args = docopt(USAGE, version = VERSION)
 
@@ -129,9 +412,20 @@ when isMainModule:
 
     let compact = args["--compact"]
 
+    let expectations = parseExpectations(args)
+
+    signal(SIGINT, handleSignal)
+    signal(SIGTERM, handleSignal)
+
+    stdout.hideCursor
+    stdout.writeLine("") # burn a line
     var line: string = ""
-    let sin = newFileStream(stdin)
+    while stdin.readLine(line) and not stopFlag.load():
-    while(sin.readLine(line)):
+      if expectations.len > 0:
+        stdout.eraseLine
+        stdout.cursorUp
+        stdout.eraseLine
+
       try:
         let logJson = parseLogLine(line)
         if logJson.kind != JObject:
@@ -144,11 +438,27 @@ when isMainModule:
         if namespace.isSome and logJson.hasKey("scope"):
           if not logJson["scope"].getStr.startsWith(namespace.get): continue
 
-        if compact: stdout.writeLine(compactFormat(logJson))
+        if compact: stdout.writeLine(compactFormat(logJson, expectations))
-        else: stdout.writeLine(fullFormat(logJson))
+        else: stdout.writeLine(fullFormat(logJson, expectations))
+
       except ValueError, JsonParsingError:
-        stdout.writeLine(line)
+        stdout.writeLine(processFormattedFieldExpectations(expectations, line))
+
+      finally:
+        if expectations.len > 0:
+          let labelsMsg = formatExpectationLabels(expectations)
+          stdout.writeLine(termFmt(repeat("─", terminalWidth()),
+            fg = cDefault, bg = cDefault, style = {tsDim}))
+          stdout.write(labelsMsg)
+          stdout.cursorBackward(len(stripAnsi(labelsMsg)))
+          stdout.flushFile
+
   except:
     stderr.writeLine("slfmt - FATAL: " & getCurrentExceptionMsg())
     stderr.writeLine(getCurrentException().getStackTrace())
     quit(QuitFailure)
+
+  finally:
+    stdout.writeLine(RESET_FORMATTING)
+    stdout.write(eraseLine(emAll))
+    stdout.showCursor