Define explicit ownership for custom appenders and formatters

Add SharedLogState for synchronized value data and factories for
independently owned appender state and captured formatter closures.
Reject unsafe configuration state, update the test collector to value
snapshots, and document the breaking APIs in version 3.0.0.

Expand ORC ownership and sanitizer coverage and reject stale test
executables.

src/namespaced_logging.nim

@@ -2,6 +2,8 @@ import std/[algorithm, atomics, exitprocs, json, locks, options, os, paths,
             sequtils, strutils, tables, times]
 import timeutils
 import std/logging as stdlog
+import namespaced_logging/ownership
+export ownership
 
 from std/logging import Level
 export Level
@@ -75,7 +77,20 @@ type
     additionalData*: JsonNode
 
 
-  LogMessageFormatter* = proc (msg: LogMessage): string {.gcsafe.}
+  StatefulLogFormatterFunc* = proc (msg: LogMessage): string {.gcsafe.}
+
+  StatelessLogFormatterFunc* = proc (msg: LogMessage): string {.gcsafe, nimcall.}
+
+  LogFormatter* = ref object of RootObj
+    ## Thread-confined formatter. clone must create independently owned state.
+
+  StatelessLogFormatterImpl = ref object of LogFormatter
+    callback: StatelessLogFormatterFunc
+
+  FactoryLogFormatterImpl[S] = ref object of LogFormatter
+    seed: S
+    factory: proc(seed: S): StatefulLogFormatterFunc {.gcsafe, nimcall.}
+    local: StatefulLogFormatterFunc
 
   ConsoleMessage = object
     msg: string
@@ -98,7 +113,7 @@ type
     ## Log appender that writes log messages to the console. See
     ## *initConsoleLogAppender* for a convenient way to create instances of
     ## this appender.
-    formatter*: LogMessageFormatter
+    formatter*: LogFormatter
       ## Formatter allows for custom formatting of log messages. The default
       ## formatter uses *formatSimpleTextLog* which just writes the log message.
       ## *formatJsonStructuredLog* is also provided to support writing logs out
@@ -106,14 +121,22 @@ type
     useStderr*: bool
 
   CustomLogAppenderFunc*[T] =
-    proc(state: T, msg: LogMessage) {.gcsafe, noconv.}
+    proc(state: var T, msg: LogMessage) {.gcsafe, noconv.}
 
   CustomLogAppender*[T] = ref object of LogAppender
     doLogMessage*: CustomLogAppenderFunc[T]
     state*: T
 
+  FactoryCustomLogAppender*[S, T] = ref object of LogAppender
+    ## Creates state lazily in the thread performing the first append.
+    seed: S
+    factory: proc(seed: S): T {.gcsafe, nimcall.}
+    doLogMessage: CustomLogAppenderFunc[T]
+    local: T
+    initialized: bool
+
   FileLogAppender* = ref object of LogAppender
-    formatter*: LogMessageFormatter
+    formatter*: LogFormatter
     absPath*: Path
 
   StdLoggingAppender* = ref object of LogAppender
@@ -127,7 +150,7 @@ type
       ## when true, only forward to std/logging where there are no appenders
       ## configured on the related LogService
 
-    formatter*: LogMessageFormatter
+    formatter*: LogFormatter
 
 const UninitializedConfigVersion = low(int)
 
@@ -698,16 +721,62 @@ proc initThreadedFileLogging() =
     fileLogging.initialized.store(true)
 
 
+# -----------------------------------------------------------------------------
+# LogFormatter Implementation
+# -----------------------------------------------------------------------------
+
+method clone*(formatter: LogFormatter): LogFormatter {.base, gcsafe.} =
+  raise newException(CatchableError, "missing formatter clone implementation")
+
+method format*(formatter: LogFormatter, msg: LogMessage): string {.base, gcsafe.} =
+  raise newException(CatchableError, "missing formatter implementation")
+
+method clone(formatter: StatelessLogFormatterImpl): LogFormatter {.gcsafe.} =
+  StatelessLogFormatterImpl(callback: formatter.callback)
+
+method format(formatter: StatelessLogFormatterImpl, msg: LogMessage): string {.gcsafe.} =
+  formatter.callback(msg)
+
+proc initLogFormatter*(callback: StatelessLogFormatterFunc): LogFormatter =
+  if callback.isNil:
+    raise newException(ValueError, "Log formatter callback is nil")
+  StatelessLogFormatterImpl(callback: callback)
+
+converter toLogFormatter*(callback: StatelessLogFormatterFunc): LogFormatter =
+  ## Noncapturing formatter functions remain convenient constructor arguments.
+  initLogFormatter(callback)
+
+method clone[S](formatter: FactoryLogFormatterImpl[S]): LogFormatter {.gcsafe.} =
+  FactoryLogFormatterImpl[S](seed: formatter.seed, factory: formatter.factory)
+
+method format[S](formatter: FactoryLogFormatterImpl[S], msg: LogMessage): string {.gcsafe.} =
+  if formatter.local.isNil:
+    formatter.local = formatter.factory(formatter.seed)
+    if formatter.local.isNil:
+      raise newException(ValueError, "Log formatter factory returned nil")
+  formatter.local(msg)
+
+proc initLogFormatter*[S](seed: S,
+    factory: proc(seed: S): StatefulLogFormatterFunc {.gcsafe, nimcall.}): LogFormatter =
+  ## The seed is copied by value. The noncapturing factory creates a closure
+  ## separately for each clone on first use; live closure state is never copied.
+  requireThreadValue(S)
+  if factory.isNil:
+    raise newException(ValueError, "Log formatter factory is nil")
+  FactoryLogFormatterImpl[S](seed: seed, factory: factory)
+
 # -----------------------------------------------------------------------------
 # CustomLogAppender Implementation
 # -----------------------------------------------------------------------------
 
-func initCustomLogAppender*[T](
+proc initCustomLogAppender*[T](
     state: T,
     doLogMessage: CustomLogAppenderFunc[T],
     namespace = "",
     threshold = lvlAll): CustomLogAppender[T] {.gcsafe.} =
-
+  ## Copy value state into each appender clone. For shared mutable state, pass
+  ## a SharedLogState handle. For local refs or closures, use a factory.
+  requireThreadValue(T)
   if doLogMessage.isNil:
     raise newException(ValueError, "initCustomLogAppender: doLogMessage is nil")
 
@@ -718,6 +787,7 @@ func initCustomLogAppender*[T](
     state: state)
 
 method clone*[T](cla: CustomLogAppender[T]): LogAppender {.gcsafe.} =
+  requireThreadValue(T)
   assert not cla.doLogMessage.isNil,
     "CustomLogAppender#clone: source doLogMessage is nil"
 
@@ -741,20 +811,51 @@ method appendLogMessage[T](
       getCurrentException(),
       "unable to append to CustomLogAppender")
 
+proc initFactoryCustomLogAppender*[S, T](seed: S,
+    factory: proc(seed: S): T {.gcsafe, nimcall.},
+    doLogMessage: CustomLogAppenderFunc[T],
+    namespace = "", threshold = lvlAll): FactoryCustomLogAppender[S, T] =
+  ## Only the seed crosses threads. State returned by the factory may contain
+  ## refs and closures, and must be independently owned by the calling thread.
+  requireThreadValue(S)
+  if factory.isNil or doLogMessage.isNil:
+    raise newException(ValueError, "Factory custom appender callbacks must not be nil")
+  FactoryCustomLogAppender[S, T](seed: seed, factory: factory,
+    doLogMessage: doLogMessage, namespace: namespace, threshold: threshold)
+
+method clone[S, T](app: FactoryCustomLogAppender[S, T]): LogAppender {.gcsafe.} =
+  FactoryCustomLogAppender[S, T](seed: app.seed, factory: app.factory,
+    doLogMessage: app.doLogMessage, namespace: app.namespace, threshold: app.threshold)
+
+method appendLogMessage[S, T](app: FactoryCustomLogAppender[S, T],
+    ls: ThreadLocalLogService, msg: LogMessage) {.gcsafe.} =
+  try:
+    if not app.initialized:
+      app.local = app.factory(app.seed)
+      app.initialized = true
+    app.doLogMessage(app.local, msg)
+  except Exception:
+    ls.shared.reportLoggingError(getCurrentException(),
+      "unable to append to FactoryCustomLogAppender")
+
 # -----------------------------------------------------------------------------
 # ConsoleLogAppender Implementation
 # -----------------------------------------------------------------------------
 
 proc initConsoleLogAppender*(
-    formatter = formatSimpleTextLog,
+    formatter: LogFormatter = nil,
     useStderr = false,
     namespace = "",
     threshold = lvlAll): ConsoleLogAppender {.gcsafe.} =
+  ## Initialize a new ConsoleLogAppender. formatter defaults to
+  ## formatSimpleTextLog. formatJsonStructuredLog is another useful formatter
+  ## provided (or you can write your own)
 
   result = ConsoleLogAppender(
     namespace: namespace,
     threshold: threshold,
-    formatter: formatter,
+    formatter: (if formatter.isNil: initLogFormatter(formatSimpleTextLog)
+                else: clone(formatter)),
     useStderr: useStderr)
 
 
@@ -762,7 +863,7 @@ method clone*(cla: ConsoleLogAppender): LogAppender {.gcsafe.} =
   result = ConsoleLogAppender(
     namespace: cla.namespace,
     threshold: cla.threshold,
-    formatter: cla.formatter,
+    formatter: clone(cla.formatter),
     useStderr: cla.useStderr)
 
 
@@ -776,15 +877,15 @@ proc appendLogMessageMultiThreaded(
 
   try:
     consoleLogging.chan.send(ConsoleMessage(
-      msg: cla.formatter(msg),
+      msg: cla.formatter.format(msg),
       useStderr: cla.useStderr))
   except Exception:
     try:
       let output =
         if cla.useStderr: stderr
         else: stdout
-      output.writeLine(cla.formatter(msg))
-      output.writeLine(cla.formatter(LogMessage(
+      output.writeLine(cla.formatter.format(msg))
+      output.writeLine(cla.formatter.format(LogMessage(
         scope: "namespaced_logging",
         level: lvlError,
         err: some(getCurrentException()),
@@ -804,7 +905,7 @@ proc appendLogMessageSingleThreaded(
     let output =
       if cla.useStderr: stderr
       else: stdout
-    output.writeLine(cla.formatter(msg))
+    output.writeLine(cla.formatter.format(msg))
     output.flushFile()
   except Exception: discard
 
@@ -832,14 +933,18 @@ method appendLogMessage(
 
 proc initFileLogAppender*(
     filePath: string,
-    formatter = formatSimpleTextLog,
+    formatter: LogFormatter = nil,
     namespace = "",
     threshold = lvlAll): FileLogAppender {.gcsafe.} =
+  ## Initialize a new FileLogAppender. formatter defaults to
+  ## formatSimpleTextLog. formatJsonStructuredLog is another useful formatter
+  ## provided (or you can write your own)
 
   result = FileLogAppender(
     namespace: namespace,
     threshold: threshold,
-    formatter: formatter,
+    formatter: (if formatter.isNil: initLogFormatter(formatSimpleTextLog)
+                else: clone(formatter)),
     absPath: absolutePath(Path(filePath)))
 
 
@@ -847,7 +952,7 @@ method clone*(fla: FileLogAppender): LogAppender {.gcsafe.} =
   result = FileLogAppender(
     namespace: fla.namespace,
     threshold: fla.threshold,
-    formatter: fla.formatter,
+    formatter: clone(fla.formatter),
     absPath: fla.absPath)
 
 
@@ -861,7 +966,7 @@ proc appendLogMessageMultiThreaded(
 
   try:
     fileLogging.chan.send(FileMessage(
-      msg: fla.formatter(msg),
+      msg: fla.formatter.format(msg),
       absPath: fla.absPath))
   except Exception: discard
 
@@ -874,7 +979,7 @@ proc appendLogMessageSingleThreaded(
   try:
     var f: File
     if not open(f, $fla.absPath, fmAppend): return
-    writeLine(f, fla.formatter(msg))
+    writeLine(f, fla.formatter.format(msg))
     flushFile(f)
     close(f)
   except Exception: discard
@@ -907,9 +1012,9 @@ func formatForwardedLog*(lm: LogMessage): string =
   "[" & lm.scope & "] " & formatSimpleTextLog(lm)
 
 
-func initStdLoggingAppender*(
+proc initStdLoggingAppender*(
     fallbackOnly = true,
-    formatter = formatForwardedLog,
+    formatter: LogFormatter = nil,
     namespace = "",
     threshold = lvlAll): StdLoggingAppender {.gcsafe.} =
 
@@ -917,7 +1022,8 @@ func initStdLoggingAppender*(
     namespace: namespace,
     threshold: threshold,
     fallbackOnly: fallbackOnly,
-    formatter: formatter)
+    formatter: (if formatter.isNil: initLogFormatter(formatForwardedLog)
+                else: clone(formatter)))
 
 
 method clone*(sla: StdLoggingAppender): LogAppender {.gcsafe.} =
@@ -925,7 +1031,7 @@ method clone*(sla: StdLoggingAppender): LogAppender {.gcsafe.} =
     namespace: sla.namespace,
     threshold: sla.threshold,
     fallbackOnly: sla.fallbackOnly,
-    formatter: sla.formatter)
+    formatter: clone(sla.formatter))
 
 
 method appendLogMessage*(
@@ -935,7 +1041,7 @@ method appendLogMessage*(
 
   if sla.fallbackOnly and ls.appenders.len > 1: return
 
-  stdlog.log(msg.level, sla.formatter(msg))
+  stdlog.log(msg.level, sla.formatter.format(msg))
 
 
 # -----------------------------------------------------------------------------