Define explicit ownership for custom appenders and formatters

Add SharedLogState for synchronized value data and factories for
independently owned appender state and captured formatter closures.
Reject unsafe configuration state, update the test collector to value
snapshots, and document the breaking APIs in version 3.0.0.

Expand ORC ownership and sanitizer coverage and reject stale test
executables.

src/namespaced_logging.nim

@@ -2,6 +2,8 @@ import std/[algorithm, atomics, exitprocs, json, locks, options, os, paths,
             sequtils, strutils, tables, times]
 import timeutils
 import std/logging as stdlog
+import namespaced_logging/ownership
+export ownership
 
 from std/logging import Level
 export Level
@@ -75,7 +77,20 @@ type
     additionalData*: JsonNode
 
 
-  LogMessageFormatter* = proc (msg: LogMessage): string {.gcsafe.}
+  StatefulLogFormatterFunc* = proc (msg: LogMessage): string {.gcsafe.}
+
+  StatelessLogFormatterFunc* = proc (msg: LogMessage): string {.gcsafe, nimcall.}
+
+  LogFormatter* = ref object of RootObj
+    ## Thread-confined formatter. clone must create independently owned state.
+
+  StatelessLogFormatterImpl = ref object of LogFormatter
+    callback: StatelessLogFormatterFunc
+
+  FactoryLogFormatterImpl[S] = ref object of LogFormatter
+    seed: S
+    factory: proc(seed: S): StatefulLogFormatterFunc {.gcsafe, nimcall.}
+    local: StatefulLogFormatterFunc
 
   ConsoleMessage = object
     msg: string
@@ -98,7 +113,7 @@ type
     ## Log appender that writes log messages to the console. See
     ## *initConsoleLogAppender* for a convenient way to create instances of
     ## this appender.
-    formatter*: LogMessageFormatter
+    formatter*: LogFormatter
       ## Formatter allows for custom formatting of log messages. The default
       ## formatter uses *formatSimpleTextLog* which just writes the log message.
       ## *formatJsonStructuredLog* is also provided to support writing logs out
@@ -106,14 +121,22 @@ type
     useStderr*: bool
 
   CustomLogAppenderFunc*[T] =
-    proc(state: T, msg: LogMessage) {.gcsafe, noconv.}
+    proc(state: var T, msg: LogMessage) {.gcsafe, noconv.}
 
   CustomLogAppender*[T] = ref object of LogAppender
     doLogMessage*: CustomLogAppenderFunc[T]
     state*: T
 
+  FactoryCustomLogAppender*[S, T] = ref object of LogAppender
+    ## Creates state lazily in the thread performing the first append.
+    seed: S
+    factory: proc(seed: S): T {.gcsafe, nimcall.}
+    doLogMessage: CustomLogAppenderFunc[T]
+    local: T
+    initialized: bool
+
   FileLogAppender* = ref object of LogAppender
-    formatter*: LogMessageFormatter
+    formatter*: LogFormatter
     absPath*: Path
 
   StdLoggingAppender* = ref object of LogAppender
@@ -127,7 +150,7 @@ type
       ## when true, only forward to std/logging where there are no appenders
       ## configured on the related LogService
 
-    formatter*: LogMessageFormatter
+    formatter*: LogFormatter
 
 const UninitializedConfigVersion = low(int)
 
@@ -698,16 +721,62 @@ proc initThreadedFileLogging() =
     fileLogging.initialized.store(true)
 
 
+# -----------------------------------------------------------------------------
+# LogFormatter Implementation
+# -----------------------------------------------------------------------------
+
+method clone*(formatter: LogFormatter): LogFormatter {.base, gcsafe.} =
+  raise newException(CatchableError, "missing formatter clone implementation")
+
+method format*(formatter: LogFormatter, msg: LogMessage): string {.base, gcsafe.} =
+  raise newException(CatchableError, "missing formatter implementation")
+
+method clone(formatter: StatelessLogFormatterImpl): LogFormatter {.gcsafe.} =
+  StatelessLogFormatterImpl(callback: formatter.callback)
+
+method format(formatter: StatelessLogFormatterImpl, msg: LogMessage): string {.gcsafe.} =
+  formatter.callback(msg)
+
+proc initLogFormatter*(callback: StatelessLogFormatterFunc): LogFormatter =
+  if callback.isNil:
+    raise newException(ValueError, "Log formatter callback is nil")
+  StatelessLogFormatterImpl(callback: callback)
+
+converter toLogFormatter*(callback: StatelessLogFormatterFunc): LogFormatter =
+  ## Noncapturing formatter functions remain convenient constructor arguments.
+  initLogFormatter(callback)
+
+method clone[S](formatter: FactoryLogFormatterImpl[S]): LogFormatter {.gcsafe.} =
+  FactoryLogFormatterImpl[S](seed: formatter.seed, factory: formatter.factory)
+
+method format[S](formatter: FactoryLogFormatterImpl[S], msg: LogMessage): string {.gcsafe.} =
+  if formatter.local.isNil:
+    formatter.local = formatter.factory(formatter.seed)
+    if formatter.local.isNil:
+      raise newException(ValueError, "Log formatter factory returned nil")
+  formatter.local(msg)
+
+proc initLogFormatter*[S](seed: S,
+    factory: proc(seed: S): StatefulLogFormatterFunc {.gcsafe, nimcall.}): LogFormatter =
+  ## The seed is copied by value. The noncapturing factory creates a closure
+  ## separately for each clone on first use; live closure state is never copied.
+  requireThreadValue(S)
+  if factory.isNil:
+    raise newException(ValueError, "Log formatter factory is nil")
+  FactoryLogFormatterImpl[S](seed: seed, factory: factory)
+
 # -----------------------------------------------------------------------------
 # CustomLogAppender Implementation
 # -----------------------------------------------------------------------------
 
-func initCustomLogAppender*[T](
+proc initCustomLogAppender*[T](
     state: T,
     doLogMessage: CustomLogAppenderFunc[T],
     namespace = "",
     threshold = lvlAll): CustomLogAppender[T] {.gcsafe.} =
-
+  ## Copy value state into each appender clone. For shared mutable state, pass
+  ## a SharedLogState handle. For local refs or closures, use a factory.
+  requireThreadValue(T)
   if doLogMessage.isNil:
     raise newException(ValueError, "initCustomLogAppender: doLogMessage is nil")
 
@@ -718,6 +787,7 @@ func initCustomLogAppender*[T](
     state: state)
 
 method clone*[T](cla: CustomLogAppender[T]): LogAppender {.gcsafe.} =
+  requireThreadValue(T)
   assert not cla.doLogMessage.isNil,
     "CustomLogAppender#clone: source doLogMessage is nil"
 
@@ -741,20 +811,51 @@ method appendLogMessage[T](
       getCurrentException(),
       "unable to append to CustomLogAppender")
 
+proc initFactoryCustomLogAppender*[S, T](seed: S,
+    factory: proc(seed: S): T {.gcsafe, nimcall.},
+    doLogMessage: CustomLogAppenderFunc[T],
+    namespace = "", threshold = lvlAll): FactoryCustomLogAppender[S, T] =
+  ## Only the seed crosses threads. State returned by the factory may contain
+  ## refs and closures, and must be independently owned by the calling thread.
+  requireThreadValue(S)
+  if factory.isNil or doLogMessage.isNil:
+    raise newException(ValueError, "Factory custom appender callbacks must not be nil")
+  FactoryCustomLogAppender[S, T](seed: seed, factory: factory,
+    doLogMessage: doLogMessage, namespace: namespace, threshold: threshold)
+
+method clone[S, T](app: FactoryCustomLogAppender[S, T]): LogAppender {.gcsafe.} =
+  FactoryCustomLogAppender[S, T](seed: app.seed, factory: app.factory,
+    doLogMessage: app.doLogMessage, namespace: app.namespace, threshold: app.threshold)
+
+method appendLogMessage[S, T](app: FactoryCustomLogAppender[S, T],
+    ls: ThreadLocalLogService, msg: LogMessage) {.gcsafe.} =
+  try:
+    if not app.initialized:
+      app.local = app.factory(app.seed)
+      app.initialized = true
+    app.doLogMessage(app.local, msg)
+  except Exception:
+    ls.shared.reportLoggingError(getCurrentException(),
+      "unable to append to FactoryCustomLogAppender")
+
 # -----------------------------------------------------------------------------
 # ConsoleLogAppender Implementation
 # -----------------------------------------------------------------------------
 
 proc initConsoleLogAppender*(
-    formatter = formatSimpleTextLog,
+    formatter: LogFormatter = nil,
     useStderr = false,
     namespace = "",
     threshold = lvlAll): ConsoleLogAppender {.gcsafe.} =
+  ## Initialize a new ConsoleLogAppender. formatter defaults to
+  ## formatSimpleTextLog. formatJsonStructuredLog is another useful formatter
+  ## provided (or you can write your own)
 
   result = ConsoleLogAppender(
     namespace: namespace,
     threshold: threshold,
-    formatter: formatter,
+    formatter: (if formatter.isNil: initLogFormatter(formatSimpleTextLog)
+                else: clone(formatter)),
     useStderr: useStderr)
 
 
@@ -762,7 +863,7 @@ method clone*(cla: ConsoleLogAppender): LogAppender {.gcsafe.} =
   result = ConsoleLogAppender(
     namespace: cla.namespace,
     threshold: cla.threshold,
-    formatter: cla.formatter,
+    formatter: clone(cla.formatter),
     useStderr: cla.useStderr)
 
 
@@ -776,15 +877,15 @@ proc appendLogMessageMultiThreaded(
 
   try:
     consoleLogging.chan.send(ConsoleMessage(
-      msg: cla.formatter(msg),
+      msg: cla.formatter.format(msg),
       useStderr: cla.useStderr))
   except Exception:
     try:
       let output =
         if cla.useStderr: stderr
         else: stdout
-      output.writeLine(cla.formatter(msg))
-      output.writeLine(cla.formatter(LogMessage(
+      output.writeLine(cla.formatter.format(msg))
+      output.writeLine(cla.formatter.format(LogMessage(
         scope: "namespaced_logging",
         level: lvlError,
         err: some(getCurrentException()),
@@ -804,7 +905,7 @@ proc appendLogMessageSingleThreaded(
     let output =
       if cla.useStderr: stderr
       else: stdout
-    output.writeLine(cla.formatter(msg))
+    output.writeLine(cla.formatter.format(msg))
     output.flushFile()
   except Exception: discard
 
@@ -832,14 +933,18 @@ method appendLogMessage(
 
 proc initFileLogAppender*(
     filePath: string,
-    formatter = formatSimpleTextLog,
+    formatter: LogFormatter = nil,
     namespace = "",
     threshold = lvlAll): FileLogAppender {.gcsafe.} =
+  ## Initialize a new FileLogAppender. formatter defaults to
+  ## formatSimpleTextLog. formatJsonStructuredLog is another useful formatter
+  ## provided (or you can write your own)
 
   result = FileLogAppender(
     namespace: namespace,
     threshold: threshold,
-    formatter: formatter,
+    formatter: (if formatter.isNil: initLogFormatter(formatSimpleTextLog)
+                else: clone(formatter)),
     absPath: absolutePath(Path(filePath)))
 
 
@@ -847,7 +952,7 @@ method clone*(fla: FileLogAppender): LogAppender {.gcsafe.} =
   result = FileLogAppender(
     namespace: fla.namespace,
     threshold: fla.threshold,
-    formatter: fla.formatter,
+    formatter: clone(fla.formatter),
     absPath: fla.absPath)
 
 
@@ -861,7 +966,7 @@ proc appendLogMessageMultiThreaded(
 
   try:
     fileLogging.chan.send(FileMessage(
-      msg: fla.formatter(msg),
+      msg: fla.formatter.format(msg),
       absPath: fla.absPath))
   except Exception: discard
 
@@ -874,7 +979,7 @@ proc appendLogMessageSingleThreaded(
   try:
     var f: File
     if not open(f, $fla.absPath, fmAppend): return
-    writeLine(f, fla.formatter(msg))
+    writeLine(f, fla.formatter.format(msg))
     flushFile(f)
     close(f)
   except Exception: discard
@@ -907,9 +1012,9 @@ func formatForwardedLog*(lm: LogMessage): string =
   "[" & lm.scope & "] " & formatSimpleTextLog(lm)
 
 
-func initStdLoggingAppender*(
+proc initStdLoggingAppender*(
     fallbackOnly = true,
-    formatter = formatForwardedLog,
+    formatter: LogFormatter = nil,
     namespace = "",
     threshold = lvlAll): StdLoggingAppender {.gcsafe.} =
 
@@ -917,7 +1022,8 @@ func initStdLoggingAppender*(
     namespace: namespace,
     threshold: threshold,
     fallbackOnly: fallbackOnly,
-    formatter: formatter)
+    formatter: (if formatter.isNil: initLogFormatter(formatForwardedLog)
+                else: clone(formatter)))
 
 
 method clone*(sla: StdLoggingAppender): LogAppender {.gcsafe.} =
@@ -925,7 +1031,7 @@ method clone*(sla: StdLoggingAppender): LogAppender {.gcsafe.} =
     namespace: sla.namespace,
     threshold: sla.threshold,
     fallbackOnly: sla.fallbackOnly,
-    formatter: sla.formatter)
+    formatter: clone(sla.formatter))
 
 
 method appendLogMessage*(
@@ -935,7 +1041,7 @@ method appendLogMessage*(
 
   if sla.fallbackOnly and ls.appenders.len > 1: return
 
-  stdlog.log(msg.level, sla.formatter(msg))
+  stdlog.log(msg.level, sla.formatter.format(msg))
 
 
 # -----------------------------------------------------------------------------

src/namespaced_logging/ownership.nim

@@ -0,0 +1,93 @@
+## Ownership for custom logging state shared between threads under ORC.
+import std/[atomics, locks, typetraits]
+
+proc checkThreadValue[T]() =
+  when T is Lock or T is Cond or T is Atomic:
+    {.error: "Do not copy synchronization primitives as logging configuration".}
+  elif T is ref:
+    {.error: "Logging configuration cannot contain ref values; use a thread-local factory or SharedLogState with value data".}
+  elif T is string:
+    discard
+  elif T is seq:
+    checkThreadValue[typeof(default(T)[0])]()
+  elif T is array:
+    checkThreadValue[typeof(default(T)[low(T)])]()
+  elif T is object or T is tuple:
+    for field in fields(default(T)):
+      checkThreadValue[typeof(field)]()
+  elif supportsCopyMem(T):
+    discard
+  else:
+    {.error: "Logging configuration cannot contain closures; create them with a thread-local factory".}
+
+template requireThreadValue*(T: typedesc) =
+  ## Reject managed references and closures, including nested ones. Strings
+  ## and sequences have value semantics under ORC. Raw pointers and custom
+  ## copy hooks still require the caller's lifetime and synchronization contract.
+  ## Type-check only: inspecting fields must not construct a payload at runtime.
+  when not compiles(block:
+    checkThreadValue[T]()
+  ):
+    {.error: "Logging configuration must be value data without refs, closures, or synchronization primitives; use SharedLogState or a thread-local factory".}
+
+type
+  SharedLogStateData[T] = object
+    owners: Atomic[int]
+    lock: Lock
+    value: T
+
+  SharedLogState*[T] = object
+    ## Copyable owner of synchronized value data. Each thread owns its own
+    ## handle. The payload cannot contain ordinary refs or captured closures.
+    data: ptr SharedLogStateData[T]
+
+proc release[T](data: ptr SharedLogStateData[T]) =
+  if data != nil and data.owners.fetchSub(1, moAcquireRelease) == 1:
+    `=destroy`(data.value)
+    deinitLock(data.lock)
+    deallocShared(data)
+
+proc `=destroy`*[T](state: SharedLogState[T]) = release(state.data)
+
+proc `=copy`*[T](dest: var SharedLogState[T], source: SharedLogState[T]) =
+  if dest.data == source.data: return
+  if source.data != nil:
+    discard source.data.owners.fetchAdd(1, moRelaxed)
+  release(dest.data)
+  dest.data = source.data
+
+proc `=dup`*[T](source: SharedLogState[T]): SharedLogState[T] =
+  if source.data != nil:
+    discard source.data.owners.fetchAdd(1, moRelaxed)
+  result.data = source.data
+
+proc `=wasMoved`*[T](state: var SharedLogState[T]) = state.data = nil
+
+proc initSharedLogState*[T](value: sink T): SharedLogState[T] =
+  requireThreadValue(T)
+  result.data = cast[ptr SharedLogStateData[T]](
+    allocShared0(sizeof(SharedLogStateData[T])))
+  result.data.owners.store(1)
+  initLock(result.data.lock)
+  result.data.value = value
+
+proc withState*[T](state: SharedLogState[T],
+    action: proc(value: var T) {.gcsafe.}) =
+  ## Execute synchronously under the state's lock. Do not retain a pointer to
+  ## value or re-enter this same state from action. Exceptions release the lock.
+  if state.data == nil:
+    raise newException(ValueError, "SharedLogState is not initialized")
+  if action.isNil:
+    raise newException(ValueError, "SharedLogState callback is nil")
+  # The callback can release its caller's handle. Retain an explicit owner
+  # until after the lock is released; a borrowed parameter is insufficient.
+  let owner = `=dup`(state)
+  withLock owner.data.lock:
+    action(owner.data.value)
+
+proc snapshot*[T](state: SharedLogState[T]): T =
+  ## Return an independently owned value copy, made under the state's lock.
+  if state.data == nil:
+    raise newException(ValueError, "SharedLogState is not initialized")
+  withLock state.data.lock:
+    result = state.data.value

src/namespaced_logging/testutil.nim

@@ -1,42 +1,51 @@
-import std/[locks, sequtils, syncio, os, times]
+import std/[json, options, syncio, os, times]
 from logging import Level
 from ../namespaced_logging import CustomLogAppender, initCustomLogAppender, LogMessage
+import ./ownership
 
 type
-  LoggedMessages* = ref object
-    messages*: seq[LogMessage]
-    lock: Lock
+  RecordedException* = object
+    name*, msg*, stacktrace*: string
+
+  RecordedLogMessage* = object
+    ## A value snapshot: JSON is serialized, exceptions retain diagnostic text,
+    ## and timestamps retain the instant without sharing a Timezone reference.
+    scope*: string
+    level*: Level
+    ts*: Time
+    msg*: string
+    err*: Option[RecordedException]
+    additionalData*: string
+
+  LoggedMessages* = SharedLogState[seq[RecordedLogMessage]]
 
 proc initLoggedMessages*(): LoggedMessages =
-  result = LoggedMessages(messages: @[])
-  initLock(result.lock)
-
+  initSharedLogState(newSeq[RecordedLogMessage]())
 
 proc add*(lm: LoggedMessages, msg: LogMessage) =
-  withLock lm.lock: lm.messages.add(msg)
-
+  var record = RecordedLogMessage(scope: msg.scope, level: msg.level,
+    ts: msg.ts.toTime, msg: msg.msg,
+    additionalData: (if msg.additionalData.isNil: "null" else: $msg.additionalData))
+  if msg.err.isSome:
+    let err = msg.err.get
+    record.err = some(RecordedException(name: $err.name, msg: err.msg,
+      stacktrace: $err.trace))
+  lm.withState(proc(messages: var seq[RecordedLogMessage]) = messages.add(record))
 
 proc clear*(lm: LoggedMessages) =
-  withLock lm.lock: lm.messages = @[]
+  lm.withState(proc(messages: var seq[RecordedLogMessage]) = messages.setLen(0))
 
+proc get*(lm: LoggedMessages): seq[RecordedLogMessage] = lm.snapshot()
 
-proc get*(lm: LoggedMessages): seq[LogMessage] =
-  withLock lm.lock: return lm.messages.mapIt(it)
-
-
-proc testLogAppenderProc(state: LoggedMessages, msg: LogMessage) {.gcsafe, noconv.} =
+proc testLogAppenderProc(state: var LoggedMessages, msg: LogMessage) {.gcsafe, noconv.} =
   state.add(msg)
 
 proc initTestLogAppender*(
     lm: LoggedMessages,
     namespace = "",
     threshold = lvlAll): CustomLogAppender[LoggedMessages] =
-
-  initCustomLogAppender(
-    state = lm,
-    doLogMessage = testLogAppenderProc,
-    namespace = namespace,
-    threshold = threshold)
+  initCustomLogAppender(state = lm, doLogMessage = testLogAppenderProc,
+    namespace = namespace, threshold = threshold)
 
 
 proc waitForFileContent*(