Use namespaced_logging 1.x for logging (optionally).

- Nim 2.x has moved the DB connectors outside the standard library to
  the `db_connector` package.
- Refactor the pooling implementation and macro expectations to use the
  `withConnection` name instead of `withConn`. This change allows a
  caller to use a [waterpark](https://github.com/guzba/waterpark) pool
  instance instead of the builtin pool instance. Waterpark provides
  better support for multi-threaded environments. The builtin pooling
  mechanism may be deprecated in favor of waterpark in the future.
- Add the `getModelIfItExists` generated proc to the list of standard
  procs we generate. This is a flavour of `getModel` that returns an
  `Option` instead of raising an exception when there is no model for
  the given id.
- Change `PaginationParams#orderBy` to accept a `seq[string]` to allow
  for sorting on multiple fields.

README.rst

@@ -57,7 +57,7 @@ Models may be defined as:
 
 .. code-block:: Nim
    # models.nim
-   import std/options, std/times
+   import std/[options, times]
    import uuids
 
    type
@@ -82,6 +82,8 @@ Using Fiber ORM we can generate a data access layer with:
 
 .. code-block:: Nim
    # db.nim
+   import std/[options]
+   import db_connectors/db_postgres
    import fiber_orm
    import ./models.nim
 
@@ -102,6 +104,7 @@ This will generate the following procedures:
 
 .. code-block:: Nim
    proc getTodoItem*(db: TodoDB, id: UUID): TodoItem;
+   proc getTodoItemIfItExists*(db: TodoDB, id: UUID): Option[TodoItem];
    proc getAllTodoItems*(db: TodoDB): seq[TodoItem];
    proc createTodoItem*(db: TodoDB, rec: TodoItem): TodoItem;
    proc updateTodoItem*(db: TodoDB, rec: TodoItem): bool;
@@ -112,6 +115,7 @@ This will generate the following procedures:
      values: varargs[string, dbFormat]): seq[TodoItem];
 
    proc getTimeEntry*(db: TodoDB, id: UUID): TimeEntry;
+   proc getTimeEntryIfItExists*(db: TodoDB, id: UUID): Option[TimeEntry];
    proc getAllTimeEntries*(db: TodoDB): seq[TimeEntry];
    proc createTimeEntry*(db: TodoDB, rec: TimeEntry): TimeEntry;
    proc updateTimeEntry*(db: TodoDB, rec: TimeEntry): bool;

fiber_orm.nimble

@@ -1,6 +1,6 @@
 # Package
 
-version       = "2.1.1"
+version       = "3.0.0"
 author        = "Jonathan Bernard"
 description   = "Lightweight Postgres ORM for Nim."
 license       = "GPL-3.0"
@@ -11,4 +11,4 @@ srcDir        = "src"
 # Dependencies
 
 requires @["nim >= 1.4.0", "uuids"]
-requires "namespaced_logging >= 0.3.0"
+requires "namespaced_logging >= 1.0.0"

src/fiber_orm.nim

@@ -1,6 +1,6 @@
 # Fiber ORM
 #
-# Copyright 2019 Jonathan Bernard <jonathan@jdbernard.com>
+# Copyright 2019-2024 Jonathan Bernard <jonathan@jdbernard.com>
 
 ## Lightweight ORM supporting the `Postgres`_ and `SQLite`_ databases in Nim.
 ## It supports a simple, opinionated model mapper to generate SQL queries based
@@ -107,6 +107,7 @@
 ##
 ##    proc createTodoItem*(db: TodoDB, rec: TodoItem): TodoItem;
 ##    proc updateTodoItem*(db: TodoDB, rec: TodoItem): bool;
+##    proc createOrUpdateTodoItem*(db: TodoDB, rec: TodoItem): bool;
 ##    proc deleteTodoItem*(db: TodoDB, rec: TodoItem): bool;
 ##    proc deleteTodoItem*(db: TodoDB, id: UUID): bool;
 ##
@@ -263,26 +264,28 @@
 ## In the example above the `pool.DbConnPool`_ object is used as database
 ## object type (aliased as `TodoDB`). This is the intended usage pattern, but
 ## anything can be passed as the database object type so long as there is a
-## defined `withConn` template that provides an injected `conn: DbConn` object
+## defined `withConnection` template that provides a `conn: DbConn` object
 ## to the provided statement body.
 ##
 ## For example, a valid database object implementation that opens a new
 ## connection for every request might look like this:
 ##
 ## .. code-block:: Nim
-##    import std/db_postgres
+##    import db_connector/db_postgres
 ##
 ##    type TodoDB* = object
 ##      connString: string
 ##
-##    template withConn*(db: TodoDB, stmt: untyped): untyped =
-##      let conn {.inject.} = open("", "", "", db.connString)
-##      try: stmt
-##      finally: close(conn)
+##    template withConnection*(db: TodoDB, stmt: untyped): untyped =
+##      block:
+##        let conn = open("", "", "", db.connString)
+##        try: stmt
+##        finally: close(conn)
 ##
 ## .. _pool.DbConnPool: fiber_orm/pool.html#DbConnPool
 ##
-import std/[db_common, logging, macros, options, sequtils, strutils]
+import std/[json, macros, options, sequtils, strutils]
+import db_connector/db_common
 import namespaced_logging, uuids
 
 from std/unicode import capitalize
@@ -305,21 +308,31 @@ type
   PaginationParams* = object
     pageSize*: int
     offset*: int
-    orderBy*: Option[string]
+    orderBy*: Option[seq[string]]
 
   PagedRecords*[T] = object
     pagination*: Option[PaginationParams]
     records*: seq[T]
     totalRecords*: int
 
+  DbUpdateError* = object of CatchableError ##\
+    ## Error types raised when a DB modification fails.
+
   NotFoundError* = object of CatchableError ##\
     ## Error type raised when no record matches a given ID
 
-var logNs {.threadvar.}: LoggingNamespace
+var logService {.threadvar.}: Option[LogService]
 
-template log(): untyped =
-  if logNs.isNil: logNs = getLoggerForNamespace(namespace = "fiber_orm", level = lvlNotice)
-  logNs
+proc logQuery(methodName: string, sqlStmt: string, args: openArray[(string, string)] = []) =
+  # namespaced_logging would do this check for us, but we don't want to even
+  # build the log object if we're not actually logging
+  if logService.isNone: return
+  var log = %*{ "method": methodName, "sql": sqlStmt }
+  for (k, v) in args: log[k] = %v
+  logService.getLogger("fiber_orm/query").debug(log)
+
+proc enableDbLogging*(svc: LogService) =
+  logService = some(svc)
 
 proc newMutateClauses(): MutateClauses =
   return MutateClauses(
@@ -345,7 +358,7 @@ proc createRecord*[D: DbConnType, T](db: D, rec: T): T =
     " VALUES (" & mc.placeholders.join(",") & ") " &
     " RETURNING " & columnNamesForModel(rec).join(",")
 
-  log().debug "createRecord: [" & sqlStmt & "]"
+  logQuery("createRecord", sqlStmt)
   let newRow = db.getRow(sql(sqlStmt), mc.values)
 
   result = rowToModel(T, newRow)
@@ -361,15 +374,33 @@ proc updateRecord*[D: DbConnType, T](db: D, rec: T): bool =
     " SET " & setClause &
     " WHERE id = ? "
 
-  log().debug "updateRecord: [" & sqlStmt & "] id: " & $rec.id
+  logQuery("updateRecord", sqlStmt, [("id", $rec.id)])
   let numRowsUpdated = db.execAffectedRows(sql(sqlStmt), mc.values.concat(@[$rec.id]))
 
   return numRowsUpdated > 0;
 
+proc createOrUpdateRecord*[D: DbConnType, T](db: D, rec: T): T =
+  ## Create or update a record. `rec` is expected to be a `model class`_. If
+  ## the `id` field is unset, or if there is no existing record with the given
+  ## id, a new record is inserted. Otherwise, the existing record is updated.
+  ##
+  ## Note that this does not perform partial updates, all fields are updated.
+
+  let findRecordStmt = "SELECT id FROM " & tableName(rec) & " WHERE id = ?"
+  logQuery("createOrUpdateRecord", findRecordStmt, [("id", $rec.id)])
+  let rows = db.getAllRows(sql(findRecordStmt), [$rec.id])
+
+  if rows.len == 0: result = createRecord(db, rec)
+  else:
+    result = rec
+    if not updateRecord(db, rec):
+      raise newException(DbUpdateError,
+        "unable to update " & modelName(rec) & " for id " & $rec.id)
+
 template deleteRecord*[D: DbConnType](db: D, modelType: type, id: typed): untyped =
   ## Delete a record by id.
   let sqlStmt = "DELETE FROM " & tableName(modelType) & " WHERE id = ?"
-  log().debug "deleteRecord: [" & sqlStmt & "] id: " & $id
+  logQuery("deleteRecord", sqlStmt, [("id", $id)])
   db.tryExec(sql(sqlStmt), $id)
 
 proc deleteRecord*[D: DbConnType, T](db: D, rec: T): bool =
@@ -377,7 +408,7 @@ proc deleteRecord*[D: DbConnType, T](db: D, rec: T): bool =
   ##
   ## .. _id: #model-class-id-field
   let sqlStmt = "DELETE FROM " & tableName(rec) & " WHERE id = ?"
-  log().debug "deleteRecord: [" & sqlStmt & "] id: " & $rec.id
+  logQuery("deleteRecord", sqlStmt, [("id", $rec.id)])
   return db.tryExec(sql(sqlStmt), $rec.id)
 
 template getRecord*[D: DbConnType](db: D, modelType: type, id: typed): untyped =
@@ -387,7 +418,7 @@ template getRecord*[D: DbConnType](db: D, modelType: type, id: typed): untyped =
     " FROM " & tableName(modelType) &
     " WHERE id = ?"
 
-  log().debug "getRecord: [" & sqlStmt & "] id: " & $id
+  logQuery("getRecord", sqlStmt, [("id", $id)])
   let row = db.getRow(sql(sqlStmt), @[$id])
 
   if allIt(row, it.len == 0):
@@ -416,14 +447,15 @@ template findRecordsWhere*[D: DbConnType](
   if page.isSome:
     let p = page.get
     if p.orderBy.isSome:
-      fetchStmt &= " ORDER BY " & p.orderBy.get
+      let orderByClause = p.orderBy.get.map(identNameToDb).join(",")
+      fetchStmt &= " ORDER BY " & orderByClause
     else:
       fetchStmt &= " ORDER BY id"
 
     fetchStmt &= " LIMIT " & $p.pageSize &
                  " OFFSET " & $p.offset
 
-  log().debug "findRecordsWhere: [" & fetchStmt & "] values: (" & values.join(", ") & ")"
+  logQuery("findRecordsWhere", fetchStmt, [("values", values.join(", "))])
   let records = db.getAllRows(sql(fetchStmt), values).mapIt(rowToModel(modelType, it))
 
   PagedRecords[modelType](
@@ -447,14 +479,15 @@ template getAllRecords*[D: DbConnType](
   if page.isSome:
     let p = page.get
     if p.orderBy.isSome:
-      fetchStmt &= " ORDER BY " & p.orderBy.get
+      let orderByClause = p.orderBy.get.map(identNameToDb).join(",")
+      fetchStmt &= " ORDER BY " & orderByClause
     else:
       fetchStmt &= " ORDER BY id"
 
     fetchStmt &= " LIMIT " & $p.pageSize &
                  " OFFSET " & $p.offset
 
-  log().debug "getAllRecords: [" & fetchStmt & "]"
+  logQuery("getAllRecords", fetchStmt)
   let records = db.getAllRows(sql(fetchStmt)).mapIt(rowToModel(modelType, it))
 
   PagedRecords[modelType](
@@ -486,14 +519,15 @@ template findRecordsBy*[D: DbConnType](
   if page.isSome:
     let p = page.get
     if p.orderBy.isSome:
-      fetchStmt &= " ORDER BY " & p.orderBy.get
+      let orderByClause = p.orderBy.get.map(identNameToDb).join(",")
+      fetchStmt &= " ORDER BY " & orderByClause
     else:
       fetchStmt &= " ORDER BY id"
 
     fetchStmt &= " LIMIT " & $p.pageSize &
                  " OFFSET " & $p.offset
 
-  log().debug "findRecordsBy: [" & fetchStmt & "] values (" & values.join(", ") & ")"
+  logQuery("findRecordsBy", fetchStmt, [("values", values.join(", "))])
   let records = db.getAllRows(sql(fetchStmt), values).mapIt(rowToModel(modelType, it))
 
   PagedRecords[modelType](
@@ -516,11 +550,12 @@ macro generateProcsForModels*(dbType: type, modelTypes: openarray[type]): untype
   ##    proc deleteTodoItem*(db: TodoDB, rec: TodoItem): bool;
   ##    proc deleteTodoItem*(db: TodoDB, id: idType): bool;
   ##    proc updateTodoItem*(db: TodoDB, rec: TodoItem): bool;
+  ##    proc createOrUpdateTodoItem*(db: TodoDB, rec: TodoItem): bool;
   ##
   ##    proc findTodoItemsWhere*(
   ##      db: TodoDB, whereClause: string, values: varargs[string]): TodoItem;
   ##
-  ## `dbType` is expected to be some type that has a defined `withConn`
+  ## `dbType` is expected to be some type that has a defined `withConnection`
   ## procedure (see `Database Object`_ for details).
   ##
   ## .. _Database Object: #database-object
@@ -533,38 +568,48 @@ macro generateProcsForModels*(dbType: type, modelTypes: openarray[type]): untype
 
     let modelName = $(t.getType[1])
     let getName = ident("get" & modelName)
+    let getIfExistsName = ident("get" & modelName & "IfItExists")
     let getAllName = ident("getAll" & pluralize(modelName))
     let findWhereName = ident("find" & pluralize(modelName) & "Where")
     let createName = ident("create" & modelName)
     let updateName = ident("update" & modelName)
+    let createOrUpdateName = ident("createOrUpdate" & modelName)
     let deleteName = ident("delete" & modelName)
     let idType = typeOfColumn(t, "id")
     result.add quote do:
       proc `getName`*(db: `dbType`, id: `idType`): `t` =
-        db.withConn: result = getRecord(conn, `t`, id)
+        db.withConnection conn: result = getRecord(conn, `t`, id)
+
+      proc `getIfExistsName`*(db: `dbType`, id: `idType`): Option[`t`] =
+        db.withConnection conn:
+          try: result = some(getRecord(conn, `t`, id))
+          except NotFoundError: result = none[`t`]()
 
       proc `getAllName`*(db: `dbType`, pagination = none[PaginationParams]()): PagedRecords[`t`] =
-        db.withConn: result = getAllRecords(conn, `t`, pagination)
+        db.withConnection conn: result = getAllRecords(conn, `t`, pagination)
 
       proc `findWhereName`*(
           db: `dbType`,
           whereClause: string,
           values: varargs[string, dbFormat],
           pagination = none[PaginationParams]()): PagedRecords[`t`] =
-        db.withConn:
+        db.withConnection conn:
           result = findRecordsWhere(conn, `t`, whereClause, values, pagination)
 
       proc `createName`*(db: `dbType`, rec: `t`): `t` =
-        db.withConn: result = createRecord(conn, rec)
+        db.withConnection conn: result = createRecord(conn, rec)
 
       proc `updateName`*(db: `dbType`, rec: `t`): bool =
-        db.withConn: result = updateRecord(conn, rec)
+        db.withConnection conn: result = updateRecord(conn, rec)
+
+      proc `createOrUpdateName`*(db: `dbType`, rec: `t`): `t` =
+        db.inTransaction: result = createOrUpdateRecord(conn, rec)
 
       proc `deleteName`*(db: `dbType`, rec: `t`): bool =
-        db.withConn: result = deleteRecord(conn, rec)
+        db.withConnection conn: result = deleteRecord(conn, rec)
 
       proc `deleteName`*(db: `dbType`, id: `idType`): bool =
-        db.withConn: result = deleteRecord(conn, `t`, id)
+        db.withConnection conn: result = deleteRecord(conn, `t`, id)
 
 macro generateLookup*(dbType: type, modelType: type, fields: seq[string]): untyped =
   ## Create a lookup procedure for a given set of field names. For example,
@@ -584,7 +629,7 @@ macro generateLookup*(dbType: type, modelType: type, fields: seq[string]): untyp
   # Create proc skeleton
   result = quote do:
     proc `procName`*(db: `dbType`): PagedRecords[`modelType`] =
-      db.withConn: result = findRecordsBy(conn, `modelType`)
+      db.withConnection conn: result = findRecordsBy(conn, `modelType`)
 
   var callParams = quote do: @[]
 
@@ -608,11 +653,11 @@ macro generateLookup*(dbType: type, modelType: type, fields: seq[string]): untyp
 
   # Add the call params to the inner procedure call
   # result[6][0][1][0][1] is
-  #   ProcDef -> [6]: StmtList (body) -> [0]: Call ->
-  #     [1]: StmtList (withConn body) -> [0]: Asgn (result =) ->
+  #   ProcDef -> [6]: StmtList (body) -> [0]: Command ->
+  #     [2]: StmtList (withConnection body) -> [0]: Asgn (result =) ->
   #     [1]: Call (inner findRecords invocation)
-  result[6][0][1][0][1].add(callParams)
-  result[6][0][1][0][1].add(quote do: pagination)
+  result[6][0][2][0][1].add(callParams)
+  result[6][0][2][0][1].add(quote do: pagination)
 
 macro generateProcsForFieldLookups*(dbType: type, modelsAndFields: openarray[tuple[t: type, fields: seq[string]]]): untyped =
   result = newStmtList()
@@ -626,7 +671,7 @@ macro generateProcsForFieldLookups*(dbType: type, modelsAndFields: openarray[tup
     # Create proc skeleton
     let procDefAST = quote do:
       proc `procName`*(db: `dbType`): PagedRecords[`modelType`] =
-        db.withConn: result = findRecordsBy(conn, `modelType`)
+        db.withConnection conn: result = findRecordsBy(conn, `modelType`)
 
     var callParams = quote do: @[]
 
@@ -678,8 +723,8 @@ proc initPool*[D: DbConnType](
     hardCap: hardCap,
     healthCheckQuery: healthCheckQuery))
 
-template inTransaction*[D: DbConnType](db: DbConnPool[D], body: untyped) =
-  pool.withConn(db):
+template inTransaction*(db, body: untyped) =
+  db.withConnection conn:
     conn.exec(sql"BEGIN TRANSACTION")
     try:
       body

src/fiber_orm/db_common.nim

@@ -1,3 +1,3 @@
-import std/[db_postgres, db_sqlite]
+import db_connector/[db_postgres, db_sqlite]
 
 type DbConnType* = db_postgres.DbConn or db_sqlite.DbConn

src/fiber_orm/pool.nim

@@ -4,12 +4,12 @@
 
 ## Simple database connection pooling implementation compatible with Fiber ORM.
 
-import std/[db_common, logging, sequtils, strutils, sugar]
+import std/[sequtils, strutils, sugar]
+import db_connector/db_common
 
-from std/db_sqlite import getRow, close
-from std/db_postgres import getRow, close
+from db_connector/db_sqlite import getRow, close
+from db_connector/db_postgres import getRow, close
 
-import namespaced_logging
 import ./db_common as fiber_db_common
 
 type
@@ -43,30 +43,22 @@ type
     cfg: DbConnPoolConfig[D]
     lastId: int
 
-var logNs {.threadvar.}: LoggingNamespace
-
-template log(): untyped =
-  if logNs.isNil: logNs = getLoggerForNamespace(namespace = "fiber_orm/pool", level = lvlNotice)
-  logNs
-
 proc initDbConnPool*[D: DbConnType](cfg: DbConnPoolConfig[D]): DbConnPool[D] =
-  log().debug("Initializing new pool (size: " & $cfg.poolSize)
   result = DbConnPool[D](
     conns: @[],
     cfg: cfg)
 
 proc newConn[D: DbConnType](pool: DbConnPool[D]): PooledDbConn[D] =
-  log().debug("Creating a new connection to add to the pool.")
   pool.lastId += 1
-  let conn = pool.cfg.connect()
-  result = PooledDbConn[D](
-    conn: conn,
-    id: pool.lastId,
-    free: true)
-  pool.conns.add(result)
+  {.gcsafe.}:
+    let conn = pool.cfg.connect()
+    result = PooledDbConn[D](
+      conn: conn,
+      id: pool.lastId,
+      free: true)
+    pool.conns.add(result)
 
 proc maintain[D: DbConnType](pool: DbConnPool[D]): void =
-  log().debug("Maintaining pool. $# connections." % [$pool.conns.len])
   pool.conns.keepIf(proc (pc: PooledDbConn[D]): bool =
     if not pc.free: return true
 
@@ -78,9 +70,6 @@ proc maintain[D: DbConnType](pool: DbConnPool[D]): void =
       except: discard ""
       return false
   )
-  log().debug(
-    "Pruned dead connections. $# connections remaining." %
-    [$pool.conns.len])
 
   let freeConns = pool.conns.filterIt(it.free)
   if pool.conns.len > pool.cfg.poolSize and freeConns.len > 0:
@@ -92,9 +81,6 @@ proc maintain[D: DbConnType](pool: DbConnPool[D]): void =
       for culled in toCull:
         try: culled.conn.close()
         except: discard ""
-      log().debug(
-        "Trimming pool size. Culled $# free connections. $# connections remaining." %
-        [$toCull.len, $pool.conns.len])
 
 proc take*[D: DbConnType](pool: DbConnPool[D]): tuple[id: int, conn: D] =
   ## Request a connection from the pool. Returns a DbConn if the pool has free
@@ -107,29 +93,22 @@ proc take*[D: DbConnType](pool: DbConnPool[D]): tuple[id: int, conn: D] =
   pool.maintain
   let freeConns = pool.conns.filterIt(it.free)
 
-  log().debug(
-    "Providing a new connection ($# currently free)." % [$freeConns.len])
-
   let reserved =
     if freeConns.len > 0: freeConns[0]
     else: pool.newConn()
 
   reserved.free = false
-  log().debug("Reserve connection $#" % [$reserved.id])
   return (id: reserved.id, conn: reserved.conn)
 
 proc release*[D: DbConnType](pool: DbConnPool[D], connId: int): void =
   ## Release a connection back to the pool.
-  log().debug("Reclaiming released connaction $#" % [$connId])
   let foundConn = pool.conns.filterIt(it.id == connId)
   if foundConn.len > 0: foundConn[0].free = true
 
-template withConn*[D: DbConnType](pool: DbConnPool[D], stmt: untyped): untyped =
+template withConnection*[D: DbConnType](pool: DbConnPool[D], conn, stmt: untyped): untyped =
   ## Convenience template to provide a connection from the pool for use in a
   ## statement block, automatically releasing that connnection when done.
-  ##
-  ## The provided connection is injected as the variable `conn` in the
-  ## statement body.
-  let (connId, conn {.inject.}) = take(pool)
-  try: stmt
-  finally: release(pool, connId)
+  block:
+    let (connId, conn) = take(pool)
+    try: stmt
+    finally: release(pool, connId)

src/fiber_orm/util.nim

@@ -102,7 +102,7 @@ proc dbFormat*[T](list: seq[T]): string =
 
 proc dbFormat*[T](item: T): string =
   ## For all other types, fall back on a defined `$` function to create a
-  ## string version of the value we can include in an SQL query>
+  ## string version of the value we can include in an SQL query.
   return $item
 
 type DbArrayParseState = enum