Bump package version.

Add a PostgreSQL-specific getRecordForUpdate helper that appends FOR
UPDATE to the generated SELECT statement so callers can lock a row
inside an explicit transaction.

generateProcsForModels now always emits a direct-connection
get<RecordName>ForUpdate proc that accepts db_postgres.DbConn. There is
intentionally no dbType overload for this API, because reacquiring a
connection via withConnection would defeat the lock's transactional
scope.

The source docs and README now document the new helper and show the
intended usage pattern inside inTransaction:

  db.inTransaction:
    var item = conn.getTodoItemForUpdate(todoId)
    item.priority += 1
    discard conn.updateTodoItem(item)

.gitignore

@@ -1,2 +1,4 @@
 *.sw?
 nimcache/
+nimble.develop
+nimble.paths

Makefile

@@ -3,6 +3,14 @@ SOURCES=$(shell find src -type f)
 build: $(shell find src -type f)
 	nimble build
 
+unittest:
+	nimble unittest
+.PHONY: unittest
+
+integrationtest:
+	nimble integrationtest
+.PHONY: integrationtest
+
 docs: $(shell find src -type f)
 	nim doc --project --index:on --git.url:https://github.com/jdbernard/fiber-orm --outdir:docs src/fiber_orm
 	nim rst2html --outdir:docs README.rst

README.rst

@@ -57,7 +57,7 @@ Models may be defined as:
 
 .. code-block:: Nim
    # models.nim
-   import std/options, std/times
+   import std/[options, times]
    import uuids
 
    type
@@ -82,6 +82,8 @@ Using Fiber ORM we can generate a data access layer with:
 
 .. code-block:: Nim
    # db.nim
+   import std/[options]
+   import db_connector/db_postgres
    import fiber_orm
    import ./models.nim
 
@@ -98,30 +100,90 @@ Using Fiber ORM we can generate a data access layer with:
 
    generateLookup(TodoDB, TimeEntry, @["todoItemId"])
 
-This will generate the following procedures:
+This will generate procedures like the following in two flavors:
+
+* a `dbType` flavor that acquires a connection via `withConnection`
+* a connection flavor that operates directly on an existing
+  `conn: D` where `D: DbConnType`
 
 .. code-block:: Nim
    proc getTodoItem*(db: TodoDB, id: UUID): TodoItem;
-   proc getAllTodoItems*(db: TodoDB): seq[TodoItem];
+   proc getTodoItem*[D: DbConnType](conn: D, id: UUID): TodoItem;
+   proc getTodoItemForUpdate*(conn: db_postgres.DbConn, id: UUID): TodoItem;
+   proc tryGetTodoItem*(db: TodoDB, id: UUID): Option[TodoItem];
+   proc tryGetTodoItem*[D: DbConnType](conn: D, id: UUID): Option[TodoItem];
+   proc getTodoItemIfItExists*(db: TodoDB, id: UUID): Option[TodoItem];
+   proc getTodoItemIfItExists*[D: DbConnType](
+     conn: D, id: UUID): Option[TodoItem];
+   proc getAllTodoItems*(db: TodoDB,
+     pagination = none[PaginationParams]()): PagedRecords[TodoItem];
+   proc getAllTodoItems*[D: DbConnType](conn: D,
+     pagination = none[PaginationParams]()): PagedRecords[TodoItem];
    proc createTodoItem*(db: TodoDB, rec: TodoItem): TodoItem;
+   proc createTodoItem*[D: DbConnType](conn: D, rec: TodoItem): TodoItem;
    proc updateTodoItem*(db: TodoDB, rec: TodoItem): bool;
+   proc updateTodoItem*[D: DbConnType](conn: D, rec: TodoItem): bool;
+   proc createOrUpdateTodoItem*(db: TodoDB, rec: TodoItem): TodoItem;
+   proc createOrUpdateTodoItem*[D: DbConnType](
+     conn: D, rec: TodoItem): TodoItem;
    proc deleteTodoItem*(db: TodoDB, rec: TodoItem): bool;
+   proc deleteTodoItem*[D: DbConnType](conn: D, rec: TodoItem): bool;
    proc deleteTodoItem*(db: TodoDB, id: UUID): bool;
+   proc deleteTodoItem*[D: DbConnType](conn: D, id: UUID): bool;
 
    proc findTodoItemsWhere*(db: TodoDB, whereClause: string,
-     values: varargs[string, dbFormat]): seq[TodoItem];
+     values: varargs[string, dbFormat],
+     pagination = none[PaginationParams]()): PagedRecords[TodoItem];
+   proc findTodoItemsWhere*[D: DbConnType](conn: D, whereClause: string,
+     values: varargs[string, dbFormat],
+     pagination = none[PaginationParams]()): PagedRecords[TodoItem];
 
    proc getTimeEntry*(db: TodoDB, id: UUID): TimeEntry;
-   proc getAllTimeEntries*(db: TodoDB): seq[TimeEntry];
+   proc getTimeEntry*[D: DbConnType](conn: D, id: UUID): TimeEntry;
+   proc getTimeEntryIfItExists*(db: TodoDB, id: UUID): Option[TimeEntry];
+   proc getTimeEntryIfItExists*[D: DbConnType](
+     conn: D, id: UUID): Option[TimeEntry];
+   proc getAllTimeEntries*(db: TodoDB,
+     pagination = none[PaginationParams]()): PagedRecords[TimeEntry];
+   proc getAllTimeEntries*[D: DbConnType](conn: D,
+     pagination = none[PaginationParams]()): PagedRecords[TimeEntry];
    proc createTimeEntry*(db: TodoDB, rec: TimeEntry): TimeEntry;
+   proc createTimeEntry*[D: DbConnType](conn: D, rec: TimeEntry): TimeEntry;
    proc updateTimeEntry*(db: TodoDB, rec: TimeEntry): bool;
+   proc updateTimeEntry*[D: DbConnType](conn: D, rec: TimeEntry): bool;
    proc deleteTimeEntry*(db: TodoDB, rec: TimeEntry): bool;
+   proc deleteTimeEntry*[D: DbConnType](conn: D, rec: TimeEntry): bool;
    proc deleteTimeEntry*(db: TodoDB, id: UUID): bool;
+   proc deleteTimeEntry*[D: DbConnType](conn: D, id: UUID): bool;
 
    proc findTimeEntriesWhere*(db: TodoDB, whereClause: string,
-     values: varargs[string, dbFormat]): seq[TimeEntry];
+     values: varargs[string, dbFormat],
+     pagination = none[PaginationParams]()): PagedRecords[TimeEntry];
+   proc findTimeEntriesWhere*[D: DbConnType](conn: D, whereClause: string,
+     values: varargs[string, dbFormat],
+     pagination = none[PaginationParams]()): PagedRecords[TimeEntry];
 
-   proc findTimeEntriesByTodoItemId(db: TodoDB, todoItemId: UUID): seq[TimeEntry];
+   proc findTimeEntriesByTodoItemId*(db: TodoDB, todoItemId: UUID,
+     pagination = none[PaginationParams]()): PagedRecords[TimeEntry];
+   proc findTimeEntriesByTodoItemId*[D: DbConnType](
+     conn: D, todoItemId: UUID,
+     pagination = none[PaginationParams]()): PagedRecords[TimeEntry];
+
+Use the `dbType` flavor when the caller does not already have a connection.
+Use the connection flavor inside `withConnection` or `inTransaction`.
+The generated `get<RecordName>ForUpdate` helper is PostgreSQL-specific and
+is only available for direct PostgreSQL connections.
+
+Warning: do not call the `dbType` flavor from inside `inTransaction`.
+Those overloads call `withConnection` and may acquire a different
+connection, causing the statements to execute outside the active
+transaction.
+
+.. code-block:: Nim
+   db.inTransaction:
+     var item = conn.getTodoItemForUpdate(todoId)
+     item.priority += 1
+     discard conn.updateTodoItem(item)
 
 Object-Relational Modeling
 ==========================
@@ -129,11 +191,11 @@ Object-Relational Modeling
 Model Class
 -----------
 
-Fiber ORM uses simple Nim `object`s and `ref object`s as model classes.
+Fiber ORM uses simple Nim objects and ref objects as model classes.
 Fiber ORM expects there to be one table for each model class.
 
 Name Mapping
-````````````
+^^^^^^^^^^^^
 Fiber ORM uses `snake_case` for database identifiers (column names, table
 names, etc.) and `camelCase` for Nim identifiers. We automatically convert
 model names to and from table names (`TodoItem` <-> `todo_items`), as well
@@ -164,7 +226,7 @@ procedures in the `fiber_orm/util`_ module for details.
 .. _util: fiber_orm/util.html
 
 ID Field
-````````
+^^^^^^^^
 
 Fiber ORM expects every model class to have a field named `id`, with a
 corresponding `id` column in the model table. This field must be either a
@@ -227,6 +289,35 @@ Nim Type         Postgres Type           SQLite Type
 `JsonNode`_      `jsonb`_
 ===============  ======================  =================
 
+Distinct types backed by a supported type are parsed through their backing
+type and then converted to the distinct type. This also applies when the
+distinct type is nested in an `Option`_ or `seq`.
+
+Custom Database Mappings
+------------------------
+
+A model module can customize conversion from the database's string
+representation by exporting a ``fromDbHook`` overload:
+
+.. code-block:: Nim
+
+   import std/strutils
+
+   type Money* = object
+     cents*: int64
+
+   proc fromDbHook*(_: typedesc[Money], value: string): Money =
+     Money(cents: value.parseBiggestInt)
+
+The overload must take ``typedesc[TargetType]`` and ``string``, and return
+``TargetType``. It must be visible where Fiber ORM's generated query code is
+declared, so export it when the type lives in another module.
+
+Fiber ORM uses a custom mapping before its built-in conversion for a value.
+``Option`` and ``seq`` remain structural: an absent optional value does not
+call the hook, while present optional values and individual sequence values
+do. Exceptions raised by the hook propagate to the caller.
+
 .. [#f1] Note that this implies that all `NULL`-able fields should be typed
          as optional using `Option[fieldType]`. Conversely, any fields with
          non-optional types should also be constrained to be `NOT NULL` in
@@ -253,8 +344,10 @@ Many of the Fiber ORM macros expect a database object type to be passed.
 In the example above the `pool.DbConnPool`_ object is used as database
 object type (aliased as `TodoDB`). This is the intended usage pattern, but
 anything can be passed as the database object type so long as there is a
-defined `withConn` template that provides an injected `conn: DbConn` object
+defined `withConnection` template that provides an injected `conn: DbConn` object
 to the provided statement body.
+The generated connection-flavor procedures are intended to work directly
+with that `conn` value.
 
 For example, a valid database object implementation that opens a new
 connection for every request might look like this:
@@ -265,7 +358,7 @@ connection for every request might look like this:
    type TodoDB* = object
      connString: string
 
-   template withConn*(db: TodoDB, stmt: untyped): untyped =
+   template withConnection*(db: TodoDB, stmt: untyped): untyped =
      let conn {.inject.} = open("", "", "", db.connString)
      try: stmt
      finally: close(conn)

fiber_orm.nimble

@@ -1,6 +1,6 @@
 # Package
 
-version       = "1.0.3"
+version       = "4.4.0"
 author        = "Jonathan Bernard"
 description   = "Lightweight Postgres ORM for Nim."
 license       = "GPL-3.0"
@@ -10,5 +10,16 @@ srcDir        = "src"
 
 # Dependencies
 
-requires "nim >= 1.4.0"
+requires @["nim >= 1.4.0", "uuids"]
-requires "https://git.jdb-software.com/jdb/nim-namespaced-logging.git"
+requires "namespaced_logging >= 2.0.2"
+
+
+# Tasks
+
+task unittest, "Runs the unit test suite.":
+  exec "nim c -r --path:src tests/test_datetime"
+  exec "nim c -r --path:src tests/test_distinct"
+  exec "nim c -r --path:src tests/test_custom_mapping"
+
+task integrationtest, "Runs the PostgreSQL integration test suite.":
+  exec "nim c -r --path:src tests/test_datetime_postgres"

src/fiber_orm/db_common.nim

@@ -0,0 +1,3 @@
+import db_connector/[db_postgres, db_sqlite]
+
+type DbConnType* = db_postgres.DbConn or db_sqlite.DbConn

src/fiber_orm/pool.nim

@@ -4,94 +4,77 @@
 
 ## Simple database connection pooling implementation compatible with Fiber ORM.
 
-import std/db_postgres, std/sequtils, std/strutils, std/sugar
+when (NimMajor, NimMinor, NimPatch) < (2, 0, 0):
+  when not defined(gcArc) and not defined (gcOrc):
+    {.error: "fiber_orm requires either --mm:arc or --mm:orc.".}
 
-import namespaced_logging
+import std/[deques, locks, sequtils, sugar]
+import db_connector/db_common
 
+from db_connector/db_sqlite import getRow, close
+from db_connector/db_postgres import getRow, close
+
+import ./db_common as fiber_db_common
+import ./private/logging
 
 type
-  DbConnPoolConfig* = object
+  DbConnPool*[D: DbConnType] = ptr DbConnPoolObj[D]
-    connect*: () -> DbConn  ## Factory procedure to create a new DBConn
-    poolSize*: int ## The pool capacity.
-    hardCap*: bool ## Is the pool capacity a hard cap?
-                   ##
-                   ## When `false`, the pool can grow beyond the configured
-                   ## capacity, but will release connections down to the its
-                   ## capacity (no less than `poolSize`).
-                   ##
-                   ## When `true` the pool will not create more than its
-                   ## configured capacity.  It a connection is requested, none
-                   ## are free, and the pool is at capacity, this will result
-                   ## in an Error being raised.
-    healthCheckQuery*: string ## Should be a simple and fast SQL query that the
-                              ## pool can use to test the liveliness of pooled
-                              ## connections.
 
-  PooledDbConn = ref object
+  DbConnPoolObj[D: DbConnType] = object
-    conn: DbConn
-    id: int
-    free: bool
-
-  DbConnPool* = ref object
     ## Database connection pool
-    conns: seq[PooledDbConn]
+    connect: proc (): D {.raises: [DbError].}
-    cfg: DbConnPoolConfig
+    healthCheckQuery: SqlQuery
-    lastId: int
+    entries: Deque[D]
+    cond: Cond
+    lock: Lock
 
-var logNs {.threadvar.}: LoggingNamespace
 
-template log(): untyped =
+proc close*[D: DbConnType](pool: DbConnPool[D]) =
-  if logNs.isNil: logNs = initLoggingNamespace(name = "fiber_orm/pool", level = lvlNotice)
+  ## Safely close all connections and release resources for the given pool.
-  logNs
+  getLogger("pool").debug("closing connection pool")
+  withLock(pool.lock):
+    while pool.entries.len > 0: close(pool.entries.popFirst())
 
-proc initDbConnPool*(cfg: DbConnPoolConfig): DbConnPool =
+  deinitLock(pool.lock)
-  log().debug("Initializing new pool (size: " & $cfg.poolSize)
+  deinitCond(pool.cond)
-  result = DbConnPool(
+  `=destroy`(pool[])
-    conns: @[],
+  deallocShared(pool)
-    cfg: cfg)
 
-proc newConn(pool: DbConnPool): PooledDbConn =
-  log().debug("Creating a new connection to add to the pool.")
-  pool.lastId += 1
-  let conn = pool.cfg.connect()
-  result = PooledDbConn(
-    conn: conn,
-    id: pool.lastId,
-    free: true)
-  pool.conns.add(result)
 
-proc maintain(pool: DbConnPool): void =
+proc newDbConnPool*[D: DbConnType](
-  log().debug("Maintaining pool. $# connections." % [$pool.conns.len])
+    poolSize: int,
-  pool.conns.keepIf(proc (pc: PooledDbConn): bool =
+    connectFunc: proc(): D {.raises: [DbError].},
-    if not pc.free: return true
+    healthCheckQuery = "SELECT 1;"): DbConnPool[D] =
+  ## Initialize a new DbConnPool. See the `initDb` procedure in the `Example
+  ## Fiber ORM Usage`_ for an example
+  ##
+  ## * `connect` must be a factory which creates a new `DbConn`.
+  ## * `poolSize` sets the desired capacity of the connection pool.
+  ## * `healthCheckQuery` should be a simple and fast SQL query that the pool
+  ##   can use to test the liveliness of pooled connections. By default it uses
+  ##   `SELECT 1;`
+  ##
+  ## .. _Example Fiber ORM Usage: ../fiber_orm.html#basic-usage-example-fiber-orm-usage
+
+  result = cast[DbConnPool[D]](allocShared0(sizeof(DbConnPoolObj[D])))
+  initCond(result.cond)
+  initLock(result.lock)
+  result.entries = initDeque[D](poolSize)
+  result.connect = connectFunc
+  result.healthCheckQuery = sql(healthCheckQuery)
 
   try:
-      discard getRow(pc.conn, sql(pool.cfg.healthCheckQuery), [])
+    for _ in 0 ..< poolSize: result.entries.addLast(connectFunc())
-      return true
+  except DbError as ex:
-    except:
+    try: result.close()
-      try: pc.conn.close()  # try to close the connection
+    except: discard
-      except: discard ""
+    getLogger("pool").error(
-      return false
+      msg = "unable to initialize connection pool",
-  )
+      err = ex)
-  log().debug(
+    raise ex
-    "Pruned dead connections. $# connections remaining." %
-    [$pool.conns.len])
 
-  let freeConns = pool.conns.filterIt(it.free)
-  if pool.conns.len > pool.cfg.poolSize and freeConns.len > 0:
-    let numToCull = min(freeConns.len, pool.conns.len - pool.cfg.poolSize)
 
-    if numToCull > 0:
+proc take*[D: DbConnType](pool: DbConnPool[D]): D {.raises: [DbError], gcsafe.} =
-      let toCull = freeConns[0..numToCull]
-      pool.conns.keepIf((pc) => toCull.allIt(it.id != pc.id))
-      for culled in toCull:
-        try: culled.conn.close()
-        except: discard ""
-      log().debug(
-        "Trimming pool size. Culled $# free connections. $# connections remaining." %
-        [$toCull.len, $pool.conns.len])
-
-proc take*(pool: DbConnPool): tuple[id: int, conn: DbConn] =
   ## Request a connection from the pool. Returns a DbConn if the pool has free
   ## connections, or if it has the capacity to create a new connection. If the
   ## pool is configured with a hard capacity limit and is out of free
@@ -99,32 +82,33 @@ proc take*(pool: DbConnPool): tuple[id: int, conn: DbConn] =
   ##
   ## Connections taken must be returned via `release` when the caller is
   ## finished using them in order for them to be released back to the pool.
-  pool.maintain
+  withLock(pool.lock):
-  let freeConns = pool.conns.filterIt(it.free)
+    while pool.entries.len == 0: wait(pool.cond, pool.lock)
+    result = pool.entries.popFirst()
 
-  log().debug(
+    # check that the connection is healthy
-    "Providing a new connection ($# currently free)." % [$freeConns.len])
+    try: discard getRow(result, pool.healthCheckQuery, [])
+    except DbError:
+      {.gcsafe.}:
+        # if it's not, let's try to close it and create a new connection
+        try:
+          getLogger("pool").info(
+            "pooled connection failed health check, opening a new connection")
+          close(result)
+        except: discard
+        result = pool.connect()
 
-  let reserved =
-    if freeConns.len > 0: freeConns[0]
-    else: pool.newConn()
 
-  reserved.free = false
+proc release*[D: DbConnType](pool: DbConnPool[D], conn: D) {.raises: [], gcsafe.} =
-  log().debug("Reserve connection $#" % [$reserved.id])
-  return (id: reserved.id, conn: reserved.conn)
-
-proc release*(pool: DbConnPool, connId: int): void =
   ## Release a connection back to the pool.
-  log().debug("Reclaiming released connaction $#" % [$connId])
+  withLock(pool.lock):
-  let foundConn = pool.conns.filterIt(it.id == connId)
+    pool.entries.addLast(conn)
-  if foundConn.len > 0: foundConn[0].free = true
+  signal(pool.cond)
 
-template withConn*(pool: DbConnPool, stmt: untyped): untyped =
+template withConnection*[D: DbConnType](pool: DbConnPool[D], conn, stmt: untyped): untyped =
   ## Convenience template to provide a connection from the pool for use in a
   ## statement block, automatically releasing that connnection when done.
-  ##
+  block:
-  ## The provided connection is injected as the variable `conn` in the
+    let conn = take(pool)
-  ## statement body.
-  let (connId, conn {.inject.}) = take(pool)
     try: stmt
-  finally: release(pool, connId)
+    finally: release(pool, conn)

src/fiber_orm/private/logging.nim

@@ -0,0 +1,34 @@
+import std/[json, options]
+
+import namespaced_logging
+
+export namespaced_logging.log
+export namespaced_logging.debug
+export namespaced_logging.info
+export namespaced_logging.notice
+export namespaced_logging.warn
+export namespaced_logging.error
+export namespaced_logging.fatal
+
+var logService {.threadvar.}: Option[ThreadLocalLogService]
+var logger {.threadvar.}: Option[Logger]
+
+proc makeQueryLogEntry(
+    m: string,
+    sql: string,
+    args: openArray[(string, string)] = []): JsonNode =
+  result = %*{ "method": m, "sql": sql }
+  for (k, v) in args: result[k] = %v
+
+proc logQuery*(methodName: string, sqlStmt: string, args: openArray[(string, string)] = []) =
+  # namespaced_logging would do this check for us, but we don't want to even
+  # build the log object if we're not actually logging
+  if logService.isNone: return
+  if logger.isNone: logger = logService.getLogger("fiber_orm/query")
+  logger.debug(makeQueryLogEntry(methodName, sqlStmt, args))
+
+proc enableDbLogging*(svc: ThreadLocalLogService) =
+  logService = some(svc)
+
+proc getLogger*(scope: string): Option[Logger] =
+  logService.getLogger("fiber_orm/" & scope)

src/fiber_orm/util.nim

@@ -3,12 +3,19 @@
 # Copyright 2019 Jonathan Bernard <jonathan@jdbernard.com>
 
 ## Utility methods used internally by Fiber ORM.
-import json, macros, options, sequtils, strutils, times, unicode,
+import std/[json, macros, options, sequtils, strutils, times, unicode]
-  uuids
+import uuids
 
-import nre except toSeq
+import std/nre except toSeq
 
 type
+  NoDbHook = object
+
+  PaginationParams* = object
+    pageSize*: int
+    offset*: int
+    orderBy*: Option[seq[string]]
+
   MutateClauses* = object
     ## Data structure to hold information about the clauses that should be
     ## added to a query. How these clauses are used will depend on the query.
@@ -18,12 +25,19 @@ type
     placeholders*: seq[string]
     values*: seq[string]
 
+proc fromDbHook[T](_: typedesc[T], _: string): NoDbHook =
+  ## Default overload used to detect that a consumer has not supplied a
+  ## database mapping for `T`.
+  NoDbHook()
+
 const ISO_8601_FORMATS = @[
   "yyyy-MM-dd'T'HH:mm:ssz",
   "yyyy-MM-dd'T'HH:mm:sszzz",
+  "yyyy-MM-dd'T'HH:mm:ss'.'ffffffzzz",
   "yyyy-MM-dd'T'HH:mm:ss'.'fffzzz",
   "yyyy-MM-dd HH:mm:ssz",
   "yyyy-MM-dd HH:mm:sszzz",
+  "yyyy-MM-dd HH:mm:ss'.'ffffffzzz",
   "yyyy-MM-dd HH:mm:ss'.'fffzzz"
 ]
 
@@ -102,7 +116,7 @@ proc dbFormat*[T](list: seq[T]): string =
 
 proc dbFormat*[T](item: T): string =
   ## For all other types, fall back on a defined `$` function to create a
-  ## string version of the value we can include in an SQL query>
+  ## string version of the value we can include in an SQL query.
   return $item
 
 type DbArrayParseState = enum
@@ -114,30 +128,41 @@ proc parsePGDatetime*(val: string): DateTime =
   const PG_TIMESTAMP_FORMATS = [
     "yyyy-MM-dd HH:mm:ss",
     "yyyy-MM-dd'T'HH:mm:ss",
+    "yyyy-MM-dd HH:mm:ssz",
+    "yyyy-MM-dd'T'HH:mm:ssz",
     "yyyy-MM-dd HH:mm:sszz",
     "yyyy-MM-dd'T'HH:mm:sszz",
+    "yyyy-MM-dd HH:mm:sszzz",
+    "yyyy-MM-dd'T'HH:mm:sszzz",
     "yyyy-MM-dd HH:mm:ss'.'fff",
     "yyyy-MM-dd'T'HH:mm:ss'.'fff",
     "yyyy-MM-dd HH:mm:ss'.'fffzz",
     "yyyy-MM-dd'T'HH:mm:ss'.'fffzz",
     "yyyy-MM-dd HH:mm:ss'.'fffzzz",
     "yyyy-MM-dd'T'HH:mm:ss'.'fffzzz",
+    "yyyy-MM-dd HH:mm:ss'.'ffffff",
+    "yyyy-MM-dd'T'HH:mm:ss'.'ffffff",
+    "yyyy-MM-dd HH:mm:ss'.'ffffffz",
+    "yyyy-MM-dd'T'HH:mm:ss'.'ffffffz",
+    "yyyy-MM-dd HH:mm:ss'.'ffffffzz",
+    "yyyy-MM-dd'T'HH:mm:ss'.'ffffffzz",
+    "yyyy-MM-dd HH:mm:ss'.'ffffffzzz",
+    "yyyy-MM-dd'T'HH:mm:ss'.'ffffffzzz",
   ]
 
   var correctedVal = val;
 
-  # PostgreSQL will truncate any trailing 0's in the millisecond value leading
+  # Nim's time parser requires a fixed number of fractional digits for each
-  # to values like `2020-01-01 16:42.3+00`. This cannot currently be parsed by
+  # format pattern. PostgreSQL emits between one and six digits, omitting
-  # the standard times format as it expects exactly three digits for
+  # trailing zeroes. Normalize the fraction to six digits so parsing retains
-  # millisecond values. So we have to detect this and pad out the millisecond
+  # PostgreSQL's full microsecond precision.
-  # value to 3 digits.
+  let PG_PARTIAL_FORMAT_REGEX = re"(\d{4}-\d{2}-\d{2}( |T)\d{2}:\d{2}:\d{2}\.)(\d+)(\S+)?"
-  let PG_PARTIAL_FORMAT_REGEX = re"(\d{4}-\d{2}-\d{2}( |'T')\d{2}:\d{2}:\d{2}\.)(\d{1,2})(\S+)?"
   let match = val.match(PG_PARTIAL_FORMAT_REGEX)
 
   if match.isSome:
     let c = match.get.captures
-    if c.toSeq.len == 2: correctedVal = c[0] & alignLeft(c[2], 3, '0')
+    correctedVal = c[0] & alignLeft(c[2], 6, '0')[0..5]
-    else: correctedVal = c[0] & alignLeft(c[2], 3, '0') & c[3]
+    if 3 in c: correctedVal &= c[3]
 
   var errStr = ""
 
@@ -146,7 +171,7 @@ proc parsePGDatetime*(val: string): DateTime =
     try: return correctedVal.parse(df)
     except: errStr &= "\n\t" & getCurrentExceptionMsg()
 
-  raise newException(ValueError, "Cannot parse PG date. Tried:" & errStr)
+  raise newException(ValueError, "Cannot parse PG date '" & correctedVal & "'. Tried:" & errStr)
 
 proc parseDbArray*(val: string): seq[string] =
   ## Parse a Postgres array column into a Nim seq[string]
@@ -207,47 +232,53 @@ proc parseDbArray*(val: string): seq[string] =
   if not (parseState == inQuote) and curStr.len > 0:
     result.add(curStr)
 
-proc createParseStmt*(t, value: NimNode): NimNode =
+func createParseStmt*(t, value: NimNode): NimNode
-  ## Utility method to create the Nim cod required to parse a value coming from
-  ## the a database query. This is used by functions like `rowToModel` to parse
-  ## the dataabase columns into the Nim object fields.
 
-  #echo "Creating parse statment for ", t.treeRepr
+template parseDbValueWithHook(
+    targetType,
+    value,
+    automaticParse: untyped): untyped =
+  block:
+    var target: targetType
+    mixin fromDbHook
+    when typeof(fromDbHook(type(target), value)) is targetType:
+      fromDbHook(type(target), value)
+    else:
+      automaticParse
+
+
+func unsupportedParseStmt(t: NimNode, description: string): NimNode =
+  let message = newLit("Cannot parse column with " & description)
+  result = quote do:
+    block:
+      {.error: `message`.}
+      default(`t`)
+
+
+func createAutomaticParseStmt(t, value: NimNode): NimNode =
   if t.typeKind == ntyObject:
-
     if t.getType == UUID.getType:
       result = quote do: parseUUID(`value`)
 
     elif t.getType == DateTime.getType:
       result = quote do: parsePGDatetime(`value`)
 
-    elif t.getTypeInst == Option.getType:
+    else:
-      var innerType = t.getTypeImpl[2][0] # start at the first RecList
+      result = unsupportedParseStmt(
-      # If the value is a non-pointer type, there is another inner RecList
+        t, "unknown object type: " & $t.getTypeInst)
-      if innerType.kind == nnkRecList: innerType = innerType[0]
-      innerType = innerType[1] # now we can take the field type from the first symbol
 
-      let parseStmt = createParseStmt(innerType, value)
+  elif t.typeKind == ntyDistinct:
-      result = quote do:
+    let baseType = t.getTypeImpl[0]
-        if `value`.len == 0:  none[`innerType`]()
+    let parseStmt = createParseStmt(baseType, value)
-        else:                 some(`parseStmt`)
+    result = quote do: `t`(`parseStmt`)
-
-    else: error "Unknown value object type: " & $t.getTypeInst
 
   elif t.typeKind == ntyRef:
-
     if $t.getTypeInst == "JsonNode":
       result = quote do: parseJson(`value`)
 
     else:
-      error "Unknown ref type: " & $t.getTypeInst
+      result = unsupportedParseStmt(
-
+        t, "unknown ref type: " & $t.getTypeInst)
-  elif t.typeKind == ntySequence:
-    let innerType = t[1]
-
-    let parseStmts = createParseStmt(innerType, ident("it"))
-
-    result = quote do: parseDbArray(`value`).mapIt(`parseStmts`)
 
   elif t.typeKind == ntyString:
     result = quote do: `value`
@@ -259,37 +290,121 @@ proc createParseStmt*(t, value: NimNode): NimNode =
     result = quote do: parseFloat(`value`)
 
   elif t.typeKind == ntyBool:
-    result = quote do: "true".startsWith(`value`.toLower)
+    result = quote do: "true".startsWith(`value`.toLower) or `value` == "1"
 
   elif t.typeKind == ntyEnum:
     let innerType = t.getTypeInst
     result = quote do: parseEnum[`innerType`](`value`)
 
   else:
-    error "Unknown value type: " & $t.typeKind
+    result = unsupportedParseStmt(
+      t, "unknown value type: " & $t.typeKind)
+
+
+func createParseStmt*(t, value: NimNode): NimNode =
+  ## Create code to parse a value returned by a database query.
+  ##
+  ## `Option` and `seq` remain structural mappings owned by Fiber ORM. Their
+  ## contained values are parsed recursively and may use `fromDbHook`.
+  if t.typeKind == ntyObject and t.getTypeInst == Option.getType:
+    var innerType = t.getTypeImpl[2][0] # start at the first RecList
+    # If the value is a non-pointer type, there is another inner RecList.
+    if innerType.kind == nnkRecList: innerType = innerType[0]
+    innerType = innerType[1] # take the field type from the first symbol
+
+    let parseStmt = createParseStmt(innerType, value)
+    result = quote do:
+      if `value`.len == 0:  none[`innerType`]()
+      else:                 some(`parseStmt`)
+    return
+
+  if t.typeKind == ntyGenericInst and
+     t.kind == nnkBracketExpr and
+     t.len > 0 and
+     t[0] == Option.getType:
+    let innerType = t.getTypeInst[1]
+    let parseStmt = createParseStmt(innerType, value)
+    result = quote do:
+      if `value`.len == 0:  none[`innerType`]()
+      else:                 some(`parseStmt`)
+    return
+
+  if t.typeKind == ntySequence:
+    let innerType = t[1]
+    let parseStmt = createParseStmt(innerType, ident("it"))
+    result = quote do: parseDbArray(`value`).mapIt(`parseStmt`)
+    return
+
+  let automaticParseStmt = createAutomaticParseStmt(t, value)
+  result = quote do:
+    parseDbValueWithHook(`t`, `value`, `automaticParseStmt`)
+
+func fields(t: NimNode): seq[tuple[fieldIdent: NimNode, fieldType: NimNode]] =
+  #[
+  debugEcho "T: " & t.treeRepr
+  debugEcho "T.kind: " & $t.kind
+  debugEcho "T.typeKind: " & $t.typeKind
+  debugEcho "T.GET_TYPE[1]: " & t.getType[1].treeRepr
+  debugEcho "T.GET_TYPE[1].kind: " & $t.getType[1].kind
+  debugEcho "T.GET_TYPE[1].typeKind: " & $t.getType[1].typeKind
+
+  debugEcho "T.GET_TYPE: " & t.getType.treeRepr
+  debugEcho "T.GET_TYPE[1].GET_TYPE: " & t.getType[1].getType.treeRepr
+  ]#
+
+  # Get the object type AST, with base object (if present) and record list.
+  var objDefAst: NimNode
+  if t.typeKind == ntyObject: objDefAst = t.getType
+  elif t.typeKind == ntyTypeDesc:
+    # In this case we have a type AST that is like:
+    # BracketExpr
+    #   Sym "typeDesc"
+    #   Sym "ModelType"
+    objDefAst = t.
+      getType[1].         # get the Sym "ModelType"
+      getType             # get the object definition type
+
+    if objDefAst.kind != nnkObjectTy:
+      error ("unable to enumerate the fields for model type '$#', " &
+        "tried to resolve the type of the provided symbol to an object " &
+        "definition (nnkObjectTy) but got a '$#'.\pAST:\p$#") % [
+          $t, $objDefAst.kind, objDefAst.treeRepr ]
+  else:
+    error ("unable to enumerate the fields for model type '$#', " &
+      "expected a symbol with type ntyTypeDesc but got a '$#'.\pAST:\p$#") % [
+        $t, $t.typeKind, t.treeRepr ]
+
+  # At this point objDefAst should look something like:
+  # ObjectTy
+  #   Empty
+  #   Sym "BaseObject"" | Empty
+  #   RecList
+  #     Sym "field1"
+  #     Sym "field2"
+  #     ...
+
+  if objDefAst[1].kind == nnkSym:
+    # We have a base class symbol, let's recurse and try and resolve the fields
+    # for the base class
+    for fieldDef in objDefAst[1].fields: result.add(fieldDef)
+
+  for fieldDef in objDefAst[2].children:
+    # objDefAst[2] is a RecList of
+    # ignore AST nodes that are not field definitions
+    if fieldDef.kind == nnkIdentDefs: result.add((fieldDef[0], fieldDef[1]))
+    elif fieldDef.kind == nnkSym: result.add((fieldDef, fieldDef.getTypeInst))
+    else: error "unknown object field definition AST: $#" % $fieldDef.kind
 
 template walkFieldDefs*(t: NimNode, body: untyped) =
   ## Iterate over every field of the given Nim object, yielding and defining
   ## `fieldIdent` and `fieldType`, the name of the field as a Nim Ident node
   ## and the type of the field as a Nim Type node respectively.
-  let tTypeImpl = t.getTypeImpl
+  for (fieldIdent {.inject.}, fieldType {.inject.}) in t.fields: body
 
-  var nodeToItr: NimNode
+#[ TODO: replace walkFieldDefs with things like this:
-  if tTypeImpl.typeKind == ntyObject: nodeToItr = tTypeImpl[2]
+func columnNamesForModel*(modelType: typedesc): seq[string] =
-  elif tTypeImpl.typeKind == ntyTypeDesc: nodeToItr = tTypeImpl.getType[1].getType[2]
+  modelType.fields.mapIt(identNameToDb($it[0]))
-  else: error $t & " is not an object or type desc (it's a " & $tTypeImpl.typeKind & ")."
+]#
-
-  for fieldDef {.inject.} in nodeToItr.children:
-    # ignore AST nodes that are not field definitions
-    if fieldDef.kind == nnkIdentDefs:
-      let fieldIdent {.inject.} = fieldDef[0]
-      let fieldType {.inject.} = fieldDef[1]
-      body
-
-    elif fieldDef.kind == nnkSym:
-      let fieldIdent {.inject.} = fieldDef
-      let fieldType {.inject.} = fieldDef.getType
-      body
 
 macro columnNamesForModel*(modelType: typed): seq[string] =
   ## Return the column names corresponding to the the fields of the given
@@ -316,7 +431,7 @@ macro rowToModel*(modelType: typed, row: seq[string]): untyped =
       fieldIdent,
       createParseStmt(fieldType, itemLookup)))
     idx += 1
-
+#[
 macro listFields*(t: typed): untyped =
   var fields: seq[tuple[n: string, t: string]] = @[]
   t.walkFieldDefs:
@@ -324,6 +439,7 @@ macro listFields*(t: typed): untyped =
     else: fields.add((n: $fieldIdent, t: $fieldType))
 
   result = newLit(fields)
+]#
 
 proc typeOfColumn*(modelType: NimNode, colName: string): NimNode =
   ## Given a model type and a column name, return the Nim type for that column.
@@ -388,6 +504,19 @@ macro populateMutateClauses*(t: typed, newRecord: bool, mc: var MutateClauses):
           `mc`.placeholders.add("?")
           `mc`.values.add(dbFormat(`t`.`fieldIdent`))
 
+
+proc getPagingClause*(page: PaginationParams): string =
+  ## Given a `PaginationParams` object, return the SQL clause necessary to
+  ## limit the number of records returned by a query.
+  result = ""
+  if page.orderBy.isSome:
+    let orderByClause = page.orderBy.get.map(identNameToDb).join(",")
+    result &= " ORDER BY " & orderByClause
+  else:
+    result &= " ORDER BY id"
+
+  result &= " LIMIT " & $page.pageSize & " OFFSET " & $page.offset
+
 ## .. _model class: ../fiber_orm.html#objectminusrelational-modeling-model-class
 ## .. _rules for name mapping: ../fiber_orm.html
 ## .. _table name: ../fiber_orm.html

tests/custom_db_types.nim

@@ -0,0 +1,20 @@
+import std/[strutils]
+
+
+type
+  ValidatedId* = distinct string
+
+  Money* = object
+    cents*: int64
+
+
+proc fromDbHook*(
+    _: typedesc[ValidatedId],
+    value: string): ValidatedId =
+  if not value.startsWith("id_"):
+    raise newException(ValueError, "validated ID must begin with id_")
+  ValidatedId(value.toUpperAscii)
+
+
+proc fromDbHook*(_: typedesc[Money], value: string): Money =
+  Money(cents: value.parseBiggestInt)

tests/test_custom_mapping.nim

@@ -0,0 +1,45 @@
+import std/[options, unittest]
+
+import fiber_orm/util
+
+import ./custom_db_types
+
+
+type CustomModel = object
+  id: ValidatedId
+  optionalId: Option[ValidatedId]
+  absentId: Option[ValidatedId]
+  relatedIds: seq[ValidatedId]
+  balance: Money
+
+
+suite "custom database mappings":
+
+  test "uses custom mappings for values nested in structural types":
+    let model = rowToModel(CustomModel, @[
+      "id_primary",
+      "id_optional",
+      "",
+      "{id_first,id_second}",
+      "1250",
+    ])
+
+    check:
+      string(model.id) == "ID_PRIMARY"
+      model.optionalId.isSome
+      string(model.optionalId.get) == "ID_OPTIONAL"
+      model.absentId.isNone
+      model.relatedIds.len == 2
+      string(model.relatedIds[0]) == "ID_FIRST"
+      string(model.relatedIds[1]) == "ID_SECOND"
+      model.balance.cents == 1_250
+
+  test "propagates errors from custom mappings without falling back":
+    expect ValueError:
+      discard rowToModel(CustomModel, @[
+        "invalid",
+        "",
+        "",
+        "{}",
+        "1250",
+      ])

tests/test_datetime.nim

@@ -0,0 +1,53 @@
+import std/[times, unittest]
+
+import fiber_orm/util
+
+proc utcDateTime(nanosecond: NanosecondRange): DateTime =
+  dateTime(
+    2026, mAug, 26, 14, 32, 10, nanosecond,
+    utc())
+
+suite "PostgreSQL DateTime conversion":
+  test "formats DateTime values with microsecond precision":
+    check dbFormat(utcDateTime(123_457_000)) ==
+      "2026-08-26T14:32:10.123457Z"
+
+  test "parses whole-second, millisecond, and microsecond values":
+    let cases = [
+      ("2026-08-26 14:32:10+00", 0),
+      ("2026-08-26 14:32:10.123+00", 123_000_000),
+      ("2026-08-26 14:32:10.9007+00", 900_700_000),
+      ("2026-08-26 14:32:10.12345+00", 123_450_000),
+      ("2026-08-26 14:32:10.123457+00", 123_457_000),
+    ]
+
+    for (value, expectedNanosecond) in cases:
+      let parsed = parsePGDatetime(value)
+      check parsed.nanosecond == expectedNanosecond
+      check parsed.utc.format("yyyy-MM-dd'T'HH:mm:ss") ==
+        "2026-08-26T14:32:10"
+
+  test "preserves supported separators and timezone representations":
+    let timezoneCases = [
+      "2026-08-26 14:32:10+00",
+      "2026-08-26T14:32:10+00",
+      "2026-08-26T14:32:10+00:00",
+      "2026-08-26 14:32:10.123+00",
+      "2026-08-26T14:32:10.123457+00:00",
+      "2026-08-26T14:32:10.123457Z",
+    ]
+
+    for value in timezoneCases:
+      check parsePGDatetime(value).utc.format("yyyy-MM-dd'T'HH:mm:ss") ==
+        "2026-08-26T14:32:10"
+
+    let localCases = [
+      "2026-08-26 14:32:10",
+      "2026-08-26T14:32:10",
+      "2026-08-26 14:32:10.1",
+      "2026-08-26T14:32:10.12",
+    ]
+
+    for value in localCases:
+      check parsePGDatetime(value).format("yyyy-MM-dd'T'HH:mm:ss") ==
+        "2026-08-26T14:32:10"

tests/test_datetime_postgres.nim

@@ -0,0 +1,30 @@
+import std/[os, times, unittest]
+import db_connector/[db_common, db_postgres]
+
+import fiber_orm/util
+
+proc utcDateTime(nanosecond: NanosecondRange): DateTime =
+  dateTime(
+    2026, mAug, 26, 14, 32, 10, nanosecond,
+    utc())
+
+let connectionString = getEnv("FIBER_ORM_TEST_DB")
+if connectionString.len == 0:
+  quit "FIBER_ORM_TEST_DB must contain a PostgreSQL connection string"
+
+suite "PostgreSQL DateTime round trips":
+  test "preserves whole-second, millisecond, and microsecond values":
+    let db = db_postgres.open("", "", "", connectionString)
+    defer: db.close()
+    db.exec(sql"SET TIME ZONE 'UTC'")
+
+    for expected in [
+      utcDateTime(0),
+      utcDateTime(123_000_000),
+      utcDateTime(123_457_000),
+    ]:
+      let dbValue = db.getValue(
+        sql"SELECT ?::timestamp with time zone",
+        dbFormat(expected))
+      let actual = parsePGDatetime(dbValue)
+      check actual.toTime == expected.toTime

tests/test_distinct.nim

@@ -0,0 +1,84 @@
+import std/[options, times, unittest]
+
+import uuids
+
+import fiber_orm/util
+
+
+type
+  StringId = distinct string
+  NestedStringId = distinct StringId
+  UuidId = distinct UUID
+  Count = distinct int
+  Ratio = distinct float
+  Enabled = distinct bool
+
+  State = enum
+    pending
+    complete
+
+  DistinctState = distinct State
+
+  DistinctModel = object
+    id: StringId
+    nestedId: NestedStringId
+    userId: UuidId
+    count: Count
+    ratio: Ratio
+    enabled: Enabled
+    state: DistinctState
+    optionalId: Option[StringId]
+    absentId: Option[StringId]
+    relatedIds: seq[StringId]
+    occurredAt: DateTime
+
+
+suite "distinct type row mapping":
+
+  test "parses each distinct type through its backing type":
+    let model = rowToModel(DistinctModel, @[
+      "item_123",
+      "nested_123",
+      "07e268ed-a3c1-4952-bc45-778b3000b76c",
+      "42",
+      "1.25",
+      "true",
+      "complete",
+      "optional_123",
+      "",
+      "{related_1,related_2}",
+      "2026-08-26 14:32:10.123457+00",
+    ])
+
+    check:
+      string(model.id) == "item_123"
+      string(StringId(model.nestedId)) == "nested_123"
+      UUID(model.userId) ==
+        parseUUID("07e268ed-a3c1-4952-bc45-778b3000b76c")
+      int(model.count) == 42
+      float(model.ratio) == 1.25
+      bool(model.enabled)
+      State(model.state) == complete
+      model.optionalId.isSome
+      string(model.optionalId.get) == "optional_123"
+      model.absentId.isNone
+      model.relatedIds.len == 2
+      string(model.relatedIds[0]) == "related_1"
+      string(model.relatedIds[1]) == "related_2"
+      model.occurredAt.nanosecond == 123_457_000
+
+  test "preserves backing-type parse failures":
+    expect ValueError:
+      discard rowToModel(DistinctModel, @[
+        "item_123",
+        "nested_123",
+        "not-a-uuid",
+        "42",
+        "1.25",
+        "true",
+        "complete",
+        "",
+        "",
+        "{}",
+        "2026-08-26 14:32:10+00",
+      ])