Add support for records associated via join tables.

fiber_orm.nimble

@@ -1,6 +1,6 @@
 # Package
 
-version       = "3.0.0"
+version       = "3.1.1"
 author        = "Jonathan Bernard"
 description   = "Lightweight Postgres ORM for Nim."
 license       = "GPL-3.0"

src/fiber_orm.nim

@@ -294,22 +294,9 @@ import ./fiber_orm/db_common as fiber_db_common
 import ./fiber_orm/pool
 import ./fiber_orm/util
 
-export
-  pool,
-  util.columnNamesForModel,
-  util.dbFormat,
-  util.dbNameToIdent,
-  util.identNameToDb,
-  util.modelName,
-  util.rowToModel,
-  util.tableName
+export pool, util
 
 type
-  PaginationParams* = object
-    pageSize*: int
-    offset*: int
-    orderBy*: Option[seq[string]]
-
   PagedRecords*[T] = object
     pagination*: Option[PaginationParams]
     records*: seq[T]
@@ -322,14 +309,16 @@ type
     ## Error type raised when no record matches a given ID
 
 var logService {.threadvar.}: Option[LogService]
+var logger {.threadvar.}: Option[Logger]
 
-proc logQuery(methodName: string, sqlStmt: string, args: openArray[(string, string)] = []) =
+proc logQuery*(methodName: string, sqlStmt: string, args: openArray[(string, string)] = []) =
   # namespaced_logging would do this check for us, but we don't want to even
   # build the log object if we're not actually logging
   if logService.isNone: return
+  if logger.isNone: logger = logService.getLogger("fiber_orm/query")
   var log = %*{ "method": methodName, "sql": sqlStmt }
   for (k, v) in args: log[k] = %v
-  logService.getLogger("fiber_orm/query").debug(log)
+  logger.debug(log)
 
 proc enableDbLogging*(svc: LogService) =
   logService = some(svc)
@@ -359,6 +348,8 @@ proc createRecord*[D: DbConnType, T](db: D, rec: T): T =
     " RETURNING " & columnNamesForModel(rec).join(",")
 
   logQuery("createRecord", sqlStmt)
+  debug(logService.getLogger("fiber_orm/query"), %*{ "values": mc.values })
+
   let newRow = db.getRow(sql(sqlStmt), mc.values)
 
   result = rowToModel(T, newRow)
@@ -444,16 +435,7 @@ template findRecordsWhere*[D: DbConnType](
     "SELECT COUNT(*) FROM " & tableName(modelType) &
     " WHERE " & whereClause
 
-  if page.isSome:
-    let p = page.get
-    if p.orderBy.isSome:
-      let orderByClause = p.orderBy.get.map(identNameToDb).join(",")
-      fetchStmt &= " ORDER BY " & orderByClause
-    else:
-      fetchStmt &= " ORDER BY id"
-
-    fetchStmt &= " LIMIT " & $p.pageSize &
-                 " OFFSET " & $p.offset
+  if page.isSome: fetchStmt &= getPagingClause(page.get)
 
   logQuery("findRecordsWhere", fetchStmt, [("values", values.join(", "))])
   let records = db.getAllRows(sql(fetchStmt), values).mapIt(rowToModel(modelType, it))
@@ -476,16 +458,7 @@ template getAllRecords*[D: DbConnType](
 
   var countStmt = "SELECT COUNT(*) FROM " & tableName(modelType)
 
-  if page.isSome:
-    let p = page.get
-    if p.orderBy.isSome:
-      let orderByClause = p.orderBy.get.map(identNameToDb).join(",")
-      fetchStmt &= " ORDER BY " & orderByClause
-    else:
-      fetchStmt &= " ORDER BY id"
-
-    fetchStmt &= " LIMIT " & $p.pageSize &
-                 " OFFSET " & $p.offset
+  if page.isSome: fetchStmt &= getPagingClause(page.get)
 
   logQuery("getAllRecords", fetchStmt)
   let records = db.getAllRows(sql(fetchStmt)).mapIt(rowToModel(modelType, it))
@@ -516,16 +489,7 @@ template findRecordsBy*[D: DbConnType](
     "SELECT COUNT(*) FROM " & tableName(modelType) &
     " WHERE " & whereClause
 
-  if page.isSome:
-    let p = page.get
-    if p.orderBy.isSome:
-      let orderByClause = p.orderBy.get.map(identNameToDb).join(",")
-      fetchStmt &= " ORDER BY " & orderByClause
-    else:
-      fetchStmt &= " ORDER BY id"
-
-    fetchStmt &= " LIMIT " & $p.pageSize &
-                 " OFFSET " & $p.offset
+  if page.isSome: fetchStmt &= getPagingClause(page.get)
 
   logQuery("findRecordsBy", fetchStmt, [("values", values.join(", "))])
   let records = db.getAllRows(sql(fetchStmt), values).mapIt(rowToModel(modelType, it))
@@ -538,6 +502,91 @@ template findRecordsBy*[D: DbConnType](
       else: db.getRow(sql(countStmt), values)[0].parseInt)
 
 
+template associate*[D: DbConnType, I, J](
+    db: D,
+    joinTableName: string,
+    rec1: I,
+    rec2: J): void =
+  ## Associate two records via a join table.
+
+  let insertStmt =
+    "INSERT INTO " & joinTableName &
+    " (" & tableName(I) & "_id, " & tableName(J) & "_id) " &
+    " VALUES (?, ?)"
+
+  logQuery("associate", insertStmt, [("id1", $rec1.id), ("id2", $rec2.id)])
+  db.exec(sql(insertStmt), [$rec1.id, $rec2.id])
+
+
+template findViaJoinTable*[D: DbConnType, L](
+    db: D,
+    joinTableName: string,
+    targetType: type,
+    rec: L,
+    page: Option[PaginationParams]): untyped =
+  ## Find all records of `targetType` that are associated with `rec` via a
+  ## join table.
+  let columns = columnNamesForModel(targetType).mapIt("t." & it).join(",")
+
+  var fetchStmt =
+    "SELECT " & columns &
+    " FROM " & tableName(targetType) & " AS t " &
+    " JOIN " & joinTableName & " AS j " &
+    " ON t.id = jt." & tableName(targetType) & "_id " &
+    " WHERE jt." & tableName(rec) & "_id = ?"
+
+  var countStmt =
+    "SELECT COUNT(*) FROM " & joinTableName &
+    " WHERE " & tableName(rec) & "_id = ?"
+
+  if page.isSome: fetchStmt &= getPagingClause(page.get)
+
+  logQuery("findViaJoinTable", fetchStmt, [("id", $rec.id)])
+  let records = db.getAllRows(sql(fetchStmt), $rec.id)
+    .mapIt(rowToModel(targetType, it))
+
+  PagedRecords[targetType](
+    pagination: page,
+    records: records,
+    totalRecords:
+      if page.isNone: records.len
+      else: db.getRow(sql(countStmt))[0].parseInt)
+
+template findViaJoinTable*[D: DbConnType](
+    db: D,
+    joinTableName: string,
+    targetType: type,
+    lookupType: type,
+    id: typed,
+    page: Option[PaginationParams]): untyped =
+  ## Find all records of `targetType` that are associated with a record of
+  ## `lookupType` via a join table.
+  let columns = columnNamesForModel(targetType).mapIt("t." & it).join(",")
+
+  var fetchStmt =
+    "SELECT " & columns &
+    " FROM " & tableName(targetType) & " AS t " &
+    " JOIN " & joinTableName & " AS j " &
+    " ON t.id = jt." & tableName(targetType) & "_id " &
+    " WHERE jt." & tableName(lookupType) & "_id = ?"
+
+  var countStmt =
+    "SELECT COUNT(*) FROM " & joinTableName &
+    " WHERE " & tableName(lookupType) & "_id = ?"
+
+  if page.isSome: fetchStmt &= getPagingClause(page.get)
+
+  logQuery("findViaJoinTable", fetchStmt, [("id", $id)])
+  let records = db.getAllRows(sql(fetchStmt), $id)
+    .mapIt(rowToModel(targetType, it))
+
+  PagedRecords[targetType](
+    pagination: page,
+    records: records,
+    totalRecords:
+      if page.isNone: records.len
+      else: db.getRow(sql(countStmt))[0].parseInt)
+
 macro generateProcsForModels*(dbType: type, modelTypes: openarray[type]): untyped =
   ## Generate all standard access procedures for the given model types. For a
   ## `model class`_ named `TodoItem`, this will generate the following
@@ -694,6 +743,103 @@ macro generateProcsForFieldLookups*(dbType: type, modelsAndFields: openarray[tup
 
     result.add procDefAST
 
+macro generateJoinTableProcs*(
+    dbType, model1Type, model2Type: type,
+    joinTableName: string): untyped =
+  ## Generate lookup procedures for a pair of models with a join table. For
+  ## example, given the TODO database demonstrated above, where `TodoItem` and
+  ## `TimeEntry` have a many-to-many relationship, you might have a join table
+  ## `todo_items_time_entries` with columns `todo_item_id` and `time_entry_id`.
+  ## This macro will generate the following procedures:
+  ##
+  ## .. code-block:: Nim
+  ##    proc findTodoItemsByTimeEntry*(db: SampleDB, timeEntry: TimeEntry): seq[TodoItem]
+  ##    proc findTimeEntriesByTodoItem*(db: SampleDB, todoItem: TodoItem): seq[TimeEntry]
+  ##
+  ## `dbType` is expected to be some type that has a defined `withConnection`
+  ## procedure (see `Database Object`_ for details).
+  ##
+  ## .. _Database Object: #database-object
+  result = newStmtList()
+
+  if model1Type.getType[1].typeKind == ntyRef or
+     model2Type.getType[1].typeKind == ntyRef:
+    raise newException(ValueError,
+      "fiber_orm model object must be objects, not refs")
+
+  let model1Name = $(model1Type.getType[1])
+  let model2Name = $(model2Type.getType[1])
+  let getModel1Name = ident("get" & pluralize(model1Name) & "By" & model2Name)
+  let getModel2Name = ident("get" & pluralize(model2Name) & "By" & model1Name)
+  let id1Type = typeOfColumn(model1Type, "id")
+  let id2Type = typeOfColumn(model2Type, "id")
+  let joinTableNameNode = newStrLitNode($joinTableName)
+
+  result.add quote do:
+    proc `getModel1Name`*(
+        db: `dbType`,
+        id: `id2Type`,
+        pagination = none[PaginationParams]()): PagedRecords[`model1Type`] =
+      db.withConnection conn:
+        result = findViaJoinTable(
+          conn,
+          `joinTableNameNode`,
+          `model1Type`,
+          `model2Type`,
+          id,
+          pagination)
+
+    proc `getModel1Name`*(
+        db: `dbType`,
+        rec: `model2Type`,
+        pagination = none[PaginationParams]()): PagedRecords[`model1Type`] =
+      db.withConnection conn:
+        result = findViaJoinTable(
+          conn,
+          `joinTableNameNode`,
+          `model1Type`,
+          rec,
+          pagination)
+
+    proc `getModel2Name`*(
+        db: `dbType`,
+        id: `id1Type`,
+        pagination = none[PaginationParams]()): Pagedrecords[`model2Type`] =
+      db.withConnection conn:
+        result = findViaJoinTable(
+          conn,
+          `joinTableNameNode`,
+          `model2Type`,
+          `model1Type`,
+          id,
+          pagination)
+
+    proc `getModel2Name`*(
+        db: `dbType`,
+        rec: `model1Type`,
+        pagination = none[PaginationParams]()): Pagedrecords[`model2Type`] =
+      db.withConnection conn:
+        result = findViaJoinTable(
+          conn,
+          `joinTableNameNode`,
+          `model2Type`,
+          rec,
+          pagination)
+
+    proc associate*(
+        db: `dbType`,
+        rec1: `model1Type`,
+        rec2: `model2Type`): void =
+      db.withConnection conn:
+        associate(conn, `joinTableNameNode`, rec1, rec2)
+
+    proc associate*(
+        db: `dbType`,
+        rec2: `model2Type`,
+        rec1: `model1Type`): void =
+      db.withConnection conn:
+        associate(conn, `joinTableNameNode`, rec1, rec2)
+
 proc initPool*[D: DbConnType](
     connect: proc(): D,
     poolSize = 10,

src/fiber_orm/util.nim

@@ -9,6 +9,11 @@ import uuids
 import std/nre except toSeq
 
 type
+  PaginationParams* = object
+    pageSize*: int
+    offset*: int
+    orderBy*: Option[seq[string]]
+
   MutateClauses* = object
     ## Data structure to hold information about the clauses that should be
     ## added to a query. How these clauses are used will depend on the query.
@@ -22,9 +27,11 @@ const ISO_8601_FORMATS = @[
   "yyyy-MM-dd'T'HH:mm:ssz",
   "yyyy-MM-dd'T'HH:mm:sszzz",
   "yyyy-MM-dd'T'HH:mm:ss'.'fffzzz",
+  "yyyy-MM-dd'T'HH:mm:ss'.'ffffzzz",
   "yyyy-MM-dd HH:mm:ssz",
   "yyyy-MM-dd HH:mm:sszzz",
-  "yyyy-MM-dd HH:mm:ss'.'fffzzz"
+  "yyyy-MM-dd HH:mm:ss'.'fffzzz",
+  "yyyy-MM-dd HH:mm:ss'.'ffffzzz"
 ]
 
 proc parseIso8601(val: string): DateTime =
@@ -126,18 +133,20 @@ proc parsePGDatetime*(val: string): DateTime =
 
   var correctedVal = val;
 
-  # PostgreSQL will truncate any trailing 0's in the millisecond value leading
-  # to values like `2020-01-01 16:42.3+00`. This cannot currently be parsed by
-  # the standard times format as it expects exactly three digits for
-  # millisecond values. So we have to detect this and pad out the millisecond
-  # value to 3 digits.
-  let PG_PARTIAL_FORMAT_REGEX = re"(\d{4}-\d{2}-\d{2}( |'T')\d{2}:\d{2}:\d{2}\.)(\d{1,2})(\S+)?"
+  # The Nim `times#format` function only recognizes 3-digit millisecond values
+  # but PostgreSQL will sometimes send 1-2 digits, truncating any trailing 0's,
+  # or sometimes provide more than three digits of preceision in the millisecond value leading
+  # to values like `2020-01-01 16:42.3+00` or `2025-01-06 00:56:00.9007+00`.
+  # This cannot currently be parsed by the standard times format as it expects
+  # exactly three digits for millisecond values. So we have to detect this and
+  # coerce the millisecond value to exactly 3 digits.
+  let PG_PARTIAL_FORMAT_REGEX = re"(\d{4}-\d{2}-\d{2}( |'T')\d{2}:\d{2}:\d{2}\.)(\d+)(\S+)?"
   let match = val.match(PG_PARTIAL_FORMAT_REGEX)
 
   if match.isSome:
     let c = match.get.captures
-    if c.toSeq.len == 2: correctedVal = c[0] & alignLeft(c[2], 3, '0')
-    else: correctedVal = c[0] & alignLeft(c[2], 3, '0') & c[3]
+    if c.toSeq.len == 2: correctedVal = c[0] & alignLeft(c[2], 3, '0')[0..2]
+    else: correctedVal = c[0] & alignLeft(c[2], 3, '0')[0..2] & c[3]
 
   var errStr = ""
 
@@ -146,7 +155,7 @@ proc parsePGDatetime*(val: string): DateTime =
     try: return correctedVal.parse(df)
     except: errStr &= "\n\t" & getCurrentExceptionMsg()
 
-  raise newException(ValueError, "Cannot parse PG date. Tried:" & errStr)
+  raise newException(ValueError, "Cannot parse PG date '" & correctedVal & "'. Tried:" & errStr)
 
 proc parseDbArray*(val: string): seq[string] =
   ## Parse a Postgres array column into a Nim seq[string]
@@ -208,7 +217,7 @@ proc parseDbArray*(val: string): seq[string] =
     result.add(curStr)
 
 func createParseStmt*(t, value: NimNode): NimNode =
-  ## Utility method to create the Nim cod required to parse a value coming from
+  ## Utility method to create the Nim code required to parse a value coming from
   ## the a database query. This is used by functions like `rowToModel` to parse
   ## the dataabase columns into the Nim object fields.
 
@@ -231,7 +240,7 @@ func createParseStmt*(t, value: NimNode): NimNode =
     elif t.getType == DateTime.getType:
       result = quote do: parsePGDatetime(`value`)
 
-    else: error "Unknown value object type: " & $t.getTypeInst
+    else: error "Cannot parse column with unknown object type: " & $t.getTypeInst
 
   elif t.typeKind == ntyGenericInst:
 
@@ -245,7 +254,7 @@ func createParseStmt*(t, value: NimNode): NimNode =
         if `value`.len == 0:  none[`innerType`]()
         else:                 some(`parseStmt`)
 
-    else: error "Unknown generic instance type: " & $t.getTypeInst
+    else: error "Cannot parse column with unknown generic instance type: " & $t.getTypeInst
 
   elif t.typeKind == ntyRef:
 
@@ -253,7 +262,7 @@ func createParseStmt*(t, value: NimNode): NimNode =
       result = quote do: parseJson(`value`)
 
     else:
-      error "Unknown ref type: " & $t.getTypeInst
+      error "Cannot parse column with unknown ref type: " & $t.getTypeInst
 
   elif t.typeKind == ntySequence:
     let innerType = t[1]
@@ -272,14 +281,14 @@ func createParseStmt*(t, value: NimNode): NimNode =
     result = quote do: parseFloat(`value`)
 
   elif t.typeKind == ntyBool:
-    result = quote do: "true".startsWith(`value`.toLower)
+    result = quote do: "true".startsWith(`value`.toLower) or `value` == "1"
 
   elif t.typeKind == ntyEnum:
     let innerType = t.getTypeInst
     result = quote do: parseEnum[`innerType`](`value`)
 
   else:
-    error "Unknown value type: " & $t.typeKind
+    error "Cannot parse column with unknown value type: " & $t.typeKind
 
 func fields(t: NimNode): seq[tuple[fieldIdent: NimNode, fieldType: NimNode]] =
   #[
@@ -447,6 +456,19 @@ macro populateMutateClauses*(t: typed, newRecord: bool, mc: var MutateClauses):
           `mc`.placeholders.add("?")
           `mc`.values.add(dbFormat(`t`.`fieldIdent`))
 
+
+proc getPagingClause*(page: PaginationParams): string =
+  ## Given a `PaginationParams` object, return the SQL clause necessary to
+  ## limit the number of records returned by a query.
+  result = ""
+  if page.orderBy.isSome:
+    let orderByClause = page.orderBy.get.map(identNameToDb).join(",")
+    result &= " ORDER BY " & orderByClause
+  else:
+    result &= " ORDER BY id"
+
+  result &= " LIMIT " & $page.pageSize & " OFFSET " & $page.offset
+
 ## .. _model class: ../fiber_orm.html#objectminusrelational-modeling-model-class
 ## .. _rules for name mapping: ../fiber_orm.html
 ## .. _table name: ../fiber_orm.html