From af44d48df1da0f5b67ef55e4be3e2f77d37ad096 Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Fri, 10 Jan 2025 20:25:13 -0600 Subject: [PATCH] Extract pagination logic into a common, exported function. Fix PG date parsing (again). --- fiber_orm.nimble | 2 +- src/fiber_orm.nim | 50 +++++------------------------------------- src/fiber_orm/util.nim | 42 ++++++++++++++++++++++++++--------- 3 files changed, 38 insertions(+), 56 deletions(-) diff --git a/fiber_orm.nimble b/fiber_orm.nimble index 4595ebc..cd1bddd 100644 --- a/fiber_orm.nimble +++ b/fiber_orm.nimble @@ -1,6 +1,6 @@ # Package -version = "3.0.0" +version = "3.1.0" author = "Jonathan Bernard" description = "Lightweight Postgres ORM for Nim." license = "GPL-3.0" diff --git a/src/fiber_orm.nim b/src/fiber_orm.nim index f3c2cbe..357a088 100644 --- a/src/fiber_orm.nim +++ b/src/fiber_orm.nim @@ -294,22 +294,9 @@ import ./fiber_orm/db_common as fiber_db_common import ./fiber_orm/pool import ./fiber_orm/util -export - pool, - util.columnNamesForModel, - util.dbFormat, - util.dbNameToIdent, - util.identNameToDb, - util.modelName, - util.rowToModel, - util.tableName +export pool, util type - PaginationParams* = object - pageSize*: int - offset*: int - orderBy*: Option[seq[string]] - PagedRecords*[T] = object pagination*: Option[PaginationParams] records*: seq[T] @@ -323,7 +310,7 @@ type var logService {.threadvar.}: Option[LogService] -proc logQuery(methodName: string, sqlStmt: string, args: openArray[(string, string)] = []) = +proc logQuery*(methodName: string, sqlStmt: string, args: openArray[(string, string)] = []) = # namespaced_logging would do this check for us, but we don't want to even # build the log object if we're not actually logging if logService.isNone: return @@ -444,16 +431,7 @@ template findRecordsWhere*[D: DbConnType]( "SELECT COUNT(*) FROM " & tableName(modelType) & " WHERE " & whereClause - if page.isSome: - let p = page.get - if p.orderBy.isSome: - let orderByClause = p.orderBy.get.map(identNameToDb).join(",") - fetchStmt &= " ORDER BY " & orderByClause - else: - fetchStmt &= " ORDER BY id" - - fetchStmt &= " LIMIT " & $p.pageSize & - " OFFSET " & $p.offset + if page.isSome: fetchStmt &= getPagingClause(page.get) logQuery("findRecordsWhere", fetchStmt, [("values", values.join(", "))]) let records = db.getAllRows(sql(fetchStmt), values).mapIt(rowToModel(modelType, it)) @@ -476,16 +454,7 @@ template getAllRecords*[D: DbConnType]( var countStmt = "SELECT COUNT(*) FROM " & tableName(modelType) - if page.isSome: - let p = page.get - if p.orderBy.isSome: - let orderByClause = p.orderBy.get.map(identNameToDb).join(",") - fetchStmt &= " ORDER BY " & orderByClause - else: - fetchStmt &= " ORDER BY id" - - fetchStmt &= " LIMIT " & $p.pageSize & - " OFFSET " & $p.offset + if page.isSome: fetchStmt &= getPagingClause(page.get) logQuery("getAllRecords", fetchStmt) let records = db.getAllRows(sql(fetchStmt)).mapIt(rowToModel(modelType, it)) @@ -516,16 +485,7 @@ template findRecordsBy*[D: DbConnType]( "SELECT COUNT(*) FROM " & tableName(modelType) & " WHERE " & whereClause - if page.isSome: - let p = page.get - if p.orderBy.isSome: - let orderByClause = p.orderBy.get.map(identNameToDb).join(",") - fetchStmt &= " ORDER BY " & orderByClause - else: - fetchStmt &= " ORDER BY id" - - fetchStmt &= " LIMIT " & $p.pageSize & - " OFFSET " & $p.offset + if page.isSome: fetchStmt &= getPagingClause(page.get) logQuery("findRecordsBy", fetchStmt, [("values", values.join(", "))]) let records = db.getAllRows(sql(fetchStmt), values).mapIt(rowToModel(modelType, it)) diff --git a/src/fiber_orm/util.nim b/src/fiber_orm/util.nim index b0cab50..d4e04cf 100644 --- a/src/fiber_orm/util.nim +++ b/src/fiber_orm/util.nim @@ -9,6 +9,11 @@ import uuids import std/nre except toSeq type + PaginationParams* = object + pageSize*: int + offset*: int + orderBy*: Option[seq[string]] + MutateClauses* = object ## Data structure to hold information about the clauses that should be ## added to a query. How these clauses are used will depend on the query. @@ -22,9 +27,11 @@ const ISO_8601_FORMATS = @[ "yyyy-MM-dd'T'HH:mm:ssz", "yyyy-MM-dd'T'HH:mm:sszzz", "yyyy-MM-dd'T'HH:mm:ss'.'fffzzz", + "yyyy-MM-dd'T'HH:mm:ss'.'ffffzzz", "yyyy-MM-dd HH:mm:ssz", "yyyy-MM-dd HH:mm:sszzz", - "yyyy-MM-dd HH:mm:ss'.'fffzzz" + "yyyy-MM-dd HH:mm:ss'.'fffzzz", + "yyyy-MM-dd HH:mm:ss'.'ffffzzz" ] proc parseIso8601(val: string): DateTime = @@ -126,18 +133,20 @@ proc parsePGDatetime*(val: string): DateTime = var correctedVal = val; - # PostgreSQL will truncate any trailing 0's in the millisecond value leading - # to values like `2020-01-01 16:42.3+00`. This cannot currently be parsed by - # the standard times format as it expects exactly three digits for - # millisecond values. So we have to detect this and pad out the millisecond - # value to 3 digits. - let PG_PARTIAL_FORMAT_REGEX = re"(\d{4}-\d{2}-\d{2}( |'T')\d{2}:\d{2}:\d{2}\.)(\d{1,2})(\S+)?" + # The Nim `times#format` function only recognizes 3-digit millisecond values + # but PostgreSQL will sometimes send 1-2 digits, truncating any trailing 0's, + # or sometimes provide more than three digits of preceision in the millisecond value leading + # to values like `2020-01-01 16:42.3+00` or `2025-01-06 00:56:00.9007+00`. + # This cannot currently be parsed by the standard times format as it expects + # exactly three digits for millisecond values. So we have to detect this and + # coerce the millisecond value to exactly 3 digits. + let PG_PARTIAL_FORMAT_REGEX = re"(\d{4}-\d{2}-\d{2}( |'T')\d{2}:\d{2}:\d{2}\.)(\d+)(\S+)?" let match = val.match(PG_PARTIAL_FORMAT_REGEX) if match.isSome: let c = match.get.captures - if c.toSeq.len == 2: correctedVal = c[0] & alignLeft(c[2], 3, '0') - else: correctedVal = c[0] & alignLeft(c[2], 3, '0') & c[3] + if c.toSeq.len == 2: correctedVal = c[0] & alignLeft(c[2], 3, '0')[0..2] + else: correctedVal = c[0] & alignLeft(c[2], 3, '0')[0..2] & c[3] var errStr = "" @@ -146,7 +155,7 @@ proc parsePGDatetime*(val: string): DateTime = try: return correctedVal.parse(df) except: errStr &= "\n\t" & getCurrentExceptionMsg() - raise newException(ValueError, "Cannot parse PG date. Tried:" & errStr) + raise newException(ValueError, "Cannot parse PG date '" & correctedVal & "'. Tried:" & errStr) proc parseDbArray*(val: string): seq[string] = ## Parse a Postgres array column into a Nim seq[string] @@ -447,6 +456,19 @@ macro populateMutateClauses*(t: typed, newRecord: bool, mc: var MutateClauses): `mc`.placeholders.add("?") `mc`.values.add(dbFormat(`t`.`fieldIdent`)) + +proc getPagingClause*(page: PaginationParams): string = + ## Given a `PaginationParams` object, return the SQL clause necessary to + ## limit the number of records returned by a query. + result = "" + if page.orderBy.isSome: + let orderByClause = page.orderBy.get.map(identNameToDb).join(",") + result &= " ORDER BY " & orderByClause + else: + result &= " ORDER BY id" + + result &= " LIMIT " & $page.pageSize & " OFFSET " & $page.offset + ## .. _model class: ../fiber_orm.html#objectminusrelational-modeling-model-class ## .. _rules for name mapping: ../fiber_orm.html ## .. _table name: ../fiber_orm.html