From ead77534ce5449287530d62586a1f360aa5534fd Mon Sep 17 00:00:00 2001
From: Jonathan Bernard <jonathan@jdbernard.com>
Date: Sun, 9 Feb 2020 00:31:40 -0600
Subject: [PATCH] api: Extract database common code into its own library
(fiber-orm).
---
api/personal_measure_api.nimble | 3 +-
.../main/nim/personal_measure_apipkg/db.nim | 33 +-
.../nim/personal_measure_apipkg/db_common.nim | 150 ---------
.../nim/personal_measure_apipkg/db_util.nim | 287 ------------------
web/{.env.dev => .env.development} | 0
5 files changed, 21 insertions(+), 452 deletions(-)
delete mode 100644 api/src/main/nim/personal_measure_apipkg/db_common.nim
delete mode 100644 api/src/main/nim/personal_measure_apipkg/db_util.nim
rename web/{.env.dev => .env.development} (100%)
diff --git a/api/personal_measure_api.nimble b/api/personal_measure_api.nimble
index 413d79e..5f21db1 100644
--- a/api/personal_measure_api.nimble
+++ b/api/personal_measure_api.nimble
@@ -14,7 +14,8 @@ skipExt = @["nim"]
# Dependencies
requires @["nim >= 0.19.4", "bcrypt", "docopt >= 0.6.8", "isaac >= 0.1.3",
- "jester >= 0.4.1", "jwt", "tempfile", "uuids >= 0.1.10" ]
+ "jester >= 0.4.3", "jwt", "tempfile", "uuids >= 0.1.10" ]
requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git >= 0.6.3"
requires "https://git.jdb-labs.com/jdb/nim-time-utils.git >= 0.5.0"
+requires "https://git.jdb-labs.com/jdb-labs/fiber-orm-nim.git >= 0.1.2"
diff --git a/api/src/main/nim/personal_measure_apipkg/db.nim b/api/src/main/nim/personal_measure_apipkg/db.nim
index cd29b18..0a5089c 100644
--- a/api/src/main/nim/personal_measure_apipkg/db.nim
+++ b/api/src/main/nim/personal_measure_apipkg/db.nim
@@ -1,10 +1,9 @@
-import db_postgres, macros, options, postgres, sequtils, strutils,
- times, timeutils, unicode, uuids
+import db_postgres, uuids
import ./models
-import ./db_common
+import ../../../../../../jdb-labs/fiber-orm-nim/src/fiber_orm
-export db_common.NotFoundError
+export fiber_orm.NotFoundError
type
PMApiDb* = ref object
@@ -14,18 +13,24 @@ type
proc connect*(connString: string): PMApiDb =
result = PMApiDb(conn: open("", "", "", connString))
-generateProcsForModels([User, ApiToken, Measure, Measurement, ClientLogEntry])
+generateProcsForModels(PMApiDb, [
+ User,
+ ApiToken,
+ Measure,
+ Measurement,
+ ClientLogEntry
+])
-generateLookup(User, @["email"])
+generateLookup(PMApiDb, User, @["email"])
-generateLookup(ApiToken, @["userId"])
-generateLookup(ApiToken, @["hashedToken"])
+generateLookup(PMApiDb, ApiToken, @["userId"])
+generateLookup(PMApiDb, ApiToken, @["hashedToken"])
-generateLookup(Measure, @["userId"])
-generateLookup(Measure, @["userId", "id"])
-generateLookup(Measure, @["userId", "slug"])
+generateLookup(PMApiDb, Measure, @["userId"])
+generateLookup(PMApiDb, Measure, @["userId", "id"])
+generateLookup(PMApiDb, Measure, @["userId", "slug"])
-generateLookup(Measurement, @["measureId"])
-generateLookup(Measurement, @["measureId", "id"])
+generateLookup(PMApiDb, Measurement, @["measureId"])
+generateLookup(PMApiDb, Measurement, @["measureId", "id"])
-generateLookup(ClientLogEntry, @["userId"])
+generateLookup(PMApiDb, ClientLogEntry, @["userId"])
diff --git a/api/src/main/nim/personal_measure_apipkg/db_common.nim b/api/src/main/nim/personal_measure_apipkg/db_common.nim
deleted file mode 100644
index 08e8f40..0000000
--- a/api/src/main/nim/personal_measure_apipkg/db_common.nim
+++ /dev/null
@@ -1,150 +0,0 @@
-import db_postgres, macros, options, sequtils, strutils, uuids
-
-from unicode import capitalize
-
-import ./db_util
-
-type NotFoundError* = object of CatchableError
-
-proc newMutateClauses(): MutateClauses =
- return MutateClauses(
- columns: @[],
- placeholders: @[],
- values: @[])
-
-proc createRecord*[T](db: DbConn, rec: T): T =
- var mc = newMutateClauses()
- populateMutateClauses(rec, true, mc)
-
- # Confusingly, getRow allows inserts and updates. We use it to get back the ID
- # we want from the row.
- let newRow = db.getRow(sql(
- "INSERT INTO " & tableName(rec) &
- " (" & mc.columns.join(",") & ") " &
- " VALUES (" & mc.placeholders.join(",") & ") " &
- " RETURNING *"), mc.values)
-
- result = rowToModel(T, newRow)
-
-proc updateRecord*[T](db: DbConn, rec: T): bool =
- var mc = newMutateClauses()
- populateMutateClauses(rec, false, mc)
-
- let setClause = zip(mc.columns, mc.placeholders).mapIt(it.a & " = " & it.b).join(",")
- let numRowsUpdated = db.execAffectedRows(sql(
- "UPDATE " & tableName(rec) &
- " SET " & setClause &
- " WHERE id = ? "), mc.values.concat(@[$rec.id]))
-
- return numRowsUpdated > 0;
-
-template deleteRecord*(db: DbConn, modelType: type, id: typed): untyped =
- db.tryExec(sql("DELETE FROM " & tableName(modelType) & " WHERE id = ?"), $id)
-
-proc deleteRecord*[T](db: DbConn, rec: T): bool =
- return db.tryExec(sql("DELETE FROM " & tableName(rec) & " WHERE id = ?"), $rec.id)
-
-template getRecord*(db: DbConn, modelType: type, id: typed): untyped =
- let row = db.getRow(sql(
- "SELECT " & columnNamesForModel(modelType).join(",") &
- " FROM " & tableName(modelType) &
- " WHERE id = ?"), @[$id])
-
- if row.allIt(it.len == 0):
- raise newException(NotFoundError, "no record for id " & $id)
-
- rowToModel(modelType, row)
-
-template findRecordsWhere*(db: DbConn, modelType: type, whereClause: string, values: varargs[string, dbFormat]): untyped =
- db.getAllRows(sql(
- "SELECT " & columnNamesForModel(modelType).join(",") &
- " FROM " & tableName(modelType) &
- " WHERE " & whereClause), values)
- .mapIt(rowToModel(modelType, it))
-
-template getAllRecords*(db: DbConn, modelType: type): untyped =
- db.getAllRows(sql(
- "SELECT " & columnNamesForModel(modelType).join(",") &
- " FROM " & tableName(modelType)))
- .mapIt(rowToModel(modelType, it))
-
-template findRecordsBy*(db: DbConn, modelType: type, lookups: seq[tuple[field: string, value: string]]): untyped =
- db.getAllRows(sql(
- "SELECT " & columnNamesForModel(modelType).join(",") &
- " FROM " & tableName(modelType) &
- " WHERE " & lookups.mapIt(it.field & " = ?").join(" AND ")),
- lookups.mapIt(it.value))
- .mapIt(rowToModel(modelType, it))
-
-macro generateProcsForModels*(modelTypes: openarray[type]): untyped =
- result = newStmtList()
-
- for t in modelTypes:
- let modelName = $(t.getType[1])
- let getName = ident("get" & modelName)
- let getAllName = ident("getAll" & modelName & "s")
- let findWhereName = ident("find" & modelName & "sWhere")
- let createName = ident("create" & modelName)
- let updateName = ident("update" & modelName)
- let deleteName = ident("delete" & modelName)
- let idType = typeOfColumn(t, "id")
- result.add quote do:
- proc `getName`*(db: PMApiDb, id: `idType`): `t` = getRecord(db.conn, `t`, id)
- proc `getAllName`*(db: PMApiDb): seq[`t`] = getAllRecords(db.conn, `t`)
- proc `findWhereName`*(db: PMApiDb, whereClause: string, values: varargs[string, dbFormat]): seq[`t`] =
- return findRecordsWhere(db.conn, `t`, whereClause, values)
- proc `createName`*(db: PMApiDb, rec: `t`): `t` = createRecord(db.conn, rec)
- proc `updateName`*(db: PMApiDb, rec: `t`): bool = updateRecord(db.conn, rec)
- proc `deleteName`*(db: PMApiDb, rec: `t`): bool = deleteRecord(db.conn, rec)
- proc `deleteName`*(db: PMApiDb, id: `idType`): bool = deleteRecord(db.conn, `t`, id)
-
-macro generateLookup*(modelType: type, fields: seq[string]): untyped =
- let fieldNames = fields[1].mapIt($it)
- let procName = ident("find" & $modelType.getType[1] & "sBy" & fieldNames.mapIt(it.capitalize).join("And"))
-
- # Create proc skeleton
- result = quote do:
- proc `procName`*(db: PMApiDb): seq[`modelType`] =
- return findRecordsBy(db.conn, `modelType`)
-
- var callParams = quote do: @[]
-
- # Add dynamic parameters for the proc definition and inner proc call
- for n in fieldNames:
- let paramTuple = newNimNode(nnkPar)
- paramTuple.add(newColonExpr(ident("field"), newLit(identNameToDb(n))))
- paramTuple.add(newColonExpr(ident("value"), ident(n)))
-
- result[3].add(newIdentDefs(ident(n), ident("string")))
- callParams[1].add(paramTuple)
-
- result[6][0][0].add(callParams)
-
-macro generateProcsForFieldLookups*(modelsAndFields: openarray[tuple[t: type, fields: seq[string]]]): untyped =
- result = newStmtList()
-
- for i in modelsAndFields:
- var modelType = i[1][0]
- let fieldNames = i[1][1][1].mapIt($it)
-
- let procName = ident("find" & $modelType & "sBy" & fieldNames.mapIt(it.capitalize).join("And"))
-
- # Create proc skeleton
- let procDefAST = quote do:
- proc `procName`*(db: PMApiDb): seq[`modelType`] =
- return findRecordsBy(db.conn, `modelType`)
-
- var callParams = quote do: @[]
-
- # Add dynamic parameters for the proc definition and inner proc call
- for n in fieldNames:
- let paramTuple = newNimNode(nnkPar)
- paramTuple.add(newColonExpr(ident("field"), newLit(n)))
- paramTuple.add(newColonExpr(ident("value"), ident(n)))
-
- procDefAST[3].add(newIdentDefs(ident(n), ident("string")))
- callParams[1].add(paramTuple)
-
- procDefAST[6][0][0].add(callParams)
-
- result.add procDefAST
diff --git a/api/src/main/nim/personal_measure_apipkg/db_util.nim b/api/src/main/nim/personal_measure_apipkg/db_util.nim
deleted file mode 100644
index e99ec65..0000000
--- a/api/src/main/nim/personal_measure_apipkg/db_util.nim
+++ /dev/null
@@ -1,287 +0,0 @@
-import json, macros, options, sequtils, strutils, times, timeutils, unicode,
- uuids
-
-const UNDERSCORE_RUNE = "_".toRunes[0]
-const PG_TIMESTAMP_FORMATS = [
- "yyyy-MM-dd HH:mm:sszz",
- "yyyy-MM-dd HH:mm:ss'.'fzz",
- "yyyy-MM-dd HH:mm:ss'.'ffzz",
- "yyyy-MM-dd HH:mm:ss'.'fffzz"
-]
-
-type
- MutateClauses* = object
- columns*: seq[string]
- placeholders*: seq[string]
- values*: seq[string]
-
-# TODO: more complete implementation
-# see https://github.com/blakeembrey/pluralize
-proc pluralize(name: string): string =
- if name[^2..^1] == "ey": return name[0..^3] & "ies"
- if name[^1] == 'y': return name[0..^2] & "ies"
- return name & "s"
-
-macro modelName*(model: object): string =
- return $model.getTypeInst
-
-macro modelName*(modelType: type): string =
- return $modelType.getType[1]
-
-proc identNameToDb*(name: string): string =
- let nameInRunes = name.toRunes
- var prev: Rune
- var resultRunes = newSeq[Rune]()
-
- for cur in nameInRunes:
- if resultRunes.len == 0:
- resultRunes.add(toLower(cur))
- elif isLower(prev) and isUpper(cur):
- resultRunes.add(UNDERSCORE_RUNE)
- resultRunes.add(toLower(cur))
- else: resultRunes.add(toLower(cur))
-
- prev = cur
-
- return $resultRunes
-
-proc dbNameToIdent*(name: string): string =
- let parts = name.split("_")
- return @[parts[0]].concat(parts[1..^1].mapIt(capitalize(it))).join("")
-
-proc tableName*(modelType: type): string =
- return pluralize(modelName(modelType).identNameToDb)
-
-proc tableName*[T](rec: T): string =
- return pluralize(modelName(rec).identNameToDb)
-
-proc dbFormat*(s: string): string = return s
-
-proc dbFormat*(dt: DateTime): string = return dt.formatIso8601
-
-proc dbFormat*[T](list: seq[T]): string =
- return "{" & list.mapIt(dbFormat(it)).join(",") & "}"
-
-proc dbFormat*[T](item: T): string = return $item
-
-type DbArrayParseState = enum
- expectStart, inQuote, inVal, expectEnd
-
-proc parsePGDatetime*(val: string): DateTime =
- var errStr = ""
- for df in PG_TIMESTAMP_FORMATS:
- try: return val.parse(df)
- except: errStr &= "\n" & getCurrentExceptionMsg()
- raise newException(ValueError, "Cannot parse PG date. Tried:" & errStr)
-
-proc parseDbArray*(val: string): seq[string] =
- result = newSeq[string]()
-
- var parseState = DbArrayParseState.expectStart
- var curStr = ""
- var idx = 1
- var sawEscape = false
-
- while idx < val.len - 1:
- var curChar = val[idx]
- idx += 1
-
- case parseState:
-
- of expectStart:
- if curChar == ' ': continue
- elif curChar == '"':
- parseState = inQuote
- continue
- else:
- parseState = inVal
-
- of expectEnd:
- if curChar == ' ': continue
- elif curChar == ',':
- result.add(curStr)
- curStr = ""
- parseState = expectStart
- continue
-
- of inQuote:
- if curChar == '"' and not sawEscape:
- parseState = expectEnd
- continue
-
- of inVal:
- if curChar == '"' and not sawEscape:
- raise newException(ValueError, "Invalid DB array value (cannot have '\"' in the middle of an unquoted string).")
- elif curChar == ',':
- result.add(curStr)
- curStr = ""
- parseState = expectStart
- continue
-
- # if we saw an escaped \", add just the ", otherwise add both
- if sawEscape:
- if curChar != '"': curStr.add('\\')
- curStr.add(curChar)
- sawEscape = false
-
- elif curChar == '\\':
- sawEscape = true
-
- else: curStr.add(curChar)
-
- if not (parseState == inQuote) and curStr.len > 0:
- result.add(curStr)
-
-proc createParseStmt*(t, value: NimNode): NimNode =
-
- #echo "Creating parse statment for ", t.treeRepr
- if t.typeKind == ntyObject:
-
- if t.getType == UUID.getType:
- result = quote do: parseUUID(`value`)
-
- elif t.getType == DateTime.getType:
- result = quote do: parsePGDatetime(`value`)
-
- elif t.getTypeInst == Option.getType:
- let innerType = t.getTypeImpl[2][0][0][1]
- let parseStmt = createParseStmt(innerType, value)
- result = quote do:
- if `value`.len == 0: none[`innerType`]()
- else: some(`parseStmt`)
-
- else: error "Unknown value object type: " & $t.getTypeInst
-
- elif t.typeKind == ntyRef:
-
- if $t.getTypeInst == "JsonNode":
- result = quote do: parseJson(`value`)
-
- else:
- error "Unknown ref type: " & $t.getTypeInst
-
- elif t.typeKind == ntySequence:
- let innerType = t[1]
-
- let parseStmts = createParseStmt(innerType, ident("it"))
-
- result = quote do: parseDbArray(`value`).mapIt(`parseStmts`)
-
- elif t.typeKind == ntyString:
- result = quote do: `value`
-
- elif t.typeKind == ntyInt:
- result = quote do: parseInt(`value`)
-
- elif t.typeKind == ntyBool:
- result = quote do: "true".startsWith(`value`.toLower)
-
- else:
- error "Unknown value type: " & $t.typeKind
-
-template walkFieldDefs*(t: NimNode, body: untyped) =
- let tTypeImpl = t.getTypeImpl
-
- var nodeToItr: NimNode
- if tTypeImpl.typeKind == ntyObject: nodeToItr = tTypeImpl[2]
- elif tTypeImpl.typeKind == ntyTypeDesc: nodeToItr = tTypeImpl.getType[1].getType[2]
- else: error $t & " is not an object or type desc (it's a " & $tTypeImpl.typeKind & ")."
-
- for fieldDef {.inject.} in nodeToItr.children:
- # ignore AST nodes that are not field definitions
- if fieldDef.kind == nnkIdentDefs:
- let fieldIdent {.inject.} = fieldDef[0]
- let fieldType {.inject.} = fieldDef[1]
- body
-
- elif fieldDef.kind == nnkSym:
- let fieldIdent {.inject.} = fieldDef
- let fieldType {.inject.} = fieldDef.getType
- body
-
-macro columnNamesForModel*(modelType: typed): seq[string] =
- var columnNames = newSeq[string]()
-
- modelType.walkFieldDefs:
- columnNames.add(identNameToDb($fieldIdent))
-
- result = newLit(columnNames)
-
-macro rowToModel*(modelType: typed, row: seq[string]): untyped =
-
- # Create the object constructor AST node
- result = newNimNode(nnkObjConstr).add(modelType)
-
- # Create new colon expressions for each of the property initializations
- var idx = 0
- modelType.walkFieldDefs:
- let itemLookup = quote do: `row`[`idx`]
- result.add(newColonExpr(
- fieldIdent,
- createParseStmt(fieldType, itemLookup)))
- idx += 1
-
-macro listFields*(t: typed): untyped =
- var fields: seq[tuple[n: string, t: string]] = @[]
- t.walkFieldDefs:
- if fieldDef.kind == nnkSym: fields.add((n: $fieldIdent, t: fieldType.repr))
- else: fields.add((n: $fieldIdent, t: $fieldType))
-
- result = newLit(fields)
-
-proc typeOfColumn*(modelType: NimNode, colName: string): NimNode =
- modelType.walkFieldDefs:
- if $fieldIdent != colName: continue
-
- if fieldType.typeKind == ntyObject:
-
- if fieldType.getType == UUID.getType: return ident("UUID")
- elif fieldType.getType == DateTime.getType: return ident("DateTime")
- elif fieldType.getType == Option.getType: return ident("Option")
- else: error "Unknown column type: " & $fieldType.getTypeInst
-
- else: return fieldType
-
- raise newException(Exception,
- "model of type '" & $modelType & "' has no column named '" & colName & "'")
-
-proc isZero(val: int): bool = return val == 0
-
-macro populateMutateClauses*(t: typed, newRecord: bool, mc: var MutateClauses): untyped =
-
- result = newStmtList()
-
- # iterate over all the object's fields
- t.walkFieldDefs:
-
- # grab the field, it's string name, and it's type
- let fieldName = $fieldIdent
-
- # we do not update the ID, but we do check: if we're creating a new
- # record, we should not have an existing ID
- if fieldName == "id":
- result.add quote do:
- if `newRecord` and not `t`.id.isZero:
- raise newException(
- AssertionError,
- "Trying to create a new record, but the record already has an ID (" & $(`t`.id) & ").")
-
- # if we're looking at an optional field, add logic to check for presence
- elif fieldType.kind == nnkBracketExpr and
- fieldType.len > 0 and
- fieldType[0] == Option.getType:
-
- result.add quote do:
- `mc`.columns.add(identNameToDb(`fieldName`))
- if `t`.`fieldIdent`.isSome:
- `mc`.placeholders.add("?")
- `mc`.values.add(dbFormat(`t`.`fieldIdent`.get))
- else:
- `mc`.placeholders.add("NULL")
-
- # otherwise assume we can convert and go ahead.
- else:
- result.add quote do:
- `mc`.columns.add(identNameToDb(`fieldName`))
- `mc`.placeholders.add("?")
- `mc`.values.add(dbFormat(`t`.`fieldIdent`))
diff --git a/web/.env.dev b/web/.env.development
similarity index 100%
rename from web/.env.dev
rename to web/.env.development