WIP refactoring db to better test. Added parseDbArray function.

api/src/main/nim/personal_measure_apipkg/db.nim

@@ -1,17 +1,8 @@
-import db_postgres, macros, options, postgres, sequtils, strutils, times, timeutils, uuids
+import db_postgres, macros, options, postgres, sequtils, strutils, times,
+  timeutils, unicode, uuids
 
 import ./models
-
+import ./db_util
-import nre except toSeq
-from unicode import capitalize, toLower
-
-type
-  MutateClauses = object
-    columns*: seq[string]
-    placeholders*: seq[string]
-    values*: seq[string]
-
-let UPPERCASE_PATTERN = re"(.)(\p{Lu})"
 
 proc newMutateClauses(): MutateClauses =
   return MutateClauses(
@@ -19,141 +10,7 @@ proc newMutateClauses(): MutateClauses =
     placeholders: @[],
     values: @[])
 
-proc dbFormat(s: string): string = return s
+proc createRecord*[T](db: DbConn, rec: T): T =
-
-proc dbFormat(dt: DateTime): string = return dt.formatIso8601
-
-proc dbFormat[T](list: seq[T]): string =
-  return "{" & list.mapIt("'" & dbFormat(it) & "'").join(",") & "}"
-
-proc dbFormat[T](item: T): string = return $item
-
-proc createParseStmt(t, value: NimNode): NimNode =
-  result = newStmtList()
-
-  if t.typeKind == ntyObject:
-    if t.getType == UUID.getType:
-      result.add quote do:
-        parseUUID(`value`)
-    elif t.getType == DateTime.getType:
-      result.add quote do:
-        `value`.parseIso8601
-    elif t.getTypeInst == Option.getType:
-      let innerType = t.getTypeImpl[2][0][0][1]
-      let parseStmt = createParseStmt(innerType, value)
-      result.add quote do:
-        if `value`.len == 0:
-          none[`innerType`]()
-        else:
-          some(`parseStmt`)
-    else:
-      error "Unknown value object type: " & $t.getTypeInst
-  elif t.typeKind == ntyString:
-    result.add quote do:
-      `value`
-  elif t.typeKind == ntyInt:
-    result.add quote do:
-      parseInt(`value`)
-  else:
-    error "Unknown value type: " & $t.getTypeInst
-
-template walkFieldDefs(t: NimNode, body: untyped) =
-  let tTypeImpl = t.getTypeImpl
-
-  var nodeToItr: NimNode
-  if tTypeImpl.typeKind == ntyObject: nodeToItr = tTypeImpl[2]
-  elif tTypeImpl.typeKind == ntyTypeDesc: nodeToItr = tTypeImpl.getType[1].getType[2]
-  else: error $t & " is not an object or type desc (it's a " & $tTypeImpl.typeKind & ")."
-
-  for fieldDef {.inject.} in nodeToItr.children:
-    # ignore AST nodes that are not field definitions
-    if fieldDef.kind == nnkIdentDefs:
-      let fieldIdent {.inject.} = fieldDef[0]
-      let fieldType {.inject.} = fieldDef[1]
-      body
-
-    elif fieldDef.kind == nnkSym:
-      let fieldIdent {.inject.} = fieldDef
-      let fieldType {.inject.} = fieldDef.getType
-      body
-
-macro populateMutateClauses(t: typed, newRecord: bool, mc: var MutateClauses): untyped =
-
-  result = newStmtList()
-
-  # iterate over all the object's fields
-  t.walkFieldDefs:
-
-      # grab the field, it's string name, and it's type
-      let fieldName = $fieldIdent
-
-      # we do not update the ID, but we do check: if we're creating a new
-      # record, we should not have an existing ID
-      if fieldName == "id":
-        result.add quote do:
-          if `newRecord` and not `t`.id.isZero:
-            raise newException(
-              AssertionError,
-              "Trying to create a new record, but the record already has an ID.")
-
-      # if we're looking at an optional field, add logic to check for presence
-      elif fieldType.kind == nnkBracketExpr and
-         fieldType.len > 0 and
-         fieldType[0] == Option.getType:
-
-        result.add quote do:
-          `mc`.columns.add(identNameToDb(`fieldName`))
-          if `t`.`fieldIdent`.isSome:
-            `mc`.placeholders.add("?")
-            `mc`.values.add(dbFormat(`t`.`fieldIdent`.get))
-          else:
-            `mc`.placeholders.add("NULL")
-
-      # otherwise assume we can convert and go ahead.
-      else:
-        result.add quote do:
-          `mc`.columns.add(identNameToDb(`fieldName`))
-          `mc`.placeholders.add("?")
-          `mc`.values.add(dbFormat(`t`.`fieldIdent`))
-
-  #echo result.repr
-
-macro rowToModel(modelType: typed, row: seq[string]): untyped =
-
-  # Create the object constructor AST node
-  result = newNimNode(nnkObjConstr).add(modelType)
-
-  # Create new colon expressions for each of the property initializations
-  var idx = 0
-  echo modelType.getTypeImpl.treeRepr
-  # TODO: how to guarantee order?
-  modelType.walkFieldDefs:
-    let itemLookup = quote do: `row`[`idx`]
-    result.add(newColonExpr(
-      fieldIdent,
-      createParseStmt(fieldType, itemLookup)))
-    idx += 1
-
-macro modelName(model: object): string =
-  return $model.getTypeInst
-
-macro modelName(modelType: type): string =
-  return $modelType.getType[1]
-
-proc identNameToDb(name: string): string =
-  return name.replace(UPPERCASE_PATTERN, "$1_$2").toLower()
-
-proc dbNameToIdent(name: string): string =
-  let parts = name.split("_")
-  return @[parts[0]].concat(parts[1..^1].mapIt(capitalize(it))).join("")
-
-proc tableName(modelType: type): string =
-  return modelName(modelType).identNameToDb & "s"
-
-proc tableName[T](rec: T): string =
-  return modelName(rec).identNameToDb & "s"
-
-proc createRecord[T](db: DbConn, rec: T): T =
   var mc = newMutateClauses()
   populateMutateClauses(rec, true, mc)
 
@@ -168,7 +25,7 @@ proc createRecord[T](db: DbConn, rec: T): T =
   result = rec
   result.id = parseUUID(newIdStr)
   
-proc updateRecord[T](db: DbConn, rec: T): bool =
+proc updateRecord*[T](db: DbConn, rec: T): bool =
   var mc = newMutateClauses()
   populateMutateClauses(rec, false, mc)
 
@@ -180,17 +37,18 @@ proc updateRecord[T](db: DbConn, rec: T): bool =
 
   return numRowsUpdated > 0;
 
-template getRecord(db: DbConn, modelType: type, id: UUID): untyped =
+template getRecord*(db: DbConn, modelType: type, id: UUID): untyped =
-  let row = db.getRow(sql("SELECT * FROM " & tableName(modelType) & " WHERE id = ?"), @[$id])
+  let row = db.getRow(sql(
+    "SELECT " & columnNamesForModel(modelType).join(",") &
+    " FROM " & tableName(modelType) &
+    " WHERE id = ?"), @[$id])
   rowToModel(modelType, row)
 
-macro listFields(t: typed): untyped =
+template getAllRecords*(db: DbConn, modelType: type): untyped =
-  var fields: seq[tuple[n: string, t: string]] = @[]
+  db.getAllRows(sql(
-  t.walkFieldDefs:
+    "SELECT " & columnNamesForModel(modelType).join(",") &
-    if fieldDef.kind == nnkSym: fields.add((n: $fieldIdent, t: fieldType.repr))
+    " FROM " & tableName(modelType)))
-    else: fields.add((n: $fieldIdent, t: $fieldType))
+    .mapIt(rowToModel(modelType, it))
-
-  result = newLit(fields)
 
 # proc create: Typed create methods for specific records
 proc createUser*(db: DbConn, user: User): User = return db.createRecord(user)
@@ -200,20 +58,27 @@ proc createValue*(db: DbConn, value: Value): Value = return db.createRecord(valu
 
 proc getUser*(db: DbConn, id: UUID): User = return db.getRecord(User, id)
 proc getApiToken*(db: DbConn, id: UUID): ApiToken = return db.getRecord(ApiToken, id)
+proc getMeasure*(db: DbConn, id: UUID): Measure = return db.getRecord(Measure, id)
+#proc getValue*(db: DbConn, id: UUID): Value = return db.getRecord(Value, id)
 
 when isMainModule:
 
-  let db = open("", "", "", "host=192.168.99.100 port=5500 dbname=personal_measure user=postgres password=password")
+  let db = open("", "", "", "host=localhost port=5500 dbname=personal_measure user=postgres password=password")
 
-  for row in db.fastRows(sql"SELECT id FROM users"):
+  echo "Users:"
-    echo $db.getUser(parseUUID(row[0]))
+  echo $db.getAllRecords(User)
 
-  for row in db.fastRows(sql"SELECT id FROM api_tokens"):
+  echo "\nApiTokens:"
-    echo $db.getApiToken(parseUUID(row[0]))
+  echo $db.getAllRecords(ApiToken)
 
+  echo "\nMeasures:"
+  let measures = db.getAllRecords(Measure)
+  echo $measures
+  echo "\tanalysis: ", measures[0].analysis[0]
+
+  #[
   #echo tableName(ApiToken)
-  echo $rowToModel(ApiToken, @["47400441-5c3a-4119-8acf-f616ae25c16c", "9e5460dd-b580-4071-af97-c1cbdedaae12", "Test Token", "5678", ""])
+  #echo $rowToModel(ApiToken, @["47400441-5c3a-4119-8acf-f616ae25c16c", "9e5460dd-b580-4071-af97-c1cbdedaae12", "Test Token", "5678", ""])
-#[
   for row in db.fastRows(sql"SELECT * FROM api_tokens"):
     echo $rowToModel(ApiToken, row);
 
@@ -222,4 +87,5 @@ when isMainModule:
     email: "bob@bobsco.com",
     hashedPwd: "test")
 
-]#
+  ]#
+

api/src/main/nim/personal_measure_apipkg/db_util.nim

@@ -0,0 +1,249 @@
+import macros, options, sequtils, strutils, times, timeutils, unicode, uuids
+
+const underscoreRune = "_".toRunes[0]
+
+type
+  MutateClauses* = object
+    columns*: seq[string]
+    placeholders*: seq[string]
+    values*: seq[string]
+
+macro modelName*(model: object): string =
+  return $model.getTypeInst
+
+macro modelName*(modelType: type): string =
+  return $modelType.getType[1]
+
+proc identNameToDb*(name: string): string =
+  let nameInRunes = name.toRunes
+  var prev: Rune
+  var resultRunes = newSeq[Rune]()
+
+  for cur in nameInRunes:
+    if resultRunes.len == 0:
+      resultRunes.add(toLower(cur))
+    elif isLower(prev) and isUpper(cur):
+      resultRunes.add(underscoreRune)
+      resultRunes.add(toLower(cur))
+    else: resultRunes.add(toLower(cur))
+
+    prev = cur
+
+  return $resultRunes
+
+proc dbNameToIdent*(name: string): string =
+  let parts = name.split("_")
+  return @[parts[0]].concat(parts[1..^1].mapIt(capitalize(it))).join("")
+
+proc tableName*(modelType: type): string =
+  return modelName(modelType).identNameToDb & "s"
+
+proc tableName*[T](rec: T): string =
+  return modelName(rec).identNameToDb & "s"
+
+proc dbFormat*(s: string): string = return s
+
+proc dbFormat*(dt: DateTime): string = return dt.formatIso8601
+
+proc dbFormat*[T](list: seq[T]): string =
+  return "{" & list.mapIt(dbFormat(it)).join(",") & "}"
+
+proc dbFormat*[T](item: T): string = return $item
+
+type DbArrayParseState = enum
+  expectStart, inQuote, inVal, expectEnd
+
+proc parseDbArray*(val: string): seq[string] =
+  result = newSeq[string]()
+
+  var parseState = DbArrayParseState.expectStart
+  var curStr = ""
+  var idx = 1
+  var sawEscape = false
+
+  while idx < val.len - 1:
+    var curChar = val[idx]
+    idx += 1
+
+    case parseState:
+
+      of expectStart:
+        if curChar == ' ': continue
+        elif curChar == '"':
+          parseState = inQuote
+          continue
+        else:
+          parseState = inVal
+
+      of expectEnd:
+        if curChar == ' ': continue
+        elif curChar == ',':
+          result.add(curStr)
+          curStr = ""
+          parseState = expectStart
+          continue
+
+      of inQuote:
+        if curChar == '"' and not sawEscape:
+          parseState = expectEnd
+          continue
+
+      of inVal:
+        if curChar == '"' and not sawEscape:
+          raise newException(ValueError, "Invalid DB array value (cannot have '\"' in the middle of an unquoted string).")
+        elif curChar == ',':
+          result.add(curStr)
+          curStr = ""
+          parseState = expectStart
+          continue
+
+    # if we saw an escaped \", add just the ", otherwise add both
+    if sawEscape:
+      if curChar != '"': curStr.add('\\')
+      curStr.add(curChar)
+      sawEscape = false
+
+    elif curChar == '\\':
+      sawEscape = true
+
+    else: curStr.add(curChar)
+
+  if not (parseState == inQuote) and curStr.len > 0:
+    result.add(curStr)
+
+proc createParseStmt*(t, value: NimNode): NimNode =
+  result = newStmtList()
+
+  #echo "Creating parse statment for ", t.treeRepr
+  if t.typeKind == ntyObject:
+
+    if t.getType == UUID.getType:
+      result.add quote do:
+        parseUUID(`value`)
+
+    elif t.getType == DateTime.getType:
+      result.add quote do:
+        `value`.parseIso8601
+
+    elif t.getTypeInst == Option.getType:
+      let innerType = t.getTypeImpl[2][0][0][1]
+      let parseStmt = createParseStmt(innerType, value)
+      result.add quote do:
+        if `value`.len == 0:
+          none[`innerType`]()
+        else:
+          some(`parseStmt`)
+
+    else:
+      error "Unknown value object type: " & $t.getTypeInst
+
+  elif t.typeKind == ntySequence:
+    let innerType = t[1]
+
+    result.add quote do:
+      # TODO: for each value call the type-specific parsing logic.
+      #parseDbArray(`value`).mapIt(createParseStmt(it))
+      parseDbArray(`value`)
+
+  elif t.typeKind == ntyString:
+    result.add quote do:
+      `value`
+
+  elif t.typeKind == ntyInt:
+    result.add quote do:
+      parseInt(`value`)
+
+  else:
+    error "Unknown value type: " & $t.typeKind
+
+template walkFieldDefs*(t: NimNode, body: untyped) =
+  let tTypeImpl = t.getTypeImpl
+
+  var nodeToItr: NimNode
+  if tTypeImpl.typeKind == ntyObject: nodeToItr = tTypeImpl[2]
+  elif tTypeImpl.typeKind == ntyTypeDesc: nodeToItr = tTypeImpl.getType[1].getType[2]
+  else: error $t & " is not an object or type desc (it's a " & $tTypeImpl.typeKind & ")."
+
+  for fieldDef {.inject.} in nodeToItr.children:
+    # ignore AST nodes that are not field definitions
+    if fieldDef.kind == nnkIdentDefs:
+      let fieldIdent {.inject.} = fieldDef[0]
+      let fieldType {.inject.} = fieldDef[1]
+      body
+
+    elif fieldDef.kind == nnkSym:
+      let fieldIdent {.inject.} = fieldDef
+      let fieldType {.inject.} = fieldDef.getType
+      body
+
+macro columnNamesForModel*(modelType: typed): seq[string] =
+  var columnNames = newSeq[string]()
+
+  modelType.walkFieldDefs:
+    columnNames.add(identNameToDb($fieldIdent))
+
+  result = newLit(columnNames)
+
+macro rowToModel*(modelType: typed, row: seq[string]): untyped =
+
+  # Create the object constructor AST node
+  result = newNimNode(nnkObjConstr).add(modelType)
+
+  # Create new colon expressions for each of the property initializations
+  var idx = 0
+  modelType.walkFieldDefs:
+    let itemLookup = quote do: `row`[`idx`]
+    result.add(newColonExpr(
+      fieldIdent,
+      createParseStmt(fieldType, itemLookup)))
+    idx += 1
+
+macro listFields*(t: typed): untyped =
+  var fields: seq[tuple[n: string, t: string]] = @[]
+  t.walkFieldDefs:
+    if fieldDef.kind == nnkSym: fields.add((n: $fieldIdent, t: fieldType.repr))
+    else: fields.add((n: $fieldIdent, t: $fieldType))
+
+  result = newLit(fields)
+
+macro populateMutateClauses*(t: typed, newRecord: bool, mc: var MutateClauses): untyped =
+
+  result = newStmtList()
+
+  # iterate over all the object's fields
+  t.walkFieldDefs:
+
+      # grab the field, it's string name, and it's type
+      let fieldName = $fieldIdent
+
+      # we do not update the ID, but we do check: if we're creating a new
+      # record, we should not have an existing ID
+      if fieldName == "id":
+        result.add quote do:
+          if `newRecord` and not `t`.id.isZero:
+            raise newException(
+              AssertionError,
+              "Trying to create a new record, but the record already has an ID.")
+
+      # if we're looking at an optional field, add logic to check for presence
+      elif fieldType.kind == nnkBracketExpr and
+         fieldType.len > 0 and
+         fieldType[0] == Option.getType:
+
+        result.add quote do:
+          `mc`.columns.add(identNameToDb(`fieldName`))
+          if `t`.`fieldIdent`.isSome:
+            `mc`.placeholders.add("?")
+            `mc`.values.add(dbFormat(`t`.`fieldIdent`.get))
+          else:
+            `mc`.placeholders.add("NULL")
+
+      # otherwise assume we can convert and go ahead.
+      else:
+        result.add quote do:
+          `mc`.columns.add(identNameToDb(`fieldName`))
+          `mc`.placeholders.add("?")
+          `mc`.values.add(dbFormat(`t`.`fieldIdent`))
+
+  #echo result.repr
+

api/src/test/nim/personal_measure_apipkg/tdb_util.nim

@@ -0,0 +1,95 @@
+import options, unittest, uuids
+
+from langutils import sameContents
+
+import ../../../main/nim/personal_measure_apipkg/db_util
+
+type
+  TestModel = object
+    id*: int
+    name*: string
+    uuid*: UUID
+    nullableBool: Option[bool]
+
+  ValBox = object
+    id*: int
+    val*: string
+
+suite "db_util":
+  
+  let testModel1 = TestModel(
+    id: 1,
+    name: "Test",
+    uuid: parseUUID("db62c4a8-0091-42de-980c-edce4312aabb"),
+    nullableBool: none[bool]())
+
+  test "modelName(type)":
+    check modelName(TestModel) == "TestModel"
+
+  test "modelName(object)":
+    check modelName(testModel1) == "TestModel"
+
+  test "identNameToDb":
+    check:
+      identNameToDb("TestModel") == "test_model"
+      identNameToDb("Test_This") == "test_this"
+      identNameToDb("getApiTrigger") == "get_api_trigger"
+      identNameToDb("_HOLD_THIS_") == "_hold_this_"
+      identNameToDb("__camelCAse") == "__camel_case"
+
+  test "dbNameToIdent":
+    check:
+      dbNameToIdent("test_model") == "testModel"
+      dbNameToIdent("test_this") == "testThis"
+      dbNameToIdent("get_api_trigger") == "getApiTrigger"
+      dbNameToIdent("_hold_this_") == "HoldThis"
+      dbNameToIdent("__camel_case") == "CamelCase"
+
+  test "tableName(type)":
+    check:
+      tableName(TestModel) == "test_models"
+      tableName(ValBox) == "val_boxs" # NOTE lack of support currently for other pluralizations
+
+  test "tableName(type)":
+    check:
+      tableName(testModel1) == "test_models"
+      tableName(ValBox(id: 1, val: "test")) == "val_boxs" # NOTE lack of support currently for other pluralizations
+
+  test "dbFormat(string)":
+    check:
+      dbFormat("123") == "123"
+      dbFormat("this is a string") == "this is a string"
+      dbFormat("should preserve\t all \n characters \\") == "should preserve\t all \n characters \\"
+
+  test "dbFormat(seq[T])":
+    let names = @["Bob", "Sam", "Jones"]
+    let ages = @[35, 42, 18]
+
+    check:
+      dbFormat(names) == "{Bob,Sam,Jones}"
+      dbFormat(ages) == "{35,42,18}"
+
+  test "parseDbArray":
+    check:
+      sameContents(parseDbArray("{1,2,3}"), @["1", "2", "3"])
+      sameContents(parseDbArray("{\"1,2\",3}"), @["1,2", "3"])
+      sameContents(parseDbArray("{test,\"this\"}"), @["test", "this"])
+      sameContents(parseDbArray("{test,\"th,is\"}"), @["test", "th,is"])
+      sameContents(parseDbArray("{test,\"th,\\\"is\"}"), @["test", "th,\"is"])
+      sameContents(parseDbArray("{test,\"th,\\\"is\",\"what?\"}"), @["test", "th,\"is", "what?"])
+      sameContents(parseDbArray("{\"find,st\\\"uff\",ov\\\"er, there, \"\", \"what?\"}"), @["find,st\"uff", "ov\"er", "there", "", "what?"])
+
+  test "columnNamesForModel":
+    check:
+      sameContents(columnNamesForModel(TestModel), @["id", "name", "uuid", "nullable_bool"])
+      sameContents(columnNamesForModel(ValBox), @["id", "val"])
+
+#[ TODO - Tests needed
+
+  test "dbFormat(DateTime)":
+    check false
+
+  test "rowToModel"
+    check false
+
+]#