diff --git a/notion_utils.nimble b/hff_notion_api_client.nimble similarity index 60% rename from notion_utils.nimble rename to hff_notion_api_client.nimble index e0de6ef..4cc2db8 100644 --- a/notion_utils.nimble +++ b/hff_notion_api_client.nimble @@ -1,8 +1,8 @@ # Package -version = "0.2.0" +version = "0.3.0" author = "Jonathan Bernard" -description = "Utilities and bindings for the Notion API." +description = "Utilities and bindings for HFF's Notion API." license = "GPL-3.0-or-later" srcDir = "src" @@ -10,4 +10,5 @@ srcDir = "src" # Dependencies requires "nim >= 1.4.8" +requires "https://git.jdb-software.com/jdb/buffoonery" requires "https://git.jdb-software.com/jdb/nim-time-utils.git >= 0.5.0" diff --git a/src/hff_notion_api_client.nim b/src/hff_notion_api_client.nim new file mode 100644 index 0000000..c2481ea --- /dev/null +++ b/src/hff_notion_api_client.nim @@ -0,0 +1,177 @@ +import std/httpclient, std/json, std/logging, std/sequtils, std/tables, std/strutils + +import buffoonery/jsonutils + +import hff_notion_api_client/config +import hff_notion_api_client/models +import hff_notion_api_client/sequtils_ext +import hff_notion_api_client/utils + +type + NotionClientConfig* = object + apiBaseUrl*: string + apiVersion*: string + configDbId*: string + integrationToken*: string + + NotionClient* = ref object + http: HttpClient + apiBaseUrl: string + config*: HffNotionConfig + +proc loadNotionConfig(http: HttpClient, cfg: NotionClientConfig): HffNotionConfig = + let url = cfg.apiBaseUrl & "/databases/" & cfg.configDbId & "/query" + let body = $(%*{ "page_size": NOTION_MAX_PAGE_SIZE }) + + debug "loadNotionConfig\n\tPOST " & url & "\n\t" & $body + let resp = http.postContent(url, body) + let resultsJson = parseJson(resp)["results"] + + result = parseHffNotionConfig(resultsJson) + +proc initNotionClient*(cfg: NotionClientConfig): NotionClient = + result = NotionClient( + apiBaseUrl: cfg.apiBaseUrl, + http: newHttpClient(headers = newHttpHeaders([ + ("Content-Type", "application/json"), + ("Authorization", "Bearer " & cfg.integrationToken), + ("Notion-Version", cfg.apiVersion) + ], true))) + + result.config = result.http.loadNotionConfig(cfg) + +proc `%`*(c: NotionClientConfig): JsonNode = + %*{ + "apiBaseUrl": c.apiBaseUrl, + "apiVersion": c.apiVersion, + "configDbId": c.configDbId, + "integrationToken": c.integrationToken + } + +proc parseNotionClientConfig*(n: JsonNode): NotionClientConfig = + NotionClientConfig( + apiBaseUrl: n.getOrFail("apiBaseUrl").getStr, + apiVersion: n.getOrFail("apiVersion").getStr, + configDbId: n.getOrFail("configDbId").getStr, + integrationToken: n.getOrFail("integrationToken").getStr) + +proc fetchAllPages*( + http: HttpClient, + url: string, + bodyTemplate = %*{ "page_size": NOTION_MAX_PAGE_SIZE}): seq[JsonNode] = + + result = @[] + var nextCursor: string = "" + + while true: + let body = parseJson($bodyTemplate) + if not nextCursor.isEmptyOrWhitespace: body["start_cursor"] = %nextCursor + + debug "Fetching pages from database:\n\tPOST " & url & "\n\t" & $body + let jsonResp = parseJson(http.postContent(url, $body)) + + result = result & jsonResp["results"].getElems + if jsonResp.hasKey("next_cursor") and jsonResp["next_cursor"].kind != JNull: + nextCursor = jsonResp["next_cursor"].getStr + + if nextCursor.isEmptyOrWhitespace: break + +template fetchPage*(notion: NotionClient, pageId: string): untyped = + let resp = notion.http.get(notion.apiBaseUrl & "/pages/" & pageId) + + if not resp.status.startsWith("2"): + debug resp.body + raise newException(HttpRequestError, "API Request failed: " & resp.body) + parseJson(resp.body) + +template createDbPage*(notion: NotionClient, parentDbId: string, r: typed): untyped = + let page = r.toPage + page["parent"] = %*{ "database_id": parentDbId } + + let resp = notion.http.post(notion.apiBaseUrl & "/pages", $page) + + if not resp.status.startsWith("2"): + debug resp.body + raise newException(HttpRequestError, "API Request failed: " & resp.body) + parseJson(resp.body) + +template updatePage*(notion: NotionClient, r: typed): JsonNode = + let page = r.toPage + + let resp = notion.http.patch(notion.apiBaseUrl & "/pages/" & r.id, $page) + + if not resp.status.startsWith("2"): debug resp.body + parseJson(resp.body) + +template delete*(notion: NotionClient, r: typed): string = + let resp = notion.http.delete(notion.apiBaseUrl & "/blocks/" & r.id) + + if not resp.status.startsWith("2"): debug resp.body + parseJson(resp.body)["id"].getStr + +proc fetchSyncRecords*(notion: NotionClient): seq[SyncRecord] = + let respRecords = notion.http.fetchAllPages( + notion.apiBaseUrl & "/databases/" & notion.config.syncRecordsDbId & "/query") + + return respRecords.mapIt(syncRecordFromPage(it)) + +proc create*(notion: NotionClient, r: SyncRecord): SyncRecord = + return syncRecordFromPage(createDbPage(notion, notion.config.syncRecordsDbId, r)) + +proc update*(notion: NotionClient, r: SyncRecord): SyncRecord = + return syncRecordFromPage(updatePage(notion, r)) + +proc fetchAddress*(notion: NotionClient, addressId: string): Address = + return addressFromPage(notion.fetchPage(addressId)) + +proc fetchAddresses*(notion: NotionClient): seq[Address] = + let respRecords = notion.http.fetchAllPages( + notion.apiBaseUrl & "/databases/" & notion.config.addressesDbId & "/query") + + return resprecords.mapIt(addressFromPage(it)) + +proc create*(notion: NotionClient, a: Address): Address = + return addressFromPage(notion.createDbPage(notion.config.addressesDbId , a)) + +proc update*(notion: NotionClient, r: Address): Address = + return addressFromPage(updatePage(notion, r)) + +proc fetchFamily*(notion: NotionClient, familyId: string): Family = + return familyFromPage(notion.fetchPage(familyId)) + +proc fetchFamilies*(notion: NotionClient): seq[Family] = + let respRecords = notion.http.fetchAllPages( + notion.apiBaseUrl & "/databases/" & notion.config.familiesDbId & "/query") + + return respRecords.mapIt(familyFromPage(it)) + +proc create*(notion: NotionClient, f: Family): Family = + return familyFromPage(notion.createDbPage(notion.config.familiesDbId , f)) + +proc update*(notion: NotionClient, r: Family): Family = + return familyFromPage(updatePage(notion, r)) + +proc fetchPerson*(notion: NotionClient, personId: string): Person = + return personFromPage(notion.fetchPage(personId)) + +proc fetchPeople*(notion: NotionClient): seq[Person] = + let respRecords = notion.http.fetchAllPages( + notion.apiBaseUrl & "/databases/" & notion.config.peopleDbId & "/query") + + return respRecords.mapIt(personFromPage(it)) + +proc create*(notion: NotionClient, p: Person): Person = + return personFromPage(notion.createDbPage(notion.config.peopleDbId, p)) + +proc update*(notion: NotionClient, r: Person): Person = + return personFromPage(updatePage(notion, r)) + +proc persist*[T](notion: NotionClient, r: T): T = + if r.id.isEmptyOrWhitespace: notion.create(r) + else: notion.update(r) + +proc fetchMembershipDataSet*(notion: NotionClient): MembershipDataSet = + result = MembershipDataSet( + addresses: mapById(notion.fetchAddresses()), + families: mapById(notion.fetchFamilies()), + people: mapById(notion.fetchPeople())) diff --git a/src/hff_notion_api_client/config.nim b/src/hff_notion_api_client/config.nim new file mode 100644 index 0000000..d2cee79 --- /dev/null +++ b/src/hff_notion_api_client/config.nim @@ -0,0 +1,47 @@ +import std/json, std/sequtils, std/times +import timeutils + +type + HffNotionConfigProperty = tuple[key: string, textNode: JsonNode, dateNode: JsonNode] + + HffNotionConfig* = object + addressesDbId*: string + familiesDbId*: string + peopleDbId*: string + syncRecordsDbId*: string + lastSyncedAt*: DateTime + +proc getCfgProp(cfgPages: JsonNode, key: string): HffNotionConfigProperty = + + let propNodes = cfgPages.filterIt( + it{"properties", "Key", "title"}[0]["plain_text"].getStr == key) + + if propNodes.len != 1: + raise newException(Exception, + "there is no configuration value for key '" & key & + "' in the Notion Configuration table.") + + return ( + propNodes[0]{"properties", "Key", "title"}[0]["plain_text"].getStr, + propNodes[0]{"properties", "TextValue", "rich_text"}, + propNodes[0]{"properties", "DateTimeValue", "date"}) + +proc dateValue(prop: HffNotionConfigProperty): DateTime = + if isNil(prop.dateNode) or prop.dateNode.kind == JNull: + raise newException(Exception, prop.key & " does not have a date value") + + return parseIso8601(prop.dateNode["start"].getStr) + +proc textValue(prop: HffNotionConfigProperty): string = + if prop.textNode.len == 0: + raise newException(Exception, prop.key & " does not have a text value") + + return prop.textNode[0]["plain_text"].getStr + +proc parseHffNotionConfig*(n: JsonNode): HffNotionConfig = + result = HffNotionConfig( + addressesDbId: n.getCfgProp("addressesDbId").textValue, + familiesDbId: n.getCfgProp("familiesDbId").textValue, + peopleDbId: n.getCfgProp("peopleDbId").textValue, + syncRecordsDbId: n.getCfgProp("syncRecordsDbId").textValue, + lastSyncedAt: n.getCfgProp("lastSyncedAt").dateValue) diff --git a/src/notion_utils/hff_models.nim b/src/hff_notion_api_client/models.nim similarity index 66% rename from src/notion_utils/hff_models.nim rename to src/hff_notion_api_client/models.nim index 733db16..576c7a0 100644 --- a/src/notion_utils/hff_models.nim +++ b/src/hff_notion_api_client/models.nim @@ -1,11 +1,12 @@ -import std/json, std/options, std/tables, std/times +import std/json, std/options, std/sequtils, std/sets, std/strutils, std/tables, + std/times import timeutils -import ../notion_utils +import ./utils type - NAddressObj* = object + AddressObj* = object id*: string city*: string state*: string @@ -15,7 +16,7 @@ type lastUpdatedAt*: Option[DateTime] - NFamilyObj* = object + FamilyObj* = object id*: string name*: string headOfHouseholdIds*: seq[string] @@ -24,7 +25,7 @@ type createdAt*: Option[DateTime] lastUpdatedAt*: Option[DateTime] - NPersonObj* = object + PersonObj* = object id*: string preferredName*: string firstName*: string @@ -42,10 +43,27 @@ type relationshipToHff*: seq[string] createdAt*: Option[DateTime] lastUpdatedAt*: Option[DateTime] + apiPermissions*: seq[string] - NAddress* = ref NAddressObj - NFamily* = ref NFamilyObj - NPerson* = ref NPersonObj + RecordType* = enum rtPerson, rtFamily, rtAddress + + SyncRecordObj* = object + id*: string + knownIds*: HashSet[string] + recType*: RecordType + notionId*: string + pcoId*: string + + + Address* = ref AddressObj + Family* = ref FamilyObj + Person* = ref PersonObj + SyncRecord* = ref SyncRecordObj + + MembershipDataSet* = ref object + addresses*: TableRef[string, Address] + families*: TableRef[string, Family] + people*: TableRef[string, Person] func sameContents[T](a: seq[T], b: seq[T]): bool = let aCount = toCountTable(a) @@ -57,13 +75,13 @@ func sameContents[T](a: seq[T], b: seq[T]): bool = return aCount.len == bCount.len -func `==`*(a, b: NFamily): bool = +func `==`*(a, b: Family): bool = return a.name == b.name and sameContents(a.headOfHouseholdIds, b.headOfHouseholdIds) and sameContents(a.primaryAddressId, b.primaryAddressId) and sameContents(a.memberIds, b.memberIds) -func `==`*(a, b: NPerson): bool = +func `==`*(a, b: Person): bool = return a.preferredName == b.preferredName and a.firstName == b.firstName and a.middleNames == b.middleNames and @@ -79,7 +97,24 @@ func `==`*(a, b: NPerson): bool = sameContents(a.childIds, b.childIds) and sameContents(a.relationshipToHff, b.relationshipToHff) -func toPage*(a: NAddress): JsonNode = +func `$`*(rt: RecordType): string = + case rt: + of rtPerson: "Person" + of rtFamily: "Family" + of rtAddress: "Address" + +func initSyncRecord*(recType: RecordType, notionId, pcoId: string, knownIds = initHashSet[string]()): SyncRecord = + result = SyncRecord( + recType: recType, + notionId: notionId, + pcoId: pcoId, + knownIds: knownIds) + + result.knownIds.incl(notionId) + result.knownIds.incl(pcoId) + + +func toPage*(a: Address): JsonNode = %*{ "properties": { "City": makeTextProp("rich_text", a.city), @@ -89,7 +124,7 @@ func toPage*(a: NAddress): JsonNode = } } -func toPage*(f: NFamily): JsonNode = +func toPage*(f: Family): JsonNode = %*{ "properties": { "Name": makeTextProp("title", f.name), @@ -99,7 +134,7 @@ func toPage*(f: NFamily): JsonNode = } } -func toPage*(p: NPerson): JsonNode = +func toPage*(p: Person): JsonNode = %*{ "properties": { "Preferred Name": makeTextProp("title", p.preferredName), @@ -116,11 +151,26 @@ func toPage*(p: NPerson): JsonNode = "Anniversary": makeDateProp(p.anniversary), "Parents": makeRelationProp(p.parentIds), "Children": makeRelationProp(p.childIds), + "API Permissions": makeMultiSelectProp(p.apiPermissions), } } -proc parseNAddress*(page: JsonNode): NAddress = - result = NAddress( +func toPage*(sr: SyncRecord): JsonNode = + result = %*{ + "properties": { + "Notion Rec Id": makeTextProp("title", sr.notionId), + "PCO Rec Id": makeTextProp("rich_text", sr.pcoId), + "Alternate IDs": makeTextProp("rich_text", toSeq(sr.knownIds).join(";")) + } + } + + case sr.recType + of rtAddress: result{"properties", "Rec Type"}= %"Address" + of rtFamily: result{"properties", "Rec Type"}= %"Family" + of rtPerson: result{"properties", "Rec Type"}= %"Person" + +proc addressFromPage*(page: JsonNode): Address = + result = Address( id: page["id"].getStr, city: page.getText("City"), state: page.getSelect("State"), @@ -129,8 +179,8 @@ proc parseNAddress*(page: JsonNode): NAddress = createdAt: some(parseIso8601(page["created_time"].getStr)), lastUpdatedAt: some(parseIso8601(page["last_edited_time"].getStr))) -proc parseNFamily*(page: JsonNode): NFamily = - result = NFamily( +proc familyFromPage*(page: JsonNode): Family = + result = Family( id: page["id"].getStr, name: page.getTitle("Name"), headOfHouseholdIds: page.getRelationIds("Head(s) of Household"), @@ -139,8 +189,8 @@ proc parseNFamily*(page: JsonNode): NFamily = createdAt: some(parseIso8601(page["created_time"].getStr)), lastUpdatedAt: some(parseIso8601(page["last_edited_time"].getStr))) -proc parseNPerson*(page: JsonNode): NPerson = - result = NPerson( +proc personFromPage*(page: JsonNode): Person = + result = Person( id: page["id"].getStr, preferredName: page.getTitle("Preferred Name"), firstName: page.getText("First Name"), @@ -157,4 +207,17 @@ proc parseNPerson*(page: JsonNode): NPerson = parentIds: page.getRelationIds("Parents"), childIds: page.getRelationIds("Children"), createdAt: some(parseIso8601(page["created_time"].getStr)), - lastUpdatedAt: some(parseIso8601(page["last_edited_time"].getStr))) + lastUpdatedAt: some(parseIso8601(page["last_edited_time"].getStr)), + apiPermissions: page.getMultiSelect("API Permissions")) + +func syncRecordFromPage*(page: JsonNode): SyncRecord = + result = SyncRecord( + id: page["id"].getStr, + notionId: page.getTitle("Notion Rec Id"), + pcoId: page.getText("PCO Rec Id"), + knownIds: toHashSet(page.getText("Alternate IDs").split(";"))) + + case page.getSelect("Record Type") + of "Address": result.recType = rtAddress + of "Family": result.recType = rtFamily + of "Person": result.recType = rtPerson diff --git a/src/hff_notion_api_client/sequtils_ext.nim b/src/hff_notion_api_client/sequtils_ext.nim new file mode 100644 index 0000000..6a9ea29 --- /dev/null +++ b/src/hff_notion_api_client/sequtils_ext.nim @@ -0,0 +1,20 @@ +import std/options, std/sugar, std/tables + +func sameContents*[T](a: seq[T], b: seq[T]): bool = + let aCount = toCountTable(a) + let bCount = toCountTable(b) + + for k in aCount.keys: + if not bCount.hasKey(k) or aCount[k] != bCount[k]: return false + + return aCount.len == bCount.len + +func findBy*[T](s: seq[T], operation: (T) -> bool): Option[T] = + for i in s: + if operation(i): + return some(i) + return none[T]() + +proc mapById*[T](records: seq[T]): TableRef[string, T] = + result = newTable[string, T]() + for r in records: result[r.id] = r diff --git a/src/notion_utils.nim b/src/hff_notion_api_client/utils.nim similarity index 76% rename from src/notion_utils.nim rename to src/hff_notion_api_client/utils.nim index ea60f0c..4546576 100644 --- a/src/notion_utils.nim +++ b/src/hff_notion_api_client/utils.nim @@ -1,5 +1,4 @@ -import std/httpclient, std/json, std/logging, std/options, std/sequtils, - std/strutils, std/times +import std/json, std/options, std/sequtils, std/times import timeutils const NOTION_MAX_PAGE_SIZE* = 100 @@ -94,31 +93,3 @@ proc getDateTime*(page: JsonNode, propName: string): Option[DateTime] = let propNode = page.getPropNode("date", propName) if propNode.kind == JNull: result = none[DateTime]() else: result = some(parseDate(propNode["start"].getStr)) - -proc newNotionClient*(apiVersion, integrationToken: string): HttpClient = - return newHttpClient(headers = newHttpHeaders([ - ("Content-Type", "application/json"), - ("Authorization", "Bearer " & integrationToken), - ("Notion-Version", apiVersion) - ], true)) - -proc fetchAllPages*( - http: HttpClient, - url: string, - bodyTemplate = %*{ "page_size": NOTION_MAX_PAGE_SIZE}): seq[JsonNode] = - - result = @[] - var nextCursor: string = "" - - while true: - let body = parseJson($bodyTemplate) - if not nextCursor.isEmptyOrWhitespace: body["start_cursor"] = %nextCursor - - debug "Fetching pages from database:\n\tPOST " & url & "\n\t" & $body - let jsonResp = parseJson(http.postContent(url, $body)) - - result = result & jsonResp["results"].getElems - if jsonResp.hasKey("next_cursor") and jsonResp["next_cursor"].kind != JNull: - nextCursor = jsonResp["next_cursor"].getStr - - if nextCursor.isEmptyOrWhitespace: break