From 457600b93db5b2a438acd8445441d3dcac3f616f Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Sun, 5 Jul 2026 21:15:19 -0500 Subject: [PATCH] Preserve MEV passage metadata --- src/embedded_bible.nim | 24 ++++--- tests/test_offline_kjv.nim | 16 +++++ tools/generate_mev_data.nim | 122 +++++++++++++++++++++++++++++++++++- 3 files changed, 152 insertions(+), 10 deletions(-) diff --git a/src/embedded_bible.nim b/src/embedded_bible.nim index 3d89265..6b61b1e 100644 --- a/src/embedded_bible.nim +++ b/src/embedded_bible.nim @@ -69,8 +69,8 @@ proc loadBibleIndex(rows, translationName: string): BibleIndex = if line.strip.len == 0: continue - let parts = line.split('\t', maxsplit = 6) - if parts.len != 4 and parts.len != 7: + let parts = line.split('\t', maxsplit = 7) + if parts.len != 4 and parts.len != 7 and parts.len != 8: raise newException(ValueError, "invalid embedded " & translationName & " row: " & line) @@ -142,6 +142,18 @@ proc selectedVerses(index: BibleIndex, reference: PassageReference, range: RefRa discard index.requireVerse(code, chapter, verse) result[verseKey(code, chapter, verse)] = true +proc shouldSeparateSegments(previous, next: string): bool = + if previous.len == 0 or next.len == 0: + return false + if previous[^1].isSpaceAscii or next[0].isSpaceAscii: + return false + next[0] notin {'.', ',', ';', ':', '?', '!', ')', ']', '}'} + +proc appendSameLine(line: var string, content: string) = + if shouldSeparateSegments(line, content): + line.add(" ") + line.add(content) + proc addFormattedSegment( lines: var seq[string], segment: BibleSegment, @@ -159,17 +171,13 @@ proc addFormattedSegment( let hasPassageLine = lines.len > 1 and lines[^1].len > 0 if segment.breakKind == sbSame and hasPassageLine: - if lines[^1].len > 0 and not lines[^1][^1].isSpaceAscii: - lines[^1].add(" ") - lines[^1].add(content) + lines[^1].appendSameLine(content) else: let isMarkerlessParagraphContinuation = segment.breakKind == sbParagraph and marker.len == 0 and hasPassageLine if isMarkerlessParagraphContinuation: - if lines[^1].len > 0 and not lines[^1][^1].isSpaceAscii: - lines[^1].add(" ") - lines[^1].add(content) + lines[^1].appendSameLine(content) else: if segment.breakKind == sbParagraph and hasPassageLine: lines.add("") diff --git a/tests/test_offline_kjv.nim b/tests/test_offline_kjv.nim index b13ed1a..f5e0e96 100644 --- a/tests/test_offline_kjv.nim +++ b/tests/test_offline_kjv.nim @@ -93,6 +93,8 @@ JHN 8 33 0 paragraph 0 They answered Him. JHN 8 39 0 paragraph 0 They answered Him, "Abraham is our father." JHN 8 39 1 paragraph 0 Jesus said to them, "If you were Abraham's children, JHN 8 40 0 same 0 you would do the works of Abraham." +JHN 8 41 0 paragraph 0 You shall be free' +JHN 8 41 1 same 0 ? HEB 4 7 0 paragraph 0 again He establishes a certain day: HEB 4 7 1 line 0 "Today, if you will hear His voice, HEB 4 7 2 line 1 do not harden your hearts." @@ -137,12 +139,26 @@ HEB 4 7 2 line 1 do not harden your hearts." "Jesus said to them, \"If you were Abraham's children, " & "[40] you would do the works of Abraham.\"" + test "joins punctuation-only segments without extra spacing": + let passages = embedded_bible.fetchPassages(layoutRows, "John 8:41", "TEST") + + check passages[0] == "John 8:41\n [41] You shall be free'?" + test "supports legacy verse rows": const legacyRows = "JHN\t3\t16\tFor God so loved the world.\n" let passages = embedded_bible.fetchPassages(legacyRows, "John 3:16", "TEST") check passages[0] == "John 3:16\n [16] For God so loved the world." + test "ignores layout metadata while rendering": + const metadataRows = + "JHN\t8\t31\t0\tparagraph\t0\tJesus said.\t" & + """{"wordsOfChrist":true,"annotations":[{"kind":"crossReference","marker":"a","href":"index_split_1.html#x","offset":6}]}""" & + "\n" + let passages = embedded_bible.fetchPassages(metadataRows, "John 8:31", "TEST") + + check passages[0] == "John 8:31\n [31] Jesus said." + test "groups legacy rows into paragraphs using pilcrows": const legacyRows = "JHN\t8\t31\tThen Jesus said.\n" & diff --git a/tools/generate_mev_data.nim b/tools/generate_mev_data.nim index 8ef7a97..522bd7f 100644 --- a/tools/generate_mev_data.nim +++ b/tools/generate_mev_data.nim @@ -1,9 +1,11 @@ import std/[ htmlparser, + json, os, osproc, streams, strutils, + tables, xmlparser, xmltree ] @@ -29,6 +31,11 @@ type segmentText: string segmentBreak: string indent: int + wordsOfChrist: bool + annotations: seq[JsonNode] + epubPath: string + targetTextCache: Table[string, string] + targetFileCache: Table[string, Table[string, string]] rows: seq[string] proc normalizeWhitespace(s: string): string = @@ -197,6 +204,92 @@ proc hasHref(node: XmlNode): bool = if hasHref(child): return true +proc firstHref(node: XmlNode): string = + if node.kind == xnElement: + let href = node.attr("href") + if href.len > 0: + return href + + for child in node.items: + let href = firstHref(child) + if href.len > 0: + return href + +proc collectIds(node: XmlNode, ids: var seq[string]) = + if node.kind == xnElement: + let id = node.attr("id") + if id.len > 0: + ids.add(id) + for child in node.items: + collectIds(child, ids) + +proc targetParagraphs(doc: XmlNode): Table[string, string] = + var paragraphs = initTable[string, string]() + + proc walk(node: XmlNode) = + if node.kind == xnElement: + if node.tag == "p": + var ids: seq[string] = @[] + collectIds(node, ids) + if ids.len > 0: + let text = normalizeWhitespace(textContent(node)) + for id in ids: + paragraphs[id] = text + + for child in node.items: + walk(child) + + walk(doc) + paragraphs + +proc linkTargetText(state: var ParseState, href: string): string = + if href.len == 0: + return "" + + if state.targetTextCache.hasKey(href): + return state.targetTextCache[href] + + let parts = href.split('#', maxsplit = 1) + if parts.len != 2 or parts[0].len == 0 or parts[1].len == 0: + state.targetTextCache[href] = "" + return "" + + if not state.targetFileCache.hasKey(parts[0]): + try: + let html = readEpubEntry(state.epubPath, parts[0]) + let doc = parseHtml(newStringStream(html)) + state.targetFileCache[parts[0]] = targetParagraphs(doc) + except CatchableError: + state.targetFileCache[parts[0]] = initTable[string, string]() + + if state.targetFileCache[parts[0]].hasKey(parts[1]): + result = state.targetFileCache[parts[0]][parts[1]] + else: + result = "" + + state.targetTextCache[href] = result + +proc annotationKind(marker: string): string = + if marker.strip.allCharsInSet({'0'..'9'}): "note" + else: "crossReference" + +proc annotationNode(marker, href, targetText: string, offset: int): JsonNode = + result = %*{ + "kind": annotationKind(marker), + "marker": marker, + "href": href, + "offset": offset + } + if targetText.len > 0: + result["targetText"] = newJString(targetText) + +proc segmentMetadata(state: ParseState): JsonNode = + result = newJObject() + if state.wordsOfChrist: + result["wordsOfChrist"] = newJBool(true) + if state.annotations.len > 0: + result["annotations"] = %state.annotations + proc isParagraphElement(node: XmlNode): bool = node.kind == xnElement and node.tag == "p" and (node.hasClass("calibre_18") or @@ -253,6 +346,7 @@ proc flushSegment(state: var ParseState) = if state.chapter > 0 and state.verse > 0: let text = normalizeWhitespace(state.segmentText).replace("\t", " ") if text.len > 0: + let metadata = segmentMetadata(state) state.rows.add([ state.code, $state.chapter, @@ -260,12 +354,15 @@ proc flushSegment(state: var ParseState) = $state.segment, state.segmentBreak, $state.indent, - text + text, + $metadata ].join("\t")) inc state.segment state.segmentBreak = "same" state.segmentText = "" + state.wordsOfChrist = false + state.annotations = @[] proc startBlock(state: var ParseState, segmentBreak: string) = state.flushSegment() @@ -309,9 +406,30 @@ proc walkPassageText(node: XmlNode, state: var ParseState) = state.segment = 0 return + if node.tag == "sup" and node.hasHref: + let marker = markerText(textContent(node)) + let href = firstHref(node) + if marker.len > 0 and href.len > 0: + state.annotations.add(annotationNode( + marker, + href, + state.linkTargetText(href), + normalizeWhitespace(state.segmentText).len)) + return + if node.tag == "sup": return + if node.hasClass("calibre_40"): + state.flushSegment() + let oldWordsOfChrist = state.wordsOfChrist + state.wordsOfChrist = true + for child in node.items: + walkPassageText(child, state) + state.flushSegment() + state.wordsOfChrist = oldWordsOfChrist + return + if node.tag == "p": let segmentBreak = if node.isParagraphElement: "paragraph" @@ -340,7 +458,7 @@ proc indexSplitFile(index: int): string = "index_split_" & align($index, 3, '0') & ".html" proc parseBook(epubPath: string, source: BookSource): seq[string] = - var state = ParseState(code: source.code) + var state = ParseState(code: source.code, epubPath: epubPath) if bookInfo(source.code).singleChapter: state.chapter = 1