Preserve MEV passage metadata

src/embedded_bible.nim

@@ -69,8 +69,8 @@ proc loadBibleIndex(rows, translationName: string): BibleIndex =
     if line.strip.len == 0:
       continue
 
-    let parts = line.split('\t', maxsplit = 6)
+    let parts = line.split('\t', maxsplit = 7)
-    if parts.len != 4 and parts.len != 7:
+    if parts.len != 4 and parts.len != 7 and parts.len != 8:
       raise newException(ValueError,
         "invalid embedded " & translationName & " row: " & line)
 
@@ -142,6 +142,18 @@ proc selectedVerses(index: BibleIndex, reference: PassageReference, range: RefRa
       discard index.requireVerse(code, chapter, verse)
       result[verseKey(code, chapter, verse)] = true
 
+proc shouldSeparateSegments(previous, next: string): bool =
+  if previous.len == 0 or next.len == 0:
+    return false
+  if previous[^1].isSpaceAscii or next[0].isSpaceAscii:
+    return false
+  next[0] notin {'.', ',', ';', ':', '?', '!', ')', ']', '}'}
+
+proc appendSameLine(line: var string, content: string) =
+  if shouldSeparateSegments(line, content):
+    line.add(" ")
+  line.add(content)
+
 proc addFormattedSegment(
     lines: var seq[string],
     segment: BibleSegment,
@@ -159,17 +171,13 @@ proc addFormattedSegment(
   let hasPassageLine = lines.len > 1 and lines[^1].len > 0
 
   if segment.breakKind == sbSame and hasPassageLine:
-    if lines[^1].len > 0 and not lines[^1][^1].isSpaceAscii:
+    lines[^1].appendSameLine(content)
-      lines[^1].add(" ")
-    lines[^1].add(content)
   else:
     let isMarkerlessParagraphContinuation =
       segment.breakKind == sbParagraph and marker.len == 0 and hasPassageLine
 
     if isMarkerlessParagraphContinuation:
-      if lines[^1].len > 0 and not lines[^1][^1].isSpaceAscii:
+      lines[^1].appendSameLine(content)
-        lines[^1].add(" ")
-      lines[^1].add(content)
     else:
       if segment.breakKind == sbParagraph and hasPassageLine:
         lines.add("")

tests/test_offline_kjv.nim

@@ -93,6 +93,8 @@ JHN	8	33	0	paragraph	0	They answered Him.
 JHN	8	39	0	paragraph	0	They answered Him, "Abraham is our father."
 JHN	8	39	1	paragraph	0	Jesus said to them, "If you were Abraham's children,
 JHN	8	40	0	same	0	you would do the works of Abraham."
+JHN	8	41	0	paragraph	0	You shall be free'
+JHN	8	41	1	same	0	?
 HEB	4	7	0	paragraph	0	again He establishes a certain day:
 HEB	4	7	1	line	0	"Today, if you will hear His voice,
 HEB	4	7	2	line	1	do not harden your hearts."
@@ -137,12 +139,26 @@ HEB	4	7	2	line	1	do not harden your hearts."
       "Jesus said to them, \"If you were Abraham's children, " &
       "[40] you would do the works of Abraham.\""
 
+  test "joins punctuation-only segments without extra spacing":
+    let passages = embedded_bible.fetchPassages(layoutRows, "John 8:41", "TEST")
+
+    check passages[0] == "John 8:41\n  [41] You shall be free'?"
+
   test "supports legacy verse rows":
     const legacyRows = "JHN\t3\t16\tFor God so loved the world.\n"
     let passages = embedded_bible.fetchPassages(legacyRows, "John 3:16", "TEST")
 
     check passages[0] == "John 3:16\n  [16] For God so loved the world."
 
+  test "ignores layout metadata while rendering":
+    const metadataRows =
+      "JHN\t8\t31\t0\tparagraph\t0\tJesus said.\t" &
+      """{"wordsOfChrist":true,"annotations":[{"kind":"crossReference","marker":"a","href":"index_split_1.html#x","offset":6}]}""" &
+      "\n"
+    let passages = embedded_bible.fetchPassages(metadataRows, "John 8:31", "TEST")
+
+    check passages[0] == "John 8:31\n  [31] Jesus said."
+
   test "groups legacy rows into paragraphs using pilcrows":
     const legacyRows =
       "JHN\t8\t31\tThen Jesus said.\n" &

tools/generate_mev_data.nim

@@ -1,9 +1,11 @@
 import std/[
   htmlparser,
+  json,
   os,
   osproc,
   streams,
   strutils,
+  tables,
   xmlparser,
   xmltree
 ]
@@ -29,6 +31,11 @@ type
     segmentText: string
     segmentBreak: string
     indent: int
+    wordsOfChrist: bool
+    annotations: seq[JsonNode]
+    epubPath: string
+    targetTextCache: Table[string, string]
+    targetFileCache: Table[string, Table[string, string]]
     rows: seq[string]
 
 proc normalizeWhitespace(s: string): string =
@@ -197,6 +204,92 @@ proc hasHref(node: XmlNode): bool =
       if hasHref(child):
         return true
 
+proc firstHref(node: XmlNode): string =
+  if node.kind == xnElement:
+    let href = node.attr("href")
+    if href.len > 0:
+      return href
+
+    for child in node.items:
+      let href = firstHref(child)
+      if href.len > 0:
+        return href
+
+proc collectIds(node: XmlNode, ids: var seq[string]) =
+  if node.kind == xnElement:
+    let id = node.attr("id")
+    if id.len > 0:
+      ids.add(id)
+    for child in node.items:
+      collectIds(child, ids)
+
+proc targetParagraphs(doc: XmlNode): Table[string, string] =
+  var paragraphs = initTable[string, string]()
+
+  proc walk(node: XmlNode) =
+    if node.kind == xnElement:
+      if node.tag == "p":
+        var ids: seq[string] = @[]
+        collectIds(node, ids)
+        if ids.len > 0:
+          let text = normalizeWhitespace(textContent(node))
+          for id in ids:
+            paragraphs[id] = text
+
+      for child in node.items:
+        walk(child)
+
+  walk(doc)
+  paragraphs
+
+proc linkTargetText(state: var ParseState, href: string): string =
+  if href.len == 0:
+    return ""
+
+  if state.targetTextCache.hasKey(href):
+    return state.targetTextCache[href]
+
+  let parts = href.split('#', maxsplit = 1)
+  if parts.len != 2 or parts[0].len == 0 or parts[1].len == 0:
+    state.targetTextCache[href] = ""
+    return ""
+
+  if not state.targetFileCache.hasKey(parts[0]):
+    try:
+      let html = readEpubEntry(state.epubPath, parts[0])
+      let doc = parseHtml(newStringStream(html))
+      state.targetFileCache[parts[0]] = targetParagraphs(doc)
+    except CatchableError:
+      state.targetFileCache[parts[0]] = initTable[string, string]()
+
+  if state.targetFileCache[parts[0]].hasKey(parts[1]):
+    result = state.targetFileCache[parts[0]][parts[1]]
+  else:
+    result = ""
+
+  state.targetTextCache[href] = result
+
+proc annotationKind(marker: string): string =
+  if marker.strip.allCharsInSet({'0'..'9'}): "note"
+  else: "crossReference"
+
+proc annotationNode(marker, href, targetText: string, offset: int): JsonNode =
+  result = %*{
+    "kind": annotationKind(marker),
+    "marker": marker,
+    "href": href,
+    "offset": offset
+  }
+  if targetText.len > 0:
+    result["targetText"] = newJString(targetText)
+
+proc segmentMetadata(state: ParseState): JsonNode =
+  result = newJObject()
+  if state.wordsOfChrist:
+    result["wordsOfChrist"] = newJBool(true)
+  if state.annotations.len > 0:
+    result["annotations"] = %state.annotations
+
 proc isParagraphElement(node: XmlNode): bool =
   node.kind == xnElement and node.tag == "p" and
     (node.hasClass("calibre_18") or
@@ -253,6 +346,7 @@ proc flushSegment(state: var ParseState) =
   if state.chapter > 0 and state.verse > 0:
     let text = normalizeWhitespace(state.segmentText).replace("\t", " ")
     if text.len > 0:
+      let metadata = segmentMetadata(state)
       state.rows.add([
         state.code,
         $state.chapter,
@@ -260,12 +354,15 @@ proc flushSegment(state: var ParseState) =
         $state.segment,
         state.segmentBreak,
         $state.indent,
-        text
+        text,
+        $metadata
       ].join("\t"))
       inc state.segment
       state.segmentBreak = "same"
 
   state.segmentText = ""
+  state.wordsOfChrist = false
+  state.annotations = @[]
 
 proc startBlock(state: var ParseState, segmentBreak: string) =
   state.flushSegment()
@@ -309,9 +406,30 @@ proc walkPassageText(node: XmlNode, state: var ParseState) =
       state.segment = 0
       return
 
+    if node.tag == "sup" and node.hasHref:
+      let marker = markerText(textContent(node))
+      let href = firstHref(node)
+      if marker.len > 0 and href.len > 0:
+        state.annotations.add(annotationNode(
+          marker,
+          href,
+          state.linkTargetText(href),
+          normalizeWhitespace(state.segmentText).len))
+      return
+
     if node.tag == "sup":
       return
 
+    if node.hasClass("calibre_40"):
+      state.flushSegment()
+      let oldWordsOfChrist = state.wordsOfChrist
+      state.wordsOfChrist = true
+      for child in node.items:
+        walkPassageText(child, state)
+      state.flushSegment()
+      state.wordsOfChrist = oldWordsOfChrist
+      return
+
     if node.tag == "p":
       let segmentBreak =
         if node.isParagraphElement: "paragraph"
@@ -340,7 +458,7 @@ proc indexSplitFile(index: int): string =
   "index_split_" & align($index, 3, '0') & ".html"
 
 proc parseBook(epubPath: string, source: BookSource): seq[string] =
-  var state = ParseState(code: source.code)
+  var state = ParseState(code: source.code, epubPath: epubPath)
   if bookInfo(source.code).singleChapter:
     state.chapter = 1