Preserve layout in embedded Bible data

src/embedded_bible.nim

@@ -2,11 +2,27 @@ import std/[strutils, tables]
 
 import ./reference_parser
 
-type BibleIndex = object
+type
-  verses: Table[string, string]
+  SegmentBreak = enum
-  lastVerseByChapter: Table[string, int]
+    sbSame = "same"
-  lastChapterByBook: Table[string, int]
+    sbLine = "line"
-  translationName: string
+    sbParagraph = "paragraph"
+
+  BibleSegment = object
+    code: string
+    chapter: int
+    verse: int
+    segment: int
+    breakKind: SegmentBreak
+    indent: int
+    text: string
+
+  BibleIndex = object
+    segments: seq[BibleSegment]
+    verses: Table[string, bool]
+    lastVerseByChapter: Table[string, int]
+    lastChapterByBook: Table[string, int]
+    translationName: string
 
 proc verseKey(code: string, chapter, verse: int): string =
   code & "\t" & $chapter & "\t" & $verse
@@ -14,6 +30,27 @@ proc verseKey(code: string, chapter, verse: int): string =
 proc chapterKey(code: string, chapter: int): string =
   code & "\t" & $chapter
 
+proc parseSegmentBreak(s: string): SegmentBreak =
+  case s
+  of "same": sbSame
+  of "line": sbLine
+  of "paragraph": sbParagraph
+  else:
+    raise newException(ValueError, "invalid embedded Bible segment break: " & s)
+
+proc addSegment(index: var BibleIndex, segment: BibleSegment) =
+  index.segments.add(segment)
+  index.verses[verseKey(segment.code, segment.chapter, segment.verse)] = true
+
+  let cKey = chapterKey(segment.code, segment.chapter)
+  if not index.lastVerseByChapter.hasKey(cKey) or
+      segment.verse > index.lastVerseByChapter[cKey]:
+    index.lastVerseByChapter[cKey] = segment.verse
+
+  if not index.lastChapterByBook.hasKey(segment.code) or
+      segment.chapter > index.lastChapterByBook[segment.code]:
+    index.lastChapterByBook[segment.code] = segment.chapter
+
 proc loadBibleIndex(rows, translationName: string): BibleIndex =
   result.translationName = translationName
 
@@ -21,26 +58,29 @@ proc loadBibleIndex(rows, translationName: string): BibleIndex =
     if line.strip.len == 0:
       continue
 
-    let parts = line.split('\t', maxsplit = 3)
+    let parts = line.split('\t', maxsplit = 6)
-    if parts.len != 4:
+    if parts.len != 4 and parts.len != 7:
       raise newException(ValueError,
         "invalid embedded " & translationName & " row: " & line)
 
-    let code = parts[0]
+    if parts.len == 4:
-    let chapter = parseInt(parts[1])
+      result.addSegment(BibleSegment(
-    let verse = parseInt(parts[2])
+        code: parts[0],
-    let text = parts[3]
+        chapter: parseInt(parts[1]),
-
+        verse: parseInt(parts[2]),
-    result.verses[verseKey(code, chapter, verse)] = text
+        segment: 0,
-
+        breakKind: sbLine,
-    let cKey = chapterKey(code, chapter)
+        indent: 0,
-    if not result.lastVerseByChapter.hasKey(cKey) or
+        text: parts[3]))
-        verse > result.lastVerseByChapter[cKey]:
+    else:
-      result.lastVerseByChapter[cKey] = verse
+      result.addSegment(BibleSegment(
-
+        code: parts[0],
-    if not result.lastChapterByBook.hasKey(code) or
+        chapter: parseInt(parts[1]),
-        chapter > result.lastChapterByBook[code]:
+        verse: parseInt(parts[2]),
-      result.lastChapterByBook[code] = chapter
+        segment: parseInt(parts[3]),
+        breakKind: parseSegmentBreak(parts[4]),
+        indent: parseInt(parts[5]),
+        text: parts[6]))
 
 proc requireLastChapter(index: BibleIndex, code: string): int =
   if not index.lastChapterByBook.hasKey(code):
@@ -62,13 +102,10 @@ proc requireVerse(index: BibleIndex, code: string, chapter, verse: int): string
     raise newException(ValueError,
       "no embedded " & index.translationName & " data for " &
         bookInfo(code).name & " " & $chapter & ":" & $verse)
-  index.verses[vKey]
+  ""
 
-proc addVerseLines(
+proc selectedVerses(index: BibleIndex, reference: PassageReference, range: RefRange):
-    lines: var seq[string],
+    Table[string, bool] =
-    index: BibleIndex,
-    reference: PassageReference,
-    range: RefRange) =
 
   let code = reference.book.code
   discard index.requireLastChapter(code)
@@ -90,16 +127,64 @@ proc addVerseLines(
       raise newException(ValueError, "reference range starts after it ends")
 
     for verse in startVerse .. endVerse:
-      lines.add("  [" & $verse & "] " & index.requireVerse(code, chapter, verse))
+      discard index.requireVerse(code, chapter, verse)
+      result[verseKey(code, chapter, verse)] = true
+
+proc addFormattedSegment(
+    lines: var seq[string],
+    segment: BibleSegment,
+    emittedVerses: var Table[string, bool]) =
+
+  let vKey = verseKey(segment.code, segment.chapter, segment.verse)
+  let marker =
+    if emittedVerses.hasKey(vKey): ""
+    else:
+      emittedVerses[vKey] = true
+      "[" & $segment.verse & "] "
+
+  let prefix = repeat("  ", segment.indent + 1)
+  let content = marker & segment.text
+  let hasPassageLine = lines.len > 1 and lines[^1].len > 0
+
+  if segment.breakKind == sbSame and hasPassageLine:
+    if lines[^1].len > 0 and not lines[^1][^1].isSpaceAscii:
+      lines[^1].add(" ")
+    lines[^1].add(content)
+  else:
+    if segment.breakKind == sbParagraph and hasPassageLine:
+      lines.add("")
+    lines.add(prefix & content)
+
+proc addVerseLines(
+    lines: var seq[string],
+    index: BibleIndex,
+    reference: PassageReference,
+    range: RefRange) =
+
+  let code = reference.book.code
+  let selected = index.selectedVerses(reference, range)
+  var emittedVerses: Table[string, bool]
+  var firstSelected = true
+
+  for segment in index.segments:
+    if segment.code == code and
+        selected.hasKey(verseKey(segment.code, segment.chapter, segment.verse)):
+      var adjusted = segment
+      if firstSelected and adjusted.breakKind == sbSame:
+        adjusted.breakKind = sbLine
+      lines.addFormattedSegment(adjusted, emittedVerses)
+      firstSelected = false
 
 proc fetchReference(index: BibleIndex, reference: PassageReference): string =
   var lines = @[$reference]
   let code = reference.book.code
 
   if reference.ranges.len == 0:
-    for chapter in 1 .. index.requireLastChapter(code):
+    var emittedVerses: Table[string, bool]
-      for verse in 1 .. index.requireLastVerse(code, chapter):
+    discard index.requireLastChapter(code)
-        lines.add("  [" & $verse & "] " & index.requireVerse(code, chapter, verse))
+    for segment in index.segments:
+      if segment.code == code:
+        lines.addFormattedSegment(segment, emittedVerses)
   else:
     for range in reference.ranges:
       lines.addVerseLines(index, reference, range)

tests/test_offline_kjv.nim

@@ -1,5 +1,6 @@
 import std/[strutils, unittest]
 
+import ../src/embedded_bible
 import ../src/kjv
 import ../src/reference_parser
 
@@ -82,3 +83,51 @@ suite "offline KJV backend":
     check passages.len == 1
     check passages[0].startsWith("Jude 3\n")
     check passages[0].contains("  [3] ")
+
+suite "embedded Bible layout":
+  const layoutRows = """
+JHN	8	31	0	paragraph	0	Then Jesus said,
+JHN	8	31	1	same	0	"If you remain in My word,
+JHN	8	32	0	same	0	you shall know the truth."
+JHN	8	33	0	paragraph	0	They answered Him.
+HEB	4	7	0	paragraph	0	again He establishes a certain day:
+HEB	4	7	1	line	0	"Today, if you will hear His voice,
+HEB	4	7	2	line	1	do not harden your hearts."
+"""
+
+  test "preserves inline verse markers within a paragraph":
+    let passages = embedded_bible.fetchPassages(layoutRows, "John 8:31-32", "TEST")
+
+    check passages.len == 1
+    check passages[0] == "John 8:31-32\n" &
+      "  [31] Then Jesus said, \"If you remain in My word, [32] you shall know the truth.\""
+
+  test "preserves paragraph breaks":
+    let passages = embedded_bible.fetchPassages(layoutRows, "John 8:31-33", "TEST")
+
+    check passages[0] == "John 8:31-33\n" &
+      "  [31] Then Jesus said, \"If you remain in My word, [32] you shall know the truth.\"\n" &
+      "\n" &
+      "  [33] They answered Him."
+
+  test "preserves indented quote lines":
+    let passages = embedded_bible.fetchPassages(layoutRows, "Hebrews 4:7", "TEST")
+
+    check passages[0] == "Hebrews 4:7\n" &
+      "  [7] again He establishes a certain day:\n" &
+      "  \"Today, if you will hear His voice,\n" &
+      "    do not harden your hearts.\""
+
+  test "starts mid-paragraph ranges on a fresh line":
+    let passages = embedded_bible.fetchPassages(layoutRows, "John 8:32-33", "TEST")
+
+    check passages[0] == "John 8:32-33\n" &
+      "  [32] you shall know the truth.\"\n" &
+      "\n" &
+      "  [33] They answered Him."
+
+  test "supports legacy verse rows":
+    const legacyRows = "JHN\t3\t16\tFor God so loved the world.\n"
+    let passages = embedded_bible.fetchPassages(legacyRows, "John 3:16", "TEST")
+
+    check passages[0] == "John 3:16\n  [16] For God so loved the world."

tools/generate_kjv_data.nim

@@ -84,6 +84,22 @@ proc parseVerseLine(line: string): tuple[verse: int, text: string] =
   result.verse = parseInt(rest[0 ..< numberEnd])
   result.text = stripUsfmMarkup(rest[numberEnd + 1 .. ^1])
 
+proc usfmBlock(line: string): tuple[matched: bool, segmentBreak: string, indent: int] =
+  if line.startsWith("\\p") or line.startsWith("\\m"):
+    return (true, "paragraph", 0)
+
+  if line.startsWith("\\q"):
+    var idx = 2
+    var digits = ""
+    while idx < line.len and line[idx].isDigit:
+      digits.add(line[idx])
+      inc idx
+
+    let indent =
+      if digits.len > 0: parseInt(digits)
+      else: 1
+    return (true, "line", indent)
+
 proc findCanonFiles(inputDir: string): Table[string, string] =
   for path in walkFiles(inputDir / "*eng-kjv.usfm"):
     let name = path.extractFilename
@@ -104,35 +120,58 @@ proc generate(inputDir, outputPath: string) =
 
     var chapter = 0
     var verse = 0
-    var verseText = ""
+    var segment = 0
+    var segmentText = ""
+    var segmentBreak = "line"
+    var indent = 0
 
-    proc flushVerse() =
+    proc flushSegment() =
       if chapter > 0 and verse > 0:
-        let text = normalizeWhitespace(verseText).replace("\t", " ")
+        let text = normalizeWhitespace(segmentText).replace("\t", " ")
         if text.len > 0:
-          rows.add([code, $chapter, $verse, text].join("\t"))
+          rows.add([
-      verse = 0
+            code,
-      verseText = ""
+            $chapter,
+            $verse,
+            $segment,
+            segmentBreak,
+            $indent,
+            text
+          ].join("\t"))
+          inc segment
+          segmentBreak = "same"
+      segmentText = ""
+
+    proc startBlock(nextBreak: string, nextIndent: int) =
+      flushSegment()
+      segmentBreak = nextBreak
+      indent = nextIndent
 
     for rawLine in canonFiles[code].lines:
       let line = rawLine.strip
 
       if line.startsWith("\\c "):
-        flushVerse()
+        flushSegment()
         chapter = parseInt(line[3..^1].strip)
+        verse = 0
       elif line.startsWith("\\v "):
-        flushVerse()
+        flushSegment()
         let parsed = parseVerseLine(line)
         verse = parsed.verse
-        verseText = parsed.text
+        segment = 0
+        segmentText = parsed.text
       elif verse > 0:
+        let block = usfmBlock(line)
+        if block.matched:
+          startBlock(block.segmentBreak, block.indent)
+
         let continued = stripUsfmMarkup(line)
         if continued.len > 0:
-          if verseText.len > 0:
+          if segmentText.len > 0:
-            verseText.add(' ')
+            segmentText.add(' ')
-          verseText.add(continued)
+          segmentText.add(continued)
 
-    flushVerse()
+    flushSegment()
 
   createDir(outputPath.parentDir)
   writeFile(outputPath, rows.join("\n") & "\n")

tools/generate_mev_data.nim

@@ -25,7 +25,10 @@ type
     code: string
     chapter: int
     verse: int
-    verseText: string
+    segment: int
+    segmentText: string
+    segmentBreak: string
+    indent: int
     rows: seq[string]
 
 proc normalizeWhitespace(s: string): string =
@@ -194,9 +197,12 @@ proc hasHref(node: XmlNode): bool =
       if hasHref(child):
         return true
 
-proc isBlockElement(node: XmlNode): bool =
+proc isParagraphElement(node: XmlNode): bool =
-  node.kind == xnElement and
+  node.kind == xnElement and node.tag == "p" and
-    node.tag in ["blockquote", "br", "div", "h1", "h2", "h3", "li", "p"]
+    (node.hasClass("calibre_18") or
+     node.hasClass("calibre_20") or
+     node.hasClass("calibre_23") or
+     node.hasClass("calibre_34"))
 
 proc chapterMarker(node: XmlNode): int =
   if node.kind == xnElement and node.tag == "span" and node.hasClass("calibre1"):
@@ -243,13 +249,27 @@ proc leadingVerseText(s: string): tuple[verse: int, rest: string] =
   if idx < text.len:
     result.rest = text[idx .. ^1]
 
-proc flushVerse(state: var ParseState) =
+proc flushSegment(state: var ParseState) =
   if state.chapter > 0 and state.verse > 0:
-    let text = normalizeWhitespace(state.verseText).replace("\t", " ")
+    let text = normalizeWhitespace(state.segmentText).replace("\t", " ")
     if text.len > 0:
-      state.rows.add([state.code, $state.chapter, $state.verse, text].join("\t"))
+      state.rows.add([
+        state.code,
+        $state.chapter,
+        $state.verse,
+        $state.segment,
+        state.segmentBreak,
+        $state.indent,
+        text
+      ].join("\t"))
+      inc state.segment
+      state.segmentBreak = "same"
 
-  state.verseText = ""
+  state.segmentText = ""
+
+proc startBlock(state: var ParseState, segmentBreak: string) =
+  state.flushSegment()
+  state.segmentBreak = segmentBreak
 
 proc walkPassageText(node: XmlNode, state: var ParseState) =
   case node.kind
@@ -259,13 +279,14 @@ proc walkPassageText(node: XmlNode, state: var ParseState) =
         let leading = leadingVerseText(node.text)
         if leading.verse > 0:
           state.verse = leading.verse
-          state.verseText.add(leading.rest)
+          state.segment = 0
+          state.segmentText.add(leading.rest)
       elif state.verse > 0:
-        state.verseText.add(node.text)
+        state.segmentText.add(node.text)
   of xnElement:
     let headingChapter = headingChapterMarker(node, state.code)
     if headingChapter > 0:
-      state.flushVerse()
+      state.flushSegment()
       state.chapter = headingChapter
       state.verse = 0
       return
@@ -275,25 +296,43 @@ proc walkPassageText(node: XmlNode, state: var ParseState) =
 
     let chapter = chapterMarker(node)
     if chapter > 0:
-      state.flushVerse()
+      state.flushSegment()
       state.chapter = chapter
       state.verse = 1
+      state.segment = 0
       return
 
     let verse = verseMarker(node)
     if verse > 0:
-      state.flushVerse()
+      state.flushSegment()
       state.verse = verse
+      state.segment = 0
       return
 
     if node.tag == "sup":
       return
 
-    for child in node.items:
+    if node.tag == "p":
-      walkPassageText(child, state)
+      let segmentBreak =
-
+        if node.isParagraphElement: "paragraph"
-    if node.isBlockElement and state.chapter > 0 and state.verse > 0:
+        else: "line"
-      state.verseText.add(' ')
+      state.startBlock(segmentBreak)
+      for child in node.items:
+        walkPassageText(child, state)
+      state.flushSegment()
+    elif node.tag == "blockquote":
+      let oldIndent = state.indent
+      state.indent = oldIndent + 1
+      state.startBlock("line")
+      for child in node.items:
+        walkPassageText(child, state)
+      state.flushSegment()
+      state.indent = oldIndent
+    elif node.tag == "br":
+      state.startBlock("line")
+    else:
+      for child in node.items:
+        walkPassageText(child, state)
   else:
     discard
 
@@ -310,7 +349,7 @@ proc parseBook(epubPath: string, source: BookSource): seq[string] =
     let doc = parseHtml(newStringStream(html))
     walkPassageText(doc, state)
 
-  state.flushVerse()
+  state.flushSegment()
   state.rows
 
 proc generate(epubPath, outputPath: string) =