From 98253920cabc63a8be3cd1bdcad179f13ef9d8a2 Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Sun, 5 Jul 2026 20:49:18 -0500 Subject: [PATCH] Preserve layout in embedded Bible data --- src/embedded_bible.nim | 149 ++++++++++++++++++++++++++++-------- tests/test_offline_kjv.nim | 49 ++++++++++++ tools/generate_kjv_data.nim | 65 ++++++++++++---- tools/generate_mev_data.nim | 77 ++++++++++++++----- 4 files changed, 276 insertions(+), 64 deletions(-) diff --git a/src/embedded_bible.nim b/src/embedded_bible.nim index 5f45b17..6f8c2cc 100644 --- a/src/embedded_bible.nim +++ b/src/embedded_bible.nim @@ -2,11 +2,27 @@ import std/[strutils, tables] import ./reference_parser -type BibleIndex = object - verses: Table[string, string] - lastVerseByChapter: Table[string, int] - lastChapterByBook: Table[string, int] - translationName: string +type + SegmentBreak = enum + sbSame = "same" + sbLine = "line" + sbParagraph = "paragraph" + + BibleSegment = object + code: string + chapter: int + verse: int + segment: int + breakKind: SegmentBreak + indent: int + text: string + + BibleIndex = object + segments: seq[BibleSegment] + verses: Table[string, bool] + lastVerseByChapter: Table[string, int] + lastChapterByBook: Table[string, int] + translationName: string proc verseKey(code: string, chapter, verse: int): string = code & "\t" & $chapter & "\t" & $verse @@ -14,6 +30,27 @@ proc verseKey(code: string, chapter, verse: int): string = proc chapterKey(code: string, chapter: int): string = code & "\t" & $chapter +proc parseSegmentBreak(s: string): SegmentBreak = + case s + of "same": sbSame + of "line": sbLine + of "paragraph": sbParagraph + else: + raise newException(ValueError, "invalid embedded Bible segment break: " & s) + +proc addSegment(index: var BibleIndex, segment: BibleSegment) = + index.segments.add(segment) + index.verses[verseKey(segment.code, segment.chapter, segment.verse)] = true + + let cKey = chapterKey(segment.code, segment.chapter) + if not index.lastVerseByChapter.hasKey(cKey) or + segment.verse > index.lastVerseByChapter[cKey]: + index.lastVerseByChapter[cKey] = segment.verse + + if not index.lastChapterByBook.hasKey(segment.code) or + segment.chapter > index.lastChapterByBook[segment.code]: + index.lastChapterByBook[segment.code] = segment.chapter + proc loadBibleIndex(rows, translationName: string): BibleIndex = result.translationName = translationName @@ -21,26 +58,29 @@ proc loadBibleIndex(rows, translationName: string): BibleIndex = if line.strip.len == 0: continue - let parts = line.split('\t', maxsplit = 3) - if parts.len != 4: + let parts = line.split('\t', maxsplit = 6) + if parts.len != 4 and parts.len != 7: raise newException(ValueError, "invalid embedded " & translationName & " row: " & line) - let code = parts[0] - let chapter = parseInt(parts[1]) - let verse = parseInt(parts[2]) - let text = parts[3] - - result.verses[verseKey(code, chapter, verse)] = text - - let cKey = chapterKey(code, chapter) - if not result.lastVerseByChapter.hasKey(cKey) or - verse > result.lastVerseByChapter[cKey]: - result.lastVerseByChapter[cKey] = verse - - if not result.lastChapterByBook.hasKey(code) or - chapter > result.lastChapterByBook[code]: - result.lastChapterByBook[code] = chapter + if parts.len == 4: + result.addSegment(BibleSegment( + code: parts[0], + chapter: parseInt(parts[1]), + verse: parseInt(parts[2]), + segment: 0, + breakKind: sbLine, + indent: 0, + text: parts[3])) + else: + result.addSegment(BibleSegment( + code: parts[0], + chapter: parseInt(parts[1]), + verse: parseInt(parts[2]), + segment: parseInt(parts[3]), + breakKind: parseSegmentBreak(parts[4]), + indent: parseInt(parts[5]), + text: parts[6])) proc requireLastChapter(index: BibleIndex, code: string): int = if not index.lastChapterByBook.hasKey(code): @@ -62,13 +102,10 @@ proc requireVerse(index: BibleIndex, code: string, chapter, verse: int): string raise newException(ValueError, "no embedded " & index.translationName & " data for " & bookInfo(code).name & " " & $chapter & ":" & $verse) - index.verses[vKey] + "" -proc addVerseLines( - lines: var seq[string], - index: BibleIndex, - reference: PassageReference, - range: RefRange) = +proc selectedVerses(index: BibleIndex, reference: PassageReference, range: RefRange): + Table[string, bool] = let code = reference.book.code discard index.requireLastChapter(code) @@ -90,16 +127,64 @@ proc addVerseLines( raise newException(ValueError, "reference range starts after it ends") for verse in startVerse .. endVerse: - lines.add(" [" & $verse & "] " & index.requireVerse(code, chapter, verse)) + discard index.requireVerse(code, chapter, verse) + result[verseKey(code, chapter, verse)] = true + +proc addFormattedSegment( + lines: var seq[string], + segment: BibleSegment, + emittedVerses: var Table[string, bool]) = + + let vKey = verseKey(segment.code, segment.chapter, segment.verse) + let marker = + if emittedVerses.hasKey(vKey): "" + else: + emittedVerses[vKey] = true + "[" & $segment.verse & "] " + + let prefix = repeat(" ", segment.indent + 1) + let content = marker & segment.text + let hasPassageLine = lines.len > 1 and lines[^1].len > 0 + + if segment.breakKind == sbSame and hasPassageLine: + if lines[^1].len > 0 and not lines[^1][^1].isSpaceAscii: + lines[^1].add(" ") + lines[^1].add(content) + else: + if segment.breakKind == sbParagraph and hasPassageLine: + lines.add("") + lines.add(prefix & content) + +proc addVerseLines( + lines: var seq[string], + index: BibleIndex, + reference: PassageReference, + range: RefRange) = + + let code = reference.book.code + let selected = index.selectedVerses(reference, range) + var emittedVerses: Table[string, bool] + var firstSelected = true + + for segment in index.segments: + if segment.code == code and + selected.hasKey(verseKey(segment.code, segment.chapter, segment.verse)): + var adjusted = segment + if firstSelected and adjusted.breakKind == sbSame: + adjusted.breakKind = sbLine + lines.addFormattedSegment(adjusted, emittedVerses) + firstSelected = false proc fetchReference(index: BibleIndex, reference: PassageReference): string = var lines = @[$reference] let code = reference.book.code if reference.ranges.len == 0: - for chapter in 1 .. index.requireLastChapter(code): - for verse in 1 .. index.requireLastVerse(code, chapter): - lines.add(" [" & $verse & "] " & index.requireVerse(code, chapter, verse)) + var emittedVerses: Table[string, bool] + discard index.requireLastChapter(code) + for segment in index.segments: + if segment.code == code: + lines.addFormattedSegment(segment, emittedVerses) else: for range in reference.ranges: lines.addVerseLines(index, reference, range) diff --git a/tests/test_offline_kjv.nim b/tests/test_offline_kjv.nim index 46d623d..f859651 100644 --- a/tests/test_offline_kjv.nim +++ b/tests/test_offline_kjv.nim @@ -1,5 +1,6 @@ import std/[strutils, unittest] +import ../src/embedded_bible import ../src/kjv import ../src/reference_parser @@ -82,3 +83,51 @@ suite "offline KJV backend": check passages.len == 1 check passages[0].startsWith("Jude 3\n") check passages[0].contains(" [3] ") + +suite "embedded Bible layout": + const layoutRows = """ +JHN 8 31 0 paragraph 0 Then Jesus said, +JHN 8 31 1 same 0 "If you remain in My word, +JHN 8 32 0 same 0 you shall know the truth." +JHN 8 33 0 paragraph 0 They answered Him. +HEB 4 7 0 paragraph 0 again He establishes a certain day: +HEB 4 7 1 line 0 "Today, if you will hear His voice, +HEB 4 7 2 line 1 do not harden your hearts." +""" + + test "preserves inline verse markers within a paragraph": + let passages = embedded_bible.fetchPassages(layoutRows, "John 8:31-32", "TEST") + + check passages.len == 1 + check passages[0] == "John 8:31-32\n" & + " [31] Then Jesus said, \"If you remain in My word, [32] you shall know the truth.\"" + + test "preserves paragraph breaks": + let passages = embedded_bible.fetchPassages(layoutRows, "John 8:31-33", "TEST") + + check passages[0] == "John 8:31-33\n" & + " [31] Then Jesus said, \"If you remain in My word, [32] you shall know the truth.\"\n" & + "\n" & + " [33] They answered Him." + + test "preserves indented quote lines": + let passages = embedded_bible.fetchPassages(layoutRows, "Hebrews 4:7", "TEST") + + check passages[0] == "Hebrews 4:7\n" & + " [7] again He establishes a certain day:\n" & + " \"Today, if you will hear His voice,\n" & + " do not harden your hearts.\"" + + test "starts mid-paragraph ranges on a fresh line": + let passages = embedded_bible.fetchPassages(layoutRows, "John 8:32-33", "TEST") + + check passages[0] == "John 8:32-33\n" & + " [32] you shall know the truth.\"\n" & + "\n" & + " [33] They answered Him." + + test "supports legacy verse rows": + const legacyRows = "JHN\t3\t16\tFor God so loved the world.\n" + let passages = embedded_bible.fetchPassages(legacyRows, "John 3:16", "TEST") + + check passages[0] == "John 3:16\n [16] For God so loved the world." diff --git a/tools/generate_kjv_data.nim b/tools/generate_kjv_data.nim index a950aba..57d513d 100644 --- a/tools/generate_kjv_data.nim +++ b/tools/generate_kjv_data.nim @@ -84,6 +84,22 @@ proc parseVerseLine(line: string): tuple[verse: int, text: string] = result.verse = parseInt(rest[0 ..< numberEnd]) result.text = stripUsfmMarkup(rest[numberEnd + 1 .. ^1]) +proc usfmBlock(line: string): tuple[matched: bool, segmentBreak: string, indent: int] = + if line.startsWith("\\p") or line.startsWith("\\m"): + return (true, "paragraph", 0) + + if line.startsWith("\\q"): + var idx = 2 + var digits = "" + while idx < line.len and line[idx].isDigit: + digits.add(line[idx]) + inc idx + + let indent = + if digits.len > 0: parseInt(digits) + else: 1 + return (true, "line", indent) + proc findCanonFiles(inputDir: string): Table[string, string] = for path in walkFiles(inputDir / "*eng-kjv.usfm"): let name = path.extractFilename @@ -104,35 +120,58 @@ proc generate(inputDir, outputPath: string) = var chapter = 0 var verse = 0 - var verseText = "" + var segment = 0 + var segmentText = "" + var segmentBreak = "line" + var indent = 0 - proc flushVerse() = + proc flushSegment() = if chapter > 0 and verse > 0: - let text = normalizeWhitespace(verseText).replace("\t", " ") + let text = normalizeWhitespace(segmentText).replace("\t", " ") if text.len > 0: - rows.add([code, $chapter, $verse, text].join("\t")) - verse = 0 - verseText = "" + rows.add([ + code, + $chapter, + $verse, + $segment, + segmentBreak, + $indent, + text + ].join("\t")) + inc segment + segmentBreak = "same" + segmentText = "" + + proc startBlock(nextBreak: string, nextIndent: int) = + flushSegment() + segmentBreak = nextBreak + indent = nextIndent for rawLine in canonFiles[code].lines: let line = rawLine.strip if line.startsWith("\\c "): - flushVerse() + flushSegment() chapter = parseInt(line[3..^1].strip) + verse = 0 elif line.startsWith("\\v "): - flushVerse() + flushSegment() let parsed = parseVerseLine(line) verse = parsed.verse - verseText = parsed.text + segment = 0 + segmentText = parsed.text elif verse > 0: + let block = usfmBlock(line) + if block.matched: + startBlock(block.segmentBreak, block.indent) + let continued = stripUsfmMarkup(line) if continued.len > 0: - if verseText.len > 0: - verseText.add(' ') - verseText.add(continued) + if segmentText.len > 0: + segmentText.add(' ') + segmentText.add(continued) - flushVerse() + flushSegment() createDir(outputPath.parentDir) writeFile(outputPath, rows.join("\n") & "\n") diff --git a/tools/generate_mev_data.nim b/tools/generate_mev_data.nim index afb4ae7..8ef7a97 100644 --- a/tools/generate_mev_data.nim +++ b/tools/generate_mev_data.nim @@ -25,7 +25,10 @@ type code: string chapter: int verse: int - verseText: string + segment: int + segmentText: string + segmentBreak: string + indent: int rows: seq[string] proc normalizeWhitespace(s: string): string = @@ -194,9 +197,12 @@ proc hasHref(node: XmlNode): bool = if hasHref(child): return true -proc isBlockElement(node: XmlNode): bool = - node.kind == xnElement and - node.tag in ["blockquote", "br", "div", "h1", "h2", "h3", "li", "p"] +proc isParagraphElement(node: XmlNode): bool = + node.kind == xnElement and node.tag == "p" and + (node.hasClass("calibre_18") or + node.hasClass("calibre_20") or + node.hasClass("calibre_23") or + node.hasClass("calibre_34")) proc chapterMarker(node: XmlNode): int = if node.kind == xnElement and node.tag == "span" and node.hasClass("calibre1"): @@ -243,13 +249,27 @@ proc leadingVerseText(s: string): tuple[verse: int, rest: string] = if idx < text.len: result.rest = text[idx .. ^1] -proc flushVerse(state: var ParseState) = +proc flushSegment(state: var ParseState) = if state.chapter > 0 and state.verse > 0: - let text = normalizeWhitespace(state.verseText).replace("\t", " ") + let text = normalizeWhitespace(state.segmentText).replace("\t", " ") if text.len > 0: - state.rows.add([state.code, $state.chapter, $state.verse, text].join("\t")) + state.rows.add([ + state.code, + $state.chapter, + $state.verse, + $state.segment, + state.segmentBreak, + $state.indent, + text + ].join("\t")) + inc state.segment + state.segmentBreak = "same" - state.verseText = "" + state.segmentText = "" + +proc startBlock(state: var ParseState, segmentBreak: string) = + state.flushSegment() + state.segmentBreak = segmentBreak proc walkPassageText(node: XmlNode, state: var ParseState) = case node.kind @@ -259,13 +279,14 @@ proc walkPassageText(node: XmlNode, state: var ParseState) = let leading = leadingVerseText(node.text) if leading.verse > 0: state.verse = leading.verse - state.verseText.add(leading.rest) + state.segment = 0 + state.segmentText.add(leading.rest) elif state.verse > 0: - state.verseText.add(node.text) + state.segmentText.add(node.text) of xnElement: let headingChapter = headingChapterMarker(node, state.code) if headingChapter > 0: - state.flushVerse() + state.flushSegment() state.chapter = headingChapter state.verse = 0 return @@ -275,25 +296,43 @@ proc walkPassageText(node: XmlNode, state: var ParseState) = let chapter = chapterMarker(node) if chapter > 0: - state.flushVerse() + state.flushSegment() state.chapter = chapter state.verse = 1 + state.segment = 0 return let verse = verseMarker(node) if verse > 0: - state.flushVerse() + state.flushSegment() state.verse = verse + state.segment = 0 return if node.tag == "sup": return - for child in node.items: - walkPassageText(child, state) - - if node.isBlockElement and state.chapter > 0 and state.verse > 0: - state.verseText.add(' ') + if node.tag == "p": + let segmentBreak = + if node.isParagraphElement: "paragraph" + else: "line" + state.startBlock(segmentBreak) + for child in node.items: + walkPassageText(child, state) + state.flushSegment() + elif node.tag == "blockquote": + let oldIndent = state.indent + state.indent = oldIndent + 1 + state.startBlock("line") + for child in node.items: + walkPassageText(child, state) + state.flushSegment() + state.indent = oldIndent + elif node.tag == "br": + state.startBlock("line") + else: + for child in node.items: + walkPassageText(child, state) else: discard @@ -310,7 +349,7 @@ proc parseBook(epubPath: string, source: BookSource): seq[string] = let doc = parseHtml(newStringStream(html)) walkPassageText(doc, state) - state.flushVerse() + state.flushSegment() state.rows proc generate(epubPath, outputPath: string) =