jdb/bibleref
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve layout in embedded Bible data

Browse Source
This commit is contained in:
Jonathan Bernard
2026-07-05 20:49:18 -05:00
parent dbc39480f7
commit 98253920ca
4 changed files with 276 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/embedded_bible.nim
+114 -29
View File
@@ -2,8 +2,24 @@ import std/[strutils, tables]
import ./reference_parser
type BibleIndex = object
verses: Table[string, string]
type
SegmentBreak = enum
sbSame = "same"
sbLine = "line"
sbParagraph = "paragraph"
BibleSegment = object
code: string
chapter: int
verse: int
segment: int
breakKind: SegmentBreak
indent: int
text: string
BibleIndex = object
segments: seq[BibleSegment]
verses: Table[string, bool]
lastVerseByChapter: Table[string, int]
lastChapterByBook: Table[string, int]
translationName: string
@@ -14,6 +30,27 @@ proc verseKey(code: string, chapter, verse: int): string =
proc chapterKey(code: string, chapter: int): string =
code & "\t" & $chapter
proc parseSegmentBreak(s: string): SegmentBreak =
case s
of "same": sbSame
of "line": sbLine
of "paragraph": sbParagraph
else:
raise newException(ValueError, "invalid embedded Bible segment break: " & s)
proc addSegment(index: var BibleIndex, segment: BibleSegment) =
index.segments.add(segment)
index.verses[verseKey(segment.code, segment.chapter, segment.verse)] = true
let cKey = chapterKey(segment.code, segment.chapter)
if not index.lastVerseByChapter.hasKey(cKey) or
segment.verse > index.lastVerseByChapter[cKey]:
index.lastVerseByChapter[cKey] = segment.verse
if not index.lastChapterByBook.hasKey(segment.code) or
segment.chapter > index.lastChapterByBook[segment.code]:
index.lastChapterByBook[segment.code] = segment.chapter
proc loadBibleIndex(rows, translationName: string): BibleIndex =
result.translationName = translationName
@@ -21,26 +58,29 @@ proc loadBibleIndex(rows, translationName: string): BibleIndex =
if line.strip.len == 0:
continue
let parts = line.split('\t', maxsplit = 3)
if parts.len != 4:
let parts = line.split('\t', maxsplit = 6)
if parts.len != 4 and parts.len != 7:
raise newException(ValueError,
"invalid embedded " & translationName & " row: " & line)
let code = parts[0]
let chapter = parseInt(parts[1])
let verse = parseInt(parts[2])
let text = parts[3]
result.verses[verseKey(code, chapter, verse)] = text
let cKey = chapterKey(code, chapter)
if not result.lastVerseByChapter.hasKey(cKey) or
verse > result.lastVerseByChapter[cKey]:
result.lastVerseByChapter[cKey] = verse
if not result.lastChapterByBook.hasKey(code) or
chapter > result.lastChapterByBook[code]:
result.lastChapterByBook[code] = chapter
if parts.len == 4:
result.addSegment(BibleSegment(
code: parts[0],
chapter: parseInt(parts[1]),
verse: parseInt(parts[2]),
segment: 0,
breakKind: sbLine,
indent: 0,
text: parts[3]))
else:
result.addSegment(BibleSegment(
code: parts[0],
chapter: parseInt(parts[1]),
verse: parseInt(parts[2]),
segment: parseInt(parts[3]),
breakKind: parseSegmentBreak(parts[4]),
indent: parseInt(parts[5]),
text: parts[6]))
proc requireLastChapter(index: BibleIndex, code: string): int =
if not index.lastChapterByBook.hasKey(code):
@@ -62,13 +102,10 @@ proc requireVerse(index: BibleIndex, code: string, chapter, verse: int): string
raise newException(ValueError,
"no embedded " & index.translationName & " data for " &
bookInfo(code).name & " " & $chapter & ":" & $verse)
index.verses[vKey]
""
proc addVerseLines(
lines: var seq[string],
index: BibleIndex,
reference: PassageReference,
range: RefRange) =
proc selectedVerses(index: BibleIndex, reference: PassageReference, range: RefRange):
Table[string, bool] =
let code = reference.book.code
discard index.requireLastChapter(code)
@@ -90,16 +127,64 @@ proc addVerseLines(
raise newException(ValueError, "reference range starts after it ends")
for verse in startVerse .. endVerse:
lines.add(" [" & $verse & "] " & index.requireVerse(code, chapter, verse))
discard index.requireVerse(code, chapter, verse)
result[verseKey(code, chapter, verse)] = true
proc addFormattedSegment(
lines: var seq[string],
segment: BibleSegment,
emittedVerses: var Table[string, bool]) =
let vKey = verseKey(segment.code, segment.chapter, segment.verse)
let marker =
if emittedVerses.hasKey(vKey): ""
else:
emittedVerses[vKey] = true
"[" & $segment.verse & "] "
let prefix = repeat(" ", segment.indent + 1)
let content = marker & segment.text
let hasPassageLine = lines.len > 1 and lines[^1].len > 0
if segment.breakKind == sbSame and hasPassageLine:
if lines[^1].len > 0 and not lines[^1][^1].isSpaceAscii:
lines[^1].add(" ")
lines[^1].add(content)
else:
if segment.breakKind == sbParagraph and hasPassageLine:
lines.add("")
lines.add(prefix & content)
proc addVerseLines(
lines: var seq[string],
index: BibleIndex,
reference: PassageReference,
range: RefRange) =
let code = reference.book.code
let selected = index.selectedVerses(reference, range)
var emittedVerses: Table[string, bool]
var firstSelected = true
for segment in index.segments:
if segment.code == code and
selected.hasKey(verseKey(segment.code, segment.chapter, segment.verse)):
var adjusted = segment
if firstSelected and adjusted.breakKind == sbSame:
adjusted.breakKind = sbLine
lines.addFormattedSegment(adjusted, emittedVerses)
firstSelected = false
proc fetchReference(index: BibleIndex, reference: PassageReference): string =
var lines = @[$reference]
let code = reference.book.code
if reference.ranges.len == 0:
for chapter in 1 .. index.requireLastChapter(code):
for verse in 1 .. index.requireLastVerse(code, chapter):
lines.add(" [" & $verse & "] " & index.requireVerse(code, chapter, verse))
var emittedVerses: Table[string, bool]
discard index.requireLastChapter(code)
for segment in index.segments:
if segment.code == code:
lines.addFormattedSegment(segment, emittedVerses)
else:
for range in reference.ranges:
lines.addVerseLines(index, reference, range)
tests/test_offline_kjv.nim
+49
View File
@@ -1,5 +1,6 @@
import std/[strutils, unittest]
import ../src/embedded_bible
import ../src/kjv
import ../src/reference_parser
@@ -82,3 +83,51 @@ suite "offline KJV backend":
check passages.len == 1
check passages[0].startsWith("Jude 3\n")
check passages[0].contains(" [3] ")
suite "embedded Bible layout":
const layoutRows = """
JHN 8 31 0 paragraph 0 Then Jesus said,
JHN 8 31 1 same 0 "If you remain in My word,
JHN 8 32 0 same 0 you shall know the truth."
JHN 8 33 0 paragraph 0 They answered Him.
HEB 4 7 0 paragraph 0 again He establishes a certain day:
HEB 4 7 1 line 0 "Today, if you will hear His voice,
HEB 4 7 2 line 1 do not harden your hearts."
"""
test "preserves inline verse markers within a paragraph":
let passages = embedded_bible.fetchPassages(layoutRows, "John 8:31-32", "TEST")
check passages.len == 1
check passages[0] == "John 8:31-32\n" &
" [31] Then Jesus said, \"If you remain in My word, [32] you shall know the truth.\""
test "preserves paragraph breaks":
let passages = embedded_bible.fetchPassages(layoutRows, "John 8:31-33", "TEST")
check passages[0] == "John 8:31-33\n" &
" [31] Then Jesus said, \"If you remain in My word, [32] you shall know the truth.\"\n" &
"\n" &
" [33] They answered Him."
test "preserves indented quote lines":
let passages = embedded_bible.fetchPassages(layoutRows, "Hebrews 4:7", "TEST")
check passages[0] == "Hebrews 4:7\n" &
" [7] again He establishes a certain day:\n" &
" \"Today, if you will hear His voice,\n" &
" do not harden your hearts.\""
test "starts mid-paragraph ranges on a fresh line":
let passages = embedded_bible.fetchPassages(layoutRows, "John 8:32-33", "TEST")
check passages[0] == "John 8:32-33\n" &
" [32] you shall know the truth.\"\n" &
"\n" &
" [33] They answered Him."
test "supports legacy verse rows":
const legacyRows = "JHN\t3\t16\tFor God so loved the world.\n"
let passages = embedded_bible.fetchPassages(legacyRows, "John 3:16", "TEST")
check passages[0] == "John 3:16\n [16] For God so loved the world."
tools/generate_kjv_data.nim
+52 -13
View File
@@ -84,6 +84,22 @@ proc parseVerseLine(line: string): tuple[verse: int, text: string] =
result.verse = parseInt(rest[0 ..< numberEnd])
result.text = stripUsfmMarkup(rest[numberEnd + 1 .. ^1])
proc usfmBlock(line: string): tuple[matched: bool, segmentBreak: string, indent: int] =
if line.startsWith("\\p") or line.startsWith("\\m"):
return (true, "paragraph", 0)
if line.startsWith("\\q"):
var idx = 2
var digits = ""
while idx < line.len and line[idx].isDigit:
digits.add(line[idx])
inc idx
let indent =
if digits.len > 0: parseInt(digits)
else: 1
return (true, "line", indent)
proc findCanonFiles(inputDir: string): Table[string, string] =
for path in walkFiles(inputDir / "*eng-kjv.usfm"):
let name = path.extractFilename
@@ -104,35 +120,58 @@ proc generate(inputDir, outputPath: string) =
var chapter = 0
var verse = 0
var verseText = ""
var segment = 0
var segmentText = ""
var segmentBreak = "line"
var indent = 0
proc flushVerse() =
proc flushSegment() =
if chapter > 0 and verse > 0:
let text = normalizeWhitespace(verseText).replace("\t", " ")
let text = normalizeWhitespace(segmentText).replace("\t", " ")
if text.len > 0:
rows.add([code, $chapter, $verse, text].join("\t"))
verse = 0
verseText = ""
rows.add([
code,
$chapter,
$verse,
$segment,
segmentBreak,
$indent,
text
].join("\t"))
inc segment
segmentBreak = "same"
segmentText = ""
proc startBlock(nextBreak: string, nextIndent: int) =
flushSegment()
segmentBreak = nextBreak
indent = nextIndent
for rawLine in canonFiles[code].lines:
let line = rawLine.strip
if line.startsWith("\\c "):
flushVerse()
flushSegment()
chapter = parseInt(line[3..^1].strip)
verse = 0
elif line.startsWith("\\v "):
flushVerse()
flushSegment()
let parsed = parseVerseLine(line)
verse = parsed.verse
verseText = parsed.text
segment = 0
segmentText = parsed.text
elif verse > 0:
let block = usfmBlock(line)
if block.matched:
startBlock(block.segmentBreak, block.indent)
let continued = stripUsfmMarkup(line)
if continued.len > 0:
if verseText.len > 0:
verseText.add(' ')
verseText.add(continued)
if segmentText.len > 0:
segmentText.add(' ')
segmentText.add(continued)
flushVerse()
flushSegment()
createDir(outputPath.parentDir)
writeFile(outputPath, rows.join("\n") & "\n")
tools/generate_mev_data.nim
+56 -17
View File
@@ -25,7 +25,10 @@ type
code: string
chapter: int
verse: int
verseText: string
segment: int
segmentText: string
segmentBreak: string
indent: int
rows: seq[string]
proc normalizeWhitespace(s: string): string =
@@ -194,9 +197,12 @@ proc hasHref(node: XmlNode): bool =
if hasHref(child):
return true
proc isBlockElement(node: XmlNode): bool =
node.kind == xnElement and
node.tag in ["blockquote", "br", "div", "h1", "h2", "h3", "li", "p"]
proc isParagraphElement(node: XmlNode): bool =
node.kind == xnElement and node.tag == "p" and
(node.hasClass("calibre_18") or
node.hasClass("calibre_20") or
node.hasClass("calibre_23") or
node.hasClass("calibre_34"))
proc chapterMarker(node: XmlNode): int =
if node.kind == xnElement and node.tag == "span" and node.hasClass("calibre1"):
@@ -243,13 +249,27 @@ proc leadingVerseText(s: string): tuple[verse: int, rest: string] =
if idx < text.len:
result.rest = text[idx .. ^1]
proc flushVerse(state: var ParseState) =
proc flushSegment(state: var ParseState) =
if state.chapter > 0 and state.verse > 0:
let text = normalizeWhitespace(state.verseText).replace("\t", " ")
let text = normalizeWhitespace(state.segmentText).replace("\t", " ")
if text.len > 0:
state.rows.add([state.code, $state.chapter, $state.verse, text].join("\t"))
state.rows.add([
state.code,
$state.chapter,
$state.verse,
$state.segment,
state.segmentBreak,
$state.indent,
text
].join("\t"))
inc state.segment
state.segmentBreak = "same"
state.verseText = ""
state.segmentText = ""
proc startBlock(state: var ParseState, segmentBreak: string) =
state.flushSegment()
state.segmentBreak = segmentBreak
proc walkPassageText(node: XmlNode, state: var ParseState) =
case node.kind
@@ -259,13 +279,14 @@ proc walkPassageText(node: XmlNode, state: var ParseState) =
let leading = leadingVerseText(node.text)
if leading.verse > 0:
state.verse = leading.verse
state.verseText.add(leading.rest)
state.segment = 0
state.segmentText.add(leading.rest)
elif state.verse > 0:
state.verseText.add(node.text)
state.segmentText.add(node.text)
of xnElement:
let headingChapter = headingChapterMarker(node, state.code)
if headingChapter > 0:
state.flushVerse()
state.flushSegment()
state.chapter = headingChapter
state.verse = 0
return
@@ -275,25 +296,43 @@ proc walkPassageText(node: XmlNode, state: var ParseState) =
let chapter = chapterMarker(node)
if chapter > 0:
state.flushVerse()
state.flushSegment()
state.chapter = chapter
state.verse = 1
state.segment = 0
return
let verse = verseMarker(node)
if verse > 0:
state.flushVerse()
state.flushSegment()
state.verse = verse
state.segment = 0
return
if node.tag == "sup":
return
if node.tag == "p":
let segmentBreak =
if node.isParagraphElement: "paragraph"
else: "line"
state.startBlock(segmentBreak)
for child in node.items:
walkPassageText(child, state)
state.flushSegment()
elif node.tag == "blockquote":
let oldIndent = state.indent
state.indent = oldIndent + 1
state.startBlock("line")
for child in node.items:
walkPassageText(child, state)
state.flushSegment()
state.indent = oldIndent
elif node.tag == "br":
state.startBlock("line")
else:
for child in node.items:
walkPassageText(child, state)
if node.isBlockElement and state.chapter > 0 and state.verse > 0:
state.verseText.add(' ')
else:
discard
@@ -310,7 +349,7 @@ proc parseBook(epubPath: string, source: BookSource): seq[string] =
let doc = parseHtml(newStringStream(html))
walkPassageText(doc, state)
state.flushVerse()
state.flushSegment()
state.rows
proc generate(epubPath, outputPath: string) =