Preserve MEV passage metadata
This commit is contained in:
+16
-8
@@ -69,8 +69,8 @@ proc loadBibleIndex(rows, translationName: string): BibleIndex =
|
||||
if line.strip.len == 0:
|
||||
continue
|
||||
|
||||
let parts = line.split('\t', maxsplit = 6)
|
||||
if parts.len != 4 and parts.len != 7:
|
||||
let parts = line.split('\t', maxsplit = 7)
|
||||
if parts.len != 4 and parts.len != 7 and parts.len != 8:
|
||||
raise newException(ValueError,
|
||||
"invalid embedded " & translationName & " row: " & line)
|
||||
|
||||
@@ -142,6 +142,18 @@ proc selectedVerses(index: BibleIndex, reference: PassageReference, range: RefRa
|
||||
discard index.requireVerse(code, chapter, verse)
|
||||
result[verseKey(code, chapter, verse)] = true
|
||||
|
||||
proc shouldSeparateSegments(previous, next: string): bool =
|
||||
if previous.len == 0 or next.len == 0:
|
||||
return false
|
||||
if previous[^1].isSpaceAscii or next[0].isSpaceAscii:
|
||||
return false
|
||||
next[0] notin {'.', ',', ';', ':', '?', '!', ')', ']', '}'}
|
||||
|
||||
proc appendSameLine(line: var string, content: string) =
|
||||
if shouldSeparateSegments(line, content):
|
||||
line.add(" ")
|
||||
line.add(content)
|
||||
|
||||
proc addFormattedSegment(
|
||||
lines: var seq[string],
|
||||
segment: BibleSegment,
|
||||
@@ -159,17 +171,13 @@ proc addFormattedSegment(
|
||||
let hasPassageLine = lines.len > 1 and lines[^1].len > 0
|
||||
|
||||
if segment.breakKind == sbSame and hasPassageLine:
|
||||
if lines[^1].len > 0 and not lines[^1][^1].isSpaceAscii:
|
||||
lines[^1].add(" ")
|
||||
lines[^1].add(content)
|
||||
lines[^1].appendSameLine(content)
|
||||
else:
|
||||
let isMarkerlessParagraphContinuation =
|
||||
segment.breakKind == sbParagraph and marker.len == 0 and hasPassageLine
|
||||
|
||||
if isMarkerlessParagraphContinuation:
|
||||
if lines[^1].len > 0 and not lines[^1][^1].isSpaceAscii:
|
||||
lines[^1].add(" ")
|
||||
lines[^1].add(content)
|
||||
lines[^1].appendSameLine(content)
|
||||
else:
|
||||
if segment.breakKind == sbParagraph and hasPassageLine:
|
||||
lines.add("")
|
||||
|
||||
@@ -93,6 +93,8 @@ JHN 8 33 0 paragraph 0 They answered Him.
|
||||
JHN 8 39 0 paragraph 0 They answered Him, "Abraham is our father."
|
||||
JHN 8 39 1 paragraph 0 Jesus said to them, "If you were Abraham's children,
|
||||
JHN 8 40 0 same 0 you would do the works of Abraham."
|
||||
JHN 8 41 0 paragraph 0 You shall be free'
|
||||
JHN 8 41 1 same 0 ?
|
||||
HEB 4 7 0 paragraph 0 again He establishes a certain day:
|
||||
HEB 4 7 1 line 0 "Today, if you will hear His voice,
|
||||
HEB 4 7 2 line 1 do not harden your hearts."
|
||||
@@ -137,12 +139,26 @@ HEB 4 7 2 line 1 do not harden your hearts."
|
||||
"Jesus said to them, \"If you were Abraham's children, " &
|
||||
"[40] you would do the works of Abraham.\""
|
||||
|
||||
test "joins punctuation-only segments without extra spacing":
|
||||
let passages = embedded_bible.fetchPassages(layoutRows, "John 8:41", "TEST")
|
||||
|
||||
check passages[0] == "John 8:41\n [41] You shall be free'?"
|
||||
|
||||
test "supports legacy verse rows":
|
||||
const legacyRows = "JHN\t3\t16\tFor God so loved the world.\n"
|
||||
let passages = embedded_bible.fetchPassages(legacyRows, "John 3:16", "TEST")
|
||||
|
||||
check passages[0] == "John 3:16\n [16] For God so loved the world."
|
||||
|
||||
test "ignores layout metadata while rendering":
|
||||
const metadataRows =
|
||||
"JHN\t8\t31\t0\tparagraph\t0\tJesus said.\t" &
|
||||
"""{"wordsOfChrist":true,"annotations":[{"kind":"crossReference","marker":"a","href":"index_split_1.html#x","offset":6}]}""" &
|
||||
"\n"
|
||||
let passages = embedded_bible.fetchPassages(metadataRows, "John 8:31", "TEST")
|
||||
|
||||
check passages[0] == "John 8:31\n [31] Jesus said."
|
||||
|
||||
test "groups legacy rows into paragraphs using pilcrows":
|
||||
const legacyRows =
|
||||
"JHN\t8\t31\tThen Jesus said.\n" &
|
||||
|
||||
+120
-2
@@ -1,9 +1,11 @@
|
||||
import std/[
|
||||
htmlparser,
|
||||
json,
|
||||
os,
|
||||
osproc,
|
||||
streams,
|
||||
strutils,
|
||||
tables,
|
||||
xmlparser,
|
||||
xmltree
|
||||
]
|
||||
@@ -29,6 +31,11 @@ type
|
||||
segmentText: string
|
||||
segmentBreak: string
|
||||
indent: int
|
||||
wordsOfChrist: bool
|
||||
annotations: seq[JsonNode]
|
||||
epubPath: string
|
||||
targetTextCache: Table[string, string]
|
||||
targetFileCache: Table[string, Table[string, string]]
|
||||
rows: seq[string]
|
||||
|
||||
proc normalizeWhitespace(s: string): string =
|
||||
@@ -197,6 +204,92 @@ proc hasHref(node: XmlNode): bool =
|
||||
if hasHref(child):
|
||||
return true
|
||||
|
||||
proc firstHref(node: XmlNode): string =
|
||||
if node.kind == xnElement:
|
||||
let href = node.attr("href")
|
||||
if href.len > 0:
|
||||
return href
|
||||
|
||||
for child in node.items:
|
||||
let href = firstHref(child)
|
||||
if href.len > 0:
|
||||
return href
|
||||
|
||||
proc collectIds(node: XmlNode, ids: var seq[string]) =
|
||||
if node.kind == xnElement:
|
||||
let id = node.attr("id")
|
||||
if id.len > 0:
|
||||
ids.add(id)
|
||||
for child in node.items:
|
||||
collectIds(child, ids)
|
||||
|
||||
proc targetParagraphs(doc: XmlNode): Table[string, string] =
|
||||
var paragraphs = initTable[string, string]()
|
||||
|
||||
proc walk(node: XmlNode) =
|
||||
if node.kind == xnElement:
|
||||
if node.tag == "p":
|
||||
var ids: seq[string] = @[]
|
||||
collectIds(node, ids)
|
||||
if ids.len > 0:
|
||||
let text = normalizeWhitespace(textContent(node))
|
||||
for id in ids:
|
||||
paragraphs[id] = text
|
||||
|
||||
for child in node.items:
|
||||
walk(child)
|
||||
|
||||
walk(doc)
|
||||
paragraphs
|
||||
|
||||
proc linkTargetText(state: var ParseState, href: string): string =
|
||||
if href.len == 0:
|
||||
return ""
|
||||
|
||||
if state.targetTextCache.hasKey(href):
|
||||
return state.targetTextCache[href]
|
||||
|
||||
let parts = href.split('#', maxsplit = 1)
|
||||
if parts.len != 2 or parts[0].len == 0 or parts[1].len == 0:
|
||||
state.targetTextCache[href] = ""
|
||||
return ""
|
||||
|
||||
if not state.targetFileCache.hasKey(parts[0]):
|
||||
try:
|
||||
let html = readEpubEntry(state.epubPath, parts[0])
|
||||
let doc = parseHtml(newStringStream(html))
|
||||
state.targetFileCache[parts[0]] = targetParagraphs(doc)
|
||||
except CatchableError:
|
||||
state.targetFileCache[parts[0]] = initTable[string, string]()
|
||||
|
||||
if state.targetFileCache[parts[0]].hasKey(parts[1]):
|
||||
result = state.targetFileCache[parts[0]][parts[1]]
|
||||
else:
|
||||
result = ""
|
||||
|
||||
state.targetTextCache[href] = result
|
||||
|
||||
proc annotationKind(marker: string): string =
|
||||
if marker.strip.allCharsInSet({'0'..'9'}): "note"
|
||||
else: "crossReference"
|
||||
|
||||
proc annotationNode(marker, href, targetText: string, offset: int): JsonNode =
|
||||
result = %*{
|
||||
"kind": annotationKind(marker),
|
||||
"marker": marker,
|
||||
"href": href,
|
||||
"offset": offset
|
||||
}
|
||||
if targetText.len > 0:
|
||||
result["targetText"] = newJString(targetText)
|
||||
|
||||
proc segmentMetadata(state: ParseState): JsonNode =
|
||||
result = newJObject()
|
||||
if state.wordsOfChrist:
|
||||
result["wordsOfChrist"] = newJBool(true)
|
||||
if state.annotations.len > 0:
|
||||
result["annotations"] = %state.annotations
|
||||
|
||||
proc isParagraphElement(node: XmlNode): bool =
|
||||
node.kind == xnElement and node.tag == "p" and
|
||||
(node.hasClass("calibre_18") or
|
||||
@@ -253,6 +346,7 @@ proc flushSegment(state: var ParseState) =
|
||||
if state.chapter > 0 and state.verse > 0:
|
||||
let text = normalizeWhitespace(state.segmentText).replace("\t", " ")
|
||||
if text.len > 0:
|
||||
let metadata = segmentMetadata(state)
|
||||
state.rows.add([
|
||||
state.code,
|
||||
$state.chapter,
|
||||
@@ -260,12 +354,15 @@ proc flushSegment(state: var ParseState) =
|
||||
$state.segment,
|
||||
state.segmentBreak,
|
||||
$state.indent,
|
||||
text
|
||||
text,
|
||||
$metadata
|
||||
].join("\t"))
|
||||
inc state.segment
|
||||
state.segmentBreak = "same"
|
||||
|
||||
state.segmentText = ""
|
||||
state.wordsOfChrist = false
|
||||
state.annotations = @[]
|
||||
|
||||
proc startBlock(state: var ParseState, segmentBreak: string) =
|
||||
state.flushSegment()
|
||||
@@ -309,9 +406,30 @@ proc walkPassageText(node: XmlNode, state: var ParseState) =
|
||||
state.segment = 0
|
||||
return
|
||||
|
||||
if node.tag == "sup" and node.hasHref:
|
||||
let marker = markerText(textContent(node))
|
||||
let href = firstHref(node)
|
||||
if marker.len > 0 and href.len > 0:
|
||||
state.annotations.add(annotationNode(
|
||||
marker,
|
||||
href,
|
||||
state.linkTargetText(href),
|
||||
normalizeWhitespace(state.segmentText).len))
|
||||
return
|
||||
|
||||
if node.tag == "sup":
|
||||
return
|
||||
|
||||
if node.hasClass("calibre_40"):
|
||||
state.flushSegment()
|
||||
let oldWordsOfChrist = state.wordsOfChrist
|
||||
state.wordsOfChrist = true
|
||||
for child in node.items:
|
||||
walkPassageText(child, state)
|
||||
state.flushSegment()
|
||||
state.wordsOfChrist = oldWordsOfChrist
|
||||
return
|
||||
|
||||
if node.tag == "p":
|
||||
let segmentBreak =
|
||||
if node.isParagraphElement: "paragraph"
|
||||
@@ -340,7 +458,7 @@ proc indexSplitFile(index: int): string =
|
||||
"index_split_" & align($index, 3, '0') & ".html"
|
||||
|
||||
proc parseBook(epubPath: string, source: BookSource): seq[string] =
|
||||
var state = ParseState(code: source.code)
|
||||
var state = ParseState(code: source.code, epubPath: epubPath)
|
||||
if bookInfo(source.code).singleChapter:
|
||||
state.chapter = 1
|
||||
|
||||
|
||||
Reference in New Issue
Block a user