Preserve layout in embedded Bible data

2026-07-05 20:49:18 -05:00
parent dbc39480f7
commit 98253920ca
4 changed files with 276 additions and 64 deletions
@@ -84,6 +84,22 @@ proc parseVerseLine(line: string): tuple[verse: int, text: string] =
  result.verse = parseInt(rest[0 ..< numberEnd])
  result.text = stripUsfmMarkup(rest[numberEnd + 1 .. ^1])

+proc usfmBlock(line: string): tuple[matched: bool, segmentBreak: string, indent: int] =
+  if line.startsWith("\\p") or line.startsWith("\\m"):
+    return (true, "paragraph", 0)
+
+  if line.startsWith("\\q"):
+    var idx = 2
+    var digits = ""
+    while idx < line.len and line[idx].isDigit:
+      digits.add(line[idx])
+      inc idx
+
+    let indent =
+      if digits.len > 0: parseInt(digits)
+      else: 1
+    return (true, "line", indent)
+
 proc findCanonFiles(inputDir: string): Table[string, string] =
  for path in walkFiles(inputDir / "*eng-kjv.usfm"):
    let name = path.extractFilename
@@ -104,35 +120,58 @@ proc generate(inputDir, outputPath: string) =

    var chapter = 0
    var verse = 0
-    var verseText = ""
+    var segment = 0
+    var segmentText = ""
+    var segmentBreak = "line"
+    var indent = 0

-    proc flushVerse() =
+    proc flushSegment() =
      if chapter > 0 and verse > 0:
-        let text = normalizeWhitespace(verseText).replace("\t", " ")
+        let text = normalizeWhitespace(segmentText).replace("\t", " ")
        if text.len > 0:
-          rows.add([code, $chapter, $verse, text].join("\t"))
-      verse = 0
-      verseText = ""
+          rows.add([
+            code,
+            $chapter,
+            $verse,
+            $segment,
+            segmentBreak,
+            $indent,
+            text
+          ].join("\t"))
+          inc segment
+          segmentBreak = "same"
+      segmentText = ""
+
+    proc startBlock(nextBreak: string, nextIndent: int) =
+      flushSegment()
+      segmentBreak = nextBreak
+      indent = nextIndent

    for rawLine in canonFiles[code].lines:
      let line = rawLine.strip

      if line.startsWith("\\c "):
-        flushVerse()
+        flushSegment()
        chapter = parseInt(line[3..^1].strip)
+        verse = 0
      elif line.startsWith("\\v "):
-        flushVerse()
+        flushSegment()
        let parsed = parseVerseLine(line)
        verse = parsed.verse
-        verseText = parsed.text
+        segment = 0
+        segmentText = parsed.text
      elif verse > 0:
+        let block = usfmBlock(line)
+        if block.matched:
+          startBlock(block.segmentBreak, block.indent)
+
        let continued = stripUsfmMarkup(line)
        if continued.len > 0:
-          if verseText.len > 0:
-            verseText.add(' ')
-          verseText.add(continued)
+          if segmentText.len > 0:
+            segmentText.add(' ')
+          segmentText.add(continued)

-    flushVerse()
+    flushSegment()

  createDir(outputPath.parentDir)
  writeFile(outputPath, rows.join("\n") & "\n")
@@ -25,7 +25,10 @@ type
    code: string
    chapter: int
    verse: int
-    verseText: string
+    segment: int
+    segmentText: string
+    segmentBreak: string
+    indent: int
    rows: seq[string]

 proc normalizeWhitespace(s: string): string =
@@ -194,9 +197,12 @@ proc hasHref(node: XmlNode): bool =
      if hasHref(child):
        return true

-proc isBlockElement(node: XmlNode): bool =
-  node.kind == xnElement and
-    node.tag in ["blockquote", "br", "div", "h1", "h2", "h3", "li", "p"]
+proc isParagraphElement(node: XmlNode): bool =
+  node.kind == xnElement and node.tag == "p" and
+    (node.hasClass("calibre_18") or
+     node.hasClass("calibre_20") or
+     node.hasClass("calibre_23") or
+     node.hasClass("calibre_34"))

 proc chapterMarker(node: XmlNode): int =
  if node.kind == xnElement and node.tag == "span" and node.hasClass("calibre1"):
@@ -243,13 +249,27 @@ proc leadingVerseText(s: string): tuple[verse: int, rest: string] =
  if idx < text.len:
    result.rest = text[idx .. ^1]

-proc flushVerse(state: var ParseState) =
+proc flushSegment(state: var ParseState) =
  if state.chapter > 0 and state.verse > 0:
-    let text = normalizeWhitespace(state.verseText).replace("\t", " ")
+    let text = normalizeWhitespace(state.segmentText).replace("\t", " ")
    if text.len > 0:
-      state.rows.add([state.code, $state.chapter, $state.verse, text].join("\t"))
+      state.rows.add([
+        state.code,
+        $state.chapter,
+        $state.verse,
+        $state.segment,
+        state.segmentBreak,
+        $state.indent,
+        text
+      ].join("\t"))
+      inc state.segment
+      state.segmentBreak = "same"

-  state.verseText = ""
+  state.segmentText = ""
+
+proc startBlock(state: var ParseState, segmentBreak: string) =
+  state.flushSegment()
+  state.segmentBreak = segmentBreak

 proc walkPassageText(node: XmlNode, state: var ParseState) =
  case node.kind
@@ -259,13 +279,14 @@ proc walkPassageText(node: XmlNode, state: var ParseState) =
        let leading = leadingVerseText(node.text)
        if leading.verse > 0:
          state.verse = leading.verse
-          state.verseText.add(leading.rest)
+          state.segment = 0
+          state.segmentText.add(leading.rest)
      elif state.verse > 0:
-        state.verseText.add(node.text)
+        state.segmentText.add(node.text)
  of xnElement:
    let headingChapter = headingChapterMarker(node, state.code)
    if headingChapter > 0:
-      state.flushVerse()
+      state.flushSegment()
      state.chapter = headingChapter
      state.verse = 0
      return
@@ -275,25 +296,43 @@ proc walkPassageText(node: XmlNode, state: var ParseState) =

    let chapter = chapterMarker(node)
    if chapter > 0:
-      state.flushVerse()
+      state.flushSegment()
      state.chapter = chapter
      state.verse = 1
+      state.segment = 0
      return

    let verse = verseMarker(node)
    if verse > 0:
-      state.flushVerse()
+      state.flushSegment()
      state.verse = verse
+      state.segment = 0
      return

    if node.tag == "sup":
      return

-    for child in node.items:
-      walkPassageText(child, state)
-
-    if node.isBlockElement and state.chapter > 0 and state.verse > 0:
-      state.verseText.add(' ')
+    if node.tag == "p":
+      let segmentBreak =
+        if node.isParagraphElement: "paragraph"
+        else: "line"
+      state.startBlock(segmentBreak)
+      for child in node.items:
+        walkPassageText(child, state)
+      state.flushSegment()
+    elif node.tag == "blockquote":
+      let oldIndent = state.indent
+      state.indent = oldIndent + 1
+      state.startBlock("line")
+      for child in node.items:
+        walkPassageText(child, state)
+      state.flushSegment()
+      state.indent = oldIndent
+    elif node.tag == "br":
+      state.startBlock("line")
+    else:
+      for child in node.items:
+        walkPassageText(child, state)
  else:
    discard

@@ -310,7 +349,7 @@ proc parseBook(epubPath: string, source: BookSource): seq[string] =
    let doc = parseHtml(newStringStream(html))
    walkPassageText(doc, state)

-  state.flushVerse()
+  state.flushSegment()
  state.rows

 proc generate(epubPath, outputPath: string) =