Preserve layout in embedded Bible data

tools/generate_mev_data.nim

@@ -25,7 +25,10 @@ type
     code: string
     chapter: int
     verse: int
-    verseText: string
+    segment: int
+    segmentText: string
+    segmentBreak: string
+    indent: int
     rows: seq[string]
 
 proc normalizeWhitespace(s: string): string =
@@ -194,9 +197,12 @@ proc hasHref(node: XmlNode): bool =
       if hasHref(child):
         return true
 
-proc isBlockElement(node: XmlNode): bool =
-  node.kind == xnElement and
-    node.tag in ["blockquote", "br", "div", "h1", "h2", "h3", "li", "p"]
+proc isParagraphElement(node: XmlNode): bool =
+  node.kind == xnElement and node.tag == "p" and
+    (node.hasClass("calibre_18") or
+     node.hasClass("calibre_20") or
+     node.hasClass("calibre_23") or
+     node.hasClass("calibre_34"))
 
 proc chapterMarker(node: XmlNode): int =
   if node.kind == xnElement and node.tag == "span" and node.hasClass("calibre1"):
@@ -243,13 +249,27 @@ proc leadingVerseText(s: string): tuple[verse: int, rest: string] =
   if idx < text.len:
     result.rest = text[idx .. ^1]
 
-proc flushVerse(state: var ParseState) =
+proc flushSegment(state: var ParseState) =
   if state.chapter > 0 and state.verse > 0:
-    let text = normalizeWhitespace(state.verseText).replace("\t", " ")
+    let text = normalizeWhitespace(state.segmentText).replace("\t", " ")
     if text.len > 0:
-      state.rows.add([state.code, $state.chapter, $state.verse, text].join("\t"))
+      state.rows.add([
+        state.code,
+        $state.chapter,
+        $state.verse,
+        $state.segment,
+        state.segmentBreak,
+        $state.indent,
+        text
+      ].join("\t"))
+      inc state.segment
+      state.segmentBreak = "same"
 
-  state.verseText = ""
+  state.segmentText = ""
+
+proc startBlock(state: var ParseState, segmentBreak: string) =
+  state.flushSegment()
+  state.segmentBreak = segmentBreak
 
 proc walkPassageText(node: XmlNode, state: var ParseState) =
   case node.kind
@@ -259,13 +279,14 @@ proc walkPassageText(node: XmlNode, state: var ParseState) =
         let leading = leadingVerseText(node.text)
         if leading.verse > 0:
           state.verse = leading.verse
-          state.verseText.add(leading.rest)
+          state.segment = 0
+          state.segmentText.add(leading.rest)
       elif state.verse > 0:
-        state.verseText.add(node.text)
+        state.segmentText.add(node.text)
   of xnElement:
     let headingChapter = headingChapterMarker(node, state.code)
     if headingChapter > 0:
-      state.flushVerse()
+      state.flushSegment()
       state.chapter = headingChapter
       state.verse = 0
       return
@@ -275,25 +296,43 @@ proc walkPassageText(node: XmlNode, state: var ParseState) =
 
     let chapter = chapterMarker(node)
     if chapter > 0:
-      state.flushVerse()
+      state.flushSegment()
       state.chapter = chapter
       state.verse = 1
+      state.segment = 0
       return
 
     let verse = verseMarker(node)
     if verse > 0:
-      state.flushVerse()
+      state.flushSegment()
       state.verse = verse
+      state.segment = 0
       return
 
     if node.tag == "sup":
       return
 
-    for child in node.items:
-      walkPassageText(child, state)
-
-    if node.isBlockElement and state.chapter > 0 and state.verse > 0:
-      state.verseText.add(' ')
+    if node.tag == "p":
+      let segmentBreak =
+        if node.isParagraphElement: "paragraph"
+        else: "line"
+      state.startBlock(segmentBreak)
+      for child in node.items:
+        walkPassageText(child, state)
+      state.flushSegment()
+    elif node.tag == "blockquote":
+      let oldIndent = state.indent
+      state.indent = oldIndent + 1
+      state.startBlock("line")
+      for child in node.items:
+        walkPassageText(child, state)
+      state.flushSegment()
+      state.indent = oldIndent
+    elif node.tag == "br":
+      state.startBlock("line")
+    else:
+      for child in node.items:
+        walkPassageText(child, state)
   else:
     discard
 
@@ -310,7 +349,7 @@ proc parseBook(epubPath: string, source: BookSource): seq[string] =
     let doc = parseHtml(newStringStream(html))
     walkPassageText(doc, state)
 
-  state.flushVerse()
+  state.flushSegment()
   state.rows
 
 proc generate(epubPath, outputPath: string) =