Add ANSI output format

bibleref.nimble

@@ -1,6 +1,6 @@
 # Package
 
-version       = "1.1.1"
+version       = "1.2.0"
 author        = "Jonathan Bernard"
 description   = "Simple Nim CLI for retrieving Biblical passages"
 license       = "MIT"

src/bibleref.nim

@@ -98,6 +98,66 @@ proc fetchPassages(reference, translation: string, cfg: CombinedConfig): seq[str
       "unsupported translation '" & translation &
         "'; supported translations: " & supportedTranslationsList())
 
+proc fetchPlainPassages(
+    reference,
+    translation: string,
+    keepVerseNumbers: bool,
+    cfg: CombinedConfig): seq[string] =
+
+  case translation
+  of "akjv", "kjv":
+    result = kjv.fetchPlainPassages(
+      reference,
+      keepVerseNumbers,
+      cfg.getVal("annotations", "all"))
+  of "mev":
+    result = mev.fetchPlainPassages(
+      reference,
+      keepVerseNumbers,
+      cfg.getVal("annotations", "all"))
+  else:
+    for passage in fetchPassages(reference, translation, cfg):
+      result.add(formatPlain(passage, translation, keepVerseNumbers))
+
+proc fetchMarkdownPassages(
+    reference,
+    translation: string,
+    cfg: CombinedConfig): seq[string] =
+
+  case translation
+  of "akjv", "kjv":
+    result = kjv.fetchMarkdownPassages(
+      reference,
+      cfg.getVal("annotations", "all"))
+  of "mev":
+    result = mev.fetchMarkdownPassages(
+      reference,
+      cfg.getVal("annotations", "all"))
+  else:
+    for passage in fetchPassages(reference, translation, cfg):
+      result.add(formatMarkdown(passage, translation))
+
+proc fetchAnsiPassages(
+    reference,
+    translation: string,
+    keepVerseNumbers: bool,
+    cfg: CombinedConfig): seq[string] =
+
+  case translation
+  of "akjv", "kjv":
+    result = kjv.fetchAnsiPassages(
+      reference,
+      keepVerseNumbers,
+      cfg.getVal("annotations", "all"))
+  of "mev":
+    result = mev.fetchAnsiPassages(
+      reference,
+      keepVerseNumbers,
+      cfg.getVal("annotations", "all"))
+  else:
+    for passage in fetchPassages(reference, translation, cfg):
+      result.add(formatPlain(passage, translation, keepVerseNumbers))
+
 when isMainModule:
   const USAGE = """Usage:
   bibleref <reference> [options]
@@ -110,7 +170,13 @@ Options:
                                 command line for debugging purposes.
 
   -f, --output-format <format>  Select a specific output format. Valid values
-                                are 'raw', 'markdown', 'plain', 'reading'.
+                                are 'ansi', 'raw', 'markdown', 'plain',
+                                'reading'.
+
+  --annotations <annotations>   Select embedded Bible annotations to render.
+                                Valid values are 'none', 'all', or a
+                                comma-separated list of annotation kinds such
+                                as 'note,crossReference'. Defaults to 'all'.
 
   -t, --translation <translation>
                                 Select a specific translation. Supported values
@@ -138,7 +204,7 @@ Options:
   --api-bible-nkjv-bible-id <id>
                                 Override the API.Bible Bible ID for NKJV.
 """
-  const VERSION = "1.1.1"
+  const VERSION = "1.2.0"
 
   let consoleLogger = newConsoleLogger(
     levelThreshold=lvlInfo,
@@ -167,19 +233,36 @@ Options:
 
     var formattedPassages: seq[string] = @[]
     for query in queries:
-      for passage in fetchPassages(query.referenceText, query.translation, cfg):
+      case $args["--output-format"]
-        formattedPassages.add(
+      of "ansi":
-          case $args["--output-format"]:
+        formattedPassages.add(fetchAnsiPassages(
-          of "plain":
+          query.referenceText,
-            formatPlain(passage, query.translation)
+          query.translation,
-          of "reading":
+          keepVerseNumbers = true,
-            formatPlain(passage, query.translation, keepVerseNumbers = false)
+          cfg))
-          of "text":
+      of "plain":
-            passage.multiReplace([(re"\[(\d+)\]", "$1")])
+        formattedPassages.add(fetchPlainPassages(
-          of "raw":
+          query.referenceText,
-            passage
+          query.translation,
-          else:
+          keepVerseNumbers = true,
-            formatMarkdown(passage, query.translation))
+          cfg))
+      of "reading", "text", "raw":
+        for passage in fetchPassages(query.referenceText, query.translation, cfg):
+          formattedPassages.add(
+            case $args["--output-format"]:
+            of "reading":
+              formatPlain(passage, query.translation, keepVerseNumbers = false)
+            of "text":
+              passage.multiReplace([(re"\[(\d+)\]", "$1")])
+            of "raw":
+              passage
+            else:
+              formatMarkdown(passage, query.translation))
+      else:
+        formattedPassages.add(fetchMarkdownPassages(
+          query.referenceText,
+          query.translation,
+          cfg))
 
     echo formattedPassages.join("\p\p")
 

src/embedded_bible.nim

@@ -1,4 +1,4 @@
-import std/[strutils, tables]
+import std/[algorithm, json, strutils, tables, wordwrap]
 
 import ./reference_parser
 
@@ -16,6 +16,27 @@ type
     breakKind: SegmentBreak
     indent: int
     text: string
+    annotations: seq[BibleAnnotation]
+    wordsOfChrist: bool
+
+  BibleAnnotation = object
+    kind: string
+    offset: int
+    targetText: string
+
+  EmbeddedOutputFormat = enum
+    eofPlain
+    eofAnsi
+    eofMarkdown
+
+  AnnotationFilterKind = enum
+    afNone
+    afAll
+    afKinds
+
+  AnnotationFilter = object
+    kind: AnnotationFilterKind
+    kinds: seq[string]
 
   BibleIndex = object
     segments: seq[BibleSegment]
@@ -24,6 +45,12 @@ type
     lastChapterByBook: Table[string, int]
     translationName: string
 
+const
+  ansiReset = "\x1b[0m"
+  ansiDarkRed = "\x1b[31m"
+  ansiGray = "\x1b[90m"
+  ansiCyan = "\x1b[36m"
+
 proc verseKey(code: string, chapter, verse: int): string =
   code & "\t" & $chapter & "\t" & $verse
 
@@ -49,6 +76,39 @@ proc parseSegmentBreak(s: string): SegmentBreak =
   else:
     raise newException(ValueError, "invalid embedded Bible segment break: " & s)
 
+proc parseMetadata(metadata: string):
+    tuple[annotations: seq[BibleAnnotation], wordsOfChrist: bool] =
+
+  if metadata.strip.len == 0:
+    return
+
+  let node = parseJson(metadata)
+  if node.kind != JObject:
+    return
+
+  result.wordsOfChrist =
+    node.hasKey("wordsOfChrist") and node["wordsOfChrist"].getBool
+
+  if not node.hasKey("annotations"):
+    return
+
+  for item in node["annotations"].getElems:
+    let targetText =
+      if item.hasKey("targetText"): item["targetText"].getStr
+      else: ""
+
+    if targetText.strip.len == 0:
+      continue
+
+    result.annotations.add(BibleAnnotation(
+      kind:
+        if item.hasKey("kind"): item["kind"].getStr
+        else: "",
+      offset:
+        if item.hasKey("offset"): item["offset"].getInt
+        else: 0,
+      targetText: targetText))
+
 proc addSegment(index: var BibleIndex, segment: BibleSegment) =
   index.segments.add(segment)
   index.verses[verseKey(segment.code, segment.chapter, segment.verse)] = true
@@ -69,8 +129,8 @@ proc loadBibleIndex(rows, translationName: string): BibleIndex =
     if line.strip.len == 0:
       continue
 
-    let parts = line.split('\t', maxsplit = 6)
+    let parts = line.split('\t', maxsplit = 7)
-    if parts.len != 4 and parts.len != 7:
+    if parts.len != 4 and parts.len != 7 and parts.len != 8:
       raise newException(ValueError,
         "invalid embedded " & translationName & " row: " & line)
 
@@ -85,6 +145,10 @@ proc loadBibleIndex(rows, translationName: string): BibleIndex =
         indent: 0,
         text: legacy.text))
     else:
+      let metadata =
+        if parts.len == 8: parseMetadata(parts[7])
+        else: (annotations: newSeq[BibleAnnotation](), wordsOfChrist: false)
+
       result.addSegment(BibleSegment(
         code: parts[0],
         chapter: parseInt(parts[1]),
@@ -92,7 +156,9 @@ proc loadBibleIndex(rows, translationName: string): BibleIndex =
         segment: parseInt(parts[3]),
         breakKind: parseSegmentBreak(parts[4]),
         indent: parseInt(parts[5]),
-        text: parts[6]))
+        text: parts[6],
+        annotations: metadata.annotations,
+        wordsOfChrist: metadata.wordsOfChrist))
 
 proc requireLastChapter(index: BibleIndex, code: string): int =
   if not index.lastChapterByBook.hasKey(code):
@@ -142,6 +208,18 @@ proc selectedVerses(index: BibleIndex, reference: PassageReference, range: RefRa
       discard index.requireVerse(code, chapter, verse)
       result[verseKey(code, chapter, verse)] = true
 
+proc shouldSeparateSegments(previous, next: string): bool =
+  if previous.len == 0 or next.len == 0:
+    return false
+  if previous[^1].isSpaceAscii or next[0].isSpaceAscii:
+    return false
+  next[0] notin {'.', ',', ';', ':', '?', '!', ')', ']', '}'}
+
+proc appendSameLine(line: var string, content: string) =
+  if shouldSeparateSegments(line, content):
+    line.add(" ")
+  line.add(content)
+
 proc addFormattedSegment(
     lines: var seq[string],
     segment: BibleSegment,
@@ -159,17 +237,13 @@ proc addFormattedSegment(
   let hasPassageLine = lines.len > 1 and lines[^1].len > 0
 
   if segment.breakKind == sbSame and hasPassageLine:
-    if lines[^1].len > 0 and not lines[^1][^1].isSpaceAscii:
+    lines[^1].appendSameLine(content)
-      lines[^1].add(" ")
-    lines[^1].add(content)
   else:
     let isMarkerlessParagraphContinuation =
       segment.breakKind == sbParagraph and marker.len == 0 and hasPassageLine
 
     if isMarkerlessParagraphContinuation:
-      if lines[^1].len > 0 and not lines[^1][^1].isSpaceAscii:
+      lines[^1].appendSameLine(content)
-        lines[^1].add(" ")
-      lines[^1].add(content)
     else:
       if segment.breakKind == sbParagraph and hasPassageLine:
         lines.add("")
@@ -237,3 +311,379 @@ proc fetchPassages*(rows, reference, translationName: string): seq[string] =
   let index = loadBibleIndex(rows, translationName)
   for parsedReference in parseReferences(reference):
     result.add(fetchReference(index, parsedReference))
+
+proc normalizeAnnotationKind(kind: string): string =
+  case kind.strip
+  of "note":
+    "note"
+  of "crossReference", "crossreference", "cross-reference", "cross_reference":
+    "crossReference"
+  else:
+    raise newException(ValueError,
+      "unsupported annotation kind '" & kind &
+        "'; supported annotation kinds: note, crossReference")
+
+proc parseAnnotationFilter(value: string): AnnotationFilter =
+  let normalized = value.strip
+
+  case normalized
+  of "", "all":
+    return AnnotationFilter(kind: afAll)
+  of "none":
+    return AnnotationFilter(kind: afNone)
+  else:
+    result.kind = afKinds
+    for part in normalized.split(','):
+      let annotationKind = normalizeAnnotationKind(part)
+      if annotationKind notin result.kinds:
+        result.kinds.add(annotationKind)
+
+proc includes(filter: AnnotationFilter, annotation: BibleAnnotation): bool =
+  case filter.kind
+  of afNone:
+    false
+  of afAll:
+    true
+  of afKinds:
+    annotation.kind in filter.kinds
+
+proc footnoteLabel(index: int): string =
+  var n = index
+  while true:
+    result.insert($char(ord('a') + (n mod 26)), 0)
+    n = (n div 26) - 1
+    if n < 0:
+      break
+
+proc annotationMarker(
+    label: string,
+    outputFormat: EmbeddedOutputFormat,
+    resumeWordsOfChrist: bool): string =
+
+  case outputFormat
+  of eofPlain:
+    "[" & label & "]"
+  of eofAnsi:
+    ansiCyan & "[" & label & "]" & ansiReset &
+      (if resumeWordsOfChrist: ansiDarkRed else: "")
+  of eofMarkdown:
+    "[^" & label & "]"
+
+proc visibleLen(s: string): int =
+  var idx = 0
+  while idx < s.len:
+    if s[idx] == '\x1b' and idx + 1 < s.len and s[idx + 1] == '[':
+      idx += 2
+      while idx < s.len and not s[idx].isAlphaAscii:
+        inc idx
+      if idx < s.len:
+        inc idx
+    else:
+      inc result
+      inc idx
+
+proc wrapAnsiWords(line: string, maxLineWidth = 74): string =
+  let indentLen = line.len - line.strip(leading = true, trailing = false).len
+  let indent = repeat(" ", indentLen)
+  let words = line.strip.splitWhitespace
+  if words.len == 0:
+    return ""
+
+  var lines: seq[string]
+  var current = indent
+
+  for word in words:
+    let separator =
+      if current.strip.len == 0: ""
+      else: " "
+    if current.strip.len > 0 and
+        current.visibleLen + separator.len + word.visibleLen > maxLineWidth:
+      lines.add(current)
+      current = word
+    else:
+      current.add(separator & word)
+
+  if current.len > 0:
+    lines.add(current)
+
+  lines.join("\p")
+
+proc addSegmentWithFootnotes(
+    line: var string,
+    segment: BibleSegment,
+    keepVerseNumbers: bool,
+    notes: var seq[string],
+    outputFormat: EmbeddedOutputFormat,
+    annotationFilter: AnnotationFilter) =
+
+  var text = segment.text
+  var annotations = segment.annotations
+  annotations.sort(proc(a, b: BibleAnnotation): int = cmp(a.offset, b.offset))
+
+  var inserted = 0
+  for annotation in annotations:
+    if not annotationFilter.includes(annotation):
+      continue
+
+    let label = footnoteLabel(notes.len)
+    notes.add(annotation.targetText)
+
+    let insertAt = max(0, min(text.len, annotation.offset + inserted))
+    let marker = annotationMarker(
+      label,
+      outputFormat,
+      resumeWordsOfChrist = segment.wordsOfChrist)
+    text.insert(marker, insertAt)
+    inserted += marker.len
+
+  if outputFormat == eofAnsi and segment.wordsOfChrist:
+    text = ansiDarkRed & text & ansiReset
+
+  let marker =
+    if keepVerseNumbers:
+      case outputFormat
+      of eofPlain: $segment.verse & " "
+      of eofAnsi: ansiGray & $segment.verse & ansiReset & " "
+      of eofMarkdown: "**" & $segment.verse & "** "
+    else:
+      ""
+
+  let content = marker & text
+  if line.len == 0:
+    line.add(repeat("  ", segment.indent) & content)
+  elif segment.breakKind == sbSame:
+    line.appendSameLine(content)
+  else:
+    line.add("\n" & repeat("  ", segment.indent) & content)
+
+proc addFootnotes(
+    lines: var seq[string],
+    notes: seq[string],
+    outputFormat: EmbeddedOutputFormat) =
+  if notes.len == 0:
+    return
+
+  lines.add("")
+  for idx, note in notes:
+    let label = footnoteLabel(idx)
+    lines.add(
+      case outputFormat
+      of eofPlain: label & ". " & note
+      of eofAnsi: ansiCyan & label & ". " & note & ansiReset
+      of eofMarkdown: "[^" & label & "]: " & note)
+
+proc selectedChapters(index: BibleIndex, reference: PassageReference):
+    seq[tuple[chapter: int, verses: Table[string, bool]]] =
+
+  let code = reference.book.code
+
+  if reference.ranges.len == 0:
+    for chapter in 1 .. index.requireLastChapter(code):
+      var verses: Table[string, bool]
+      for verse in 1 .. index.requireLastVerse(code, chapter):
+        discard index.requireVerse(code, chapter, verse)
+        verses[verseKey(code, chapter, verse)] = true
+      result.add((chapter, verses))
+  else:
+    var chapterPositions: Table[int, int]
+    for range in reference.ranges:
+      let selected = index.selectedVerses(reference, range)
+      for key in selected.keys:
+        let parts = key.split('\t')
+        let chapter = parseInt(parts[1])
+        if not chapterPositions.hasKey(chapter):
+          chapterPositions[chapter] = result.len
+          var verses: Table[string, bool]
+          result.add((chapter, verses))
+        result[chapterPositions[chapter]].verses[key] = true
+
+proc formattedChapter(
+    index: BibleIndex,
+    code: string,
+    selected: Table[string, bool],
+    keepVerseNumbers: bool,
+    outputFormat: EmbeddedOutputFormat,
+    annotationFilter: AnnotationFilter): string =
+
+  var lines: seq[string]
+  var currentLine = ""
+  var notes: seq[string]
+  var emittedVerses: Table[string, bool]
+  var firstSelected = true
+
+  proc flushLine() =
+    if currentLine.len > 0:
+      lines.add(currentLine)
+      currentLine = ""
+
+  for segment in index.segments:
+    if segment.code != code:
+      continue
+
+    let key = verseKey(segment.code, segment.chapter, segment.verse)
+    if not selected.hasKey(key):
+      continue
+
+    var adjusted = segment
+    if firstSelected:
+      adjusted = normalizeFirstSelectedSegment(adjusted)
+
+    let startsNewParagraph =
+      adjusted.breakKind == sbParagraph and
+      not firstSelected and
+      (not emittedVerses.hasKey(key))
+
+    if startsNewParagraph:
+      flushLine()
+      lines.add("")
+    elif adjusted.breakKind == sbLine:
+      flushLine()
+
+    let marker =
+      if emittedVerses.hasKey(key): false
+      else:
+        emittedVerses[key] = true
+        true
+
+    var renderSegment = adjusted
+    if not marker:
+      renderSegment.verse = 0
+
+    if marker:
+      currentLine.addSegmentWithFootnotes(
+        renderSegment,
+        keepVerseNumbers,
+        notes,
+        outputFormat,
+        annotationFilter)
+    else:
+      var noVerseSegment = renderSegment
+      noVerseSegment.text = renderSegment.text
+      noVerseSegment.breakKind = sbSame
+      currentLine.addSegmentWithFootnotes(
+        noVerseSegment,
+        false,
+        notes,
+        outputFormat,
+        annotationFilter)
+
+    firstSelected = false
+
+  flushLine()
+  lines.addFootnotes(notes, outputFormat)
+
+  var wrapped: seq[string]
+  for line in lines:
+    if line.len == 0:
+      wrapped.add("")
+      continue
+
+    let prepared =
+      if line.len > 90: line.strip
+      else: line & "  "
+    if outputFormat == eofAnsi:
+      wrapped.add(wrapAnsiWords(prepared))
+    else:
+      wrapped.add(wrapWords(prepared, maxLineWidth = 74, newLine = "\p"))
+  wrapped.join("\p")
+
+proc fetchFormattedPassages(
+    rows,
+    reference,
+    translationName: string,
+    keepVerseNumbers: bool,
+    outputFormat: EmbeddedOutputFormat,
+    annotations: string): seq[string] =
+
+  let index = loadBibleIndex(rows, translationName)
+  let annotationFilter = parseAnnotationFilter(annotations)
+
+  for parsedReference in parseReferences(reference):
+    var parts: seq[string]
+    for chapter in index.selectedChapters(parsedReference):
+      parts.add(index.formattedChapter(
+        parsedReference.book.code,
+        chapter.verses,
+        keepVerseNumbers,
+        outputFormat,
+        annotationFilter))
+
+    let body = parts.join("\p\p")
+    let referenceSeparator =
+      case outputFormat
+      of eofPlain, eofAnsi:
+        if body.contains("\pa. "): "\p\p"
+        elif body.contains("\p" & ansiCyan & "a. "): "\p\p"
+        else: "\p"
+      of eofMarkdown:
+        if body.contains("\p[^a]: "): "\p\p"
+        else: "\p"
+
+    let referenceLine =
+      case outputFormat
+      of eofPlain, eofAnsi:
+        "– " & $parsedReference & " (" & translationName.toUpperAscii & ")"
+      of eofMarkdown:
+        "> -- *" & $parsedReference & " (" &
+          translationName.toUpperAscii & ")*"
+
+    let formattedBody =
+      case outputFormat
+      of eofPlain, eofAnsi:
+        body
+      of eofMarkdown:
+        var markdownLines: seq[string]
+        for line in body.splitLines:
+          if line.len == 0:
+            markdownLines.add("")
+          elif line.startsWith("[^"):
+            markdownLines.add(line)
+          else:
+            markdownLines.add("> " & line)
+        markdownLines.join("\p")
+
+    result.add(formattedBody & referenceSeparator & referenceLine)
+
+proc fetchPlainPassages*(
+    rows,
+    reference,
+    translationName: string,
+    keepVerseNumbers = true,
+    annotations = "all"): seq[string] =
+
+  fetchFormattedPassages(
+    rows,
+    reference,
+    translationName,
+    keepVerseNumbers,
+    eofPlain,
+    annotations)
+
+proc fetchAnsiPassages*(
+    rows,
+    reference,
+    translationName: string,
+    keepVerseNumbers = true,
+    annotations = "all"): seq[string] =
+
+  fetchFormattedPassages(
+    rows,
+    reference,
+    translationName,
+    keepVerseNumbers,
+    eofAnsi,
+    annotations)
+
+proc fetchMarkdownPassages*(
+    rows,
+    reference,
+    translationName: string,
+    annotations = "all"): seq[string] =
+
+  fetchFormattedPassages(
+    rows,
+    reference,
+    translationName,
+    keepVerseNumbers = true,
+    eofMarkdown,
+    annotations)

src/kjv.nim

@@ -5,3 +5,30 @@ const kjvRows = embeddedTranslationData("kjv")
 
 proc fetchPassages*(reference: string): seq[string] =
   embedded_bible.fetchPassages(kjvRows, reference, "KJV")
+
+proc fetchPlainPassages*(
+    reference: string,
+    keepVerseNumbers = true,
+    annotations = "all"): seq[string] =
+
+  embedded_bible.fetchPlainPassages(
+    kjvRows,
+    reference,
+    "KJV",
+    keepVerseNumbers,
+    annotations)
+
+proc fetchMarkdownPassages*(reference: string, annotations = "all"): seq[string] =
+  embedded_bible.fetchMarkdownPassages(kjvRows, reference, "KJV", annotations)
+
+proc fetchAnsiPassages*(
+    reference: string,
+    keepVerseNumbers = true,
+    annotations = "all"): seq[string] =
+
+  embedded_bible.fetchAnsiPassages(
+    kjvRows,
+    reference,
+    "KJV",
+    keepVerseNumbers,
+    annotations)

src/mev.nim

@@ -11,3 +11,42 @@ proc fetchPassages*(reference: string): seq[string] =
   else:
     raise newException(ValueError,
       "MEV data is not embedded; generate data/private/mev.tsv and rebuild")
+
+proc fetchPlainPassages*(
+    reference: string,
+    keepVerseNumbers = true,
+    annotations = "all"): seq[string] =
+
+  when hasEmbeddedTranslationData("mev"):
+    embedded_bible.fetchPlainPassages(
+      mevRows,
+      reference,
+      "MEV",
+      keepVerseNumbers,
+      annotations)
+  else:
+    raise newException(ValueError,
+      "MEV data is not embedded; generate data/private/mev.tsv and rebuild")
+
+proc fetchMarkdownPassages*(reference: string, annotations = "all"): seq[string] =
+  when hasEmbeddedTranslationData("mev"):
+    embedded_bible.fetchMarkdownPassages(mevRows, reference, "MEV", annotations)
+  else:
+    raise newException(ValueError,
+      "MEV data is not embedded; generate data/private/mev.tsv and rebuild")
+
+proc fetchAnsiPassages*(
+    reference: string,
+    keepVerseNumbers = true,
+    annotations = "all"): seq[string] =
+
+  when hasEmbeddedTranslationData("mev"):
+    embedded_bible.fetchAnsiPassages(
+      mevRows,
+      reference,
+      "MEV",
+      keepVerseNumbers,
+      annotations)
+  else:
+    raise newException(ValueError,
+      "MEV data is not embedded; generate data/private/mev.tsv and rebuild")

tests/test_offline_kjv.nim

@@ -93,6 +93,8 @@ JHN	8	33	0	paragraph	0	They answered Him.
 JHN	8	39	0	paragraph	0	They answered Him, "Abraham is our father."
 JHN	8	39	1	paragraph	0	Jesus said to them, "If you were Abraham's children,
 JHN	8	40	0	same	0	you would do the works of Abraham."
+JHN	8	41	0	paragraph	0	You shall be free'
+JHN	8	41	1	same	0	?
 HEB	4	7	0	paragraph	0	again He establishes a certain day:
 HEB	4	7	1	line	0	"Today, if you will hear His voice,
 HEB	4	7	2	line	1	do not harden your hearts."
@@ -137,12 +139,26 @@ HEB	4	7	2	line	1	do not harden your hearts."
       "Jesus said to them, \"If you were Abraham's children, " &
       "[40] you would do the works of Abraham.\""
 
+  test "joins punctuation-only segments without extra spacing":
+    let passages = embedded_bible.fetchPassages(layoutRows, "John 8:41", "TEST")
+
+    check passages[0] == "John 8:41\n  [41] You shall be free'?"
+
   test "supports legacy verse rows":
     const legacyRows = "JHN\t3\t16\tFor God so loved the world.\n"
     let passages = embedded_bible.fetchPassages(legacyRows, "John 3:16", "TEST")
 
     check passages[0] == "John 3:16\n  [16] For God so loved the world."
 
+  test "ignores layout metadata while rendering":
+    const metadataRows =
+      "JHN\t8\t31\t0\tparagraph\t0\tJesus said.\t" &
+      """{"wordsOfChrist":true,"annotations":[{"kind":"crossReference","marker":"a","href":"index_split_1.html#x","offset":6}]}""" &
+      "\n"
+    let passages = embedded_bible.fetchPassages(metadataRows, "John 8:31", "TEST")
+
+    check passages[0] == "John 8:31\n  [31] Jesus said."
+
   test "groups legacy rows into paragraphs using pilcrows":
     const legacyRows =
       "JHN\t8\t31\tThen Jesus said.\n" &
@@ -154,3 +170,158 @@ HEB	4	7	2	line	1	do not harden your hearts."
       "  [31] Then Jesus said. [32] You shall know the truth.\n" &
       "\n" &
       "  [33] They answered Him."
+
+  test "plain output renders annotation footnotes":
+    const rows =
+      "JHN\t8\t1\t0\tparagraph\t0\tBut Jesus went to the Mount of Olives.\t" &
+      """{"annotations":[{"kind":"crossReference","marker":"a","offset":40,"targetText":"Mt 21:1"}]}""" & "\n" &
+      "JHN\t8\t2\t0\tsame\t0\tEarly in the morning He returned to the temple.\t" &
+      """{"annotations":[{"kind":"crossReference","marker":"a","offset":49,"targetText":"Mt 26:55; Lk 4:20"}]}""" & "\n"
+    let passages = embedded_bible.fetchPlainPassages(rows, "John 8:1-2", "TEST")
+
+    check passages[0] == "1 But Jesus went to the Mount of Olives.[a] 2 Early in the morning He\n" &
+      "returned to the temple.[b]\n" &
+      "\n" &
+      "a. Mt 21:1\n" &
+      "b. Mt 26:55; Lk 4:20\n" &
+      "\n" &
+      "– John 8:1-2 (TEST)"
+
+  test "plain output can suppress annotation footnotes":
+    const rows =
+      "JHN\t8\t1\t0\tparagraph\t0\tJesus spoke.\t" &
+      """{"annotations":[{"kind":"crossReference","offset":12,"targetText":"Mt 21:1"}]}""" & "\n"
+    let passages = embedded_bible.fetchPlainPassages(
+      rows,
+      "John 8:1",
+      "TEST",
+      annotations = "none")
+
+    check passages[0] == "1 Jesus spoke.\n" &
+      "– John 8:1 (TEST)"
+
+  test "plain output filters annotation footnotes by kind":
+    const rows =
+      "JHN\t8\t1\t0\tparagraph\t0\tJesus spoke plainly.\t" &
+      """{"annotations":[{"kind":"note","offset":20,"targetText":"A note."}]}""" & "\n" &
+      "JHN\t8\t2\t0\tsame\t0\tHe taught.\t" &
+      """{"annotations":[{"kind":"crossReference","offset":10,"targetText":"Mt 26:55"}]}""" & "\n"
+
+    let notesOnly = embedded_bible.fetchPlainPassages(
+      rows,
+      "John 8:1-2",
+      "TEST",
+      annotations = "note")
+    let crossReferencesOnly = embedded_bible.fetchPlainPassages(
+      rows,
+      "John 8:1-2",
+      "TEST",
+      annotations = "crossReference")
+    let both = embedded_bible.fetchPlainPassages(
+      rows,
+      "John 8:1-2",
+      "TEST",
+      annotations = "note,crossReference")
+
+    check notesOnly[0] == "1 Jesus spoke plainly.[a] 2 He taught.\n" &
+      "\n" &
+      "a. A note.\n" &
+      "\n" &
+      "– John 8:1-2 (TEST)"
+    check crossReferencesOnly[0] == "1 Jesus spoke plainly. 2 He taught.[a]\n" &
+      "\n" &
+      "a. Mt 26:55\n" &
+      "\n" &
+      "– John 8:1-2 (TEST)"
+    check both[0] == "1 Jesus spoke plainly.[a] 2 He taught.[b]\n" &
+      "\n" &
+      "a. A note.\n" &
+      "b. Mt 26:55\n" &
+      "\n" &
+      "– John 8:1-2 (TEST)"
+
+  test "rejects unknown annotation filter kinds":
+    const rows =
+      "JHN\t8\t1\t0\tparagraph\t0\tJesus spoke.\t" &
+      """{"annotations":[{"kind":"note","offset":12,"targetText":"A note."}]}""" & "\n"
+
+    expect ValueError:
+      discard embedded_bible.fetchPlainPassages(
+        rows,
+        "John 8:1",
+        "TEST",
+        annotations = "gloss")
+
+  test "plain output resets footnote labels per chapter":
+    const rows =
+      "JHN\t8\t59\t0\tparagraph\t0\tThey took up stones.\t" &
+      """{"annotations":[{"kind":"crossReference","marker":"a","offset":21,"targetText":"Lev 24:16"}]}""" & "\n" &
+      "JHN\t9\t1\t0\tparagraph\t0\tAs Jesus passed by.\t" &
+      """{"annotations":[{"kind":"crossReference","marker":"a","offset":20,"targetText":"Jn 8:12"}]}""" & "\n"
+    let passages = embedded_bible.fetchPlainPassages(rows, "John 8:59-9:1", "TEST")
+
+    check passages[0] == "59 They took up stones.[a]\n" &
+      "\n" &
+      "a. Lev 24:16\n" &
+      "\n" &
+      "1 As Jesus passed by.[a]\n" &
+      "\n" &
+      "a. Jn 8:12\n" &
+      "\n" &
+      "– John 8:59-9:1 (TEST)"
+
+  test "markdown output renders native footnotes":
+    const rows =
+      "JHN\t8\t1\t0\tparagraph\t0\tBut Jesus went to the Mount of Olives.\t" &
+      """{"annotations":[{"kind":"crossReference","marker":"a","offset":40,"targetText":"Mt 21:1"}]}""" & "\n" &
+      "JHN\t8\t2\t0\tsame\t0\tEarly in the morning He returned to the temple.\t" &
+      """{"annotations":[{"kind":"crossReference","marker":"a","offset":49,"targetText":"Mt 26:55; Lk 4:20"}]}""" & "\n"
+    let passages = embedded_bible.fetchMarkdownPassages(rows, "John 8:1-2", "TEST")
+
+    check passages[0] == "> **1** But Jesus went to the Mount of Olives.[^a] **2** Early in the\n" &
+      "> morning He returned to the temple.[^b]\n" &
+      "\n" &
+      "[^a]: Mt 21:1\n" &
+      "[^b]: Mt 26:55; Lk 4:20\n" &
+      "\n" &
+      "> -- *John 8:1-2 (TEST)*"
+
+  test "markdown output resets native footnotes per chapter":
+    const rows =
+      "JHN\t8\t59\t0\tparagraph\t0\tThey took up stones.\t" &
+      """{"annotations":[{"kind":"crossReference","marker":"a","offset":21,"targetText":"Lev 24:16"}]}""" & "\n" &
+      "JHN\t9\t1\t0\tparagraph\t0\tAs Jesus passed by.\t" &
+      """{"annotations":[{"kind":"crossReference","marker":"a","offset":20,"targetText":"Jn 8:12"}]}""" & "\n"
+    let passages = embedded_bible.fetchMarkdownPassages(rows, "John 8:59-9:1", "TEST")
+
+    check passages[0] == "> **59** They took up stones.[^a]\n" &
+      "\n" &
+      "[^a]: Lev 24:16\n" &
+      "\n" &
+      "> **1** As Jesus passed by.[^a]\n" &
+      "\n" &
+      "[^a]: Jn 8:12\n" &
+      "\n" &
+      "> -- *John 8:59-9:1 (TEST)*"
+
+  test "ansi output renders verse numbers in gray and words of Christ in dark red":
+    const rows =
+      "JHN\t8\t1\t0\tparagraph\t0\tJesus said.\t" &
+      """{"wordsOfChrist":true}""" & "\n"
+    let passages = embedded_bible.fetchAnsiPassages(rows, "John 8:1", "TEST")
+
+    check passages[0] == "\x1b[90m1\x1b[0m \x1b[31mJesus said.\x1b[0m\n" &
+      "– John 8:1 (TEST)"
+
+  test "ansi output renders footnotes in cyan":
+    const rows =
+      "JHN\t8\t1\t0\tparagraph\t0\tJesus said.\t" &
+      """{"wordsOfChrist":true,"annotations":[{"kind":"crossReference","offset":11,"targetText":"Mt 21:1"}]}""" &
+      "\n"
+    let passages = embedded_bible.fetchAnsiPassages(rows, "John 8:1", "TEST")
+
+    check passages[0] == "\x1b[90m1\x1b[0m \x1b[31mJesus said.\x1b[36m[a]\x1b[0m\x1b[31m\x1b[0m\n" &
+      "\n" &
+      "\x1b[36ma. Mt 21:1\x1b[0m\n" &
+      "\n" &
+      "– John 8:1 (TEST)"

tools/generate_mev_data.nim

@@ -1,9 +1,11 @@
 import std/[
   htmlparser,
+  json,
   os,
   osproc,
   streams,
   strutils,
+  tables,
   xmlparser,
   xmltree
 ]
@@ -29,6 +31,11 @@ type
     segmentText: string
     segmentBreak: string
     indent: int
+    wordsOfChrist: bool
+    annotations: seq[JsonNode]
+    epubPath: string
+    targetTextCache: Table[string, string]
+    targetFileCache: Table[string, Table[string, string]]
     rows: seq[string]
 
 proc normalizeWhitespace(s: string): string =
@@ -197,6 +204,101 @@ proc hasHref(node: XmlNode): bool =
       if hasHref(child):
         return true
 
+proc firstHref(node: XmlNode): string =
+  if node.kind == xnElement:
+    let href = node.attr("href")
+    if href.len > 0:
+      return href
+
+    for child in node.items:
+      let href = firstHref(child)
+      if href.len > 0:
+        return href
+
+proc collectIds(node: XmlNode, ids: var seq[string]) =
+  if node.kind == xnElement:
+    let id = node.attr("id")
+    if id.len > 0:
+      ids.add(id)
+    for child in node.items:
+      collectIds(child, ids)
+
+proc targetParagraphs(doc: XmlNode): Table[string, string] =
+  var paragraphs = initTable[string, string]()
+
+  proc walk(node: XmlNode) =
+    if node.kind == xnElement:
+      if node.tag == "p":
+        var ids: seq[string] = @[]
+        collectIds(node, ids)
+        if ids.len > 0:
+          let text = normalizeWhitespace(textContent(node))
+          for id in ids:
+            paragraphs[id] = text
+
+      for child in node.items:
+        walk(child)
+
+  walk(doc)
+  paragraphs
+
+proc linkTargetText(state: var ParseState, href: string): string =
+  if href.len == 0:
+    return ""
+
+  if state.targetTextCache.hasKey(href):
+    return state.targetTextCache[href]
+
+  let parts = href.split('#', maxsplit = 1)
+  if parts.len != 2 or parts[0].len == 0 or parts[1].len == 0:
+    state.targetTextCache[href] = ""
+    return ""
+
+  if not state.targetFileCache.hasKey(parts[0]):
+    try:
+      let html = readEpubEntry(state.epubPath, parts[0])
+      let doc = parseHtml(newStringStream(html))
+      state.targetFileCache[parts[0]] = targetParagraphs(doc)
+    except CatchableError:
+      state.targetFileCache[parts[0]] = initTable[string, string]()
+
+  if state.targetFileCache[parts[0]].hasKey(parts[1]):
+    result = state.targetFileCache[parts[0]][parts[1]]
+  else:
+    result = ""
+
+  state.targetTextCache[href] = result
+
+proc annotationKind(marker: string): string =
+  if marker.strip.allCharsInSet({'0'..'9'}): "note"
+  else: "crossReference"
+
+proc cleanTargetText(targetText: string): string =
+  var parts = targetText.splitWhitespace
+  if parts.len >= 3 and parts[0].allCharsInSet({'0'..'9'}):
+    parts.delete(0)
+    parts.delete(0)
+    return parts.join(" ")
+
+  targetText
+
+proc annotationNode(marker, href, targetText: string, offset: int): JsonNode =
+  let kind = annotationKind(marker)
+  let cleanedTargetText = cleanTargetText(targetText)
+  result = %*{
+    "kind": kind,
+    "offset": offset
+  }
+  if cleanedTargetText.len > 0:
+    result["targetText"] = newJString(cleanedTargetText)
+
+proc segmentMetadata(state: ParseState): JsonNode =
+  result = newJObject()
+  if state.wordsOfChrist:
+    result["wordsOfChrist"] = newJBool(true)
+  if state.annotations.len > 0:
+    result["annotations"] = %state.annotations
+
 proc isParagraphElement(node: XmlNode): bool =
   node.kind == xnElement and node.tag == "p" and
     (node.hasClass("calibre_18") or
@@ -253,6 +355,7 @@ proc flushSegment(state: var ParseState) =
   if state.chapter > 0 and state.verse > 0:
     let text = normalizeWhitespace(state.segmentText).replace("\t", " ")
     if text.len > 0:
+      let metadata = segmentMetadata(state)
       state.rows.add([
         state.code,
         $state.chapter,
@@ -260,12 +363,15 @@ proc flushSegment(state: var ParseState) =
         $state.segment,
         state.segmentBreak,
         $state.indent,
-        text
+        text,
+        $metadata
       ].join("\t"))
       inc state.segment
       state.segmentBreak = "same"
 
   state.segmentText = ""
+  state.wordsOfChrist = false
+  state.annotations = @[]
 
 proc startBlock(state: var ParseState, segmentBreak: string) =
   state.flushSegment()
@@ -309,9 +415,30 @@ proc walkPassageText(node: XmlNode, state: var ParseState) =
       state.segment = 0
       return
 
+    if node.tag == "sup" and node.hasHref:
+      let marker = markerText(textContent(node))
+      let href = firstHref(node)
+      if marker.len > 0 and href.len > 0:
+        state.annotations.add(annotationNode(
+          marker,
+          href,
+          state.linkTargetText(href),
+          normalizeWhitespace(state.segmentText).len))
+      return
+
     if node.tag == "sup":
       return
 
+    if node.hasClass("calibre_40"):
+      state.flushSegment()
+      let oldWordsOfChrist = state.wordsOfChrist
+      state.wordsOfChrist = true
+      for child in node.items:
+        walkPassageText(child, state)
+      state.flushSegment()
+      state.wordsOfChrist = oldWordsOfChrist
+      return
+
     if node.tag == "p":
       let segmentBreak =
         if node.isParagraphElement: "paragraph"
@@ -340,7 +467,7 @@ proc indexSplitFile(index: int): string =
   "index_split_" & align($index, 3, '0') & ".html"
 
 proc parseBook(epubPath: string, source: BookSource): seq[string] =
-  var state = ParseState(code: source.code)
+  var state = ParseState(code: source.code, epubPath: epubPath)
   if bookInfo(source.code).singleChapter:
     state.chapter = 1