lexer, common: More descriptive error messages.

The lexer now tracks the data that has been read since the start of the
current line. While this may have use in parsers, the immediate use is
by the common error reporting procedure.

The `common#error` procedure already reports the column and line number
where an error occurs. The `common#expect` function is broadly used by
parsers and generates the majority of parser errors. It now uses the
lexer's record of the current line to format its error message with a
direct pointer to the location of the unmet expectation.

src/vcard/private/common.nim

@@ -57,7 +57,7 @@ func serialize*(s: seq[VC_XParam]): string =
 # =============================================================================
 
 proc error*(p: VCardParser, msg: string) =
-  raise newException(VCardParsingError, "$1($2, $3) Error: $4] " %
+  raise newException(VCardParsingError, "$1($2, $3) Error: $4" %
     [ p.filename, $p.lineNumber, $p.getColNumber(p.pos), msg ])
 
 proc isNext*[T](p: var T, expected: string, caseSensitive = false): bool =
@@ -79,21 +79,23 @@ proc isNext*[T](p: var T, expected: string, caseSensitive = false): bool =
   p.returnToBookmark
 
 proc expect*[T](p: var T, expected: string, caseSensitive = false) =
-  p.setBookmark
+  try:
+    p.setBookmark
 
-  if caseSensitive:
-    for ch in expected:
-      if p.read != ch:
-        p.error("expected '$1' but found '$2'" %
-          [expected, p.readSinceBookmark])
+    if caseSensitive:
+      for ch in expected:
+        if p.read != ch: raise newException(ValueError, "")
+    else:
+      for rune in expected.runes:
+        if p.readRune.toLower != rune.toLower:
+          raise newException(ValueError, "")
 
-  else:
-    for rune in expected.runes:
-      if p.readRune.toLower != rune.toLower:
-        p.error("expected '$1' but found '$2'" %
-          [ expected, p.readSinceBookmark ])
+  except ValueError:
+    p.error("expected '$1' but found '$2':\n\t$3\n\t$4" %
+      [expected, p.readSinceBookmark, p.lineVal,
+       " ".repeat(p.getColNumber(p.pos) - 1) & "^\n"])
 
-  p.unsetBookmark
+  finally: p.unsetBookmark
 
 proc readGroup*[T](p: var T): Option[string] =
   ## All VCARD content items can be optionally prefixed with a group name. This

src/vcard/private/lexer.nim

@@ -11,6 +11,7 @@ type VCardLexer* = object of RootObj
   bookmarkVal*: seq[string] # value read since the bookmark was set
   lineNumber*: int          # how many newlines have we seen so far
   lineStart: int            # buffer index buffer for the start of the current line
+  lineVal*: string          # value read since the start of the current line
 
 proc skipUtf8Bom(vcl: var VCardLexer) =
   if (vcl.buffer[0] == '\xEF') and (vcl.buffer[1] == '\xBB') and (vcl.buffer[2] == '\xBF'):
@@ -137,16 +138,18 @@ proc read*(vcl: var VCardLexer, peek = false): char =
     vcl.pos += 3
     vcl.lineNumber += 1
     vcl.lineStart = vcl.pos
+    vcl.lineVal = newStringOfCap(84)
     if vcl.atEnd: vcl.fillBuffer()
 
   elif vcl.buffer[vcl.pos] == '\n':
     vcl.lineNumber += 1
     vcl.lineStart = wrappedIdx(vcl.pos + 1)
+    vcl.lineVal = newStringOfCap(84)
 
   result = vcl.buffer[vcl.pos]
   if not peek:
-    for idx in 0..<vcl.bookmarkVal.len:
-      vcl.bookmarkVal[idx].add(result)
+    for idx in 0..<vcl.bookmarkVal.len: vcl.bookmarkVal[idx].add(result)
+    vcl.lineVal.add(result)
     vcl.pos = wrappedIdx(vcl.pos + 1)
 
 proc readLen*(vcl: var VCardLexer, bytesToRead: int, peek = false): string =
@@ -160,14 +163,19 @@ proc readRune*(vcl: var VCardLexer, peek = false): Rune =
     vcl.pos += 3
     vcl.lineNumber += 1
     vcl.lineStart = vcl.pos
+    vcl.lineVal = newStringOfCap(84)
     if vcl.atEnd: vcl.fillBuffer()
 
   elif vcl.buffer[vcl.pos] == '\n':
     vcl.lineNumber += 1
     vcl.lineStart = wrappedIdx(vcl.pos + 1)
+    vcl.lineVal = newStringOfCap(84)
 
   result = vcl.buffer.runeAt(vcl.pos)
-  if not peek: vcl.pos += vcl.buffer.runeLenAt(vcl.pos)
+  if not peek:
+    for idx in 0..<vcl.bookmarkVal.len: vcl.bookmarkVal[idx].add(result)
+    vcl.lineVal.add(result)
+    vcl.pos += vcl.buffer.runeLenAt(vcl.pos)
 
 proc readRunesLen*(vcl: var VCardLexer, runesToRead: int, peek = false): string =
   result = newStringOfCap(runesToRead * 4)