Add env configuration, bug in SQL parsing for Nim db_migrate.

* The Nim binary now recognizes the following environment and allows them to
  override configured values:

  - `DATABASE_DRIVER`: overrides the `driver` config value, selects which
    kind of database we expect to connect to.
  - `MIGRATIONS_DIR`: overrides the `sqlDir` config value, sets the path to
    the directory containing the migration SQL files.
  - `DATABASE_URL`: overrides the `connectionString` config value, supplies
    connection parameters to the database.
  - `DBM_LOG_LEVEL`: overrides the `logLevel` config value, sets the logging
    level of db_migrate.

* Fixes a bug in the parsing of migration files. Previously the file was split
  on `;` characters and chunks that started with `--` were ignored as comments.
  This was wrong in the common case where a block starts with a comment but
  then contains SQL further. Such a block was being ignored. The new behavior
  is to consider each line and build up queries that way.

* Fixes a logging bug when parsing the configuration. If there was an that
  prevented the configuration from loading this in turn prevented logging from
  being setup correctly, and so the error was not logged. Now errors that may
  occur before logging is setup are hard-coded to be logged to STDERR.

* Changes the logic for creating the `migrations` table to check before
  performing the query that creates the table. This is to avoid continually
  printing messages about skipping this creation when the table already exists
  (which is the normal case). This change is PostgreSQL-specific, but of course
  the entire tool is currently PostgreSQL-specific.

build.gradle

@@ -3,7 +3,7 @@ apply plugin: 'application'
 apply plugin: 'maven'
 
 group = 'com.jdblabs'
-version = '0.2.4'
+version = '0.2.5'
 
 mainClassName = 'com.jdblabs.dbmigrate.DbMigrate'
 

db_migrate.nimble

@@ -1,7 +1,7 @@
 # Package
 
 bin           = @["db_migrate"]
-version       = "0.2.4"
+version       = "0.2.5"
 author        = "Jonathan Bernard"
 description   = "Simple tool to handle database migrations."
 license       = "BSD"

src/main/groovy/com/jdblabs/dbmigrate/DbMigrate.groovy

@@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory
 
 public class DbMigrate {
 
-  public static final VERSION = "0.2.4"
+  public static final VERSION = "0.2.5"
 
   public static final def DOC = """\
 db-migrate.groovy v${VERSION}

src/main/nim/db_migrate.nim

@@ -9,8 +9,14 @@ import algorithm, json, times, os, strutils, docopt, db_postgres, sets,
 type
   DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: Level ]
 
-proc createMigrationsTable(conn: DbConn): void =
-  conn.exec(sql("""
+proc ensureMigrationsTableExists(conn: DbConn): void =
+  let tableCount = conn.getValue(sql"""
+SELECT COUNT(*) FROM information_schema.tables 
+WHERE table_name = 'migrations';""")
+
+  if tableCount.strip == "0":
+    info "Creating the migrations table as it does not already exist."
+    conn.exec(sql("""
 CREATE TABLE IF NOT EXISTS migrations (
   id SERIAL PRIMARY KEY,
   name VARCHAR NOT NULL,
@@ -24,15 +30,27 @@ proc loadConfig*(filename: string): DbMigrateConfig =
   ## Load DbMigrateConfig from a file.
   let cfg = json.parseFile(filename)
 
-  var logLevel: Level
-  if cfg.hasKey("logLevel"):
+  var logLevel: Level = lvlInfo
+  if existsEnv("DBM_LOG_LEVEL"):
+    let idx = find(LevelNames, $getEnv("DBM_LOG_LEVEL").toUpper)
+    logLevel = if idx == -1: lvlInfo else: (Level)(idx)
+  elif cfg.hasKey("logLevel"):
     let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper)
     logLevel = if idx == -1: lvlInfo else: (Level)(idx)
 
   return (
-    driver: if cfg.hasKey("driver"): cfg["driver"].getStr else: "postres",
-    sqlDir: if cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr else: "migrations",
-    connectionString: cfg["connectionString"].getStr,
+    driver:
+      if existsEnv("DATABASE_DRIVER"): $getEnv("DATABASE_DRIVER")
+      elif cfg.hasKey("driver"): cfg["driver"].getStr
+      else: "postres",
+    sqlDir:
+      if existsEnv("MIGRATIONS_DIR"): $getEnv("MIGRATIONS_DIR")
+      elif cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr
+      else: "migrations",
+    connectionString:
+      if existsEnv("DATABASE_URL"): $getEnv("DATABASE_URL")
+      elif cfg.hasKey("connectionString"): cfg["connectionString"].getStr
+      else: "",
     logLevel: logLevel)
 
 proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[string] =
@@ -96,10 +114,21 @@ proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig):
           missing: missingMigrations)
 
 proc readStatements*(filename: string): seq[SqlQuery] =
-  let migrationSql = filename.readFile
-  result = migrationSql.split(';').
-    filter(proc(st: string): bool = st.strip.len > 0 and not st.startsWith("--")).
-    map(proc(st: string): SqlQuery = sql(st & ";"))
+  result = @[]
+  var stmt: string = ""
+
+  for line in filename.lines:
+    let l = line.strip
+    if l.len == 0 or l.startsWith("--"): continue
+
+    let parts = line.split(';')
+    stmt &= "\n" & parts[0]
+
+    if parts.len > 1:
+      result.add(sql(stmt & ";"))
+      stmt = parts[1] & "";
+
+  if stmt.strip.len > 0: result.add(sql(stmt))
 
 proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[string] =
   var migrationsRun = newSeq[string]()
@@ -118,7 +147,8 @@ proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[strin
     let statements = filename.readStatements
 
     try:
-      for statement in statements: pgConn.exec(statement)
+      for statement in statements:
+        pgConn.exec(statement)
       pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration)
     except DbError:
       pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
@@ -182,7 +212,7 @@ Options:
 """
 
   # Parse arguments
-  let args = docopt(doc, version = "db-migrate 0.2.4")
+  let args = docopt(doc, version = "db-migrate 0.2.5")
 
   let exitErr = proc(msg: string): void =
     fatal("db_migrate: " & msg)
@@ -197,9 +227,10 @@ Options:
   try:
     config = loadConfig(configFilename)
   except IOError:
-    exitErr "Cannot open config file: " & configFilename
+    writeLine(stderr, "db_migrate: Cannot open config file: " & configFilename)
   except:
-    exitErr "Error parsing config file: " & configFilename & "\L\t" & getCurrentExceptionMsg()
+    writeLine(stderr, "db_migrate: Error parsing config file: " &
+      configFilename & "\L\t" & getCurrentExceptionMsg())
 
 
   logging.addHandler(newConsoleLogger())
@@ -233,7 +264,7 @@ Options:
         exitErr "Unable to connect to the database: " & getCurrentExceptionMsg()
         (DbConn)(nil)
 
-    pgConn.createMigrationsTable
+    pgConn.ensureMigrationsTableExists
 
     let (run, notRun, missing) = diffMigrations(pgConn, config)