Update for Nim 2.x

* The Nim binary now recognizes the following environment and allows them to
  override configured values:

  - `DATABASE_DRIVER`: overrides the `driver` config value, selects which
    kind of database we expect to connect to.
  - `MIGRATIONS_DIR`: overrides the `sqlDir` config value, sets the path to
    the directory containing the migration SQL files.
  - `DATABASE_URL`: overrides the `connectionString` config value, supplies
    connection parameters to the database.
  - `DBM_LOG_LEVEL`: overrides the `logLevel` config value, sets the logging
    level of db_migrate.

* Fixes a bug in the parsing of migration files. Previously the file was split
  on `;` characters and chunks that started with `--` were ignored as comments.
  This was wrong in the common case where a block starts with a comment but
  then contains SQL further. Such a block was being ignored. The new behavior
  is to consider each line and build up queries that way.

* Fixes a logging bug when parsing the configuration. If there was an that
  prevented the configuration from loading this in turn prevented logging from
  being setup correctly, and so the error was not logged. Now errors that may
  occur before logging is setup are hard-coded to be logged to STDERR.

* Changes the logic for creating the `migrations` table to check before
  performing the query that creates the table. This is to avoid continually
  printing messages about skipping this creation when the table already exists
  (which is the normal case). This change is PostgreSQL-specific, but of course
  the entire tool is currently PostgreSQL-specific.

README.md

@@ -1,4 +1,36 @@
-DB Migrate
-==========
+# DB Migrate
 
 Small tool(s) to manage database migrations in various languages.
+
+## Usage
+
+```
+Usage:
+  db_migrate [options] create <migration-name>
+  db_migrate [options] up [<count>]
+  db_migrate [options] down [<count>]
+  db_migrate [options] init <schema-name>
+  db_migrate (-V | --version)
+  db_migrate (-h | --help)
+
+Options:
+  -c --config <config-file>   Use the given configuration file (defaults to
+                              "database.properties").
+  -q --quiet                  Suppress log information.
+  -v --verbose                Print detailed log information.
+     --very-verbose           Print very detailed log information.
+  -V --version                Print the tools version information.
+  -h --help                   Print this usage information.
+```
+
+
+## Database Config Format
+
+The database config is formatted as JSON. The following keys are supported by
+all of the implementations:
+
+* `sqlDir` -- Directory to store SQL files.
+
+The following keys are supported by the Nim implementation:
+
+* `connectionString` -- 

build.gradle

@@ -3,7 +3,7 @@ apply plugin: 'application'
 apply plugin: 'maven'
 
 group = 'com.jdblabs'
-version = '0.2.3'
+version = '0.2.5'
 
 mainClassName = 'com.jdblabs.dbmigrate.DbMigrate'
 

db_migrate.nimble

@@ -1,7 +1,7 @@
 # Package
 
 bin           = @["db_migrate"]
-version       = "0.2.3"
+version       = "0.3.2"
 author        = "Jonathan Bernard"
 description   = "Simple tool to handle database migrations."
 license       = "BSD"
@@ -9,5 +9,4 @@ srcDir        = "src/main/nim"
 
 # Dependencies
 
-requires: @["nim >= 0.13.0", "docopt >= 0.1.0"]
-
+requires: @["nim >= 2.0.0", "docopt >= 0.1.0", "db_connector"]

src/main/groovy/com/jdblabs/dbmigrate/DbMigrate.groovy

@@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory
 
 public class DbMigrate {
 
-  public static final VERSION = "0.2.3"
+  public static final VERSION = "0.2.5"
 
   public static final def DOC = """\
 db-migrate.groovy v${VERSION}
@@ -42,12 +42,12 @@ Options:
   private static Logger LOGGER = LoggerFactory.getLogger(DbMigrate)
 
   Sql sql
-  File migrationsDir
+  File sqlDir
 
   public static void main(String[] args) {
 
     // Parse arguments
-    def opts = new Docopt(DOC).withVersion("wdiwtlt v$VERSION").parse(args)
+    def opts = new Docopt(DOC).withVersion("db-migrate.groovy v$VERSION").parse(args)
 
     if (opts['--version']) {
       println "db-migrate.groovy v$VERSION"
@@ -90,14 +90,14 @@ Options:
         givenCfg.clear() } }
 
     // Check for migrations directory
-    File migrationsDir = new File(givenCfg["migrations.dir"] ?: 'migrations')
-    if (!migrationsDir.exists() || !migrationsDir.isDirectory()) {
+    File sqlDir = new File(givenCfg["sqlDir"] ?: 'migrations')
+    if (!sqlDir.exists() || !sqlDir.isDirectory()) {
       clilog.error("'{}' does not exist or is not a directory.",
-        migrationsDir.canonicalPath)
+        sqlDir.canonicalPath)
       System.exit(1) }
 
     // Instantiate the DbMigrate instance
-    DbMigrate dbmigrate = new DbMigrate(migrationsDir: migrationsDir)
+    DbMigrate dbmigrate = new DbMigrate(sqlDir: sqlDir)
 
     // If we've only been asked to create a new migration, we don't need to
     // setup the DB connection.
@@ -112,7 +112,7 @@ Options:
 
     // Create the datasource.
     Properties dsProps = new Properties()
-    dsProps.putAll(givenCfg.findAll { it.key != 'migrations.dir' })
+    dsProps.putAll(givenCfg.findAll { it.key != 'sqlDir' })
 
     HikariDataSource hds = new HikariDataSource(new HikariConfig(dsProps))
 
@@ -125,8 +125,8 @@ Options:
   public List<File> createMigration(String migrationName) {
     String timestamp = sdf.format(new Date())
 
-    File upFile = new File(migrationsDir, "$timestamp-$migrationName-up.sql")
-    File downFile = new File(migrationsDir, "$timestamp-$migrationName-down.sql")
+    File upFile = new File(sqlDir, "$timestamp-$migrationName-up.sql")
+    File downFile = new File(sqlDir, "$timestamp-$migrationName-down.sql")
 
     upFile.text = "-- UP script for $migrationName ($timestamp)"
     downFile.text = "-- DOWN script for $migrationName ($timestamp)"
@@ -140,7 +140,7 @@ Options:
 CREATE TABLE IF NOT EXISTS migrations (
   id SERIAL PRIMARY KEY,
   name VARCHAR NOT NULL,
-  run_at TIMESTAMP NOT NULL DEFAULT NOW())''') }
+  run_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW())''') }
 
   public def diffMigrations() {
     def results = [notRun: [], missing: []]
@@ -150,7 +150,7 @@ CREATE TABLE IF NOT EXISTS migrations (
       .collect { it.name }.sort()
 
     SortedSet<String> available = new TreeSet<>()
-    available.addAll(migrationsDir
+    available.addAll(sqlDir
       .listFiles({ d, n -> n ==~ /.+-(up|down).sql$/ } as FilenameFilter)
       .collect { f -> f.name.replaceAll(/-(up|down).sql$/, '') })
 
@@ -215,7 +215,7 @@ CREATE TABLE IF NOT EXISTS migrations (
 
       toRun.each { migrationName ->
         LOGGER.info(migrationName)
-        File migrationFile = new File(migrationsDir,
+        File migrationFile = new File(sqlDir,
           "$migrationName-${up ? 'up' : 'down'}.sql")
 
         if (!migrationFile.exists() || !migrationFile.isFile())

src/main/nim/db_migrate.nim

@@ -3,18 +3,31 @@
 ##
 ## Simple tool to manage database migrations.
 
-import algorithm, json, times, os, strutils, docopt, db_postgres, sets,
-  sequtils, logging
+import std/[algorithm, json, logging, os, sequtils, sets, strutils, tables,
+            times]
+import db_connector/db_postgres
+import docopt
 
 type
-  DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: Level ]
+  DbMigrateConfig* = object
+    driver, connectionString: string
+    sqlDirs: seq[string]
+    logLevel: Level
 
-proc createMigrationsTable(conn: DbConn): void =
-  conn.exec(sql("""
+  MigrationEntry* = tuple[name, upPath, downPath: string]
+
+proc ensureMigrationsTableExists(conn: DbConn): void =
+  let tableCount = conn.getValue(sql"""
+SELECT COUNT(*) FROM information_schema.tables
+WHERE table_name = 'migrations';""")
+
+  if tableCount.strip == "0":
+    info "Creating the migrations table as it does not already exist."
+    conn.exec(sql("""
 CREATE TABLE IF NOT EXISTS migrations (
   id SERIAL PRIMARY KEY,
   name VARCHAR NOT NULL,
-  run_at TIMESTAMP NOT NULL DEFAULT NOW())"""))
+  run_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW())"""))
 
 proc rollbackWithErr (pgConn: DbConn, errMsg: string): void =
   pgConn.exec(sql"ROLLBACK")
@@ -24,29 +37,43 @@ proc loadConfig*(filename: string): DbMigrateConfig =
   ## Load DbMigrateConfig from a file.
   let cfg = json.parseFile(filename)
 
-  var logLevel: Level
-  if cfg.hasKey("logLevel"):
+  var logLevel: Level = lvlInfo
+  if existsEnv("DBM_LOG_LEVEL"):
+    let idx = find(LevelNames, $getEnv("DBM_LOG_LEVEL").toUpper)
+    logLevel = if idx == -1: lvlInfo else: (Level)(idx)
+  elif cfg.hasKey("logLevel"):
     let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper)
     logLevel = if idx == -1: lvlInfo else: (Level)(idx)
 
-  return (
-    driver: if cfg.hasKey("driver"): cfg["driver"].getStr else: "postres",
-    sqlDir: if cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr else: "migrations",
-    connectionString: cfg["connectionString"].getStr,
+  return DbMigrateConfig(
+    driver:
+      if existsEnv("DATABASE_DRIVER"): $getEnv("DATABASE_DRIVER")
+      elif cfg.hasKey("driver"): cfg["driver"].getStr
+      else: "postres",
+    connectionString:
+      if existsEnv("DATABASE_URL"): $getEnv("DATABASE_URL")
+      elif cfg.hasKey("connectionString"): cfg["connectionString"].getStr
+      else: "",
+    sqlDirs:
+      if existsEnv("MIGRATIONS_DIRS"): getEnv("MIGRATIONS_DIRS").split(';')
+      elif cfg.hasKey("sqlDirs"): cfg["sqlDirs"].getElems.mapIt(it.getStr)
+      else: @["migrations"],
     logLevel: logLevel)
 
-proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[string] =
+proc createMigration*(config: DbMigrateConfig, migrationName: string): MigrationEntry =
   ## Create a new set of database migration files.
-  let timestamp = getTime().getLocalTime().format("yyyyMMddHHmmss")
+  let timestamp = now().format("yyyyMMddHHmmss")
   let filenamePrefix =  timestamp & "-" & migrationName
 
-  let upFilename = joinPath(config.sqlDir, filenamePrefix & "-up.sql")
-  let downFilename = joinPath(config.sqlDir, filenamePrefix & "-down.sql")
+  let migration = (
+    name: filenamePrefix & "-up.sql",
+    upPath: joinPath(config.sqlDirs[0], filenamePrefix & "-up.sql"),
+    downPath: joinPath(config.sqlDirs[0], filenamePrefix & "-down.sql"))
 
   let scriptDesc = migrationName & " (" & timestamp & ")"
 
-  let upFile = open(upFilename, fmWrite)
-  let downFile = open(downFilename, fmWrite)
+  let upFile = open(migration.upPath, fmWrite)
+  let downFile = open(migration.downPath, fmWrite)
 
   upFile.writeLine "-- UP script for " & scriptDesc
   downFile.writeLine "-- DOWN script for " & scriptDesc
@@ -54,74 +81,110 @@ proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[strin
   upFile.close()
   downFile.close()
 
-  return @[upFilename, downFilename]
+  return migration
 
-proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig):
-  tuple[ run, notRun, missing: seq[string] ] =
+proc diffMigrations*(
+    pgConn: DbConn,
+    config: DbMigrateConfig
+  ): tuple[
+    available: TableRef[string, MigrationEntry],
+    run: seq[string],
+    notRun, missing: seq[MigrationEntry] ] =
+
+  debug "diffMigrations: inspecting database and configured directories " &
+    "for migrations"
 
   # Query the database to find out what migrations have been run.
-  var migrationsRun = initSet[string]()
+  var migrationsRun = initHashSet[string]()
 
   for row in pgConn.fastRows(sql"SELECT * FROM migrations ORDER BY name", @[]):
     migrationsRun.incl(row[1])
 
   # Inspect the filesystem to see what migrations are available.
-  var migrationsAvailable = initSet[string]()
-  for filePath in walkFiles(joinPath(config.sqlDir, "*.sql")):
-    var migrationName = filePath.extractFilename
-    migrationName.removeSuffix("-up.sql")
-    migrationName.removeSuffix("-down.sql")
-    migrationsAvailable.incl(migrationName)
+  var migrationsAvailable = newTable[string, MigrationEntry]()
+  for sqlDir in config.sqlDirs:
+    debug "Looking in " & sqlDir
+    for filePath in walkFiles(joinPath(sqlDir, "*.sql")):
+      debug "Saw migration file: " & filePath
+      var migrationName = filePath.extractFilename
+      migrationName.removeSuffix("-up.sql")
+      migrationName.removeSuffix("-down.sql")
+      migrationsAvailable[migrationName] = (
+        name: migrationName,
+        upPath: joinPath(sqlDir, migrationName) & "-up.sql",
+        downPath: joinPath(sqlDir, migrationName) & "-down.sql")
 
   # Diff with the list of migrations that we have in our migrations
   # directory.
   let migrationsInOrder =
-    toSeq(migrationsAvailable.items).sorted(system.cmp)
+    toSeq(migrationsAvailable.keys).sorted(system.cmp)
 
-  var migrationsNotRun = newSeq[string]()
-  var missingMigrations = newSeq[string]()
+  var migrationsNotRun = newSeq[MigrationEntry]()
+  var missingMigrations = newSeq[MigrationEntry]()
 
-  for migration in migrationsInOrder:
-    if not migrationsRun.contains(migration):
-      migrationsNotRun.add(migration)
+  for migName in migrationsInOrder:
+    if not migrationsRun.contains(migName):
+      migrationsNotRun.add(migrationsAvailable[migName])
 
     # if we've already seen some migrations that have not been run, but this
     # one has been, that means we have a gap and are missing migrations
     elif migrationsNotRun.len > 0:
       missingMigrations.add(migrationsNotRun)
-      migrationsNotRun = newSeq[string]()
+      migrationsNotRun = newSeq[MigrationEntry]()
 
-  return (run: toSeq(migrationsRun.items).sorted(system.cmp),
+  result = (available: migrationsAvailable,
+          run: toSeq(migrationsRun.items).sorted(system.cmp),
           notRun: migrationsNotRun,
           missing: missingMigrations)
 
-proc readStatements*(filename: string): seq[SqlQuery] =
-  let migrationSql = filename.readFile
-  result = migrationSql.split(';').
-    filter(proc(st: string): bool = st.strip.len > 0 and not st.startsWith("--")).
-    map(proc(st: string): SqlQuery = sql(st & ";"))
+  debug "diffMigration: Results" &
+    "\n\tavailable: " & $toSeq(result[0].keys) &
+    "\n\trun:       " & $result[1] &
+    "\n\tnotRun:    " & $(result[2].mapIt(it.name)) &
+    "\n\tmissing:   " & $(result[3].mapIt(it.name))
 
-proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[string] =
-  var migrationsRun = newSeq[string]()
+
+proc readStatements*(filename: string): seq[SqlQuery] =
+  result = @[]
+  var stmt: string = ""
+
+  for line in filename.lines:
+    let l = line.strip
+    if l.len == 0 or l.startsWith("--"): continue
+
+    let parts = line.split(';')
+    stmt &= "\n" & parts[0]
+
+    if parts.len > 1:
+      result.add(sql(stmt & ";"))
+      stmt = parts[1] & "";
+
+  if stmt.strip.len > 0: result.add(sql(stmt))
+
+proc up*(
+    pgConn: DbConn,
+    config: DbMigrateConfig,
+    toRun: seq[MigrationEntry]): seq[MigrationEntry] =
+  var migrationsRun = newSeq[MigrationEntry]()
   # Begin a transaction.
   pgConn.exec(sql"BEGIN")
 
   # Apply each of the migrations.
   for migration in toRun:
-    info migration
-    let filename = joinPath(config.sqlDir, migration & "-up.sql")
+    info migration.name
 
-    if not filename.fileExists:
-      pgConn.rollbackWithErr "Can not find UP file for " & migration &
-        ". Expected '" & filename & "'."
+    if not migration.upPath.fileExists:
+      pgConn.rollbackWithErr "Can not find UP file for " & migration.name &
+        ". Expected '" & migration.upPath & "'."
 
-    let statements = filename.readStatements
+    let statements = migration.upPath.readStatements
 
     try:
-      for statement in statements: pgConn.exec(statement)
-      pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration)
+      for statement in statements:
+        pgConn.exec(statement)
+      pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration.name)
     except DbError:
-      pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
+      pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
         getCurrentExceptionMsg()
 
     migrationsRun.add(migration)
@@ -130,27 +193,28 @@ proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[strin
 
   return migrationsRun
 
-proc down*(pgConn: DbConn, config: DbMigrateConfig, migrationsToDown: seq[string]): seq[string] =
-  var migrationsDowned = newSeq[string]()
+proc down*(
+    pgConn: DbConn,
+    config: DbMigrateConfig,
+    migrationsToDown: seq[MigrationEntry]): seq[MigrationEntry] =
+  var migrationsDowned = newSeq[MigrationEntry]()
 
   pgConn.exec(sql"BEGIN")
 
   for migration in migrationsToDown:
-    info migration
+    info migration.name
 
-    let filename = joinPath(config.sqlDir, migration & "-down.sql")
+    if not migration.downPath.fileExists:
+      pgConn.rollbackWithErr "Can not find DOWN file for " & migration.name &
+        ". Expected '" & migration.downPath & "'."
 
-    if not filename.fileExists:
-      pgConn.rollbackWithErr "Can not find DOWN file for " & migration &
-        ". Expected '" & filename & "'."
-
-    let statements = filename.readStatements
+    let statements = migration.downPath.readStatements
 
     try:
       for statement in statements: pgConn.exec(statement)
-      pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration)
+      pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration.name)
     except DbError:
-      pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
+      pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
         getCurrentExceptionMsg()
 
     migrationsDowned.add(migration)
@@ -166,12 +230,15 @@ Usage:
   db_migrate [options] down [<count>]
   db_migrate [options] init <schema-name>
   db_migrate (-V | --version)
+  db_migrate (-h | --help)
 
 Options:
 
   -c --config <config-file>   Use the given configuration file (defaults to
                               "database.json").
 
+  -h --help                   Show this usage information.
+
   -q --quiet                  Suppress log information.
 
   -v --verbose                Print detailed log information.
@@ -182,7 +249,7 @@ Options:
 """
 
   # Parse arguments
-  let args = docopt(doc, version = "db-migrate 0.2.3")
+  let args = docopt(doc, version = "db-migrate (Nim) 0.3.2\nhttps://git.jdb-software.com/jdb/db-migrate")
 
   let exitErr = proc(msg: string): void =
     fatal("db_migrate: " & msg)
@@ -190,16 +257,17 @@ Options:
 
   # Load configuration file
   let configFilename =
-    if args["--config"]: $args["<config-file>"]
+    if args["--config"]: $args["--config"]
     else: "database.json"
 
   var config: DbMigrateConfig
   try:
     config = loadConfig(configFilename)
   except IOError:
-    exitErr "Cannot open config file: " & configFilename
+    writeLine(stderr, "db_migrate: Cannot open config file: " & configFilename)
   except:
-    exitErr "Error parsing config file: " & configFilename & "\L\t" & getCurrentExceptionMsg()
+    writeLine(stderr, "db_migrate: Error parsing config file: " &
+      configFilename & "\L\t" & getCurrentExceptionMsg())
 
 
   logging.addHandler(newConsoleLogger())
@@ -209,20 +277,22 @@ Options:
   else: logging.setLogFilter(config.logLevel)
 
   # Check for migrations directory
-  if not existsDir config.sqlDir:
-    try:
-      warn "SQL directory '" & config.sqlDir &
-        "' does not exist and will be created."
-      createDir config.sqlDir
-    except IOError:
-      exitErr "Unable to create directory: " & config.sqlDir & ":\L\T" & getCurrentExceptionMsg()
+  for sqlDir in config.sqlDirs:
+    if not dirExists sqlDir:
+      try:
+        warn "SQL directory '" & sqlDir &
+          "' does not exist and will be created."
+        createDir sqlDir
+      except IOError:
+        exitErr "Unable to create directory: " & sqlDir & ":\L\T" & getCurrentExceptionMsg()
 
   # Execute commands
   if args["create"]:
     try:
-      let filesCreated = createMigration(config, $args["<migration-name>"])
+      let newMigration = createMigration(config, $args["<migration-name>"])
       info "Created new migration files:"
-      for filename in filesCreated: info "\t" & filename
+      info "\t" & newMigration.upPath
+      info "\t" & newMigration.downPath
     except IOError:
       exitErr "Unable to create migration scripts: " & getCurrentExceptionMsg()
 
@@ -233,9 +303,9 @@ Options:
         exitErr "Unable to connect to the database: " & getCurrentExceptionMsg()
         (DbConn)(nil)
 
-    pgConn.createMigrationsTable
+    pgConn.ensureMigrationsTableExists
 
-    let (run, notRun, missing) = diffMigrations(pgConn, config)
+    let (available, run, notRun, missing) = diffMigrations(pgConn, config)
 
     # Make sure we have no gaps (database is in an unknown state)
     if missing.len > 0:
@@ -254,7 +324,8 @@ Options:
     elif args["down"]:
       try:
         let count = if args["<count>"]: parseInt($args["<count>"]) else: 1
-        let toRun = if count < run.len: run.reversed[0..<count] else: run.reversed
+        let toRunNames = if count < run.len: run.reversed[0..<count] else: run.reversed
+        let toRun = toRunNames.mapIt(available[it])
         let migrationsRun = pgConn.down(config, toRun)
         info "Went down " & $(migrationsRun.len) & "."
       except DbError: