Update for Nim 2.x

README.md

@@ -1,4 +1,36 @@
-DB Migrate
-==========
+# DB Migrate
 
 Small tool(s) to manage database migrations in various languages.
+
+## Usage
+
+```
+Usage:
+  db_migrate [options] create <migration-name>
+  db_migrate [options] up [<count>]
+  db_migrate [options] down [<count>]
+  db_migrate [options] init <schema-name>
+  db_migrate (-V | --version)
+  db_migrate (-h | --help)
+
+Options:
+  -c --config <config-file>   Use the given configuration file (defaults to
+                              "database.properties").
+  -q --quiet                  Suppress log information.
+  -v --verbose                Print detailed log information.
+     --very-verbose           Print very detailed log information.
+  -V --version                Print the tools version information.
+  -h --help                   Print this usage information.
+```
+
+
+## Database Config Format
+
+The database config is formatted as JSON. The following keys are supported by
+all of the implementations:
+
+* `sqlDir` -- Directory to store SQL files.
+
+The following keys are supported by the Nim implementation:
+
+* `connectionString` -- 

db_migrate.nimble

@@ -1,7 +1,7 @@
 # Package
 
 bin           = @["db_migrate"]
-version       = "0.2.6"
+version       = "0.3.2"
 author        = "Jonathan Bernard"
 description   = "Simple tool to handle database migrations."
 license       = "BSD"
@@ -9,5 +9,4 @@ srcDir        = "src/main/nim"
 
 # Dependencies
 
-requires: @["nim >= 0.13.0", "docopt >= 0.1.0"]
-
+requires: @["nim >= 2.0.0", "docopt >= 0.1.0", "db_connector"]

src/main/groovy/com/jdblabs/dbmigrate/DbMigrate.groovy

@@ -42,7 +42,7 @@ Options:
   private static Logger LOGGER = LoggerFactory.getLogger(DbMigrate)
 
   Sql sql
-  File migrationsDir
+  File sqlDir
 
   public static void main(String[] args) {
 
@@ -90,14 +90,14 @@ Options:
         givenCfg.clear() } }
 
     // Check for migrations directory
-    File migrationsDir = new File(givenCfg["migrations.dir"] ?: 'migrations')
-    if (!migrationsDir.exists() || !migrationsDir.isDirectory()) {
+    File sqlDir = new File(givenCfg["sqlDir"] ?: 'migrations')
+    if (!sqlDir.exists() || !sqlDir.isDirectory()) {
       clilog.error("'{}' does not exist or is not a directory.",
-        migrationsDir.canonicalPath)
+        sqlDir.canonicalPath)
       System.exit(1) }
 
     // Instantiate the DbMigrate instance
-    DbMigrate dbmigrate = new DbMigrate(migrationsDir: migrationsDir)
+    DbMigrate dbmigrate = new DbMigrate(sqlDir: sqlDir)
 
     // If we've only been asked to create a new migration, we don't need to
     // setup the DB connection.
@@ -112,7 +112,7 @@ Options:
 
     // Create the datasource.
     Properties dsProps = new Properties()
-    dsProps.putAll(givenCfg.findAll { it.key != 'migrations.dir' })
+    dsProps.putAll(givenCfg.findAll { it.key != 'sqlDir' })
 
     HikariDataSource hds = new HikariDataSource(new HikariConfig(dsProps))
 
@@ -125,8 +125,8 @@ Options:
   public List<File> createMigration(String migrationName) {
     String timestamp = sdf.format(new Date())
 
-    File upFile = new File(migrationsDir, "$timestamp-$migrationName-up.sql")
-    File downFile = new File(migrationsDir, "$timestamp-$migrationName-down.sql")
+    File upFile = new File(sqlDir, "$timestamp-$migrationName-up.sql")
+    File downFile = new File(sqlDir, "$timestamp-$migrationName-down.sql")
 
     upFile.text = "-- UP script for $migrationName ($timestamp)"
     downFile.text = "-- DOWN script for $migrationName ($timestamp)"
@@ -140,7 +140,7 @@ Options:
 CREATE TABLE IF NOT EXISTS migrations (
   id SERIAL PRIMARY KEY,
   name VARCHAR NOT NULL,
-  run_at TIMESTAMP NOT NULL DEFAULT NOW())''') }
+  run_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW())''') }
 
   public def diffMigrations() {
     def results = [notRun: [], missing: []]
@@ -150,7 +150,7 @@ CREATE TABLE IF NOT EXISTS migrations (
       .collect { it.name }.sort()
 
     SortedSet<String> available = new TreeSet<>()
-    available.addAll(migrationsDir
+    available.addAll(sqlDir
       .listFiles({ d, n -> n ==~ /.+-(up|down).sql$/ } as FilenameFilter)
       .collect { f -> f.name.replaceAll(/-(up|down).sql$/, '') })
 
@@ -215,7 +215,7 @@ CREATE TABLE IF NOT EXISTS migrations (
 
       toRun.each { migrationName ->
         LOGGER.info(migrationName)
-        File migrationFile = new File(migrationsDir,
+        File migrationFile = new File(sqlDir,
           "$migrationName-${up ? 'up' : 'down'}.sql")
 
         if (!migrationFile.exists() || !migrationFile.isFile())

src/main/nim/db_migrate.nim

@@ -3,11 +3,18 @@
 ##
 ## Simple tool to manage database migrations.
 
-import algorithm, json, times, os, strutils, docopt, db_postgres, sets,
-  sequtils, logging
+import std/[algorithm, json, logging, os, sequtils, sets, strutils, tables,
+            times]
+import db_connector/db_postgres
+import docopt
 
 type
-  DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: Level ]
+  DbMigrateConfig* = object
+    driver, connectionString: string
+    sqlDirs: seq[string]
+    logLevel: Level
+
+  MigrationEntry* = tuple[name, upPath, downPath: string]
 
 proc ensureMigrationsTableExists(conn: DbConn): void =
   let tableCount = conn.getValue(sql"""
@@ -38,33 +45,35 @@ proc loadConfig*(filename: string): DbMigrateConfig =
     let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper)
     logLevel = if idx == -1: lvlInfo else: (Level)(idx)
 
-  return (
+  return DbMigrateConfig(
     driver:
       if existsEnv("DATABASE_DRIVER"): $getEnv("DATABASE_DRIVER")
       elif cfg.hasKey("driver"): cfg["driver"].getStr
       else: "postres",
-    sqlDir:
-      if existsEnv("MIGRATIONS_DIR"): $getEnv("MIGRATIONS_DIR")
-      elif cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr
-      else: "migrations",
     connectionString:
       if existsEnv("DATABASE_URL"): $getEnv("DATABASE_URL")
       elif cfg.hasKey("connectionString"): cfg["connectionString"].getStr
       else: "",
+    sqlDirs:
+      if existsEnv("MIGRATIONS_DIRS"): getEnv("MIGRATIONS_DIRS").split(';')
+      elif cfg.hasKey("sqlDirs"): cfg["sqlDirs"].getElems.mapIt(it.getStr)
+      else: @["migrations"],
     logLevel: logLevel)
 
-proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[string] =
+proc createMigration*(config: DbMigrateConfig, migrationName: string): MigrationEntry =
   ## Create a new set of database migration files.
-  let timestamp = getTime().getLocalTime().format("yyyyMMddHHmmss")
+  let timestamp = now().format("yyyyMMddHHmmss")
   let filenamePrefix =  timestamp & "-" & migrationName
 
-  let upFilename = joinPath(config.sqlDir, filenamePrefix & "-up.sql")
-  let downFilename = joinPath(config.sqlDir, filenamePrefix & "-down.sql")
+  let migration = (
+    name: filenamePrefix & "-up.sql",
+    upPath: joinPath(config.sqlDirs[0], filenamePrefix & "-up.sql"),
+    downPath: joinPath(config.sqlDirs[0], filenamePrefix & "-down.sql"))
 
   let scriptDesc = migrationName & " (" & timestamp & ")"
 
-  let upFile = open(upFilename, fmWrite)
-  let downFile = open(downFilename, fmWrite)
+  let upFile = open(migration.upPath, fmWrite)
+  let downFile = open(migration.downPath, fmWrite)
 
   upFile.writeLine "-- UP script for " & scriptDesc
   downFile.writeLine "-- DOWN script for " & scriptDesc
@@ -72,47 +81,69 @@ proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[strin
   upFile.close()
   downFile.close()
 
-  return @[upFilename, downFilename]
+  return migration
 
-proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig):
-  tuple[ run, notRun, missing: seq[string] ] =
+proc diffMigrations*(
+    pgConn: DbConn,
+    config: DbMigrateConfig
+  ): tuple[
+    available: TableRef[string, MigrationEntry],
+    run: seq[string],
+    notRun, missing: seq[MigrationEntry] ] =
+
+  debug "diffMigrations: inspecting database and configured directories " &
+    "for migrations"
 
   # Query the database to find out what migrations have been run.
-  var migrationsRun = initSet[string]()
+  var migrationsRun = initHashSet[string]()
 
   for row in pgConn.fastRows(sql"SELECT * FROM migrations ORDER BY name", @[]):
     migrationsRun.incl(row[1])
 
   # Inspect the filesystem to see what migrations are available.
-  var migrationsAvailable = initSet[string]()
-  for filePath in walkFiles(joinPath(config.sqlDir, "*.sql")):
-    var migrationName = filePath.extractFilename
-    migrationName.removeSuffix("-up.sql")
-    migrationName.removeSuffix("-down.sql")
-    migrationsAvailable.incl(migrationName)
+  var migrationsAvailable = newTable[string, MigrationEntry]()
+  for sqlDir in config.sqlDirs:
+    debug "Looking in " & sqlDir
+    for filePath in walkFiles(joinPath(sqlDir, "*.sql")):
+      debug "Saw migration file: " & filePath
+      var migrationName = filePath.extractFilename
+      migrationName.removeSuffix("-up.sql")
+      migrationName.removeSuffix("-down.sql")
+      migrationsAvailable[migrationName] = (
+        name: migrationName,
+        upPath: joinPath(sqlDir, migrationName) & "-up.sql",
+        downPath: joinPath(sqlDir, migrationName) & "-down.sql")
 
   # Diff with the list of migrations that we have in our migrations
   # directory.
   let migrationsInOrder =
-    toSeq(migrationsAvailable.items).sorted(system.cmp)
+    toSeq(migrationsAvailable.keys).sorted(system.cmp)
 
-  var migrationsNotRun = newSeq[string]()
-  var missingMigrations = newSeq[string]()
+  var migrationsNotRun = newSeq[MigrationEntry]()
+  var missingMigrations = newSeq[MigrationEntry]()
 
-  for migration in migrationsInOrder:
-    if not migrationsRun.contains(migration):
-      migrationsNotRun.add(migration)
+  for migName in migrationsInOrder:
+    if not migrationsRun.contains(migName):
+      migrationsNotRun.add(migrationsAvailable[migName])
 
     # if we've already seen some migrations that have not been run, but this
     # one has been, that means we have a gap and are missing migrations
     elif migrationsNotRun.len > 0:
       missingMigrations.add(migrationsNotRun)
-      migrationsNotRun = newSeq[string]()
+      migrationsNotRun = newSeq[MigrationEntry]()
 
-  return (run: toSeq(migrationsRun.items).sorted(system.cmp),
+  result = (available: migrationsAvailable,
+          run: toSeq(migrationsRun.items).sorted(system.cmp),
           notRun: migrationsNotRun,
           missing: missingMigrations)
 
+  debug "diffMigration: Results" &
+    "\n\tavailable: " & $toSeq(result[0].keys) &
+    "\n\trun:       " & $result[1] &
+    "\n\tnotRun:    " & $(result[2].mapIt(it.name)) &
+    "\n\tmissing:   " & $(result[3].mapIt(it.name))
+
+
 proc readStatements*(filename: string): seq[SqlQuery] =
   result = @[]
   var stmt: string = ""
@@ -130,28 +161,30 @@ proc readStatements*(filename: string): seq[SqlQuery] =
 
   if stmt.strip.len > 0: result.add(sql(stmt))
 
-proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[string] =
-  var migrationsRun = newSeq[string]()
+proc up*(
+    pgConn: DbConn,
+    config: DbMigrateConfig,
+    toRun: seq[MigrationEntry]): seq[MigrationEntry] =
+  var migrationsRun = newSeq[MigrationEntry]()
   # Begin a transaction.
   pgConn.exec(sql"BEGIN")
 
   # Apply each of the migrations.
   for migration in toRun:
-    info migration
-    let filename = joinPath(config.sqlDir, migration & "-up.sql")
+    info migration.name
 
-    if not filename.fileExists:
-      pgConn.rollbackWithErr "Can not find UP file for " & migration &
-        ". Expected '" & filename & "'."
+    if not migration.upPath.fileExists:
+      pgConn.rollbackWithErr "Can not find UP file for " & migration.name &
+        ". Expected '" & migration.upPath & "'."
 
-    let statements = filename.readStatements
+    let statements = migration.upPath.readStatements
 
     try:
       for statement in statements:
         pgConn.exec(statement)
-      pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration)
+      pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration.name)
     except DbError:
-      pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
+      pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
         getCurrentExceptionMsg()
 
     migrationsRun.add(migration)
@@ -160,27 +193,28 @@ proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[strin
 
   return migrationsRun
 
-proc down*(pgConn: DbConn, config: DbMigrateConfig, migrationsToDown: seq[string]): seq[string] =
-  var migrationsDowned = newSeq[string]()
+proc down*(
+    pgConn: DbConn,
+    config: DbMigrateConfig,
+    migrationsToDown: seq[MigrationEntry]): seq[MigrationEntry] =
+  var migrationsDowned = newSeq[MigrationEntry]()
 
   pgConn.exec(sql"BEGIN")
 
   for migration in migrationsToDown:
-    info migration
+    info migration.name
 
-    let filename = joinPath(config.sqlDir, migration & "-down.sql")
+    if not migration.downPath.fileExists:
+      pgConn.rollbackWithErr "Can not find DOWN file for " & migration.name &
+        ". Expected '" & migration.downPath & "'."
 
-    if not filename.fileExists:
-      pgConn.rollbackWithErr "Can not find DOWN file for " & migration &
-        ". Expected '" & filename & "'."
-
-    let statements = filename.readStatements
+    let statements = migration.downPath.readStatements
 
     try:
       for statement in statements: pgConn.exec(statement)
-      pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration)
+      pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration.name)
     except DbError:
-      pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
+      pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
         getCurrentExceptionMsg()
 
     migrationsDowned.add(migration)
@@ -196,12 +230,15 @@ Usage:
   db_migrate [options] down [<count>]
   db_migrate [options] init <schema-name>
   db_migrate (-V | --version)
+  db_migrate (-h | --help)
 
 Options:
 
   -c --config <config-file>   Use the given configuration file (defaults to
                               "database.json").
 
+  -h --help                   Show this usage information.
+
   -q --quiet                  Suppress log information.
 
   -v --verbose                Print detailed log information.
@@ -212,7 +249,7 @@ Options:
 """
 
   # Parse arguments
-  let args = docopt(doc, version = "db-migrate 0.2.6")
+  let args = docopt(doc, version = "db-migrate (Nim) 0.3.2\nhttps://git.jdb-software.com/jdb/db-migrate")
 
   let exitErr = proc(msg: string): void =
     fatal("db_migrate: " & msg)
@@ -240,20 +277,22 @@ Options:
   else: logging.setLogFilter(config.logLevel)
 
   # Check for migrations directory
-  if not existsDir config.sqlDir:
-    try:
-      warn "SQL directory '" & config.sqlDir &
-        "' does not exist and will be created."
-      createDir config.sqlDir
-    except IOError:
-      exitErr "Unable to create directory: " & config.sqlDir & ":\L\T" & getCurrentExceptionMsg()
+  for sqlDir in config.sqlDirs:
+    if not dirExists sqlDir:
+      try:
+        warn "SQL directory '" & sqlDir &
+          "' does not exist and will be created."
+        createDir sqlDir
+      except IOError:
+        exitErr "Unable to create directory: " & sqlDir & ":\L\T" & getCurrentExceptionMsg()
 
   # Execute commands
   if args["create"]:
     try:
-      let filesCreated = createMigration(config, $args["<migration-name>"])
+      let newMigration = createMigration(config, $args["<migration-name>"])
       info "Created new migration files:"
-      for filename in filesCreated: info "\t" & filename
+      info "\t" & newMigration.upPath
+      info "\t" & newMigration.downPath
     except IOError:
       exitErr "Unable to create migration scripts: " & getCurrentExceptionMsg()
 
@@ -266,7 +305,7 @@ Options:
 
     pgConn.ensureMigrationsTableExists
 
-    let (run, notRun, missing) = diffMigrations(pgConn, config)
+    let (available, run, notRun, missing) = diffMigrations(pgConn, config)
 
     # Make sure we have no gaps (database is in an unknown state)
     if missing.len > 0:
@@ -285,7 +324,8 @@ Options:
     elif args["down"]:
       try:
         let count = if args["<count>"]: parseInt($args["<count>"]) else: 1
-        let toRun = if count < run.len: run.reversed[0..<count] else: run.reversed
+        let toRunNames = if count < run.len: run.reversed[0..<count] else: run.reversed
+        let toRun = toRunNames.mapIt(available[it])
         let migrationsRun = pgConn.down(config, toRun)
         info "Went down " & $(migrationsRun.len) & "."
       except DbError: