README.md

@@ -1,36 +1,4 @@
-# DB Migrate
+DB Migrate
+==========
 
 Small tool(s) to manage database migrations in various languages.
-
-## Usage
-
-```
-Usage:
-  db_migrate [options] create <migration-name>
-  db_migrate [options] up [<count>]
-  db_migrate [options] down [<count>]
-  db_migrate [options] init <schema-name>
-  db_migrate (-V | --version)
-  db_migrate (-h | --help)
-
-Options:
-  -c --config <config-file>   Use the given configuration file (defaults to
-                              "database.properties").
-  -q --quiet                  Suppress log information.
-  -v --verbose                Print detailed log information.
-     --very-verbose           Print very detailed log information.
-  -V --version                Print the tools version information.
-  -h --help                   Print this usage information.
-```
-
-
-## Database Config Format
-
-The database config is formatted as JSON. The following keys are supported by
-all of the implementations:
-
-* `sqlDir` -- Directory to store SQL files.
-
-The following keys are supported by the Nim implementation:
-
-* `connectionString` -- 

db_migrate.nimble

@@ -1,7 +1,7 @@
 # Package
 
 bin           = @["db_migrate"]
-version       = "0.3.2"
+version       = "0.2.6"
 author        = "Jonathan Bernard"
 description   = "Simple tool to handle database migrations."
 license       = "BSD"
@@ -9,4 +9,5 @@ srcDir        = "src/main/nim"
 
 # Dependencies
 
-requires: @["nim >= 2.0.0", "docopt >= 0.1.0", "db_connector"]
+requires: @["nim >= 0.13.0", "docopt >= 0.1.0"]
+

src/main/groovy/com/jdblabs/dbmigrate/DbMigrate.groovy

@@ -42,7 +42,7 @@ Options:
   private static Logger LOGGER = LoggerFactory.getLogger(DbMigrate)
 
   Sql sql
-  File sqlDir
+  File migrationsDir
 
   public static void main(String[] args) {
 
@@ -90,14 +90,14 @@ Options:
         givenCfg.clear() } }
 
     // Check for migrations directory
-    File sqlDir = new File(givenCfg["sqlDir"] ?: 'migrations')
+    File migrationsDir = new File(givenCfg["migrations.dir"] ?: 'migrations')
-    if (!sqlDir.exists() || !sqlDir.isDirectory()) {
+    if (!migrationsDir.exists() || !migrationsDir.isDirectory()) {
       clilog.error("'{}' does not exist or is not a directory.",
-        sqlDir.canonicalPath)
+        migrationsDir.canonicalPath)
       System.exit(1) }
 
     // Instantiate the DbMigrate instance
-    DbMigrate dbmigrate = new DbMigrate(sqlDir: sqlDir)
+    DbMigrate dbmigrate = new DbMigrate(migrationsDir: migrationsDir)
 
     // If we've only been asked to create a new migration, we don't need to
     // setup the DB connection.
@@ -112,7 +112,7 @@ Options:
 
     // Create the datasource.
     Properties dsProps = new Properties()
-    dsProps.putAll(givenCfg.findAll { it.key != 'sqlDir' })
+    dsProps.putAll(givenCfg.findAll { it.key != 'migrations.dir' })
 
     HikariDataSource hds = new HikariDataSource(new HikariConfig(dsProps))
 
@@ -125,8 +125,8 @@ Options:
   public List<File> createMigration(String migrationName) {
     String timestamp = sdf.format(new Date())
 
-    File upFile = new File(sqlDir, "$timestamp-$migrationName-up.sql")
+    File upFile = new File(migrationsDir, "$timestamp-$migrationName-up.sql")
-    File downFile = new File(sqlDir, "$timestamp-$migrationName-down.sql")
+    File downFile = new File(migrationsDir, "$timestamp-$migrationName-down.sql")
 
     upFile.text = "-- UP script for $migrationName ($timestamp)"
     downFile.text = "-- DOWN script for $migrationName ($timestamp)"
@@ -140,7 +140,7 @@ Options:
 CREATE TABLE IF NOT EXISTS migrations (
   id SERIAL PRIMARY KEY,
   name VARCHAR NOT NULL,
-  run_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW())''') }
+  run_at TIMESTAMP NOT NULL DEFAULT NOW())''') }
 
   public def diffMigrations() {
     def results = [notRun: [], missing: []]
@@ -150,7 +150,7 @@ CREATE TABLE IF NOT EXISTS migrations (
       .collect { it.name }.sort()
 
     SortedSet<String> available = new TreeSet<>()
-    available.addAll(sqlDir
+    available.addAll(migrationsDir
       .listFiles({ d, n -> n ==~ /.+-(up|down).sql$/ } as FilenameFilter)
       .collect { f -> f.name.replaceAll(/-(up|down).sql$/, '') })
 
@@ -215,7 +215,7 @@ CREATE TABLE IF NOT EXISTS migrations (
 
       toRun.each { migrationName ->
         LOGGER.info(migrationName)
-        File migrationFile = new File(sqlDir,
+        File migrationFile = new File(migrationsDir,
           "$migrationName-${up ? 'up' : 'down'}.sql")
 
         if (!migrationFile.exists() || !migrationFile.isFile())

src/main/nim/db_migrate.nim

@@ -3,22 +3,15 @@
 ##
 ## Simple tool to manage database migrations.
 
-import std/[algorithm, json, logging, os, sequtils, sets, strutils, tables,
+import algorithm, json, times, os, strutils, docopt, db_postgres, sets,
-            times]
+  sequtils, logging
-import db_connector/db_postgres
-import docopt
 
 type
-  DbMigrateConfig* = object
+  DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: Level ]
-    driver, connectionString: string
-    sqlDirs: seq[string]
-    logLevel: Level
-
-  MigrationEntry* = tuple[name, upPath, downPath: string]
 
 proc ensureMigrationsTableExists(conn: DbConn): void =
   let tableCount = conn.getValue(sql"""
-SELECT COUNT(*) FROM information_schema.tables
+SELECT COUNT(*) FROM information_schema.tables 
 WHERE table_name = 'migrations';""")
 
   if tableCount.strip == "0":
@@ -45,35 +38,33 @@ proc loadConfig*(filename: string): DbMigrateConfig =
     let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper)
     logLevel = if idx == -1: lvlInfo else: (Level)(idx)
 
-  return DbMigrateConfig(
+  return (
     driver:
       if existsEnv("DATABASE_DRIVER"): $getEnv("DATABASE_DRIVER")
       elif cfg.hasKey("driver"): cfg["driver"].getStr
       else: "postres",
+    sqlDir:
+      if existsEnv("MIGRATIONS_DIR"): $getEnv("MIGRATIONS_DIR")
+      elif cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr
+      else: "migrations",
     connectionString:
       if existsEnv("DATABASE_URL"): $getEnv("DATABASE_URL")
       elif cfg.hasKey("connectionString"): cfg["connectionString"].getStr
       else: "",
-    sqlDirs:
-      if existsEnv("MIGRATIONS_DIRS"): getEnv("MIGRATIONS_DIRS").split(';')
-      elif cfg.hasKey("sqlDirs"): cfg["sqlDirs"].getElems.mapIt(it.getStr)
-      else: @["migrations"],
     logLevel: logLevel)
 
-proc createMigration*(config: DbMigrateConfig, migrationName: string): MigrationEntry =
+proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[string] =
   ## Create a new set of database migration files.
-  let timestamp = now().format("yyyyMMddHHmmss")
+  let timestamp = getTime().getLocalTime().format("yyyyMMddHHmmss")
   let filenamePrefix =  timestamp & "-" & migrationName
 
-  let migration = (
+  let upFilename = joinPath(config.sqlDir, filenamePrefix & "-up.sql")
-    name: filenamePrefix & "-up.sql",
+  let downFilename = joinPath(config.sqlDir, filenamePrefix & "-down.sql")
-    upPath: joinPath(config.sqlDirs[0], filenamePrefix & "-up.sql"),
-    downPath: joinPath(config.sqlDirs[0], filenamePrefix & "-down.sql"))
 
   let scriptDesc = migrationName & " (" & timestamp & ")"
 
-  let upFile = open(migration.upPath, fmWrite)
+  let upFile = open(upFilename, fmWrite)
-  let downFile = open(migration.downPath, fmWrite)
+  let downFile = open(downFilename, fmWrite)
 
   upFile.writeLine "-- UP script for " & scriptDesc
   downFile.writeLine "-- DOWN script for " & scriptDesc
@@ -81,69 +72,47 @@ proc createMigration*(config: DbMigrateConfig, migrationName: string): Migration
   upFile.close()
   downFile.close()
 
-  return migration
+  return @[upFilename, downFilename]
 
-proc diffMigrations*(
+proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig):
-    pgConn: DbConn,
+  tuple[ run, notRun, missing: seq[string] ] =
-    config: DbMigrateConfig
-  ): tuple[
-    available: TableRef[string, MigrationEntry],
-    run: seq[string],
-    notRun, missing: seq[MigrationEntry] ] =
-
-  debug "diffMigrations: inspecting database and configured directories " &
-    "for migrations"
 
   # Query the database to find out what migrations have been run.
-  var migrationsRun = initHashSet[string]()
+  var migrationsRun = initSet[string]()
 
   for row in pgConn.fastRows(sql"SELECT * FROM migrations ORDER BY name", @[]):
     migrationsRun.incl(row[1])
 
   # Inspect the filesystem to see what migrations are available.
-  var migrationsAvailable = newTable[string, MigrationEntry]()
+  var migrationsAvailable = initSet[string]()
-  for sqlDir in config.sqlDirs:
+  for filePath in walkFiles(joinPath(config.sqlDir, "*.sql")):
-    debug "Looking in " & sqlDir
+    var migrationName = filePath.extractFilename
-    for filePath in walkFiles(joinPath(sqlDir, "*.sql")):
+    migrationName.removeSuffix("-up.sql")
-      debug "Saw migration file: " & filePath
+    migrationName.removeSuffix("-down.sql")
-      var migrationName = filePath.extractFilename
+    migrationsAvailable.incl(migrationName)
-      migrationName.removeSuffix("-up.sql")
-      migrationName.removeSuffix("-down.sql")
-      migrationsAvailable[migrationName] = (
-        name: migrationName,
-        upPath: joinPath(sqlDir, migrationName) & "-up.sql",
-        downPath: joinPath(sqlDir, migrationName) & "-down.sql")
 
   # Diff with the list of migrations that we have in our migrations
   # directory.
   let migrationsInOrder =
-    toSeq(migrationsAvailable.keys).sorted(system.cmp)
+    toSeq(migrationsAvailable.items).sorted(system.cmp)
 
-  var migrationsNotRun = newSeq[MigrationEntry]()
+  var migrationsNotRun = newSeq[string]()
-  var missingMigrations = newSeq[MigrationEntry]()
+  var missingMigrations = newSeq[string]()
 
-  for migName in migrationsInOrder:
+  for migration in migrationsInOrder:
-    if not migrationsRun.contains(migName):
+    if not migrationsRun.contains(migration):
-      migrationsNotRun.add(migrationsAvailable[migName])
+      migrationsNotRun.add(migration)
 
     # if we've already seen some migrations that have not been run, but this
     # one has been, that means we have a gap and are missing migrations
     elif migrationsNotRun.len > 0:
       missingMigrations.add(migrationsNotRun)
-      migrationsNotRun = newSeq[MigrationEntry]()
+      migrationsNotRun = newSeq[string]()
 
-  result = (available: migrationsAvailable,
+  return (run: toSeq(migrationsRun.items).sorted(system.cmp),
-          run: toSeq(migrationsRun.items).sorted(system.cmp),
           notRun: migrationsNotRun,
           missing: missingMigrations)
 
-  debug "diffMigration: Results" &
-    "\n\tavailable: " & $toSeq(result[0].keys) &
-    "\n\trun:       " & $result[1] &
-    "\n\tnotRun:    " & $(result[2].mapIt(it.name)) &
-    "\n\tmissing:   " & $(result[3].mapIt(it.name))
-
-
 proc readStatements*(filename: string): seq[SqlQuery] =
   result = @[]
   var stmt: string = ""
@@ -161,30 +130,28 @@ proc readStatements*(filename: string): seq[SqlQuery] =
 
   if stmt.strip.len > 0: result.add(sql(stmt))
 
-proc up*(
+proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[string] =
-    pgConn: DbConn,
+  var migrationsRun = newSeq[string]()
-    config: DbMigrateConfig,
-    toRun: seq[MigrationEntry]): seq[MigrationEntry] =
-  var migrationsRun = newSeq[MigrationEntry]()
   # Begin a transaction.
   pgConn.exec(sql"BEGIN")
 
   # Apply each of the migrations.
   for migration in toRun:
-    info migration.name
+    info migration
+    let filename = joinPath(config.sqlDir, migration & "-up.sql")
 
-    if not migration.upPath.fileExists:
+    if not filename.fileExists:
-      pgConn.rollbackWithErr "Can not find UP file for " & migration.name &
+      pgConn.rollbackWithErr "Can not find UP file for " & migration &
-        ". Expected '" & migration.upPath & "'."
+        ". Expected '" & filename & "'."
 
-    let statements = migration.upPath.readStatements
+    let statements = filename.readStatements
 
     try:
       for statement in statements:
         pgConn.exec(statement)
-      pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration.name)
+      pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration)
     except DbError:
-      pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
+      pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
         getCurrentExceptionMsg()
 
     migrationsRun.add(migration)
@@ -193,28 +160,27 @@ proc up*(
 
   return migrationsRun
 
-proc down*(
+proc down*(pgConn: DbConn, config: DbMigrateConfig, migrationsToDown: seq[string]): seq[string] =
-    pgConn: DbConn,
+  var migrationsDowned = newSeq[string]()
-    config: DbMigrateConfig,
-    migrationsToDown: seq[MigrationEntry]): seq[MigrationEntry] =
-  var migrationsDowned = newSeq[MigrationEntry]()
 
   pgConn.exec(sql"BEGIN")
 
   for migration in migrationsToDown:
-    info migration.name
+    info migration
 
-    if not migration.downPath.fileExists:
+    let filename = joinPath(config.sqlDir, migration & "-down.sql")
-      pgConn.rollbackWithErr "Can not find DOWN file for " & migration.name &
-        ". Expected '" & migration.downPath & "'."
 
-    let statements = migration.downPath.readStatements
+    if not filename.fileExists:
+      pgConn.rollbackWithErr "Can not find DOWN file for " & migration &
+        ". Expected '" & filename & "'."
+
+    let statements = filename.readStatements
 
     try:
       for statement in statements: pgConn.exec(statement)
-      pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration.name)
+      pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration)
     except DbError:
-      pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
+      pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
         getCurrentExceptionMsg()
 
     migrationsDowned.add(migration)
@@ -230,15 +196,12 @@ Usage:
   db_migrate [options] down [<count>]
   db_migrate [options] init <schema-name>
   db_migrate (-V | --version)
-  db_migrate (-h | --help)
 
 Options:
 
   -c --config <config-file>   Use the given configuration file (defaults to
                               "database.json").
 
-  -h --help                   Show this usage information.
-
   -q --quiet                  Suppress log information.
 
   -v --verbose                Print detailed log information.
@@ -249,7 +212,7 @@ Options:
 """
 
   # Parse arguments
-  let args = docopt(doc, version = "db-migrate (Nim) 0.3.2\nhttps://git.jdb-software.com/jdb/db-migrate")
+  let args = docopt(doc, version = "db-migrate 0.2.6")
 
   let exitErr = proc(msg: string): void =
     fatal("db_migrate: " & msg)
@@ -277,22 +240,20 @@ Options:
   else: logging.setLogFilter(config.logLevel)
 
   # Check for migrations directory
-  for sqlDir in config.sqlDirs:
+  if not existsDir config.sqlDir:
-    if not dirExists sqlDir:
+    try:
-      try:
+      warn "SQL directory '" & config.sqlDir &
-        warn "SQL directory '" & sqlDir &
+        "' does not exist and will be created."
-          "' does not exist and will be created."
+      createDir config.sqlDir
-        createDir sqlDir
+    except IOError:
-      except IOError:
+      exitErr "Unable to create directory: " & config.sqlDir & ":\L\T" & getCurrentExceptionMsg()
-        exitErr "Unable to create directory: " & sqlDir & ":\L\T" & getCurrentExceptionMsg()
 
   # Execute commands
   if args["create"]:
     try:
-      let newMigration = createMigration(config, $args["<migration-name>"])
+      let filesCreated = createMigration(config, $args["<migration-name>"])
       info "Created new migration files:"
-      info "\t" & newMigration.upPath
+      for filename in filesCreated: info "\t" & filename
-      info "\t" & newMigration.downPath
     except IOError:
       exitErr "Unable to create migration scripts: " & getCurrentExceptionMsg()
 
@@ -305,7 +266,7 @@ Options:
 
     pgConn.ensureMigrationsTableExists
 
-    let (available, run, notRun, missing) = diffMigrations(pgConn, config)
+    let (run, notRun, missing) = diffMigrations(pgConn, config)
 
     # Make sure we have no gaps (database is in an unknown state)
     if missing.len > 0:
@@ -324,8 +285,7 @@ Options:
     elif args["down"]:
       try:
         let count = if args["<count>"]: parseInt($args["<count>"]) else: 1
-        let toRunNames = if count < run.len: run.reversed[0..<count] else: run.reversed
+        let toRun = if count < run.len: run.reversed[0..<count] else: run.reversed
-        let toRun = toRunNames.mapIt(available[it])
         let migrationsRun = pgConn.down(config, toRun)
         info "Went down " & $(migrationsRun.len) & "."
       except DbError: