|
|
|
@ -3,15 +3,22 @@
|
|
|
|
|
##
|
|
|
|
|
## Simple tool to manage database migrations.
|
|
|
|
|
|
|
|
|
|
import algorithm, json, times, os, strutils, docopt, db_postgres, sets,
|
|
|
|
|
sequtils, logging
|
|
|
|
|
import std/[algorithm, json, logging, os, sequtils, sets, strutils, tables,
|
|
|
|
|
times]
|
|
|
|
|
import db_connector/db_postgres
|
|
|
|
|
import docopt
|
|
|
|
|
|
|
|
|
|
type
|
|
|
|
|
DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: Level ]
|
|
|
|
|
DbMigrateConfig* = object
|
|
|
|
|
driver, connectionString: string
|
|
|
|
|
sqlDirs: seq[string]
|
|
|
|
|
logLevel: Level
|
|
|
|
|
|
|
|
|
|
MigrationEntry* = tuple[name, upPath, downPath: string]
|
|
|
|
|
|
|
|
|
|
proc ensureMigrationsTableExists(conn: DbConn): void =
|
|
|
|
|
let tableCount = conn.getValue(sql"""
|
|
|
|
|
SELECT COUNT(*) FROM information_schema.tables
|
|
|
|
|
SELECT COUNT(*) FROM information_schema.tables
|
|
|
|
|
WHERE table_name = 'migrations';""")
|
|
|
|
|
|
|
|
|
|
if tableCount.strip == "0":
|
|
|
|
@ -38,33 +45,35 @@ proc loadConfig*(filename: string): DbMigrateConfig =
|
|
|
|
|
let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper)
|
|
|
|
|
logLevel = if idx == -1: lvlInfo else: (Level)(idx)
|
|
|
|
|
|
|
|
|
|
return (
|
|
|
|
|
return DbMigrateConfig(
|
|
|
|
|
driver:
|
|
|
|
|
if existsEnv("DATABASE_DRIVER"): $getEnv("DATABASE_DRIVER")
|
|
|
|
|
elif cfg.hasKey("driver"): cfg["driver"].getStr
|
|
|
|
|
else: "postres",
|
|
|
|
|
sqlDir:
|
|
|
|
|
if existsEnv("MIGRATIONS_DIR"): $getEnv("MIGRATIONS_DIR")
|
|
|
|
|
elif cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr
|
|
|
|
|
else: "migrations",
|
|
|
|
|
connectionString:
|
|
|
|
|
if existsEnv("DATABASE_URL"): $getEnv("DATABASE_URL")
|
|
|
|
|
elif cfg.hasKey("connectionString"): cfg["connectionString"].getStr
|
|
|
|
|
else: "",
|
|
|
|
|
sqlDirs:
|
|
|
|
|
if existsEnv("MIGRATIONS_DIRS"): getEnv("MIGRATIONS_DIRS").split(';')
|
|
|
|
|
elif cfg.hasKey("sqlDirs"): cfg["sqlDirs"].getElems.mapIt(it.getStr)
|
|
|
|
|
else: @["migrations"],
|
|
|
|
|
logLevel: logLevel)
|
|
|
|
|
|
|
|
|
|
proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[string] =
|
|
|
|
|
proc createMigration*(config: DbMigrateConfig, migrationName: string): MigrationEntry =
|
|
|
|
|
## Create a new set of database migration files.
|
|
|
|
|
let timestamp = getTime().getLocalTime().format("yyyyMMddHHmmss")
|
|
|
|
|
let timestamp = now().format("yyyyMMddHHmmss")
|
|
|
|
|
let filenamePrefix = timestamp & "-" & migrationName
|
|
|
|
|
|
|
|
|
|
let upFilename = joinPath(config.sqlDir, filenamePrefix & "-up.sql")
|
|
|
|
|
let downFilename = joinPath(config.sqlDir, filenamePrefix & "-down.sql")
|
|
|
|
|
let migration = (
|
|
|
|
|
name: filenamePrefix & "-up.sql",
|
|
|
|
|
upPath: joinPath(config.sqlDirs[0], filenamePrefix & "-up.sql"),
|
|
|
|
|
downPath: joinPath(config.sqlDirs[0], filenamePrefix & "-down.sql"))
|
|
|
|
|
|
|
|
|
|
let scriptDesc = migrationName & " (" & timestamp & ")"
|
|
|
|
|
|
|
|
|
|
let upFile = open(upFilename, fmWrite)
|
|
|
|
|
let downFile = open(downFilename, fmWrite)
|
|
|
|
|
let upFile = open(migration.upPath, fmWrite)
|
|
|
|
|
let downFile = open(migration.downPath, fmWrite)
|
|
|
|
|
|
|
|
|
|
upFile.writeLine "-- UP script for " & scriptDesc
|
|
|
|
|
downFile.writeLine "-- DOWN script for " & scriptDesc
|
|
|
|
@ -72,47 +81,69 @@ proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[strin
|
|
|
|
|
upFile.close()
|
|
|
|
|
downFile.close()
|
|
|
|
|
|
|
|
|
|
return @[upFilename, downFilename]
|
|
|
|
|
return migration
|
|
|
|
|
|
|
|
|
|
proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig):
|
|
|
|
|
tuple[ run, notRun, missing: seq[string] ] =
|
|
|
|
|
proc diffMigrations*(
|
|
|
|
|
pgConn: DbConn,
|
|
|
|
|
config: DbMigrateConfig
|
|
|
|
|
): tuple[
|
|
|
|
|
available: TableRef[string, MigrationEntry],
|
|
|
|
|
run: seq[string],
|
|
|
|
|
notRun, missing: seq[MigrationEntry] ] =
|
|
|
|
|
|
|
|
|
|
debug "diffMigrations: inspecting database and configured directories " &
|
|
|
|
|
"for migrations"
|
|
|
|
|
|
|
|
|
|
# Query the database to find out what migrations have been run.
|
|
|
|
|
var migrationsRun = initSet[string]()
|
|
|
|
|
var migrationsRun = initHashSet[string]()
|
|
|
|
|
|
|
|
|
|
for row in pgConn.fastRows(sql"SELECT * FROM migrations ORDER BY name", @[]):
|
|
|
|
|
migrationsRun.incl(row[1])
|
|
|
|
|
|
|
|
|
|
# Inspect the filesystem to see what migrations are available.
|
|
|
|
|
var migrationsAvailable = initSet[string]()
|
|
|
|
|
for filePath in walkFiles(joinPath(config.sqlDir, "*.sql")):
|
|
|
|
|
var migrationName = filePath.extractFilename
|
|
|
|
|
migrationName.removeSuffix("-up.sql")
|
|
|
|
|
migrationName.removeSuffix("-down.sql")
|
|
|
|
|
migrationsAvailable.incl(migrationName)
|
|
|
|
|
var migrationsAvailable = newTable[string, MigrationEntry]()
|
|
|
|
|
for sqlDir in config.sqlDirs:
|
|
|
|
|
debug "Looking in " & sqlDir
|
|
|
|
|
for filePath in walkFiles(joinPath(sqlDir, "*.sql")):
|
|
|
|
|
debug "Saw migration file: " & filePath
|
|
|
|
|
var migrationName = filePath.extractFilename
|
|
|
|
|
migrationName.removeSuffix("-up.sql")
|
|
|
|
|
migrationName.removeSuffix("-down.sql")
|
|
|
|
|
migrationsAvailable[migrationName] = (
|
|
|
|
|
name: migrationName,
|
|
|
|
|
upPath: joinPath(sqlDir, migrationName) & "-up.sql",
|
|
|
|
|
downPath: joinPath(sqlDir, migrationName) & "-down.sql")
|
|
|
|
|
|
|
|
|
|
# Diff with the list of migrations that we have in our migrations
|
|
|
|
|
# directory.
|
|
|
|
|
let migrationsInOrder =
|
|
|
|
|
toSeq(migrationsAvailable.items).sorted(system.cmp)
|
|
|
|
|
toSeq(migrationsAvailable.keys).sorted(system.cmp)
|
|
|
|
|
|
|
|
|
|
var migrationsNotRun = newSeq[string]()
|
|
|
|
|
var missingMigrations = newSeq[string]()
|
|
|
|
|
var migrationsNotRun = newSeq[MigrationEntry]()
|
|
|
|
|
var missingMigrations = newSeq[MigrationEntry]()
|
|
|
|
|
|
|
|
|
|
for migration in migrationsInOrder:
|
|
|
|
|
if not migrationsRun.contains(migration):
|
|
|
|
|
migrationsNotRun.add(migration)
|
|
|
|
|
for migName in migrationsInOrder:
|
|
|
|
|
if not migrationsRun.contains(migName):
|
|
|
|
|
migrationsNotRun.add(migrationsAvailable[migName])
|
|
|
|
|
|
|
|
|
|
# if we've already seen some migrations that have not been run, but this
|
|
|
|
|
# one has been, that means we have a gap and are missing migrations
|
|
|
|
|
elif migrationsNotRun.len > 0:
|
|
|
|
|
missingMigrations.add(migrationsNotRun)
|
|
|
|
|
migrationsNotRun = newSeq[string]()
|
|
|
|
|
migrationsNotRun = newSeq[MigrationEntry]()
|
|
|
|
|
|
|
|
|
|
return (run: toSeq(migrationsRun.items).sorted(system.cmp),
|
|
|
|
|
result = (available: migrationsAvailable,
|
|
|
|
|
run: toSeq(migrationsRun.items).sorted(system.cmp),
|
|
|
|
|
notRun: migrationsNotRun,
|
|
|
|
|
missing: missingMigrations)
|
|
|
|
|
|
|
|
|
|
debug "diffMigration: Results" &
|
|
|
|
|
"\n\tavailable: " & $toSeq(result[0].keys) &
|
|
|
|
|
"\n\trun: " & $result[1] &
|
|
|
|
|
"\n\tnotRun: " & $(result[2].mapIt(it.name)) &
|
|
|
|
|
"\n\tmissing: " & $(result[3].mapIt(it.name))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
proc readStatements*(filename: string): seq[SqlQuery] =
|
|
|
|
|
result = @[]
|
|
|
|
|
var stmt: string = ""
|
|
|
|
@ -130,28 +161,30 @@ proc readStatements*(filename: string): seq[SqlQuery] =
|
|
|
|
|
|
|
|
|
|
if stmt.strip.len > 0: result.add(sql(stmt))
|
|
|
|
|
|
|
|
|
|
proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[string] =
|
|
|
|
|
var migrationsRun = newSeq[string]()
|
|
|
|
|
proc up*(
|
|
|
|
|
pgConn: DbConn,
|
|
|
|
|
config: DbMigrateConfig,
|
|
|
|
|
toRun: seq[MigrationEntry]): seq[MigrationEntry] =
|
|
|
|
|
var migrationsRun = newSeq[MigrationEntry]()
|
|
|
|
|
# Begin a transaction.
|
|
|
|
|
pgConn.exec(sql"BEGIN")
|
|
|
|
|
|
|
|
|
|
# Apply each of the migrations.
|
|
|
|
|
for migration in toRun:
|
|
|
|
|
info migration
|
|
|
|
|
let filename = joinPath(config.sqlDir, migration & "-up.sql")
|
|
|
|
|
info migration.name
|
|
|
|
|
|
|
|
|
|
if not filename.fileExists:
|
|
|
|
|
pgConn.rollbackWithErr "Can not find UP file for " & migration &
|
|
|
|
|
". Expected '" & filename & "'."
|
|
|
|
|
if not migration.upPath.fileExists:
|
|
|
|
|
pgConn.rollbackWithErr "Can not find UP file for " & migration.name &
|
|
|
|
|
". Expected '" & migration.upPath & "'."
|
|
|
|
|
|
|
|
|
|
let statements = filename.readStatements
|
|
|
|
|
let statements = migration.upPath.readStatements
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
for statement in statements:
|
|
|
|
|
pgConn.exec(statement)
|
|
|
|
|
pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration)
|
|
|
|
|
pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration.name)
|
|
|
|
|
except DbError:
|
|
|
|
|
pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
|
|
|
|
|
pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
|
|
|
|
|
getCurrentExceptionMsg()
|
|
|
|
|
|
|
|
|
|
migrationsRun.add(migration)
|
|
|
|
@ -160,27 +193,28 @@ proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[strin
|
|
|
|
|
|
|
|
|
|
return migrationsRun
|
|
|
|
|
|
|
|
|
|
proc down*(pgConn: DbConn, config: DbMigrateConfig, migrationsToDown: seq[string]): seq[string] =
|
|
|
|
|
var migrationsDowned = newSeq[string]()
|
|
|
|
|
proc down*(
|
|
|
|
|
pgConn: DbConn,
|
|
|
|
|
config: DbMigrateConfig,
|
|
|
|
|
migrationsToDown: seq[MigrationEntry]): seq[MigrationEntry] =
|
|
|
|
|
var migrationsDowned = newSeq[MigrationEntry]()
|
|
|
|
|
|
|
|
|
|
pgConn.exec(sql"BEGIN")
|
|
|
|
|
|
|
|
|
|
for migration in migrationsToDown:
|
|
|
|
|
info migration
|
|
|
|
|
info migration.name
|
|
|
|
|
|
|
|
|
|
let filename = joinPath(config.sqlDir, migration & "-down.sql")
|
|
|
|
|
if not migration.downPath.fileExists:
|
|
|
|
|
pgConn.rollbackWithErr "Can not find DOWN file for " & migration.name &
|
|
|
|
|
". Expected '" & migration.downPath & "'."
|
|
|
|
|
|
|
|
|
|
if not filename.fileExists:
|
|
|
|
|
pgConn.rollbackWithErr "Can not find DOWN file for " & migration &
|
|
|
|
|
". Expected '" & filename & "'."
|
|
|
|
|
|
|
|
|
|
let statements = filename.readStatements
|
|
|
|
|
let statements = migration.downPath.readStatements
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
for statement in statements: pgConn.exec(statement)
|
|
|
|
|
pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration)
|
|
|
|
|
pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration.name)
|
|
|
|
|
except DbError:
|
|
|
|
|
pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
|
|
|
|
|
pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
|
|
|
|
|
getCurrentExceptionMsg()
|
|
|
|
|
|
|
|
|
|
migrationsDowned.add(migration)
|
|
|
|
@ -196,12 +230,15 @@ Usage:
|
|
|
|
|
db_migrate [options] down [<count>]
|
|
|
|
|
db_migrate [options] init <schema-name>
|
|
|
|
|
db_migrate (-V | --version)
|
|
|
|
|
db_migrate (-h | --help)
|
|
|
|
|
|
|
|
|
|
Options:
|
|
|
|
|
|
|
|
|
|
-c --config <config-file> Use the given configuration file (defaults to
|
|
|
|
|
"database.json").
|
|
|
|
|
|
|
|
|
|
-h --help Show this usage information.
|
|
|
|
|
|
|
|
|
|
-q --quiet Suppress log information.
|
|
|
|
|
|
|
|
|
|
-v --verbose Print detailed log information.
|
|
|
|
@ -212,7 +249,7 @@ Options:
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
# Parse arguments
|
|
|
|
|
let args = docopt(doc, version = "db-migrate 0.2.6")
|
|
|
|
|
let args = docopt(doc, version = "db-migrate (Nim) 0.3.2\nhttps://git.jdb-software.com/jdb/db-migrate")
|
|
|
|
|
|
|
|
|
|
let exitErr = proc(msg: string): void =
|
|
|
|
|
fatal("db_migrate: " & msg)
|
|
|
|
@ -240,20 +277,22 @@ Options:
|
|
|
|
|
else: logging.setLogFilter(config.logLevel)
|
|
|
|
|
|
|
|
|
|
# Check for migrations directory
|
|
|
|
|
if not existsDir config.sqlDir:
|
|
|
|
|
try:
|
|
|
|
|
warn "SQL directory '" & config.sqlDir &
|
|
|
|
|
"' does not exist and will be created."
|
|
|
|
|
createDir config.sqlDir
|
|
|
|
|
except IOError:
|
|
|
|
|
exitErr "Unable to create directory: " & config.sqlDir & ":\L\T" & getCurrentExceptionMsg()
|
|
|
|
|
for sqlDir in config.sqlDirs:
|
|
|
|
|
if not dirExists sqlDir:
|
|
|
|
|
try:
|
|
|
|
|
warn "SQL directory '" & sqlDir &
|
|
|
|
|
"' does not exist and will be created."
|
|
|
|
|
createDir sqlDir
|
|
|
|
|
except IOError:
|
|
|
|
|
exitErr "Unable to create directory: " & sqlDir & ":\L\T" & getCurrentExceptionMsg()
|
|
|
|
|
|
|
|
|
|
# Execute commands
|
|
|
|
|
if args["create"]:
|
|
|
|
|
try:
|
|
|
|
|
let filesCreated = createMigration(config, $args["<migration-name>"])
|
|
|
|
|
let newMigration = createMigration(config, $args["<migration-name>"])
|
|
|
|
|
info "Created new migration files:"
|
|
|
|
|
for filename in filesCreated: info "\t" & filename
|
|
|
|
|
info "\t" & newMigration.upPath
|
|
|
|
|
info "\t" & newMigration.downPath
|
|
|
|
|
except IOError:
|
|
|
|
|
exitErr "Unable to create migration scripts: " & getCurrentExceptionMsg()
|
|
|
|
|
|
|
|
|
@ -266,7 +305,7 @@ Options:
|
|
|
|
|
|
|
|
|
|
pgConn.ensureMigrationsTableExists
|
|
|
|
|
|
|
|
|
|
let (run, notRun, missing) = diffMigrations(pgConn, config)
|
|
|
|
|
let (available, run, notRun, missing) = diffMigrations(pgConn, config)
|
|
|
|
|
|
|
|
|
|
# Make sure we have no gaps (database is in an unknown state)
|
|
|
|
|
if missing.len > 0:
|
|
|
|
@ -285,7 +324,8 @@ Options:
|
|
|
|
|
elif args["down"]:
|
|
|
|
|
try:
|
|
|
|
|
let count = if args["<count>"]: parseInt($args["<count>"]) else: 1
|
|
|
|
|
let toRun = if count < run.len: run.reversed[0..<count] else: run.reversed
|
|
|
|
|
let toRunNames = if count < run.len: run.reversed[0..<count] else: run.reversed
|
|
|
|
|
let toRun = toRunNames.mapIt(available[it])
|
|
|
|
|
let migrationsRun = pgConn.down(config, toRun)
|
|
|
|
|
info "Went down " & $(migrationsRun.len) & "."
|
|
|
|
|
except DbError:
|
|
|
|
|