Allow multiple SQL directories (to support, for example, test configurations).
This commit is contained in:
parent
6837e5448b
commit
7cf53a4702
@ -1,7 +1,7 @@
|
||||
# Package
|
||||
|
||||
bin = @["db_migrate"]
|
||||
version = "0.2.8"
|
||||
version = "0.3.0"
|
||||
author = "Jonathan Bernard"
|
||||
description = "Simple tool to handle database migrations."
|
||||
license = "BSD"
|
||||
|
||||
@ -4,10 +4,15 @@
|
||||
## Simple tool to manage database migrations.
|
||||
|
||||
import algorithm, db_postgres, docopt, json, logging, os, sequtils, sets,
|
||||
strutils, times
|
||||
strutils, tables, times
|
||||
|
||||
type
|
||||
DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: Level ]
|
||||
DbMigrateConfig* = object
|
||||
driver, connectionString: string
|
||||
sqlDirs: seq[string]
|
||||
logLevel: Level
|
||||
|
||||
MigrationEntry* = tuple[name, upPath, downPath: string]
|
||||
|
||||
proc ensureMigrationsTableExists(conn: DbConn): void =
|
||||
let tableCount = conn.getValue(sql"""
|
||||
@ -38,33 +43,35 @@ proc loadConfig*(filename: string): DbMigrateConfig =
|
||||
let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper)
|
||||
logLevel = if idx == -1: lvlInfo else: (Level)(idx)
|
||||
|
||||
return (
|
||||
return DbMigrateConfig(
|
||||
driver:
|
||||
if existsEnv("DATABASE_DRIVER"): $getEnv("DATABASE_DRIVER")
|
||||
elif cfg.hasKey("driver"): cfg["driver"].getStr
|
||||
else: "postres",
|
||||
sqlDir:
|
||||
if existsEnv("MIGRATIONS_DIR"): $getEnv("MIGRATIONS_DIR")
|
||||
elif cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr
|
||||
else: "migrations",
|
||||
connectionString:
|
||||
if existsEnv("DATABASE_URL"): $getEnv("DATABASE_URL")
|
||||
elif cfg.hasKey("connectionString"): cfg["connectionString"].getStr
|
||||
else: "",
|
||||
sqlDirs:
|
||||
if existsEnv("MIGRATIONS_DIRS"): getEnv("MIGRATIONS_DIRS").split(';')
|
||||
elif cfg.hasKey("sqlDirs"): cfg["sqlDirs"].getElems.mapIt(it.getStr)
|
||||
else: @["migrations"],
|
||||
logLevel: logLevel)
|
||||
|
||||
proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[string] =
|
||||
proc createMigration*(config: DbMigrateConfig, migrationName: string): MigrationEntry =
|
||||
## Create a new set of database migration files.
|
||||
let timestamp = now().format("yyyyMMddHHmmss")
|
||||
let filenamePrefix = timestamp & "-" & migrationName
|
||||
|
||||
let upFilename = joinPath(config.sqlDir, filenamePrefix & "-up.sql")
|
||||
let downFilename = joinPath(config.sqlDir, filenamePrefix & "-down.sql")
|
||||
let migration = (
|
||||
name: filenamePrefix & "-up.sql",
|
||||
upPath: joinPath(config.sqlDirs[0], filenamePrefix & "-up.sql"),
|
||||
downPath: joinPath(config.sqlDirs[0], filenamePrefix & "-down.sql"))
|
||||
|
||||
let scriptDesc = migrationName & " (" & timestamp & ")"
|
||||
|
||||
let upFile = open(upFilename, fmWrite)
|
||||
let downFile = open(downFilename, fmWrite)
|
||||
let upFile = open(migration.upPath, fmWrite)
|
||||
let downFile = open(migration.downPath, fmWrite)
|
||||
|
||||
upFile.writeLine "-- UP script for " & scriptDesc
|
||||
downFile.writeLine "-- DOWN script for " & scriptDesc
|
||||
@ -72,10 +79,15 @@ proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[strin
|
||||
upFile.close()
|
||||
downFile.close()
|
||||
|
||||
return @[upFilename, downFilename]
|
||||
return migration
|
||||
|
||||
proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig):
|
||||
tuple[ run, notRun, missing: seq[string] ] =
|
||||
proc diffMigrations*(
|
||||
pgConn: DbConn,
|
||||
config: DbMigrateConfig
|
||||
): tuple[
|
||||
available: TableRef[string, MigrationEntry],
|
||||
run: seq[string],
|
||||
notRun, missing: seq[MigrationEntry] ] =
|
||||
|
||||
# Query the database to find out what migrations have been run.
|
||||
var migrationsRun = initHashSet[string]()
|
||||
@ -84,32 +96,37 @@ proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig):
|
||||
migrationsRun.incl(row[1])
|
||||
|
||||
# Inspect the filesystem to see what migrations are available.
|
||||
var migrationsAvailable = initHashSet[string]()
|
||||
for filePath in walkFiles(joinPath(config.sqlDir, "*.sql")):
|
||||
var migrationName = filePath.extractFilename
|
||||
migrationName.removeSuffix("-up.sql")
|
||||
migrationName.removeSuffix("-down.sql")
|
||||
migrationsAvailable.incl(migrationName)
|
||||
var migrationsAvailable = newTable[string, MigrationEntry]()
|
||||
for sqlDir in config.sqlDirs:
|
||||
for filePath in walkFiles(joinPath(sqlDir, "*.sql")):
|
||||
var migrationName = filePath.extractFilename
|
||||
migrationName.removeSuffix("-up.sql")
|
||||
migrationName.removeSuffix("-down.sql")
|
||||
migrationsAvailable[migrationName] = (
|
||||
name: migrationName,
|
||||
upPath: joinPath(sqlDir, migrationName) & "-up.sql",
|
||||
downPath: joinPath(sqlDir, migrationName) & "-down.sql")
|
||||
|
||||
# Diff with the list of migrations that we have in our migrations
|
||||
# directory.
|
||||
let migrationsInOrder =
|
||||
toSeq(migrationsAvailable.items).sorted(system.cmp)
|
||||
toSeq(migrationsAvailable.keys).sorted(system.cmp)
|
||||
|
||||
var migrationsNotRun = newSeq[string]()
|
||||
var missingMigrations = newSeq[string]()
|
||||
var migrationsNotRun = newSeq[MigrationEntry]()
|
||||
var missingMigrations = newSeq[MigrationEntry]()
|
||||
|
||||
for migration in migrationsInOrder:
|
||||
if not migrationsRun.contains(migration):
|
||||
migrationsNotRun.add(migration)
|
||||
for migName in migrationsInOrder:
|
||||
if not migrationsRun.contains(migName):
|
||||
migrationsNotRun.add(migrationsAvailable[migName])
|
||||
|
||||
# if we've already seen some migrations that have not been run, but this
|
||||
# one has been, that means we have a gap and are missing migrations
|
||||
elif migrationsNotRun.len > 0:
|
||||
missingMigrations.add(migrationsNotRun)
|
||||
migrationsNotRun = newSeq[string]()
|
||||
migrationsNotRun = newSeq[MigrationEntry]()
|
||||
|
||||
return (run: toSeq(migrationsRun.items).sorted(system.cmp),
|
||||
return (available: migrationsAvailable,
|
||||
run: toSeq(migrationsRun.items).sorted(system.cmp),
|
||||
notRun: migrationsNotRun,
|
||||
missing: missingMigrations)
|
||||
|
||||
@ -130,28 +147,30 @@ proc readStatements*(filename: string): seq[SqlQuery] =
|
||||
|
||||
if stmt.strip.len > 0: result.add(sql(stmt))
|
||||
|
||||
proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[string] =
|
||||
var migrationsRun = newSeq[string]()
|
||||
proc up*(
|
||||
pgConn: DbConn,
|
||||
config: DbMigrateConfig,
|
||||
toRun: seq[MigrationEntry]): seq[MigrationEntry] =
|
||||
var migrationsRun = newSeq[MigrationEntry]()
|
||||
# Begin a transaction.
|
||||
pgConn.exec(sql"BEGIN")
|
||||
|
||||
# Apply each of the migrations.
|
||||
for migration in toRun:
|
||||
info migration
|
||||
let filename = joinPath(config.sqlDir, migration & "-up.sql")
|
||||
info migration.name
|
||||
|
||||
if not filename.fileExists:
|
||||
pgConn.rollbackWithErr "Can not find UP file for " & migration &
|
||||
". Expected '" & filename & "'."
|
||||
if not migration.upPath.fileExists:
|
||||
pgConn.rollbackWithErr "Can not find UP file for " & migration.name &
|
||||
". Expected '" & migration.upPath & "'."
|
||||
|
||||
let statements = filename.readStatements
|
||||
let statements = migration.upPath.readStatements
|
||||
|
||||
try:
|
||||
for statement in statements:
|
||||
pgConn.exec(statement)
|
||||
pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration)
|
||||
pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration.name)
|
||||
except DbError:
|
||||
pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
|
||||
pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
|
||||
getCurrentExceptionMsg()
|
||||
|
||||
migrationsRun.add(migration)
|
||||
@ -160,27 +179,28 @@ proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[strin
|
||||
|
||||
return migrationsRun
|
||||
|
||||
proc down*(pgConn: DbConn, config: DbMigrateConfig, migrationsToDown: seq[string]): seq[string] =
|
||||
var migrationsDowned = newSeq[string]()
|
||||
proc down*(
|
||||
pgConn: DbConn,
|
||||
config: DbMigrateConfig,
|
||||
migrationsToDown: seq[MigrationEntry]): seq[MigrationEntry] =
|
||||
var migrationsDowned = newSeq[MigrationEntry]()
|
||||
|
||||
pgConn.exec(sql"BEGIN")
|
||||
|
||||
for migration in migrationsToDown:
|
||||
info migration
|
||||
info migration.name
|
||||
|
||||
let filename = joinPath(config.sqlDir, migration & "-down.sql")
|
||||
if not migration.downPath.fileExists:
|
||||
pgConn.rollbackWithErr "Can not find DOWN file for " & migration.name &
|
||||
". Expected '" & migration.downPath & "'."
|
||||
|
||||
if not filename.fileExists:
|
||||
pgConn.rollbackWithErr "Can not find DOWN file for " & migration &
|
||||
". Expected '" & filename & "'."
|
||||
|
||||
let statements = filename.readStatements
|
||||
let statements = migration.downPath.readStatements
|
||||
|
||||
try:
|
||||
for statement in statements: pgConn.exec(statement)
|
||||
pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration)
|
||||
pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration.name)
|
||||
except DbError:
|
||||
pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
|
||||
pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
|
||||
getCurrentExceptionMsg()
|
||||
|
||||
migrationsDowned.add(migration)
|
||||
@ -215,7 +235,7 @@ Options:
|
||||
"""
|
||||
|
||||
# Parse arguments
|
||||
let args = docopt(doc, version = "db-migrate (Nim) 0.2.8\nhttps://git.jdb-software.com/jdb/db-migrate")
|
||||
let args = docopt(doc, version = "db-migrate (Nim) 0.3.0\nhttps://git.jdb-software.com/jdb/db-migrate")
|
||||
|
||||
let exitErr = proc(msg: string): void =
|
||||
fatal("db_migrate: " & msg)
|
||||
@ -243,20 +263,22 @@ Options:
|
||||
else: logging.setLogFilter(config.logLevel)
|
||||
|
||||
# Check for migrations directory
|
||||
if not dirExists config.sqlDir:
|
||||
try:
|
||||
warn "SQL directory '" & config.sqlDir &
|
||||
"' does not exist and will be created."
|
||||
createDir config.sqlDir
|
||||
except IOError:
|
||||
exitErr "Unable to create directory: " & config.sqlDir & ":\L\T" & getCurrentExceptionMsg()
|
||||
for sqlDir in config.sqlDirs:
|
||||
if not dirExists sqlDir:
|
||||
try:
|
||||
warn "SQL directory '" & sqlDir &
|
||||
"' does not exist and will be created."
|
||||
createDir sqlDir
|
||||
except IOError:
|
||||
exitErr "Unable to create directory: " & sqlDir & ":\L\T" & getCurrentExceptionMsg()
|
||||
|
||||
# Execute commands
|
||||
if args["create"]:
|
||||
try:
|
||||
let filesCreated = createMigration(config, $args["<migration-name>"])
|
||||
let newMigration = createMigration(config, $args["<migration-name>"])
|
||||
info "Created new migration files:"
|
||||
for filename in filesCreated: info "\t" & filename
|
||||
info "\t" & newMigration.upPath
|
||||
info "\t" & newMigration.downPath
|
||||
except IOError:
|
||||
exitErr "Unable to create migration scripts: " & getCurrentExceptionMsg()
|
||||
|
||||
@ -269,7 +291,7 @@ Options:
|
||||
|
||||
pgConn.ensureMigrationsTableExists
|
||||
|
||||
let (run, notRun, missing) = diffMigrations(pgConn, config)
|
||||
let (available, run, notRun, missing) = diffMigrations(pgConn, config)
|
||||
|
||||
# Make sure we have no gaps (database is in an unknown state)
|
||||
if missing.len > 0:
|
||||
@ -288,7 +310,8 @@ Options:
|
||||
elif args["down"]:
|
||||
try:
|
||||
let count = if args["<count>"]: parseInt($args["<count>"]) else: 1
|
||||
let toRun = if count < run.len: run.reversed[0..<count] else: run.reversed
|
||||
let toRunNames = if count < run.len: run.reversed[0..<count] else: run.reversed
|
||||
let toRun = toRunNames.mapIt(available[it])
|
||||
let migrationsRun = pgConn.down(config, toRun)
|
||||
info "Went down " & $(migrationsRun.len) & "."
|
||||
except DbError:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user