jdb/db-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow multiple SQL directories (to support, for example, test configurations).

Browse Source
This commit is contained in:
Jonathan Bernard 2021-07-19 22:57:47 -05:00
parent 6837e5448b
commit 7cf53a4702
2 changed files with 86 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
db_migrate.nimble
View File

@ -1,7 +1,7 @@
# Package # Package
bin = @["db_migrate"] bin = @["db_migrate"]
version = "0.2.8" version = "0.3.0"
author = "Jonathan Bernard" author = "Jonathan Bernard"
description = "Simple tool to handle database migrations." description = "Simple tool to handle database migrations."
license = "BSD" license = "BSD"

147
src/main/nim/db_migrate.nim
View File

@ -4,10 +4,15 @@
## Simple tool to manage database migrations. ## Simple tool to manage database migrations.
import algorithm, db_postgres, docopt, json, logging, os, sequtils, sets, import algorithm, db_postgres, docopt, json, logging, os, sequtils, sets,
strutils, times strutils, tables, times
type type
DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: Level ] DbMigrateConfig* = object
driver, connectionString: string
sqlDirs: seq[string]
logLevel: Level
MigrationEntry* = tuple[name, upPath, downPath: string]
proc ensureMigrationsTableExists(conn: DbConn): void = proc ensureMigrationsTableExists(conn: DbConn): void =
let tableCount = conn.getValue(sql""" let tableCount = conn.getValue(sql"""
@ -38,33 +43,35 @@ proc loadConfig*(filename: string): DbMigrateConfig =
let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper) let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper)
logLevel = if idx == -1: lvlInfo else: (Level)(idx) logLevel = if idx == -1: lvlInfo else: (Level)(idx)
return ( return DbMigrateConfig(
driver: driver:
if existsEnv("DATABASE_DRIVER"): $getEnv("DATABASE_DRIVER") if existsEnv("DATABASE_DRIVER"): $getEnv("DATABASE_DRIVER")
elif cfg.hasKey("driver"): cfg["driver"].getStr elif cfg.hasKey("driver"): cfg["driver"].getStr
else: "postres", else: "postres",
sqlDir:
if existsEnv("MIGRATIONS_DIR"): $getEnv("MIGRATIONS_DIR")
elif cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr
else: "migrations",
connectionString: connectionString:
if existsEnv("DATABASE_URL"): $getEnv("DATABASE_URL") if existsEnv("DATABASE_URL"): $getEnv("DATABASE_URL")
elif cfg.hasKey("connectionString"): cfg["connectionString"].getStr elif cfg.hasKey("connectionString"): cfg["connectionString"].getStr
else: "", else: "",
sqlDirs:
if existsEnv("MIGRATIONS_DIRS"): getEnv("MIGRATIONS_DIRS").split(';')
elif cfg.hasKey("sqlDirs"): cfg["sqlDirs"].getElems.mapIt(it.getStr)
else: @["migrations"],
logLevel: logLevel) logLevel: logLevel)
proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[string] = proc createMigration*(config: DbMigrateConfig, migrationName: string): MigrationEntry =
## Create a new set of database migration files. ## Create a new set of database migration files.
let timestamp = now().format("yyyyMMddHHmmss") let timestamp = now().format("yyyyMMddHHmmss")
let filenamePrefix = timestamp & "-" & migrationName let filenamePrefix = timestamp & "-" & migrationName
let upFilename = joinPath(config.sqlDir, filenamePrefix & "-up.sql") let migration = (
let downFilename = joinPath(config.sqlDir, filenamePrefix & "-down.sql") name: filenamePrefix & "-up.sql",
upPath: joinPath(config.sqlDirs[0], filenamePrefix & "-up.sql"),
downPath: joinPath(config.sqlDirs[0], filenamePrefix & "-down.sql"))
let scriptDesc = migrationName & " (" & timestamp & ")" let scriptDesc = migrationName & " (" & timestamp & ")"
let upFile = open(upFilename, fmWrite) let upFile = open(migration.upPath, fmWrite)
let downFile = open(downFilename, fmWrite) let downFile = open(migration.downPath, fmWrite)
upFile.writeLine "-- UP script for " & scriptDesc upFile.writeLine "-- UP script for " & scriptDesc
downFile.writeLine "-- DOWN script for " & scriptDesc downFile.writeLine "-- DOWN script for " & scriptDesc
@ -72,10 +79,15 @@ proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[strin
upFile.close() upFile.close()
downFile.close() downFile.close()
return @[upFilename, downFilename] return migration
proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig): proc diffMigrations*(
tuple[ run, notRun, missing: seq[string] ] = pgConn: DbConn,
config: DbMigrateConfig
): tuple[
available: TableRef[string, MigrationEntry],
run: seq[string],
notRun, missing: seq[MigrationEntry] ] =
# Query the database to find out what migrations have been run. # Query the database to find out what migrations have been run.
var migrationsRun = initHashSet[string]() var migrationsRun = initHashSet[string]()
@ -84,32 +96,37 @@ proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig):
migrationsRun.incl(row[1]) migrationsRun.incl(row[1])
# Inspect the filesystem to see what migrations are available. # Inspect the filesystem to see what migrations are available.
var migrationsAvailable = initHashSet[string]() var migrationsAvailable = newTable[string, MigrationEntry]()
for filePath in walkFiles(joinPath(config.sqlDir, "*.sql")): for sqlDir in config.sqlDirs:
var migrationName = filePath.extractFilename for filePath in walkFiles(joinPath(sqlDir, "*.sql")):
migrationName.removeSuffix("-up.sql") var migrationName = filePath.extractFilename
migrationName.removeSuffix("-down.sql") migrationName.removeSuffix("-up.sql")
migrationsAvailable.incl(migrationName) migrationName.removeSuffix("-down.sql")
migrationsAvailable[migrationName] = (
name: migrationName,
upPath: joinPath(sqlDir, migrationName) & "-up.sql",
downPath: joinPath(sqlDir, migrationName) & "-down.sql")
# Diff with the list of migrations that we have in our migrations # Diff with the list of migrations that we have in our migrations
# directory. # directory.
let migrationsInOrder = let migrationsInOrder =
toSeq(migrationsAvailable.items).sorted(system.cmp) toSeq(migrationsAvailable.keys).sorted(system.cmp)
var migrationsNotRun = newSeq[string]() var migrationsNotRun = newSeq[MigrationEntry]()
var missingMigrations = newSeq[string]() var missingMigrations = newSeq[MigrationEntry]()
for migration in migrationsInOrder: for migName in migrationsInOrder:
if not migrationsRun.contains(migration): if not migrationsRun.contains(migName):
migrationsNotRun.add(migration) migrationsNotRun.add(migrationsAvailable[migName])
# if we've already seen some migrations that have not been run, but this # if we've already seen some migrations that have not been run, but this
# one has been, that means we have a gap and are missing migrations # one has been, that means we have a gap and are missing migrations
elif migrationsNotRun.len > 0: elif migrationsNotRun.len > 0:
missingMigrations.add(migrationsNotRun) missingMigrations.add(migrationsNotRun)
migrationsNotRun = newSeq[string]() migrationsNotRun = newSeq[MigrationEntry]()
return (run: toSeq(migrationsRun.items).sorted(system.cmp), return (available: migrationsAvailable,
run: toSeq(migrationsRun.items).sorted(system.cmp),
notRun: migrationsNotRun, notRun: migrationsNotRun,
missing: missingMigrations) missing: missingMigrations)
@ -130,28 +147,30 @@ proc readStatements*(filename: string): seq[SqlQuery] =
if stmt.strip.len > 0: result.add(sql(stmt)) if stmt.strip.len > 0: result.add(sql(stmt))
proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[string] = proc up*(
var migrationsRun = newSeq[string]() pgConn: DbConn,
config: DbMigrateConfig,
toRun: seq[MigrationEntry]): seq[MigrationEntry] =
var migrationsRun = newSeq[MigrationEntry]()
# Begin a transaction. # Begin a transaction.
pgConn.exec(sql"BEGIN") pgConn.exec(sql"BEGIN")
# Apply each of the migrations. # Apply each of the migrations.
for migration in toRun: for migration in toRun:
info migration info migration.name
let filename = joinPath(config.sqlDir, migration & "-up.sql")
if not filename.fileExists: if not migration.upPath.fileExists:
pgConn.rollbackWithErr "Can not find UP file for " & migration & pgConn.rollbackWithErr "Can not find UP file for " & migration.name &
". Expected '" & filename & "'." ". Expected '" & migration.upPath & "'."
let statements = filename.readStatements let statements = migration.upPath.readStatements
try: try:
for statement in statements: for statement in statements:
pgConn.exec(statement) pgConn.exec(statement)
pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration) pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration.name)
except DbError: except DbError:
pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" & pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
getCurrentExceptionMsg() getCurrentExceptionMsg()
migrationsRun.add(migration) migrationsRun.add(migration)
@ -160,27 +179,28 @@ proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[strin
return migrationsRun return migrationsRun
proc down*(pgConn: DbConn, config: DbMigrateConfig, migrationsToDown: seq[string]): seq[string] = proc down*(
var migrationsDowned = newSeq[string]() pgConn: DbConn,
config: DbMigrateConfig,
migrationsToDown: seq[MigrationEntry]): seq[MigrationEntry] =
var migrationsDowned = newSeq[MigrationEntry]()
pgConn.exec(sql"BEGIN") pgConn.exec(sql"BEGIN")
for migration in migrationsToDown: for migration in migrationsToDown:
info migration info migration.name
let filename = joinPath(config.sqlDir, migration & "-down.sql") if not migration.downPath.fileExists:
pgConn.rollbackWithErr "Can not find DOWN file for " & migration.name &
". Expected '" & migration.downPath & "'."
if not filename.fileExists: let statements = migration.downPath.readStatements
pgConn.rollbackWithErr "Can not find DOWN file for " & migration &
". Expected '" & filename & "'."
let statements = filename.readStatements
try: try:
for statement in statements: pgConn.exec(statement) for statement in statements: pgConn.exec(statement)
pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration) pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration.name)
except DbError: except DbError:
pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" & pgConn.rollbackWithErr "Migration '" & migration.name & "' failed:\n\t" &
getCurrentExceptionMsg() getCurrentExceptionMsg()
migrationsDowned.add(migration) migrationsDowned.add(migration)
@ -215,7 +235,7 @@ Options:
""" """
# Parse arguments # Parse arguments
let args = docopt(doc, version = "db-migrate (Nim) 0.2.8\nhttps://git.jdb-software.com/jdb/db-migrate") let args = docopt(doc, version = "db-migrate (Nim) 0.3.0\nhttps://git.jdb-software.com/jdb/db-migrate")
let exitErr = proc(msg: string): void = let exitErr = proc(msg: string): void =
fatal("db_migrate: " & msg) fatal("db_migrate: " & msg)
@ -243,20 +263,22 @@ Options:
else: logging.setLogFilter(config.logLevel) else: logging.setLogFilter(config.logLevel)
# Check for migrations directory # Check for migrations directory
if not dirExists config.sqlDir: for sqlDir in config.sqlDirs:
try: if not dirExists sqlDir:
warn "SQL directory '" & config.sqlDir & try:
"' does not exist and will be created." warn "SQL directory '" & sqlDir &
createDir config.sqlDir "' does not exist and will be created."
except IOError: createDir sqlDir
exitErr "Unable to create directory: " & config.sqlDir & ":\L\T" & getCurrentExceptionMsg() except IOError:
exitErr "Unable to create directory: " & sqlDir & ":\L\T" & getCurrentExceptionMsg()
# Execute commands # Execute commands
if args["create"]: if args["create"]:
try: try:
let filesCreated = createMigration(config, $args["<migration-name>"]) let newMigration = createMigration(config, $args["<migration-name>"])
info "Created new migration files:" info "Created new migration files:"
for filename in filesCreated: info "\t" & filename info "\t" & newMigration.upPath
info "\t" & newMigration.downPath
except IOError: except IOError:
exitErr "Unable to create migration scripts: " & getCurrentExceptionMsg() exitErr "Unable to create migration scripts: " & getCurrentExceptionMsg()
@ -269,7 +291,7 @@ Options:
pgConn.ensureMigrationsTableExists pgConn.ensureMigrationsTableExists
let (run, notRun, missing) = diffMigrations(pgConn, config) let (available, run, notRun, missing) = diffMigrations(pgConn, config)
# Make sure we have no gaps (database is in an unknown state) # Make sure we have no gaps (database is in an unknown state)
if missing.len > 0: if missing.len > 0:
@ -288,7 +310,8 @@ Options:
elif args["down"]: elif args["down"]:
try: try:
let count = if args["<count>"]: parseInt($args["<count>"]) else: 1 let count = if args["<count>"]: parseInt($args["<count>"]) else: 1
let toRun = if count < run.len: run.reversed[0..<count] else: run.reversed let toRunNames = if count < run.len: run.reversed[0..<count] else: run.reversed
let toRun = toRunNames.mapIt(available[it])
let migrationsRun = pgConn.down(config, toRun) let migrationsRun = pgConn.down(config, toRun)
info "Went down " & $(migrationsRun.len) & "." info "Went down " & $(migrationsRun.len) & "."
except DbError: except DbError: