Change line feeds to unix.
This commit is contained in:
Jonathan Bernard
536
db_migrate.nim
536
db_migrate.nim
@@ -1,268 +1,268 @@
|
|||||||
## DB Migrate
|
## DB Migrate
|
||||||
## ==========
|
## ==========
|
||||||
##
|
##
|
||||||
## Simple tool to manage database migrations.
|
## Simple tool to manage database migrations.
|
||||||
|
|
||||||
import algorithm, json, times, os, strutils, docopt, db_postgres, sets,
|
import algorithm, json, times, os, strutils, docopt, db_postgres, sets,
|
||||||
sequtils, logging
|
sequtils, logging
|
||||||
|
|
||||||
type
|
type
|
||||||
DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: Level ]
|
DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: Level ]
|
||||||
|
|
||||||
proc createMigrationsTable(conn: DbConn): void =
|
proc createMigrationsTable(conn: DbConn): void =
|
||||||
conn.exec(sql("""
|
conn.exec(sql("""
|
||||||
CREATE TABLE IF NOT EXISTS migrations (
|
CREATE TABLE IF NOT EXISTS migrations (
|
||||||
id SERIAL PRIMARY KEY,
|
id SERIAL PRIMARY KEY,
|
||||||
name VARCHAR NOT NULL,
|
name VARCHAR NOT NULL,
|
||||||
run_at TIMESTAMP NOT NULL DEFAULT NOW())"""))
|
run_at TIMESTAMP NOT NULL DEFAULT NOW())"""))
|
||||||
|
|
||||||
proc rollbackWithErr (pgConn: DbConn, errMsg: string): void =
|
proc rollbackWithErr (pgConn: DbConn, errMsg: string): void =
|
||||||
pgConn.exec(sql"ROLLBACK")
|
pgConn.exec(sql"ROLLBACK")
|
||||||
dbError(errMsg)
|
dbError(errMsg)
|
||||||
|
|
||||||
proc loadConfig*(filename: string): DbMigrateConfig =
|
proc loadConfig*(filename: string): DbMigrateConfig =
|
||||||
## Load DbMigrateConfig from a file.
|
## Load DbMigrateConfig from a file.
|
||||||
let cfg = json.parseFile(filename)
|
let cfg = json.parseFile(filename)
|
||||||
|
|
||||||
var logLevel: Level
|
var logLevel: Level
|
||||||
if cfg.hasKey("logLevel"):
|
if cfg.hasKey("logLevel"):
|
||||||
let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper)
|
let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper)
|
||||||
logLevel = if idx == -1: lvlInfo else: (Level)(idx)
|
logLevel = if idx == -1: lvlInfo else: (Level)(idx)
|
||||||
|
|
||||||
return (
|
return (
|
||||||
driver: if cfg.hasKey("driver"): cfg["driver"].getStr else: "postres",
|
driver: if cfg.hasKey("driver"): cfg["driver"].getStr else: "postres",
|
||||||
sqlDir: if cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr else: "migrations",
|
sqlDir: if cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr else: "migrations",
|
||||||
connectionString: cfg["connectionString"].getStr,
|
connectionString: cfg["connectionString"].getStr,
|
||||||
logLevel: logLevel)
|
logLevel: logLevel)
|
||||||
|
|
||||||
proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[string] =
|
proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[string] =
|
||||||
## Create a new set of database migration files.
|
## Create a new set of database migration files.
|
||||||
let timestamp = getTime().getLocalTime().format("yyyyMMddHHmmss")
|
let timestamp = getTime().getLocalTime().format("yyyyMMddHHmmss")
|
||||||
let filenamePrefix = timestamp & "-" & migrationName
|
let filenamePrefix = timestamp & "-" & migrationName
|
||||||
|
|
||||||
let upFilename = joinPath(config.sqlDir, filenamePrefix & "-up.sql")
|
let upFilename = joinPath(config.sqlDir, filenamePrefix & "-up.sql")
|
||||||
let downFilename = joinPath(config.sqlDir, filenamePrefix & "-down.sql")
|
let downFilename = joinPath(config.sqlDir, filenamePrefix & "-down.sql")
|
||||||
|
|
||||||
let scriptDesc = migrationName & " (" & timestamp & ")"
|
let scriptDesc = migrationName & " (" & timestamp & ")"
|
||||||
|
|
||||||
let upFile = open(upFilename, fmWrite)
|
let upFile = open(upFilename, fmWrite)
|
||||||
let downFile = open(downFilename, fmWrite)
|
let downFile = open(downFilename, fmWrite)
|
||||||
|
|
||||||
upFile.writeLine "-- UP script for " & scriptDesc
|
upFile.writeLine "-- UP script for " & scriptDesc
|
||||||
downFile.writeLine "-- DOWN script for " & scriptDesc
|
downFile.writeLine "-- DOWN script for " & scriptDesc
|
||||||
|
|
||||||
upFile.close()
|
upFile.close()
|
||||||
downFile.close()
|
downFile.close()
|
||||||
|
|
||||||
return @[upFilename, downFilename]
|
return @[upFilename, downFilename]
|
||||||
|
|
||||||
proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig):
|
proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig):
|
||||||
tuple[ run, notRun, missing: seq[string] ] =
|
tuple[ run, notRun, missing: seq[string] ] =
|
||||||
|
|
||||||
# Query the database to find out what migrations have been run.
|
# Query the database to find out what migrations have been run.
|
||||||
var migrationsRun = initSet[string]()
|
var migrationsRun = initSet[string]()
|
||||||
|
|
||||||
for row in pgConn.fastRows(sql"SELECT * FROM migrations ORDER BY name", @[]):
|
for row in pgConn.fastRows(sql"SELECT * FROM migrations ORDER BY name", @[]):
|
||||||
migrationsRun.incl(row[1])
|
migrationsRun.incl(row[1])
|
||||||
|
|
||||||
# Inspect the filesystem to see what migrations are available.
|
# Inspect the filesystem to see what migrations are available.
|
||||||
var migrationsAvailable = initSet[string]()
|
var migrationsAvailable = initSet[string]()
|
||||||
for filePath in walkFiles(joinPath(config.sqlDir, "*.sql")):
|
for filePath in walkFiles(joinPath(config.sqlDir, "*.sql")):
|
||||||
var migrationName = filePath.extractFilename
|
var migrationName = filePath.extractFilename
|
||||||
migrationName.removeSuffix("-up.sql")
|
migrationName.removeSuffix("-up.sql")
|
||||||
migrationName.removeSuffix("-down.sql")
|
migrationName.removeSuffix("-down.sql")
|
||||||
migrationsAvailable.incl(migrationName)
|
migrationsAvailable.incl(migrationName)
|
||||||
|
|
||||||
# Diff with the list of migrations that we have in our migrations
|
# Diff with the list of migrations that we have in our migrations
|
||||||
# directory.
|
# directory.
|
||||||
let migrationsInOrder =
|
let migrationsInOrder =
|
||||||
toSeq(migrationsAvailable.items).sorted(system.cmp)
|
toSeq(migrationsAvailable.items).sorted(system.cmp)
|
||||||
|
|
||||||
var migrationsNotRun = newSeq[string]()
|
var migrationsNotRun = newSeq[string]()
|
||||||
var missingMigrations = newSeq[string]()
|
var missingMigrations = newSeq[string]()
|
||||||
|
|
||||||
for migration in migrationsInOrder:
|
for migration in migrationsInOrder:
|
||||||
if not migrationsRun.contains(migration):
|
if not migrationsRun.contains(migration):
|
||||||
migrationsNotRun.add(migration)
|
migrationsNotRun.add(migration)
|
||||||
|
|
||||||
# if we've already seen some migrations that have not been run, but this
|
# if we've already seen some migrations that have not been run, but this
|
||||||
# one has been, that means we have a gap and are missing migrations
|
# one has been, that means we have a gap and are missing migrations
|
||||||
elif migrationsNotRun.len > 0:
|
elif migrationsNotRun.len > 0:
|
||||||
missingMigrations.add(migrationsNotRun)
|
missingMigrations.add(migrationsNotRun)
|
||||||
migrationsNotRun = newSeq[string]()
|
migrationsNotRun = newSeq[string]()
|
||||||
|
|
||||||
return (run: toSeq(migrationsRun.items).sorted(system.cmp),
|
return (run: toSeq(migrationsRun.items).sorted(system.cmp),
|
||||||
notRun: migrationsNotRun,
|
notRun: migrationsNotRun,
|
||||||
missing: missingMigrations)
|
missing: missingMigrations)
|
||||||
|
|
||||||
proc readStatements*(filename: string): seq[SqlQuery] =
|
proc readStatements*(filename: string): seq[SqlQuery] =
|
||||||
let migrationSql = filename.readFile
|
let migrationSql = filename.readFile
|
||||||
result = migrationSql.split(';').
|
result = migrationSql.split(';').
|
||||||
filter(proc(st: string): bool = st.strip.len > 0 and not st.startsWith("--")).
|
filter(proc(st: string): bool = st.strip.len > 0 and not st.startsWith("--")).
|
||||||
map(proc(st: string): SqlQuery = sql(st & ";"))
|
map(proc(st: string): SqlQuery = sql(st & ";"))
|
||||||
|
|
||||||
proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[string] =
|
proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[string] =
|
||||||
var migrationsRun = newSeq[string]()
|
var migrationsRun = newSeq[string]()
|
||||||
# Begin a transaction.
|
# Begin a transaction.
|
||||||
pgConn.exec(sql"BEGIN")
|
pgConn.exec(sql"BEGIN")
|
||||||
|
|
||||||
# Apply each of the migrations.
|
# Apply each of the migrations.
|
||||||
for migration in toRun:
|
for migration in toRun:
|
||||||
info migration
|
info migration
|
||||||
let filename = joinPath(config.sqlDir, migration & "-up.sql")
|
let filename = joinPath(config.sqlDir, migration & "-up.sql")
|
||||||
|
|
||||||
if not filename.fileExists:
|
if not filename.fileExists:
|
||||||
pgConn.rollbackWithErr "Can not find UP file for " & migration &
|
pgConn.rollbackWithErr "Can not find UP file for " & migration &
|
||||||
". Expected '" & filename & "'."
|
". Expected '" & filename & "'."
|
||||||
|
|
||||||
let statements = filename.readStatements
|
let statements = filename.readStatements
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for statement in statements: pgConn.exec(statement)
|
for statement in statements: pgConn.exec(statement)
|
||||||
pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration)
|
pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration)
|
||||||
except DbError:
|
except DbError:
|
||||||
pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
|
pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
|
||||||
getCurrentExceptionMsg()
|
getCurrentExceptionMsg()
|
||||||
|
|
||||||
migrationsRun.add(migration)
|
migrationsRun.add(migration)
|
||||||
|
|
||||||
pgConn.exec(sql"COMMIT")
|
pgConn.exec(sql"COMMIT")
|
||||||
|
|
||||||
return migrationsRun
|
return migrationsRun
|
||||||
|
|
||||||
proc down*(pgConn: DbConn, config: DbMigrateConfig, migrationsToDown: seq[string]): seq[string] =
|
proc down*(pgConn: DbConn, config: DbMigrateConfig, migrationsToDown: seq[string]): seq[string] =
|
||||||
var migrationsDowned = newSeq[string]()
|
var migrationsDowned = newSeq[string]()
|
||||||
|
|
||||||
pgConn.exec(sql"BEGIN")
|
pgConn.exec(sql"BEGIN")
|
||||||
|
|
||||||
for migration in migrationsToDown:
|
for migration in migrationsToDown:
|
||||||
info migration
|
info migration
|
||||||
|
|
||||||
let filename = joinPath(config.sqlDir, migration & "-down.sql")
|
let filename = joinPath(config.sqlDir, migration & "-down.sql")
|
||||||
|
|
||||||
if not filename.fileExists:
|
if not filename.fileExists:
|
||||||
pgConn.rollbackWithErr "Can not find DOWN file for " & migration &
|
pgConn.rollbackWithErr "Can not find DOWN file for " & migration &
|
||||||
". Expected '" & filename & "'."
|
". Expected '" & filename & "'."
|
||||||
|
|
||||||
let statements = filename.readStatements
|
let statements = filename.readStatements
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for statement in statements: pgConn.exec(statement)
|
for statement in statements: pgConn.exec(statement)
|
||||||
pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration)
|
pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration)
|
||||||
except DbError:
|
except DbError:
|
||||||
pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
|
pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" &
|
||||||
getCurrentExceptionMsg()
|
getCurrentExceptionMsg()
|
||||||
|
|
||||||
migrationsDowned.add(migration)
|
migrationsDowned.add(migration)
|
||||||
|
|
||||||
pgConn.exec(sql"COMMIT")
|
pgConn.exec(sql"COMMIT")
|
||||||
return migrationsDowned
|
return migrationsDowned
|
||||||
|
|
||||||
when isMainModule:
|
when isMainModule:
|
||||||
let doc = """
|
let doc = """
|
||||||
Usage:
|
Usage:
|
||||||
db_migrate [options] create <migration-name>
|
db_migrate [options] create <migration-name>
|
||||||
db_migrate [options] up [<count>]
|
db_migrate [options] up [<count>]
|
||||||
db_migrate [options] down [<count>]
|
db_migrate [options] down [<count>]
|
||||||
db_migrate [options] init <schema-name>
|
db_migrate [options] init <schema-name>
|
||||||
db_migrate (-V | --version)
|
db_migrate (-V | --version)
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
|
|
||||||
-c --config <config-file> Use the given configuration file (defaults to
|
-c --config <config-file> Use the given configuration file (defaults to
|
||||||
"database.json").
|
"database.json").
|
||||||
|
|
||||||
-q --quiet Suppress log information.
|
-q --quiet Suppress log information.
|
||||||
|
|
||||||
-v --verbose Print detailed log information.
|
-v --verbose Print detailed log information.
|
||||||
|
|
||||||
--very-verbose Print very detailed log information.
|
--very-verbose Print very detailed log information.
|
||||||
|
|
||||||
-V --version Print the tools version information.
|
-V --version Print the tools version information.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Parse arguments
|
# Parse arguments
|
||||||
let args = docopt(doc, version = "db-migrate 0.2.0")
|
let args = docopt(doc, version = "db-migrate 0.2.0")
|
||||||
|
|
||||||
let exitErr = proc(msg: string): void =
|
let exitErr = proc(msg: string): void =
|
||||||
fatal("db_migrate: " & msg)
|
fatal("db_migrate: " & msg)
|
||||||
quit(QuitFailure)
|
quit(QuitFailure)
|
||||||
|
|
||||||
# Load configuration file
|
# Load configuration file
|
||||||
let configFilename =
|
let configFilename =
|
||||||
if args["--config"]: $args["<config-file>"]
|
if args["--config"]: $args["<config-file>"]
|
||||||
else: "database.json"
|
else: "database.json"
|
||||||
|
|
||||||
var config: DbMigrateConfig
|
var config: DbMigrateConfig
|
||||||
try:
|
try:
|
||||||
config = loadConfig(configFilename)
|
config = loadConfig(configFilename)
|
||||||
except IOError:
|
except IOError:
|
||||||
exitErr "Cannot open config file: " & configFilename
|
exitErr "Cannot open config file: " & configFilename
|
||||||
except:
|
except:
|
||||||
exitErr "Error parsing config file: " & configFilename & "\L\t" & getCurrentExceptionMsg()
|
exitErr "Error parsing config file: " & configFilename & "\L\t" & getCurrentExceptionMsg()
|
||||||
|
|
||||||
|
|
||||||
logging.addHandler(newConsoleLogger())
|
logging.addHandler(newConsoleLogger())
|
||||||
if args["--quiet"]: logging.setLogFilter(lvlError)
|
if args["--quiet"]: logging.setLogFilter(lvlError)
|
||||||
elif args["--very-verbose"]: logging.setLogFilter(lvlAll)
|
elif args["--very-verbose"]: logging.setLogFilter(lvlAll)
|
||||||
elif args["--verbose"]: logging.setlogFilter(lvlDebug)
|
elif args["--verbose"]: logging.setlogFilter(lvlDebug)
|
||||||
else: logging.setLogFilter(config.logLevel)
|
else: logging.setLogFilter(config.logLevel)
|
||||||
|
|
||||||
# Check for migrations directory
|
# Check for migrations directory
|
||||||
if not existsDir config.sqlDir:
|
if not existsDir config.sqlDir:
|
||||||
try:
|
try:
|
||||||
warn "SQL directory '" & config.sqlDir &
|
warn "SQL directory '" & config.sqlDir &
|
||||||
"' does not exist and will be created."
|
"' does not exist and will be created."
|
||||||
createDir config.sqlDir
|
createDir config.sqlDir
|
||||||
except IOError:
|
except IOError:
|
||||||
exitErr "Unable to create directory: " & config.sqlDir & ":\L\T" & getCurrentExceptionMsg()
|
exitErr "Unable to create directory: " & config.sqlDir & ":\L\T" & getCurrentExceptionMsg()
|
||||||
|
|
||||||
# Execute commands
|
# Execute commands
|
||||||
if args["create"]:
|
if args["create"]:
|
||||||
try:
|
try:
|
||||||
let filesCreated = createMigration(config, $args["<migration-name>"])
|
let filesCreated = createMigration(config, $args["<migration-name>"])
|
||||||
info "Created new migration files:"
|
info "Created new migration files:"
|
||||||
for filename in filesCreated: info "\t" & filename
|
for filename in filesCreated: info "\t" & filename
|
||||||
except IOError:
|
except IOError:
|
||||||
exitErr "Unable to create migration scripts: " & getCurrentExceptionMsg()
|
exitErr "Unable to create migration scripts: " & getCurrentExceptionMsg()
|
||||||
|
|
||||||
else:
|
else:
|
||||||
let pgConn: DbConn =
|
let pgConn: DbConn =
|
||||||
try: open("", "", "", config.connectionString)
|
try: open("", "", "", config.connectionString)
|
||||||
except DbError:
|
except DbError:
|
||||||
exitErr "Unable to connect to the database."
|
exitErr "Unable to connect to the database."
|
||||||
(DbConn)(nil)
|
(DbConn)(nil)
|
||||||
|
|
||||||
pgConn.createMigrationsTable
|
pgConn.createMigrationsTable
|
||||||
|
|
||||||
let (run, notRun, missing) = diffMigrations(pgConn, config)
|
let (run, notRun, missing) = diffMigrations(pgConn, config)
|
||||||
|
|
||||||
# Make sure we have no gaps (database is in an unknown state)
|
# Make sure we have no gaps (database is in an unknown state)
|
||||||
if missing.len > 0:
|
if missing.len > 0:
|
||||||
exitErr "Database is in an inconsistent state. Migrations have been " &
|
exitErr "Database is in an inconsistent state. Migrations have been " &
|
||||||
"run that are not sequential."
|
"run that are not sequential."
|
||||||
|
|
||||||
if args["up"]:
|
if args["up"]:
|
||||||
try:
|
try:
|
||||||
let count = if args["<count>"]: parseInt($args["<count>"]) else: high(int)
|
let count = if args["<count>"]: parseInt($args["<count>"]) else: high(int)
|
||||||
let toRun = if count < notRun.len: notRun[0..<count] else: notRun
|
let toRun = if count < notRun.len: notRun[0..<count] else: notRun
|
||||||
let migrationsRun = pgConn.up(config, toRun)
|
let migrationsRun = pgConn.up(config, toRun)
|
||||||
if count < high(int): info "Went up " & $(migrationsRun.len) & "."
|
if count < high(int): info "Went up " & $(migrationsRun.len) & "."
|
||||||
except DbError:
|
except DbError:
|
||||||
exitErr "Unable to migrate database: " & getCurrentExceptionMsg()
|
exitErr "Unable to migrate database: " & getCurrentExceptionMsg()
|
||||||
|
|
||||||
elif args["down"]:
|
elif args["down"]:
|
||||||
try:
|
try:
|
||||||
let count = if args["<count>"]: parseInt($args["<count>"]) else: 1
|
let count = if args["<count>"]: parseInt($args["<count>"]) else: 1
|
||||||
let toRun = if count < run.len: run.reversed[0..<count] else: run.reversed
|
let toRun = if count < run.len: run.reversed[0..<count] else: run.reversed
|
||||||
let migrationsRun = pgConn.down(config, toRun)
|
let migrationsRun = pgConn.down(config, toRun)
|
||||||
info "Went down " & $(migrationsRun.len) & "."
|
info "Went down " & $(migrationsRun.len) & "."
|
||||||
except DbError:
|
except DbError:
|
||||||
exitErr "Unable to migrate database: " & getCurrentExceptionMsg()
|
exitErr "Unable to migrate database: " & getCurrentExceptionMsg()
|
||||||
|
|
||||||
elif args["init"]: discard
|
elif args["init"]: discard
|
||||||
|
|
||||||
let newResults = diffMigrations(pgConn, config)
|
let newResults = diffMigrations(pgConn, config)
|
||||||
if newResults.notRun.len > 0:
|
if newResults.notRun.len > 0:
|
||||||
info "Database is behind by " & $(newResults.notRun.len) & " migrations."
|
info "Database is behind by " & $(newResults.notRun.len) & " migrations."
|
||||||
else: info "Database is up to date."
|
else: info "Database is up to date."
|
||||||
|
|||||||
Reference in New Issue
Block a user