From 02e326e66155aab43e000fe2d3824215dc53b5f0 Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Sun, 7 Feb 2016 07:37:06 -0600 Subject: [PATCH] Added logging, implemented diff, up, and down. --- db_migrate.nim | 193 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 127 insertions(+), 66 deletions(-) diff --git a/db_migrate.nim b/db_migrate.nim index f733cb6..fdb2f03 100644 --- a/db_migrate.nim +++ b/db_migrate.nim @@ -4,11 +4,10 @@ ## Simple tool to manage database migrations. import algorithm, json, times, os, strutils, docopt, db_postgres, sets, - sequtils + sequtils, logging -type - LogLevel = enum quiet, normal, verbose, very_verbose - DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: LogLevel ] +type + DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: Level ] proc createMigrationsTable(conn: DbConn): void = conn.exec(sql(""" @@ -17,15 +16,24 @@ CREATE TABLE IF NOT EXISTS migrations ( name VARCHAR NOT NULL, run_at TIMESTAMP NOT NULL DEFAULT NOW())""")) +proc rollbackWithErr (pgConn: DbConn, errMsg: string): void = + pgConn.exec(sql"ROLLBACK") + dbError(errMsg) + proc loadConfig*(filename: string): DbMigrateConfig = - ## Load DbMigrateConfig from a file. + ## Load DbMigrateConfig from a file. let cfg = json.parseFile(filename) + var logLevel: Level + if cfg.hasKey("logLevel"): + let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper) + logLevel = if idx == -1: lvlInfo else: (Level)(idx) + return ( driver: if cfg.hasKey("driver"): cfg["driver"].getStr else: "postres", sqlDir: if cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr else: "migrations", connectionString: cfg["connectionString"].getStr, - logLevel: if cfg.hasKey("logLevel"): parseEnum[LogLevel](cfg["logLevel"].getStr) else: normal) + logLevel: logLevel) proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[string] = ## Create a new set of database migration files. @@ -56,7 +64,7 @@ proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig): for row in pgConn.fastRows(sql"SELECT * FROM migrations ORDER BY name", @[]): migrationsRun.incl(row[1]) - + # Inspect the filesystem to see what migrations are available. var migrationsAvailable = initSet[string]() for filePath in walkFiles(joinPath(config.sqlDir, "*.sql")): @@ -70,8 +78,8 @@ proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig): let migrationsInOrder = toSeq(migrationsAvailable.items).sorted(system.cmp) - var migrationsNotRun: seq[string] = @[] - var missingMigrations: seq[string] = @[] + var migrationsNotRun = newSeq[string]() + var missingMigrations = newSeq[string]() for migration in migrationsInOrder: if not migrationsRun.contains(migration): @@ -81,49 +89,75 @@ proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig): # one has been, that means we have a gap and are missing migrations elif migrationsNotRun.len > 0: missingMigrations.add(migrationsNotRun) - migrationsNotRun = @[] - - return (run: toSeq(migrationsRun.items), notRun: migrationsNotRun, missing: missingMigrations) + migrationsNotRun = newSeq[string]() -proc up*(config: DbMigrateConfig, count: int): seq[string] = - let pgConn = open("", "", "", config.connectionString) - pgConn.createMigrationsTable + return (run: toSeq(migrationsRun.items).sorted(system.cmp), + notRun: migrationsNotRun, + missing: missingMigrations) - let (run, toRun, missing) = diffMigrations(pgConn, config) - - # Make sure we have no gaps (database is in an unknown state) - if missing.len > 0: - dbError("Database is in an inconsistent state. Migrations have been " & - "run that are not sequential.") +proc readStatements*(filename: string): seq[SqlQuery] = + let migrationSql = filename.readFile + result = migrationSql.split(';'). + filter(proc(st: string): bool = st.strip.len > 0 and not st.startsWith("--")). + map(proc(st: string): SqlQuery = sql(st & ";")) - var migrationsRun: seq[string] = @[] - if toRun.len > 0: - # Begin a transaction. - pgConn.exec(sql"BEGIN") +proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[string] = + var migrationsRun = newSeq[string]() + # Begin a transaction. + pgConn.exec(sql"BEGIN") - let rollbackWithErr = proc (errMsg: string): void = - pgConn.exec(sql"ROLLBACK") - dbError(errMsg) + # Apply each of the migrations. + for migration in toRun: + info migration + let filename = joinPath(config.sqlDir, migration & "-up.sql") - # Apply each of the migrations. - for migration in toRun: - let filename = joinPath(config.sqlDir, migration & "-up.sql") + if not filename.fileExists: + pgConn.rollbackWithErr "Can not find UP file for " & migration & + ". Expected '" & filename & "'." - if not filename.fileExists: - rollbackWithErr "Can not find up file for " & migration & - ". Expected '" & filename & "'." + let statements = filename.readStatements - let migrationSql = filename.readFile - try: - pgConn.exec(sql(migrationSql)) - migrationsRun.add(migration) - except DbError: - rollbackWithErr "Migration '" & migration & "'failed:\n\t" & - getCurrentExceptionMsg() + try: + for statement in statements: pgConn.exec(statement) + pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration) + except DbError: + pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" & + getCurrentExceptionMsg() + migrationsRun.add(migration) + + pgConn.exec(sql"COMMIT") return migrationsRun - + +proc down*(pgConn: DbConn, config: DbMigrateConfig, migrationsToDown: seq[string]): seq[string] = + var migrationsDowned = newSeq[string]() + + pgConn.exec(sql"BEGIN") + + for migration in migrationsToDown: + info migration + + let filename = joinPath(config.sqlDir, migration & "-down.sql") + + if not filename.fileExists: + pgConn.rollbackWithErr "Can not find DOWN file for " & migration & + ". Expected '" & filename & "'." + + let statements = filename.readStatements + + try: + for statement in statements: pgConn.exec(statement) + pgConn.exec(sql"DELETE FROM migrations WHERE name = ?;", migration) + except DbError: + pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" & + getCurrentExceptionMsg() + + migrationsDowned.add(migration) + + pgConn.exec(sql"COMMIT") + return migrationsDowned + when isMainModule: let doc = """ Usage: @@ -135,11 +169,14 @@ Usage: Options: - -c --config Use the given configuration file (defaults to + -c --config Use the given configuration file (defaults to "database.json"). - -v --verbose Print detailed log information (use -vv to - print even more details). + -q --quiet Suppress log information. + + -v --verbose Print detailed log information. + + --very-verbose Print very detailed log information. -V --version Print the tools version information. """ @@ -148,7 +185,7 @@ Options: let args = docopt(doc, version = "db-migrate 0.2.0") let exitErr = proc(msg: string): void = - stderr.writeLine("db_migrate: " & msg) + fatal("db_migrate: " & msg) quit(QuitFailure) # Load configuration file @@ -164,10 +201,17 @@ Options: except: exitErr "Error parsing config file: " & configFilename & "\L\t" & getCurrentExceptionMsg() + + logging.addHandler(newConsoleLogger()) + if args["--quiet"]: logging.setLogFilter(lvlError) + elif args["--very-verbose"]: logging.setLogFilter(lvlAll) + elif args["--verbose"]: logging.setlogFilter(lvlDebug) + else: logging.setLogFilter(config.logLevel) + # Check for migrations directory if not existsDir config.sqlDir: try: - echo "SQL directory '" & config.sqlDir & + warn "SQL directory '" & config.sqlDir & "' does not exist and will be created." createDir config.sqlDir except IOError: @@ -177,31 +221,48 @@ Options: if args["create"]: try: let filesCreated = createMigration(config, $args[""]) - echo "Created new migration files:" - for filename in filesCreated: echo "\t" & filename + info "Created new migration files:" + for filename in filesCreated: info "\t" & filename except IOError: exitErr "Unable to create migration scripts: " & getCurrentExceptionMsg() - elif args["up"]: - try: - let migrationsRun = up(config, if args[""]: parseInt($args[""]) else: 1) - for migration in migrationsRun: - echo migration - echo "Up to date." - except DbError: - exitErr "Unable to migrate database: " & getCurrentExceptionMsg() + else: + let pgConn: DbConn = + try: open("", "", "", config.connectionString) + except DbError: + exitErr "Unable to connect to the database." + (DbConn)(nil) - elif args["down"]: discard - # Query the database to find out what migrations have been run. + pgConn.createMigrationsTable - # Find how many we need to go down (default to 1 if the count parameter was - # not passed. + let (run, notRun, missing) = diffMigrations(pgConn, config) - # Begin transaction + # Make sure we have no gaps (database is in an unknown state) + if missing.len > 0: + exitErr "Database is in an inconsistent state. Migrations have been " & + "run that are not sequential." - # Apply each down script - # If any fail, roll back the transaction - # Otherwise report success + if args["up"]: + try: + let count = if args[""]: parseInt($args[""]) else: high(int) + let toRun = if count < notRun.len: notRun[0.."]: parseInt($args[""]) else: 1 + let toRun = if count < run.len: run.reversed[0.. 0: + info "Database is behind by " & $(newResults.notRun.len) & " migrations." + else: info "Database is up to date."