From e35a5177ef7ef884223a24fe4c393b95eef70d2c Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Sat, 11 Feb 2017 19:52:12 -0600 Subject: [PATCH] Add env configuration, bug in SQL parsing for Nim db_migrate. * The Nim binary now recognizes the following environment and allows them to override configured values: - `DATABASE_DRIVER`: overrides the `driver` config value, selects which kind of database we expect to connect to. - `MIGRATIONS_DIR`: overrides the `sqlDir` config value, sets the path to the directory containing the migration SQL files. - `DATABASE_URL`: overrides the `connectionString` config value, supplies connection parameters to the database. - `DBM_LOG_LEVEL`: overrides the `logLevel` config value, sets the logging level of db_migrate. * Fixes a bug in the parsing of migration files. Previously the file was split on `;` characters and chunks that started with `--` were ignored as comments. This was wrong in the common case where a block starts with a comment but then contains SQL further. Such a block was being ignored. The new behavior is to consider each line and build up queries that way. * Fixes a logging bug when parsing the configuration. If there was an that prevented the configuration from loading this in turn prevented logging from being setup correctly, and so the error was not logged. Now errors that may occur before logging is setup are hard-coded to be logged to STDERR. * Changes the logic for creating the `migrations` table to check before performing the query that creates the table. This is to avoid continually printing messages about skipping this creation when the table already exists (which is the normal case). This change is PostgreSQL-specific, but of course the entire tool is currently PostgreSQL-specific. --- build.gradle | 2 +- db_migrate.nimble | 2 +- .../com/jdblabs/dbmigrate/DbMigrate.groovy | 2 +- src/main/nim/db_migrate.nim | 63 ++++++++++++++----- 4 files changed, 50 insertions(+), 19 deletions(-) diff --git a/build.gradle b/build.gradle index dbfd732..5bc776a 100644 --- a/build.gradle +++ b/build.gradle @@ -3,7 +3,7 @@ apply plugin: 'application' apply plugin: 'maven' group = 'com.jdblabs' -version = '0.2.4' +version = '0.2.5' mainClassName = 'com.jdblabs.dbmigrate.DbMigrate' diff --git a/db_migrate.nimble b/db_migrate.nimble index 6c17d68..87f19e4 100644 --- a/db_migrate.nimble +++ b/db_migrate.nimble @@ -1,7 +1,7 @@ # Package bin = @["db_migrate"] -version = "0.2.4" +version = "0.2.5" author = "Jonathan Bernard" description = "Simple tool to handle database migrations." license = "BSD" diff --git a/src/main/groovy/com/jdblabs/dbmigrate/DbMigrate.groovy b/src/main/groovy/com/jdblabs/dbmigrate/DbMigrate.groovy index 87e1e44..9da2fb7 100644 --- a/src/main/groovy/com/jdblabs/dbmigrate/DbMigrate.groovy +++ b/src/main/groovy/com/jdblabs/dbmigrate/DbMigrate.groovy @@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory public class DbMigrate { - public static final VERSION = "0.2.4" + public static final VERSION = "0.2.5" public static final def DOC = """\ db-migrate.groovy v${VERSION} diff --git a/src/main/nim/db_migrate.nim b/src/main/nim/db_migrate.nim index 5d8e297..1fe68b1 100644 --- a/src/main/nim/db_migrate.nim +++ b/src/main/nim/db_migrate.nim @@ -9,8 +9,14 @@ import algorithm, json, times, os, strutils, docopt, db_postgres, sets, type DbMigrateConfig* = tuple[ driver, sqlDir, connectionString: string, logLevel: Level ] -proc createMigrationsTable(conn: DbConn): void = - conn.exec(sql(""" +proc ensureMigrationsTableExists(conn: DbConn): void = + let tableCount = conn.getValue(sql""" +SELECT COUNT(*) FROM information_schema.tables +WHERE table_name = 'migrations';""") + + if tableCount.strip == "0": + info "Creating the migrations table as it does not already exist." + conn.exec(sql(""" CREATE TABLE IF NOT EXISTS migrations ( id SERIAL PRIMARY KEY, name VARCHAR NOT NULL, @@ -24,15 +30,27 @@ proc loadConfig*(filename: string): DbMigrateConfig = ## Load DbMigrateConfig from a file. let cfg = json.parseFile(filename) - var logLevel: Level - if cfg.hasKey("logLevel"): + var logLevel: Level = lvlInfo + if existsEnv("DBM_LOG_LEVEL"): + let idx = find(LevelNames, $getEnv("DBM_LOG_LEVEL").toUpper) + logLevel = if idx == -1: lvlInfo else: (Level)(idx) + elif cfg.hasKey("logLevel"): let idx = find(LevelNames, cfg["logLevel"].getStr.toUpper) logLevel = if idx == -1: lvlInfo else: (Level)(idx) return ( - driver: if cfg.hasKey("driver"): cfg["driver"].getStr else: "postres", - sqlDir: if cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr else: "migrations", - connectionString: cfg["connectionString"].getStr, + driver: + if existsEnv("DATABASE_DRIVER"): $getEnv("DATABASE_DRIVER") + elif cfg.hasKey("driver"): cfg["driver"].getStr + else: "postres", + sqlDir: + if existsEnv("MIGRATIONS_DIR"): $getEnv("MIGRATIONS_DIR") + elif cfg.hasKey("sqlDir"): cfg["sqlDir"].getStr + else: "migrations", + connectionString: + if existsEnv("DATABASE_URL"): $getEnv("DATABASE_URL") + elif cfg.hasKey("connectionString"): cfg["connectionString"].getStr + else: "", logLevel: logLevel) proc createMigration*(config: DbMigrateConfig, migrationName: string): seq[string] = @@ -96,10 +114,21 @@ proc diffMigrations*(pgConn: DbConn, config: DbMigrateConfig): missing: missingMigrations) proc readStatements*(filename: string): seq[SqlQuery] = - let migrationSql = filename.readFile - result = migrationSql.split(';'). - filter(proc(st: string): bool = st.strip.len > 0 and not st.startsWith("--")). - map(proc(st: string): SqlQuery = sql(st & ";")) + result = @[] + var stmt: string = "" + + for line in filename.lines: + let l = line.strip + if l.len == 0 or l.startsWith("--"): continue + + let parts = line.split(';') + stmt &= "\n" & parts[0] + + if parts.len > 1: + result.add(sql(stmt & ";")) + stmt = parts[1] & ""; + + if stmt.strip.len > 0: result.add(sql(stmt)) proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[string] = var migrationsRun = newSeq[string]() @@ -118,7 +147,8 @@ proc up*(pgConn: DbConn, config: DbMigrateConfig, toRun: seq[string]): seq[strin let statements = filename.readStatements try: - for statement in statements: pgConn.exec(statement) + for statement in statements: + pgConn.exec(statement) pgConn.exec(sql"INSERT INTO migrations (name) VALUES (?);", migration) except DbError: pgConn.rollbackWithErr "Migration '" & migration & "' failed:\n\t" & @@ -182,7 +212,7 @@ Options: """ # Parse arguments - let args = docopt(doc, version = "db-migrate 0.2.4") + let args = docopt(doc, version = "db-migrate 0.2.5") let exitErr = proc(msg: string): void = fatal("db_migrate: " & msg) @@ -197,9 +227,10 @@ Options: try: config = loadConfig(configFilename) except IOError: - exitErr "Cannot open config file: " & configFilename + writeLine(stderr, "db_migrate: Cannot open config file: " & configFilename) except: - exitErr "Error parsing config file: " & configFilename & "\L\t" & getCurrentExceptionMsg() + writeLine(stderr, "db_migrate: Error parsing config file: " & + configFilename & "\L\t" & getCurrentExceptionMsg()) logging.addHandler(newConsoleLogger()) @@ -233,7 +264,7 @@ Options: exitErr "Unable to connect to the database: " & getCurrentExceptionMsg() (DbConn)(nil) - pgConn.createMigrationsTable + pgConn.ensureMigrationsTableExists let (run, notRun, missing) = diffMigrations(pgConn, config)