diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c0a0b95 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.sw? +nimcache/ +/strawboss diff --git a/example.json b/example.json new file mode 100644 index 0000000..72e8fd9 --- /dev/null +++ b/example.json @@ -0,0 +1,22 @@ +{ + "name": "new-life-intro-band", + "steps": { + "build": { + "workingDir": "website", + "expectedEnv": ["VERSION"], + "artifacts": ["nlib-web-$VERSION.zip"], + "cmdInput": [ + "nim c -r make_site", + "(cd rendered && zip -r ../nlib-web-$VERSION.zip *)" + ] + }, + "deploy": { + "dontCache": "true", + "depends": ["build"], + "expectedEnv": ["VERSION"], + "cmdInput": [ + "netlify deploy -s newlifeintroband -p '$build_DIR/nlib-web-$VERSION.zip'" + ] + } + } +} diff --git a/strawboss.config.json b/strawboss.config.json new file mode 100644 index 0000000..c20fbdd --- /dev/null +++ b/strawboss.config.json @@ -0,0 +1,6 @@ +{ + "artifactsRepo": "artifacts", + "projects": [ + { "name": "new-life-intro-band", + "repo": "/home/jdb/projects/new-life-introductory-band" } ] +} diff --git a/strawboss.nim b/strawboss.nim new file mode 100644 index 0000000..aae8472 --- /dev/null +++ b/strawboss.nim @@ -0,0 +1,320 @@ +import docopt, json, logging, nre, os, osproc, sequtils, streams, strtabs, strutils, tables, tempfile + +let SB_VER = "0.1.0" + +# Types +# +type + Step* = object + name*, stepCmd*, workingDir*: string + artifacts*, cmdInput*, depends*, expectedEnv*: seq[string] + dontCache*: bool + + ProjectCfg* = object + name*: string + versionCmd*: string + steps*: Table[string, Step] + + ProjectDef* = object + cfgFilePath*, defaultBranch*, name*, repo*: string + envVars*: StringTableRef + + StrawBossCfg* = object + artifactsRepo*: string + projects*: seq[ProjectDef] + + Workspace* = ref object + artifactsRepo*: string + env*: StringTableRef + workingDir*: string + project: ProjectCfg + + HandleProcMsgCB = proc (cmd: string, outMsg: TaintedString, errMsg: TaintedString): void + +# Misc. helpers + +proc raiseEx(reason: string): void = + raise newException(Exception, reason) + +let nullNode = newJNull() +proc getIfExists(n: JsonNode, key: string): JsonNode = + result = if n.hasKey(key): n[key] + else: nullNode + + +proc loadEnv(): StringTableRef = + result = newStringTable() + + for k, v in envPairs(): + result[k] = v + +proc waitForWithOutput(p: Process, msgCB: HandleProcMsgCB, + procCmd: string = ""): + tuple[output: TaintedString, error: TaintedString, exitCode: int] = + + var pout = outputStream(p) + var perr = errorStream(p) + + result = (TaintedString"", TaintedString"", -1) + var line = newStringOfCap(120).TaintedString + while true: + if pout.readLine(line): + if msgCB != nil: msgCB(procCmd, line, nil) + result[0].string.add(line.string) + result[0].string.add("\n") + elif perr.readLine(line): + if msgCB != nil: msgCB(procCmd, nil, line) + result[1].string.add(line.string) + result[1].string.add("\n") + else: + result[2] = peekExitCode(p) + if result[2] != -1: break + close(p) + +proc exec(command: string, workingDir: string = "", + args: openArray[string] = [], env: StringTableRef = nil, + options: set[ProcessOption] = {poUsePath}, + msgCB: HandleProcMsgCB = nil): + tuple[output: TaintedString, error: TaintedString, exitCode: int] + {.tags: [ExecIOEffect, ReadIOEffect], gcsafe.} = + + var p = startProcess(command, workingDir, args, env, options) + result = waitForWithOutput(p, msgCb, command) + +let ENV = loadEnv() +let logProcOutput: HandleProcMsgCB = proc (cmd: string, outMsg: TaintedString, errMsg: TaintedString) = + if outMsg != nil: info cmd & "(stdout): " & outMsg + if errMsg != nil: info cmd & "(stderr): " & errMsg + +let envVarRe = re"\$\w+|\$\{[^}]+\}" +proc resolveEnvVars(line: string, env: StringTableRef): string = + result = line + for found in line.findAll(envVarRe): + let key = if found[1] == '{': found[2..^2] else: found[1..^1] + if env.hasKey(key): result = result.replace(found, env[key]) + +let SB_EXPECTED_VARS = ["VERSION"] +# Configuration parsing code + +proc loadStrawBossConfig(cfgFile: string): StrawBossCfg = + let jsonCfg = parseFile(cfgFile) + + var projectDefs: seq[ProjectDef] = @[] + + for pJson in jsonCfg.getIfExists("projects").getElems: + if not pJson.hasKey("name"): + raiseEx "a project definition is missing the project name" + + if not pJson.hasKey("repo"): + raiseEx "a project definition is missing the project repo configuration" + + var envVars = newStringTable(modeCaseSensitive) + for k, v in pJson.getIfExists("envVars").getFields: envVars[k] = v.getStr("") + + projectDefs.add( + ProjectDef( + cfgFilePath: pJson.getIfExists("cfgFilePath").getStr("strawboss.json"), + defaultBranch: pJson.getIfExists("defaultBranch").getStr("master"), + name: pJson["name"].getStr, + envVars: envVars, + repo: pJson["repo"].getStr)) + + result = StrawBossCfg( + artifactsRepo: jsonCfg.getIfExists("artifactsRepo").getStr("artifacts"), + projects: projectDefs) + +proc loadProjectConfig(cfgFile: string): ProjectCfg = + let jsonCfg = parseFile(cfgFile) + + if not jsonCfg.hasKey("name"): + raise newException(Exception, "project configuration is missing a name") + + if not jsonCfg.hasKey("steps"): + raise newException(Exception, "project configuration is missing steps definition") + + var steps = initTable[string, Step]() + for sName, pJson in jsonCfg["steps"].getFields: + steps[sName] = Step( + name: sName, + workingDir: pJson.getIfExists("workingDir").getStr("."), + stepCmd: pJson.getIfExists("stepCmd").getStr("sh"), + depends: pJson.getIfExists("depends").getElems.mapIt(it.getStr), + artifacts: pJson.getIfExists("artifacts").getElems.mapIt(it.getStr), + cmdInput: pJson.getIfExists("cmdInput").getElems.mapIt(it.getStr), + expectedEnv: pJson.getIfExists("expectedEnv").getElems.mapIt(it.getStr), + dontCache: pJson.getIfExists("dontCache").getStr("false") != "false") + + if steps[sName].stepCmd == "sh" and steps[sName].cmdInput.len == 0: + warn "Step " & sName & " uses 'sh' as its command but has no cmdInput." + + result = ProjectCfg( + name: jsonCfg["name"].getStr, + versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"), + steps: steps) + +proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string): Workspace = + + info "Setting up to do work for '" & projectDef.name & "' at ref " & buildRef & "." + + # Create a temp directory that we'll work in + let projDir = mkdtemp() + + # Clone the project into the $temp/repo directory + let cloneResult = exec("git", projDir, ["clone", projectDef.repo, "repo"], + ENV, {poUsePath}, logProcOutput) + + if cloneResult.exitCode != 0: + removeDir(projDir) + raiseEx "unable to clone repo for '" & projectDef.name & "'" + + # Checkout the requested ref + let checkoutResult = exec("git", projDir & "/repo", ["checkout", buildRef], + ENV, {poUsePath}, logProcOutput) + + if checkoutResult.exitCode != 0: + removeDir(projDir) + raiseEx "unable to checkout ref " & buildRef & " for '" & projectDef.name & "'" + + # Find the strawboss project configuration + let projCfgFile = projDir & "/repo/" & projectDef.cfgFilePath + if not existsFile(projCfgFile): + removeDir(projDir) + raiseEx "Cannot find strawboss project configuration in the project " & + "repo (expected at '" & projectDef.cfgFilePath & "')." + + let projectCfg = loadProjectConfig(projCfgFile) + result = Workspace(env: ENV, workingDir: projDir, project: projectCfg, + artifactsRepo: artifactsRepo) + + # Merge in the project-defined env vars + for k, v in projectDef.envVars: result.env[k] = v + + # Get the build version + let versionProc = startProcess( + projectCfg.versionCmd, # command + projDir & "/repo", # working dir + [], # args + result.env, # environment + {poUsePath, poEvalCommand}) # options + + let versionResult = waitForWithOutput(versionProc, logProcOutput, + projectCfg.versionCmd) + + if versionResult.exitCode != 0: + removeDir(projDir) + raiseEx "Version command (" & projectCfg.versionCmd & ") returned non-zero exit code." + + debug "Building version " & versionResult.output.strip + result.env["VERSION"] = versionResult.output.strip + + debug "Workspace for '" & projectCfg.name & ": " & projDir + +## TODO +proc runStep(step: Step, wksp: Workspace): void = + + let stepArtifactDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & step.name & "/" & wksp.env["VERSION"] + + if existsDir(stepArtifactDir) and not step.dontCache: + info "Skipping step '" & step.name & "': already completed." + return + + info "Running step '" & step.name & "' for " & wksp.project.name + + # Ensure all expected environment variables are present. + for k in (step.expectedEnv & @SB_EXPECTED_VARS): + if not wksp.env.hasKey(k): + debug "workspace.env = " & $(wksp.env) + raiseEx "step " & step.name & " failed: missing required env variable: " & k + + # Ensure that artifacts in steps we depend on are present + for dep in step.depends: + if not wksp.project.steps.hasKey(dep): + raiseEx step.name & " depends on " & dep & " but there is no step named " & dep + let depStep = wksp.project.steps[dep] + + # Run that step (may get skipped) + let depDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & dep & "/" & wksp.env["VERSION"] + runStep(depStep, wksp) + + # Add the artifacts directory for the dependent step to our env so that + # further steps can reference it via $_DIR + echo "FP: " & depDir + wksp.env[depStep.name & "_DIR"] = depDir + + # Run the step command, piping in cmdInput + debug step.name & ": starting stepCmd: " & step.stepCmd + let cmdProc = startProcess(step.stepCmd, wksp.workingDir & "/repo/" & step.workingDir, + [], wksp.env, {poUsePath, poEvalCommand}) + + let cmdInStream = inputStream(cmdProc) + + # Replace env variables in step cmdInput as we pipe it in + for line in step.cmdInput: cmdInStream.writeLine(line.resolveEnvVars(wksp.env)) + cmdInStream.flush() + cmdInStream.close() + + let cmdResult = waitForWithOutput(cmdProc, logProcOutput, step.stepCmd) + + if cmdResult.exitCode != 0: + raiseEx "step " & step.name & " failed: step command returned non-zero exit code" + + # Gather the output artifacts (if we have any) + if not existsDir(stepArtifactDir): createDir(stepArtifactDir) + if step.artifacts.len > 0: + for a in step.artifacts: + let artifactPath = a.resolveEnvVars(wksp.env) + let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1] + try: + copyFile(wksp.workingDir & "/repo/" & step.workingDir & "/" & artifactPath, + stepArtifactDir & "/" & artifactName) + except: + removeDir(stepArtifactDir) + raiseEx "step " & step.name & " failed: unable to copy artifact " & + artifactPath & ":\n" & getCurrentExceptionMsg() + +when isMainModule: + + logging.addHandler(newConsoleLogger()) + + let cfg = loadStrawBossConfig("strawboss.config.json") + let artifactsRepo = expandFilename(cfg.artifactsRepo) + + if not existsDir(artifactsRepo): + info "Artifacts repo (" & artifactsRepo & ") does not exist. Creating..." + createDir(artifactsRepo) + + let doc = """ +Usage: + strawboss serve + strawboss run [-r ]""" + + let args = docopt(doc, version = "strawboss v" & SB_VER) + + if args["run"]: + + # Find the project + let projName = $args[""] + let matching = cfg.projects.filterIt(it.name == projName) + if matching.len == 0: + fatal "strawboss: no such project: " & projName + quit(QuitFailure) + elif matching.len > 1: + fatal "strawboss: more than one project named : " & projName + quit(QuitFailure) + + let projectDef = matching[0] + let buildRef = if args["-r"]: $args[""] else: projectDef.defaultBranch + + try: + let wksp = setupProjectForWork(projectDef, buildRef, artifactsRepo) + + # Find the step + let stepName = $args[""] + if not wksp.project.steps.hasKey(stepName): + raiseEx "no step name '" & stepName & "' for " & projName + + let step = wksp.project.steps[stepName] + runStep(step, wksp) + except: + fatal "strawboss: " & getCurrentExceptionMsg() + quit(QuitFailure) diff --git a/strawboss.nimble b/strawboss.nimble index 13f034b..56d480f 100644 --- a/strawboss.nimble +++ b/strawboss.nimble @@ -8,5 +8,5 @@ license = "MIT" # Dependencies -requires @["nim >= 0.16.1", "docopt >= 0.1.0"] +requires @["nim >= 0.16.1", "docopt >= 0.1.0", "tempfile"]