diff --git a/src/main/json/project.config.schema.json b/src/main/json/project.config.schema.json index 40eeed8..bf36c23 100644 --- a/src/main/json/project.config.schema.json +++ b/src/main/json/project.config.schema.json @@ -28,7 +28,7 @@ "expectedEnv": { "type": "array", "items": { "type": "string" }}, - "dontCache": "bool" + "dontSkip": "bool" }, "additionalProperties": false } diff --git a/src/main/nim/strawboss.nim b/src/main/nim/strawboss.nim index 69a99b4..ff9f60c 100644 --- a/src/main/nim/strawboss.nim +++ b/src/main/nim/strawboss.nim @@ -1,12 +1,18 @@ -import docopt, logging, os, sequtils +import docopt, logging, os, sequtils, tempfile import strawboss/private/util import strawboss/configuration import strawboss/core import strawboss/server +import strawboss/supervisor let SB_VER = "0.1.0" +proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) = + let prefix = if cmd != nil: cmd else: "" + if outMsg != nil: info prefix & "(stdout): " & outMsg + if errMsg != nil: info prefix & "(stderr): " & errMsg + when isMainModule: if logging.getHandlers().len == 0: @@ -22,19 +28,41 @@ when isMainModule: let doc = """ Usage: strawboss serve - strawboss run [-r ]""" + strawboss supervisor [-i ] [-o ] + strawboss run [options] + +Options + + -f --force-rebuild Force a build step to re-run even we have cached + results from building that step before for this + version of the project. + + -r --reference Build the project at this commit reference. + + -w --workspace Use the given directory as the build workspace. + +""" let args = docopt(doc, version = "strawboss v" & SB_VER) if args["run"]: - let projName = $args[""] - let stepName = $args[""] - let buildRef = if args["-r"]: $args[""] else: nil + let req = RunRequest( + projectName: $args[""], + stepName: $args[""], + buildRef: if args["--rreference"]: $args[""] else: nil, + forceRebuild: args["--force-rebuild"], + workspaceDir: if args["--workspace"]: $args[""] else: mkdtemp()) - try: core.runStep(cfg, projName, stepName, buildRef) + try: + let summary = core.runStep(cfg, req, logProcOutput) + # TODO: inspect result except: - fatal "strawboss: " & getCurrentExceptionMsg() + fatal "strawboss: " & getCurrentExceptionMsg() & "." quit(QuitFailure) + info "strawboss: build passed" + + elif elif args["serve"]: server.start(cfg) + diff --git a/src/main/nim/strawboss.nim.cfg b/src/main/nim/strawboss.nim.cfg new file mode 100644 index 0000000..aed303e --- /dev/null +++ b/src/main/nim/strawboss.nim.cfg @@ -0,0 +1 @@ +--threads:on diff --git a/src/main/nim/strawboss/configuration.nim b/src/main/nim/strawboss/configuration.nim index d27d74a..652a19a 100644 --- a/src/main/nim/strawboss/configuration.nim +++ b/src/main/nim/strawboss/configuration.nim @@ -1,13 +1,16 @@ -import logging, json, nre, sequtils, strtabs, tables +import logging, json, os, nre, sequtils, strtabs, tables import private/util # Types # type + BuildStatus* = object + state*, details*: string + Step* = object name*, stepCmd*, workingDir*: string artifacts*, cmdInput*, depends*, expectedEnv*: seq[string] - dontCache*: bool + dontSkip*: bool ProjectCfg* = object name*: string @@ -22,7 +25,6 @@ type artifactsRepo*: string projects*: seq[ProjectDef] - # internal utils let nullNode = newJNull() @@ -33,6 +35,9 @@ proc getIfExists(n: JsonNode, key: string): JsonNode = # Configuration parsing code proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig = + if not existsFile(cfgFile): + raiseEx "strawboss config file not found: " & cfgFile + let jsonCfg = parseFile(cfgFile) var projectDefs: seq[ProjectDef] = @[] @@ -60,13 +65,16 @@ proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig = projects: projectDefs) proc loadProjectConfig*(cfgFile: string): ProjectCfg = + if not existsFile(cfgFile): + raiseEx "project config file not found: " & cfgFile + let jsonCfg = parseFile(cfgFile) if not jsonCfg.hasKey("name"): - raise newException(Exception, "project configuration is missing a name") + raiseEx "project configuration is missing a name" if not jsonCfg.hasKey("steps"): - raise newException(Exception, "project configuration is missing steps definition") + raiseEx "project configuration is missing steps definition" var steps = initTable[string, Step]() for sName, pJson in jsonCfg["steps"].getFields: @@ -78,7 +86,7 @@ proc loadProjectConfig*(cfgFile: string): ProjectCfg = artifacts: pJson.getIfExists("artifacts").getElems.mapIt(it.getStr), cmdInput: pJson.getIfExists("cmdInput").getElems.mapIt(it.getStr), expectedEnv: pJson.getIfExists("expectedEnv").getElems.mapIt(it.getStr), - dontCache: pJson.getIfExists("dontCache").getStr("false") != "false") + dontSkip: pJson.getIfExists("dontSkip").getStr("false") != "false") if steps[sName].stepCmd == "sh" and steps[sName].cmdInput.len == 0: warn "Step " & sName & " uses 'sh' as its command but has no cmdInput." @@ -88,3 +96,22 @@ proc loadProjectConfig*(cfgFile: string): ProjectCfg = versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"), steps: steps) +proc loadBuildStatus*(statusFile: string): BuildStatus = + if not existsFile(statusFile): raiseEx "status file not found: " & statusFile + let jsonObj = parseFile(statusFile) + + if not jsonObj.hasKey("state"): + raiseEx "project status is missing the 'state' field" + + result = BuildStatus( + state: jsonObj["state"].getStr, + details: jsonObj.getIfExists("details").getStr("") ) + +proc `%`*(s: BuildStatus): JsonNode = + result = %* { + "state": s.state, + "details": s.details + } + +proc `$`*(s: BuildStatus): string = + result =pretty(%s) diff --git a/src/main/nim/strawboss/core.nim b/src/main/nim/strawboss/core.nim index 1c2a920..bde74e1 100644 --- a/src/main/nim/strawboss/core.nim +++ b/src/main/nim/strawboss/core.nim @@ -1,8 +1,8 @@ -import logging, nre, os, osproc, sequtils, streams, strtabs, strutils, tables, - tempfile +import logging, nre, os, osproc, sequtils, streams, strtabs, strutils, tables, tempfile import private/util import configuration +from posix import fork, Pid type Workspace* = ref object @@ -11,26 +11,37 @@ type workingDir*: string project*: ProjectCfg + RunSummary* = object + project*: ProjectCfg + step*: Step + buildVersion*, statusFile*: string + workerPid*: Pid -let ENV = loadEnv() + RunRequest* = object + projectName*, stepName*, buildRef*, workspaceDir*: string + forceRebuild*: bool -let logProcOutput: HandleProcMsgCB = proc (cmd: string, outMsg: TaintedString, errMsg: TaintedString) = - if outMsg != nil: info cmd & "(stdout): " & outMsg - if errMsg != nil: info cmd & "(stderr): " & errMsg - -let envVarRe = re"\$\w+|\$\{[^}]+\}" proc resolveEnvVars(line: string, env: StringTableRef): string = result = line - for found in line.findAll(envVarRe): + for found in line.findAll(re"\$\w+|\$\{[^}]+\}"): let key = if found[1] == '{': found[2..^2] else: found[1..^1] if env.hasKey(key): result = result.replace(found, env[key]) -let SB_EXPECTED_VARS = ["VERSION"] +proc combineProcMsgHandlers(a, b: HandleProcMsgCB): HandleProcMsgCB = + if a == nil: result = b + elif b == nil: result = a + else: + result = proc(cmd: string, outMsg, errMsg: TaintedString): void = + a(cmd, outMsg, errMsg) + b(cmd, outMsg, errMsg) -proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string): Workspace = +proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string, + outputHandler: HandleProcMsgCB = nil): Workspace = - info "Setting up to do work for '" & projectDef.name & "' at ref " & buildRef & "." - var env = ENV + outputHandler.sendMsg "Setting up to do work for '" & projectDef.name & + "' at ref " & buildRef & "." + + var env = loadEnv() env["GIT_DIR"] = ".git" # Create a temp directory that we'll work in @@ -40,7 +51,7 @@ proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string # Clone the project into the $temp/repo directory let cloneResult = exec("git", projDir, ["clone", projectDef.repo, "repo"], - env, {poUsePath}, logProcOutput) + env, {poUsePath}, outputHandler) if cloneResult.exitCode != 0: removeDir(projDir) @@ -48,7 +59,7 @@ proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string # Checkout the requested ref let checkoutResult = exec("git", projDir & "/repo", ["checkout", buildRef], - env, {poUsePath}, logProcOutput) + env, {poUsePath}, outputHandler) if checkoutResult.exitCode != 0: removeDir(projDir) @@ -76,94 +87,169 @@ proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string result.env, # environment {poUsePath, poEvalCommand}) # options - let versionResult = waitForWithOutput(versionProc, logProcOutput, + let versionResult = waitForWithOutput(versionProc, outputHandler, projectCfg.versionCmd) if versionResult.exitCode != 0: removeDir(projDir) raiseEx "Version command (" & projectCfg.versionCmd & ") returned non-zero exit code." - debug "Building version " & versionResult.output.strip + outputHandler.sendMsg "Building version " & versionResult.output.strip result.env["VERSION"] = versionResult.output.strip -proc runStep*(step: Step, wksp: Workspace): void = - let stepArtifactDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & step.name & "/" & wksp.env["VERSION"] +proc runStep*(step: Step, wksp: Workspace, + givenOutputHandler: HandleProcMsgCB = nil): void = + let SB_EXPECTED_VARS = ["VERSION"] + let stepArtifactDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & + step.name & "/" & wksp.env["VERSION"] + let statusFile = stepArtifactDir & ".status.json" - if existsDir(stepArtifactDir) and not step.dontCache: - info "Skipping step '" & step.name & "': already completed." - return - - info "Running step '" & step.name & "' for " & wksp.project.name - - # Ensure all expected environment variables are present. - for k in (step.expectedEnv & @SB_EXPECTED_VARS): - if not wksp.env.hasKey(k): - debug "workspace.env = " & $(wksp.env) - raiseEx "step " & step.name & " failed: missing required env variable: " & k - - # Ensure that artifacts in steps we depend on are present - for dep in step.depends: - if not wksp.project.steps.hasKey(dep): - raiseEx step.name & " depends on " & dep & " but there is no step named " & dep - let depStep = wksp.project.steps[dep] - - # Run that step (may get skipped) - let depDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & dep & "/" & wksp.env["VERSION"] - runStep(depStep, wksp) - - # Add the artifacts directory for the dependent step to our env so that - # further steps can reference it via $_DIR - echo "FP: " & depDir - wksp.env[depStep.name & "_DIR"] = depDir - - # Run the step command, piping in cmdInput - debug step.name & ": starting stepCmd: " & step.stepCmd - let cmdProc = startProcess(step.stepCmd, wksp.workingDir & "/repo/" & step.workingDir, - [], wksp.env, {poUsePath, poEvalCommand}) - - let cmdInStream = inputStream(cmdProc) - - # Replace env variables in step cmdInput as we pipe it in - for line in step.cmdInput: cmdInStream.writeLine(line.resolveEnvVars(wksp.env)) - cmdInStream.flush() - cmdInStream.close() - - let cmdResult = waitForWithOutput(cmdProc, logProcOutput, step.stepCmd) - - if cmdResult.exitCode != 0: - raiseEx "step " & step.name & " failed: step command returned non-zero exit code" - - # Gather the output artifacts (if we have any) if not existsDir(stepArtifactDir): createDir(stepArtifactDir) - if step.artifacts.len > 0: - for a in step.artifacts: - let artifactPath = a.resolveEnvVars(wksp.env) - let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1] - try: - copyFile(wksp.workingDir & "/repo/" & step.workingDir & "/" & artifactPath, - stepArtifactDir & "/" & artifactName) - except: - removeDir(stepArtifactDir) - raiseEx "step " & step.name & " failed: unable to copy artifact " & - artifactPath & ":\n" & getCurrentExceptionMsg() -# TODO: change return to return logs, results, some compound, useful object. -proc runStep*(cfg: StrawBossConfig, projectName, stepName, buildRef: string): void = + # Have we tried to build this before and are we caching the results? + if existsFile(statusFile) and not step.dontSkip: + let status = loadBuildStatus(statusFile) - let matching = cfg.projects.filterIt(it.name == projectName) - if matching.len == 0: raiseEx "no such project: " & projectName - elif matching.len > 1: raiseEx "more than one project named : " & projectName + # If we succeeded last time, no need to rebuild + if status.state == "complete": + givenOutputHandler.sendMsg "Skipping step '" & step.name & "' for version '" & + wksp.env["VERSION"] & "': already completed." + return + else: + givenOutputHandler.sendMsg "Rebuilding failed step '" & step.name & "' for version '" & + wksp.env["VERSION"] & "'." - let projectDef = matching[0] + givenOutputHandler.sendMsg "Running step '" & step.name & "' for " & wksp.project.name + writeFile(statusFile, $BuildStatus(state: "running", details: "")) - let foundBuildRef = if buildRef != nil: buildRef else: projectDef.defaultBranch - let wksp = setupProjectForWork(projectDef, foundBuildRef, cfg.artifactsRepo) + var stdoutLogFile, stderrLogFile: File - # Find the step - if not wksp.project.steps.hasKey(stepName): - raiseEx "no step name '" & stepName & "' for " & projectName + try: - let step = wksp.project.steps[stepName] + var outputHandler: HandleProcMsgCB - runStep(step, wksp) + # Make sure we log output to the stdout and sterr log files + if not (open(stdoutLogFile, stepArtifactDir & "/stdout.log", fmWrite) and + open(stderrLogFile, stepArtifactDir & "/stderr.log", fmWrite)): + givenOutputHandler.sendMsg nil, "Failed to open log files for STDOUT and STDERR." + outputHandler = givenOutputHandler + if stdoutLogFile != nil: close(stdoutLogFile) + if stderrLogFile != nil: close(stderrLogFile) + else: + outputHandler = combineProcMsgHandlers( + givenOutputHandler, + makeProcMsgHandler(stdoutLogFile, stderrLogFile)) + # Ensure all expected environment variables are present. + for k in (step.expectedEnv & @SB_EXPECTED_VARS): + if not wksp.env.hasKey(k): + debug "workspace.env = " & $(wksp.env) + raiseEx "step " & step.name & " failed: missing required env variable: " & k + + # Ensure that artifacts in steps we depend on are present + # TODO: detect circular-references in dependency trees. + for dep in step.depends: + if not wksp.project.steps.hasKey(dep): + raiseEx step.name & " depends on " & dep & + " but there is no step named " & dep + let depStep = wksp.project.steps[dep] + + # Run that step (may get skipped) + let depDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & + dep & "/" & wksp.env["VERSION"] + + runStep(depStep, wksp) + + # Add the artifacts directory for the dependent step to our env so that + # further steps can reference it via $_DIR + echo "FP: " & depDir + wksp.env[depStep.name & "_DIR"] = depDir + + # Run the step command, piping in cmdInput + outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd + let cmdProc = startProcess(step.stepCmd, + wksp.workingDir & "/repo/" & step.workingDir, + [], wksp.env, {poUsePath, poEvalCommand}) + + let cmdInStream = inputStream(cmdProc) + + # Replace env variables in step cmdInput as we pipe it in + for line in step.cmdInput: cmdInStream.writeLine(line.resolveEnvVars(wksp.env)) + cmdInStream.flush() + cmdInStream.close() + + let cmdResult = waitForWithOutput(cmdProc, outputHandler, step.stepCmd) + + if cmdResult.exitCode != 0: + raiseEx "step " & step.name & " failed: step command returned non-zero exit code" + + # Gather the output artifacts (if we have any) + if step.artifacts.len > 0: + for a in step.artifacts: + let artifactPath = a.resolveEnvVars(wksp.env) + let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1] + try: + copyFile(wksp.workingDir & "/repo/" & step.workingDir & "/" & artifactPath, + stepArtifactDir & "/" & artifactName) + except: + raiseEx "step " & step.name & " failed: unable to copy artifact " & + artifactPath & ":\n" & getCurrentExceptionMsg() + + writeFile(statusFile, $BuildStatus(state: "complete", details: "")) + + except: + writeFile(statusFile, $BuildStatus( + state: "failed", + details: getCurrentExceptionMsg())) + + finally: + if stdoutLogFile != nil: close(stdoutLogFile) + if stderrLogFile != nil: close(stderrLogFile) + +proc runStep*(cfg: StrawBossConfig, req: RunRequest, + outputHandler: HandleProcMsgCB = nil): RunSummary = + + if not existsDir(req.workspaceDir): createDir(req.workspaceDir) + let statusFile = req.workspaceDir & "/" & "status.json" + + try: + writeFile(statusFile, $BuildStatus( + state: "setup", + details: "Preparing working environment.")) + + # Find the project definition + let matching = cfg.projects.filterIt(it.name == req.projectName) + if matching.len == 0: raiseEx "no such project: " & req.projectName + elif matching.len > 1: raiseEx "more than one project named : " & req.projectName + + let projectDef = matching[0] + + # Find the commit reference we're building + let foundBuildRef = + if req.buildRef != nil and req.buildRef.len > 0: req.buildRef + else: projectDef.defaultBranch + + let wksp = setupProjectForWork(projectDef, foundBuildRef, cfg.artifactsRepo, + outputHandler) + + # Find the step + if not wksp.project.steps.hasKey(req.stepName): + raiseEx "no step name '" & req.stepName & "' for " & req.projectName + + var step = wksp.project.steps[req.stepName] + + # Enfore forceRebuild + if req.forceRebuild: step.dontSkip = true + + result = RunSummary( + project: wksp.project, + step: step, + buildVersion: wksp.env["VERSION"], + statusFile: wksp.artifactsRepo & "/" & wksp.project.name & "/" & step.name & + "/" & wksp.env["VERSION"] & ".status.json") + if req.async: + let pid = fork() + if pid == 0: runStep(step, wksp, outputHandler) # if we are the child + else: result.workerPid = pid # if we are the parent + + else: runStep(step, wksp, outputHandler) diff --git a/src/main/nim/strawboss/private/util.nim b/src/main/nim/strawboss/private/util.nim index 491ddfe..285b043 100644 --- a/src/main/nim/strawboss/private/util.nim +++ b/src/main/nim/strawboss/private/util.nim @@ -1,6 +1,11 @@ import os, osproc, streams, strtabs -type HandleProcMsgCB* = proc (cmd: string, outMsg: TaintedString, errMsg: TaintedString): void +from posix import kill + +type HandleProcMsgCB* = proc (outMsg: TaintedString, errMsg: TaintedString, cmd: string): void + +proc sendMsg*(h: HandleProcMsgCB, outMsg: TaintedString, errMsg: TaintedString = nil, cmd: string = "strawboss"): void = + if h != nil: h(outMsg, errMsg, cmd) proc raiseEx*(reason: string): void = raise newException(Exception, reason) @@ -22,11 +27,11 @@ proc waitForWithOutput*(p: Process, msgCB: HandleProcMsgCB, var line = newStringOfCap(120).TaintedString while true: if pout.readLine(line): - if msgCB != nil: msgCB(procCmd, line, nil) + msgCB.sendMsg(line, nil, procCmd) result[0].string.add(line.string) result[0].string.add("\n") elif perr.readLine(line): - if msgCB != nil: msgCB(procCmd, nil, line) + msgCB.sendMsg(nil, line, procCmd) result[1].string.add(line.string) result[1].string.add("\n") else: @@ -49,3 +54,15 @@ proc loadEnv*(): StringTableRef = for k, v in envPairs(): result[k] = v + +proc makeProcMsgHandler*(outSink, errSink: File): HandleProcMsgCB = + result = proc(outMsg, errMsg: TaintedString, cmd: string): void {.closure.} = + let prefix = if cmd != nil: cmd & ": " else: "" + if outMsg != nil: outSink.writeLine(prefix & outMsg) + if errMsg != nil: errSink.writeLine(prefix & errMsg) + +proc makeProcMsgHandler*(outSink, errSink: Stream): HandleProcMsgCB = + result = proc(outMsg, errMsg: TaintedString, cmd: string): void {.closure.} = + let prefix = if cmd != nil: cmd & ": " else: "" + if outMsg != nil: outSink.writeLine(prefix & outMsg) + if errMsg != nil: errSink.writeLine(prefix & errMsg) diff --git a/src/main/nim/strawboss/server.nim b/src/main/nim/strawboss/server.nim index 307f247..ae36b5b 100644 --- a/src/main/nim/strawboss/server.nim +++ b/src/main/nim/strawboss/server.nim @@ -5,12 +5,29 @@ import ./configuration, ./core settings: port = Port(8180) -routes: - get "/api/ping": - resp $(%*"pong"), "application/json" - get "/api/projects": - resp $(%*[]), "application/json" +proc start*(givenCfg: StrawBossConfig): void = + + routes: + get "/api/ping": + resp $(%*"pong"), "application/json" + + get "/api/projects": + resp $(%*[]), "application/json" + + post "/api/project/@projectName/@stepName/run/@buildRef?": + let req = RunRequest( + projectName: @"projectName", + stepName: @"stepName", + buildRef: if @"buildRef" != "": @"buildRef" else: nil, + async: true, + forceRebuild: false) # TODO support this with optional query params + +# try: +# let runSummary = core.runStep(givenCfg, req) +# except: +# discard "" +# # TODO + -proc start*(cfg: StrawBossConfig): void = runForever() diff --git a/strawboss.config.json b/strawboss.config.json index 563b967..0811599 100644 --- a/strawboss.config.json +++ b/strawboss.config.json @@ -4,5 +4,7 @@ "tokens": [], "projects": [ { "name": "new-life-intro-band", - "repo": "/home/jdb/projects/new-life-introductory-band" } ] + "repo": "/home/jdb/projects/new-life-introductory-band" }, + { "name": "test-strawboss", + "repo": "/home/jdb/projects/test-strawboss" } ] }