From 0976871563037c2eda07ece316d07e5411da2f4a Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Mon, 13 Mar 2017 07:29:37 -0500 Subject: [PATCH] Finished refactor towards process-based workers. --- src/main/nim/strawboss.nim | 30 +- src/main/nim/strawboss/configuration.nim | 54 ++-- src/main/nim/strawboss/core.nim | 375 ++++++++++++----------- src/main/nim/strawboss/private/util.nim | 10 + src/main/nim/strawboss/server.nim | 29 +- strawboss.nimble | 2 +- 6 files changed, 271 insertions(+), 229 deletions(-) diff --git a/src/main/nim/strawboss.nim b/src/main/nim/strawboss.nim index ff9f60c..f16b93d 100644 --- a/src/main/nim/strawboss.nim +++ b/src/main/nim/strawboss.nim @@ -1,26 +1,22 @@ -import docopt, logging, os, sequtils, tempfile +import docopt, os, sequtils, tempfile import strawboss/private/util import strawboss/configuration import strawboss/core import strawboss/server -import strawboss/supervisor -let SB_VER = "0.1.0" +let SB_VER = "0.2.0" proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) = let prefix = if cmd != nil: cmd else: "" - if outMsg != nil: info prefix & "(stdout): " & outMsg - if errMsg != nil: info prefix & "(stderr): " & errMsg + if outMsg != nil: echo prefix & "(stdout): " & outMsg + if errMsg != nil: echo prefix & "(stderr): " & errMsg when isMainModule: - if logging.getHandlers().len == 0: - logging.addHandler(newConsoleLogger()) - var cfg = loadStrawBossConfig("strawboss.config.json") if not existsDir(cfg.artifactsRepo): - info "Artifacts repo (" & cfg.artifactsRepo & ") does not exist. Creating..." + echo "Artifacts repo (" & cfg.artifactsRepo & ") does not exist. Creating..." createDir(cfg.artifactsRepo) cfg.artifactsRepo = expandFilename(cfg.artifactsRepo) @@ -28,7 +24,6 @@ when isMainModule: let doc = """ Usage: strawboss serve - strawboss supervisor [-i ] [-o ] strawboss run [options] Options @@ -45,24 +40,25 @@ Options let args = docopt(doc, version = "strawboss v" & SB_VER) + echo $args if args["run"]: let req = RunRequest( projectName: $args[""], stepName: $args[""], - buildRef: if args["--rreference"]: $args[""] else: nil, + buildRef: if args["--reference"]: $args["--reference"] else: nil, forceRebuild: args["--force-rebuild"], workspaceDir: if args["--workspace"]: $args[""] else: mkdtemp()) try: - let summary = core.runStep(cfg, req, logProcOutput) - # TODO: inspect result + let status = core.runStep(cfg, req, logProcOutput) + if status.state == "failed": raiseEx status.details + echo "strawboss: build passed." except: - fatal "strawboss: " & getCurrentExceptionMsg() & "." + echo "strawboss: build FAILED: " & getCurrentExceptionMsg() & "." quit(QuitFailure) + finally: + if existsDir(req.workspaceDir): removeDir(req.workspaceDir) - info "strawboss: build passed" - - elif elif args["serve"]: server.start(cfg) diff --git a/src/main/nim/strawboss/configuration.nim b/src/main/nim/strawboss/configuration.nim index 652a19a..0935497 100644 --- a/src/main/nim/strawboss/configuration.nim +++ b/src/main/nim/strawboss/configuration.nim @@ -25,6 +25,10 @@ type artifactsRepo*: string projects*: seq[ProjectDef] + RunRequest* = object + projectName*, stepName*, buildRef*, workspaceDir*: string + forceRebuild*: bool + # internal utils let nullNode = newJNull() @@ -32,6 +36,10 @@ proc getIfExists(n: JsonNode, key: string): JsonNode = result = if n.hasKey(key): n[key] else: nullNode +proc getOrFail(n: JsonNode, key: string, objName: string = ""): JsonNode = + if not n.hasKey(key): raiseEx objName & " missing key " & key + return n[key] + # Configuration parsing code proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig = @@ -43,12 +51,6 @@ proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig = var projectDefs: seq[ProjectDef] = @[] for pJson in jsonCfg.getIfExists("projects").getElems: - if not pJson.hasKey("name"): - raiseEx "a project definition is missing the project name" - - if not pJson.hasKey("repo"): - raiseEx "a project definition is missing the project repo configuration" - var envVars = newStringTable(modeCaseSensitive) for k, v in pJson.getIfExists("envVars").getFields: envVars[k] = v.getStr("") @@ -56,9 +58,9 @@ proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig = ProjectDef( cfgFilePath: pJson.getIfExists("cfgFilePath").getStr("strawboss.json"), defaultBranch: pJson.getIfExists("defaultBranch").getStr("master"), - name: pJson["name"].getStr, + name: pJson.getOrFail("name", "project definition").getStr, envVars: envVars, - repo: pJson["repo"].getStr)) + repo: pJson.getOrFail("repo", "project definition").getStr)) result = StrawBossConfig( artifactsRepo: jsonCfg.getIfExists("artifactsRepo").getStr("artifacts"), @@ -70,14 +72,11 @@ proc loadProjectConfig*(cfgFile: string): ProjectCfg = let jsonCfg = parseFile(cfgFile) - if not jsonCfg.hasKey("name"): - raiseEx "project configuration is missing a name" - if not jsonCfg.hasKey("steps"): raiseEx "project configuration is missing steps definition" var steps = initTable[string, Step]() - for sName, pJson in jsonCfg["steps"].getFields: + for sName, pJson in jsonCfg.getOrFail("steps", "project configuration").getFields: steps[sName] = Step( name: sName, workingDir: pJson.getIfExists("workingDir").getStr("."), @@ -92,7 +91,7 @@ proc loadProjectConfig*(cfgFile: string): ProjectCfg = warn "Step " & sName & " uses 'sh' as its command but has no cmdInput." result = ProjectCfg( - name: jsonCfg["name"].getStr, + name: jsonCfg.getOrFail("name", "project configuration").getStr, versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"), steps: steps) @@ -100,18 +99,35 @@ proc loadBuildStatus*(statusFile: string): BuildStatus = if not existsFile(statusFile): raiseEx "status file not found: " & statusFile let jsonObj = parseFile(statusFile) - if not jsonObj.hasKey("state"): - raiseEx "project status is missing the 'state' field" - result = BuildStatus( - state: jsonObj["state"].getStr, + state: jsonObj.getOrFail("state", "build status").getStr, details: jsonObj.getIfExists("details").getStr("") ) +proc parseRunRequest*(reqStr: string): RunRequest = + let reqJson = parseJson(reqStr) + + result = RunRequest( + projectName: reqJson.getOrFail("projectName", "RunRequest").getStr, + stepName: reqJson.getOrFail("stepName", "RunRequest").getStr, + buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr, + workspaceDir: reqJson.getOrFail("workspaceDir", "RunRequest").getStr, + forceRebuild: reqJson.getOrFail("forceRebuild", "RunRequest").getBVal) + +# TODO: can we use the marshal module for this? proc `%`*(s: BuildStatus): JsonNode = result = %* { "state": s.state, "details": s.details } -proc `$`*(s: BuildStatus): string = - result =pretty(%s) +proc `%`*(req: RunRequest): JsonNode = + result = %* { + "projectName": req.projectName, + "stepName": req.stepName, + "buildRef": req.buildRef, + "workspaceDir": req.workspaceDir, + "forceRebuild": req.forceRebuild + } + +proc `$`*(s: BuildStatus): string = result = pretty(%s) +proc `$`*(req: RunRequest): string = result = pretty(%req) diff --git a/src/main/nim/strawboss/core.nim b/src/main/nim/strawboss/core.nim index bde74e1..f3096e4 100644 --- a/src/main/nim/strawboss/core.nim +++ b/src/main/nim/strawboss/core.nim @@ -2,24 +2,23 @@ import logging, nre, os, osproc, sequtils, streams, strtabs, strutils, tables, t import private/util import configuration -from posix import fork, Pid +from posix import link type - Workspace* = ref object - artifactsRepo*: string - env*: StringTableRef - workingDir*: string - project*: ProjectCfg - - RunSummary* = object - project*: ProjectCfg - step*: Step - buildVersion*, statusFile*: string - workerPid*: Pid - - RunRequest* = object - projectName*, stepName*, buildRef*, workspaceDir*: string - forceRebuild*: bool + Workspace = ref object ## Data needed by internal build process + artifactsDir*: string ## absolute path to the directory for this version + artifactsRepo*: string ## absolute path to the global artifacts repo + buildRef*: string ## git-style commit reference to the revision we are building + dir*: string ## absolute path to the working directory + env*: StringTableRef ## environment variables for all build processes + openedFiles*: seq[File] ## all files that we have opened that need to be closed + outputHandler*: HandleProcMsgCB ## handler for process output + project*: ProjectCfg ## the project configuration + projectDef*: ProjectDef ## the StrawBoss project definition + status*: BuildStatus ## the current status of the build + statusFile*: string ## absolute path to the build status file + step*: Step ## the step we're building + version*: string ## project version as returned by versionCmd proc resolveEnvVars(line: string, env: StringTableRef): string = result = line @@ -27,212 +26,188 @@ proc resolveEnvVars(line: string, env: StringTableRef): string = let key = if found[1] == '{': found[2..^2] else: found[1..^1] if env.hasKey(key): result = result.replace(found, env[key]) -proc combineProcMsgHandlers(a, b: HandleProcMsgCB): HandleProcMsgCB = - if a == nil: result = b - elif b == nil: result = a - else: - result = proc(cmd: string, outMsg, errMsg: TaintedString): void = - a(cmd, outMsg, errMsg) - b(cmd, outMsg, errMsg) +proc emitStatus(status: BuildStatus, statusFilePath: string, + outputHandler: HandleProcMsgCB): BuildStatus = + if statusFilePath != nil: writeFile(statusFilePath, $status) + if outputHandler != nil: + outputHandler.sendMsg(status.state & ": " & status.details) + result = status -proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string, - outputHandler: HandleProcMsgCB = nil): Workspace = +proc publishStatus(wksp: Workspace, state, details: string) = + let status = BuildStatus(state: state, details: details) + wksp.status = emitStatus(status, wksp.statusFile, wksp.outputHandler) - outputHandler.sendMsg "Setting up to do work for '" & projectDef.name & - "' at ref " & buildRef & "." - - var env = loadEnv() - env["GIT_DIR"] = ".git" - - # Create a temp directory that we'll work in - let projDir = mkdtemp() - debug "Workspace for '" & projectDef.name & ": " & projDir - assert projDir.isAbsolute +proc setupProject(wksp: Workspace) = # Clone the project into the $temp/repo directory - let cloneResult = exec("git", projDir, ["clone", projectDef.repo, "repo"], - env, {poUsePath}, outputHandler) + let cloneResult = exec("git", wksp.dir, + ["clone", wksp.projectDef.repo, "repo"], + wksp.env, {poUsePath}, wksp.outputHandler) if cloneResult.exitCode != 0: - removeDir(projDir) - raiseEx "unable to clone repo for '" & projectDef.name & "'" + raiseEx "unable to clone repo for '" & wksp.projectDef.name & "'" # Checkout the requested ref - let checkoutResult = exec("git", projDir & "/repo", ["checkout", buildRef], - env, {poUsePath}, outputHandler) + let checkoutResult = exec("git", wksp.dir & "/repo", + ["checkout", wksp.buildRef], + wksp.env, {poUsePath}, wksp.outputHandler) if checkoutResult.exitCode != 0: - removeDir(projDir) - raiseEx "unable to checkout ref " & buildRef & " for '" & projectDef.name & "'" + raiseEx "unable to checkout ref " & wksp.buildRef & + " for '" & wksp.projectDef.name & "'" # Find the strawboss project configuration - let projCfgFile = projDir & "/repo/" & projectDef.cfgFilePath + let projCfgFile = wksp.dir & "/repo/" & wksp.projectDef.cfgFilePath if not existsFile(projCfgFile): - removeDir(projDir) raiseEx "Cannot find strawboss project configuration in the project " & - "repo (expected at '" & projectDef.cfgFilePath & "')." + "repo (expected at '" & wksp.projectDef.cfgFilePath & "')." - let projectCfg = loadProjectConfig(projCfgFile) - result = Workspace(env: env, workingDir: projDir, project: projectCfg, - artifactsRepo: artifactsRepo) + wksp.project = loadProjectConfig(projCfgFile) # Merge in the project-defined env vars - for k, v in projectDef.envVars: result.env[k] = v + for k, v in wksp.projectDef.envVars: wksp.env[k] = v # Get the build version let versionProc = startProcess( - projectCfg.versionCmd, # command - projDir & "/repo", # working dir + wksp.project.versionCmd, # command + wksp.dir & "/repo", # working dir [], # args - result.env, # environment + wksp.env, # environment {poUsePath, poEvalCommand}) # options - let versionResult = waitForWithOutput(versionProc, outputHandler, - projectCfg.versionCmd) + let versionResult = waitForWithOutput(versionProc, wksp.outputHandler, + wksp.project.versionCmd) if versionResult.exitCode != 0: - removeDir(projDir) - raiseEx "Version command (" & projectCfg.versionCmd & ") returned non-zero exit code." + raiseEx "Version command (" & wksp.project.versionCmd & + ") returned non-zero exit code." - outputHandler.sendMsg "Building version " & versionResult.output.strip - result.env["VERSION"] = versionResult.output.strip + wksp.outputHandler.sendMsg "Building version " & versionResult.output.strip + wksp.version = versionResult.output.strip + wksp.env["VERSION"] = wksp.version + +proc runStep*(wksp: Workspace, step: Step) = -proc runStep*(step: Step, wksp: Workspace, - givenOutputHandler: HandleProcMsgCB = nil): void = let SB_EXPECTED_VARS = ["VERSION"] - let stepArtifactDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & - step.name & "/" & wksp.env["VERSION"] - let statusFile = stepArtifactDir & ".status.json" - if not existsDir(stepArtifactDir): createDir(stepArtifactDir) + wksp.publishStatus("running", + "running '" & step.name & "' for version " & wksp.version & + " from " & wksp.buildRef) - # Have we tried to build this before and are we caching the results? - if existsFile(statusFile) and not step.dontSkip: - let status = loadBuildStatus(statusFile) + # Ensure all expected environment variables are present. + for k in (step.expectedEnv & @SB_EXPECTED_VARS): + if not wksp.env.hasKey(k): + raiseEx "step " & step.name & " failed: missing required env variable: " & k - # If we succeeded last time, no need to rebuild - if status.state == "complete": - givenOutputHandler.sendMsg "Skipping step '" & step.name & "' for version '" & - wksp.env["VERSION"] & "': already completed." - return - else: - givenOutputHandler.sendMsg "Rebuilding failed step '" & step.name & "' for version '" & - wksp.env["VERSION"] & "'." + # Ensure that artifacts in steps we depend on are present + # TODO: detect circular-references in dependency trees. + for dep in step.depends: + if not wksp.project.steps.hasKey(dep): + raiseEx step.name & " depends on " & dep & + " but there is no step named " & dep + let depStep = wksp.project.steps[dep] - givenOutputHandler.sendMsg "Running step '" & step.name & "' for " & wksp.project.name - writeFile(statusFile, $BuildStatus(state: "running", details: "")) + # Run that step (may get skipped) + runStep(wksp, depStep) - var stdoutLogFile, stderrLogFile: File + # Add the artifacts directory for the dependent step to our env so that + # further steps can reference it via $_DIR + wksp.env[depStep.name & "_DIR"] = wksp.artifactsRepo & "/" & + wksp.project.name & "/" & dep & "/" & wksp.version - try: + # Run the step command, piping in cmdInput + wksp.outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd + let cmdProc = startProcess(step.stepCmd, + wksp.dir & "/repo/" & step.workingDir, + [], wksp.env, {poUsePath, poEvalCommand}) - var outputHandler: HandleProcMsgCB + let cmdInStream = inputStream(cmdProc) - # Make sure we log output to the stdout and sterr log files - if not (open(stdoutLogFile, stepArtifactDir & "/stdout.log", fmWrite) and - open(stderrLogFile, stepArtifactDir & "/stderr.log", fmWrite)): - givenOutputHandler.sendMsg nil, "Failed to open log files for STDOUT and STDERR." - outputHandler = givenOutputHandler - if stdoutLogFile != nil: close(stdoutLogFile) - if stderrLogFile != nil: close(stderrLogFile) - else: - outputHandler = combineProcMsgHandlers( - givenOutputHandler, - makeProcMsgHandler(stdoutLogFile, stderrLogFile)) + # Replace env variables in step cmdInput as we pipe it in + for line in step.cmdInput: cmdInStream.writeLine(line.resolveEnvVars(wksp.env)) + cmdInStream.flush() + cmdInStream.close() + + let cmdResult = waitForWithOutput(cmdProc, wksp.outputHandler, step.stepCmd) - # Ensure all expected environment variables are present. - for k in (step.expectedEnv & @SB_EXPECTED_VARS): - if not wksp.env.hasKey(k): - debug "workspace.env = " & $(wksp.env) - raiseEx "step " & step.name & " failed: missing required env variable: " & k + if cmdResult.exitCode != 0: + raiseEx "step " & step.name & " failed: step command returned non-zero exit code" - # Ensure that artifacts in steps we depend on are present - # TODO: detect circular-references in dependency trees. - for dep in step.depends: - if not wksp.project.steps.hasKey(dep): - raiseEx step.name & " depends on " & dep & - " but there is no step named " & dep - let depStep = wksp.project.steps[dep] + # Gather the output artifacts (if we have any) + wksp.outputHandler.sendMsg "artifacts: " & $step.artifacts + if step.artifacts.len > 0: + for a in step.artifacts: + let artifactPath = a.resolveEnvVars(wksp.env) + let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1] + try: + wksp.outputHandler.sendMsg "copy " & wksp.dir & "/repo/" & step.workingDir & "/" & artifactPath & " -> " & wksp.artifactsDir & "/" & artifactName + copyFile(wksp.dir & "/repo/" & step.workingDir & "/" & artifactPath, + wksp.artifactsDir & "/" & artifactName) + except: + raiseEx "step " & step.name & " failed: unable to copy artifact " & + artifactPath & ":\n" & getCurrentExceptionMsg() - # Run that step (may get skipped) - let depDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & - dep & "/" & wksp.env["VERSION"] - - runStep(depStep, wksp) - - # Add the artifacts directory for the dependent step to our env so that - # further steps can reference it via $_DIR - echo "FP: " & depDir - wksp.env[depStep.name & "_DIR"] = depDir - - # Run the step command, piping in cmdInput - outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd - let cmdProc = startProcess(step.stepCmd, - wksp.workingDir & "/repo/" & step.workingDir, - [], wksp.env, {poUsePath, poEvalCommand}) - - let cmdInStream = inputStream(cmdProc) - - # Replace env variables in step cmdInput as we pipe it in - for line in step.cmdInput: cmdInStream.writeLine(line.resolveEnvVars(wksp.env)) - cmdInStream.flush() - cmdInStream.close() - - let cmdResult = waitForWithOutput(cmdProc, outputHandler, step.stepCmd) - - if cmdResult.exitCode != 0: - raiseEx "step " & step.name & " failed: step command returned non-zero exit code" - - # Gather the output artifacts (if we have any) - if step.artifacts.len > 0: - for a in step.artifacts: - let artifactPath = a.resolveEnvVars(wksp.env) - let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1] - try: - copyFile(wksp.workingDir & "/repo/" & step.workingDir & "/" & artifactPath, - stepArtifactDir & "/" & artifactName) - except: - raiseEx "step " & step.name & " failed: unable to copy artifact " & - artifactPath & ":\n" & getCurrentExceptionMsg() - - writeFile(statusFile, $BuildStatus(state: "complete", details: "")) - - except: - writeFile(statusFile, $BuildStatus( - state: "failed", - details: getCurrentExceptionMsg())) - - finally: - if stdoutLogFile != nil: close(stdoutLogFile) - if stderrLogFile != nil: close(stderrLogFile) + wksp.publishStatus("complete", "") proc runStep*(cfg: StrawBossConfig, req: RunRequest, - outputHandler: HandleProcMsgCB = nil): RunSummary = + outputHandler: HandleProcMsgCB = nil): BuildStatus = - if not existsDir(req.workspaceDir): createDir(req.workspaceDir) - let statusFile = req.workspaceDir & "/" & "status.json" + result = BuildStatus( + state: "setup", + details: "initializing build workspace") + discard emitStatus(result, nil, outputHandler) + + var wksp: Workspace try: - writeFile(statusFile, $BuildStatus( - state: "setup", - details: "Preparing working environment.")) + assert req.workspaceDir.isAbsolute + if not existsDir(req.workspaceDir): createDir(req.workspaceDir) # Find the project definition let matching = cfg.projects.filterIt(it.name == req.projectName) if matching.len == 0: raiseEx "no such project: " & req.projectName elif matching.len > 1: raiseEx "more than one project named : " & req.projectName - let projectDef = matching[0] + # Read in the existing system environment + var env = loadEnv() + env["GIT_DIR"] = ".git" - # Find the commit reference we're building - let foundBuildRef = - if req.buildRef != nil and req.buildRef.len > 0: req.buildRef - else: projectDef.defaultBranch + # Setup our STDOUT and STDERR files + let stdoutFile = open(req.workspaceDir & "/stdout.log", fmWrite) + let stderrFile = open(req.workspaceDir & "/stderr.log", fmWrite) - let wksp = setupProjectForWork(projectDef, foundBuildRef, cfg.artifactsRepo, - outputHandler) + let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile) - # Find the step + wksp = Workspace( + artifactsDir: nil, + artifactsRepo: cfg.artifactsRepo, + buildRef: + if req.buildRef != nil and req.buildRef.len > 0: req.buildRef + else: matching[0].defaultBranch, + dir: req.workspaceDir, + env: env, + openedFiles: @[stdoutFile, stderrFile], + outputHandler: combineProcMsgHandlers(outputHandler, logFilesOH), + project: ProjectCfg(), + projectDef: matching[0], + status: result, + statusFile: req.workspaceDir & "/" & "status.json", + step: Step(), + version: nil) + + except: + result = BuildStatus(state: "failed", + details: getCurrentExceptionMsg()) + try: discard emitStatus(result, nil, outputHandler) + except: discard "" + + try: + # Clone the repo and setup the working environment + wksp.publishStatus("setup", + "cloning project repo and preparing to run '" & req.stepName & "'") + wksp.setupProject() + + # Find the requested step if not wksp.project.steps.hasKey(req.stepName): raiseEx "no step name '" & req.stepName & "' for " & req.projectName @@ -241,15 +216,55 @@ proc runStep*(cfg: StrawBossConfig, req: RunRequest, # Enfore forceRebuild if req.forceRebuild: step.dontSkip = true - result = RunSummary( - project: wksp.project, - step: step, - buildVersion: wksp.env["VERSION"], - statusFile: wksp.artifactsRepo & "/" & wksp.project.name & "/" & step.name & - "/" & wksp.env["VERSION"] & ".status.json") - if req.async: - let pid = fork() - if pid == 0: runStep(step, wksp, outputHandler) # if we are the child - else: result.workerPid = pid # if we are the parent + # Compose the path to the artifacts directory for this step and version + wksp.artifactsDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & + step.name & "/" & wksp.version + + # Have we tried to build this before and are we caching the results? + if existsFile(wksp.artifactsDir & "/status.json") and not step.dontSkip: + let prevStatus = loadBuildStatus(wksp.artifactsDir & "/status.json") + + # If we succeeded last time, no need to rebuild + if prevStatus.state == "complete": + wksp.outputHandler.sendMsg( + "Skipping step '" & step.name & "' for version '" & + wksp.version & "': already completed.") + return prevStatus + else: + wksp.outputHandler.sendMsg( + "Rebuilding failed step '" & step.name & "' for version '" & + wksp.version & "'.") + + # Make the artifacts directory if it doesn't already exist + if not existsDir(wksp.artifactsDir): createDir(wksp.artifactsDir) + + # Link status file and output logs to the artifacts dir + for fn in @["status.json", "stdout.log", "stderr.log"]: + # TODO: roll old files instead of delete them? + if existsFile(wksp.artifactsDir & "/" & fn): + removeFile(wksp.artifactsDir & "/" & fn) + + if link(wksp.dir & "/" & fn, wksp.artifactsDir & "/" & fn) != 0: + wksp.outputHandler.sendMsg(nil, + "WARN: could not link " & fn & " to artifacts dir.") + + runStep(wksp, step) + + result = wksp.status + + except: + let msg = getCurrentExceptionMsg() + try: + wksp.publishStatus("failed", msg) + result = wksp.status + except: + result = BuildStatus(state: "failed", details: msg) + try: discard emitStatus(result, nil, outputHandler) + except: discard "" + + finally: + if wksp != nil: + for f in wksp.openedFiles: + try: close(f) + except: discard "" - else: runStep(step, wksp, outputHandler) diff --git a/src/main/nim/strawboss/private/util.nim b/src/main/nim/strawboss/private/util.nim index 285b043..878fa1f 100644 --- a/src/main/nim/strawboss/private/util.nim +++ b/src/main/nim/strawboss/private/util.nim @@ -66,3 +66,13 @@ proc makeProcMsgHandler*(outSink, errSink: Stream): HandleProcMsgCB = let prefix = if cmd != nil: cmd & ": " else: "" if outMsg != nil: outSink.writeLine(prefix & outMsg) if errMsg != nil: errSink.writeLine(prefix & errMsg) + +proc combineProcMsgHandlers*(a, b: HandleProcMsgCB): HandleProcMsgCB = + if a == nil: result = b + elif b == nil: result = a + else: + result = proc(cmd: string, outMsg, errMsg: TaintedString): void = + a(cmd, outMsg, errMsg) + b(cmd, outMsg, errMsg) + + diff --git a/src/main/nim/strawboss/server.nim b/src/main/nim/strawboss/server.nim index ae36b5b..85023ad 100644 --- a/src/main/nim/strawboss/server.nim +++ b/src/main/nim/strawboss/server.nim @@ -1,13 +1,26 @@ -import asyncdispatch, jester, json +import asyncdispatch, jester, json, osproc, tempfile -import ./configuration, ./core +import ./configuration, ./core, private/util settings: port = Port(8180) +type Worker = object + process*: Process + workingDir*: string + +proc spawnWorker(req: RunRequest): Worker = + let dir = mkdtemp() + var args = @["run", req.projectName, req.stepName, "-r", req.buildRef, "-w", dir] + if req.forceRebuild: args.add("-f") + result = Worker( + process: startProcess("strawboss", ".", args, loadEnv(), {poUsePath}), + workingDir: dir) proc start*(givenCfg: StrawBossConfig): void = + var workers: seq[Worker] = @[] + routes: get "/api/ping": resp $(%*"pong"), "application/json" @@ -16,18 +29,10 @@ proc start*(givenCfg: StrawBossConfig): void = resp $(%*[]), "application/json" post "/api/project/@projectName/@stepName/run/@buildRef?": - let req = RunRequest( + workers.add(spawnWorker(RunRequest( projectName: @"projectName", stepName: @"stepName", buildRef: if @"buildRef" != "": @"buildRef" else: nil, - async: true, - forceRebuild: false) # TODO support this with optional query params - -# try: -# let runSummary = core.runStep(givenCfg, req) -# except: -# discard "" -# # TODO - + forceRebuild: false))) # TODO support this with optional query params runForever() diff --git a/strawboss.nimble b/strawboss.nimble index 182beed..be3cccc 100644 --- a/strawboss.nimble +++ b/strawboss.nimble @@ -1,7 +1,7 @@ # Package bin = @["strawboss"] -version = "0.1.0" +version = "0.2.0" author = "Jonathan Bernard" description = "My personal continious integration worker." license = "MIT"