jdb/strawboss
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finished refactor towards process-based workers.

Browse Source
This commit is contained in:
Jonathan Bernard 2017-03-13 07:29:37 -05:00
parent cc28e7f4bf
commit 0976871563
6 changed files with 271 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/main/nim/strawboss.nim
View File

@ -1,26 +1,22 @@
import docopt, logging, os, sequtils, tempfile import docopt, os, sequtils, tempfile
import strawboss/private/util import strawboss/private/util
import strawboss/configuration import strawboss/configuration
import strawboss/core import strawboss/core
import strawboss/server import strawboss/server
import strawboss/supervisor
let SB_VER = "0.1.0" let SB_VER = "0.2.0"
proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) = proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) =
let prefix = if cmd != nil: cmd else: "" let prefix = if cmd != nil: cmd else: ""
if outMsg != nil: info prefix & "(stdout): " & outMsg if outMsg != nil: echo prefix & "(stdout): " & outMsg
if errMsg != nil: info prefix & "(stderr): " & errMsg if errMsg != nil: echo prefix & "(stderr): " & errMsg
when isMainModule: when isMainModule:
if logging.getHandlers().len == 0:
logging.addHandler(newConsoleLogger())
var cfg = loadStrawBossConfig("strawboss.config.json") var cfg = loadStrawBossConfig("strawboss.config.json")
if not existsDir(cfg.artifactsRepo): if not existsDir(cfg.artifactsRepo):
info "Artifacts repo (" & cfg.artifactsRepo & ") does not exist. Creating..." echo "Artifacts repo (" & cfg.artifactsRepo & ") does not exist. Creating..."
createDir(cfg.artifactsRepo) createDir(cfg.artifactsRepo)
cfg.artifactsRepo = expandFilename(cfg.artifactsRepo) cfg.artifactsRepo = expandFilename(cfg.artifactsRepo)
@ -28,7 +24,6 @@ when isMainModule:
let doc = """ let doc = """
Usage: Usage:
strawboss serve strawboss serve
strawboss supervisor [-i <in-file>] [-o <out-file>]
strawboss run <project> <step> [options] strawboss run <project> <step> [options]
Options Options
@ -45,24 +40,25 @@ Options
let args = docopt(doc, version = "strawboss v" & SB_VER) let args = docopt(doc, version = "strawboss v" & SB_VER)
echo $args
if args["run"]: if args["run"]:
let req = RunRequest( let req = RunRequest(
projectName: $args["<project>"], projectName: $args["<project>"],
stepName: $args["<step>"], stepName: $args["<step>"],
buildRef: if args["--rreference"]: $args["<ref>"] else: nil, buildRef: if args["--reference"]: $args["--reference"] else: nil,
forceRebuild: args["--force-rebuild"], forceRebuild: args["--force-rebuild"],
workspaceDir: if args["--workspace"]: $args["<workspace>"] else: mkdtemp()) workspaceDir: if args["--workspace"]: $args["<workspace>"] else: mkdtemp())
try: try:
let summary = core.runStep(cfg, req, logProcOutput) let status = core.runStep(cfg, req, logProcOutput)
# TODO: inspect result if status.state == "failed": raiseEx status.details
echo "strawboss: build passed."
except: except:
fatal "strawboss: " & getCurrentExceptionMsg() & "." echo "strawboss: build FAILED: " & getCurrentExceptionMsg() & "."
quit(QuitFailure) quit(QuitFailure)
finally:
if existsDir(req.workspaceDir): removeDir(req.workspaceDir)
info "strawboss: build passed"
elif
elif args["serve"]: server.start(cfg) elif args["serve"]: server.start(cfg)

54
src/main/nim/strawboss/configuration.nim
View File

@ -25,6 +25,10 @@ type
artifactsRepo*: string artifactsRepo*: string
projects*: seq[ProjectDef] projects*: seq[ProjectDef]
RunRequest* = object
projectName*, stepName*, buildRef*, workspaceDir*: string
forceRebuild*: bool
# internal utils # internal utils
let nullNode = newJNull() let nullNode = newJNull()
@ -32,6 +36,10 @@ proc getIfExists(n: JsonNode, key: string): JsonNode =
result = if n.hasKey(key): n[key] result = if n.hasKey(key): n[key]
else: nullNode else: nullNode
proc getOrFail(n: JsonNode, key: string, objName: string = ""): JsonNode =
if not n.hasKey(key): raiseEx objName & " missing key " & key
return n[key]
# Configuration parsing code # Configuration parsing code
proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig = proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig =
@ -43,12 +51,6 @@ proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig =
var projectDefs: seq[ProjectDef] = @[] var projectDefs: seq[ProjectDef] = @[]
for pJson in jsonCfg.getIfExists("projects").getElems: for pJson in jsonCfg.getIfExists("projects").getElems:
if not pJson.hasKey("name"):
raiseEx "a project definition is missing the project name"
if not pJson.hasKey("repo"):
raiseEx "a project definition is missing the project repo configuration"
var envVars = newStringTable(modeCaseSensitive) var envVars = newStringTable(modeCaseSensitive)
for k, v in pJson.getIfExists("envVars").getFields: envVars[k] = v.getStr("") for k, v in pJson.getIfExists("envVars").getFields: envVars[k] = v.getStr("")
@ -56,9 +58,9 @@ proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig =
ProjectDef( ProjectDef(
cfgFilePath: pJson.getIfExists("cfgFilePath").getStr("strawboss.json"), cfgFilePath: pJson.getIfExists("cfgFilePath").getStr("strawboss.json"),
defaultBranch: pJson.getIfExists("defaultBranch").getStr("master"), defaultBranch: pJson.getIfExists("defaultBranch").getStr("master"),
name: pJson["name"].getStr, name: pJson.getOrFail("name", "project definition").getStr,
envVars: envVars, envVars: envVars,
repo: pJson["repo"].getStr)) repo: pJson.getOrFail("repo", "project definition").getStr))
result = StrawBossConfig( result = StrawBossConfig(
artifactsRepo: jsonCfg.getIfExists("artifactsRepo").getStr("artifacts"), artifactsRepo: jsonCfg.getIfExists("artifactsRepo").getStr("artifacts"),
@ -70,14 +72,11 @@ proc loadProjectConfig*(cfgFile: string): ProjectCfg =
let jsonCfg = parseFile(cfgFile) let jsonCfg = parseFile(cfgFile)
if not jsonCfg.hasKey("name"):
raiseEx "project configuration is missing a name"
if not jsonCfg.hasKey("steps"): if not jsonCfg.hasKey("steps"):
raiseEx "project configuration is missing steps definition" raiseEx "project configuration is missing steps definition"
var steps = initTable[string, Step]() var steps = initTable[string, Step]()
for sName, pJson in jsonCfg["steps"].getFields: for sName, pJson in jsonCfg.getOrFail("steps", "project configuration").getFields:
steps[sName] = Step( steps[sName] = Step(
name: sName, name: sName,
workingDir: pJson.getIfExists("workingDir").getStr("."), workingDir: pJson.getIfExists("workingDir").getStr("."),
@ -92,7 +91,7 @@ proc loadProjectConfig*(cfgFile: string): ProjectCfg =
warn "Step " & sName & " uses 'sh' as its command but has no cmdInput." warn "Step " & sName & " uses 'sh' as its command but has no cmdInput."
result = ProjectCfg( result = ProjectCfg(
name: jsonCfg["name"].getStr, name: jsonCfg.getOrFail("name", "project configuration").getStr,
versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"), versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"),
steps: steps) steps: steps)
@ -100,18 +99,35 @@ proc loadBuildStatus*(statusFile: string): BuildStatus =
if not existsFile(statusFile): raiseEx "status file not found: " & statusFile if not existsFile(statusFile): raiseEx "status file not found: " & statusFile
let jsonObj = parseFile(statusFile) let jsonObj = parseFile(statusFile)
if not jsonObj.hasKey("state"):
raiseEx "project status is missing the 'state' field"
result = BuildStatus( result = BuildStatus(
state: jsonObj["state"].getStr, state: jsonObj.getOrFail("state", "build status").getStr,
details: jsonObj.getIfExists("details").getStr("") ) details: jsonObj.getIfExists("details").getStr("") )
proc parseRunRequest*(reqStr: string): RunRequest =
let reqJson = parseJson(reqStr)
result = RunRequest(
projectName: reqJson.getOrFail("projectName", "RunRequest").getStr,
stepName: reqJson.getOrFail("stepName", "RunRequest").getStr,
buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr,
workspaceDir: reqJson.getOrFail("workspaceDir", "RunRequest").getStr,
forceRebuild: reqJson.getOrFail("forceRebuild", "RunRequest").getBVal)
# TODO: can we use the marshal module for this?
proc `%`*(s: BuildStatus): JsonNode = proc `%`*(s: BuildStatus): JsonNode =
result = %* { result = %* {
"state": s.state, "state": s.state,
"details": s.details "details": s.details
} }
proc `$`*(s: BuildStatus): string = proc `%`*(req: RunRequest): JsonNode =
result =pretty(%s) result = %* {
"projectName": req.projectName,
"stepName": req.stepName,
"buildRef": req.buildRef,
"workspaceDir": req.workspaceDir,
"forceRebuild": req.forceRebuild
}
proc `$`*(s: BuildStatus): string = result = pretty(%s)
proc `$`*(req: RunRequest): string = result = pretty(%req)

375
src/main/nim/strawboss/core.nim
View File

@ -2,24 +2,23 @@ import logging, nre, os, osproc, sequtils, streams, strtabs, strutils, tables, t
import private/util import private/util
import configuration import configuration
from posix import fork, Pid from posix import link
type type
Workspace* = ref object Workspace = ref object ## Data needed by internal build process
artifactsRepo*: string artifactsDir*: string ## absolute path to the directory for this version
env*: StringTableRef artifactsRepo*: string ## absolute path to the global artifacts repo
workingDir*: string buildRef*: string ## git-style commit reference to the revision we are building
project*: ProjectCfg dir*: string ## absolute path to the working directory
env*: StringTableRef ## environment variables for all build processes
RunSummary* = object openedFiles*: seq[File] ## all files that we have opened that need to be closed
project*: ProjectCfg outputHandler*: HandleProcMsgCB ## handler for process output
step*: Step project*: ProjectCfg ## the project configuration
buildVersion*, statusFile*: string projectDef*: ProjectDef ## the StrawBoss project definition
workerPid*: Pid status*: BuildStatus ## the current status of the build
statusFile*: string ## absolute path to the build status file
RunRequest* = object step*: Step ## the step we're building
projectName*, stepName*, buildRef*, workspaceDir*: string version*: string ## project version as returned by versionCmd
forceRebuild*: bool
proc resolveEnvVars(line: string, env: StringTableRef): string = proc resolveEnvVars(line: string, env: StringTableRef): string =
result = line result = line
@ -27,212 +26,188 @@ proc resolveEnvVars(line: string, env: StringTableRef): string =
let key = if found[1] == '{': found[2..^2] else: found[1..^1] let key = if found[1] == '{': found[2..^2] else: found[1..^1]
if env.hasKey(key): result = result.replace(found, env[key]) if env.hasKey(key): result = result.replace(found, env[key])
proc combineProcMsgHandlers(a, b: HandleProcMsgCB): HandleProcMsgCB = proc emitStatus(status: BuildStatus, statusFilePath: string,
if a == nil: result = b outputHandler: HandleProcMsgCB): BuildStatus =
elif b == nil: result = a if statusFilePath != nil: writeFile(statusFilePath, $status)
else: if outputHandler != nil:
result = proc(cmd: string, outMsg, errMsg: TaintedString): void = outputHandler.sendMsg(status.state & ": " & status.details)
a(cmd, outMsg, errMsg) result = status
b(cmd, outMsg, errMsg)
proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string, proc publishStatus(wksp: Workspace, state, details: string) =
outputHandler: HandleProcMsgCB = nil): Workspace = let status = BuildStatus(state: state, details: details)
wksp.status = emitStatus(status, wksp.statusFile, wksp.outputHandler)
outputHandler.sendMsg "Setting up to do work for '" & projectDef.name & proc setupProject(wksp: Workspace) =
"' at ref " & buildRef & "."
var env = loadEnv()
env["GIT_DIR"] = ".git"
# Create a temp directory that we'll work in
let projDir = mkdtemp()
debug "Workspace for '" & projectDef.name & ": " & projDir
assert projDir.isAbsolute
# Clone the project into the $temp/repo directory # Clone the project into the $temp/repo directory
let cloneResult = exec("git", projDir, ["clone", projectDef.repo, "repo"], let cloneResult = exec("git", wksp.dir,
env, {poUsePath}, outputHandler) ["clone", wksp.projectDef.repo, "repo"],
wksp.env, {poUsePath}, wksp.outputHandler)
if cloneResult.exitCode != 0: if cloneResult.exitCode != 0:
removeDir(projDir) raiseEx "unable to clone repo for '" & wksp.projectDef.name & "'"
raiseEx "unable to clone repo for '" & projectDef.name & "'"
# Checkout the requested ref # Checkout the requested ref
let checkoutResult = exec("git", projDir & "/repo", ["checkout", buildRef], let checkoutResult = exec("git", wksp.dir & "/repo",
env, {poUsePath}, outputHandler) ["checkout", wksp.buildRef],
wksp.env, {poUsePath}, wksp.outputHandler)
if checkoutResult.exitCode != 0: if checkoutResult.exitCode != 0:
removeDir(projDir) raiseEx "unable to checkout ref " & wksp.buildRef &
raiseEx "unable to checkout ref " & buildRef & " for '" & projectDef.name & "'" " for '" & wksp.projectDef.name & "'"
# Find the strawboss project configuration # Find the strawboss project configuration
let projCfgFile = projDir & "/repo/" & projectDef.cfgFilePath let projCfgFile = wksp.dir & "/repo/" & wksp.projectDef.cfgFilePath
if not existsFile(projCfgFile): if not existsFile(projCfgFile):
removeDir(projDir)
raiseEx "Cannot find strawboss project configuration in the project " & raiseEx "Cannot find strawboss project configuration in the project " &
"repo (expected at '" & projectDef.cfgFilePath & "')." "repo (expected at '" & wksp.projectDef.cfgFilePath & "')."
let projectCfg = loadProjectConfig(projCfgFile) wksp.project = loadProjectConfig(projCfgFile)
result = Workspace(env: env, workingDir: projDir, project: projectCfg,
artifactsRepo: artifactsRepo)
# Merge in the project-defined env vars # Merge in the project-defined env vars
for k, v in projectDef.envVars: result.env[k] = v for k, v in wksp.projectDef.envVars: wksp.env[k] = v
# Get the build version # Get the build version
let versionProc = startProcess( let versionProc = startProcess(
projectCfg.versionCmd, # command wksp.project.versionCmd, # command
projDir & "/repo", # working dir wksp.dir & "/repo", # working dir
[], # args [], # args
result.env, # environment wksp.env, # environment
{poUsePath, poEvalCommand}) # options {poUsePath, poEvalCommand}) # options
let versionResult = waitForWithOutput(versionProc, outputHandler, let versionResult = waitForWithOutput(versionProc, wksp.outputHandler,
projectCfg.versionCmd) wksp.project.versionCmd)
if versionResult.exitCode != 0: if versionResult.exitCode != 0:
removeDir(projDir) raiseEx "Version command (" & wksp.project.versionCmd &
raiseEx "Version command (" & projectCfg.versionCmd & ") returned non-zero exit code." ") returned non-zero exit code."
outputHandler.sendMsg "Building version " & versionResult.output.strip wksp.outputHandler.sendMsg "Building version " & versionResult.output.strip
result.env["VERSION"] = versionResult.output.strip wksp.version = versionResult.output.strip
wksp.env["VERSION"] = wksp.version
proc runStep*(wksp: Workspace, step: Step) =
proc runStep*(step: Step, wksp: Workspace,
givenOutputHandler: HandleProcMsgCB = nil): void =
let SB_EXPECTED_VARS = ["VERSION"] let SB_EXPECTED_VARS = ["VERSION"]
let stepArtifactDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" &
step.name & "/" & wksp.env["VERSION"]
let statusFile = stepArtifactDir & ".status.json"
if not existsDir(stepArtifactDir): createDir(stepArtifactDir) wksp.publishStatus("running",
"running '" & step.name & "' for version " & wksp.version &
" from " & wksp.buildRef)
# Have we tried to build this before and are we caching the results? # Ensure all expected environment variables are present.
if existsFile(statusFile) and not step.dontSkip: for k in (step.expectedEnv & @SB_EXPECTED_VARS):
let status = loadBuildStatus(statusFile) if not wksp.env.hasKey(k):
raiseEx "step " & step.name & " failed: missing required env variable: " & k
# If we succeeded last time, no need to rebuild # Ensure that artifacts in steps we depend on are present
if status.state == "complete": # TODO: detect circular-references in dependency trees.
givenOutputHandler.sendMsg "Skipping step '" & step.name & "' for version '" & for dep in step.depends:
wksp.env["VERSION"] & "': already completed." if not wksp.project.steps.hasKey(dep):
return raiseEx step.name & " depends on " & dep &
else: " but there is no step named " & dep
givenOutputHandler.sendMsg "Rebuilding failed step '" & step.name & "' for version '" & let depStep = wksp.project.steps[dep]
wksp.env["VERSION"] & "'."
givenOutputHandler.sendMsg "Running step '" & step.name & "' for " & wksp.project.name # Run that step (may get skipped)
writeFile(statusFile, $BuildStatus(state: "running", details: "")) runStep(wksp, depStep)
var stdoutLogFile, stderrLogFile: File # Add the artifacts directory for the dependent step to our env so that
# further steps can reference it via $<stepname>_DIR
wksp.env[depStep.name & "_DIR"] = wksp.artifactsRepo & "/" &
wksp.project.name & "/" & dep & "/" & wksp.version
try: # Run the step command, piping in cmdInput
wksp.outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd
let cmdProc = startProcess(step.stepCmd,
wksp.dir & "/repo/" & step.workingDir,
[], wksp.env, {poUsePath, poEvalCommand})
var outputHandler: HandleProcMsgCB let cmdInStream = inputStream(cmdProc)
# Make sure we log output to the stdout and sterr log files # Replace env variables in step cmdInput as we pipe it in
if not (open(stdoutLogFile, stepArtifactDir & "/stdout.log", fmWrite) and for line in step.cmdInput: cmdInStream.writeLine(line.resolveEnvVars(wksp.env))
open(stderrLogFile, stepArtifactDir & "/stderr.log", fmWrite)): cmdInStream.flush()
givenOutputHandler.sendMsg nil, "Failed to open log files for STDOUT and STDERR." cmdInStream.close()
outputHandler = givenOutputHandler
if stdoutLogFile != nil: close(stdoutLogFile) let cmdResult = waitForWithOutput(cmdProc, wksp.outputHandler, step.stepCmd)
if stderrLogFile != nil: close(stderrLogFile)
else:
outputHandler = combineProcMsgHandlers(
givenOutputHandler,
makeProcMsgHandler(stdoutLogFile, stderrLogFile))
# Ensure all expected environment variables are present. if cmdResult.exitCode != 0:
for k in (step.expectedEnv & @SB_EXPECTED_VARS): raiseEx "step " & step.name & " failed: step command returned non-zero exit code"
if not wksp.env.hasKey(k):
debug "workspace.env = " & $(wksp.env)
raiseEx "step " & step.name & " failed: missing required env variable: " & k
# Ensure that artifacts in steps we depend on are present # Gather the output artifacts (if we have any)
# TODO: detect circular-references in dependency trees. wksp.outputHandler.sendMsg "artifacts: " & $step.artifacts
for dep in step.depends: if step.artifacts.len > 0:
if not wksp.project.steps.hasKey(dep): for a in step.artifacts:
raiseEx step.name & " depends on " & dep & let artifactPath = a.resolveEnvVars(wksp.env)
" but there is no step named " & dep let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
let depStep = wksp.project.steps[dep] try:
wksp.outputHandler.sendMsg "copy " & wksp.dir & "/repo/" & step.workingDir & "/" & artifactPath & " -> " & wksp.artifactsDir & "/" & artifactName
copyFile(wksp.dir & "/repo/" & step.workingDir & "/" & artifactPath,
wksp.artifactsDir & "/" & artifactName)
except:
raiseEx "step " & step.name & " failed: unable to copy artifact " &
artifactPath & ":\n" & getCurrentExceptionMsg()
# Run that step (may get skipped) wksp.publishStatus("complete", "")
let depDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" &
dep & "/" & wksp.env["VERSION"]
runStep(depStep, wksp)
# Add the artifacts directory for the dependent step to our env so that
# further steps can reference it via $<stepname>_DIR
echo "FP: " & depDir
wksp.env[depStep.name & "_DIR"] = depDir
# Run the step command, piping in cmdInput
outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd
let cmdProc = startProcess(step.stepCmd,
wksp.workingDir & "/repo/" & step.workingDir,
[], wksp.env, {poUsePath, poEvalCommand})
let cmdInStream = inputStream(cmdProc)
# Replace env variables in step cmdInput as we pipe it in
for line in step.cmdInput: cmdInStream.writeLine(line.resolveEnvVars(wksp.env))
cmdInStream.flush()
cmdInStream.close()
let cmdResult = waitForWithOutput(cmdProc, outputHandler, step.stepCmd)
if cmdResult.exitCode != 0:
raiseEx "step " & step.name & " failed: step command returned non-zero exit code"
# Gather the output artifacts (if we have any)
if step.artifacts.len > 0:
for a in step.artifacts:
let artifactPath = a.resolveEnvVars(wksp.env)
let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
try:
copyFile(wksp.workingDir & "/repo/" & step.workingDir & "/" & artifactPath,
stepArtifactDir & "/" & artifactName)
except:
raiseEx "step " & step.name & " failed: unable to copy artifact " &
artifactPath & ":\n" & getCurrentExceptionMsg()
writeFile(statusFile, $BuildStatus(state: "complete", details: ""))
except:
writeFile(statusFile, $BuildStatus(
state: "failed",
details: getCurrentExceptionMsg()))
finally:
if stdoutLogFile != nil: close(stdoutLogFile)
if stderrLogFile != nil: close(stderrLogFile)
proc runStep*(cfg: StrawBossConfig, req: RunRequest, proc runStep*(cfg: StrawBossConfig, req: RunRequest,
outputHandler: HandleProcMsgCB = nil): RunSummary = outputHandler: HandleProcMsgCB = nil): BuildStatus =
if not existsDir(req.workspaceDir): createDir(req.workspaceDir) result = BuildStatus(
let statusFile = req.workspaceDir & "/" & "status.json" state: "setup",
details: "initializing build workspace")
discard emitStatus(result, nil, outputHandler)
var wksp: Workspace
try: try:
writeFile(statusFile, $BuildStatus( assert req.workspaceDir.isAbsolute
state: "setup", if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
details: "Preparing working environment."))
# Find the project definition # Find the project definition
let matching = cfg.projects.filterIt(it.name == req.projectName) let matching = cfg.projects.filterIt(it.name == req.projectName)
if matching.len == 0: raiseEx "no such project: " & req.projectName if matching.len == 0: raiseEx "no such project: " & req.projectName
elif matching.len > 1: raiseEx "more than one project named : " & req.projectName elif matching.len > 1: raiseEx "more than one project named : " & req.projectName
let projectDef = matching[0] # Read in the existing system environment
var env = loadEnv()
env["GIT_DIR"] = ".git"
# Find the commit reference we're building # Setup our STDOUT and STDERR files
let foundBuildRef = let stdoutFile = open(req.workspaceDir & "/stdout.log", fmWrite)
if req.buildRef != nil and req.buildRef.len > 0: req.buildRef let stderrFile = open(req.workspaceDir & "/stderr.log", fmWrite)
else: projectDef.defaultBranch
let wksp = setupProjectForWork(projectDef, foundBuildRef, cfg.artifactsRepo, let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile)
outputHandler)
# Find the step wksp = Workspace(
artifactsDir: nil,
artifactsRepo: cfg.artifactsRepo,
buildRef:
if req.buildRef != nil and req.buildRef.len > 0: req.buildRef
else: matching[0].defaultBranch,
dir: req.workspaceDir,
env: env,
openedFiles: @[stdoutFile, stderrFile],
outputHandler: combineProcMsgHandlers(outputHandler, logFilesOH),
project: ProjectCfg(),
projectDef: matching[0],
status: result,
statusFile: req.workspaceDir & "/" & "status.json",
step: Step(),
version: nil)
except:
result = BuildStatus(state: "failed",
details: getCurrentExceptionMsg())
try: discard emitStatus(result, nil, outputHandler)
except: discard ""
try:
# Clone the repo and setup the working environment
wksp.publishStatus("setup",
"cloning project repo and preparing to run '" & req.stepName & "'")
wksp.setupProject()
# Find the requested step
if not wksp.project.steps.hasKey(req.stepName): if not wksp.project.steps.hasKey(req.stepName):
raiseEx "no step name '" & req.stepName & "' for " & req.projectName raiseEx "no step name '" & req.stepName & "' for " & req.projectName
@ -241,15 +216,55 @@ proc runStep*(cfg: StrawBossConfig, req: RunRequest,
# Enfore forceRebuild # Enfore forceRebuild
if req.forceRebuild: step.dontSkip = true if req.forceRebuild: step.dontSkip = true
result = RunSummary( # Compose the path to the artifacts directory for this step and version
project: wksp.project, wksp.artifactsDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" &
step: step, step.name & "/" & wksp.version
buildVersion: wksp.env["VERSION"],
statusFile: wksp.artifactsRepo & "/" & wksp.project.name & "/" & step.name & # Have we tried to build this before and are we caching the results?
"/" & wksp.env["VERSION"] & ".status.json") if existsFile(wksp.artifactsDir & "/status.json") and not step.dontSkip:
if req.async: let prevStatus = loadBuildStatus(wksp.artifactsDir & "/status.json")
let pid = fork()
if pid == 0: runStep(step, wksp, outputHandler) # if we are the child # If we succeeded last time, no need to rebuild
else: result.workerPid = pid # if we are the parent if prevStatus.state == "complete":
wksp.outputHandler.sendMsg(
"Skipping step '" & step.name & "' for version '" &
wksp.version & "': already completed.")
return prevStatus
else:
wksp.outputHandler.sendMsg(
"Rebuilding failed step '" & step.name & "' for version '" &
wksp.version & "'.")
# Make the artifacts directory if it doesn't already exist
if not existsDir(wksp.artifactsDir): createDir(wksp.artifactsDir)
# Link status file and output logs to the artifacts dir
for fn in @["status.json", "stdout.log", "stderr.log"]:
# TODO: roll old files instead of delete them?
if existsFile(wksp.artifactsDir & "/" & fn):
removeFile(wksp.artifactsDir & "/" & fn)
if link(wksp.dir & "/" & fn, wksp.artifactsDir & "/" & fn) != 0:
wksp.outputHandler.sendMsg(nil,
"WARN: could not link " & fn & " to artifacts dir.")
runStep(wksp, step)
result = wksp.status
except:
let msg = getCurrentExceptionMsg()
try:
wksp.publishStatus("failed", msg)
result = wksp.status
except:
result = BuildStatus(state: "failed", details: msg)
try: discard emitStatus(result, nil, outputHandler)
except: discard ""
finally:
if wksp != nil:
for f in wksp.openedFiles:
try: close(f)
except: discard ""
else: runStep(step, wksp, outputHandler)

10
src/main/nim/strawboss/private/util.nim
View File

@ -66,3 +66,13 @@ proc makeProcMsgHandler*(outSink, errSink: Stream): HandleProcMsgCB =
let prefix = if cmd != nil: cmd & ": " else: "" let prefix = if cmd != nil: cmd & ": " else: ""
if outMsg != nil: outSink.writeLine(prefix & outMsg) if outMsg != nil: outSink.writeLine(prefix & outMsg)
if errMsg != nil: errSink.writeLine(prefix & errMsg) if errMsg != nil: errSink.writeLine(prefix & errMsg)
proc combineProcMsgHandlers*(a, b: HandleProcMsgCB): HandleProcMsgCB =
if a == nil: result = b
elif b == nil: result = a
else:
result = proc(cmd: string, outMsg, errMsg: TaintedString): void =
a(cmd, outMsg, errMsg)
b(cmd, outMsg, errMsg)

29
src/main/nim/strawboss/server.nim
View File

@ -1,13 +1,26 @@
import asyncdispatch, jester, json import asyncdispatch, jester, json, osproc, tempfile
import ./configuration, ./core import ./configuration, ./core, private/util
settings: settings:
port = Port(8180) port = Port(8180)
type Worker = object
process*: Process
workingDir*: string
proc spawnWorker(req: RunRequest): Worker =
let dir = mkdtemp()
var args = @["run", req.projectName, req.stepName, "-r", req.buildRef, "-w", dir]
if req.forceRebuild: args.add("-f")
result = Worker(
process: startProcess("strawboss", ".", args, loadEnv(), {poUsePath}),
workingDir: dir)
proc start*(givenCfg: StrawBossConfig): void = proc start*(givenCfg: StrawBossConfig): void =
var workers: seq[Worker] = @[]
routes: routes:
get "/api/ping": get "/api/ping":
resp $(%*"pong"), "application/json" resp $(%*"pong"), "application/json"
@ -16,18 +29,10 @@ proc start*(givenCfg: StrawBossConfig): void =
resp $(%*[]), "application/json" resp $(%*[]), "application/json"
post "/api/project/@projectName/@stepName/run/@buildRef?": post "/api/project/@projectName/@stepName/run/@buildRef?":
let req = RunRequest( workers.add(spawnWorker(RunRequest(
projectName: @"projectName", projectName: @"projectName",
stepName: @"stepName", stepName: @"stepName",
buildRef: if @"buildRef" != "": @"buildRef" else: nil, buildRef: if @"buildRef" != "": @"buildRef" else: nil,
async: true, forceRebuild: false))) # TODO support this with optional query params
forceRebuild: false) # TODO support this with optional query params
# try:
# let runSummary = core.runStep(givenCfg, req)
# except:
# discard ""
# # TODO
runForever() runForever()

2
strawboss.nimble
View File

@ -1,7 +1,7 @@
# Package # Package
bin = @["strawboss"] bin = @["strawboss"]
version = "0.1.0" version = "0.2.0"
author = "Jonathan Bernard" author = "Jonathan Bernard"
description = "My personal continious integration worker." description = "My personal continious integration worker."
license = "MIT" license = "MIT"