Finished refactor towards process-based workers.
This commit is contained in:
Jonathan Bernard
parent
cc28e7f4bf
commit
0976871563
@ -1,26 +1,22 @@
|
||||
import docopt, logging, os, sequtils, tempfile
|
||||
import docopt, os, sequtils, tempfile
|
||||
|
||||
import strawboss/private/util
|
||||
import strawboss/configuration
|
||||
import strawboss/core
|
||||
import strawboss/server
|
||||
import strawboss/supervisor
|
||||
|
||||
let SB_VER = "0.1.0"
|
||||
let SB_VER = "0.2.0"
|
||||
|
||||
proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) =
|
||||
let prefix = if cmd != nil: cmd else: ""
|
||||
if outMsg != nil: info prefix & "(stdout): " & outMsg
|
||||
if errMsg != nil: info prefix & "(stderr): " & errMsg
|
||||
if outMsg != nil: echo prefix & "(stdout): " & outMsg
|
||||
if errMsg != nil: echo prefix & "(stderr): " & errMsg
|
||||
|
||||
when isMainModule:
|
||||
|
||||
if logging.getHandlers().len == 0:
|
||||
logging.addHandler(newConsoleLogger())
|
||||
|
||||
var cfg = loadStrawBossConfig("strawboss.config.json")
|
||||
if not existsDir(cfg.artifactsRepo):
|
||||
info "Artifacts repo (" & cfg.artifactsRepo & ") does not exist. Creating..."
|
||||
echo "Artifacts repo (" & cfg.artifactsRepo & ") does not exist. Creating..."
|
||||
createDir(cfg.artifactsRepo)
|
||||
|
||||
cfg.artifactsRepo = expandFilename(cfg.artifactsRepo)
|
||||
@ -28,7 +24,6 @@ when isMainModule:
|
||||
let doc = """
|
||||
Usage:
|
||||
strawboss serve
|
||||
strawboss supervisor [-i <in-file>] [-o <out-file>]
|
||||
strawboss run <project> <step> [options]
|
||||
|
||||
Options
|
||||
@ -45,24 +40,25 @@ Options
|
||||
|
||||
let args = docopt(doc, version = "strawboss v" & SB_VER)
|
||||
|
||||
echo $args
|
||||
if args["run"]:
|
||||
|
||||
let req = RunRequest(
|
||||
projectName: $args["<project>"],
|
||||
stepName: $args["<step>"],
|
||||
buildRef: if args["--rreference"]: $args["<ref>"] else: nil,
|
||||
buildRef: if args["--reference"]: $args["--reference"] else: nil,
|
||||
forceRebuild: args["--force-rebuild"],
|
||||
workspaceDir: if args["--workspace"]: $args["<workspace>"] else: mkdtemp())
|
||||
|
||||
try:
|
||||
let summary = core.runStep(cfg, req, logProcOutput)
|
||||
# TODO: inspect result
|
||||
let status = core.runStep(cfg, req, logProcOutput)
|
||||
if status.state == "failed": raiseEx status.details
|
||||
echo "strawboss: build passed."
|
||||
except:
|
||||
fatal "strawboss: " & getCurrentExceptionMsg() & "."
|
||||
echo "strawboss: build FAILED: " & getCurrentExceptionMsg() & "."
|
||||
quit(QuitFailure)
|
||||
finally:
|
||||
if existsDir(req.workspaceDir): removeDir(req.workspaceDir)
|
||||
|
||||
info "strawboss: build passed"
|
||||
|
||||
elif
|
||||
elif args["serve"]: server.start(cfg)
|
||||
|
||||
|
||||
@ -25,6 +25,10 @@ type
|
||||
artifactsRepo*: string
|
||||
projects*: seq[ProjectDef]
|
||||
|
||||
RunRequest* = object
|
||||
projectName*, stepName*, buildRef*, workspaceDir*: string
|
||||
forceRebuild*: bool
|
||||
|
||||
# internal utils
|
||||
|
||||
let nullNode = newJNull()
|
||||
@ -32,6 +36,10 @@ proc getIfExists(n: JsonNode, key: string): JsonNode =
|
||||
result = if n.hasKey(key): n[key]
|
||||
else: nullNode
|
||||
|
||||
proc getOrFail(n: JsonNode, key: string, objName: string = ""): JsonNode =
|
||||
if not n.hasKey(key): raiseEx objName & " missing key " & key
|
||||
return n[key]
|
||||
|
||||
# Configuration parsing code
|
||||
|
||||
proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig =
|
||||
@ -43,12 +51,6 @@ proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig =
|
||||
var projectDefs: seq[ProjectDef] = @[]
|
||||
|
||||
for pJson in jsonCfg.getIfExists("projects").getElems:
|
||||
if not pJson.hasKey("name"):
|
||||
raiseEx "a project definition is missing the project name"
|
||||
|
||||
if not pJson.hasKey("repo"):
|
||||
raiseEx "a project definition is missing the project repo configuration"
|
||||
|
||||
var envVars = newStringTable(modeCaseSensitive)
|
||||
for k, v in pJson.getIfExists("envVars").getFields: envVars[k] = v.getStr("")
|
||||
|
||||
@ -56,9 +58,9 @@ proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig =
|
||||
ProjectDef(
|
||||
cfgFilePath: pJson.getIfExists("cfgFilePath").getStr("strawboss.json"),
|
||||
defaultBranch: pJson.getIfExists("defaultBranch").getStr("master"),
|
||||
name: pJson["name"].getStr,
|
||||
name: pJson.getOrFail("name", "project definition").getStr,
|
||||
envVars: envVars,
|
||||
repo: pJson["repo"].getStr))
|
||||
repo: pJson.getOrFail("repo", "project definition").getStr))
|
||||
|
||||
result = StrawBossConfig(
|
||||
artifactsRepo: jsonCfg.getIfExists("artifactsRepo").getStr("artifacts"),
|
||||
@ -70,14 +72,11 @@ proc loadProjectConfig*(cfgFile: string): ProjectCfg =
|
||||
|
||||
let jsonCfg = parseFile(cfgFile)
|
||||
|
||||
if not jsonCfg.hasKey("name"):
|
||||
raiseEx "project configuration is missing a name"
|
||||
|
||||
if not jsonCfg.hasKey("steps"):
|
||||
raiseEx "project configuration is missing steps definition"
|
||||
|
||||
var steps = initTable[string, Step]()
|
||||
for sName, pJson in jsonCfg["steps"].getFields:
|
||||
for sName, pJson in jsonCfg.getOrFail("steps", "project configuration").getFields:
|
||||
steps[sName] = Step(
|
||||
name: sName,
|
||||
workingDir: pJson.getIfExists("workingDir").getStr("."),
|
||||
@ -92,7 +91,7 @@ proc loadProjectConfig*(cfgFile: string): ProjectCfg =
|
||||
warn "Step " & sName & " uses 'sh' as its command but has no cmdInput."
|
||||
|
||||
result = ProjectCfg(
|
||||
name: jsonCfg["name"].getStr,
|
||||
name: jsonCfg.getOrFail("name", "project configuration").getStr,
|
||||
versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"),
|
||||
steps: steps)
|
||||
|
||||
@ -100,18 +99,35 @@ proc loadBuildStatus*(statusFile: string): BuildStatus =
|
||||
if not existsFile(statusFile): raiseEx "status file not found: " & statusFile
|
||||
let jsonObj = parseFile(statusFile)
|
||||
|
||||
if not jsonObj.hasKey("state"):
|
||||
raiseEx "project status is missing the 'state' field"
|
||||
|
||||
result = BuildStatus(
|
||||
state: jsonObj["state"].getStr,
|
||||
state: jsonObj.getOrFail("state", "build status").getStr,
|
||||
details: jsonObj.getIfExists("details").getStr("") )
|
||||
|
||||
proc parseRunRequest*(reqStr: string): RunRequest =
|
||||
let reqJson = parseJson(reqStr)
|
||||
|
||||
result = RunRequest(
|
||||
projectName: reqJson.getOrFail("projectName", "RunRequest").getStr,
|
||||
stepName: reqJson.getOrFail("stepName", "RunRequest").getStr,
|
||||
buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr,
|
||||
workspaceDir: reqJson.getOrFail("workspaceDir", "RunRequest").getStr,
|
||||
forceRebuild: reqJson.getOrFail("forceRebuild", "RunRequest").getBVal)
|
||||
|
||||
# TODO: can we use the marshal module for this?
|
||||
proc `%`*(s: BuildStatus): JsonNode =
|
||||
result = %* {
|
||||
"state": s.state,
|
||||
"details": s.details
|
||||
}
|
||||
|
||||
proc `$`*(s: BuildStatus): string =
|
||||
result =pretty(%s)
|
||||
proc `%`*(req: RunRequest): JsonNode =
|
||||
result = %* {
|
||||
"projectName": req.projectName,
|
||||
"stepName": req.stepName,
|
||||
"buildRef": req.buildRef,
|
||||
"workspaceDir": req.workspaceDir,
|
||||
"forceRebuild": req.forceRebuild
|
||||
}
|
||||
|
||||
proc `$`*(s: BuildStatus): string = result = pretty(%s)
|
||||
proc `$`*(req: RunRequest): string = result = pretty(%req)
|
||||
|
||||
@ -2,24 +2,23 @@ import logging, nre, os, osproc, sequtils, streams, strtabs, strutils, tables, t
|
||||
|
||||
import private/util
|
||||
import configuration
|
||||
from posix import fork, Pid
|
||||
from posix import link
|
||||
|
||||
type
|
||||
Workspace* = ref object
|
||||
artifactsRepo*: string
|
||||
env*: StringTableRef
|
||||
workingDir*: string
|
||||
project*: ProjectCfg
|
||||
|
||||
RunSummary* = object
|
||||
project*: ProjectCfg
|
||||
step*: Step
|
||||
buildVersion*, statusFile*: string
|
||||
workerPid*: Pid
|
||||
|
||||
RunRequest* = object
|
||||
projectName*, stepName*, buildRef*, workspaceDir*: string
|
||||
forceRebuild*: bool
|
||||
Workspace = ref object ## Data needed by internal build process
|
||||
artifactsDir*: string ## absolute path to the directory for this version
|
||||
artifactsRepo*: string ## absolute path to the global artifacts repo
|
||||
buildRef*: string ## git-style commit reference to the revision we are building
|
||||
dir*: string ## absolute path to the working directory
|
||||
env*: StringTableRef ## environment variables for all build processes
|
||||
openedFiles*: seq[File] ## all files that we have opened that need to be closed
|
||||
outputHandler*: HandleProcMsgCB ## handler for process output
|
||||
project*: ProjectCfg ## the project configuration
|
||||
projectDef*: ProjectDef ## the StrawBoss project definition
|
||||
status*: BuildStatus ## the current status of the build
|
||||
statusFile*: string ## absolute path to the build status file
|
||||
step*: Step ## the step we're building
|
||||
version*: string ## project version as returned by versionCmd
|
||||
|
||||
proc resolveEnvVars(line: string, env: StringTableRef): string =
|
||||
result = line
|
||||
@ -27,123 +26,77 @@ proc resolveEnvVars(line: string, env: StringTableRef): string =
|
||||
let key = if found[1] == '{': found[2..^2] else: found[1..^1]
|
||||
if env.hasKey(key): result = result.replace(found, env[key])
|
||||
|
||||
proc combineProcMsgHandlers(a, b: HandleProcMsgCB): HandleProcMsgCB =
|
||||
if a == nil: result = b
|
||||
elif b == nil: result = a
|
||||
else:
|
||||
result = proc(cmd: string, outMsg, errMsg: TaintedString): void =
|
||||
a(cmd, outMsg, errMsg)
|
||||
b(cmd, outMsg, errMsg)
|
||||
proc emitStatus(status: BuildStatus, statusFilePath: string,
|
||||
outputHandler: HandleProcMsgCB): BuildStatus =
|
||||
if statusFilePath != nil: writeFile(statusFilePath, $status)
|
||||
if outputHandler != nil:
|
||||
outputHandler.sendMsg(status.state & ": " & status.details)
|
||||
result = status
|
||||
|
||||
proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string,
|
||||
outputHandler: HandleProcMsgCB = nil): Workspace =
|
||||
proc publishStatus(wksp: Workspace, state, details: string) =
|
||||
let status = BuildStatus(state: state, details: details)
|
||||
wksp.status = emitStatus(status, wksp.statusFile, wksp.outputHandler)
|
||||
|
||||
outputHandler.sendMsg "Setting up to do work for '" & projectDef.name &
|
||||
"' at ref " & buildRef & "."
|
||||
|
||||
var env = loadEnv()
|
||||
env["GIT_DIR"] = ".git"
|
||||
|
||||
# Create a temp directory that we'll work in
|
||||
let projDir = mkdtemp()
|
||||
debug "Workspace for '" & projectDef.name & ": " & projDir
|
||||
assert projDir.isAbsolute
|
||||
proc setupProject(wksp: Workspace) =
|
||||
|
||||
# Clone the project into the $temp/repo directory
|
||||
let cloneResult = exec("git", projDir, ["clone", projectDef.repo, "repo"],
|
||||
env, {poUsePath}, outputHandler)
|
||||
let cloneResult = exec("git", wksp.dir,
|
||||
["clone", wksp.projectDef.repo, "repo"],
|
||||
wksp.env, {poUsePath}, wksp.outputHandler)
|
||||
|
||||
if cloneResult.exitCode != 0:
|
||||
removeDir(projDir)
|
||||
raiseEx "unable to clone repo for '" & projectDef.name & "'"
|
||||
raiseEx "unable to clone repo for '" & wksp.projectDef.name & "'"
|
||||
|
||||
# Checkout the requested ref
|
||||
let checkoutResult = exec("git", projDir & "/repo", ["checkout", buildRef],
|
||||
env, {poUsePath}, outputHandler)
|
||||
let checkoutResult = exec("git", wksp.dir & "/repo",
|
||||
["checkout", wksp.buildRef],
|
||||
wksp.env, {poUsePath}, wksp.outputHandler)
|
||||
|
||||
if checkoutResult.exitCode != 0:
|
||||
removeDir(projDir)
|
||||
raiseEx "unable to checkout ref " & buildRef & " for '" & projectDef.name & "'"
|
||||
raiseEx "unable to checkout ref " & wksp.buildRef &
|
||||
" for '" & wksp.projectDef.name & "'"
|
||||
|
||||
# Find the strawboss project configuration
|
||||
let projCfgFile = projDir & "/repo/" & projectDef.cfgFilePath
|
||||
let projCfgFile = wksp.dir & "/repo/" & wksp.projectDef.cfgFilePath
|
||||
if not existsFile(projCfgFile):
|
||||
removeDir(projDir)
|
||||
raiseEx "Cannot find strawboss project configuration in the project " &
|
||||
"repo (expected at '" & projectDef.cfgFilePath & "')."
|
||||
"repo (expected at '" & wksp.projectDef.cfgFilePath & "')."
|
||||
|
||||
let projectCfg = loadProjectConfig(projCfgFile)
|
||||
result = Workspace(env: env, workingDir: projDir, project: projectCfg,
|
||||
artifactsRepo: artifactsRepo)
|
||||
wksp.project = loadProjectConfig(projCfgFile)
|
||||
|
||||
# Merge in the project-defined env vars
|
||||
for k, v in projectDef.envVars: result.env[k] = v
|
||||
for k, v in wksp.projectDef.envVars: wksp.env[k] = v
|
||||
|
||||
# Get the build version
|
||||
let versionProc = startProcess(
|
||||
projectCfg.versionCmd, # command
|
||||
projDir & "/repo", # working dir
|
||||
wksp.project.versionCmd, # command
|
||||
wksp.dir & "/repo", # working dir
|
||||
[], # args
|
||||
result.env, # environment
|
||||
wksp.env, # environment
|
||||
{poUsePath, poEvalCommand}) # options
|
||||
|
||||
let versionResult = waitForWithOutput(versionProc, outputHandler,
|
||||
projectCfg.versionCmd)
|
||||
let versionResult = waitForWithOutput(versionProc, wksp.outputHandler,
|
||||
wksp.project.versionCmd)
|
||||
|
||||
if versionResult.exitCode != 0:
|
||||
removeDir(projDir)
|
||||
raiseEx "Version command (" & projectCfg.versionCmd & ") returned non-zero exit code."
|
||||
raiseEx "Version command (" & wksp.project.versionCmd &
|
||||
") returned non-zero exit code."
|
||||
|
||||
outputHandler.sendMsg "Building version " & versionResult.output.strip
|
||||
result.env["VERSION"] = versionResult.output.strip
|
||||
wksp.outputHandler.sendMsg "Building version " & versionResult.output.strip
|
||||
wksp.version = versionResult.output.strip
|
||||
wksp.env["VERSION"] = wksp.version
|
||||
|
||||
proc runStep*(wksp: Workspace, step: Step) =
|
||||
|
||||
proc runStep*(step: Step, wksp: Workspace,
|
||||
givenOutputHandler: HandleProcMsgCB = nil): void =
|
||||
let SB_EXPECTED_VARS = ["VERSION"]
|
||||
let stepArtifactDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" &
|
||||
step.name & "/" & wksp.env["VERSION"]
|
||||
let statusFile = stepArtifactDir & ".status.json"
|
||||
|
||||
if not existsDir(stepArtifactDir): createDir(stepArtifactDir)
|
||||
|
||||
# Have we tried to build this before and are we caching the results?
|
||||
if existsFile(statusFile) and not step.dontSkip:
|
||||
let status = loadBuildStatus(statusFile)
|
||||
|
||||
# If we succeeded last time, no need to rebuild
|
||||
if status.state == "complete":
|
||||
givenOutputHandler.sendMsg "Skipping step '" & step.name & "' for version '" &
|
||||
wksp.env["VERSION"] & "': already completed."
|
||||
return
|
||||
else:
|
||||
givenOutputHandler.sendMsg "Rebuilding failed step '" & step.name & "' for version '" &
|
||||
wksp.env["VERSION"] & "'."
|
||||
|
||||
givenOutputHandler.sendMsg "Running step '" & step.name & "' for " & wksp.project.name
|
||||
writeFile(statusFile, $BuildStatus(state: "running", details: ""))
|
||||
|
||||
var stdoutLogFile, stderrLogFile: File
|
||||
|
||||
try:
|
||||
|
||||
var outputHandler: HandleProcMsgCB
|
||||
|
||||
# Make sure we log output to the stdout and sterr log files
|
||||
if not (open(stdoutLogFile, stepArtifactDir & "/stdout.log", fmWrite) and
|
||||
open(stderrLogFile, stepArtifactDir & "/stderr.log", fmWrite)):
|
||||
givenOutputHandler.sendMsg nil, "Failed to open log files for STDOUT and STDERR."
|
||||
outputHandler = givenOutputHandler
|
||||
if stdoutLogFile != nil: close(stdoutLogFile)
|
||||
if stderrLogFile != nil: close(stderrLogFile)
|
||||
else:
|
||||
outputHandler = combineProcMsgHandlers(
|
||||
givenOutputHandler,
|
||||
makeProcMsgHandler(stdoutLogFile, stderrLogFile))
|
||||
wksp.publishStatus("running",
|
||||
"running '" & step.name & "' for version " & wksp.version &
|
||||
" from " & wksp.buildRef)
|
||||
|
||||
# Ensure all expected environment variables are present.
|
||||
for k in (step.expectedEnv & @SB_EXPECTED_VARS):
|
||||
if not wksp.env.hasKey(k):
|
||||
debug "workspace.env = " & $(wksp.env)
|
||||
raiseEx "step " & step.name & " failed: missing required env variable: " & k
|
||||
|
||||
# Ensure that artifacts in steps we depend on are present
|
||||
@ -155,20 +108,17 @@ proc runStep*(step: Step, wksp: Workspace,
|
||||
let depStep = wksp.project.steps[dep]
|
||||
|
||||
# Run that step (may get skipped)
|
||||
let depDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" &
|
||||
dep & "/" & wksp.env["VERSION"]
|
||||
|
||||
runStep(depStep, wksp)
|
||||
runStep(wksp, depStep)
|
||||
|
||||
# Add the artifacts directory for the dependent step to our env so that
|
||||
# further steps can reference it via $<stepname>_DIR
|
||||
echo "FP: " & depDir
|
||||
wksp.env[depStep.name & "_DIR"] = depDir
|
||||
wksp.env[depStep.name & "_DIR"] = wksp.artifactsRepo & "/" &
|
||||
wksp.project.name & "/" & dep & "/" & wksp.version
|
||||
|
||||
# Run the step command, piping in cmdInput
|
||||
outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd
|
||||
wksp.outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd
|
||||
let cmdProc = startProcess(step.stepCmd,
|
||||
wksp.workingDir & "/repo/" & step.workingDir,
|
||||
wksp.dir & "/repo/" & step.workingDir,
|
||||
[], wksp.env, {poUsePath, poEvalCommand})
|
||||
|
||||
let cmdInStream = inputStream(cmdProc)
|
||||
@ -178,61 +128,86 @@ proc runStep*(step: Step, wksp: Workspace,
|
||||
cmdInStream.flush()
|
||||
cmdInStream.close()
|
||||
|
||||
let cmdResult = waitForWithOutput(cmdProc, outputHandler, step.stepCmd)
|
||||
let cmdResult = waitForWithOutput(cmdProc, wksp.outputHandler, step.stepCmd)
|
||||
|
||||
if cmdResult.exitCode != 0:
|
||||
raiseEx "step " & step.name & " failed: step command returned non-zero exit code"
|
||||
|
||||
# Gather the output artifacts (if we have any)
|
||||
wksp.outputHandler.sendMsg "artifacts: " & $step.artifacts
|
||||
if step.artifacts.len > 0:
|
||||
for a in step.artifacts:
|
||||
let artifactPath = a.resolveEnvVars(wksp.env)
|
||||
let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
|
||||
try:
|
||||
copyFile(wksp.workingDir & "/repo/" & step.workingDir & "/" & artifactPath,
|
||||
stepArtifactDir & "/" & artifactName)
|
||||
wksp.outputHandler.sendMsg "copy " & wksp.dir & "/repo/" & step.workingDir & "/" & artifactPath & " -> " & wksp.artifactsDir & "/" & artifactName
|
||||
copyFile(wksp.dir & "/repo/" & step.workingDir & "/" & artifactPath,
|
||||
wksp.artifactsDir & "/" & artifactName)
|
||||
except:
|
||||
raiseEx "step " & step.name & " failed: unable to copy artifact " &
|
||||
artifactPath & ":\n" & getCurrentExceptionMsg()
|
||||
|
||||
writeFile(statusFile, $BuildStatus(state: "complete", details: ""))
|
||||
|
||||
except:
|
||||
writeFile(statusFile, $BuildStatus(
|
||||
state: "failed",
|
||||
details: getCurrentExceptionMsg()))
|
||||
|
||||
finally:
|
||||
if stdoutLogFile != nil: close(stdoutLogFile)
|
||||
if stderrLogFile != nil: close(stderrLogFile)
|
||||
wksp.publishStatus("complete", "")
|
||||
|
||||
proc runStep*(cfg: StrawBossConfig, req: RunRequest,
|
||||
outputHandler: HandleProcMsgCB = nil): RunSummary =
|
||||
outputHandler: HandleProcMsgCB = nil): BuildStatus =
|
||||
|
||||
if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
|
||||
let statusFile = req.workspaceDir & "/" & "status.json"
|
||||
result = BuildStatus(
|
||||
state: "setup",
|
||||
details: "initializing build workspace")
|
||||
discard emitStatus(result, nil, outputHandler)
|
||||
|
||||
var wksp: Workspace
|
||||
|
||||
try:
|
||||
writeFile(statusFile, $BuildStatus(
|
||||
state: "setup",
|
||||
details: "Preparing working environment."))
|
||||
assert req.workspaceDir.isAbsolute
|
||||
if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
|
||||
|
||||
# Find the project definition
|
||||
let matching = cfg.projects.filterIt(it.name == req.projectName)
|
||||
if matching.len == 0: raiseEx "no such project: " & req.projectName
|
||||
elif matching.len > 1: raiseEx "more than one project named : " & req.projectName
|
||||
|
||||
let projectDef = matching[0]
|
||||
# Read in the existing system environment
|
||||
var env = loadEnv()
|
||||
env["GIT_DIR"] = ".git"
|
||||
|
||||
# Find the commit reference we're building
|
||||
let foundBuildRef =
|
||||
# Setup our STDOUT and STDERR files
|
||||
let stdoutFile = open(req.workspaceDir & "/stdout.log", fmWrite)
|
||||
let stderrFile = open(req.workspaceDir & "/stderr.log", fmWrite)
|
||||
|
||||
let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile)
|
||||
|
||||
wksp = Workspace(
|
||||
artifactsDir: nil,
|
||||
artifactsRepo: cfg.artifactsRepo,
|
||||
buildRef:
|
||||
if req.buildRef != nil and req.buildRef.len > 0: req.buildRef
|
||||
else: projectDef.defaultBranch
|
||||
else: matching[0].defaultBranch,
|
||||
dir: req.workspaceDir,
|
||||
env: env,
|
||||
openedFiles: @[stdoutFile, stderrFile],
|
||||
outputHandler: combineProcMsgHandlers(outputHandler, logFilesOH),
|
||||
project: ProjectCfg(),
|
||||
projectDef: matching[0],
|
||||
status: result,
|
||||
statusFile: req.workspaceDir & "/" & "status.json",
|
||||
step: Step(),
|
||||
version: nil)
|
||||
|
||||
let wksp = setupProjectForWork(projectDef, foundBuildRef, cfg.artifactsRepo,
|
||||
outputHandler)
|
||||
except:
|
||||
result = BuildStatus(state: "failed",
|
||||
details: getCurrentExceptionMsg())
|
||||
try: discard emitStatus(result, nil, outputHandler)
|
||||
except: discard ""
|
||||
|
||||
# Find the step
|
||||
try:
|
||||
# Clone the repo and setup the working environment
|
||||
wksp.publishStatus("setup",
|
||||
"cloning project repo and preparing to run '" & req.stepName & "'")
|
||||
wksp.setupProject()
|
||||
|
||||
# Find the requested step
|
||||
if not wksp.project.steps.hasKey(req.stepName):
|
||||
raiseEx "no step name '" & req.stepName & "' for " & req.projectName
|
||||
|
||||
@ -241,15 +216,55 @@ proc runStep*(cfg: StrawBossConfig, req: RunRequest,
|
||||
# Enfore forceRebuild
|
||||
if req.forceRebuild: step.dontSkip = true
|
||||
|
||||
result = RunSummary(
|
||||
project: wksp.project,
|
||||
step: step,
|
||||
buildVersion: wksp.env["VERSION"],
|
||||
statusFile: wksp.artifactsRepo & "/" & wksp.project.name & "/" & step.name &
|
||||
"/" & wksp.env["VERSION"] & ".status.json")
|
||||
if req.async:
|
||||
let pid = fork()
|
||||
if pid == 0: runStep(step, wksp, outputHandler) # if we are the child
|
||||
else: result.workerPid = pid # if we are the parent
|
||||
# Compose the path to the artifacts directory for this step and version
|
||||
wksp.artifactsDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" &
|
||||
step.name & "/" & wksp.version
|
||||
|
||||
# Have we tried to build this before and are we caching the results?
|
||||
if existsFile(wksp.artifactsDir & "/status.json") and not step.dontSkip:
|
||||
let prevStatus = loadBuildStatus(wksp.artifactsDir & "/status.json")
|
||||
|
||||
# If we succeeded last time, no need to rebuild
|
||||
if prevStatus.state == "complete":
|
||||
wksp.outputHandler.sendMsg(
|
||||
"Skipping step '" & step.name & "' for version '" &
|
||||
wksp.version & "': already completed.")
|
||||
return prevStatus
|
||||
else:
|
||||
wksp.outputHandler.sendMsg(
|
||||
"Rebuilding failed step '" & step.name & "' for version '" &
|
||||
wksp.version & "'.")
|
||||
|
||||
# Make the artifacts directory if it doesn't already exist
|
||||
if not existsDir(wksp.artifactsDir): createDir(wksp.artifactsDir)
|
||||
|
||||
# Link status file and output logs to the artifacts dir
|
||||
for fn in @["status.json", "stdout.log", "stderr.log"]:
|
||||
# TODO: roll old files instead of delete them?
|
||||
if existsFile(wksp.artifactsDir & "/" & fn):
|
||||
removeFile(wksp.artifactsDir & "/" & fn)
|
||||
|
||||
if link(wksp.dir & "/" & fn, wksp.artifactsDir & "/" & fn) != 0:
|
||||
wksp.outputHandler.sendMsg(nil,
|
||||
"WARN: could not link " & fn & " to artifacts dir.")
|
||||
|
||||
runStep(wksp, step)
|
||||
|
||||
result = wksp.status
|
||||
|
||||
except:
|
||||
let msg = getCurrentExceptionMsg()
|
||||
try:
|
||||
wksp.publishStatus("failed", msg)
|
||||
result = wksp.status
|
||||
except:
|
||||
result = BuildStatus(state: "failed", details: msg)
|
||||
try: discard emitStatus(result, nil, outputHandler)
|
||||
except: discard ""
|
||||
|
||||
finally:
|
||||
if wksp != nil:
|
||||
for f in wksp.openedFiles:
|
||||
try: close(f)
|
||||
except: discard ""
|
||||
|
||||
else: runStep(step, wksp, outputHandler)
|
||||
|
||||
@ -66,3 +66,13 @@ proc makeProcMsgHandler*(outSink, errSink: Stream): HandleProcMsgCB =
|
||||
let prefix = if cmd != nil: cmd & ": " else: ""
|
||||
if outMsg != nil: outSink.writeLine(prefix & outMsg)
|
||||
if errMsg != nil: errSink.writeLine(prefix & errMsg)
|
||||
|
||||
proc combineProcMsgHandlers*(a, b: HandleProcMsgCB): HandleProcMsgCB =
|
||||
if a == nil: result = b
|
||||
elif b == nil: result = a
|
||||
else:
|
||||
result = proc(cmd: string, outMsg, errMsg: TaintedString): void =
|
||||
a(cmd, outMsg, errMsg)
|
||||
b(cmd, outMsg, errMsg)
|
||||
|
||||
|
||||
|
||||
@ -1,13 +1,26 @@
|
||||
import asyncdispatch, jester, json
|
||||
import asyncdispatch, jester, json, osproc, tempfile
|
||||
|
||||
import ./configuration, ./core
|
||||
import ./configuration, ./core, private/util
|
||||
|
||||
settings:
|
||||
port = Port(8180)
|
||||
|
||||
type Worker = object
|
||||
process*: Process
|
||||
workingDir*: string
|
||||
|
||||
proc spawnWorker(req: RunRequest): Worker =
|
||||
let dir = mkdtemp()
|
||||
var args = @["run", req.projectName, req.stepName, "-r", req.buildRef, "-w", dir]
|
||||
if req.forceRebuild: args.add("-f")
|
||||
result = Worker(
|
||||
process: startProcess("strawboss", ".", args, loadEnv(), {poUsePath}),
|
||||
workingDir: dir)
|
||||
|
||||
proc start*(givenCfg: StrawBossConfig): void =
|
||||
|
||||
var workers: seq[Worker] = @[]
|
||||
|
||||
routes:
|
||||
get "/api/ping":
|
||||
resp $(%*"pong"), "application/json"
|
||||
@ -16,18 +29,10 @@ proc start*(givenCfg: StrawBossConfig): void =
|
||||
resp $(%*[]), "application/json"
|
||||
|
||||
post "/api/project/@projectName/@stepName/run/@buildRef?":
|
||||
let req = RunRequest(
|
||||
workers.add(spawnWorker(RunRequest(
|
||||
projectName: @"projectName",
|
||||
stepName: @"stepName",
|
||||
buildRef: if @"buildRef" != "": @"buildRef" else: nil,
|
||||
async: true,
|
||||
forceRebuild: false) # TODO support this with optional query params
|
||||
|
||||
# try:
|
||||
# let runSummary = core.runStep(givenCfg, req)
|
||||
# except:
|
||||
# discard ""
|
||||
# # TODO
|
||||
|
||||
forceRebuild: false))) # TODO support this with optional query params
|
||||
|
||||
runForever()
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# Package
|
||||
|
||||
bin = @["strawboss"]
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
author = "Jonathan Bernard"
|
||||
description = "My personal continious integration worker."
|
||||
license = "MIT"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user