321 lines
11 KiB
Nim
321 lines
11 KiB
Nim
import docopt, json, logging, nre, os, osproc, sequtils, streams, strtabs, strutils, tables, tempfile
|
|
|
|
let SB_VER = "0.1.0"
|
|
|
|
# Types
|
|
#
|
|
type
|
|
Step* = object
|
|
name*, stepCmd*, workingDir*: string
|
|
artifacts*, cmdInput*, depends*, expectedEnv*: seq[string]
|
|
dontCache*: bool
|
|
|
|
ProjectCfg* = object
|
|
name*: string
|
|
versionCmd*: string
|
|
steps*: Table[string, Step]
|
|
|
|
ProjectDef* = object
|
|
cfgFilePath*, defaultBranch*, name*, repo*: string
|
|
envVars*: StringTableRef
|
|
|
|
StrawBossCfg* = object
|
|
artifactsRepo*: string
|
|
projects*: seq[ProjectDef]
|
|
|
|
Workspace* = ref object
|
|
artifactsRepo*: string
|
|
env*: StringTableRef
|
|
workingDir*: string
|
|
project: ProjectCfg
|
|
|
|
HandleProcMsgCB = proc (cmd: string, outMsg: TaintedString, errMsg: TaintedString): void
|
|
|
|
# Misc. helpers
|
|
|
|
proc raiseEx(reason: string): void =
|
|
raise newException(Exception, reason)
|
|
|
|
let nullNode = newJNull()
|
|
proc getIfExists(n: JsonNode, key: string): JsonNode =
|
|
result = if n.hasKey(key): n[key]
|
|
else: nullNode
|
|
|
|
|
|
proc loadEnv(): StringTableRef =
|
|
result = newStringTable()
|
|
|
|
for k, v in envPairs():
|
|
result[k] = v
|
|
|
|
proc waitForWithOutput(p: Process, msgCB: HandleProcMsgCB,
|
|
procCmd: string = ""):
|
|
tuple[output: TaintedString, error: TaintedString, exitCode: int] =
|
|
|
|
var pout = outputStream(p)
|
|
var perr = errorStream(p)
|
|
|
|
result = (TaintedString"", TaintedString"", -1)
|
|
var line = newStringOfCap(120).TaintedString
|
|
while true:
|
|
if pout.readLine(line):
|
|
if msgCB != nil: msgCB(procCmd, line, nil)
|
|
result[0].string.add(line.string)
|
|
result[0].string.add("\n")
|
|
elif perr.readLine(line):
|
|
if msgCB != nil: msgCB(procCmd, nil, line)
|
|
result[1].string.add(line.string)
|
|
result[1].string.add("\n")
|
|
else:
|
|
result[2] = peekExitCode(p)
|
|
if result[2] != -1: break
|
|
close(p)
|
|
|
|
proc exec(command: string, workingDir: string = "",
|
|
args: openArray[string] = [], env: StringTableRef = nil,
|
|
options: set[ProcessOption] = {poUsePath},
|
|
msgCB: HandleProcMsgCB = nil):
|
|
tuple[output: TaintedString, error: TaintedString, exitCode: int]
|
|
{.tags: [ExecIOEffect, ReadIOEffect], gcsafe.} =
|
|
|
|
var p = startProcess(command, workingDir, args, env, options)
|
|
result = waitForWithOutput(p, msgCb, command)
|
|
|
|
let ENV = loadEnv()
|
|
let logProcOutput: HandleProcMsgCB = proc (cmd: string, outMsg: TaintedString, errMsg: TaintedString) =
|
|
if outMsg != nil: info cmd & "(stdout): " & outMsg
|
|
if errMsg != nil: info cmd & "(stderr): " & errMsg
|
|
|
|
let envVarRe = re"\$\w+|\$\{[^}]+\}"
|
|
proc resolveEnvVars(line: string, env: StringTableRef): string =
|
|
result = line
|
|
for found in line.findAll(envVarRe):
|
|
let key = if found[1] == '{': found[2..^2] else: found[1..^1]
|
|
if env.hasKey(key): result = result.replace(found, env[key])
|
|
|
|
let SB_EXPECTED_VARS = ["VERSION"]
|
|
# Configuration parsing code
|
|
|
|
proc loadStrawBossConfig(cfgFile: string): StrawBossCfg =
|
|
let jsonCfg = parseFile(cfgFile)
|
|
|
|
var projectDefs: seq[ProjectDef] = @[]
|
|
|
|
for pJson in jsonCfg.getIfExists("projects").getElems:
|
|
if not pJson.hasKey("name"):
|
|
raiseEx "a project definition is missing the project name"
|
|
|
|
if not pJson.hasKey("repo"):
|
|
raiseEx "a project definition is missing the project repo configuration"
|
|
|
|
var envVars = newStringTable(modeCaseSensitive)
|
|
for k, v in pJson.getIfExists("envVars").getFields: envVars[k] = v.getStr("")
|
|
|
|
projectDefs.add(
|
|
ProjectDef(
|
|
cfgFilePath: pJson.getIfExists("cfgFilePath").getStr("strawboss.json"),
|
|
defaultBranch: pJson.getIfExists("defaultBranch").getStr("master"),
|
|
name: pJson["name"].getStr,
|
|
envVars: envVars,
|
|
repo: pJson["repo"].getStr))
|
|
|
|
result = StrawBossCfg(
|
|
artifactsRepo: jsonCfg.getIfExists("artifactsRepo").getStr("artifacts"),
|
|
projects: projectDefs)
|
|
|
|
proc loadProjectConfig(cfgFile: string): ProjectCfg =
|
|
let jsonCfg = parseFile(cfgFile)
|
|
|
|
if not jsonCfg.hasKey("name"):
|
|
raise newException(Exception, "project configuration is missing a name")
|
|
|
|
if not jsonCfg.hasKey("steps"):
|
|
raise newException(Exception, "project configuration is missing steps definition")
|
|
|
|
var steps = initTable[string, Step]()
|
|
for sName, pJson in jsonCfg["steps"].getFields:
|
|
steps[sName] = Step(
|
|
name: sName,
|
|
workingDir: pJson.getIfExists("workingDir").getStr("."),
|
|
stepCmd: pJson.getIfExists("stepCmd").getStr("sh"),
|
|
depends: pJson.getIfExists("depends").getElems.mapIt(it.getStr),
|
|
artifacts: pJson.getIfExists("artifacts").getElems.mapIt(it.getStr),
|
|
cmdInput: pJson.getIfExists("cmdInput").getElems.mapIt(it.getStr),
|
|
expectedEnv: pJson.getIfExists("expectedEnv").getElems.mapIt(it.getStr),
|
|
dontCache: pJson.getIfExists("dontCache").getStr("false") != "false")
|
|
|
|
if steps[sName].stepCmd == "sh" and steps[sName].cmdInput.len == 0:
|
|
warn "Step " & sName & " uses 'sh' as its command but has no cmdInput."
|
|
|
|
result = ProjectCfg(
|
|
name: jsonCfg["name"].getStr,
|
|
versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"),
|
|
steps: steps)
|
|
|
|
proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string): Workspace =
|
|
|
|
info "Setting up to do work for '" & projectDef.name & "' at ref " & buildRef & "."
|
|
|
|
# Create a temp directory that we'll work in
|
|
let projDir = mkdtemp()
|
|
|
|
# Clone the project into the $temp/repo directory
|
|
let cloneResult = exec("git", projDir, ["clone", projectDef.repo, "repo"],
|
|
ENV, {poUsePath}, logProcOutput)
|
|
|
|
if cloneResult.exitCode != 0:
|
|
removeDir(projDir)
|
|
raiseEx "unable to clone repo for '" & projectDef.name & "'"
|
|
|
|
# Checkout the requested ref
|
|
let checkoutResult = exec("git", projDir & "/repo", ["checkout", buildRef],
|
|
ENV, {poUsePath}, logProcOutput)
|
|
|
|
if checkoutResult.exitCode != 0:
|
|
removeDir(projDir)
|
|
raiseEx "unable to checkout ref " & buildRef & " for '" & projectDef.name & "'"
|
|
|
|
# Find the strawboss project configuration
|
|
let projCfgFile = projDir & "/repo/" & projectDef.cfgFilePath
|
|
if not existsFile(projCfgFile):
|
|
removeDir(projDir)
|
|
raiseEx "Cannot find strawboss project configuration in the project " &
|
|
"repo (expected at '" & projectDef.cfgFilePath & "')."
|
|
|
|
let projectCfg = loadProjectConfig(projCfgFile)
|
|
result = Workspace(env: ENV, workingDir: projDir, project: projectCfg,
|
|
artifactsRepo: artifactsRepo)
|
|
|
|
# Merge in the project-defined env vars
|
|
for k, v in projectDef.envVars: result.env[k] = v
|
|
|
|
# Get the build version
|
|
let versionProc = startProcess(
|
|
projectCfg.versionCmd, # command
|
|
projDir & "/repo", # working dir
|
|
[], # args
|
|
result.env, # environment
|
|
{poUsePath, poEvalCommand}) # options
|
|
|
|
let versionResult = waitForWithOutput(versionProc, logProcOutput,
|
|
projectCfg.versionCmd)
|
|
|
|
if versionResult.exitCode != 0:
|
|
removeDir(projDir)
|
|
raiseEx "Version command (" & projectCfg.versionCmd & ") returned non-zero exit code."
|
|
|
|
debug "Building version " & versionResult.output.strip
|
|
result.env["VERSION"] = versionResult.output.strip
|
|
|
|
debug "Workspace for '" & projectCfg.name & ": " & projDir
|
|
|
|
## TODO
|
|
proc runStep(step: Step, wksp: Workspace): void =
|
|
|
|
let stepArtifactDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & step.name & "/" & wksp.env["VERSION"]
|
|
|
|
if existsDir(stepArtifactDir) and not step.dontCache:
|
|
info "Skipping step '" & step.name & "': already completed."
|
|
return
|
|
|
|
info "Running step '" & step.name & "' for " & wksp.project.name
|
|
|
|
# Ensure all expected environment variables are present.
|
|
for k in (step.expectedEnv & @SB_EXPECTED_VARS):
|
|
if not wksp.env.hasKey(k):
|
|
debug "workspace.env = " & $(wksp.env)
|
|
raiseEx "step " & step.name & " failed: missing required env variable: " & k
|
|
|
|
# Ensure that artifacts in steps we depend on are present
|
|
for dep in step.depends:
|
|
if not wksp.project.steps.hasKey(dep):
|
|
raiseEx step.name & " depends on " & dep & " but there is no step named " & dep
|
|
let depStep = wksp.project.steps[dep]
|
|
|
|
# Run that step (may get skipped)
|
|
let depDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & dep & "/" & wksp.env["VERSION"]
|
|
runStep(depStep, wksp)
|
|
|
|
# Add the artifacts directory for the dependent step to our env so that
|
|
# further steps can reference it via $<stepname>_DIR
|
|
echo "FP: " & depDir
|
|
wksp.env[depStep.name & "_DIR"] = depDir
|
|
|
|
# Run the step command, piping in cmdInput
|
|
debug step.name & ": starting stepCmd: " & step.stepCmd
|
|
let cmdProc = startProcess(step.stepCmd, wksp.workingDir & "/repo/" & step.workingDir,
|
|
[], wksp.env, {poUsePath, poEvalCommand})
|
|
|
|
let cmdInStream = inputStream(cmdProc)
|
|
|
|
# Replace env variables in step cmdInput as we pipe it in
|
|
for line in step.cmdInput: cmdInStream.writeLine(line.resolveEnvVars(wksp.env))
|
|
cmdInStream.flush()
|
|
cmdInStream.close()
|
|
|
|
let cmdResult = waitForWithOutput(cmdProc, logProcOutput, step.stepCmd)
|
|
|
|
if cmdResult.exitCode != 0:
|
|
raiseEx "step " & step.name & " failed: step command returned non-zero exit code"
|
|
|
|
# Gather the output artifacts (if we have any)
|
|
if not existsDir(stepArtifactDir): createDir(stepArtifactDir)
|
|
if step.artifacts.len > 0:
|
|
for a in step.artifacts:
|
|
let artifactPath = a.resolveEnvVars(wksp.env)
|
|
let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
|
|
try:
|
|
copyFile(wksp.workingDir & "/repo/" & step.workingDir & "/" & artifactPath,
|
|
stepArtifactDir & "/" & artifactName)
|
|
except:
|
|
removeDir(stepArtifactDir)
|
|
raiseEx "step " & step.name & " failed: unable to copy artifact " &
|
|
artifactPath & ":\n" & getCurrentExceptionMsg()
|
|
|
|
when isMainModule:
|
|
|
|
logging.addHandler(newConsoleLogger())
|
|
|
|
let cfg = loadStrawBossConfig("strawboss.config.json")
|
|
let artifactsRepo = expandFilename(cfg.artifactsRepo)
|
|
|
|
if not existsDir(artifactsRepo):
|
|
info "Artifacts repo (" & artifactsRepo & ") does not exist. Creating..."
|
|
createDir(artifactsRepo)
|
|
|
|
let doc = """
|
|
Usage:
|
|
strawboss serve
|
|
strawboss run <project> <step> [-r <ref>]"""
|
|
|
|
let args = docopt(doc, version = "strawboss v" & SB_VER)
|
|
|
|
if args["run"]:
|
|
|
|
# Find the project
|
|
let projName = $args["<project>"]
|
|
let matching = cfg.projects.filterIt(it.name == projName)
|
|
if matching.len == 0:
|
|
fatal "strawboss: no such project: " & projName
|
|
quit(QuitFailure)
|
|
elif matching.len > 1:
|
|
fatal "strawboss: more than one project named : " & projName
|
|
quit(QuitFailure)
|
|
|
|
let projectDef = matching[0]
|
|
let buildRef = if args["-r"]: $args["<ref>"] else: projectDef.defaultBranch
|
|
|
|
try:
|
|
let wksp = setupProjectForWork(projectDef, buildRef, artifactsRepo)
|
|
|
|
# Find the step
|
|
let stepName = $args["<step>"]
|
|
if not wksp.project.steps.hasKey(stepName):
|
|
raiseEx "no step name '" & stepName & "' for " & projName
|
|
|
|
let step = wksp.project.steps[stepName]
|
|
runStep(step, wksp)
|
|
except:
|
|
fatal "strawboss: " & getCurrentExceptionMsg()
|
|
quit(QuitFailure)
|