WIP

src/main/json/project.config.schema.json

@@ -28,7 +28,7 @@
               "expectedEnv": {
                 "type": "array",
                 "items": { "type": "string" }},
-              "dontCache": "bool"
+              "dontSkip": "bool"
             },
             "additionalProperties": false
           }

src/main/nim/strawboss.nim

@@ -1,12 +1,18 @@
-import docopt, logging, os, sequtils
+import docopt, logging, os, sequtils, tempfile
 
 import strawboss/private/util
 import strawboss/configuration
 import strawboss/core
 import strawboss/server
+import strawboss/supervisor
 
 let SB_VER = "0.1.0"
 
+proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) =
+  let prefix = if cmd != nil: cmd else: ""
+  if outMsg != nil: info prefix & "(stdout): " & outMsg
+  if errMsg != nil: info prefix & "(stderr): " & errMsg
+
 when isMainModule:
 
   if logging.getHandlers().len == 0:
@@ -22,19 +28,41 @@ when isMainModule:
   let doc = """
 Usage:
   strawboss serve
-  strawboss run <project> <step> [-r <ref>]"""
+  strawboss supervisor [-i <in-file>] [-o <out-file>]
+  strawboss run <project> <step> [options]
+  
+Options
+  
+  -f --force-rebuild          Force a build step to re-run even we have cached
+                              results from building that step before for this
+                              version of the project.
+
+  -r --reference <ref>        Build the project at this commit reference.
+
+  -w --workspace <workspace>  Use the given directory as the build workspace.
+
+"""
 
   let args = docopt(doc, version = "strawboss v" & SB_VER)
 
   if args["run"]:
 
-    let projName = $args["<project>"]
+    let req = RunRequest(
-    let stepName = $args["<step>"]
+      projectName: $args["<project>"],
-    let buildRef = if args["-r"]: $args["<ref>"] else: nil
+      stepName: $args["<step>"],
+      buildRef: if args["--rreference"]: $args["<ref>"] else: nil,
+      forceRebuild: args["--force-rebuild"],
+      workspaceDir: if args["--workspace"]: $args["<workspace>"] else: mkdtemp())
 
-    try: core.runStep(cfg, projName, stepName, buildRef)
+    try:
+      let summary = core.runStep(cfg, req, logProcOutput)
+      # TODO: inspect result
     except:
-      fatal "strawboss: " & getCurrentExceptionMsg()
+      fatal "strawboss: " & getCurrentExceptionMsg() & "."
       quit(QuitFailure)
 
+    info "strawboss: build passed"
+
+  elif
   elif args["serve"]: server.start(cfg)
+

src/main/nim/strawboss.nim.cfg

@@ -0,0 +1 @@
+--threads:on

src/main/nim/strawboss/configuration.nim

@@ -1,13 +1,16 @@
-import logging, json, nre, sequtils, strtabs, tables
+import logging, json, os, nre, sequtils, strtabs, tables
 import private/util
 
 # Types
 #
 type
+  BuildStatus* = object
+    state*, details*: string
+
   Step* = object
     name*, stepCmd*, workingDir*: string
     artifacts*, cmdInput*, depends*, expectedEnv*: seq[string]
-    dontCache*: bool
+    dontSkip*: bool
 
   ProjectCfg* = object
     name*: string
@@ -22,7 +25,6 @@ type
     artifactsRepo*: string
     projects*: seq[ProjectDef]
 
-
 # internal utils
 
 let nullNode = newJNull()
@@ -33,6 +35,9 @@ proc getIfExists(n: JsonNode, key: string): JsonNode =
 # Configuration parsing code
 
 proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig =
+  if not existsFile(cfgFile):
+    raiseEx "strawboss config file not found: " & cfgFile
+
   let jsonCfg = parseFile(cfgFile)
 
   var projectDefs: seq[ProjectDef] = @[]
@@ -60,13 +65,16 @@ proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig =
     projects: projectDefs)
 
 proc loadProjectConfig*(cfgFile: string): ProjectCfg =
+  if not existsFile(cfgFile):
+    raiseEx "project config file not found: " & cfgFile
+
   let jsonCfg = parseFile(cfgFile)
 
   if not jsonCfg.hasKey("name"):
-    raise newException(Exception, "project configuration is missing a name")
+    raiseEx "project configuration is missing a name"
 
   if not jsonCfg.hasKey("steps"):
-    raise newException(Exception, "project configuration is missing steps definition")
+    raiseEx "project configuration is missing steps definition"
 
   var steps = initTable[string, Step]()
   for sName, pJson in jsonCfg["steps"].getFields:
@@ -78,7 +86,7 @@ proc loadProjectConfig*(cfgFile: string): ProjectCfg =
       artifacts:    pJson.getIfExists("artifacts").getElems.mapIt(it.getStr),
       cmdInput:     pJson.getIfExists("cmdInput").getElems.mapIt(it.getStr),
       expectedEnv:  pJson.getIfExists("expectedEnv").getElems.mapIt(it.getStr),
-      dontCache:    pJson.getIfExists("dontCache").getStr("false") != "false")
+      dontSkip:     pJson.getIfExists("dontSkip").getStr("false") != "false")
 
     if steps[sName].stepCmd == "sh" and steps[sName].cmdInput.len == 0:
       warn "Step " & sName & " uses 'sh' as its command but has no cmdInput."
@@ -88,3 +96,22 @@ proc loadProjectConfig*(cfgFile: string): ProjectCfg =
     versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"),
     steps: steps)
 
+proc loadBuildStatus*(statusFile: string): BuildStatus =
+  if not existsFile(statusFile): raiseEx "status file not found: " & statusFile
+  let jsonObj = parseFile(statusFile)
+
+  if not jsonObj.hasKey("state"):
+    raiseEx "project status is missing the 'state' field"
+
+  result = BuildStatus(
+    state: jsonObj["state"].getStr,
+    details: jsonObj.getIfExists("details").getStr("") )
+
+proc `%`*(s: BuildStatus): JsonNode =
+  result = %* {
+    "state": s.state,
+    "details": s.details
+  }
+
+proc `$`*(s: BuildStatus): string =
+  result =pretty(%s)

src/main/nim/strawboss/core.nim

@@ -1,8 +1,8 @@
-import logging, nre, os, osproc, sequtils, streams, strtabs, strutils, tables,
+import logging, nre, os, osproc, sequtils, streams, strtabs, strutils, tables, tempfile
-  tempfile
 
 import private/util
 import configuration
+from posix import fork, Pid
 
 type
   Workspace* = ref object
@@ -11,26 +11,37 @@ type
     workingDir*: string
     project*: ProjectCfg
 
+  RunSummary* = object
+    project*: ProjectCfg
+    step*: Step
+    buildVersion*, statusFile*: string
+    workerPid*: Pid
 
-let ENV = loadEnv()
+  RunRequest* = object
+    projectName*, stepName*, buildRef*, workspaceDir*: string
+    forceRebuild*: bool
 
-let logProcOutput: HandleProcMsgCB = proc (cmd: string, outMsg: TaintedString, errMsg: TaintedString) =
-  if outMsg != nil: info cmd & "(stdout): " & outMsg
-  if errMsg != nil: info cmd & "(stderr): " & errMsg
-
-let envVarRe = re"\$\w+|\$\{[^}]+\}"
 proc resolveEnvVars(line: string, env: StringTableRef): string =
   result = line
-  for found in line.findAll(envVarRe):
+  for found in line.findAll(re"\$\w+|\$\{[^}]+\}"):
     let key = if found[1] == '{': found[2..^2] else: found[1..^1]
     if env.hasKey(key): result = result.replace(found, env[key])
 
-let SB_EXPECTED_VARS = ["VERSION"]
+proc combineProcMsgHandlers(a, b: HandleProcMsgCB): HandleProcMsgCB =
+  if a == nil: result = b
+  elif b == nil: result = a
+  else:
+    result = proc(cmd: string, outMsg, errMsg: TaintedString): void =
+      a(cmd, outMsg, errMsg)
+      b(cmd, outMsg, errMsg)
 
-proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string): Workspace =
+proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string,
+                         outputHandler: HandleProcMsgCB = nil): Workspace =
 
-  info "Setting up to do work for '" & projectDef.name & "' at ref " & buildRef & "."
+  outputHandler.sendMsg "Setting up to do work for '" & projectDef.name &
-  var env = ENV
+                        "' at ref " & buildRef & "."
+
+  var env = loadEnv()
   env["GIT_DIR"] = ".git"
 
   # Create a temp directory that we'll work in
@@ -40,7 +51,7 @@ proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string
 
   # Clone the project into the $temp/repo directory
   let cloneResult = exec("git", projDir, ["clone", projectDef.repo, "repo"],
-    env, {poUsePath}, logProcOutput)
+    env, {poUsePath}, outputHandler)
 
   if cloneResult.exitCode != 0:
     removeDir(projDir)
@@ -48,7 +59,7 @@ proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string
 
   # Checkout the requested ref
   let checkoutResult = exec("git", projDir & "/repo", ["checkout", buildRef],
-    env, {poUsePath}, logProcOutput)
+    env, {poUsePath}, outputHandler)
 
   if checkoutResult.exitCode != 0:
     removeDir(projDir)
@@ -76,94 +87,169 @@ proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string
     result.env,                 # environment
     {poUsePath, poEvalCommand}) # options
 
-  let versionResult = waitForWithOutput(versionProc, logProcOutput,
+  let versionResult = waitForWithOutput(versionProc, outputHandler,
                                         projectCfg.versionCmd)
 
   if versionResult.exitCode != 0:
     removeDir(projDir)
     raiseEx "Version command (" & projectCfg.versionCmd & ") returned non-zero exit code."
 
-  debug "Building version " & versionResult.output.strip
+  outputHandler.sendMsg "Building version " & versionResult.output.strip
   result.env["VERSION"] = versionResult.output.strip
 
-proc runStep*(step: Step, wksp: Workspace): void =
+proc runStep*(step: Step, wksp: Workspace,
-  let stepArtifactDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & step.name & "/" & wksp.env["VERSION"]
+              givenOutputHandler: HandleProcMsgCB = nil): void =
+  let SB_EXPECTED_VARS = ["VERSION"]
+  let stepArtifactDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" &
+                        step.name & "/" & wksp.env["VERSION"]
+  let statusFile = stepArtifactDir & ".status.json"
 
-  if existsDir(stepArtifactDir) and not step.dontCache:
-    info "Skipping step '" & step.name & "': already completed."
-    return
-
-  info "Running step '" & step.name & "' for " & wksp.project.name
-
-  # Ensure all expected environment variables are present.
-  for k in (step.expectedEnv & @SB_EXPECTED_VARS):
-    if not wksp.env.hasKey(k):
-      debug "workspace.env = " & $(wksp.env)
-      raiseEx "step " & step.name & " failed: missing required env variable: " & k
-
-  # Ensure that artifacts in steps we depend on are present
-  for dep in step.depends:
-    if not wksp.project.steps.hasKey(dep):
-      raiseEx step.name & " depends on " & dep & " but there is no step named " & dep
-    let depStep = wksp.project.steps[dep]
-
-    # Run that step (may get skipped)
-    let depDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & dep & "/" & wksp.env["VERSION"]
-    runStep(depStep, wksp)
-
-    # Add the artifacts directory for the dependent step to our env so that
-    # further steps can reference it via $<stepname>_DIR
-    echo "FP: " & depDir
-    wksp.env[depStep.name & "_DIR"] = depDir
-
-  # Run the step command, piping in cmdInput
-  debug step.name & ": starting stepCmd: " & step.stepCmd
-  let cmdProc = startProcess(step.stepCmd, wksp.workingDir & "/repo/" & step.workingDir,
-    [], wksp.env, {poUsePath, poEvalCommand})
-
-  let cmdInStream = inputStream(cmdProc)
-
-  # Replace env variables in step cmdInput as we pipe it in
-  for line in step.cmdInput: cmdInStream.writeLine(line.resolveEnvVars(wksp.env))
-  cmdInStream.flush()
-  cmdInStream.close()
-  
-  let cmdResult = waitForWithOutput(cmdProc, logProcOutput, step.stepCmd)
-
-  if cmdResult.exitCode != 0:
-    raiseEx "step " & step.name & " failed: step command returned non-zero exit code"
-
-  # Gather the output artifacts (if we have any)
   if not existsDir(stepArtifactDir): createDir(stepArtifactDir)
-  if step.artifacts.len > 0:
-    for a in step.artifacts:
-      let artifactPath = a.resolveEnvVars(wksp.env)
-      let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
-      try:
-        copyFile(wksp.workingDir & "/repo/" & step.workingDir & "/" & artifactPath,
-                 stepArtifactDir & "/" & artifactName)
-      except:
-        removeDir(stepArtifactDir)
-        raiseEx "step " & step.name & " failed: unable to copy artifact " &
-          artifactPath & ":\n" & getCurrentExceptionMsg()
 
-# TODO: change return to return logs, results, some compound, useful object.
+  # Have we tried to build this before and are we caching the results?
-proc runStep*(cfg: StrawBossConfig, projectName, stepName, buildRef: string): void =
+  if existsFile(statusFile) and not step.dontSkip:
+    let status = loadBuildStatus(statusFile)
 
-  let matching = cfg.projects.filterIt(it.name == projectName)
+    # If we succeeded last time, no need to rebuild
-  if matching.len == 0: raiseEx "no such project: " & projectName
+    if status.state == "complete":
-  elif matching.len > 1: raiseEx "more than one project named : " & projectName
+      givenOutputHandler.sendMsg "Skipping step '" & step.name & "' for version '" &
+        wksp.env["VERSION"] & "': already completed."
+      return
+    else:
+      givenOutputHandler.sendMsg "Rebuilding failed step '" & step.name & "' for version '" &
+        wksp.env["VERSION"] & "'."
 
-  let projectDef = matching[0]
+  givenOutputHandler.sendMsg "Running step '" & step.name & "' for " & wksp.project.name
+  writeFile(statusFile, $BuildStatus(state: "running", details: ""))
 
-  let foundBuildRef = if buildRef != nil: buildRef else: projectDef.defaultBranch
+  var stdoutLogFile, stderrLogFile: File
-  let wksp = setupProjectForWork(projectDef, foundBuildRef, cfg.artifactsRepo)
 
-  # Find the step
+  try:
-  if not wksp.project.steps.hasKey(stepName):
-    raiseEx "no step name '" & stepName & "' for " & projectName
 
-  let step = wksp.project.steps[stepName]
+    var outputHandler: HandleProcMsgCB
 
-  runStep(step, wksp)
+    # Make sure we log output to the stdout and sterr log files
+    if not (open(stdoutLogFile, stepArtifactDir & "/stdout.log", fmWrite) and
+            open(stderrLogFile, stepArtifactDir & "/stderr.log", fmWrite)):
+      givenOutputHandler.sendMsg nil, "Failed to open log files for STDOUT and STDERR."
+      outputHandler = givenOutputHandler
+      if stdoutLogFile != nil: close(stdoutLogFile)
+      if stderrLogFile != nil: close(stderrLogFile)
+    else:
+      outputHandler = combineProcMsgHandlers(
+        givenOutputHandler,
+        makeProcMsgHandler(stdoutLogFile, stderrLogFile))
 
+    # Ensure all expected environment variables are present.
+    for k in (step.expectedEnv & @SB_EXPECTED_VARS):
+      if not wksp.env.hasKey(k):
+        debug "workspace.env = " & $(wksp.env)
+        raiseEx "step " & step.name & " failed: missing required env variable: " & k
+
+    # Ensure that artifacts in steps we depend on are present
+    # TODO: detect circular-references in dependency trees.
+    for dep in step.depends:
+      if not wksp.project.steps.hasKey(dep):
+        raiseEx step.name & " depends on " & dep &
+                " but there is no step named " & dep
+      let depStep = wksp.project.steps[dep]
+
+      # Run that step (may get skipped)
+      let depDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" &
+                   dep & "/" & wksp.env["VERSION"]
+
+      runStep(depStep, wksp)
+
+      # Add the artifacts directory for the dependent step to our env so that
+      # further steps can reference it via $<stepname>_DIR
+      echo "FP: " & depDir
+      wksp.env[depStep.name & "_DIR"] = depDir
+
+    # Run the step command, piping in cmdInput
+    outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd
+    let cmdProc = startProcess(step.stepCmd,
+      wksp.workingDir & "/repo/" & step.workingDir,
+      [], wksp.env, {poUsePath, poEvalCommand})
+
+    let cmdInStream = inputStream(cmdProc)
+
+    # Replace env variables in step cmdInput as we pipe it in
+    for line in step.cmdInput: cmdInStream.writeLine(line.resolveEnvVars(wksp.env))
+    cmdInStream.flush()
+    cmdInStream.close()
+    
+    let cmdResult = waitForWithOutput(cmdProc, outputHandler, step.stepCmd)
+
+    if cmdResult.exitCode != 0:
+      raiseEx "step " & step.name & " failed: step command returned non-zero exit code"
+
+    # Gather the output artifacts (if we have any)
+    if step.artifacts.len > 0:
+      for a in step.artifacts:
+        let artifactPath = a.resolveEnvVars(wksp.env)
+        let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
+        try:
+          copyFile(wksp.workingDir & "/repo/" & step.workingDir & "/" & artifactPath,
+                   stepArtifactDir & "/" & artifactName)
+        except:
+          raiseEx "step " & step.name & " failed: unable to copy artifact " &
+            artifactPath & ":\n" & getCurrentExceptionMsg()
+
+    writeFile(statusFile, $BuildStatus(state: "complete", details: ""))
+
+  except:
+    writeFile(statusFile, $BuildStatus(
+      state: "failed",
+      details: getCurrentExceptionMsg()))
+
+  finally:
+    if stdoutLogFile != nil: close(stdoutLogFile)
+    if stderrLogFile != nil: close(stderrLogFile)
+
+proc runStep*(cfg: StrawBossConfig, req: RunRequest,
+              outputHandler: HandleProcMsgCB = nil): RunSummary =
+
+  if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
+  let statusFile = req.workspaceDir & "/" & "status.json"
+
+  try:
+    writeFile(statusFile, $BuildStatus(
+      state: "setup",
+      details: "Preparing working environment."))
+
+    # Find the project definition
+    let matching = cfg.projects.filterIt(it.name == req.projectName)
+    if matching.len == 0: raiseEx "no such project: " & req.projectName
+    elif matching.len > 1: raiseEx "more than one project named : " & req.projectName
+
+    let projectDef = matching[0]
+
+    # Find the commit reference we're building
+    let foundBuildRef =
+      if req.buildRef != nil and req.buildRef.len > 0: req.buildRef
+      else: projectDef.defaultBranch
+
+    let wksp = setupProjectForWork(projectDef, foundBuildRef, cfg.artifactsRepo,
+                                   outputHandler)
+
+    # Find the step
+    if not wksp.project.steps.hasKey(req.stepName):
+      raiseEx "no step name '" & req.stepName & "' for " & req.projectName
+
+    var step = wksp.project.steps[req.stepName]
+
+    # Enfore forceRebuild
+    if req.forceRebuild: step.dontSkip = true
+
+    result = RunSummary(
+      project: wksp.project,
+      step: step,
+      buildVersion: wksp.env["VERSION"],
+      statusFile: wksp.artifactsRepo & "/" & wksp.project.name & "/" & step.name &
+                                       "/" & wksp.env["VERSION"] & ".status.json")
+    if req.async:
+      let pid = fork()
+      if pid == 0: runStep(step, wksp, outputHandler) # if we are the child
+      else: result.workerPid = pid                    # if we are the parent
+
+    else: runStep(step, wksp, outputHandler)

src/main/nim/strawboss/private/util.nim

@@ -1,6 +1,11 @@
 import os, osproc, streams, strtabs
 
-type HandleProcMsgCB* = proc (cmd: string, outMsg: TaintedString, errMsg: TaintedString): void
+from posix import kill
+
+type HandleProcMsgCB* = proc (outMsg: TaintedString, errMsg: TaintedString, cmd: string): void
+
+proc sendMsg*(h: HandleProcMsgCB, outMsg: TaintedString, errMsg: TaintedString = nil, cmd: string = "strawboss"): void =
+  if h != nil: h(outMsg, errMsg, cmd)
 
 proc raiseEx*(reason: string): void =
   raise newException(Exception, reason)
@@ -22,11 +27,11 @@ proc waitForWithOutput*(p: Process, msgCB: HandleProcMsgCB,
   var line = newStringOfCap(120).TaintedString
   while true:
     if pout.readLine(line):
-      if msgCB != nil: msgCB(procCmd, line, nil)
+      msgCB.sendMsg(line, nil, procCmd)
       result[0].string.add(line.string)
       result[0].string.add("\n")
     elif perr.readLine(line):
-      if msgCB != nil: msgCB(procCmd, nil, line)
+      msgCB.sendMsg(nil, line, procCmd)
       result[1].string.add(line.string)
       result[1].string.add("\n")
     else:
@@ -49,3 +54,15 @@ proc loadEnv*(): StringTableRef =
 
   for k, v in envPairs():
     result[k] = v
+
+proc makeProcMsgHandler*(outSink, errSink: File): HandleProcMsgCB =
+  result = proc(outMsg, errMsg: TaintedString, cmd: string): void {.closure.} =
+    let prefix = if cmd != nil: cmd & ": " else: ""
+    if outMsg != nil: outSink.writeLine(prefix & outMsg)
+    if errMsg != nil: errSink.writeLine(prefix & errMsg)
+
+proc makeProcMsgHandler*(outSink, errSink: Stream): HandleProcMsgCB =
+  result = proc(outMsg, errMsg: TaintedString, cmd: string): void {.closure.} =
+    let prefix = if cmd != nil: cmd & ": " else: ""
+    if outMsg != nil: outSink.writeLine(prefix & outMsg)
+    if errMsg != nil: errSink.writeLine(prefix & errMsg)

src/main/nim/strawboss/server.nim

@@ -5,12 +5,29 @@ import ./configuration, ./core
 settings:
   port = Port(8180)
 
-routes:
-  get "/api/ping":
-    resp $(%*"pong"), "application/json"
 
-  get "/api/projects":
+proc start*(givenCfg: StrawBossConfig): void =
-    resp $(%*[]), "application/json"
+
+  routes:
+    get "/api/ping":
+      resp $(%*"pong"), "application/json"
+
+    get "/api/projects":
+      resp $(%*[]), "application/json"
+
+    post "/api/project/@projectName/@stepName/run/@buildRef?":
+      let req = RunRequest(
+        projectName: @"projectName",
+        stepName: @"stepName",
+        buildRef: if @"buildRef" != "": @"buildRef" else: nil,
+        async: true,
+        forceRebuild: false)  # TODO support this with optional query params
+
+#      try:
+#        let runSummary = core.runStep(givenCfg, req)
+#      except:
+#        discard ""
+#        # TODO
+
 
-proc start*(cfg: StrawBossConfig): void =
   runForever()

strawboss.config.json

@@ -4,5 +4,7 @@
   "tokens": [],
   "projects": [
     { "name": "new-life-intro-band",
-      "repo": "/home/jdb/projects/new-life-introductory-band" } ]
+      "repo": "/home/jdb/projects/new-life-introductory-band" },
+    { "name": "test-strawboss",
+      "repo": "/home/jdb/projects/test-strawboss" } ]
 }