WIP Moving back towards using named runs.

* Rename artifactsRepo -> buildDataDir to be more explicit about the fact that
  it holds more than just the artifacts.
* Revert removal of run ids.
* Move Worker definition into core as part of making the core responsible for
  accepting run requests.
* Make the core module more responsible for internal details of data structure
  and storage. External callers should not need to construct paths to
  artifacts, versions, etc. but should be able to call method in the core
  module to do this work for them.
* The working directory no longer contains anything but the checked-out code.
  All StrawBoss-specific data is stored by StrawBoss elsewhere.
* Add a regular maintenance cycle to the server module.

README.md

@@ -53,8 +53,9 @@ the `strawboss` executable. This is the configuration file for StrawBoss
 itself. The contents are expected to be a valid JSON object. The top level keys
 are:
 
-* `artifactsRepo`: A string denoting the path to the artifacts repository
+* `buildDataDir`: A string denoting the path to the directory where StrawBoss
-  directory.
+  keeps metadata about builds it has performed and the artifacts resulting from
+  the builds.
 
 * `authSecret`: Secret key used to sign JWT session tokens.
 
@@ -113,12 +114,7 @@ object. The top level keys are:
 
 * `versionCmd` *(optional)*: a command to be run in a shell (`sh`-compatible)
   that is expected to print the current version of the project on `stdout`.
-  It is important to note that if you supply a verion command it should provide
+  *(defaults to `git describe --tags --always`)*.
-  a unique result for every commit in the repository. StrawBoss is built around
-  the assumptions that builds are repeatable and that every buildable point has
-  a unique version id. This is the reason that StrawBoss does not create uniue
-  IDs for individual builds. The combination of project name, build step, and
-  version *is* the build ID. *(defaults to `git describe --tags --always`)*.
 
 ####  Step Definition
 

api.rst

@@ -6,10 +6,10 @@
 - GET    /api/project/<proj-id>                          -- TODO
 * GET    /api/project/<proj-id>/runs                     -- list summary information for all runs
 * GET    /api/project/<proj-id>/runs/active              -- list summary information about all currently active runs
+- GET    /api/project/<proj-id>/runs/<run-id>            -- list detailed information about a specific run
 ✓ GET    /api/project/<proj-id>/versions                 -- list the versions of this project that have been built
 * GET    /api/project/<proj-id>/version/<ref>            -- return detailed project definition (include steps) at a specific version
-- GET    /api/project/<proj-id>/step/<step-id>           -- return detailed step information (include runs for different versions)
+- GET    /api/project/<proj-id>/step/<step-id>           -- return detailed step information (include runs)
-- GET    /api/project/<proj-id>/step/<step-id>/run/<ref> -- list detailed information about a specific run
 * POST   /api/project/<proj-id>/step/<step-id>/run/<ref> -- kick off a run
 
 

file-structure.txt

@@ -0,0 +1,17 @@
+build-data/
+  <project-name>/
+    configurations/
+      <version>.json
+    runs/
+      <id>.request.json
+      <id>.stdout.log
+      <id>.stderr.log
+      <id>.status.json
+    status/
+      <version>.json
+    artifacts/
+      <step-name>/
+        <version>/
+          <artifact-file>
+
+workspace/

src/main/nim/strawboss.nim

@@ -1,4 +1,4 @@
-import cliutils, docopt, os, sequtils, tempfile
+import cliutils, docopt, os, sequtils, tempfile, uuids
 
 import strawbosspkg/configuration
 import strawbosspkg/core
@@ -30,6 +30,9 @@ Options
 
   -r --reference <ref>        Build the project at this commit reference.
 
+  -i --run-id <id>            Use the given UUID as the run ID. If not given, a
+                              new UUID is generated for this run.
+
   -w --workspace <workspace>  Use the given directory as the build workspace.
 
 """
@@ -41,11 +44,11 @@ Options
 
   var cfg = loadStrawBossConfig(cfgFile)
   cfg.pathToExe = paramStr(0)
-  if not existsDir(cfg.artifactsRepo):
+  if not existsDir(cfg.buildDataDir):
-    echo "Artifacts repo (" & cfg.artifactsRepo & ") does not exist. Creating..."
+    echo "Build data directory (" & cfg.buildDataDir & ") does not exist. Creating..."
-    createDir(cfg.artifactsRepo)
+    createDir(cfg.buildDataDir)
 
-  cfg.artifactsRepo = expandFilename(cfg.artifactsRepo)
+  cfg.buildDataDir = expandFilename(cfg.buildDataDir)
 
 
   if args["run"]:
@@ -54,6 +57,7 @@ Options
 
     try:
       let req = RunRequest(
+        id: if args["--run-id"]: parseUUID($args["--run-id"]) else: genUUID(),
         projectName: $args["<project>"],
         stepName: $args["<step>"],
         buildRef: if args["--reference"]: $args["--reference"] else: nil,

src/main/nim/strawbosspkg/configuration.nim

@@ -1,4 +1,4 @@
-import cliutils, logging, json, os, nre, sequtils, strtabs, tables, times
+import cliutils, logging, json, os, nre, sequtils, strtabs, tables, times, uuids
 
 from langutils import sameContents
 from typeinfo import toAny
@@ -7,7 +7,7 @@ from typeinfo import toAny
 #
 type
   BuildStatus* = object
-    state*, details*: string
+    runId*, state*, details*: string
 
   Step* = object
     name*, stepCmd*, workingDir*: string
@@ -24,6 +24,7 @@ type
     envVars*: StringTableRef
 
   RunRequest* = object
+    id*: UUID
     projectName*, stepName*, buildRef*, workspaceDir*: string
     forceRebuild*: bool
 
@@ -34,7 +35,7 @@ type
   UserRef* = ref User
 
   StrawBossConfig* = object
-    artifactsRepo*: string
+    buildDataDir*: string
     authSecret*: string
     filePath*: string
     debug*: bool
@@ -60,7 +61,7 @@ proc `==`*(a, b: ProjectDef): bool =
 
 proc `==`*(a, b: StrawBossConfig): bool =
   result =
-    a.artifactsRepo == b.artifactsRepo and
+    a.buildDataDir == b.buildDataDir and
     a.authSecret == b.authSecret and
     a.pwdCost == b.pwdCost and
     sameContents(a.users, b.users) and
@@ -68,6 +69,7 @@ proc `==`*(a, b: StrawBossConfig): bool =
 
 proc `==`*(a, b: RunRequest): bool =
   result =
+    a.id == b.id and
     a.projectName == b.projectName and
     a.stepName == b.stepName and
     a.buildRef == b.buildRef and
@@ -130,7 +132,7 @@ proc parseStrawBossConfig*(jsonCfg: JsonNode): StrawBossConfig =
       hashedPwd: uJson.getOrFail("hashedPwd", "user record").getStr))
 
   result = StrawBossConfig(
-    artifactsRepo: jsonCfg.getIfExists("artifactsRepo").getStr("artifacts"),
+    buildDataDir: jsonCfg.getIfExists("buildDataDir").getStr("build-data"),
     authSecret: jsonCfg.getOrFail("authSecret", "strawboss config").getStr,
     debug: jsonCfg.getIfExists("debug").getBVal(false),
     pwdCost: int8(jsonCfg.getOrFail("pwdCost", "strawboss config").getNum),
@@ -189,11 +191,13 @@ proc loadBuildStatus*(statusFile: string): BuildStatus =
   let jsonObj = parseFile(statusFile)
 
   result = BuildStatus(
+    runId: jsonObj.getOrFail("runId", "run ID").getStr,
     state: jsonObj.getOrFail("state", "build status").getStr,
     details: jsonObj.getIfExists("details").getStr("") )
 
 proc parseRunRequest*(reqJson: JsonNode): RunRequest =
   result = RunRequest(
+    id: parseUUID(reqJson.getOrFail("id", "RunRequest").getStr),
     projectName: reqJson.getOrFail("projectName", "RunRequest").getStr,
     stepName: reqJson.getOrFail("stepName", "RunRequest").getStr,
     buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr,
@@ -203,6 +207,7 @@ proc parseRunRequest*(reqJson: JsonNode): RunRequest =
 # TODO: can we use the marshal module for this?
 proc `%`*(s: BuildStatus): JsonNode =
   result = %* {
+    "runId": s.runId,
     "state": s.state,
     "details": s.details }
 
@@ -238,6 +243,7 @@ proc `%`*(p: ProjectConfig): JsonNode =
 
 proc `%`*(req: RunRequest): JsonNode =
   result = %* {
+    "id": $(req.id),
     "projectName": req.projectName,
     "stepName": req.stepName,
     "buildRef": req.buildRef,
@@ -251,7 +257,7 @@ proc `%`*(user: User): JsonNode =
 
 proc `%`*(cfg: StrawBossConfig): JsonNode =
   result = %* {
-    "artifactsRepo": cfg.artifactsRepo,
+    "buildDataDir": cfg.buildDataDir,
     "authSecret": cfg.authSecret,
     "debug": cfg.debug,
     "projects": %cfg.projects,

src/main/nim/strawbosspkg/core.nim

@@ -1,14 +1,14 @@
 import cliutils, logging, json, options, os, osproc, sequtils, streams,
-  strtabs, strutils, tables
+  strtabs, strutils, tables, uuids
 
-import nre except toSeq
 import ./configuration
+import nre except toSeq
 from posix import link
 
 type
   Workspace = ref object      ## Data needed by internal build process
     artifactsDir*: string     ## absolute path to the directory for this version
-    artifactsRepo*: string    ## absolute path to the global artifacts repo
+    buildDataDir*: string     ## absolute path to the global build data directory for this project
     buildRef*: string         ## git-style commit reference to the revision we are building
     dir*: string              ## absolute path to the working directory
     env*: StringTableRef      ## environment variables for all build processes
@@ -16,11 +16,16 @@ type
     outputHandler*: HandleProcMsgCB ## handler for process output
     project*: ProjectConfig   ## the project configuration
     projectDef*: ProjectDef   ## the StrawBoss project definition
+    runRequest*: RunRequest   ## the RunRequest that initated the current build
     status*: BuildStatus      ## the current status of the build
     statusFile*: string       ## absolute path to the build status file
     step*: Step               ## the step we're building
     version*: string          ## project version as returned by versionCmd
 
+  Worker* = object
+    runId*: UUID
+    process*: Process
+
 proc sendMsg(h: HandleProcMsgCB, msg: TaintedString): void =
   h.sendMsg(msg, nil, "strawboss")
 
@@ -45,21 +50,53 @@ proc emitStatus(status: BuildStatus, statusFilePath: string,
 proc publishStatus(wksp: Workspace, state, details: string) =
   ## Update the status for a Workspace and publish this status to the
   ## Workspace's status file and any output message handlers.
-  let status = BuildStatus(state: state, details: details)
+  let status = BuildStatus(
+    runId: $wksp.runRequest.id, state: state, details: details)
   wksp.status = emitStatus(status, wksp.statusFile, wksp.outputHandler)
 
+proc ensureProjectDirsExist(cfg: StrawBossConfig, p: ProjectDef): void =
+  for subdir in ["configurations", "runs", "status", "artifacts"]:
+    let fullPath = cfg.buildDataDir & "/" & p.name & "/" & subdir
+    if not existsDir(fullPath):
+      createDir(fullPath)
+
+proc listVersions*(cfg: StrawBossConfig, project: ProjectDef): seq[string] =
+  ## List the versions that have been built for a project.
+  ensureProjectDirsExist(cfg, project)
+
+  let versionFiles = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/configurations/*.json"))
+  result = versionFiles.map(proc(s: string): string =
+    let slashIdx = s.rfind('/')
+    result = s[(slashIdx + 1)..^6])
+
+proc listRuns*(cfg: StrawBossConfig, project: ProjectDef): seq[RunRequest] =
+  ## List the runs that have been performed for a project.
+  ensureProjectDirsExist(cfg, project)
+
+  let runPaths = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/runs/*.request.json"))
+  return runPaths.mapIt(parseRunRequest(parseFile(it)))
+
+proc getCurrentProjectConfig*(cfg: StrawBossConfig, project: ProjectDef): Option[ProjectConfig] =
+  let projCfgFile = "nope.json" # TODO
+  if not existsFile(projCfgFile): result = none(ProjectConfig)
+  else:
+    try:
+      let projectConfig: ProjectConfig = loadProjectConfig(projCfgFile) #ProjectConfig(name: "test")
+      result = some(projectConfig)
+    except: result = none(ProjectConfig)
+
 proc setupProject(wksp: Workspace) =
 
-  # Clone the project into the $temp/repo directory
+  # Clone the project into the $temp directory
-  let cloneResult = exec("git", wksp.dir,
+  let cloneResult = exec("git", ".",
-    ["clone", wksp.projectDef.repo, "repo"],
+    ["clone", wksp.projectDef.repo, wksp.dir],
     wksp.env, {poUsePath}, wksp.outputHandler)
 
   if cloneResult != 0:
     raiseEx "unable to clone repo for '" & wksp.projectDef.name & "'"
 
   # Checkout the requested ref
-  let checkoutResult = exec("git", wksp.dir & "/repo",
+  let checkoutResult = exec("git", wksp.dir,
     ["checkout", wksp.buildRef],
     wksp.env, {poUsePath}, wksp.outputHandler)
 
@@ -68,7 +105,7 @@ proc setupProject(wksp: Workspace) =
             " for '" & wksp.projectDef.name & "'"
 
   # Find the strawboss project configuration
-  let projCfgFile = wksp.dir & "/repo/" & wksp.projectDef.cfgFilePath
+  let projCfgFile = wksp.dir & wksp.projectDef.cfgFilePath
   if not existsFile(projCfgFile):
     raiseEx "Cannot find strawboss project configuration in the project " &
       "repo (expected at '" & wksp.projectDef.cfgFilePath & "')."
@@ -81,7 +118,7 @@ proc setupProject(wksp: Workspace) =
   # Get the build version
   let versionResult = execWithOutput(
     wksp.project.versionCmd,    # command
-    wksp.dir & "/repo",         # working dir
+    wksp.dir,                   # working dir
     [],                         # args
     wksp.env,                   # environment
     {poUsePath, poEvalCommand}) # options
@@ -93,22 +130,6 @@ proc setupProject(wksp: Workspace) =
   wksp.version = versionResult.output.strip
   wksp.env["VERSION"] = wksp.version
 
-proc listRuns*(cfg: StrawBossConfig, project: ProjectDef): seq[RunRequest] =
-  let runsDir = cfg.artifactsRepo & "/" & project.name & "/runs"
-  if not existsDir(runsDir): return @[]
-
-  let runPaths = toSeq(walkFiles(runsDir & "/*.json"))
-  return runPaths.mapIt(parseRunRequest(parseFile(it)))
-
-proc getCurrentProjectConfig*(cfg: StrawBossConfig, project: ProjectDef): Option[ProjectConfig] =
-  let projCfgFile = cfg.artifactsRepo & "/" & project.name & "/" & project.cfgFilePath
-  if not existsFile(projCfgFile): result = none(ProjectConfig)
-  else:
-    try:
-      let projectConfig: ProjectConfig = loadProjectConfig(projCfgFile) #ProjectConfig(name: "test")
-      result = some(projectConfig)
-    except: result = none(ProjectConfig)
-
 proc runStep*(wksp: Workspace, step: Step) =
 
   ## Lower-level method to execute a given step within the context of a project
@@ -139,14 +160,13 @@ proc runStep*(wksp: Workspace, step: Step) =
 
     # Add the artifacts directory for the dependent step to our env so that
     # further steps can reference it via $<stepname>_DIR
-    wksp.env[depStep.name & "_DIR"] = wksp.artifactsRepo & "/" &
+    wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir & "/artifacts/" &
-      wksp.project.name & "/" & dep & "/" & wksp.version
+      "/" & dep & "/" & wksp.version
 
   # Run the step command, piping in cmdInput
   wksp.outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd
   let cmdProc = startProcess(step.stepCmd,
-    wksp.dir & "/repo/" & step.workingDir,
+    wksp.dir & step.workingDir, [], wksp.env, {poUsePath, poEvalCommand})
-    [], wksp.env, {poUsePath, poEvalCommand})
 
   let cmdInStream = inputStream(cmdProc)
 
@@ -167,11 +187,11 @@ proc runStep*(wksp: Workspace, step: Step) =
       let artifactPath = a.resolveEnvVars(wksp.env)
       let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
       try:
-        wksp.outputHandler.sendMsg "copy " & wksp.dir & "/repo/" &
+        wksp.outputHandler.sendMsg "copy " & wksp.dir &
           step.workingDir & "/" & artifactPath & " -> " &
           wksp.artifactsDir & "/" & artifactName
 
-        copyFile(wksp.dir & "/repo/" & step.workingDir & "/" & artifactPath,
+        copyFile(wksp.dir & step.workingDir & "/" & artifactPath,
                  wksp.artifactsDir & "/" & artifactName)
       except:
         raiseEx "step " & step.name & " failed: unable to copy artifact " &
@@ -186,6 +206,7 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
   ## entrypoint to  running a build step.
 
   result = BuildStatus(
+    runId: $req.id,
     state: "setup",
     details: "initializing build workspace")
   discard emitStatus(result, nil, outputHandler)
@@ -193,29 +214,33 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
   var wksp: Workspace
 
   try:
-    assert req.workspaceDir.isAbsolute
-    if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
-
     # Find the project definition
     let projectDef = cfg.findProject(req.projectName)
 
-    # Make sure our directory in the artifacts repo exists
+    # Make sure the build data directories for this project exist.
-    if not existsDir(cfg.artifactsRepo & "/" & projectDef.name & "/run-requests"):
+    ensureProjectDirsExist(cfg, projectDef)
-      createDir(cfg.artifactsRepo & "/" & projectDef.name & "/run-requests")
+
+    # Update our run status
+    let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs"
+    discard emitStatus(result, runDir & "/" & $req.id & ".status.json", nil)
 
     # Read in the existing system environment
     var env = loadEnv()
     env["GIT_DIR"] = ".git"
 
+    # Make sure we have a workspace directory
+    assert req.workspaceDir.isAbsolute
+    if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
+
     # Setup our STDOUT and STDERR files
-    let stdoutFile = open(req.workspaceDir & "/stdout.log", fmWrite)
+    let stdoutFile = open(runDir & "/" & $req.id & ".stdout.log", fmWrite)
-    let stderrFile = open(req.workspaceDir & "/stderr.log", fmWrite)
+    let stderrFile = open(runDir & "/" & $req.id & ".stderr.log", fmWrite)
 
     let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile)
 
     wksp = Workspace(
       artifactsDir: nil,
-      artifactsRepo: cfg.artifactsRepo,
+      buildDataDir: cfg.buildDataDir & "/" & projectDef.name,
       buildRef:
         if req.buildRef != nil and req.buildRef.len > 0: req.buildRef
         else: projectDef.defaultBranch,
@@ -225,14 +250,15 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
       outputHandler: combineProcMsgHandlers(outputHandler, logFilesOH),
       project: ProjectConfig(),
       projectDef: projectDef,
+      runRequest: req,
       status: result,
-      statusFile: req.workspaceDir & "/" & "status.json",
+      statusFile: runDir & "/" & $req.id & ".status.json",
       step: Step(),
       version: nil)
 
   except:
     when not defined(release): echo getCurrentException().getStackTrace()
-    result = BuildStatus(state: "failed",
+    result = BuildStatus(runId: $req.id, state: "failed",
                          details: getCurrentExceptionMsg())
     try: discard emitStatus(result, nil, outputHandler)
     except: discard ""
@@ -244,33 +270,26 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
       "cloning project repo and preparing to run '" & req.stepName & "'")
     wksp.setupProject()
 
-    # Make sure our project directory exists in the artifacts repo
+    # Update our cache of project configurations.
-    if not existsDir(wksp.artifactsRepo & "/" & wksp.project.name):
+    # TODO: what happens if this fails?
-      createDir(wksp.artifactsRepo & "/" & wksp.project.name)
-
-    # Update our cache of project configurations by copying the configuration
-    # file to our artifacts directory.
     copyFile(
-      wksp.dir & "/repo/" & wksp.projectDef.cfgFilePath,
+      wksp.dir & "/" & wksp.projectDef.cfgFilePath,
-      cfg.artifactsRepo & "/" & wksp.project.name & "/configuration." &
+      wksp.buildDataDir & "/configurations/" & wksp.version & ".json")
-        wksp.version & ".json")
 
     # Find the requested step
     if not wksp.project.steps.hasKey(req.stepName):
       raiseEx "no step name '" & req.stepName & "' for " & req.projectName
-
     var step = wksp.project.steps[req.stepName]
 
-    # Enfore forceRebuild
     if req.forceRebuild: step.dontSkip = true
 
-    # Compose the path to the artifacts directory for this step and version
+    wksp.artifactsDir = wksp.buildDataDir & "/artifacts/" &
-    wksp.artifactsDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" &
                         step.name & "/" & wksp.version
 
     # Have we tried to build this before and are we caching the results?
-    if existsFile(wksp.artifactsDir & "/status.json") and not step.dontSkip:
+    let statusFilePath = wksp.buildDataDir & "/status/" & wksp.version & ".json"
-      let prevStatus = loadBuildStatus(wksp.artifactsDir & "/status.json")
+    if existsFile(statusFilePath) and not step.dontSkip:
+      let prevStatus = loadBuildStatus(statusFilePath)
 
       # If we succeeded last time, no need to rebuild
       if prevStatus.state == "complete":
@@ -283,21 +302,13 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
           "Rebuilding failed step '" & step.name & "' for version '" &
           wksp.version & "'.")
 
-    # Make the artifacts directory if it doesn't already exist
     if not existsDir(wksp.artifactsDir): createDir(wksp.artifactsDir)
 
-    # Link status file and output logs to the artifacts dir
-    for fn in @["status.json", "stdout.log", "stderr.log"]:
-      # TODO: roll old files instead of delete them?
-      if existsFile(wksp.artifactsDir & "/" & fn):
-        removeFile(wksp.artifactsDir & "/" & fn)
-
-      if link(wksp.dir & "/" & fn, wksp.artifactsDir & "/" & fn) != 0:
-        wksp.outputHandler.sendErrMsg(
-          "WARN: could not link " & fn & " to artifacts dir.")
-
     runStep(wksp, step)
 
+    # Record the results of this build as the status for this version. 
+    writeFile(wksp.buildDataDir & "/status/" & wksp.version & ".json", $wksp.status)
+
     result = wksp.status
 
   except:
@@ -307,7 +318,7 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
       wksp.publishStatus("failed", msg)
       result = wksp.status
     except:
-      result = BuildStatus(state: "failed", details: msg)
+      result = BuildStatus(runId: $req.id, state: "failed", details: msg)
       try: discard emitStatus(result, nil, outputHandler)
       except: discard ""
 
@@ -317,3 +328,38 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
         try: close(f)
         except: discard ""
 
+proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest): Worker =
+ 
+  # Find the project definition (will throw appropriate exceptions)
+  let projectDef = cfg.findProject(req.projectName)
+  let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs"
+  let reqFile = runDir & "/" & $req.id & ".request.json"
+  let statusFile = runDir & "/" & $req.id & ".status.json"
+
+  try:
+    # Make sure the build data directories for this project exist.
+    ensureProjectDirsExist(cfg, projectDef)
+
+    # Save the run request
+    writeFile(reqFile, $req)
+
+    # Write the initial build status (queued).
+    let queuedStatus = BuildStatus(
+      runId: $req.id,
+      state: "queued",
+      details: "request queued for execution")
+    writeFile(statusFile, $queuedStatus)
+
+    var args = @["run", reqFile]
+    debug "Launching worker: " & cfg.pathToExe & " " & args.join(" ")
+    result = Worker(
+      runId: req.id,
+      process: startProcess(cfg.pathToExe, ".", args, loadEnv(), {poUsePath}))
+
+  except:
+    let exMsg = "run request rejected: " & getCurrentExceptionMsg()
+    raiseEx exMsg
+    try:
+      writeFile(statusFile,
+        $(BuildStatus(runId: $req.id, state: "rejected", details: exMsg)))
+    except: discard ""

src/main/nim/strawbosspkg/server.nim

@@ -1,12 +1,8 @@
 import algorithm, asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging,
-  options, os, osproc, sequtils, strutils, tempfile, times, unittest
+  options, os, osproc, sequtils, strutils, tempfile, times, unittest, uuids
 
 import ./configuration, ./core
 
-type Worker = object
-  process*: Process
-  workingDir*: string
-
 type
   Session = object
     user*: UserRef
@@ -14,6 +10,7 @@ type
 
 #const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"
 const JSON = "application/json"
+const CLEANUP_PERIOD_MS = 1000
 
 proc makeJsonResp(status: HttpCode, details: string = ""): string =
   result = $(%* {
@@ -71,18 +68,6 @@ proc extractSession(cfg: StrawBossConfig, request: Request): Session =
 
   result = fromJWT(cfg, headerVal[7..^1])
 
-proc spawnWorker(cfg: StrawBossConfig, req: RunRequest): Worker =
-  ## Kick off a new worker process with the given run information
-
-  let dir = mkdtemp()
-  var args = @["run", req.projectName, req.stepName, "-r", req.buildRef,
-               "-w", dir, "-c", cfg.filePath]
-  if req.forceRebuild: args.add("-f")
-  debug "Launching worker: " & cfg.pathToExe & " " & args.join(" ")
-  result = Worker(
-    process: startProcess(cfg.pathToExe, ".", args, loadEnv(), {poUsePath}),
-    workingDir: dir)
-
 proc hashPwd*(pwd: string, cost: int8): string =
   let salt = genSalt(cost)
   result = hash(pwd, salt)
@@ -124,9 +109,19 @@ template checkAuth() =
     debug "Auth failed: " & getCurrentExceptionMsg()
     resp(Http401, makeJsonResp(Http401), JSON)
 
+proc performMaintenance(cfg: StrawBossConfig): void =
+  # Prune workers
+  workers = workers.filterIt(it.running())
+
+  let fut = sleepAsync(CLEANUP_PERIOD_MS)
+  fut.callback =
+    proc(): void =
+      callSoon(proc(): void = performMaintenance(cfg))
+
+
 proc start*(cfg: StrawBossConfig): void =
 
-  let stopFuture = newFuture[void]()
+  var stopFuture = newFuture[void]()
   var workers: seq[Worker] = @[]
 
   # TODO: add recurring clean-up down to clear completed workers from the
@@ -138,7 +133,7 @@ proc start*(cfg: StrawBossConfig): void =
 
   routes:
 
-    get "/ping": resp($(%*"pong"), JSON)
+    get "/ping": resp($(%"pong"), JSON)
 
     post "/auth-token":
       var uname, pwd: string
@@ -150,7 +145,7 @@ proc start*(cfg: StrawBossConfig): void =
 
       try:
         let authToken = makeAuthToken(cfg, uname, pwd)
-        resp("\"" & $authToken & "\"", JSON)
+        resp($(%authToken), JSON)
       except: resp(Http401, makeJsonResp(Http401, getCurrentExceptionMsg()), JSON)
 
     get "/verify-auth":
@@ -163,7 +158,7 @@ proc start*(cfg: StrawBossConfig): void =
 
       checkAuth(); if not authed: return true
 
-      resp($(%(cfg.projects)), JSON)
+      resp($(%cfg.projects), JSON)
 
     post "/projects":
       ## Create a new project definition
@@ -183,13 +178,7 @@ proc start*(cfg: StrawBossConfig): void =
       try: project = cfg.findProject(@"projectName")
       except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
 
-      var versions: seq[string] = @[]
+      var versions: seq[string] = listVersions(cfg, project)
-      for cfgFilePath in walkFiles(cfg.artifactsRepo & "/" & project.name & "/configuration.*.json"):
-        let vstart = cfgFilePath.rfind("/configuration.") + 15
-        versions.add(cfgFilePath[vstart..^6])
-
-      if versions.len == 0:
-        resp(Http404, makeJsonResp(Http404, "I have not built any versions of " & project.name), JSON)
 
       resp($(%(versions)), JSON)
 
@@ -206,8 +195,8 @@ proc start*(cfg: StrawBossConfig): void =
       # Given version
       var cachedFilePath: string
       if @"version" != "":
-        cachedFilePath = cfg.artifactsRepo & "/" & project.name &
+        cachedFilePath = cfg.buildDataDir & "/" & project.name &
-          "/configuration." & @"version" & ".json"
+          "/configurations/" & @"version" & ".json"
 
         if not existsFile(cachedFilePath):
           resp(Http404,
@@ -216,7 +205,7 @@ proc start*(cfg: StrawBossConfig): void =
 
       # No version requested, use "latest"
       else:
-        let confFilePaths = toSeq(walkFiles(cfg.artifactsRepo & "/" & project.name & "/configuration.*.json"))
+        let confFilePaths = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/configurations/*.json"))
         if confFilePaths.len == 0:
           resp(Http404, makeJsonResp(Http404, "I have not built any versions of " & project.name), JSON)
         let modTimes = confFilePaths.mapIt(it.getLastModificationTime)
@@ -230,7 +219,7 @@ proc start*(cfg: StrawBossConfig): void =
         resp(Http500, makeJsonResp(Http500, "could not read cached project configuration"), JSON)
 
     get "/project/@projectName":
-      ## TBD
+      ## Return a project's configuration, as well as it's versions.
 
       checkAuth(); if not authed: return true
 
@@ -239,17 +228,11 @@ proc start*(cfg: StrawBossConfig): void =
       try: projDef = cfg.findProject(@"projectName")
       except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
 
-      # Get the project configuration.
+      let respJson = newJObject()
-      let projConf = getCurrentProjectConfig(cfg, projDef)
+      respJson["definition"] = %projDef
+      respJson["versions"] = %listVersions(cfg, projDef)
 
-      var respObj = newJObject()
+      resp(pretty(respJson), JSON)
-      respObj["definition"] = %projDef
-
-      if projConf.isSome():
-        let pc: ProjectConfig = projConf.get()
-        respObj["configuration"] = %pc
-
-      resp($respObj, JSON)
 
     get "/project/@projectName/runs":
       ## List all runs
@@ -263,7 +246,7 @@ proc start*(cfg: StrawBossConfig): void =
 
       let runRequests = listRuns(cfg, project)
 
-      resp($(%runRequests), JSON)
+      resp($runRequests, JSON)
 
     get "/project/@projectName/runs/active":
       ## List all currently active runs
@@ -275,6 +258,14 @@ proc start*(cfg: StrawBossConfig): void =
       #resp($(%statuses), JSON)
       resp(Http501, makeJsonResp(Http501), JSON)
 
+    get "/project/@projectName/runs/@runId":
+      ## Details for a specific run
+
+      checkAuth(); if not authed: return true
+
+      # TODO
+      resp(Http501, makeJsonResp(Http501), JSON)
+
     get "/project/@projectName/step/@stepName":
       ## Get step details including runs.
 
@@ -297,6 +288,7 @@ proc start*(cfg: StrawBossConfig): void =
       checkAuth(); if not authed: return true
 
       let runRequest = RunRequest(
+        id: genUUID(),
         projectName: @"projectName",
         stepName: @"stepName",
         buildRef: if @"buildRef" != "": @"buildRef" else: nil,
@@ -316,8 +308,9 @@ proc start*(cfg: StrawBossConfig): void =
     post "/service/debug/stop":
       if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON)
       else:
-        callSoon(proc(): void = complete(stopFuture))
+        let shutdownFut = sleepAsync(100)
-        resp($(%*"shutting down"), JSON)
+        shutdownFut.callback = proc(): void = complete(stopFuture)
+        resp($(%"shutting down"), JSON)
 
   #[
     get re".*":
@@ -327,4 +320,5 @@ proc start*(cfg: StrawBossConfig): void =
       resp(Http404, makeJsonResp(Http404), JSON)
   ]#
 
+  callSoon(proc(): void = performMaintenance(cfg))
   waitFor(stopFuture)

src/test/json/test-status.json

@@ -1,4 +1,5 @@
 {
+  "runId": "90843e0c-6113-4462-af33-a89ff9731031",
   "state": "failed",
   "details": "some very good reason"
 }

src/test/nim/functional/tserver.nim

@@ -17,7 +17,7 @@ let TIMEOUT = 2.minutes
 # configuration and working files.
 template keepEnv(): untyped =
   preserveEnv = true
-  echo "artifacts dir: " & tempArtifactsDir
+  echo "artifacts dir: " & tempBuildDataDir
   echo "strawboss serve -c " & tempCfgPath
 
 suite "strawboss server":
@@ -30,13 +30,13 @@ suite "strawboss server":
 
   # per-test setup: spin up a fresh strawboss instance
   setup:
-    let tempArtifactsDir = mkdtemp()
+    let tempBuildDataDir = mkdtemp()
     let (_, tempCfgPath) = mkstemp()
     var preserveEnv = false
 
     # copy our test config
     var newCfg = cfg
-    newCfg.artifactsRepo = tempArtifactsDir
+    newCfg.buildDataDir = tempBuildDataDir
 
     # update the repo string for the extracted test project
     var testProjDef = newCfg.findProject(testProjName)
@@ -55,7 +55,7 @@ suite "strawboss server":
     discard newAsyncHttpClient().post(apiBase & "/service/debug/stop")
 
     if not preserveEnv:
-      removeDir(tempArtifactsDir)
+      removeDir(tempBuildDataDir)
       removeFile(tempCfgPath)
 
     # give the server time to spin down but kill it after that
@@ -73,14 +73,14 @@ suite "strawboss server":
     check resp.status.startsWith("404")
 
   test "GET /api/project/@projectName/versions":
-    let projArtifactsDir = tempArtifactsDir & "/" & testProjName
+    let cachedConfsDir = tempBuildDataDir & "/" & testProjName & "/configurations"
     let expectedVersions = @["alpha", "beta", "1.0.0", "1.0.1"]
 
     # Touch configuration files
-    createDir(projArtifactsDir)
+    createDir(cachedConfsDir)
     for v in expectedVersions:
       var f: File
-      check open(f, projArtifactsDir & "/configuration." & v & ".json", fmWrite)
+      check open(f, cachedConfsDir & "/" & v & ".json", fmWrite)
       close(f)
 
     let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
@@ -96,12 +96,13 @@ suite "strawboss server":
     # give the filesystem time to create stuff
     sleep(100)
 
+    # check that the run request has been 
     # check that the project directory has been created in the artifacts repo
-    let runArtifactsDir = tempArtifactsDir & "/" & testProjName & "/build/0.1.0"
+    let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build/0.1.0"
     check existsDir(runArtifactsDir)
 
     # check that the run status file has been created in the artifacts repo
-    let statusFile = runArtifactsDir & "/status.json"
+    let statusFile = tempBuildDataDir & "/" & testProjName & "/status/0.1.0.json"
     check fileExists(statusFile)
 
     # check that the run status is not failed

src/test/nim/unit/tconfiguration.nim

@@ -21,6 +21,7 @@ suite "load and save configuration objects":
 
   test "parseRunRequest":
     let rr1 = RunRequest(
+      id: genUUID(),
       projectName: testProjDef.name,
       stepName: "build",
       buildRef: "master",
@@ -85,7 +86,7 @@ suite "load and save configuration objects":
                  envVars: newStringTable(modeCaseSensitive))]
 
     check:
-      cfg.artifactsRepo == "artifacts"
+      cfg.buildDataDir == "build-data"
       cfg.authSecret == "change me"
       cfg.pwdCost == 11
       sameContents(expectedUsers, cfg.users)
@@ -141,5 +142,6 @@ suite "load and save configuration objects":
     let st = loadBuildStatus("src/test/json/test-status.json")
 
     check:
+      st.runId == "90843e0c-6113-4462-af33-a89ff9731031"
       st.state == "failed"
       st.details == "some very good reason"

strawboss.nimble

@@ -19,7 +19,9 @@ requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git"
 # Tasks
 #
 task functest, "Runs the functional test suite.":
+  exec "nimble build"
   exec "nim c -r src/test/nim/run_functional_tests.nim"
 
 task unittest, "Runs the unit test suite.":
+  exec "nimble build"
   exec "nim c -r src/test/nim/run_unit_tests.nim"