Finished refactor to base the build process around explicit run instances.

* Implemented periodic maintenance window.
* Moved worker creation into the core module.
* Worker processes no longer create run requests, but read queued requests from
  the file system.
* Build status and logs have been moved into the StrawBoss data directory.
* An initial build status is recorded when the job is queued.
* Build status is recorded for build references as well as actual versions.
  So there will be a build status for "master", for example, that is
  overwritten whenever "master" is built for that step.
* RunRequests now include a timestamp.
* Added a Run object to contain both a RunRequest and the corresponding
  BuildStatus for that run.
* API endpoints that talk about runs now return Run objects instead of
  RunRequests.
* Moved all data layer operations into the core module so that the
  "database API" only lives in one place.

.editorconfig

@@ -0,0 +1,10 @@
+[*]
+charset=utf-8
+end_of_line=lf
+indent_style=space
+indent_size=2
+max_line_length=79
+
+[{.babelrc,.stylelintrc,jest.config,.eslintrc,*.bowerrc,*.jsb3,*.jsb2,*.json,*.js}]
+indent_style=space
+indent_size=2

README.md

@@ -53,16 +53,26 @@ the `strawboss` executable. This is the configuration file for StrawBoss
 itself. The contents are expected to be a valid JSON object. The top level keys
 are:
 
-* `buildDataDir`: A string denoting the path to the directory where StrawBoss
+* `buildDataDir`: *(optional)* A string denoting the path to the directory
-  keeps metadata about builds it has performed and the artifacts resulting from
+  where StrawBoss keeps metadata about builds it has performed and the
-  the builds.
+  artifacts resulting from the builds. *(defaults to `build-data`)*
 
-* `authSecret`: Secret key used to sign JWT session tokens.
+* `authSecret`: *(required)* Secret key used to sign JWT session tokens.
 
-* `users`: the array of user definition objects. Each user object is required
+* `users`: *(required)* the array of user definition objects. Each user object is required
   to have `username` and `hashedPwd` keys, both string.
 
-* `projects`: an array of project definitions (detailed below).
+* `projects`: *(required)* an array of project definitions (detailed below).
+
+* `pwdCost`: *(required)* parameter to the user password hashing algorithm determining the
+  computational cost of the hash.
+
+* `maintenancePeriod`: *(optional)* how often, in milliseconds, should the
+  StrawBoss server perform maintenance (clear finished workers, etc).
+  *(defaults to `10000`, every 10 seconds)*.
+
+* `debug`: boolean, should debug behavior be enabled. This is primarily
+  intended for testing during StrawBoss development. *(defaults to `false`)*
 
 All are required.
 

file-structure.txt

@@ -8,7 +8,8 @@ build-data/
       <id>.stderr.log
       <id>.status.json
     status/
-      <version>.json
+      <step-name>/
+        <version>.json
     artifacts/
       <step-name>/
         <version>/

src/main/nim/strawboss.nim

@@ -1,4 +1,4 @@
-import cliutils, docopt, os, sequtils, tempfile, uuids
+import cliutils, docopt, os, sequtils, strutils, tempfile, uuids
 
 import strawbosspkg/configuration
 import strawbosspkg/core
@@ -16,25 +16,13 @@ when isMainModule:
   let doc = """
 Usage:
   strawboss serve [options]
-  strawboss run <project> <step> [options]
+  strawboss run <requestFile>
   strawboss hashpwd <pwd>
   
 Options
   
   -c --config-file <cfgFile>  Use this config file instead of the default
                               (strawboss.config.json).
-
-  -f --force-rebuild          Force a build step to re-run even we have cached
-                              results from building that step before for this
-                              version of the project.
-
-  -r --reference <ref>        Build the project at this commit reference.
-
-  -i --run-id <id>            Use the given UUID as the run ID. If not given, a
-                              new UUID is generated for this run.
-
-  -w --workspace <workspace>  Use the given directory as the build workspace.
-
 """
 
   let args = docopt(doc, version = "strawboss v" & SB_VER)
@@ -53,25 +41,24 @@ Options
 
   if args["run"]:
 
-    let wkspDir = if args["--workspace"]: $args["--workspace"] else: mkdtemp()
+    var req: RunRequest
+    try: req = loadRunRequest($args["<requestFile>"])
+    except:
+      echo "strawboss: unable to parse run request (" & $args["<requestFile>"] & ")"
+      quit(QuitFailure)
 
     try:
-      let req = RunRequest(
+
-        id: if args["--run-id"]: parseUUID($args["--run-id"]) else: genUUID(),
+      if req.workspaceDir.isNilOrEmpty: req.workspaceDir = mkdtemp()
-        projectName: $args["<project>"],
-        stepName: $args["<step>"],
-        buildRef: if args["--reference"]: $args["--reference"] else: nil,
-        forceRebuild: args["--force-rebuild"],
-        workspaceDir: wkspDir)
 
       let status = core.initiateRun(cfg, req, logProcOutput)
-      if status.state == "failed": raiseEx status.details
+      if status.state == BuildState.failed: raiseEx status.details
       echo "strawboss: build passed."
     except:
       echo "strawboss: build FAILED: " & getCurrentExceptionMsg() & "."
       quit(QuitFailure)
     finally:
-      if existsDir(wkspDir): removeDir(wkspDir)
+      if existsDir(req.workspaceDir): removeDir(req.workspaceDir)
 
   elif args["serve"]: server.start(cfg)
 

src/main/nim/strawbosspkg/configuration.nim

@@ -1,13 +1,20 @@
-import cliutils, logging, json, os, nre, sequtils, strtabs, tables, times, uuids
+import cliutils, logging, json, os, sequtils, strtabs, tables, times, uuids
 
 from langutils import sameContents
 from typeinfo import toAny
+from strutils import parseEnum
+
+const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz"
 
 # Types
 #
 type
+  BuildState* {.pure.} = enum
+    queued, complete, failed, running, setup, rejected
+
   BuildStatus* = object
-    runId*, state*, details*: string
+    runId*, details*: string
+    state*: BuildState
 
   Step* = object
     name*, stepCmd*, workingDir*: string
@@ -24,10 +31,16 @@ type
     envVars*: StringTableRef
 
   RunRequest* = object
-    id*: UUID
+    runId*: UUID
     projectName*, stepName*, buildRef*, workspaceDir*: string
+    timestamp*: TimeInfo
     forceRebuild*: bool
 
+  Run* = object
+    id*: UUID
+    request*: RunRequest
+    status*: BuildStatus
+
   User* = object
     name*: string
     hashedPwd*: string
@@ -43,6 +56,7 @@ type
     projects*: seq[ProjectDef]
     pwdCost*: int8
     users*: seq[UserRef]
+    maintenancePeriod*: int
 
 # Equality on custom types
 proc `==`*(a, b: UserRef): bool = result = a.name == b.name
@@ -64,38 +78,23 @@ proc `==`*(a, b: StrawBossConfig): bool =
     a.buildDataDir == b.buildDataDir and
     a.authSecret == b.authSecret and
     a.pwdCost == b.pwdCost and
+    a.maintenancePeriod == b.maintenancePeriod and
     sameContents(a.users, b.users) and
     sameContents(a.projects, b.projects)
 
 proc `==`*(a, b: RunRequest): bool =
   result =
-    a.id == b.id and
+    a.runId == b.runId and
     a.projectName == b.projectName and
     a.stepName == b.stepName and
     a.buildRef == b.buildRef and
+    a.timestamp == b.timestamp and
     a.workspaceDir == b.workspaceDir and
     a.forceRebuild == b.forceRebuild
 
-# Util methods on custom types
+# Useful utilities
-proc findProject*(cfg: StrawBossConfig, projectName: string): ProjectDef =
+proc filesMatching*(pat: string): seq[string] = toSeq(walkFiles(pat))
-  let candidates = cfg.projects.filterIt(it.name == projectName)
-  if candidates.len == 0:
-    raise newException(KeyError, "no project named " & projectName)
-  elif candidates.len > 1:
-    raise newException(KeyError, "multiple projects named " & projectName)
-  else: result = candidates[0]
 
-proc setProject*(cfg: var StrawBossConfig, projectName: string, newDef: ProjectDef): void =
-  var found = false
-  for idx in 0..<cfg.projects.len:
-    if cfg.projects[idx].name == projectName:
-      cfg.projects[idx] = newDef
-      found = true
-      break
-
-  if not found: cfg.projects.add(newDef)
-
-# other utils
 proc raiseEx*(reason: string): void =
   raise newException(Exception, reason)
 
@@ -137,6 +136,7 @@ proc parseStrawBossConfig*(jsonCfg: JsonNode): StrawBossConfig =
     debug: jsonCfg.getIfExists("debug").getBVal(false),
     pwdCost: int8(jsonCfg.getOrFail("pwdCost", "strawboss config").getNum),
     projects: jsonCfg.getIfExists("projects").getElems.mapIt(parseProjectDef(it)),
+    maintenancePeriod: int(jsonCfg.getIfExists("maintenancePeriod").getNum(10000)),
     users: users)
 
 
@@ -192,23 +192,30 @@ proc loadBuildStatus*(statusFile: string): BuildStatus =
 
   result = BuildStatus(
     runId: jsonObj.getOrFail("runId", "run ID").getStr,
-    state: jsonObj.getOrFail("state", "build status").getStr,
+    state: parseEnum[BuildState](jsonObj.getOrFail("state", "build status").getStr),
     details: jsonObj.getIfExists("details").getStr("") )
 
 proc parseRunRequest*(reqJson: JsonNode): RunRequest =
   result = RunRequest(
-    id: parseUUID(reqJson.getOrFail("id", "RunRequest").getStr),
+    runId: parseUUID(reqJson.getOrFail("runId", "RunRequest").getStr),
     projectName: reqJson.getOrFail("projectName", "RunRequest").getStr,
     stepName: reqJson.getOrFail("stepName", "RunRequest").getStr,
     buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr,
     workspaceDir: reqJson.getOrFail("workspaceDir", "RunRequest").getStr,
+    timestamp: times.parse(reqJson.getOrFail("timestamp", "RunRequest").getStr, ISO_TIME_FORMAT),
     forceRebuild: reqJson.getOrFail("forceRebuild", "RunRequest").getBVal)
 
+proc loadRunRequest*(reqFilePath: string): RunRequest =
+  if not existsFile(reqFilePath):
+    raiseEx "request file not found: " & reqFilePath
+
+  parseRunRequest(parseFile(reqFilePath))
+
 # TODO: can we use the marshal module for this?
 proc `%`*(s: BuildStatus): JsonNode =
   result = %* {
     "runId": s.runId,
-    "state": s.state,
+    "state": $s.state,
     "details": s.details }
 
 proc `%`*(p: ProjectDef): JsonNode =
@@ -243,12 +250,13 @@ proc `%`*(p: ProjectConfig): JsonNode =
 
 proc `%`*(req: RunRequest): JsonNode =
   result = %* {
-    "id": $(req.id),
+    "runId": $(req.runId),
     "projectName": req.projectName,
     "stepName": req.stepName,
     "buildRef": req.buildRef,
     "workspaceDir": req.workspaceDir,
-    "forceRebuild": req.forceRebuild }
+    "forceRebuild": req.forceRebuild,
+    "timestamp": req.timestamp.format(ISO_TIME_FORMAT) }
 
 proc `%`*(user: User): JsonNode =
   result = %* {
@@ -262,9 +270,17 @@ proc `%`*(cfg: StrawBossConfig): JsonNode =
     "debug": cfg.debug,
     "projects": %cfg.projects,
     "pwdCost": cfg.pwdCost,
+    "maintenancePeriod": cfg.maintenancePeriod,
     "users": %cfg.users }
 
+proc `%`*(run: Run): JsonNode =
+  result = %* {
+    "id": $run.id,
+    "request": %run.request,
+    "status": %run.status }
+
 proc `$`*(s: BuildStatus): string = result = pretty(%s)
 proc `$`*(req: RunRequest): string = result = pretty(%req)
 proc `$`*(pd: ProjectDef): string = result = pretty(%pd)
 proc `$`*(cfg: StrawBossConfig): string = result = pretty(%cfg)
+proc `$`*(run: Run): string = result = pretty(%run)

src/main/nim/strawbosspkg/core.nim

@@ -1,9 +1,10 @@
-import cliutils, logging, json, options, os, osproc, sequtils, streams,
+import cliutils, logging, json, os, osproc, sequtils, streams,
-  strtabs, strutils, tables, uuids
+  strtabs, strutils, tables, times, uuids
 
 import ./configuration
 import nre except toSeq
 from posix import link
+from algorithm import sorted
 
 type
   Workspace = ref object      ## Data needed by internal build process
@@ -18,19 +19,26 @@ type
     projectDef*: ProjectDef   ## the StrawBoss project definition
     runRequest*: RunRequest   ## the RunRequest that initated the current build
     status*: BuildStatus      ## the current status of the build
-    statusFile*: string       ## absolute path to the build status file
     step*: Step               ## the step we're building
     version*: string          ## project version as returned by versionCmd
 
   Worker* = object
     runId*: UUID
+    projectName*: string
     process*: Process
 
-proc sendMsg(h: HandleProcMsgCB, msg: TaintedString): void =
+  NotFoundException = object of Exception
-  h.sendMsg(msg, nil, "strawboss")
 
-proc sendErrMsg(h: HandleProcMsgCB, msg: TaintedString): void =
+# Utility methods for Workspace activities
-  h.sendMsg(nil, msg, "strawboss")
+proc sendStatusMsg(oh: HandleProcMsgCB, status: BuildStatus): void =
+  if not oh.isNil:
+    oh.sendMsg($status.state & ": " & status.details, nil, "strawboss")
+
+proc sendMsg(w: Workspace, msg: TaintedString): void =
+  w.outputHandler.sendMsg(msg, nil, "strawboss")
+
+proc sendErrMsg(w: Workspace, msg: TaintedString): void =
+  w.outputHandler.sendMsg(nil, msg, "strawboss")
 
 proc resolveEnvVars(line: string, env: StringTableRef): string =
   result = line
@@ -38,21 +46,30 @@ proc resolveEnvVars(line: string, env: StringTableRef): string =
     let key = if found[1] == '{': found[2..^2] else: found[1..^1]
     if env.hasKey(key): result = result.replace(found, env[key])
 
-proc emitStatus(status: BuildStatus, statusFilePath: string,
+proc publishStatus(wksp: Workspace, state: BuildState, details: string): void =
-                outputHandler: HandleProcMsgCB): BuildStatus =
-  ## Emit a BuildStatus to the given file as a JSON object and to the given
-  ## message handlers.
-  if statusFilePath != nil: writeFile(statusFilePath, $status)
-  if outputHandler != nil:
-    outputHandler.sendMsg(status.state & ": " & status.details)
-  result = status
-
-proc publishStatus(wksp: Workspace, state, details: string) =
   ## Update the status for a Workspace and publish this status to the
   ## Workspace's status file and any output message handlers.
-  let status = BuildStatus(
+  wksp.status = BuildStatus(
-    runId: $wksp.runRequest.id, state: state, details: details)
+    runId: $wksp.runRequest.runId, state: state, details: details)
-  wksp.status = emitStatus(status, wksp.statusFile, wksp.outputHandler)
+
+  # Write to our run directory, and to our version status
+  writeFile(wksp.buildDataDir & "/runs/" &
+    $wksp.runRequest.runId & ".status.json", $wksp.status)
+
+  # If we have our step we can save status to the step status
+  if not wksp.step.name.isNilOrEmpty():
+    let stepStatusDir = wksp.buildDataDir & "/status/" & wksp.step.name
+    if not existsDir(stepStatusDir): createDir(stepStatusDir)
+    writeFile(stepStatusDir & "/" & wksp.version & ".json", $wksp.status)
+
+    # If we were asked to build a ref that is not the version directly (like
+    # "master" or something), then let's also save our status under that name.
+    # We're probably overwriting a prior status, but that's OK.
+    if wksp.runRequest.buildRef != wksp.version:
+      writeFile(wksp.buildDataDir & "/status/" & wksp.step.name & "/" &
+        wksp.runRequest.buildRef & ".json", $wksp.status)
+
+  wksp.outputHandler.sendStatusMsg(wksp.status)
 
 proc ensureProjectDirsExist(cfg: StrawBossConfig, p: ProjectDef): void =
   for subdir in ["configurations", "runs", "status", "artifacts"]:
@@ -60,31 +77,123 @@ proc ensureProjectDirsExist(cfg: StrawBossConfig, p: ProjectDef): void =
     if not existsDir(fullPath):
       createDir(fullPath)
 
-proc listVersions*(cfg: StrawBossConfig, project: ProjectDef): seq[string] =
+# Data and configuration access
+
+proc getProject*(cfg: StrawBossConfig, projectName: string): ProjectDef =
+  ## Get a project definition by name from the service configuration
+  let candidates = cfg.projects.filterIt(it.name == projectName)
+  if candidates.len == 0:
+    raise newException(KeyError, "no project named " & projectName)
+  elif candidates.len > 1:
+    raise newException(KeyError, "multiple projects named " & projectName)
+  else: result = candidates[0]
+
+proc setProject*(cfg: var StrawBossConfig, projectName: string, newDef: ProjectDef): void =
+  ## Add a project definition to the service configuration
+  var found = false
+  for idx in 0..<cfg.projects.len:
+    if cfg.projects[idx].name == projectName:
+      cfg.projects[idx] = newDef
+      found = true
+      break
+
+  if not found: cfg.projects.add(newDef)
+
+proc listVersions*(cfg: StrawBossConfig, projectName: string): seq[string] =
   ## List the versions that have been built for a project.
+
+  let project = cfg.getProject(projectName)
+
   ensureProjectDirsExist(cfg, project)
 
-  let versionFiles = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/configurations/*.json"))
+  let versionFiles = filesMatching(
+    cfg.buildDataDir & "/" & project.name & "/configurations/*.json")
+
   result = versionFiles.map(proc(s: string): string =
     let slashIdx = s.rfind('/')
     result = s[(slashIdx + 1)..^6])
 
-proc listRuns*(cfg: StrawBossConfig, project: ProjectDef): seq[RunRequest] =
+proc existsRun*(cfg: StrawBossConfig, projectName, runId: string): bool =
+  existsFile(cfg.buildDataDir & "/" & projectName & "/runs/" & runId & ".request.json")
+
+proc getRun*(cfg: StrawBossConfig, projectName, runId: string): Run =
+  let project = cfg.getProject(projectName)
+  let runsPath = cfg.buildDataDir & "/" & project.name & "/runs"
+
+  try: result = Run(
+    id: parseUUID(runId),
+    request: loadRunRequest(runsPath & "/" & runId & ".request.json"),
+    status: loadBuildStatus(runsPath & "/" & runId & ".status.json"))
+  except: raiseEx "unable to load run information for id " & runId
+
+proc listRuns*(cfg: StrawBossConfig, projectName: string): seq[Run] =
   ## List the runs that have been performed for a project.
+  let project = cfg.getProject(projectName)
   ensureProjectDirsExist(cfg, project)
 
-  let runPaths = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/runs/*.request.json"))
+  let runsPath = cfg.buildDataDir & "/" & project.name & "/runs"
-  return runPaths.mapIt(parseRunRequest(parseFile(it)))
+  let reqPaths = filesMatching(runsPath & "/*.request.json")
 
-proc getCurrentProjectConfig*(cfg: StrawBossConfig, project: ProjectDef): Option[ProjectConfig] =
+  result = reqPaths.map(proc(reqPath: string): Run =
-  let projCfgFile = "nope.json" # TODO
+    let runId = reqPath[(runsPath.len + 1)..^14]
-  if not existsFile(projCfgFile): result = none(ProjectConfig)
+    result = Run(
+      id: parseUUID(runId),
+      request: loadRunRequest(reqPath),
+      status: loadBuildStatus(runsPath & "/" & runId & ".status.json")))
+
+proc getBuildStatus*(cfg: StrawBossConfig,
+                     projectName, stepName, buildRef: string): BuildStatus =
+
+  let project = cfg.getProject(projectName)
+
+  let statusFile = cfg.buildDataDir & "/" & project.name & "/status/" &
+    stepName & "/" & buildRef & ".json"
+
+  if not existsFile(statusFile):
+    raise newException(NotFoundException,
+      stepName & " has never been built for reference '" & buildRef)
+
+  result = loadBuildStatus(statusFile)
+
+proc getProjectConfig*(cfg: StrawBossConfig,
+                       projectName, version: string): ProjectConfig =
+
+  let project = cfg.getProject(projectName)
+  ensureProjectDirsExist(cfg, project)
+
+  # If they didn't give us a version, let try to figure out what is the latest one.
+  var confFilePath: string
+
+  if version.isNilOrEmpty:
+
+    let candidatePaths = filesMatching(
+      cfg.buildDataDir & "/" & project.name & "/configurations/*.json")
+
+    if candidatePaths.len == 0:
+      raise newException(NotFoundException,
+        "no versions of this project have been built")
+
+    let modTimes = candidatePaths.mapIt(it.getLastModificationTime)
+    confFilePath = sorted(zip(candidatePaths, modTimes),
+                              proc(a, b: tuple): int = cmp(a.b, b.b))[0].a
+
+    #cachedFilePath = sorted(zip(confFilePaths, modTimes),
+    #                        proc (a, b: tuple): int = cmp(a.b, b.b))[0].a
+
+  # If they did, let's try to load that
   else:
-    try:
+    confFilePath =
-      let projectConfig: ProjectConfig = loadProjectConfig(projCfgFile) #ProjectConfig(name: "test")
+      cfg.buildDataDir & "/" & project.name & "/configurations/" &
-      result = some(projectConfig)
+      version & ".json"
-    except: result = none(ProjectConfig)
 
+  if not existsFile(confFilePath):
+    raise newException(NotFoundException,
+      projectName & " version " & version & " has never been built")
+
+  result = loadProjectConfig(confFilePath)
+
+
+# Internal working methods.
 proc setupProject(wksp: Workspace) =
 
   # Clone the project into the $temp directory
@@ -105,7 +214,7 @@ proc setupProject(wksp: Workspace) =
             " for '" & wksp.projectDef.name & "'"
 
   # Find the strawboss project configuration
-  let projCfgFile = wksp.dir & wksp.projectDef.cfgFilePath
+  let projCfgFile = wksp.dir & "/" & wksp.projectDef.cfgFilePath
   if not existsFile(projCfgFile):
     raiseEx "Cannot find strawboss project configuration in the project " &
       "repo (expected at '" & wksp.projectDef.cfgFilePath & "')."
@@ -138,7 +247,9 @@ proc runStep*(wksp: Workspace, step: Step) =
 
   let SB_EXPECTED_VARS = ["VERSION"]
 
-  wksp.publishStatus("running",
+  wksp.step = step
+
+  wksp.publishStatus(BuildState.running,
     "running '" & step.name & "' for version " & wksp.version &
     " from " & wksp.buildRef)
 
@@ -164,9 +275,9 @@ proc runStep*(wksp: Workspace, step: Step) =
       "/" & dep & "/" & wksp.version
 
   # Run the step command, piping in cmdInput
-  wksp.outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd
+  wksp.sendMsg step.name & ": starting stepCmd: " & step.stepCmd
   let cmdProc = startProcess(step.stepCmd,
-    wksp.dir & step.workingDir, [], wksp.env, {poUsePath, poEvalCommand})
+    wksp.dir & "/" & step.workingDir, [], wksp.env, {poUsePath, poEvalCommand})
 
   let cmdInStream = inputStream(cmdProc)
 
@@ -181,23 +292,23 @@ proc runStep*(wksp: Workspace, step: Step) =
     raiseEx "step " & step.name & " failed: step command returned non-zero exit code"
 
   # Gather the output artifacts (if we have any)
-  wksp.outputHandler.sendMsg "artifacts: " & $step.artifacts
+  wksp.sendMsg "artifacts: " & $step.artifacts
   if step.artifacts.len > 0:
     for a in step.artifacts:
       let artifactPath = a.resolveEnvVars(wksp.env)
       let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
       try:
-        wksp.outputHandler.sendMsg "copy " & wksp.dir &
+        wksp.sendMsg "copy " &
-          step.workingDir & "/" & artifactPath & " -> " &
+          wksp.dir & "/" & step.workingDir & "/" & artifactPath & " -> " &
           wksp.artifactsDir & "/" & artifactName
 
-        copyFile(wksp.dir & step.workingDir & "/" & artifactPath,
+        copyFile(wksp.dir & "/" & step.workingDir & "/" & artifactPath,
                  wksp.artifactsDir & "/" & artifactName)
       except:
         raiseEx "step " & step.name & " failed: unable to copy artifact " &
           artifactPath & ":\n" & getCurrentExceptionMsg()
 
-  wksp.publishStatus("complete", "")
+  wksp.publishStatus(BuildState.complete, "")
 
 proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
               outputHandler: HandleProcMsgCB = nil): BuildStatus =
@@ -206,23 +317,23 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
   ## entrypoint to  running a build step.
 
   result = BuildStatus(
-    runId: $req.id,
+    runId: $req.runId,
-    state: "setup",
+    state: BuildState.setup,
     details: "initializing build workspace")
-  discard emitStatus(result, nil, outputHandler)
+  outputHandler.sendStatusMsg(result)
 
   var wksp: Workspace
 
   try:
     # Find the project definition
-    let projectDef = cfg.findProject(req.projectName)
+    let projectDef = cfg.getProject(req.projectName)
 
     # Make sure the build data directories for this project exist.
     ensureProjectDirsExist(cfg, projectDef)
 
     # Update our run status
     let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs"
-    discard emitStatus(result, runDir & "/" & $req.id & ".status.json", nil)
+    writeFile(runDir & "/" & $req.runId & ".status.json", $result)
 
     # Read in the existing system environment
     var env = loadEnv()
@@ -233,8 +344,8 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
     if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
 
     # Setup our STDOUT and STDERR files
-    let stdoutFile = open(runDir & "/" & $req.id & ".stdout.log", fmWrite)
+    let stdoutFile = open(runDir & "/" & $req.runId & ".stdout.log", fmWrite)
-    let stderrFile = open(runDir & "/" & $req.id & ".stderr.log", fmWrite)
+    let stderrFile = open(runDir & "/" & $req.runId & ".stderr.log", fmWrite)
 
     let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile)
 
@@ -252,21 +363,20 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
       projectDef: projectDef,
       runRequest: req,
       status: result,
-      statusFile: runDir & "/" & $req.id & ".status.json",
       step: Step(),
       version: nil)
 
   except:
     when not defined(release): echo getCurrentException().getStackTrace()
-    result = BuildStatus(runId: $req.id, state: "failed",
+    result = BuildStatus(runId: $req.runId, state: BuildState.failed,
                          details: getCurrentExceptionMsg())
-    try: discard emitStatus(result, nil, outputHandler)
+    try: outputHandler.sendStatusMsg(result)
     except: discard ""
     return
 
   try:
     # Clone the repo and setup the working environment
-    wksp.publishStatus("setup",
+    wksp.publishStatus(BuildState.setup,
       "cloning project repo and preparing to run '" & req.stepName & "'")
     wksp.setupProject()
 
@@ -287,18 +397,20 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
                         step.name & "/" & wksp.version
 
     # Have we tried to build this before and are we caching the results?
-    let statusFilePath = wksp.buildDataDir & "/status/" & wksp.version & ".json"
+    let statusFilePath = wksp.buildDataDir & "/status/" & step.name &
+      "/" & wksp.version & ".json"
+
     if existsFile(statusFilePath) and not step.dontSkip:
       let prevStatus = loadBuildStatus(statusFilePath)
 
       # If we succeeded last time, no need to rebuild
-      if prevStatus.state == "complete":
+      if prevStatus.state == BuildState.complete:
-        wksp.outputHandler.sendMsg(
+        wksp.publishStatus(BuildState.complete,
-          "Skipping step '" & step.name & "' for version '" &
+          "Skipping step '" & step.name & "' for version '" & wksp.version &
-          wksp.version & "': already completed.")
+            "': already completed.")
         return prevStatus
       else:
-        wksp.outputHandler.sendMsg(
+        wksp.sendMsg(
           "Rebuilding failed step '" & step.name & "' for version '" &
           wksp.version & "'.")
 
@@ -306,20 +418,17 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
 
     runStep(wksp, step)
 
-    # Record the results of this build as the status for this version. 
-    writeFile(wksp.buildDataDir & "/status/" & wksp.version & ".json", $wksp.status)
-
     result = wksp.status
 
   except:
     when not defined(release): echo getCurrentException().getStackTrace()
     let msg = getCurrentExceptionMsg()
     try:
-      wksp.publishStatus("failed", msg)
+      wksp.publishStatus(BuildState.failed, msg)
       result = wksp.status
     except:
-      result = BuildStatus(runId: $req.id, state: "failed", details: msg)
+      result = BuildStatus(runId: $req.runId, state: BuildState.failed, details: msg)
-      try: discard emitStatus(result, nil, outputHandler)
+      try: outputHandler.sendStatusMsg(result)
       except: discard ""
 
   finally:
@@ -328,13 +437,14 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
         try: close(f)
         except: discard ""
 
-proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest): Worker =
+proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
+  tuple[status: BuildStatus, worker: Worker] =
  
   # Find the project definition (will throw appropriate exceptions)
-  let projectDef = cfg.findProject(req.projectName)
+  let projectDef = cfg.getProject(req.projectName)
   let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs"
-  let reqFile = runDir & "/" & $req.id & ".request.json"
+  let reqFile = runDir & "/" & $req.runId & ".request.json"
-  let statusFile = runDir & "/" & $req.id & ".status.json"
+  let statusFile = runDir & "/" & $req.runId & ".status.json"
 
   try:
     # Make sure the build data directories for this project exist.
@@ -345,21 +455,25 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest): Worker =
 
     # Write the initial build status (queued).
     let queuedStatus = BuildStatus(
-      runId: $req.id,
+      runId: $req.runId,
-      state: "queued",
+      state: BuildState.queued,
       details: "request queued for execution")
     writeFile(statusFile, $queuedStatus)
 
     var args = @["run", reqFile]
     debug "Launching worker: " & cfg.pathToExe & " " & args.join(" ")
-    result = Worker(
+
-      runId: req.id,
+    let worker = Worker(
+      runId: req.runId,
+      projectName: projectDef.name,
       process: startProcess(cfg.pathToExe, ".", args, loadEnv(), {poUsePath}))
 
+    result = (queuedStatus, worker)
+
   except:
     let exMsg = "run request rejected: " & getCurrentExceptionMsg()
     raiseEx exMsg
     try:
       writeFile(statusFile,
-        $(BuildStatus(runId: $req.id, state: "rejected", details: exMsg)))
+        $(BuildStatus(runId: $req.runId, state: BuildState.rejected, details: exMsg)))
     except: discard ""

src/main/nim/strawbosspkg/server.nim

@@ -1,5 +1,5 @@
-import algorithm, asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging,
+import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging,
-  options, os, osproc, sequtils, strutils, tempfile, times, unittest, uuids
+  os, osproc, sequtils, strutils, tempfile, times, unittest, uuids
 
 import ./configuration, ./core
 
@@ -10,7 +10,6 @@ type
 
 #const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"
 const JSON = "application/json"
-const CLEANUP_PERIOD_MS = 1000
 
 proc makeJsonResp(status: HttpCode, details: string = ""): string =
   result = $(%* {
@@ -109,24 +108,11 @@ template checkAuth() =
     debug "Auth failed: " & getCurrentExceptionMsg()
     resp(Http401, makeJsonResp(Http401), JSON)
 
-proc performMaintenance(cfg: StrawBossConfig): void =
-  # Prune workers
-  workers = workers.filterIt(it.running())
-
-  let fut = sleepAsync(CLEANUP_PERIOD_MS)
-  fut.callback =
-    proc(): void =
-      callSoon(proc(): void = performMaintenance(cfg))
-
-
 proc start*(cfg: StrawBossConfig): void =
 
   var stopFuture = newFuture[void]()
   var workers: seq[Worker] = @[]
 
-  # TODO: add recurring clean-up down to clear completed workers from the
-  # workers queu and kick off pending requests as worker slots free up.
-
   settings:
     port = Port(8180)
     appName = "/api"
@@ -168,56 +154,6 @@ proc start*(cfg: StrawBossConfig): void =
       # TODO
       resp(Http501, makeJsonResp(Http501), JSON)
 
-    get "/project/@projectName/versions":
-      ## Get a list of all versions that we have built
-
-      checkAuth(); if not authed: return true
-
-      # Make sure we know about that project
-      var project: ProjectDef
-      try: project = cfg.findProject(@"projectName")
-      except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
-
-      var versions: seq[string] = listVersions(cfg, project)
-
-      resp($(%(versions)), JSON)
-
-    get "/project/@projectName/version/@version?":
-      ## Get a detailed project record including step definitions (ProjectConfig).
-
-      checkAuth(); if not authed: return true
-
-      # Make sure we know about that project
-      var project: ProjectDef
-      try: project = cfg.findProject(@"projectName")
-      except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
-
-      # Given version
-      var cachedFilePath: string
-      if @"version" != "":
-        cachedFilePath = cfg.buildDataDir & "/" & project.name &
-          "/configurations/" & @"version" & ".json"
-
-        if not existsFile(cachedFilePath):
-          resp(Http404,
-               makeJsonResp(Http404, "I have never built version " & @"version"),
-               JSON)
-
-      # No version requested, use "latest"
-      else:
-        let confFilePaths = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/configurations/*.json"))
-        if confFilePaths.len == 0:
-          resp(Http404, makeJsonResp(Http404, "I have not built any versions of " & project.name), JSON)
-        let modTimes = confFilePaths.mapIt(it.getLastModificationTime)
-        cachedFilePath = sorted(zip(confFilePaths, modTimes),
-                                proc (a, b: tuple): int = cmp(a.b, b.b))[0].a
-
-      try: resp(readFile(cachedFilePath), JSON)
-      except:
-        debug "Could not serve cached project configuration at: " &
-          cachedFilePath & "\n\t Reason: " & getCurrentExceptionMsg()
-        resp(Http500, makeJsonResp(Http500, "could not read cached project configuration"), JSON)
-
     get "/project/@projectName":
       ## Return a project's configuration, as well as it's versions.
 
@@ -225,46 +161,85 @@ proc start*(cfg: StrawBossConfig): void =
 
       # Make sure we know about that project
       var projDef: ProjectDef
-      try: projDef = cfg.findProject(@"projectName")
+      try: projDef = cfg.getProject(@"projectName")
       except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
 
+      var projConf: ProjectConfig
+      try: projConf = getProjectConfig(cfg, @"projectName", "")
+      except: discard ""
+
       let respJson = newJObject()
       respJson["definition"] = %projDef
-      respJson["versions"] = %listVersions(cfg, projDef)
+      respJson["versions"] = %listVersions(cfg, @"projectName")
+      if not projConf.name.isNil:
+        respJson["latestConfig"] = %projConf
 
       resp(pretty(respJson), JSON)
 
+    get "/project/@projectName/versions":
+      ## Get a list of all versions that we have built
+
+      checkAuth(); if not authed: return true
+
+      try: resp($(%listVersions(cfg, @"projectName")), JSON)
+      except:
+        if getCurrentException() is KeyError:
+          resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
+        else:
+          when not defined(release): debug getCurrentException().getStackTrace()
+          error "unable to list versions for project " & @"projectName" &
+            ":\n" & getCurrentExceptionMsg()
+          resp(Http500, makeJsonResp(Http500, "internal server error"), JSON)
+
+    get "/project/@projectName/version/@version?":
+      ## Get a detailed project record including step definitions (ProjectConfig).
+
+      checkAuth(); if not authed: return true
+
+      # Make sure we know about that project
+      try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON)
+      except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
+
     get "/project/@projectName/runs":
       ## List all runs
 
       checkAuth(); if not authed: return true
 
-      # Make sure we know about that project
+      try: resp($(%listRuns(cfg, @"projectName")), JSON)
-      var project: ProjectDef
-      try: project = cfg.findProject(@"projectName")
       except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
 
-      let runRequests = listRuns(cfg, project)
-
-      resp($runRequests, JSON)
-
     get "/project/@projectName/runs/active":
       ## List all currently active runs
 
       checkAuth(); if not authed: return true
 
-      #let statusFiles = workers.mapIt(it.workingDir & "/status.json")
+      try:
-      #let statuses = statusFiles.mapIt(loadBuildStatus(it)).filterIt(it.state != "completed" && it.)
+        let activeRuns = workers
-      #resp($(%statuses), JSON)
+          .filterIt(it.process.running and it.projectName == @"projectName")
-      resp(Http501, makeJsonResp(Http501), JSON)
+          .mapIt(cfg.getRun(@"projecName", $it.runId));
+        resp($(%activeRuns), JSON)
+      except:
+        if getCurrentException() is KeyError:
+          resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
+        else:
+          when not defined(release): debug getCurrentException().getStackTrace()
+          error "problem loading active runs: " & getCurrentExceptionMsg()
+          resp(Http500, makeJsonResp(Http500, "internal server error"), JSON)
 
-    get "/project/@projectName/runs/@runId":
+    get "/project/@projectName/run/@runId":
       ## Details for a specific run
 
       checkAuth(); if not authed: return true
 
-      # TODO
+      # Make sure we know about that project
-      resp(Http501, makeJsonResp(Http501), JSON)
+      try: discard cfg.getProject(@"projectName")
+      except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
+
+      if not existsRun(cfg, @"projectName", @"runId"):
+        resp(Http404, makeJsonResp(Http404, "no such run for project"), JSON)
+
+      try: resp($getRun(cfg, @"projectName", @"runId"), JSON)
+      except: resp(Http500, makeJsonResp(Http500, getCurrentExceptionMsg()), JSON)
 
     get "/project/@projectName/step/@stepName":
       ## Get step details including runs.
@@ -274,13 +249,13 @@ proc start*(cfg: StrawBossConfig): void =
       # TODO
       resp(Http501, makeJsonResp(Http501), JSON)
 
-    get "/project/@projectName/step/@stepName/run/@buildRef":
+    get "/project/@projectName/step/@stepName/status/@buildRef":
-      ## Get detailed information about a run
+      ## Get detailed information about the status of a step (assuming it has been built)
 
       checkAuth(); if not authed: return true
 
-      # TODO
+      try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON)
-      resp(Http501, makeJsonResp(Http501), JSON)
+      except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
 
     post "/project/@projectName/step/@stepName/run/@buildRef?":
       # Kick off a run
@@ -288,22 +263,23 @@ proc start*(cfg: StrawBossConfig): void =
       checkAuth(); if not authed: return true
 
       let runRequest = RunRequest(
-        id: genUUID(),
+        runId: genUUID(),
         projectName: @"projectName",
         stepName: @"stepName",
         buildRef: if @"buildRef" != "": @"buildRef" else: nil,
+        timestamp: getLocalTime(getTime()),
         forceRebuild: false) # TODO support this with optional query params
 
       # TODO: instead of immediately spawning a worker, add the request to a
       # queue to be picked up by a worker. Allows capping the number of worker
       # prcesses, distributing, etc.
-      let worker = spawnWorker(cfg, runRequest)
+      let (status, worker) = spawnWorker(cfg, runRequest)
       workers.add(worker)
 
-      resp($(%*{
+      resp($Run(
-        "runRequest": runRequest,
+        id: runRequest.runId,
-        "status": { "state": "accepted", "details": "Run request has been queued." }
+        request: runRequest,
-      }))
+        status: status), JSON)
 
     post "/service/debug/stop":
       if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON)
@@ -312,13 +288,27 @@ proc start*(cfg: StrawBossConfig): void =
         shutdownFut.callback = proc(): void = complete(stopFuture)
         resp($(%"shutting down"), JSON)
 
-  #[
+    #[
     get re".*":
       resp(Http404, makeJsonResp(Http404), JSON)
 
     post re".*":
       resp(Http404, makeJsonResp(Http404), JSON)
-  ]#
+    ]#
+
+  proc performMaintenance(cfg: StrawBossConfig): void =
+    # Prune workers
+    workers = workers.filterIt(it.process.running())
+    debug "Performing maintanance: " & $len(workers) & " active workers after pruning."
+
+    let fut = sleepAsync(cfg.maintenancePeriod)
+    fut.callback =
+      proc(): void =
+        callSoon(proc(): void = performMaintenance(cfg))
+
+
+  info "StrawBoss is bossing people around."
+  #debug "configuration:\n\n" & $cfg & "\n\n"
 
   callSoon(proc(): void = performMaintenance(cfg))
   waitFor(stopFuture)

src/test/nim/functional/tcore.nim

@@ -0,0 +1,7 @@
+import unittest
+
+from langutils import sameContents
+
+import ../testutil
+import ../../../main/nim/strawbosspkg/configuration
+

strawboss.config.json

@@ -1,9 +1,10 @@
 {
-  "artifactsRepo": "artifacts",
+  "buildDataDir": "build-data",
   "debug": true,
   "users": [],
   "authSecret": "change me",
   "pwdCost": 11,
+  "maintenancePeriod": 5000,
   "projects": [
     { "name": "new-life-intro-band",
       "repo": "/home/jdb/projects/new-life-introductory-band" },

strawboss.nimble

@@ -14,7 +14,7 @@ requires @["nim >= 0.16.1", "docopt >= 0.6.5", "isaac >= 0.1.2", "tempfile", "je
 
 requires "https://github.com/yglukhov/nim-jwt"
 requires "https://git.jdb-labs.com/jdb/nim-lang-utils.git"
-requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git"
+requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git >= 0.3.1"
 
 # Tasks
 #