Fix unit tests for latest changes to API.

* Added endopint to return the logs from a run.
* Refactored the `pathVar & "/" & pathvar` pattern into `pathVar / pathVar`
  using the `ospaths` module. Cleaner code, more resistant to extra `/` bugs.
* Added endpoints and core methods to list artifacts for a build, as well as to
  retrieve specific artifacts.
* Fixed a problem with the `complete` status being overloaded. The problem was
  that in the case of a multi-step build all of the prerequisite steps will
  return a state of `complete` which will get recorded in the status file for
  the run. The run will continue, but anyone watching the run state file (via
  the API for example) had no definitive way to tell the difference between a
  sub-step completing and the requested (last) step completing. This was caught
  in the functional tests (race condition based on when it polls for the run
  status). The fix was to introduce a new build state: `stepComplete`. The
  inner `doRun` procedure uses this instead of `complete`. Now the only place
  that `complete` is set is at the end of the original call to `run`, right
  before the worker terminates. It checks the last result (from the originally
  requested step) and if this result is `stepComplete` it "finishes" the build
  by setting the state to `complete`. Because this is the only place where
  `complete` is set, an observer is now guaranteed not to see `complete` until
  all steps have run successfully.
* Fixed a long-standing bug with the request handling logic in error cases
  (like requested resources not being available). Issue has something to do
  with the way that `except` blocks become special when in an async context.
  The main jester routes block invokes the handlers in an async context. The
  effect is that one `except` block is fine, but adding more than one (to catch
  different exception types, for example) causes the return type of the route
  handler to change and not match what the outer block is expecting (a Future).
  The fix here is to wrap any exception discrimination within a single outer
  except block, re-raise the exception, and catch it inside this new
  synchronous context. Ex:

  ```nim
    try: someCall(mayFail)
    except:
      try: raise getCurrentException()
      except ExceptionType1:
        # do whatever...
      except ExceptionType2:
        # do whatever
      except:
        # general catch-all
    return true
  ```

  The return at the end is also part of the story. Jester's match handler
  allows a route to defer making a decision about whether it matches. If you
  return true from a route block Jester accepts the result as a matched route.
  If you return false, Jester discards the result and looks for another
  matching route. Normally this is taken care of by the `resp` templates
  provided by Jester, but invoking those templates within the except blocks
  also causes problems, so we manually setup the response and `return true` to
  tell Jester that, yes this route matched, use this response.
* Moved the `/service/debug/ping` endpoint back to `/ping` and removed the
  debug-only fence. I envision this as being useful as a simple healthcheck URL.

TODO.md

@@ -1,6 +1,11 @@
-* Write a tool to convert JSON Schema into a human-readable format suitable for
-  documentation. Should use the description, title, and other fields from the
-  JSON spec. Use this for writing the JSON schema docs instead of duplicating
-  the description of configuration files between JSON schema and the
-  documentation. In other words, use the schemas as the single source of truth
-  and generate everything else from that.
+TODO
+
+* Orchestration of docker containers for running builds.
+* Write API docs.
+
+NICE TO HAVE
+
+* Use/create some json-schema -> nim code generator to auto-generate json
+  handling code from schemas.
+* Use some json-schema -> docs generator to document the API.
+* Support unique UUID prefixes in URLs.

src/main/nim/strawboss.nim

@@ -4,7 +4,7 @@ import strawbosspkg/configuration
 import strawbosspkg/core
 import strawbosspkg/server
 
-let SB_VER = "0.3.1"
+let SB_VER = "0.4.0"
 
 proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) =
   let prefix = if cmd != nil: cmd & ": " else: ""

src/main/nim/strawbosspkg/configuration.nim

@@ -10,7 +10,7 @@ const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz"
 #
 type
   BuildState* {.pure.} = enum
-    queued, complete, failed, running, setup, rejected
+    complete, failed, queued, rejected, running, setup, stepComplete
 
   BuildStatus* = object
     runId*, details*: string
@@ -41,6 +41,10 @@ type
     request*: RunRequest
     status*: BuildStatus
 
+  RunLogs* = object
+    runId*: UUID
+    stdout*, stderr*: seq[string]
+
   User* = object
     name*: string
     hashedPwd*: string
@@ -296,8 +300,15 @@ proc `%`*(run: Run): JsonNode =
     "request": %run.request,
     "status": %run.status }
 
+proc `%`*(logs: RunLogs): JsonNode =
+  result = %* {
+    "runId": $logs.runId,
+    "stdout": %logs.stdout,
+    "stderr": %logs.stderr }
+
 proc `$`*(s: BuildStatus): string = result = pretty(%s)
 proc `$`*(req: RunRequest): string = result = pretty(%req)
 proc `$`*(pd: ProjectDef): string = result = pretty(%pd)
 proc `$`*(cfg: StrawBossConfig): string = result = pretty(%cfg)
 proc `$`*(run: Run): string = result = pretty(%run)
+proc `$`*(logs: RunLogs): string = result = pretty(%logs)

src/main/nim/strawbosspkg/core.nim

@@ -1,9 +1,9 @@
-import cliutils, logging, json, os, osproc, sequtils, streams,
+import cliutils, logging, json, os, ospaths, osproc, sequtils, streams,
   strtabs, strutils, tables, times, uuids
 
 import ./configuration
 import nre except toSeq
-from posix import link
+from posix import link, realpath
 from algorithm import sorted
 
 type
@@ -27,7 +27,7 @@ type
     projectName*: string
     process*: Process
 
-  NotFoundException = object of Exception
+  NotFoundException* = object of Exception
 
 proc newCopy(w: Workspace): Workspace =
   var newEnv: StringTableRef = newStringTable()
@@ -82,27 +82,27 @@ proc publishStatus(wksp: Workspace, state: BuildState, details: string): void =
     runId: $wksp.runRequest.runId, state: state, details: details)
 
   # Write to our run directory, and to our version status
-  writeFile(wksp.buildDataDir & "/runs/" &
+  writeFile(wksp.buildDataDir / "runs" /
     $wksp.runRequest.runId & ".status.json", $wksp.status)
 
   # If we have our step we can save status to the step status
   if not wksp.step.name.isNilOrEmpty():
-    let stepStatusDir = wksp.buildDataDir & "/status/" & wksp.step.name
+    let stepStatusDir = wksp.buildDataDir / "status" / wksp.step.name
     if not existsDir(stepStatusDir): createDir(stepStatusDir)
-    writeFile(stepStatusDir & "/" & wksp.version & ".json", $wksp.status)
+    writeFile(stepStatusDir / wksp.version & ".json", $wksp.status)
 
     # If we were asked to build a ref that is not the version directly (like
     # "master" or something), then let's also save our status under that name.
     # We're probably overwriting a prior status, but that's OK.
     if wksp.runRequest.buildRef != wksp.version:
-      writeFile(stepStatusDir & "/" & wksp.runRequest.buildRef & ".json",
+      writeFile(stepStatusDir / wksp.runRequest.buildRef & ".json",
                 $wksp.status)
 
   wksp.outputHandler.sendStatusMsg(wksp.status)
 
 proc ensureProjectDirsExist(cfg: StrawBossConfig, p: ProjectDef): void =
   for subdir in ["configurations", "runs", "status", "artifacts"]:
-    let fullPath = cfg.buildDataDir & "/" & p.name & "/" & subdir
+    let fullPath = cfg.buildDataDir / p.name / subdir
     if not existsDir(fullPath):
       createDir(fullPath)
 
@@ -112,9 +112,9 @@ proc getProject*(cfg: StrawBossConfig, projectName: string): ProjectDef =
   ## Get a project definition by name from the service configuration
   let candidates = cfg.projects.filterIt(it.name == projectName)
   if candidates.len == 0:
-    raise newException(KeyError, "no project named " & projectName)
+    raise newException(NotFoundException, "no project named " & projectName)
   elif candidates.len > 1:
-    raise newException(KeyError, "multiple projects named " & projectName)
+    raise newException(NotFoundException, "multiple projects named " & projectName)
   else: result = candidates[0]
 
 proc setProject*(cfg: var StrawBossConfig, projectName: string, newDef: ProjectDef): void =
@@ -136,23 +136,67 @@ proc listVersions*(cfg: StrawBossConfig, projectName: string): seq[string] =
   ensureProjectDirsExist(cfg, project)
 
   let versionFiles = filesMatching(
-    cfg.buildDataDir & "/" & project.name & "/configurations/*.json")
+    cfg.buildDataDir / project.name / "configurations/*.json")
 
   result = versionFiles.map(proc(s: string): string =
     let slashIdx = s.rfind('/')
     result = s[(slashIdx + 1)..^6])
 
+proc getBuildStatus*(cfg: StrawBossConfig,
+                     projectName, stepName, buildRef: string): BuildStatus =
+
+  let project = cfg.getProject(projectName)
+
+  let statusFile = cfg.buildDataDir / project.name / "status" /
+    stepName / buildRef & ".json"
+
+  if not existsFile(statusFile):
+    raise newException(NotFoundException,
+      stepName & " has never been built for " & projectName & "@" & buildRef)
+
+  result = loadBuildStatus(statusFile)
+
+
+proc listArtifacts*(cfg: StrawBossConfig,
+                    projectName, stepName, version: string): seq[string] =
+  ## List the artifacts that have been built for a step.
+
+  let project = cfg.getProject(projectName)
+
+  ensureProjectDirsExist(cfg, project)
+
+  let buildStatus = cfg.getBuildStatus(projectName, stepName, version)
+
+  if buildStatus.state != BuildState.complete:
+    raise newException(NotFoundException, "step " & stepName &
+      " has never been successfully built for " & projectName & "@" & version)
+
+  result = filesMatching(
+      cfg.buildDataDir / project.name / "artifacts" / stepName / version / "*")
+    .mapIt(it.extractFilename)
+
+proc getArtifactPath*(cfg: StrawBossConfig,
+  projectName, stepName, version, artifactName: string): string =
+
+  let artifacts = cfg.listArtifacts(projectName, stepName, version)
+  if not artifacts.contains(artifactName):
+    raise newException(NotFoundException, "no artifact named " &
+      artifactName & " exists for step " & stepName & " in project " &
+      projectName & "@" & version)
+
+  result = cfg.buildDataDir / projectName / "artifacts" / stepName / version / artifactName
+
 proc existsRun*(cfg: StrawBossConfig, projectName, runId: string): bool =
-  existsFile(cfg.buildDataDir & "/" & projectName & "/runs/" & runId & ".request.json")
+  existsFile(cfg.buildDataDir / projectName / "runs" / runId & ".request.json")
 
 proc getRun*(cfg: StrawBossConfig, projectName, runId: string): Run =
   let project = cfg.getProject(projectName)
-  let runsPath = cfg.buildDataDir & "/" & project.name & "/runs"
+  let runsPath = cfg.buildDataDir / project.name / "runs"
 
   try: result = Run(
     id: parseUUID(runId),
-    request: loadRunRequest(runsPath & "/" & runId & ".request.json"),
-    status: loadBuildStatus(runsPath & "/" & runId & ".status.json"))
+    request: loadRunRequest(runsPath / runId & ".request.json"),
+    status: loadBuildStatus(runsPath / runId & ".status.json"))
   except: raiseEx "unable to load run information for id " & runId
 
 proc listRuns*(cfg: StrawBossConfig, projectName: string): seq[Run] =
@@ -160,29 +204,25 @@ proc listRuns*(cfg: StrawBossConfig, projectName: string): seq[Run] =
   let project = cfg.getProject(projectName)
   ensureProjectDirsExist(cfg, project)
 
-  let runsPath = cfg.buildDataDir & "/" & project.name & "/runs"
-  let reqPaths = filesMatching(runsPath & "/*.request.json")
+  let runsPath = cfg.buildDataDir / project.name / "runs"
+  let reqPaths = filesMatching(runsPath / "*.request.json")
 
   result = reqPaths.map(proc(reqPath: string): Run =
     let runId = reqPath[(runsPath.len + 1)..^14]
     result = Run(
       id: parseUUID(runId),
       request: loadRunRequest(reqPath),
-      status: loadBuildStatus(runsPath & "/" & runId & ".status.json")))
-
-proc getBuildStatus*(cfg: StrawBossConfig,
-                     projectName, stepName, buildRef: string): BuildStatus =
+      status: loadBuildStatus(runsPath / runId & ".status.json")))
 
+proc getLogs*(cfg: StrawBossConfig, projectname, runId: string): RunLogs =
   let project = cfg.getProject(projectName)
+  let runsPath = cfg.buildDataDir / project.name / "runs"
 
-  let statusFile = cfg.buildDataDir & "/" & project.name & "/status/" &
-    stepName & "/" & buildRef & ".json"
-
-  if not existsFile(statusFile):
-    raise newException(NotFoundException,
-      stepName & " has never been built for reference '" & buildRef)
-
-  result = loadBuildStatus(statusFile)
+  try: result = RunLogs(
+    runId: parseUUID(runId),
+    stdout: toSeq(lines(runsPath / runId & ".stdout.log")),
+    stderr: toSeq(lines(runsPath / runId & ".stderr.log")))
+  except: raiseEx "unable to load logs for run " & runId
 
 proc getProjectConfig*(cfg: StrawBossConfig,
                        projectName, version: string): ProjectConfig =
@@ -196,7 +236,7 @@ proc getProjectConfig*(cfg: StrawBossConfig,
   if version.isNilOrEmpty:
 
     let candidatePaths = filesMatching(
-      cfg.buildDataDir & "/" & project.name & "/configurations/*.json")
+      cfg.buildDataDir / project.name / "configurations/*.json")
 
     if candidatePaths.len == 0:
       raise newException(NotFoundException,
@@ -212,8 +252,7 @@ proc getProjectConfig*(cfg: StrawBossConfig,
   # If they did, let's try to load that
   else:
     confFilePath =
-      cfg.buildDataDir & "/" & project.name & "/configurations/" &
-      version & ".json"
+      cfg.buildDataDir / project.name / "configurations" / version & ".json"
 
   if not existsFile(confFilePath):
     raise newException(NotFoundException,
@@ -228,7 +267,7 @@ proc setupProject(wksp: Workspace) =
   wksp.sendMsg(lvlDebug, "Setting up project.")
 
   # Clone the project into the $temp directory
-  let cloneArgs = ["clone", wksp.projectDef.repo, wksp.dir]
+  let cloneArgs = @["clone", wksp.projectDef.repo, wksp.dir]
   wksp.sendMsg(lvlDebug, "git " & $cloneArgs)
 
   let cloneResult = exec("git", ".", cloneArgs, wksp.env, {poUsePath},
@@ -238,7 +277,7 @@ proc setupProject(wksp: Workspace) =
     raiseEx "unable to clone repo for '" & wksp.projectDef.name & "'"
 
   # Checkout the requested ref
-  let checkoutArgs = ["checkout", wksp.buildRef]
+  let checkoutArgs = @["checkout", wksp.buildRef]
   wksp.sendMsg(lvlDebug, "git " & $checkoutArgs)
 
   let checkoutResult = exec("git", wksp.dir, checkoutArgs,
@@ -249,7 +288,7 @@ proc setupProject(wksp: Workspace) =
             " for '" & wksp.projectDef.name & "'"
 
   # Find the strawboss project configuration
-  let projCfgFile = wksp.dir & "/" & wksp.projectDef.cfgFilePath
+  let projCfgFile = wksp.dir / wksp.projectDef.cfgFilePath
   wksp.sendMsg(lvlDebug, "Looking for project configuration at '" & projCfgFile & "'")
   if not existsFile(projCfgFile):
     raiseEx "Cannot find strawboss project configuration in the project " &
@@ -283,21 +322,20 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
 
   wksp.step = step
 
-  let artifactsDir = wksp.buildDataDir & "/artifacts/" &
-                     step.name & "/" & wksp.version
+  let artifactsDir = wksp.buildDataDir / "artifacts" / step.name / wksp.version
 
   if not existsDir(artifactsDir): createDir(artifactsDir)
 
   # Have we tried to build this before and are we caching the results?
-  let statusFilePath = wksp.buildDataDir & "/status/" & step.name &
-    "/" & wksp.version & ".json"
+  let statusFilePath = wksp.buildDataDir / "status" / step.name /
+    wksp.version & ".json"
 
   if existsFile(statusFilePath) and not step.dontSkip:
     let prevStatus = loadBuildStatus(statusFilePath)
 
     # If we succeeded last time, no need to rebuild
     if prevStatus.state == BuildState.complete:
-      wksp.publishStatus(BuildState.complete,
+      wksp.publishStatus(BuildState.stepComplete,
         "Skipping step '" & step.name & "' for version '" & wksp.version &
           "': already completed.")
       return wksp.status
@@ -328,7 +366,7 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
     # Run that step (may get skipped)
     let runStatus = doStep(core.newCopy(wksp), depStep)
 
-    if not (runStatus.state == BuildState.complete):
+    if not (runStatus.state == BuildState.stepComplete):
       raiseEx "dependent step failed: " & depStep.name
 
     wksp.sendMsg(lvlDebug, "dependent step '" & depStep.name &
@@ -336,8 +374,8 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
 
     # Add the artifacts directory for the dependent step to our env so that
     # further steps can reference it via $<stepname>_DIR
-    wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir & "/artifacts/" &
-      dep & "/" & wksp.version
+    wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir / "artifacts" /
+      dep / wksp.version
 
   # Run the step command, piping in cmdInput
   let stepCmd = wksp.resolveEnvVars(step.stepCmd)
@@ -345,7 +383,7 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
                 else: stepCmd
   wksp.sendMsg step.name & ": starting stepCmd: " & stepCmd
   let cmdProc = startProcess(stepCmd,
-    wksp.dir & "/" & step.workingDir, [], wksp.env, {poUsePath, poEvalCommand})
+    wksp.dir / step.workingDir, [], wksp.env, {poUsePath, poEvalCommand})
 
   let cmdInStream = inputStream(cmdProc)
 
@@ -367,16 +405,16 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
       let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
       try:
         wksp.sendMsg "copy " &
-          wksp.dir & "/" & step.workingDir & "/" & artifactPath & " -> " &
-          artifactsDir & "/" & artifactName
+          wksp.dir / step.workingDir / artifactPath & " -> " &
+          artifactsDir / artifactName
 
-        copyFileWithPermissions(wksp.dir & "/" & step.workingDir & "/" &
-          artifactPath, artifactsDir & "/" & artifactName)
+        copyFileWithPermissions(wksp.dir / step.workingDir / artifactPath,
+                                artifactsDir / artifactName)
       except:
         raiseEx "step " & step.name & " failed: unable to copy artifact " &
           artifactPath & ":\n" & getCurrentExceptionMsg()
 
-  wksp.publishStatus(BuildState.complete, "")
+  wksp.publishStatus(BuildState.stepComplete, "step " & step.name & " complete")
   result = wksp.status
 
 proc run*(cfg: StrawBossConfig, req: RunRequest,
@@ -401,8 +439,8 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
     ensureProjectDirsExist(cfg, projectDef)
 
     # Update our run status
-    let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs"
-    writeFile(runDir & "/" & $req.runId & ".status.json", $result)
+    let runDir = cfg.buildDataDir / projectDef.name / "runs"
+    writeFile(runDir / $req.runId & ".status.json", $result)
 
     # Read in the existing system environment
     var env = loadEnv()
@@ -413,13 +451,13 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
     if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
 
     # Setup our STDOUT and STDERR files
-    let stdoutFile = open(runDir & "/" & $req.runId & ".stdout.log", fmWrite)
-    let stderrFile = open(runDir & "/" & $req.runId & ".stderr.log", fmWrite)
+    let stdoutFile = open(runDir / $req.runId & ".stdout.log", fmWrite)
+    let stderrFile = open(runDir / $req.runId & ".stderr.log", fmWrite)
 
     let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile)
 
     wksp = Workspace(
-      buildDataDir: cfg.buildDataDir & "/" & projectDef.name,
+      buildDataDir: cfg.buildDataDir / projectDef.name,
       buildRef:
         if req.buildRef != nil and req.buildRef.len > 0: req.buildRef
         else: projectDef.defaultBranch,
@@ -452,8 +490,8 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
     # Update our cache of project configurations.
     # TODO: what happens if this fails?
     copyFileWithPermissions(
-      wksp.dir & "/" & wksp.projectDef.cfgFilePath,
-      wksp.buildDataDir & "/configurations/" & wksp.version & ".json")
+      wksp.dir / wksp.projectDef.cfgFilePath,
+      wksp.buildDataDir / "configurations" / wksp.version & ".json")
 
     # Find the requested step
     if not wksp.project.steps.hasKey(req.stepName):
@@ -462,7 +500,12 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
 
     if req.forceRebuild: step.dontSkip = true
 
-    result = doStep(wksp, step)
+    var buildStatus = doStep(wksp, step)
+    if buildStatus.state == BuildState.stepComplete:
+      buildStatus.state = BuildState.complete
+      wksp.publishStatus(buildStatus.state, "all steps complete")
+
+    result = wksp.status
 
   except:
     when not defined(release): echo getCurrentException().getStackTrace()
@@ -487,9 +530,9 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
 
   # Find the project definition (will throw appropriate exceptions)
   let projectDef = cfg.getProject(req.projectName)
-  let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs"
-  let reqFile = runDir & "/" & $req.runId & ".request.json"
-  let statusFile = runDir & "/" & $req.runId & ".status.json"
+  let runDir = cfg.buildDataDir / projectDef.name / "runs"
+  let reqFile = runDir / $req.runId & ".request.json"
+  let statusFile = runDir / $req.runId & ".status.json"
 
   try:
     # Make sure the build data directories for this project exist.
@@ -517,8 +560,8 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
 
   except:
     let exMsg = "run request rejected: " & getCurrentExceptionMsg()
-    raiseEx exMsg
     try:
       writeFile(statusFile,
         $(BuildStatus(runId: $req.runId, state: BuildState.rejected, details: exMsg)))
     except: discard ""
+    raiseEx exMsg

src/main/nim/strawbosspkg/server.nim

@@ -1,6 +1,11 @@
-import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging,
+import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, md5,
   os, osproc, sequtils, strutils, tempfile, times, unittest, uuids
 
+from mimetypes import getMimeType
+from asyncfile import openAsync, readToStream, close
+from asyncnet import send
+from re import re, find
+
 import ./configuration, ./core
 
 type
@@ -17,13 +22,22 @@ proc newSession*(user: UserRef): Session =
     issuedAt: getTime(),
     expires: daysForward(7).toTime())
 
-proc makeJsonResp(status: HttpCode, details: string = ""): string =
-  result = $(%* {
-    "statusCode": status.int,
-    "status": $status,
+proc buildJson(resp: Response, code: HttpCode, details: string = ""): void =
+  resp.data[0] = CallbackAction.TCActionSend
+  resp.data[1] = code
+  resp.data[2]["Content-Type"] = JSON
+  resp.data[3] = $(%* {
+    "statusCode": code.int,
+    "status": $code,
     "details": details
   })
 
+# Work-around for weirdness trying to use resp(Http500... in exception blocks
+proc build500Json(resp: Response, ex: ref Exception, msg: string): void =
+  when not defined(release): debug ex.getStackTrace()
+  error msg & ":\n" & ex.msg
+  resp.buildJson(Http500)
+
 proc toJWT*(cfg: StrawBossConfig, session: Session): string =
   ## Make a JST token for this session.
   var jwt = JWT(
@@ -46,7 +60,6 @@ proc fromJWT*(cfg: StrawBossConfig, strTok: string): Session =
   # Find the user record (if authenticated)
   let username = jwt.claims["sub"].node.str
   let users = cfg.users.filterIt(it.name == username)
-  debug "username: " & username & "\n\tusers: " & $users.mapIt(it.name) & "\n\tall users: " & cfg.users.mapIt(it.name)
   if users.len != 1: raiseEx "Could not find session user."
 
   result = Session(
@@ -131,7 +144,7 @@ template checkAuth() =
   except:
     debug "Auth failed: " & getCurrentExceptionMsg()
     response.data[2]["WWW-Authenticate"] = "Bearer"
-    resp(Http401, makeJsonResp(Http401), JSON)
+    response.buildJson(Http401)
 
 proc start*(cfg: StrawBossConfig): void =
 
@@ -144,18 +157,21 @@ proc start*(cfg: StrawBossConfig): void =
 
   routes:
 
+    get "/ping":
+      resp($(%"pong"), JSON)
+
     post "/auth-token":
       var uname, pwd: string
       try:
         let jsonBody = parseJson(request.body)
         uname = jsonBody["username"].getStr
         pwd = jsonBody["password"].getStr
-      except: resp(Http400, makeJsonResp(Http400), JSON)
+      except: response.buildJson(Http400); return true
 
       try:
         let authToken = makeAuthToken(cfg, uname, pwd)
         resp($(%authToken), JSON)
-      except: resp(Http401, makeJsonResp(Http401, getCurrentExceptionMsg()), JSON)
+      except: response.buildJson(Http401, getCurrentExceptionMsg()); return true
 
     get "/verify-auth":
       checkAuth(); if not authed: return true
@@ -175,7 +191,7 @@ proc start*(cfg: StrawBossConfig): void =
       checkAuth(); if not authed: return true
 
       # TODO
-      resp(Http501, makeJsonResp(Http501), JSON)
+      response.buildJson(Http501); return true
 
     get "/project/@projectName":
       ## Return a project's configuration, as well as it's versions.
@@ -185,7 +201,14 @@ proc start*(cfg: StrawBossConfig): void =
       # Make sure we know about that project
       var projDef: ProjectDef
       try: projDef = cfg.getProject(@"projectName")
-      except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
+      except:
+        try: raise getCurrentException()
+        except NotFoundException:
+          response.buildJson(Http404, getCurrentExceptionMsg())
+        except:
+          response.build500Json(getCurrentException(),
+            "unable to load project definition for project " & @"projectName")
+        return true
 
       var projConf: ProjectConfig
       try: projConf = getProjectConfig(cfg, @"projectName", "")
@@ -206,13 +229,13 @@ proc start*(cfg: StrawBossConfig): void =
 
       try: resp($(%listVersions(cfg, @"projectName")), JSON)
       except:
-        if getCurrentException() is KeyError:
-          resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
-        else:
-          when not defined(release): debug getCurrentException().getStackTrace()
-          error "unable to list versions for project " & @"projectName" &
-            ":\n" & getCurrentExceptionMsg()
-          resp(Http500, makeJsonResp(Http500, "internal server error"), JSON)
+        try: raise getCurrentException()
+        except NotFoundException:
+          response.buildJson(Http404, getCurrentExceptionMsg())
+        except:
+          response.build500Json(getCurrentException(),
+            "unable to list versions for project " & @"projectName")
+        return true
 
     get "/project/@projectName/version/@version?":
       ## Get a detailed project record including step definitions (ProjectConfig).
@@ -221,7 +244,7 @@ proc start*(cfg: StrawBossConfig): void =
 
       # Make sure we know about that project
       try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON)
-      except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
+      except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
 
     get "/project/@projectName/runs":
       ## List all runs
@@ -229,7 +252,7 @@ proc start*(cfg: StrawBossConfig): void =
       checkAuth(); if not authed: return true
 
       try: resp($(%listRuns(cfg, @"projectName")), JSON)
-      except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
+      except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
 
     get "/project/@projectName/runs/active":
       ## List all currently active runs
@@ -242,12 +265,13 @@ proc start*(cfg: StrawBossConfig): void =
           .mapIt(cfg.getRun(@"projecName", $it.runId));
         resp($(%activeRuns), JSON)
       except:
-        if getCurrentException() is KeyError:
-          resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
-        else:
-          when not defined(release): debug getCurrentException().getStackTrace()
-          error "problem loading active runs: " & getCurrentExceptionMsg()
-          resp(Http500, makeJsonResp(Http500, "internal server error"), JSON)
+        try: raise getCurrentException()
+        except NotFoundException:
+          response.buildJson(Http404, getCurrentExceptionMsg())
+        except:
+          response.build500Json(getCurrentException(),
+                                "problem loading active runs")
+        return true
 
     get "/project/@projectName/run/@runId":
       ## Details for a specific run
@@ -256,13 +280,115 @@ proc start*(cfg: StrawBossConfig): void =
 
       # Make sure we know about that project
       try: discard cfg.getProject(@"projectName")
-      except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
+      except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
 
       if not existsRun(cfg, @"projectName", @"runId"):
-        resp(Http404, makeJsonResp(Http404, "no such run for project"), JSON)
+        response.buildJson(Http404, "no such run for project"); return true
 
       try: resp($getRun(cfg, @"projectName", @"runId"), JSON)
-      except: resp(Http500, makeJsonResp(Http500, getCurrentExceptionMsg()), JSON)
+      except:
+        response.build500Json(getCurrentException(),
+          "unable to load run details for project " & @"projectName" &
+          " run " & @"runId")
+        return true
+
+    get "/project/@projectName/run/@runId/logs":
+      ## Get logs from a specific run
+
+      checkAuth(); if not authed: return true
+
+      try: discard cfg.getProject(@"projectName")
+      except:
+        response.buildJson(Http404, getCurrentExceptionMsg())
+        return true
+
+      if not existsRun(cfg, @"projectName", @"runId"):
+        response.buildJson(Http404, "no such run for project")
+        return true
+
+      try: resp($getLogs(cfg, @"projectName", @"runId"))
+      except:
+        response.build500Json(getCurrentException(),
+          "unable to load run logs for " & @"projectName" & " run " & @"runId")
+        return true
+
+    get "/project/@projectName/step/@stepName/artifacts/@version":
+      ## Get the list of artifacts that were built for
+
+      checkAuth(); if not authed: return true
+
+      debug "Matched artifacts list request: " & $(%*{
+        "project": @"projectName",
+        "step": @"stepName",
+        "version": @"version"
+      })
+
+      try: resp($(%listArtifacts(cfg, @"projectName", @"stepName", @"version")), JSON)
+      except:
+        try: raise getCurrentException()
+        except NotFoundException:
+          response.buildJson(Http404, getCurrentExceptionMsg())
+        except:
+          response.build500Json(getCurrentException(), "unable to list artifacts for " &
+            @"projectName" & ":" & @"stepName" & "@" & @"buildRef")
+        return true
+
+    get "/project/@projectName/step/@stepName/artifact/@version/@artifactName":
+      ## Get a specific artifact that was built.
+
+      checkAuth(); if not authed: return true
+
+      var artifactPath: string
+      try: artifactPath = getArtifactPath(cfg,
+          @"projectName", @"stepName", @"version", @"artifactName")
+      except:
+        try: raise getCurrentException()
+        except NotFoundException:
+          response.buildJson(Http404, getCurrentExceptionMsg())
+        except:
+          response.build500Json(getCurrentException(), "unable to check artifact path for " &
+            @"projectName" & ":" & @"stepName" & "@" & @"version")
+        return true
+
+      debug "Preparing: " & artifactPath
+      let fileSize = getFileSize(artifactPath)
+      let mimetype = request.settings.mimes.getMimetype(artifactPath.splitFile.ext[1 .. ^1])
+      if fileSize < 10_000_000: # 10 mb
+        var file = readFile(artifactPath)
+
+        var hashed = getMD5(file)
+
+        # If the user has a cached version of this file and it matches our
+        # version, let them use it
+        if request.headers.hasKey("If-None-Match") and request.headers["If-None-Match"] == hashed:
+          resp(Http304, [], "")
+        else:
+          resp(Http200, [
+            ("Content-Disposition", "; filename=\"" & @"artifactName" & "\""),
+            ("Content-Type", mimetype),
+            ("ETag", hashed )], file)
+      else:
+        let headers = {
+          "Content-Disposition": "; filename=\"" & @"artifactName" & "\"",
+          "Content-Type": mimetype,
+          "Content-Length": $fileSize
+        }.newStringTable
+        await response.sendHeaders(Http200, headers)
+
+        var fileStream = newFutureStream[string]("sendStaticIfExists")
+        var file = openAsync(artifactPath, fmRead)
+        # Let `readToStream` write file data into fileStream in the
+        # background.
+        asyncCheck file.readToStream(fileStream)
+        # The `writeFromStream` proc will complete once all the data in the
+        # `bodyStream` has been written to the file.
+        while true:
+          let (hasValue, value) = await fileStream.read()
+          if hasValue:
+            await response.client.send(value)
+          else:
+            break
+        file.close()
 
     get "/project/@projectName/step/@stepName/status/@buildRef":
       ## Get detailed information about the status of a step (assuming it has been built)
@@ -270,7 +396,19 @@ proc start*(cfg: StrawBossConfig): void =
       checkAuth(); if not authed: return true
 
       try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON)
-      except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
+      except:
+        try: raise getCurrentException()
+        except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg())
+        except:
+          response.build500Json(getCurrentException(), "unable to load the build state for " &
+            @"projectName" & ":" & @"stepName" & "@" & @"buildRef")
+        return true
+
+    #get "/project/@projectName/step/@stepName/status/@buildRef.svg":
+      ## Get an image representing the status of a build
+
+      ## TODO: how do we want to handle auth for this? Unlike
+      #checkAuth(): if not authed: return true
 
     post "/project/@projectName/step/@stepName/run/@buildRef?":
       # Kick off a run
@@ -296,26 +434,26 @@ proc start*(cfg: StrawBossConfig): void =
           id: runRequest.runId,
           request: runRequest,
           status: status), JSON)
-      except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
-
-    get "/service/debug/ping":
-      if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON)
-      else: resp($(%"pong"), JSON)
+      except:
+        try: raise getCurrentException()
+        except NotFoundException:
+          response.buildJson(Http404, getCurrentExceptionMsg())
+        except: response.buildJson(Http400, getCurrentExceptionMsg())
+        return true
 
     post "/service/debug/stop":
-      if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON)
+      if not cfg.debug: response.buildJson(Http404); return true
       else:
         let shutdownFut = sleepAsync(100)
         shutdownFut.callback = proc(): void = complete(stopFuture)
         resp($(%"shutting down"), JSON)
 
-    #[
+
     get re".*":
-      resp(Http404, makeJsonResp(Http404), JSON)
+      response.buildJson(Http404); return true
 
     post re".*":
-      resp(Http404, makeJsonResp(Http404), JSON)
-    ]#
+      response.buildJson(Http404); return true
 
   proc performMaintenance(cfg: StrawBossConfig): void =
     # Prune workers

src/main/systemd/strawboss.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=StrawBoss build server.
+
+[Service]
+Type=simple
+User=strawboss
+WorkingDirectory=/home/strawboss
+ExecStart=/home/strawboss/strawboss
+Restart=on-failure

src/test/nim/functional/tserver.nim

@@ -135,12 +135,12 @@ suite "strawboss server":
     let completedRun = http.waitForBuild(apiBase, testProjName, $queuedRun.id)
 
     # there should be successful status files for both the build and test steps
-    for stepName in ["build", "test"]:
-      let statusFile = tempBuildDataDir & "/" & testProjName & "/status/" & stepName & "/0.2.1.json"
+    for step in [("build", BuildState.stepComplete), ("test", BuildState.complete)]:
+      let statusFile = tempBuildDataDir & "/" & testProjName & "/status/" & step[0] & "/0.2.1.json"
       check fileExists(statusFile)
 
       let status = loadBuildStatus(statusFile)
-      check status.state == BuildState.complete
+      check status.state == step[1]
 
   #test "already completed steps should not be rebuilt":
   #  let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")

src/test/nim/unit/tserver.nim

@@ -41,7 +41,7 @@ suite "strawboss server":
     check fromJWT(cfg, tok) == session
 
   test "ping":
-    let resp = http.get(apiBase & "/service/debug/ping")
+    let resp = http.get(apiBase & "/ping")
     check:
       resp.status.startsWith("200")
       resp.body == "\"pong\""

src/util/bash/client.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+host="${STRAWBOSS_HOST:-localhost:8180}"
+if [ $# -eq 1 ]; then
+  url="$1"
+  method="GET"
+  data=""
+elif [ $# -eq 2 ]; then
+  method="$1"
+  url="$2"
+  data=""
+else 
+  method="$1"
+  url="$2"
+  data="$3"
+fi
+
+curl -X "$method" -H "Authorization: Bearer $(cat token.txt)" "http://${host}/api/$url" -d "$data"
+echo ""
+#echo "curl -X \"$method\" -H \"Authorization: Bearer $(cat token.txt)\" \"localhost:8180/api/$url\" | jq . "

strawboss.nimble

@@ -1,7 +1,7 @@
 # Package
 
 bin           = @["strawboss"]
-version       = "0.3.1"
+version       = "0.4.0"
 author        = "Jonathan Bernard"
 description   = "My personal continious integration worker."
 license       = "MIT"
@@ -12,12 +12,18 @@ srcDir        = "src/main/nim"
 requires @["nim >= 0.16.1", "docopt >= 0.6.5", "isaac >= 0.1.2", "tempfile", "jester", "bcrypt",
            "untar", "uuids"]
 
-requires "https://github.com/yglukhov/nim-jwt"
+# Hacky to point to a specific hash. But there is some bug building in the
+# docker image we use to build the project with the next version. It adds an
+# ifdef branch to support libssl 1.1 but for some reason that ifdef is set
+# wrong and it tries to build against the 1.1 API even though the image only
+# has the 1.0 API. I'm crossing my fingers and hoping that our base image
+# supports libssl 1.1 before I need to update this library.
+requires "https://github.com/yglukhov/nim-jwt#549aa1eb13b8ddc0c6861d15cc2cc5b52bcbef01"
+
 requires "https://git.jdb-labs.com/jdb/nim-lang-utils.git >= 0.3.0"
 requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git >= 0.3.1"
 
 # Tasks
-#
 task functest, "Runs the functional test suite.":
   exec "nimble build"
   exec "nim c -r src/test/nim/run_functional_tests.nim"

strawboss.projectdef.json

@@ -1,12 +1,36 @@
 {
   "name": "strawboss",
   "steps": {
-    "build": {
+    "compile": {
       "artifacts": ["strawboss"],
-      "stepCmd": "nimble build"
+      "stepCmd": "docker run -v `pwd`:/usr/src/strawboss -w /usr/src/strawboss jdbernard/nim:0.17.2 nimble build"
     },
-    "test": { "depends": ["unittest", "functest"] },
-    "functest": { "stepCmd": "nimble functest" },
-    "unittest": { "stepCmd": "nimble unittest" }
+    "unittest": {
+      "depends": ["compile"],
+      "stepCmd": "docker run -v `pwd`:/usr/src/strawboss -v $compile_DIR:/usr/build/strawboss -w /usr/src/strawboss -i jdbernard/nim:0.17.2 /bin/bash",
+      "cmdInput": [
+        "cp /usr/build/strawboss/strawboss .",
+        "nimble install --depsOnly",
+        "nim c -r src/test/nim/run_unit_tests"
+      ]
+    },
+    "functest": {
+      "depends": ["compile"],
+      "stepCmd": "docker run -v `pwd`:/usr/src/strawboss -v $compile_DIR:/usr/build/strawboss -w /usr/src/strawboss -i jdbernard/nim:0.17.2 /bin/bash",
+      "cmdInput": [
+        "cp /usr/build/strawboss/strawboss .",
+        "nimble install --depsOnly",
+        "nim c -r src/test/nim/run_functional_tests"
+      ]
+    },
+    "build": {
+      "artifacts": ["strawboss-$VERSION.zip"],
+      "depends": ["compile", "unittest", "functest"],
+      "stepCmd": "docker run -v `pwd`:/usr/src/strawboss -v $compile_DIR:/usr/build/strawboss -w /usr/src/strawboss -i jdbernard/nim:0.17.2 /bin/bash",
+      "cmdInput": [
+        "cp /usr/build/strawboss/strawboss .",
+        "zip strawboss-$VERSION.zip strawboss strawboss.config.json example.json src/main/systemd/strawboss.service"
+      ]
+    }
   }
 }