Added more functional tests, fix bugs discovered.

* Fixed the formatting of command line logging of strawboss workers.
* Fixed a bug in the (de)serialization of log levels in the strawboss service
  config file.
* Pulled `parseBuildStatus` logic out of `loadBuildStatus` so that we could
  parse a JSON that didn't come from a file.
* Added `parseRun` for Run objects.
* Moved `/ping` to `/service/debug/ping` for symmetry with
  `/service/debug/stop`
* Added functional tests of full builds.

src/main/nim/strawboss.nim

@@ -7,7 +7,7 @@ import strawbosspkg/server
 let SB_VER = "0.2.0"
 
 proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) =
-  let prefix = if cmd != nil: cmd else: ""
+  let prefix = if cmd != nil: cmd & ": " else: ""
   if outMsg != nil: stdout.writeLine prefix & outMsg
   if errMsg != nil: stderr.writeLine prefix & errMsg
 
@@ -16,7 +16,7 @@ when isMainModule:
   let doc = """
 Usage:
   strawboss serve [options]
-  strawboss run <requestFile>
+  strawboss run <requestFile> [options]
   strawboss hashpwd <pwd>
   
 Options

src/main/nim/strawbosspkg/configuration.nim

@@ -143,7 +143,7 @@ proc parseStrawBossConfig*(jsonCfg: JsonNode): StrawBossConfig =
     pwdCost: int8(jsonCfg.getOrFail("pwdCost", "strawboss config").getNum),
     projects: jsonCfg.getIfExists("projects").getElems.mapIt(parseProjectDef(it)),
     maintenancePeriod: int(jsonCfg.getIfExists("maintenancePeriod").getNum(10000)),
-    logLevel: parseLogLevel(jsonCfg.getIfExists("logLevel").getStr("lvlInfo")),
+    logLevel: parseLogLevel(jsonCfg.getIfExists("logLevel").getStr("info")),
     users: users)
 
 
@@ -193,14 +193,17 @@ proc loadProjectConfig*(cfgFile: string): ProjectConfig =
     versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"),
     steps: steps)
 
+proc parseBuildStatus*(statusJson: JsonNode): BuildStatus =
+  result = BuildStatus(
+    runId: statusJson.getOrFail("runId", "run ID").getStr,
+    state: parseEnum[BuildState](statusJson.getOrFail("state", "build status").getStr),
+    details: statusJson.getIfExists("details").getStr("") )
+
 proc loadBuildStatus*(statusFile: string): BuildStatus =
   if not existsFile(statusFile): raiseEx "status file not found: " & statusFile
   let jsonObj = parseFile(statusFile)
 
-  result = BuildStatus(
+  result = parseBuildStatus(jsonObj)
-    runId: jsonObj.getOrFail("runId", "run ID").getStr,
-    state: parseEnum[BuildState](jsonObj.getOrFail("state", "build status").getStr),
-    details: jsonObj.getIfExists("details").getStr("") )
 
 proc parseRunRequest*(reqJson: JsonNode): RunRequest =
   result = RunRequest(
@@ -218,6 +221,12 @@ proc loadRunRequest*(reqFilePath: string): RunRequest =
 
   parseRunRequest(parseFile(reqFilePath))
 
+proc parseRun*(runJson: JsonNode): Run =
+  result = Run(
+    id: parseUUID(runJson.getOrFail("id", "Run").getStr),
+    request: parseRunRequest(runJson.getOrFail("request", "Run")),
+    status: parseBuildStatus(runJson.getOrFail("status", "Run")))
+
 # TODO: can we use the marshal module for this?
 proc `%`*(s: BuildStatus): JsonNode =
   result = %* {
@@ -278,7 +287,7 @@ proc `%`*(cfg: StrawBossConfig): JsonNode =
     "projects": %cfg.projects,
     "pwdCost": cfg.pwdCost,
     "maintenancePeriod": cfg.maintenancePeriod,
-    "logLevel": cfg.logLevel,
+    "logLevel": toLower(($cfg.logLevel)[3]) & ($cfg.logLevel)[4..^1],
     "users": %cfg.users }
 
 proc `%`*(run: Run): JsonNode =

src/main/nim/strawbosspkg/core.nim

@@ -505,7 +505,7 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
       details: "request queued for execution")
     writeFile(statusFile, $queuedStatus)
 
-    var args = @["run", reqFile]
+    var args = @["run", reqFile, "-c", cfg.filePath]
     debug "Launching worker: " & cfg.pathToExe & " " & args.join(" ")
 
     let worker = Worker(

src/main/nim/strawbosspkg/server.nim

@@ -119,8 +119,6 @@ proc start*(cfg: StrawBossConfig): void =
 
   routes:
 
-    get "/ping": resp($(%"pong"), JSON)
-
     post "/auth-token":
       var uname, pwd: string
       try:
@@ -241,14 +239,6 @@ proc start*(cfg: StrawBossConfig): void =
       try: resp($getRun(cfg, @"projectName", @"runId"), JSON)
       except: resp(Http500, makeJsonResp(Http500, getCurrentExceptionMsg()), JSON)
 
-    get "/project/@projectName/step/@stepName":
-      ## Get step details including runs.
-
-      checkAuth(); if not authed: return true
-
-      # TODO
-      resp(Http501, makeJsonResp(Http501), JSON)
-
     get "/project/@projectName/step/@stepName/status/@buildRef":
       ## Get detailed information about the status of a step (assuming it has been built)
 
@@ -283,6 +273,10 @@ proc start*(cfg: StrawBossConfig): void =
           status: status), JSON)
       except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
 
+    get "/service/debug/ping":
+      if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON)
+      else: resp($(%"pong"), JSON)
+
     post "/service/debug/stop":
       if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON)
       else:
@@ -310,7 +304,6 @@ proc start*(cfg: StrawBossConfig): void =
 
 
   info "StrawBoss is bossing people around."
-  #debug "configuration:\n\n" & $cfg & "\n\n"
 
   callSoon(proc(): void = performMaintenance(cfg))
   waitFor(stopFuture)

src/test/nim/functional/tserver.nim

@@ -1,10 +1,11 @@
 import cliutils, httpclient, json, os, osproc, sequtils, strutils, tempfile,
-  times, unittest, untar
+  times, unittest, untar, uuids
 
 from langutils import sameContents
 
 import ../testutil
 import ../../../main/nim/strawbosspkg/configuration
+import ../../../main/nim/strawbosspkg/core
 
 let apiBase = "http://localhost:8180/api"
 let cfgFilePath = "src/test/json/strawboss.config.json"
@@ -39,7 +40,7 @@ suite "strawboss server":
     newCfg.buildDataDir = tempBuildDataDir
 
     # update the repo string for the extracted test project
-    var testProjDef = newCfg.findProject(testProjName)
+    var testProjDef = newCfg.getProject(testProjName)
     testProjDef.repo = testProjTempDir
     newCfg.setProject(testProjName, testProjDef)
 
@@ -90,45 +91,64 @@ suite "strawboss server":
 
   test "run a successful build with artifacts":
     let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
-    let resp = http.post(apiBase & "/project/" & testProjName & "/step/build/run/0.1.0")
+    let resp = http.post(apiBase & "/project/" & testProjName & "/step/build/run/0.2.1")
     check resp.status.startsWith("200")
 
-    # give the filesystem time to create stuff
+    # Check that the run was queued
-    sleep(100)
+    let queuedRun = parseRun(parseJson(resp.body))
+    check queuedRun.status.state == BuildState.queued
 
-    # check that the run request has been saved
+    # Wait for the build to complete
+    let completedRun = http.waitForBuild(apiBase, testProjname, $queuedRun.id)
+
+    # check that the run directory, run request, status, and output logs exist
     let runsDir = tempBuildDataDir & "/" & testProjName & "/runs"
+    let runId = $completedRun.id
     check existsDir(runsDir)
+    for suffix in [".request.json", ".status.json", ".stdout.log", ".stderr.log"]:
+      check existsFile(runsDir & "/" & runId & suffix)
 
-    let reqFile = runsDir
     # check that the project directory has been created in the artifacts repo
-    let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build/0.1.0"
+    let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build/0.2.1"
     check existsDir(runArtifactsDir)
 
-    # check that the run status file has been created in the artifacts repo
+    # check that the build step status file has been created
-    let statusFile = tempBuildDataDir & "/" & testProjName & "/status/0.1.0.json"
+    let statusFile = tempBuildDataDir & "/" & testProjName & "/status/build/0.2.1.json"
     check fileExists(statusFile)
 
-    # check that the run status is not failed
+    # check that the status is complete
     var status = loadBuildStatus(statusFile)
-    check status.state != "failed"
+    check status.state == BuildState.complete
-
-    # wait for the build to complete
-    let expTime = getTime() + TIMEOUT
-    while getTime() < expTime and not contains(["complete", "failed"], status.state):
-      sleep(1000)
-      status = loadBuildStatus(statusFile)
-
-    # check that the status is "complete"
-    check status.state == "complete"
 
     # check that the artifacts we expect are present
     let binFile = runArtifactsDir & "/test_project"
     check existsFile(binFile)
 
-  # TODO
+  test "run a multi-step build":
-  test "run a time-consuming build and check the status via the API":
+    let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
-    check false
+
+    # Run the "test" step (depends on "build")
+    var resp = http.post(apiBase & "/project/" & testProjname & "/step/test/run/0.2.1")
+    check resp.status.startsWith("200")
+
+    let queuedRun = parseRun(parseJson(resp.body))
+    let completedRun = http.waitForBuild(apiBase, testProjName, $queuedRun.id)
+
+    # there should be successful status files for both the build and test steps
+    for stepName in ["build", "test"]:
+      let statusFile = tempBuildDataDir & "/" & testProjName & "/status/" & stepName & "/0.2.1.json"
+      check fileExists(statusFile)
+
+      let status = loadBuildStatus(statusFile)
+      check status.state == BuildState.complete
+
+  #test "already completed steps should not be rebuilt":
+  #  let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
+  #  let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build/0.2.1"
+  #  let exeModTime = getLastModificationTime(runArtifactsDir & "/test_project")
+
+    # Run the "build" step
+    # Kick off a build that depends on "build" (which was run in the last test)
 
   # TODO
   #test "kick off multiple runs and check the list of active runs via the API":

src/test/nim/testutil.nim

@@ -1,4 +1,7 @@
-import httpclient, json, strutils
+import httpclient, json, os, strutils, times
+
+import ../../main/nim/strawbosspkg/core
+import ../../main/nim/strawbosspkg/configuration
 
 proc newAuthenticatedHttpClient*(apiBase, uname, pwd: string): HttpClient =
   result = newHttpClient()
@@ -6,4 +9,38 @@ proc newAuthenticatedHttpClient*(apiBase, uname, pwd: string): HttpClient =
   assert authResp.status.startsWith("200")
   result.headers = newHttpHeaders({"Authorization": "Bearer " & parseJson(authResp.body).getStr})
 
+proc waitForBuild*(client: HttpClient, apiBase, projectName, runId: string,
+                         expectedState = BuildState.complete,
+                         failedState = BuildState.failed,
+                         timeout = 10): Run =
 
+  let startTime = epochTime()
+  var run: Run
+
+  #echo "Waiting for '" & $expectedState & "' from run:\n\t" &
+  #  apiBase & "/project/" & projectName & "/run/" & runId
+
+  while true:
+    var curElapsed = epochTime() - startTime
+
+    #echo "Checking (" & $curElapsed & " has passed)."
+
+    if curElapsed > toFloat(timeout):
+      raise newException(SystemError, "Timeout exceeded waiting for build.")
+
+    let resp = client.get(apiBase & "/project/" & projectName & "/run/" & runId)
+
+    #echo "Received resp:\n\n" & $resp.status & "\n\n" & $resp.body
+
+    if not resp.status.startsWith("200"):
+      raise newException(IOError, "Unable to retrieve status. Received response: " & resp.body)
+
+    run = parseRun(parseJson(resp.body))
+
+    if run.status.state == failedState:
+      raise newException(IOError, "Run transitioned to failed state '" & $failedState & "'")
+
+    if run.status.state == expectedState:
+      return run
+
+    sleep(200)

src/test/nim/unit/tserver.nim

@@ -41,7 +41,7 @@ suite "strawboss server":
     check fromJWT(cfg, tok) == session
 
   test "ping":
-    let resp = http.get(apiBase & "/ping")
+    let resp = http.get(apiBase & "/service/debug/ping")
     check:
       resp.status.startsWith("200")
       resp.body == "\"pong\""

strawboss.nimble

@@ -1,7 +1,7 @@
 # Package
 
 bin           = @["strawboss"]
-version       = "0.2.0"
+version       = "0.3.0"
 author        = "Jonathan Bernard"
 description   = "My personal continious integration worker."
 license       = "MIT"
@@ -13,7 +13,7 @@ requires @["nim >= 0.16.1", "docopt >= 0.6.5", "isaac >= 0.1.2", "tempfile", "je
            "untar", "uuids"]
 
 requires "https://github.com/yglukhov/nim-jwt"
-requires "https://git.jdb-labs.com/jdb/nim-lang-utils.git"
+requires "https://git.jdb-labs.com/jdb/nim-lang-utils.git >= 0.3.0"
 requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git >= 0.3.1"
 
 # Tasks
@@ -25,3 +25,15 @@ task functest, "Runs the functional test suite.":
 task unittest, "Runs the unit test suite.":
   exec "nimble build"
   exec "nim c -r src/test/nim/run_unit_tests.nim"
+
+task test, "Runs both the unit and functional test suites.":
+  exec "nimble build"
+  echo "Building test suites..."
+  exec "nim c src/test/nim/run_unit_tests.nim"
+  exec "nim c src/test/nim/run_functional_tests.nim"
+  echo "\nRunning unit tests."
+  echo "-------------------"
+  exec "src/test/nim/run_unit_tests"
+  echo "\nRunning functional tests."
+  echo "-------------------------"
+  exec "src/test/nim/run_functional_tests"