Split testing into unit and functional tests.

* Split the `test` nimble task into `unittest` and `functest`, with
  corresponding test directories and test runners.
* Added documentation in README regarding building and testing StrawBoss.
* Created a small, simple test project for use in the functional tests.
* Added a `keepEnv` template in the server unit test code to make it easy to
  preserve the working environment for a single unit test to invistigate
  failures manually.

.gitignore

@@ -2,3 +2,4 @@
 nimcache/
 /strawboss
 src/test/nim/runtests
+src/test/nim/run_*_tests

README.md

@@ -219,3 +219,43 @@ using the handler to update the supervisor's knowledge of the build results and
 
 When launched in single-build mode there is no supervisory process. The main
 process directly executes the requested build steps.
+
+### Building StrawBoss
+
+To build StrawBoss locally, checkout the repository and in the repo root run:
+
+    nimble build
+
+### Testing
+
+StrawBoss has two test suites, a set of unit tests and a set of functional
+tests. All the test code and assets live under the `src/test` subdirectory.
+
+Each test suite has a runner file that serves as an entry for the test process,
+named `run_unit_tests.nim` and `run_functional_tests.nim`.
+
+#### Unit Tests
+
+The unit test soruce files live in the `nim/unit` subdirectory and have a
+one-to-one correspondence with the StrawBoss source files following this
+naming convention: `t<module>.nim`. The unit tests are intended to be run any
+time the code is recompiled.
+
+To run the unit tests, use the `unittest` nimble task:
+
+    nimble unittest
+
+#### Functional Tests
+
+The functional test source files live in the `nim/functional` subdirectory.
+There is a test project that is used to excercise StrawBoss functionality. To
+avoid external coupling it is stored within the StrawBoss repository as a test
+asset. To avoid `git` complications it is stored as a Gzipped TAR file and
+unpacked to a temporary directory as part of the functional test process.
+
+As the functional tests are more time-consuming and intensive, they are
+expected to bu run when performing a build.
+
+To run the functional tests, use the `functest` nimble task:
+
+    nimble functest

src/main/nim/strawbosspkg/configuration.nim

@@ -74,8 +74,17 @@ proc findProject*(cfg: StrawBossConfig, projectName: string): ProjectDef =
     raise newException(KeyError, "multiple projects named " & projectName)
   else: result = candidates[0]
 
-# internal utils
+proc setProject*(cfg: var StrawBossConfig, projectName: string, newDef: ProjectDef): void =
+  var found = false
+  for idx in 0..<cfg.projects.len:
+    if cfg.projects[idx].name == projectName:
+      cfg.projects[idx] = newDef
+      found = true
+      break
 
+  if not found: cfg.projects.add(newDef)
+
+# internal utils
 let nullNode = newJNull()
 proc getIfExists(n: JsonNode, key: string): JsonNode =
   # convenience method to get a key from a JObject or return null

src/main/nim/strawbosspkg/server.nim

@@ -1,4 +1,4 @@
-import algorithm, asyncdispatch, bcrypt, jester, json, jwt, os, osproc,
+import algorithm, asyncdispatch, bcrypt, jester, json, jwt, logging, os, osproc,
   sequtils, strutils, tempfile, times, unittest
 
 import ./configuration, ./core, private/util

src/test/json/dummy-project.config.json

@@ -1,5 +1,5 @@
 {
-  "name": "test-project-1",
+  "name": "dummy-project",
   "versionCmd": "git describe --all --always",
   "steps": {
     "build": {

src/test/json/strawboss.config.json

@@ -8,12 +8,12 @@
   ],
   "pwdCost": 11,
   "projects": [
-    { "name": "test-project-1",
+    { "name": "dummy-project",
       "repo": "/non-existent/dir",
       "cfgFilePath": "strawhat.json",
       "defaultBranch": "deploy",
       "envVars": { "VAR1": "value" }
     },
-    { "name": "test-strawboss",
+    { "name": "test-project",
-      "repo": "https://git.jdb-labs.com:jdb/test-strawboss.git" } ]
+      "repo": "" } ]
 }

src/test/nim/functional/tserver.nim

@@ -0,0 +1,100 @@
+import httpclient, json, os, osproc, sequtils, strutils, tempfile, unittest, untar
+
+from langutils import sameContents
+
+import ../testutil
+import ../../../main/nim/strawbosspkg/configuration
+import ../../../main/nim/strawbosspkg/private/util
+
+let apiBase = "http://localhost:8180/api"
+let cfgFilePath = "src/test/json/strawboss.config.json"
+let cfg = loadStrawBossConfig(cfgFilePath)
+
+# Util template intended for use to manually review test case.
+# Inserting into a test case will prevent the test case from cleaning up it's
+# working files and echo the command to start StrawBoss using that test's
+# configuration and working files.
+template keepEnv(): untyped =
+  preserveEnv = true
+  echo "artifacts dir: " & tempArtifactsDir
+  echo "strawboss serve -c " & tempCfgPath
+
+suite "strawboss server":
+
+  # Suite setup: extract test project
+  let testProjTempDir = mkdtemp()
+  let testProjTarFile = newTarFile("src/test/test-project.tar.gz")
+  let testProjName = "test-project"
+  testProjTarFile.extract(testProjTempDir)
+
+  # per-test setup: spin up a fresh strawboss instance
+  setup:
+    let tempArtifactsDir = mkdtemp()
+    let (_, tempCfgPath) = mkstemp()
+    var preserveEnv = false
+
+    # copy our test config
+    var newCfg = cfg
+    newCfg.artifactsRepo = tempArtifactsDir
+
+    # update the repo string for the extracted test project
+    var testProjDef = newCfg.findProject(testProjName)
+    testProjDef.repo = testProjTempDir & "/" & testProjName
+    newCfg.setProject(testProjName, testProjDef)
+
+    # save the updated config and start the strawboss instance using it
+    writeFile(tempCfgPath, $newCfg)
+    let serverProcess = startProcess("./strawboss", ".",
+      @["serve", "-c", tempCfgPath], loadEnv(), {poUsePath})
+
+    # give the server time to spin up
+    sleep(100)
+
+  teardown:
+    discard newAsyncHttpClient().post(apiBase & "/service/debug/stop")
+
+    if not preserveEnv:
+      removeDir(tempArtifactsDir)
+      removeFile(tempCfgPath)
+
+    # give the server time to spin down but kill it after that
+    sleep(100)
+    if serverProcess.running: kill(serverProcess)
+
+  test "handle missing project configuration":
+    let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
+    let resp = http.get(apiBase & "/projects/" & cfg.projects[0].name)
+    check resp.status.startsWith("404")
+
+  test "gives 404 when no versions built":
+    let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
+    let resp = http.get(apiBase & "/projects/" & testProjName & "/versions")
+    check resp.status.startsWith("404")
+
+  test "GET /api/project/@projectName/versions":
+    let projArtifactsDir = tempArtifactsDir & "/" & testProjName
+    let expectedVersions = @["alpha", "beta", "1.0.0", "1.0.1"]
+
+    # Touch configuration files
+    createDir(projArtifactsDir)
+    for v in expectedVersions:
+      var f: File
+      check open(f, projArtifactsDir & "/configuration." & v & ".json", fmWrite)
+      close(f)
+
+    let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
+    let resp = http.get(apiBase & "/project/" & testProjName & "/versions")
+    let returnedVersions = parseJson(resp.body).getElems.mapIt(it.getStr)
+    check sameContents(expectedVersions, returnedVersions)
+
+  #test "enqueue a build":
+  #  let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
+  #  let resp = http.get(apiBase & "/project/" & testProjName & "
+
+  # Last-chance catch to kill the server in case some test err'ed and didn't
+  # reach it's teardown handler
+  discard newAsyncHttpClient().post(apiBase & "/service/debug/stop")
+
+  # Also, delete the extracted test project "source" repo
+  removeDir(testProjTempDir)
+

src/test/nim/run_functional_tests.nim

@@ -0,0 +1,3 @@
+import unittest
+
+import ./functional/tserver.nim

src/test/nim/run_unit_tests.nim

@@ -0,0 +1,4 @@
+import unittest
+
+import ./unit/tserver.nim
+import ./unit/tconfiguration.nim

src/test/nim/runtests.nim

@@ -1,4 +0,0 @@
-import unittest
-
-import ./tserver.nim
-import ./tconfiguration.nim

src/test/nim/testutil.nim

@@ -0,0 +1,9 @@
+import httpclient, json, strutils
+
+proc newAuthenticatedHttpClient*(apiBase, uname, pwd: string): HttpClient =
+  result = newHttpClient()
+  let authResp = result.post(apiBase & "/auth-token", $(%*{"username": uname, "password": pwd}))
+  assert authResp.status.startsWith("200")
+  result.headers = newHttpHeaders({"Authorization": "Bearer " & parseJson(authResp.body).getStr})
+
+

src/test/nim/unit/tconfiguration.nim

@@ -1,6 +1,6 @@
 import json, strtabs, tables, unittest
 from langutils import sameContents
-import ../../main/nim/strawbosspkg/configuration
+import ../../../main/nim/strawbosspkg/configuration
 
 suite "load and save configuration objects":
 

src/test/nim/unit/tserver.nim

@@ -1,18 +1,12 @@
 import asyncdispatch, httpclient, json, os, osproc, sequtils, strutils,
-  tempfile, times, unittest
+  times, unittest
 
 from langutils import sameContents
 
-import ../../main/nim/strawbosspkg/configuration
+import ../testutil
-import ../../main/nim/strawbosspkg/server
+import ../../../main/nim/strawbosspkg/configuration
-import ../../main/nim/strawbosspkg/private/util
+import ../../../main/nim/strawbosspkg/server
-
+import ../../../main/nim/strawbosspkg/private/util
-# test helpers
-proc newAuthenticatedHttpClient(apiBase, uname, pwd: string): HttpClient =
-  result = newHttpClient()
-  let authResp = result.post(apiBase & "/auth-token", $(%*{"username": uname, "password": pwd}))
-  assert authResp.status.startsWith("200")
-  result.headers = newHttpHeaders({"Authorization": "Bearer " & parseJson(authResp.body).getStr})
 
 let apiBase = "http://localhost:8180/api"
 let cfgFilePath = "src/test/json/strawboss.config.json"
@@ -98,56 +92,3 @@ suite "strawboss server":
   sleep(100)
   if serverProcess.running: kill(serverProcess)
 
-suite "strawboss server continued":
-
-  setup:
-    let tmpArtifactsDir = mkdtemp()
-    let (_, tmpCfgPath) = mkstemp()
-    var newCfg = cfg
-    newCfg.artifactsRepo = tmpArtifactsDir
-    writeFile(tmpCfgPath, $newCfg)
-    let serverProcess = startProcess("./strawboss", ".",
-      @["serve", "-c", tmpCfgPath], loadEnv(), {poUsePath})
-
-    # give the server time to spin up
-    sleep(100)
-
-  teardown:
-    discard newAsyncHttpClient().post(apiBase & "/service/debug/stop")
-    removeDir(tmpArtifactsDir)
-    removeFile(tmpCfgPath)
-
-    # give the server time to spin down but kill it after that
-    sleep(100)
-    if serverProcess.running: kill(serverProcess)
-
-  test "handle missing project configuration":
-    let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
-    let resp = http.get(apiBase & "/projects/" & cfg.projects[0].name)
-    check resp.status.startsWith("404")
-
-  test "gives 404 when no versions built":
-    let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
-    let resp = http.get(apiBase & "/projects/" & cfg.projects[0].name & "/versions")
-    check resp.status.startsWith("404")
-
-  test "GET /api/project/@projectName/versions":
-    let projArtifactsDir = tmpArtifactsDir & "/" & cfg.projects[0].name
-    let expectedVersions = @["alpha", "beta", "1.0.0", "1.0.1"]
-
-    # Touch configuration files
-    createDir(projArtifactsDir)
-    for v in expectedVersions:
-      var f: File
-      check open(f, projArtifactsDir & "/configuration." & v & ".json", fmWrite)
-      close(f)
-
-    let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
-    let resp = http.get(apiBase & "/project/" & cfg.projects[0].name & "/versions")
-    let returnedVersions = parseJson(resp.body).getElems.mapIt(it.getStr)
-    check sameContents(expectedVersions, returnedVersions)
-
-  # Last-chance catch to kill the server in case some test err'ed and didn't
-  # reach it's teardown handler
-  discard newAsyncHttpClient().post(apiBase & "/service/debug/stop")
-

strawboss.nimble

@@ -9,9 +9,12 @@ srcDir        = "src/main/nim"
 
 # Dependencies
 
-requires @["nim >= 0.16.1", "docopt >= 0.1.0", "tempfile", "jester", "bcrypt"]
+requires @["nim >= 0.16.1", "docopt >= 0.1.0", "tempfile", "jester", "bcrypt", "untar"]
 requires "https://github.com/yglukhov/nim-jwt"
 requires "https://git.jdb-labs.com/jdb/nim-lang-utils.git"
 
-task test, "Runs the test suite.":
+task functest, "Runs the functional test suite.":
-  exec "nim c -r src/test/nim/runtests.nim"
+  exec "nim c -r src/test/nim/run_functional_tests.nim"
+
+task unittest, "Runs the unit test suite.":
+  exec "nim c -r src/test/nim/run_unit_tests.nim"