Compare commits

...

11 Commits

Author SHA1 Message Date
deac844d02 Add testing protocol. 2020-09-05 19:16:00 -05:00
80a3ba4621 Add more descriptive message for 404s. 2019-02-21 23:46:45 -06:00
Jonathan Bernard
774d0b446f Fix typo in active runs API, add functional test for same. 2018-12-26 23:43:44 -06:00
Jonathan Bernard
ee1147a1a5 Add configurability of the server port. 2018-12-23 18:20:22 -06:00
Jonathan Bernard
186b7d5b29 Fix /ping unit test (now /version) 2018-12-23 17:42:43 -06:00
Jonathan Bernard
52eaa63f25 Rework build configuration to take advantage of new built-in docker build capabilities. 2018-12-23 17:39:04 -06:00
Jonathan Bernard
e61fe3b01e Add functional tests for docker-based build configurations. 2018-12-23 16:08:36 -06:00
Jonathan Bernard
e83e64273b Minor fixes to unused test file. 2018-12-21 21:12:15 -06:00
Jonathan Bernard
b2d4df0aac WIP Upgrading to Nim 0.19. Getting docker pieces compiling.
* Addressing breaking changes in migration from Nim 0.18 to 0.19.
* Finishing the initial pass at the refactor required to include
  docker-based builds.
* Regaining confidence in the existing functionality by getting all
  tests passing again after docker introduction (still need new tests to
  cover new docker functionality).
2018-12-09 07:09:23 -06:00
Jonathan Bernard
c827beab5e WIP Adding native support for docker. 2017-12-02 20:47:26 -06:00
Jonathan Bernard
0574f0ec6a Include the version being built in BuildStatus objects. 2017-12-02 19:08:29 -06:00
17 changed files with 352 additions and 178 deletions

View File

@ -3,13 +3,12 @@ import cliutils, docopt, os, sequtils, strutils, tempfile, uuids
import strawbosspkg/configuration import strawbosspkg/configuration
import strawbosspkg/core import strawbosspkg/core
import strawbosspkg/server import strawbosspkg/server
import strawbosspkg/version
let SB_VER = "0.4.0"
proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) = proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) =
let prefix = if cmd != nil: cmd & ": " else: "" let prefix = if cmd.len > 0: cmd & ": " else: ""
if outMsg != nil: stdout.writeLine prefix & outMsg if outMsg.len > 0: stdout.writeLine prefix & outMsg
if errMsg != nil: stderr.writeLine prefix & errMsg if errMsg.len > 0: stderr.writeLine prefix & errMsg
when isMainModule: when isMainModule:
@ -26,7 +25,7 @@ Options
(strawboss.config.json). (strawboss.config.json).
""" """
let args = docopt(doc, version = "strawboss v" & SB_VER) let args = docopt(doc, version = "strawboss v" & SB_VERSION)
let cfgFile = if args["--config-file"]: $args["--config-file"] let cfgFile = if args["--config-file"]: $args["--config-file"]
else: "strawboss.config.json" else: "strawboss.config.json"
@ -50,7 +49,7 @@ Options
try: try:
if req.workspaceDir.isNilOrEmpty: req.workspaceDir = mkdtemp() if req.workspaceDir.len == 0: req.workspaceDir = mkdtemp()
let status = core.run(cfg, req, logProcOutput) let status = core.run(cfg, req, logProcOutput)
if status.state == BuildState.failed: raiseEx status.details if status.state == BuildState.failed: raiseEx status.details

View File

@ -1,4 +1,5 @@
import cliutils, logging, json, os, sequtils, strtabs, strutils, tables, times, uuids import cliutils, logging, json, os, sequtils, strtabs, strutils, tables, times,
unicode, uuids
from langutils import sameContents from langutils import sameContents
from typeinfo import toAny from typeinfo import toAny
@ -7,23 +8,22 @@ from strutils import parseEnum
const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz" const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz"
# Types # Types
#
type type
BuildState* {.pure.} = enum BuildState* {.pure.} = enum
complete, failed, queued, rejected, running, setup, stepComplete complete, failed, queued, rejected, running, setup, stepComplete
BuildStatus* = object BuildStatus* = object
runId*, details*: string runId*, details*, version*: string
state*: BuildState state*: BuildState
Step* = object Step* = object
name*, stepCmd*, workingDir*: string containerImage*, name*, stepCmd*, workingDir*: string
artifacts*, cmdInput*, depends*, expectedEnv*: seq[string] artifacts*, cmdInput*, depends*, expectedEnv*: seq[string]
dontSkip*: bool dontSkip*: bool
ProjectConfig* = object ProjectConfig* = object
name*: string containerImage*, name*, versionCmd*: string
versionCmd*: string
steps*: Table[string, Step] steps*: Table[string, Step]
ProjectDef* = object ProjectDef* = object
@ -33,7 +33,7 @@ type
RunRequest* = object RunRequest* = object
runId*: UUID runId*: UUID
projectName*, stepName*, buildRef*, workspaceDir*: string projectName*, stepName*, buildRef*, workspaceDir*: string
timestamp*: TimeInfo timestamp*: DateTime
forceRebuild*: bool forceRebuild*: bool
Run* = object Run* = object
@ -58,6 +58,7 @@ type
debug*: bool debug*: bool
logLevel*: Level logLevel*: Level
pathToExe*: string pathToExe*: string
port*: int
projects*: seq[ProjectDef] projects*: seq[ProjectDef]
pwdCost*: int8 pwdCost*: int8
users*: seq[UserRef] users*: seq[UserRef]
@ -83,6 +84,7 @@ proc `==`*(a, b: StrawBossConfig): bool =
a.buildDataDir == b.buildDataDir and a.buildDataDir == b.buildDataDir and
a.authSecret == b.authSecret and a.authSecret == b.authSecret and
a.pwdCost == b.pwdCost and a.pwdCost == b.pwdCost and
a.port == b.port and
a.maintenancePeriod == b.maintenancePeriod and a.maintenancePeriod == b.maintenancePeriod and
a.logLevel == b.logLevel and a.logLevel == b.logLevel and
sameContents(a.users, b.users) and sameContents(a.users, b.users) and
@ -118,7 +120,7 @@ proc getOrFail(n: JsonNode, key: string, objName: string = ""): JsonNode =
# Configuration parsing code # Configuration parsing code
proc parseLogLevel*(level: string): Level = proc parseLogLevel*(level: string): Level =
let lvlStr = "lvl" & toUpper(level[0]) & level[1..^1] let lvlStr = "lvl" & toUpperAscii(level[0]) & level[1..^1]
result = parseEnum[Level](lvlStr) result = parseEnum[Level](lvlStr)
proc parseProjectDef*(pJson: JsonNode): ProjectDef = proc parseProjectDef*(pJson: JsonNode): ProjectDef =
@ -143,10 +145,11 @@ proc parseStrawBossConfig*(jsonCfg: JsonNode): StrawBossConfig =
result = StrawBossConfig( result = StrawBossConfig(
buildDataDir: jsonCfg.getIfExists("buildDataDir").getStr("build-data"), buildDataDir: jsonCfg.getIfExists("buildDataDir").getStr("build-data"),
authSecret: jsonCfg.getOrFail("authSecret", "strawboss config").getStr, authSecret: jsonCfg.getOrFail("authSecret", "strawboss config").getStr,
debug: jsonCfg.getIfExists("debug").getBVal(false), debug: jsonCfg.getIfExists("debug").getBool(false),
pwdCost: int8(jsonCfg.getOrFail("pwdCost", "strawboss config").getNum), port: int(jsonCfg.getIfExists("port").getInt(8180)),
pwdCost: int8(jsonCfg.getOrFail("pwdCost", "strawboss config").getInt),
projects: jsonCfg.getIfExists("projects").getElems.mapIt(parseProjectDef(it)), projects: jsonCfg.getIfExists("projects").getElems.mapIt(parseProjectDef(it)),
maintenancePeriod: int(jsonCfg.getIfExists("maintenancePeriod").getNum(10000)), maintenancePeriod: int(jsonCfg.getIfExists("maintenancePeriod").getInt(10000)),
logLevel: parseLogLevel(jsonCfg.getIfExists("logLevel").getStr("info")), logLevel: parseLogLevel(jsonCfg.getIfExists("logLevel").getStr("info")),
users: users) users: users)
@ -170,14 +173,15 @@ proc loadProjectConfig*(cfgFile: string): ProjectConfig =
var steps = initTable[string, Step]() var steps = initTable[string, Step]()
for sName, pJson in jsonCfg.getOrFail("steps", "project configuration").getFields: for sName, pJson in jsonCfg.getOrFail("steps", "project configuration").getFields:
steps[sName] = Step( steps[sName] = Step(
name: sName, name: sName,
workingDir: pJson.getIfExists("workingDir").getStr("."), workingDir: pJson.getIfExists("workingDir").getStr("."),
stepCmd: pJson.getIfExists("stepCmd").getStr("NOT GIVEN"), stepCmd: pJson.getIfExists("stepCmd").getStr("NOT GIVEN"),
depends: pJson.getIfExists("depends").getElems.mapIt(it.getStr), depends: pJson.getIfExists("depends").getElems.mapIt(it.getStr),
artifacts: pJson.getIfExists("artifacts").getElems.mapIt(it.getStr), artifacts: pJson.getIfExists("artifacts").getElems.mapIt(it.getStr),
cmdInput: pJson.getIfExists("cmdInput").getElems.mapIt(it.getStr), cmdInput: pJson.getIfExists("cmdInput").getElems.mapIt(it.getStr),
expectedEnv: pJson.getIfExists("expectedEnv").getElems.mapIt(it.getStr), expectedEnv: pJson.getIfExists("expectedEnv").getElems.mapIt(it.getStr),
dontSkip: pJson.getIfExists("dontSkip").getBVal(false)) containerImage: pJson.getIfExists("containerImage").getStr(""),
dontSkip: pJson.getIfExists("dontSkip").getBool(false))
# cmdInput and stepCmd are related, so we have a conditional defaulting. # cmdInput and stepCmd are related, so we have a conditional defaulting.
# Four possibilities: # Four possibilities:
@ -194,6 +198,7 @@ proc loadProjectConfig*(cfgFile: string): ProjectConfig =
result = ProjectConfig( result = ProjectConfig(
name: jsonCfg.getOrFail("name", "project configuration").getStr, name: jsonCfg.getOrFail("name", "project configuration").getStr,
containerImage: jsonCfg.getIfExists("containerImage").getStr(""),
versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"), versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"),
steps: steps) steps: steps)
@ -217,7 +222,7 @@ proc parseRunRequest*(reqJson: JsonNode): RunRequest =
buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr, buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr,
workspaceDir: reqJson.getOrFail("workspaceDir", "RunRequest").getStr, workspaceDir: reqJson.getOrFail("workspaceDir", "RunRequest").getStr,
timestamp: times.parse(reqJson.getOrFail("timestamp", "RunRequest").getStr, ISO_TIME_FORMAT), timestamp: times.parse(reqJson.getOrFail("timestamp", "RunRequest").getStr, ISO_TIME_FORMAT),
forceRebuild: reqJson.getOrFail("forceRebuild", "RunRequest").getBVal) forceRebuild: reqJson.getOrFail("forceRebuild", "RunRequest").getBool)
proc loadRunRequest*(reqFilePath: string): RunRequest = proc loadRunRequest*(reqFilePath: string): RunRequest =
if not existsFile(reqFilePath): if not existsFile(reqFilePath):
@ -259,6 +264,9 @@ proc `%`*(s: Step): JsonNode =
"expectedEnv": s.expectedEnv, "expectedEnv": s.expectedEnv,
"dontSkip": s.dontSkip } "dontSkip": s.dontSkip }
if s.containerImage.len > 0:
result["containerImage"] = %s.containerImage
proc `%`*(p: ProjectConfig): JsonNode = proc `%`*(p: ProjectConfig): JsonNode =
result = %* { result = %* {
"name": p.name, "name": p.name,
@ -268,6 +276,9 @@ proc `%`*(p: ProjectConfig): JsonNode =
for name, step in p.steps: for name, step in p.steps:
result["steps"][name] = %step result["steps"][name] = %step
if p.containerImage.len > 0:
result["containerImage"] = %p.containerImage
proc `%`*(req: RunRequest): JsonNode = proc `%`*(req: RunRequest): JsonNode =
result = %* { result = %* {
"runId": $(req.runId), "runId": $(req.runId),
@ -288,10 +299,11 @@ proc `%`*(cfg: StrawBossConfig): JsonNode =
"buildDataDir": cfg.buildDataDir, "buildDataDir": cfg.buildDataDir,
"authSecret": cfg.authSecret, "authSecret": cfg.authSecret,
"debug": cfg.debug, "debug": cfg.debug,
"port": cfg.port,
"projects": %cfg.projects, "projects": %cfg.projects,
"pwdCost": cfg.pwdCost, "pwdCost": cfg.pwdCost,
"maintenancePeriod": cfg.maintenancePeriod, "maintenancePeriod": cfg.maintenancePeriod,
"logLevel": toLower(($cfg.logLevel)[3]) & ($cfg.logLevel)[4..^1], "logLevel": toLowerAscii(($cfg.logLevel)[3]) & ($cfg.logLevel)[4..^1],
"users": %cfg.users } "users": %cfg.users }
proc `%`*(run: Run): JsonNode = proc `%`*(run: Run): JsonNode =

View File

@ -1,5 +1,5 @@
import cliutils, logging, json, os, ospaths, osproc, sequtils, streams, import cliutils, logging, json, os, ospaths, osproc, sequtils, streams,
strtabs, strutils, tables, times, uuids strtabs, strutils, tables, tempfile, times, uuids
import ./configuration import ./configuration
import nre except toSeq import nre except toSeq
@ -49,20 +49,64 @@ proc newCopy(w: Workspace): Workspace =
step: w.step, step: w.step,
version: w.version) version: w.version)
# Logging wrappers around const WKSP_ROOT = "/strawboss/wksp"
const ARTIFACTS_ROOT = "/strawboss/artifacts"
proc execWithOutput(wksp: Workspace, cmd, workingDir: string,
args: openarray[string], env: StringTableRef,
options: set[ProcessOption] = {poUsePath},
msgCB: HandleProcMsgCB = nil):
tuple[output: TaintedString, error: TaintedString, exitCode: int]
{.tags: [ExecIOEffect, ReadIOEffect, RootEffect] .} =
# Look for a container image to use
let containerImage =
if wksp.step.containerImage.len > 0: wksp.step.containerImage
else: wksp.project.containerImage
if containerImage.len == 0:
return execWithOutput(cmd, workingDir, args, env, options, msgCB)
var fullEnv = newStringTable(modeCaseSensitive)
for k,v in env: fullEnv[k] = v
var fullArgs = @["run", "-w", WKSP_ROOT, "-v", wksp.dir & ":" & WKSP_ROOT ]
if wksp.step.name.len == 0:
for depStep in wksp.step.depends:
fullArgs.add(["-v", ARTIFACTS_ROOT / depStep])
fullEnv[depStep & "_DIR"] = ARTIFACTS_ROOT / depStep
let envFile = mkstemp().name
writeFile(envFile, toSeq(fullEnv.pairs()).mapIt(it[0] & "=" & it[1]).join("\n"))
fullArgs.add(["--env-file", envFile])
fullArgs.add(containerImage)
fullArgs.add(cmd)
echo "Executing docker command: \n\t" & "docker " & $(fullArgs & @args)
return execWithOutput("docker", wksp.dir, fullArgs & @args, fullEnv, options, msgCB)
proc exec(w: Workspace, cmd, workingDir: string, args: openarray[string],
env: StringTableRef, options: set[ProcessOption] = {poUsePath},
msgCB: HandleProcMsgCB = nil): int
{.tags: [ExecIOEffect, ReadIOEffect, RootEffect] .} =
return execWithOutput(w, cmd, workingDir, args, env, options, msgCB)[2]
# Utility methods for Workspace activities # Utility methods for Workspace activities
proc sendStatusMsg(oh: HandleProcMsgCB, status: BuildStatus): void = proc sendStatusMsg(oh: HandleProcMsgCB, status: BuildStatus): void =
if not oh.isNil: if not oh.isNil:
oh.sendMsg($status.state & ": " & status.details, nil, "strawboss") oh.sendMsg($status.state & ": " & status.details, "", "strawboss")
proc sendMsg(w: Workspace, msg: TaintedString): void = proc sendMsg(w: Workspace, msg: TaintedString): void =
w.outputHandler.sendMsg(msg, nil, "strawboss") w.outputHandler.sendMsg(msg, "", "strawboss")
proc sendMsg(w: Workspace, l: Level, msg: TaintedString): void = proc sendMsg(w: Workspace, l: Level, msg: TaintedString): void =
if l >= w.logLevel: w.sendMsg(msg) if l >= w.logLevel: w.sendMsg(msg)
proc sendErrMsg(w: Workspace, msg: TaintedString): void = proc sendErrMsg(w: Workspace, msg: TaintedString): void =
w.outputHandler.sendMsg(nil, msg, "strawboss") w.outputHandler.sendMsg("", msg, "strawboss")
proc sendErrMsg(w: Workspace, l: Level, msg: TaintedString): void = proc sendErrMsg(w: Workspace, l: Level, msg: TaintedString): void =
if l >= w.logLevel: w.sendErrMsg(msg) if l >= w.logLevel: w.sendErrMsg(msg)
@ -79,14 +123,17 @@ proc publishStatus(wksp: Workspace, state: BuildState, details: string): void =
## Update the status for a Workspace and publish this status to the ## Update the status for a Workspace and publish this status to the
## Workspace's status file and any output message handlers. ## Workspace's status file and any output message handlers.
wksp.status = BuildStatus( wksp.status = BuildStatus(
runId: $wksp.runRequest.runId, state: state, details: details) runId: $wksp.runRequest.runId,
state: state,
details: details,
version: wksp.version)
# Write to our run directory, and to our version status # Write to our run directory, and to our version status
writeFile(wksp.buildDataDir / "runs" / writeFile(wksp.buildDataDir / "runs" /
$wksp.runRequest.runId & ".status.json", $wksp.status) $wksp.runRequest.runId & ".status.json", $wksp.status)
# If we have our step we can save status to the step status # If we have our step we can save status to the step status
if not wksp.step.name.isNilOrEmpty(): if wksp.step.name.len > 0:
let stepStatusDir = wksp.buildDataDir / "status" / wksp.step.name let stepStatusDir = wksp.buildDataDir / "status" / wksp.step.name
if not existsDir(stepStatusDir): createDir(stepStatusDir) if not existsDir(stepStatusDir): createDir(stepStatusDir)
writeFile(stepStatusDir / wksp.version & ".json", $wksp.status) writeFile(stepStatusDir / wksp.version & ".json", $wksp.status)
@ -233,7 +280,7 @@ proc getProjectConfig*(cfg: StrawBossConfig,
# If they didn't give us a version, let try to figure out what is the latest one. # If they didn't give us a version, let try to figure out what is the latest one.
var confFilePath: string var confFilePath: string
if version.isNilOrEmpty: if version.len == 0:
let candidatePaths = filesMatching( let candidatePaths = filesMatching(
cfg.buildDataDir / project.name / "configurations/*.json") cfg.buildDataDir / project.name / "configurations/*.json")
@ -426,7 +473,8 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
result = BuildStatus( result = BuildStatus(
runId: $req.runId, runId: $req.runId,
state: BuildState.setup, state: BuildState.setup,
details: "initializing build workspace") details: "initializing build workspace",
version: "")
outputHandler.sendStatusMsg(result) outputHandler.sendStatusMsg(result)
var wksp: Workspace var wksp: Workspace
@ -459,7 +507,7 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
wksp = Workspace( wksp = Workspace(
buildDataDir: cfg.buildDataDir / projectDef.name, buildDataDir: cfg.buildDataDir / projectDef.name,
buildRef: buildRef:
if req.buildRef != nil and req.buildRef.len > 0: req.buildRef if req.buildRef.len > 0: req.buildRef
else: projectDef.defaultBranch, else: projectDef.defaultBranch,
dir: req.workspaceDir, dir: req.workspaceDir,
env: env, env: env,
@ -471,12 +519,12 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
runRequest: req, runRequest: req,
status: result, status: result,
step: Step(), step: Step(),
version: nil) version: "")
except: except:
when not defined(release): echo getCurrentException().getStackTrace() when not defined(release): echo getCurrentException().getStackTrace()
result = BuildStatus(runId: $req.runId, state: BuildState.failed, result = BuildStatus(runId: $req.runId, state: BuildState.failed,
details: getCurrentExceptionMsg()) details: getCurrentExceptionMsg(), version: "")
try: outputHandler.sendStatusMsg(result) try: outputHandler.sendStatusMsg(result)
except: discard "" except: discard ""
return return
@ -514,7 +562,8 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
wksp.publishStatus(BuildState.failed, msg) wksp.publishStatus(BuildState.failed, msg)
result = wksp.status result = wksp.status
except: except:
result = BuildStatus(runId: $req.runId, state: BuildState.failed, details: msg) result = BuildStatus(runId: $req.runId, state: BuildState.failed,
details: msg, version: "")
try: outputHandler.sendStatusMsg(result) try: outputHandler.sendStatusMsg(result)
except: discard "" except: discard ""
@ -545,7 +594,8 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
let queuedStatus = BuildStatus( let queuedStatus = BuildStatus(
runId: $req.runId, runId: $req.runId,
state: BuildState.queued, state: BuildState.queued,
details: "request queued for execution") details: "request queued for execution",
version: "")
writeFile(statusFile, $queuedStatus) writeFile(statusFile, $queuedStatus)
var args = @["run", reqFile, "-c", cfg.filePath] var args = @["run", reqFile, "-c", cfg.filePath]
@ -562,6 +612,7 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
let exMsg = "run request rejected: " & getCurrentExceptionMsg() let exMsg = "run request rejected: " & getCurrentExceptionMsg()
try: try:
writeFile(statusFile, writeFile(statusFile,
$(BuildStatus(runId: $req.runId, state: BuildState.rejected, details: exMsg))) $(BuildStatus(runId: $req.runId, state: BuildState.rejected,
details: exMsg, version: "")))
except: discard "" except: discard ""
raiseEx exMsg raiseEx exMsg

View File

@ -1,12 +1,13 @@
import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, md5, import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, md5,
os, osproc, sequtils, strutils, tempfile, times, unittest, uuids options, os, osproc, sequtils, strutils, tempfile, times, unittest, uuids
from mimetypes import getMimeType from mimetypes import getMimeType
from asyncfile import openAsync, readToStream, close from asyncfile import openAsync, readToStream, close
from asyncnet import send from asyncnet import send
from re import re, find from re import re, find
from timeutils import trimNanoSec
import ./configuration, ./core import ./configuration, ./core, ./version
type type
Session = object Session = object
@ -19,24 +20,38 @@ const JSON = "application/json"
proc newSession*(user: UserRef): Session = proc newSession*(user: UserRef): Session =
result = Session( result = Session(
user: user, user: user,
issuedAt: getTime(), issuedAt: getTime().local.trimNanoSec.toTime,
expires: daysForward(7).toTime()) expires: daysForward(7).trimNanoSec.toTime)
proc buildJson(resp: Response, code: HttpCode, details: string = ""): void = template halt(code: HttpCode,
resp.data[0] = CallbackAction.TCActionSend headers: RawHeaders,
resp.data[1] = code content: string): typed =
resp.data[2]["Content-Type"] = JSON ## Immediately replies with the specified request. This means any further
resp.data[3] = $(%* { ## code will not be executed after calling this template in the current
"statusCode": code.int, ## route.
"status": $code, bind TCActionSend, newHttpHeaders
"details": details result[0] = CallbackAction.TCActionSend
}) result[1] = code
result[2] = some(headers)
result[3] = content
result.matched = true
break allRoutes
# Work-around for weirdness trying to use resp(Http500... in exception blocks template jsonResp(code: HttpCode, details: string = "", headers: RawHeaders = @{:} ) =
proc build500Json(resp: Response, ex: ref Exception, msg: string): void = halt(
code,
headers & @{"Content-Type": JSON},
$(%* {
"statusCode": code.int,
"status": $code,
"details": details
})
)
template json500Resp(ex: ref Exception, details: string = ""): void =
when not defined(release): debug ex.getStackTrace() when not defined(release): debug ex.getStackTrace()
error msg & ":\n" & ex.msg error details & ":\n" & ex.msg
resp.buildJson(Http500) jsonResp(Http500)
proc toJWT*(cfg: StrawBossConfig, session: Session): string = proc toJWT*(cfg: StrawBossConfig, session: Session): string =
## Make a JST token for this session. ## Make a JST token for this session.
@ -44,8 +59,8 @@ proc toJWT*(cfg: StrawBossConfig, session: Session): string =
header: JOSEHeader(alg: HS256, typ: "jwt"), header: JOSEHeader(alg: HS256, typ: "jwt"),
claims: toClaims(%*{ claims: toClaims(%*{
"sub": session.user.name, "sub": session.user.name,
"iat": session.issuedAt.toSeconds().int, "iat": session.issuedAt.toUnix.int,
"exp": session.expires.toSeconds().int })) "exp": session.expires.toUnix.int }))
jwt.sign(cfg.authSecret) jwt.sign(cfg.authSecret)
result = $jwt result = $jwt
@ -64,8 +79,8 @@ proc fromJWT*(cfg: StrawBossConfig, strTok: string): Session =
result = Session( result = Session(
user: users[0], user: users[0],
issuedAt: fromSeconds(jwt.claims["iat"].node.num), issuedAt: fromUnix(jwt.claims["iat"].node.num),
expires: fromSeconds(jwt.claims["exp"].node.num)) expires: fromUnix(jwt.claims["exp"].node.num))
proc extractSession(cfg: StrawBossConfig, request: Request): Session = proc extractSession(cfg: StrawBossConfig, request: Request): Session =
## Helper to extract a session from a reqest. ## Helper to extract a session from a reqest.
@ -93,7 +108,7 @@ proc makeAuthToken*(cfg: StrawBossConfig, uname, pwd: string): string =
## Given a username and pwd, validate the combination and generate a JWT ## Given a username and pwd, validate the combination and generate a JWT
## token string. ## token string.
if uname == nil or pwd == nil: if uname.len == 0 or pwd.len == 0:
raiseEx "fields 'username' and 'password' required" raiseEx "fields 'username' and 'password' required"
# find the user record # find the user record
@ -115,7 +130,7 @@ proc makeApiKey*(cfg: StrawBossConfig, uname: string): string =
## function for an administrator to setup a unsupervised account (git access ## function for an administrator to setup a unsupervised account (git access
## for example). ## for example).
if uname == nil: raiseEx "no username given" if uname.len == 0: raiseEx "no username given"
# find the user record # find the user record
let users = cfg.users.filterIt(it.name == uname) let users = cfg.users.filterIt(it.name == uname)
@ -130,21 +145,15 @@ proc makeApiKey*(cfg: StrawBossConfig, uname: string): string =
template checkAuth() = template checkAuth() =
## Check this request for authentication and authorization information. ## Check this request for authentication and authorization information.
## Injects two variables into the running context: the session and authed: ## Injects the session into the running context. If the request is not
## true if the request is authorized, false otherwise. If the request is not ## authorized, this template returns an appropriate 401 response.
## authorized, this template sets up the 401 response correctly. The calling
## context needs only to return from the route.
var session {.inject.}: Session var session {.inject.}: Session
var authed {.inject.} = false
try: try: session = extractSession(cfg, request)
session = extractSession(cfg, request)
authed = true
except: except:
debug "Auth failed: " & getCurrentExceptionMsg() debug "Auth failed: " & getCurrentExceptionMsg()
response.data[2]["WWW-Authenticate"] = "Bearer" jsonResp(Http401, "Unauthorized", @{"WWW-Authenticate": "Bearer"})
response.buildJson(Http401)
proc start*(cfg: StrawBossConfig): void = proc start*(cfg: StrawBossConfig): void =
@ -152,13 +161,13 @@ proc start*(cfg: StrawBossConfig): void =
var workers: seq[Worker] = @[] var workers: seq[Worker] = @[]
settings: settings:
port = Port(8180) port = Port(cfg.port)
appName = "/api" appName = "/api"
routes: routes:
get "/ping": get "/version":
resp($(%"pong"), JSON) resp($(%("strawboss v" & SB_VERSION)), JSON)
post "/auth-token": post "/auth-token":
var uname, pwd: string var uname, pwd: string
@ -166,37 +175,39 @@ proc start*(cfg: StrawBossConfig): void =
let jsonBody = parseJson(request.body) let jsonBody = parseJson(request.body)
uname = jsonBody["username"].getStr uname = jsonBody["username"].getStr
pwd = jsonBody["password"].getStr pwd = jsonBody["password"].getStr
except: response.buildJson(Http400); return true except: jsonResp(Http400)
try: try:
let authToken = makeAuthToken(cfg, uname, pwd) let authToken = makeAuthToken(cfg, uname, pwd)
resp($(%authToken), JSON) resp($(%authToken), JSON)
except: response.buildJson(Http401, getCurrentExceptionMsg()); return true except:
jsonResp(Http401, getCurrentExceptionMsg())
if ctx.cfg.debug: echo getStackTrace()
get "/verify-auth": get "/verify-auth":
checkAuth(); if not authed: return true checkAuth()
resp(Http200, $(%*{ "username": session.user.name }), JSON) resp(Http200, $(%*{ "username": session.user.name }), JSON)
get "/projects": get "/projects":
## List project summaries (ProjectDefs only) ## List project summaries (ProjectDefs only)
checkAuth(); if not authed: return true checkAuth()
resp($(%cfg.projects), JSON) resp($(%cfg.projects), JSON)
post "/projects": post "/projects":
## Create a new project definition ## Create a new project definition
checkAuth(); if not authed: return true checkAuth()
# TODO # TODO
response.buildJson(Http501); return true jsonResp(Http501)
get "/project/@projectName": get "/project/@projectName":
## Return a project's configuration, as well as it's versions. ## Return a project's configuration, as well as it's versions.
checkAuth(); if not authed: return true checkAuth()
# Make sure we know about that project # Make sure we know about that project
var projDef: ProjectDef var projDef: ProjectDef
@ -204,11 +215,10 @@ proc start*(cfg: StrawBossConfig): void =
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: except NotFoundException:
response.buildJson(Http404, getCurrentExceptionMsg()) jsonResp(Http404, getCurrentExceptionMsg())
except: except:
response.build500Json(getCurrentException(), let msg = "unable to load project definition for project " & @"projectName"
"unable to load project definition for project " & @"projectName") json500Resp(getCurrentException(), msg)
return true
var projConf: ProjectConfig var projConf: ProjectConfig
try: projConf = getProjectConfig(cfg, @"projectName", "") try: projConf = getProjectConfig(cfg, @"projectName", "")
@ -217,7 +227,7 @@ proc start*(cfg: StrawBossConfig): void =
let respJson = newJObject() let respJson = newJObject()
respJson["definition"] = %projDef respJson["definition"] = %projDef
respJson["versions"] = %listVersions(cfg, @"projectName") respJson["versions"] = %listVersions(cfg, @"projectName")
if not projConf.name.isNil: if projConf.name.len > 0:
respJson["latestConfig"] = %projConf respJson["latestConfig"] = %projConf
resp(pretty(respJson), JSON) resp(pretty(respJson), JSON)
@ -225,97 +235,89 @@ proc start*(cfg: StrawBossConfig): void =
get "/project/@projectName/versions": get "/project/@projectName/versions":
## Get a list of all versions that we have built ## Get a list of all versions that we have built
checkAuth(); if not authed: return true checkAuth()
try: resp($(%listVersions(cfg, @"projectName")), JSON) try: resp($(%listVersions(cfg, @"projectName")), JSON)
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: except NotFoundException:
response.buildJson(Http404, getCurrentExceptionMsg()) jsonResp(Http404, getCurrentExceptionMsg())
except: except:
response.build500Json(getCurrentException(), let msg = "unable to list versions for project " & @"projectName"
"unable to list versions for project " & @"projectName") json500Resp(getCurrentException(), msg)
return true
get "/project/@projectName/version/@version?": get "/project/@projectName/version/@version?":
## Get a detailed project record including step definitions (ProjectConfig). ## Get a detailed project record including step definitions (ProjectConfig).
checkAuth(); if not authed: return true checkAuth()
# Make sure we know about that project # Make sure we know about that project
try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON) try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON)
except: response.buildJson(Http404, getCurrentExceptionMsg()); return true except: jsonResp(Http404, getCurrentExceptionMsg())
get "/project/@projectName/runs": get "/project/@projectName/runs":
## List all runs ## List all runs
checkAuth(); if not authed: return true checkAuth()
try: resp($(%listRuns(cfg, @"projectName")), JSON) try: resp($(%listRuns(cfg, @"projectName")), JSON)
except: response.buildJson(Http404, getCurrentExceptionMsg()); return true except: jsonResp(Http404, getCurrentExceptionMsg())
get "/project/@projectName/runs/active": get "/project/@projectName/runs/active":
## List all currently active runs ## List all currently active runs
checkAuth(); if not authed: return true checkAuth()
var details = ""
try: try:
let activeRuns = workers let activeRuns = workers
.filterIt(it.process.running and it.projectName == @"projectName") .filterIt(it.process.running and it.projectName == @"projectName")
.mapIt(cfg.getRun(@"projecName", $it.runId)); .mapIt(cfg.getRun(@"projectName", $it.runId));
resp($(%activeRuns), JSON) resp($(%activeRuns), JSON)
except NotFoundException:
jsonResp(Http404, getCurrentExceptionMsg())
except: except:
try: raise getCurrentException() json500Resp(getCurrentException(), "problem loading active runs")
except NotFoundException:
response.buildJson(Http404, getCurrentExceptionMsg())
except:
response.build500Json(getCurrentException(),
"problem loading active runs")
return true
get "/project/@projectName/run/@runId": get "/project/@projectName/run/@runId":
## Details for a specific run ## Details for a specific run
checkAuth(); if not authed: return true checkAuth()
# Make sure we know about that project # Make sure we know about that project
try: discard cfg.getProject(@"projectName") try: discard cfg.getProject(@"projectName")
except: response.buildJson(Http404, getCurrentExceptionMsg()); return true except: jsonResp(Http404, getCurrentExceptionMsg())
if not existsRun(cfg, @"projectName", @"runId"): if not existsRun(cfg, @"projectName", @"runId"):
response.buildJson(Http404, "no such run for project"); return true jsonResp(Http404, "no such run for project")
try: resp($getRun(cfg, @"projectName", @"runId"), JSON) try: resp($getRun(cfg, @"projectName", @"runId"), JSON)
except: except:
response.build500Json(getCurrentException(), json500Resp(getCurrentException(),
"unable to load run details for project " & @"projectName" & "unable to load run details for project " & @"projectName" &
" run " & @"runId") " run " & @"runId")
return true
get "/project/@projectName/run/@runId/logs": get "/project/@projectName/run/@runId/logs":
## Get logs from a specific run ## Get logs from a specific run
checkAuth(); if not authed: return true checkAuth()
try: discard cfg.getProject(@"projectName") try: discard cfg.getProject(@"projectName")
except: except:
response.buildJson(Http404, getCurrentExceptionMsg()) jsonResp(Http404, getCurrentExceptionMsg())
return true
if not existsRun(cfg, @"projectName", @"runId"): if not existsRun(cfg, @"projectName", @"runId"):
response.buildJson(Http404, "no such run for project") jsonResp(Http404, "no such run for project")
return true
try: resp($getLogs(cfg, @"projectName", @"runId")) try: resp($getLogs(cfg, @"projectName", @"runId"))
except: except:
response.build500Json(getCurrentException(), json500Resp(getCurrentException(),
"unable to load run logs for " & @"projectName" & " run " & @"runId") "unable to load run logs for " & @"projectName" & " run " & @"runId")
return true
get "/project/@projectName/step/@stepName/artifacts/@version": get "/project/@projectName/step/@stepName/artifacts/@version":
## Get the list of artifacts that were built for ## Get the list of artifacts that were built for
checkAuth(); if not authed: return true checkAuth()
debug "Matched artifacts list request: " & $(%*{ debug "Matched artifacts list request: " & $(%*{
"project": @"projectName", "project": @"projectName",
@ -327,16 +329,15 @@ proc start*(cfg: StrawBossConfig): void =
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: except NotFoundException:
response.buildJson(Http404, getCurrentExceptionMsg()) jsonResp(Http404, getCurrentExceptionMsg())
except: except:
response.build500Json(getCurrentException(), "unable to list artifacts for " & json500Resp(getCurrentException(), "unable to list artifacts for " &
@"projectName" & ":" & @"stepName" & "@" & @"buildRef") @"projectName" & ":" & @"stepName" & "@" & @"buildRef")
return true
get "/project/@projectName/step/@stepName/artifact/@version/@artifactName": get "/project/@projectName/step/@stepName/artifact/@version/@artifactName":
## Get a specific artifact that was built. ## Get a specific artifact that was built.
checkAuth(); if not authed: return true checkAuth()
var artifactPath: string var artifactPath: string
try: artifactPath = getArtifactPath(cfg, try: artifactPath = getArtifactPath(cfg,
@ -344,11 +345,12 @@ proc start*(cfg: StrawBossConfig): void =
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: except NotFoundException:
response.buildJson(Http404, getCurrentExceptionMsg()) jsonResp(Http404, getCurrentExceptionMsg())
except: except:
response.build500Json(getCurrentException(), "unable to check artifact path for " & json500Resp(getCurrentException(), "unable to check artifact path for " &
@"projectName" & ":" & @"stepName" & "@" & @"version") @"projectName" & ":" & @"stepName" & "@" & @"version")
return true
enableRawMode
debug "Preparing: " & artifactPath debug "Preparing: " & artifactPath
let fileSize = getFileSize(artifactPath) let fileSize = getFileSize(artifactPath)
@ -361,19 +363,19 @@ proc start*(cfg: StrawBossConfig): void =
# If the user has a cached version of this file and it matches our # If the user has a cached version of this file and it matches our
# version, let them use it # version, let them use it
if request.headers.hasKey("If-None-Match") and request.headers["If-None-Match"] == hashed: if request.headers.hasKey("If-None-Match") and request.headers["If-None-Match"] == hashed:
resp(Http304, [], "") resp(Http304)
else: else:
resp(Http200, [ resp(Http200, [
("Content-Disposition", "; filename=\"" & @"artifactName" & "\""), ("Content-Disposition", "; filename=\"" & @"artifactName" & "\""),
("Content-Type", mimetype), ("Content-Type", mimetype),
("ETag", hashed )], file) ("ETag", hashed )], file)
else: else:
let headers = { let headers = @{
"Content-Disposition": "; filename=\"" & @"artifactName" & "\"", "Content-Disposition": "; filename=\"" & @"artifactName" & "\"",
"Content-Type": mimetype, "Content-Type": mimetype,
"Content-Length": $fileSize "Content-Length": $fileSize
}.newStringTable }
await response.sendHeaders(Http200, headers) request.sendHeaders(Http200, headers)
var fileStream = newFutureStream[string]("sendStaticIfExists") var fileStream = newFutureStream[string]("sendStaticIfExists")
var file = openAsync(artifactPath, fmRead) var file = openAsync(artifactPath, fmRead)
@ -384,25 +386,22 @@ proc start*(cfg: StrawBossConfig): void =
# `bodyStream` has been written to the file. # `bodyStream` has been written to the file.
while true: while true:
let (hasValue, value) = await fileStream.read() let (hasValue, value) = await fileStream.read()
if hasValue: if hasValue: request.send(value)
await response.client.send(value) else: break
else:
break
file.close() file.close()
get "/project/@projectName/step/@stepName/status/@buildRef": get "/project/@projectName/step/@stepName/status/@buildRef":
## Get detailed information about the status of a step (assuming it has been built) ## Get detailed information about the status of a step (assuming it has been built)
checkAuth(); if not authed: return true checkAuth()
try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON) try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON)
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg()) except NotFoundException: jsonResp(Http404, getCurrentExceptionMsg())
except: except:
response.build500Json(getCurrentException(), "unable to load the build state for " & json500Resp(getCurrentException(), "unable to load the build state for " &
@"projectName" & ":" & @"stepName" & "@" & @"buildRef") @"projectName" & ":" & @"stepName" & "@" & @"buildRef")
return true
#get "/project/@projectName/step/@stepName/status/@buildRef.svg": #get "/project/@projectName/step/@stepName/status/@buildRef.svg":
## Get an image representing the status of a build ## Get an image representing the status of a build
@ -413,14 +412,14 @@ proc start*(cfg: StrawBossConfig): void =
post "/project/@projectName/step/@stepName/run/@buildRef?": post "/project/@projectName/step/@stepName/run/@buildRef?":
# Kick off a run # Kick off a run
checkAuth(); if not authed: return true checkAuth()
let runRequest = RunRequest( let runRequest = RunRequest(
runId: genUUID(), runId: genUUID(),
projectName: @"projectName", projectName: @"projectName",
stepName: @"stepName", stepName: @"stepName",
buildRef: if @"buildRef" != "": @"buildRef" else: nil, buildRef: if @"buildRef" != "": @"buildRef" else: "",
timestamp: getLocalTime(getTime()), timestamp: getTime().local,
forceRebuild: false) # TODO support this with optional query params forceRebuild: false) # TODO support this with optional query params
# TODO: instead of immediately spawning a worker, add the request to a # TODO: instead of immediately spawning a worker, add the request to a
@ -436,13 +435,11 @@ proc start*(cfg: StrawBossConfig): void =
status: status), JSON) status: status), JSON)
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: except NotFoundException: jsonResp(Http404, getCurrentExceptionMsg())
response.buildJson(Http404, getCurrentExceptionMsg()) except: jsonResp(Http400, getCurrentExceptionMsg())
except: response.buildJson(Http400, getCurrentExceptionMsg())
return true
post "/service/debug/stop": post "/service/debug/stop":
if not cfg.debug: response.buildJson(Http404); return true if not cfg.debug: jsonResp(Http404)
else: else:
let shutdownFut = sleepAsync(100) let shutdownFut = sleepAsync(100)
shutdownFut.callback = proc(): void = complete(stopFuture) shutdownFut.callback = proc(): void = complete(stopFuture)
@ -450,10 +447,10 @@ proc start*(cfg: StrawBossConfig): void =
get re".*": get re".*":
response.buildJson(Http404); return true jsonResp(Http404, "URL [" & request.path & "] is not present on this server.")
post re".*": post re".*":
response.buildJson(Http404); return true jsonResp(Http404)
proc performMaintenance(cfg: StrawBossConfig): void = proc performMaintenance(cfg: StrawBossConfig): void =
# Prune workers # Prune workers

View File

@ -0,0 +1,2 @@
const SB_VERSION* = "0.5.1"

View File

@ -1,8 +1,10 @@
{ {
"name": "dummy-project", "name": "dummy-project",
"versionCmd": "git describe --all --always", "versionCmd": "git describe --all --always",
"containerImage": "ubuntu",
"steps": { "steps": {
"build": { "build": {
"containerImage": "alpine",
"depends": ["test"], "depends": ["test"],
"workingDir": "dir1", "workingDir": "dir1",
"stepCmd": "cust-build", "stepCmd": "cust-build",

View File

@ -6,6 +6,7 @@
{ "name": "bob@builder.com", "hashedPwd": "$2a$11$lVZ9U4optQMhzPh0E9A7Yu6XndXblUF3gCa.zmEvJy4F.4C4718b." }, { "name": "bob@builder.com", "hashedPwd": "$2a$11$lVZ9U4optQMhzPh0E9A7Yu6XndXblUF3gCa.zmEvJy4F.4C4718b." },
{ "name": "sam@sousa.com", "hashedPwd": "testvalue" } { "name": "sam@sousa.com", "hashedPwd": "testvalue" }
], ],
"port": 8180,
"pwdCost": 11, "pwdCost": 11,
"projects": [ "projects": [
{ "name": "dummy-project", { "name": "dummy-project",

View File

@ -1,4 +1,4 @@
import unittest import tempfile, times, unittest, untar
from langutils import sameContents from langutils import sameContents
@ -12,8 +12,8 @@ let TIMEOUT = 2.minutes
suite "strawboss core": suite "strawboss core":
# Suite setup: extract test project # Suite setup: extract test project
let testProjTempDir = mkdir() let testProjTempDir = mkdtemp()
let testProjTarFile = newTarFile("src/test/test-project.tar.gz:) let testProjTarFile = newTarFile("src/test/test-project.tar.gz")
let testProjName = "test-project" let testProjName = "test-project"
testProjTarFile.extract(testProjTempDir) testProjTarFile.extract(testProjTempDir)

View File

@ -2,6 +2,7 @@ import cliutils, httpclient, json, os, osproc, sequtils, strutils, tempfile,
times, unittest, untar, uuids times, unittest, untar, uuids
from langutils import sameContents from langutils import sameContents
from algorithm import sorted
import ../testutil import ../testutil
import ../../../main/nim/strawbosspkg/configuration import ../../../main/nim/strawbosspkg/configuration
@ -50,7 +51,7 @@ suite "strawboss server":
@["serve", "-c", tempCfgPath], loadEnv(), {poUsePath}) @["serve", "-c", tempCfgPath], loadEnv(), {poUsePath})
# give the server time to spin up # give the server time to spin up
sleep(100) sleep(200)
teardown: teardown:
discard newAsyncHttpClient().post(apiBase & "/service/debug/stop") discard newAsyncHttpClient().post(apiBase & "/service/debug/stop")
@ -60,7 +61,7 @@ suite "strawboss server":
removeFile(tempCfgPath) removeFile(tempCfgPath)
# give the server time to spin down but kill it after that # give the server time to spin down but kill it after that
sleep(100) sleep(200)
if serverProcess.running: kill(serverProcess) if serverProcess.running: kill(serverProcess)
test "handle missing project configuration": test "handle missing project configuration":
@ -142,6 +143,61 @@ suite "strawboss server":
let status = loadBuildStatus(statusFile) let status = loadBuildStatus(statusFile)
check status.state == step[1] check status.state == step[1]
test "run a build in docker":
let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
# Run the "build-docker" step
var resp = http.post(apiBase & "/project/" & testProjName & "/step/build-docker/run/0.3.0")
check resp.status.startsWith("200")
let queuedRun = parseRun(parseJson(resp.body))
check queuedRun.status.state == BuildState.queued
# Wait for the build to complete
let completedRun = http.waitForBuild(apiBase, testProjName, $queuedRun.id)
# check that the run directory, run request, status, and output logs exist
let runsDir = tempBuildDataDir & "/" & testProjName & "/runs"
let runId = $completedRun.id
check existsDir(runsDir)
for suffix in [".request.json", ".status.json", ".stdout.log", ".stderr.log"]:
check existsFile(runsDir & "/" & runId & suffix)
# check that the project directory has been created in the artifacts repo
let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build-docker/0.3.0"
check existsDir(runArtifactsDir)
# check that the build step status file has been created
let statusFile = tempBuildDataDir & "/" & testProjName & "/status/build-docker/0.3.0.json"
check fileExists(statusFile)
# check that the status is complete
var status = loadBuildStatus(statusFile)
check status.state == BuildState.complete
# check that the artifacts we expect are present
let binFile = runArtifactsDir & "/test_project"
check existsFile(binFile)
test "run a multi-step docker-based build":
let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
# Run the "test" step (depends on "build")
var resp = http.post(apiBase & "/project/" & testProjname & "/step/test-docker/run/0.3.0")
check resp.status.startsWith("200")
let queuedRun = parseRun(parseJson(resp.body))
let completedRun = http.waitForBuild(apiBase, testProjName, $queuedRun.id)
# there should be successful status files for both the build and test steps
for step in [("build-docker", BuildState.stepComplete), ("test-docker", BuildState.complete)]:
let statusFile = tempBuildDataDir & "/" & testProjName & "/status/" & step[0] & "/0.3.0.json"
check fileExists(statusFile)
let status = loadBuildStatus(statusFile)
check status.state == step[1]
# TODO
#test "already completed steps should not be rebuilt": #test "already completed steps should not be rebuilt":
# let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password") # let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
# let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build/0.2.1" # let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build/0.2.1"
@ -150,9 +206,40 @@ suite "strawboss server":
# Run the "build" step # Run the "build" step
# Kick off a build that depends on "build" (which was run in the last test) # Kick off a build that depends on "build" (which was run in the last test)
# TODO test "kick off multiple runs and check the list of active runs via the API":
#test "kick off multiple runs and check the list of active runs via the API": let http = newAuthenticatedHttpClient(apiBase, "bob@builder.com", "password")
# check false
# Kick off multiple runs of the "long-running" job
let queuedRuns = toSeq((1..3)).map(proc (idx: int): Run =
let resp = http.post(apiBase & "/project/" & testProjName & "/step/long-running/run/0.3.1")
check resp.status.startsWith("200")
return parseRun(parseJson(resp.body)))
# Collect run ids.
let runIds = queuedRuns.mapIt($(it.id)).sorted(cmpIgnoreCase)
# Check on the runs
let getActiveResp = http.get(apiBase & "/project/" & testProjName & "/runs/active")
check getActiveResp.status.startsWith("200")
let activeRuns = parseJson(getActiveResp.body).getElems().mapIt(parseRun(it))
let activeRunIds = activeRuns.mapIt($(it.id)).sorted(cmpIgnoreCase)
# Make sure we see all runs in the active state.
check runIds == activeRunIds
let completedRuns = runIds.map(proc (runId: string): Run =
return http.waitForBuild(apiBase, testProjName, runId))
# Make sure all are completed and all are accounted for
check completedRuns.allIt(it.status.state == BuildState.complete)
check completedRuns.mapIt($(it.id)).sorted(cmpIgnoreCase) == runIds;
# Check that there are no more active runs
let getActiveResp2 = http.get(apiBase & "/project/" & testProjName & "/runs/active")
let remainingActiveRuns = parseJson(getActiveResp2.body).getElems().mapIt(parseRun(it))
check remainingActiveRuns.len == 0
# Last-chance catch to kill the server in case some test err'ed and didn't # Last-chance catch to kill the server in case some test err'ed and didn't
# reach it's teardown handler # reach it's teardown handler

View File

@ -26,7 +26,7 @@ proc waitForBuild*(client: HttpClient, apiBase, projectName, runId: string,
#echo "Checking (" & $curElapsed & " has passed)." #echo "Checking (" & $curElapsed & " has passed)."
if curElapsed > toFloat(timeout): if curElapsed > toFloat(timeout):
raise newException(SystemError, "Timeout exceeded waiting for build.") raise newException(Exception, "Timeout exceeded waiting for build.")
let resp = client.get(apiBase & "/project/" & projectName & "/run/" & runId) let resp = client.get(apiBase & "/project/" & projectName & "/run/" & runId)

View File

@ -1,6 +1,7 @@
import json, strtabs, times, tables, unittest, uuids import json, strtabs, times, tables, unittest, uuids
from langutils import sameContents from langutils import sameContents
from timeutils import trimNanoSec
import ../../../main/nim/strawbosspkg/configuration import ../../../main/nim/strawbosspkg/configuration
suite "load and save configuration objects": suite "load and save configuration objects":
@ -26,7 +27,7 @@ suite "load and save configuration objects":
stepName: "build", stepName: "build",
buildRef: "master", buildRef: "master",
workspaceDir: "/no-real/dir", workspaceDir: "/no-real/dir",
timestamp: getLocalTime(getTime()), timestamp: getTime().local.trimNanoSec,
forceRebuild: true) forceRebuild: true)
let rrStr = $rr1 let rrStr = $rr1
@ -99,6 +100,7 @@ suite "load and save configuration objects":
check: check:
pc.name == "dummy-project" pc.name == "dummy-project"
pc.versionCmd == "git describe --all --always" pc.versionCmd == "git describe --all --always"
pc.containerImage == "ubuntu"
pc.steps.len == 2 pc.steps.len == 2
# Explicitly set properties # Explicitly set properties
@ -106,6 +108,7 @@ suite "load and save configuration objects":
pc.steps["build"].dontSkip == true pc.steps["build"].dontSkip == true
pc.steps["build"].stepCmd == "cust-build" pc.steps["build"].stepCmd == "cust-build"
pc.steps["build"].workingDir == "dir1" pc.steps["build"].workingDir == "dir1"
pc.steps["build"].containerImage == "alpine"
sameContents(pc.steps["build"].artifacts, @["bin1", "doc1"]) sameContents(pc.steps["build"].artifacts, @["bin1", "doc1"])
sameContents(pc.steps["build"].depends, @["test"]) sameContents(pc.steps["build"].depends, @["test"])
sameContents(pc.steps["build"].expectedEnv, @["VAR1"]) sameContents(pc.steps["build"].expectedEnv, @["VAR1"])
@ -116,6 +119,7 @@ suite "load and save configuration objects":
pc.steps["test"].dontSkip == false pc.steps["test"].dontSkip == false
pc.steps["test"].stepCmd == "true" pc.steps["test"].stepCmd == "true"
pc.steps["test"].workingDir == "." pc.steps["test"].workingDir == "."
pc.steps["test"].containerImage.len == 0
sameContents(pc.steps["test"].artifacts, @[]) sameContents(pc.steps["test"].artifacts, @[])
sameContents(pc.steps["test"].depends, @[]) sameContents(pc.steps["test"].depends, @[])
sameContents(pc.steps["test"].expectedEnv, @[]) sameContents(pc.steps["test"].expectedEnv, @[])

View File

@ -6,6 +6,7 @@ from langutils import sameContents
import ../testutil import ../testutil
import ../../../main/nim/strawbosspkg/configuration import ../../../main/nim/strawbosspkg/configuration
import ../../../main/nim/strawbosspkg/server import ../../../main/nim/strawbosspkg/server
import ../../../main/nim/strawbosspkg/version
let apiBase = "http://localhost:8180/api" let apiBase = "http://localhost:8180/api"
let cfgFilePath = "src/test/json/strawboss.config.json" let cfgFilePath = "src/test/json/strawboss.config.json"
@ -40,11 +41,11 @@ suite "strawboss server":
let tok = toJWT(cfg, session) let tok = toJWT(cfg, session)
check fromJWT(cfg, tok) == session check fromJWT(cfg, tok) == session
test "ping": test "version":
let resp = http.get(apiBase & "/ping") let resp = http.get(apiBase & "/version")
check: check:
resp.status.startsWith("200") resp.status.startsWith("200")
resp.body == "\"pong\"" resp.body == "\"strawboss v" & SB_VERSION & "\""
test "fail auth": test "fail auth":
let resp = http.post(apiBase & "/auth-token", let resp = http.post(apiBase & "/auth-token",

@ -1 +1 @@
Subproject commit 127be8f66fcc6d4d223acf56668d42ff9c37bfb0 Subproject commit ab883bd9602a1373347a23c8bee4ed28dd475aec

Binary file not shown.

View File

@ -1,7 +1,7 @@
# Package # Package
bin = @["strawboss"] bin = @["strawboss"]
version = "0.4.0" version = "0.5.1"
author = "Jonathan Bernard" author = "Jonathan Bernard"
description = "My personal continious integration worker." description = "My personal continious integration worker."
license = "MIT" license = "MIT"
@ -9,8 +9,8 @@ srcDir = "src/main/nim"
# Dependencies # Dependencies
requires @["nim >= 0.16.1", "docopt >= 0.6.5", "isaac >= 0.1.2", "tempfile", "jester", "bcrypt", requires @["nim >= 0.19.0", "docopt >= 0.6.8", "isaac >= 0.1.3", "tempfile", "jester >= 0.4.1", "bcrypt",
"untar", "uuids"] "untar", "uuids >= 0.1.10", "jwt"]
# Hacky to point to a specific hash. But there is some bug building in the # Hacky to point to a specific hash. But there is some bug building in the
# docker image we use to build the project with the next version. It adds an # docker image we use to build the project with the next version. It adds an
@ -18,10 +18,11 @@ requires @["nim >= 0.16.1", "docopt >= 0.6.5", "isaac >= 0.1.2", "tempfile", "je
# wrong and it tries to build against the 1.1 API even though the image only # wrong and it tries to build against the 1.1 API even though the image only
# has the 1.0 API. I'm crossing my fingers and hoping that our base image # has the 1.0 API. I'm crossing my fingers and hoping that our base image
# supports libssl 1.1 before I need to update this library. # supports libssl 1.1 before I need to update this library.
requires "https://github.com/yglukhov/nim-jwt#549aa1eb13b8ddc0c6861d15cc2cc5b52bcbef01" #requires "https://github.com/yglukhov/nim-jwt#549aa1eb13b8ddc0c6861d15cc2cc5b52bcbef01"
requires "https://git.jdb-labs.com/jdb/nim-lang-utils.git >= 0.3.0" requires "https://git.jdb-labs.com/jdb/nim-lang-utils.git >= 0.4.0"
requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git >= 0.3.1" requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git >= 0.6.0"
requires "https://git.jdb-labs.com/jdb/nim-time-utils.git >= 0.4.0"
# Tasks # Tasks
task functest, "Runs the functional test suite.": task functest, "Runs the functional test suite.":
@ -43,3 +44,8 @@ task test, "Runs both the unit and functional test suites.":
echo "\nRunning functional tests." echo "\nRunning functional tests."
echo "-------------------------" echo "-------------------------"
exec "src/test/nim/run_functional_tests" exec "src/test/nim/run_functional_tests"
task dist, "Creates distributable package.":
exec "nimble build"
mkdir "dist"
exec "cp strawboss strawboss.config.json example.json dist/."

View File

@ -1,24 +1,25 @@
{ {
"name": "strawboss", "name": "strawboss",
"containerImage": "nimlang/nim:0.19.0",
"steps": { "steps": {
"compile": { "compile": {
"artifacts": ["strawboss"], "artifacts": ["strawboss"],
"stepCmd": "docker run -v `pwd`:/usr/src/strawboss -w /usr/src/strawboss jdbernard/nim:0.17.2 nimble build" "stepCmd": "nimble build"
}, },
"unittest": { "unittest": {
"depends": ["compile"], "depends": ["compile"],
"stepCmd": "docker run -v `pwd`:/usr/src/strawboss -v $compile_DIR:/usr/build/strawboss -w /usr/src/strawboss -i jdbernard/nim:0.17.2 /bin/bash", "stepCmd": "/bin/bash",
"cmdInput": [ "cmdInput": [
"cp /usr/build/strawboss/strawboss .", "cp $compile_DIR/strawboss .",
"nimble install --depsOnly", "nimble install --depsOnly",
"nim c -r src/test/nim/run_unit_tests" "nim c -r src/test/nim/run_unit_tests"
] ]
}, },
"functest": { "functest": {
"depends": ["compile"], "depends": ["compile"],
"stepCmd": "docker run -v `pwd`:/usr/src/strawboss -v $compile_DIR:/usr/build/strawboss -w /usr/src/strawboss -i jdbernard/nim:0.17.2 /bin/bash", "stepCmd": "/bin/bash",
"cmdInput": [ "cmdInput": [
"cp /usr/build/strawboss/strawboss .", "cp $compile_DIR/strawboss .",
"nimble install --depsOnly", "nimble install --depsOnly",
"nim c -r src/test/nim/run_functional_tests" "nim c -r src/test/nim/run_functional_tests"
] ]
@ -26,9 +27,9 @@
"build": { "build": {
"artifacts": ["strawboss-$VERSION.zip"], "artifacts": ["strawboss-$VERSION.zip"],
"depends": ["compile", "unittest", "functest"], "depends": ["compile", "unittest", "functest"],
"stepCmd": "docker run -v `pwd`:/usr/src/strawboss -v $compile_DIR:/usr/build/strawboss -w /usr/src/strawboss -i jdbernard/nim:0.17.2 /bin/bash", "stepCmd": "/bin/bash",
"cmdInput": [ "cmdInput": [
"cp /usr/build/strawboss/strawboss .", "cp $compile_DIR/strawboss .",
"zip strawboss-$VERSION.zip strawboss strawboss.config.json example.json src/main/systemd/strawboss.service" "zip strawboss-$VERSION.zip strawboss strawboss.config.json example.json src/main/systemd/strawboss.service"
] ]
} }

11
test-spec.txt Normal file
View File

@ -0,0 +1,11 @@
Run a build. Look for:
- Run request archived
- Output logs archived with the run request
- Artifacts archived in the build-data directory.
- Configuration for that version archived in configurations directory.
- Status for that version archived in the status directory
Run the build again for the same project and build ref:
- Build should be skipped.
- Run request should be archived.