Compare commits

..

No commits in common. "master" and "0.4.0" have entirely different histories.

17 changed files with 178 additions and 352 deletions

View File

@ -3,12 +3,13 @@ import cliutils, docopt, os, sequtils, strutils, tempfile, uuids
import strawbosspkg/configuration import strawbosspkg/configuration
import strawbosspkg/core import strawbosspkg/core
import strawbosspkg/server import strawbosspkg/server
import strawbosspkg/version
let SB_VER = "0.4.0"
proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) = proc logProcOutput*(outMsg, errMsg: TaintedString, cmd: string) =
let prefix = if cmd.len > 0: cmd & ": " else: "" let prefix = if cmd != nil: cmd & ": " else: ""
if outMsg.len > 0: stdout.writeLine prefix & outMsg if outMsg != nil: stdout.writeLine prefix & outMsg
if errMsg.len > 0: stderr.writeLine prefix & errMsg if errMsg != nil: stderr.writeLine prefix & errMsg
when isMainModule: when isMainModule:
@ -25,7 +26,7 @@ Options
(strawboss.config.json). (strawboss.config.json).
""" """
let args = docopt(doc, version = "strawboss v" & SB_VERSION) let args = docopt(doc, version = "strawboss v" & SB_VER)
let cfgFile = if args["--config-file"]: $args["--config-file"] let cfgFile = if args["--config-file"]: $args["--config-file"]
else: "strawboss.config.json" else: "strawboss.config.json"
@ -49,7 +50,7 @@ Options
try: try:
if req.workspaceDir.len == 0: req.workspaceDir = mkdtemp() if req.workspaceDir.isNilOrEmpty: req.workspaceDir = mkdtemp()
let status = core.run(cfg, req, logProcOutput) let status = core.run(cfg, req, logProcOutput)
if status.state == BuildState.failed: raiseEx status.details if status.state == BuildState.failed: raiseEx status.details

View File

@ -1,5 +1,4 @@
import cliutils, logging, json, os, sequtils, strtabs, strutils, tables, times, import cliutils, logging, json, os, sequtils, strtabs, strutils, tables, times, uuids
unicode, uuids
from langutils import sameContents from langutils import sameContents
from typeinfo import toAny from typeinfo import toAny
@ -8,22 +7,23 @@ from strutils import parseEnum
const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz" const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz"
# Types # Types
#
type type
BuildState* {.pure.} = enum BuildState* {.pure.} = enum
complete, failed, queued, rejected, running, setup, stepComplete complete, failed, queued, rejected, running, setup, stepComplete
BuildStatus* = object BuildStatus* = object
runId*, details*, version*: string runId*, details*: string
state*: BuildState state*: BuildState
Step* = object Step* = object
containerImage*, name*, stepCmd*, workingDir*: string name*, stepCmd*, workingDir*: string
artifacts*, cmdInput*, depends*, expectedEnv*: seq[string] artifacts*, cmdInput*, depends*, expectedEnv*: seq[string]
dontSkip*: bool dontSkip*: bool
ProjectConfig* = object ProjectConfig* = object
containerImage*, name*, versionCmd*: string name*: string
versionCmd*: string
steps*: Table[string, Step] steps*: Table[string, Step]
ProjectDef* = object ProjectDef* = object
@ -33,7 +33,7 @@ type
RunRequest* = object RunRequest* = object
runId*: UUID runId*: UUID
projectName*, stepName*, buildRef*, workspaceDir*: string projectName*, stepName*, buildRef*, workspaceDir*: string
timestamp*: DateTime timestamp*: TimeInfo
forceRebuild*: bool forceRebuild*: bool
Run* = object Run* = object
@ -58,7 +58,6 @@ type
debug*: bool debug*: bool
logLevel*: Level logLevel*: Level
pathToExe*: string pathToExe*: string
port*: int
projects*: seq[ProjectDef] projects*: seq[ProjectDef]
pwdCost*: int8 pwdCost*: int8
users*: seq[UserRef] users*: seq[UserRef]
@ -84,7 +83,6 @@ proc `==`*(a, b: StrawBossConfig): bool =
a.buildDataDir == b.buildDataDir and a.buildDataDir == b.buildDataDir and
a.authSecret == b.authSecret and a.authSecret == b.authSecret and
a.pwdCost == b.pwdCost and a.pwdCost == b.pwdCost and
a.port == b.port and
a.maintenancePeriod == b.maintenancePeriod and a.maintenancePeriod == b.maintenancePeriod and
a.logLevel == b.logLevel and a.logLevel == b.logLevel and
sameContents(a.users, b.users) and sameContents(a.users, b.users) and
@ -120,7 +118,7 @@ proc getOrFail(n: JsonNode, key: string, objName: string = ""): JsonNode =
# Configuration parsing code # Configuration parsing code
proc parseLogLevel*(level: string): Level = proc parseLogLevel*(level: string): Level =
let lvlStr = "lvl" & toUpperAscii(level[0]) & level[1..^1] let lvlStr = "lvl" & toUpper(level[0]) & level[1..^1]
result = parseEnum[Level](lvlStr) result = parseEnum[Level](lvlStr)
proc parseProjectDef*(pJson: JsonNode): ProjectDef = proc parseProjectDef*(pJson: JsonNode): ProjectDef =
@ -145,11 +143,10 @@ proc parseStrawBossConfig*(jsonCfg: JsonNode): StrawBossConfig =
result = StrawBossConfig( result = StrawBossConfig(
buildDataDir: jsonCfg.getIfExists("buildDataDir").getStr("build-data"), buildDataDir: jsonCfg.getIfExists("buildDataDir").getStr("build-data"),
authSecret: jsonCfg.getOrFail("authSecret", "strawboss config").getStr, authSecret: jsonCfg.getOrFail("authSecret", "strawboss config").getStr,
debug: jsonCfg.getIfExists("debug").getBool(false), debug: jsonCfg.getIfExists("debug").getBVal(false),
port: int(jsonCfg.getIfExists("port").getInt(8180)), pwdCost: int8(jsonCfg.getOrFail("pwdCost", "strawboss config").getNum),
pwdCost: int8(jsonCfg.getOrFail("pwdCost", "strawboss config").getInt),
projects: jsonCfg.getIfExists("projects").getElems.mapIt(parseProjectDef(it)), projects: jsonCfg.getIfExists("projects").getElems.mapIt(parseProjectDef(it)),
maintenancePeriod: int(jsonCfg.getIfExists("maintenancePeriod").getInt(10000)), maintenancePeriod: int(jsonCfg.getIfExists("maintenancePeriod").getNum(10000)),
logLevel: parseLogLevel(jsonCfg.getIfExists("logLevel").getStr("info")), logLevel: parseLogLevel(jsonCfg.getIfExists("logLevel").getStr("info")),
users: users) users: users)
@ -173,15 +170,14 @@ proc loadProjectConfig*(cfgFile: string): ProjectConfig =
var steps = initTable[string, Step]() var steps = initTable[string, Step]()
for sName, pJson in jsonCfg.getOrFail("steps", "project configuration").getFields: for sName, pJson in jsonCfg.getOrFail("steps", "project configuration").getFields:
steps[sName] = Step( steps[sName] = Step(
name: sName, name: sName,
workingDir: pJson.getIfExists("workingDir").getStr("."), workingDir: pJson.getIfExists("workingDir").getStr("."),
stepCmd: pJson.getIfExists("stepCmd").getStr("NOT GIVEN"), stepCmd: pJson.getIfExists("stepCmd").getStr("NOT GIVEN"),
depends: pJson.getIfExists("depends").getElems.mapIt(it.getStr), depends: pJson.getIfExists("depends").getElems.mapIt(it.getStr),
artifacts: pJson.getIfExists("artifacts").getElems.mapIt(it.getStr), artifacts: pJson.getIfExists("artifacts").getElems.mapIt(it.getStr),
cmdInput: pJson.getIfExists("cmdInput").getElems.mapIt(it.getStr), cmdInput: pJson.getIfExists("cmdInput").getElems.mapIt(it.getStr),
expectedEnv: pJson.getIfExists("expectedEnv").getElems.mapIt(it.getStr), expectedEnv: pJson.getIfExists("expectedEnv").getElems.mapIt(it.getStr),
containerImage: pJson.getIfExists("containerImage").getStr(""), dontSkip: pJson.getIfExists("dontSkip").getBVal(false))
dontSkip: pJson.getIfExists("dontSkip").getBool(false))
# cmdInput and stepCmd are related, so we have a conditional defaulting. # cmdInput and stepCmd are related, so we have a conditional defaulting.
# Four possibilities: # Four possibilities:
@ -198,7 +194,6 @@ proc loadProjectConfig*(cfgFile: string): ProjectConfig =
result = ProjectConfig( result = ProjectConfig(
name: jsonCfg.getOrFail("name", "project configuration").getStr, name: jsonCfg.getOrFail("name", "project configuration").getStr,
containerImage: jsonCfg.getIfExists("containerImage").getStr(""),
versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"), versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"),
steps: steps) steps: steps)
@ -222,7 +217,7 @@ proc parseRunRequest*(reqJson: JsonNode): RunRequest =
buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr, buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr,
workspaceDir: reqJson.getOrFail("workspaceDir", "RunRequest").getStr, workspaceDir: reqJson.getOrFail("workspaceDir", "RunRequest").getStr,
timestamp: times.parse(reqJson.getOrFail("timestamp", "RunRequest").getStr, ISO_TIME_FORMAT), timestamp: times.parse(reqJson.getOrFail("timestamp", "RunRequest").getStr, ISO_TIME_FORMAT),
forceRebuild: reqJson.getOrFail("forceRebuild", "RunRequest").getBool) forceRebuild: reqJson.getOrFail("forceRebuild", "RunRequest").getBVal)
proc loadRunRequest*(reqFilePath: string): RunRequest = proc loadRunRequest*(reqFilePath: string): RunRequest =
if not existsFile(reqFilePath): if not existsFile(reqFilePath):
@ -264,9 +259,6 @@ proc `%`*(s: Step): JsonNode =
"expectedEnv": s.expectedEnv, "expectedEnv": s.expectedEnv,
"dontSkip": s.dontSkip } "dontSkip": s.dontSkip }
if s.containerImage.len > 0:
result["containerImage"] = %s.containerImage
proc `%`*(p: ProjectConfig): JsonNode = proc `%`*(p: ProjectConfig): JsonNode =
result = %* { result = %* {
"name": p.name, "name": p.name,
@ -276,9 +268,6 @@ proc `%`*(p: ProjectConfig): JsonNode =
for name, step in p.steps: for name, step in p.steps:
result["steps"][name] = %step result["steps"][name] = %step
if p.containerImage.len > 0:
result["containerImage"] = %p.containerImage
proc `%`*(req: RunRequest): JsonNode = proc `%`*(req: RunRequest): JsonNode =
result = %* { result = %* {
"runId": $(req.runId), "runId": $(req.runId),
@ -299,11 +288,10 @@ proc `%`*(cfg: StrawBossConfig): JsonNode =
"buildDataDir": cfg.buildDataDir, "buildDataDir": cfg.buildDataDir,
"authSecret": cfg.authSecret, "authSecret": cfg.authSecret,
"debug": cfg.debug, "debug": cfg.debug,
"port": cfg.port,
"projects": %cfg.projects, "projects": %cfg.projects,
"pwdCost": cfg.pwdCost, "pwdCost": cfg.pwdCost,
"maintenancePeriod": cfg.maintenancePeriod, "maintenancePeriod": cfg.maintenancePeriod,
"logLevel": toLowerAscii(($cfg.logLevel)[3]) & ($cfg.logLevel)[4..^1], "logLevel": toLower(($cfg.logLevel)[3]) & ($cfg.logLevel)[4..^1],
"users": %cfg.users } "users": %cfg.users }
proc `%`*(run: Run): JsonNode = proc `%`*(run: Run): JsonNode =

View File

@ -1,5 +1,5 @@
import cliutils, logging, json, os, ospaths, osproc, sequtils, streams, import cliutils, logging, json, os, ospaths, osproc, sequtils, streams,
strtabs, strutils, tables, tempfile, times, uuids strtabs, strutils, tables, times, uuids
import ./configuration import ./configuration
import nre except toSeq import nre except toSeq
@ -49,64 +49,20 @@ proc newCopy(w: Workspace): Workspace =
step: w.step, step: w.step,
version: w.version) version: w.version)
const WKSP_ROOT = "/strawboss/wksp" # Logging wrappers around
const ARTIFACTS_ROOT = "/strawboss/artifacts"
proc execWithOutput(wksp: Workspace, cmd, workingDir: string,
args: openarray[string], env: StringTableRef,
options: set[ProcessOption] = {poUsePath},
msgCB: HandleProcMsgCB = nil):
tuple[output: TaintedString, error: TaintedString, exitCode: int]
{.tags: [ExecIOEffect, ReadIOEffect, RootEffect] .} =
# Look for a container image to use
let containerImage =
if wksp.step.containerImage.len > 0: wksp.step.containerImage
else: wksp.project.containerImage
if containerImage.len == 0:
return execWithOutput(cmd, workingDir, args, env, options, msgCB)
var fullEnv = newStringTable(modeCaseSensitive)
for k,v in env: fullEnv[k] = v
var fullArgs = @["run", "-w", WKSP_ROOT, "-v", wksp.dir & ":" & WKSP_ROOT ]
if wksp.step.name.len == 0:
for depStep in wksp.step.depends:
fullArgs.add(["-v", ARTIFACTS_ROOT / depStep])
fullEnv[depStep & "_DIR"] = ARTIFACTS_ROOT / depStep
let envFile = mkstemp().name
writeFile(envFile, toSeq(fullEnv.pairs()).mapIt(it[0] & "=" & it[1]).join("\n"))
fullArgs.add(["--env-file", envFile])
fullArgs.add(containerImage)
fullArgs.add(cmd)
echo "Executing docker command: \n\t" & "docker " & $(fullArgs & @args)
return execWithOutput("docker", wksp.dir, fullArgs & @args, fullEnv, options, msgCB)
proc exec(w: Workspace, cmd, workingDir: string, args: openarray[string],
env: StringTableRef, options: set[ProcessOption] = {poUsePath},
msgCB: HandleProcMsgCB = nil): int
{.tags: [ExecIOEffect, ReadIOEffect, RootEffect] .} =
return execWithOutput(w, cmd, workingDir, args, env, options, msgCB)[2]
# Utility methods for Workspace activities # Utility methods for Workspace activities
proc sendStatusMsg(oh: HandleProcMsgCB, status: BuildStatus): void = proc sendStatusMsg(oh: HandleProcMsgCB, status: BuildStatus): void =
if not oh.isNil: if not oh.isNil:
oh.sendMsg($status.state & ": " & status.details, "", "strawboss") oh.sendMsg($status.state & ": " & status.details, nil, "strawboss")
proc sendMsg(w: Workspace, msg: TaintedString): void = proc sendMsg(w: Workspace, msg: TaintedString): void =
w.outputHandler.sendMsg(msg, "", "strawboss") w.outputHandler.sendMsg(msg, nil, "strawboss")
proc sendMsg(w: Workspace, l: Level, msg: TaintedString): void = proc sendMsg(w: Workspace, l: Level, msg: TaintedString): void =
if l >= w.logLevel: w.sendMsg(msg) if l >= w.logLevel: w.sendMsg(msg)
proc sendErrMsg(w: Workspace, msg: TaintedString): void = proc sendErrMsg(w: Workspace, msg: TaintedString): void =
w.outputHandler.sendMsg("", msg, "strawboss") w.outputHandler.sendMsg(nil, msg, "strawboss")
proc sendErrMsg(w: Workspace, l: Level, msg: TaintedString): void = proc sendErrMsg(w: Workspace, l: Level, msg: TaintedString): void =
if l >= w.logLevel: w.sendErrMsg(msg) if l >= w.logLevel: w.sendErrMsg(msg)
@ -123,17 +79,14 @@ proc publishStatus(wksp: Workspace, state: BuildState, details: string): void =
## Update the status for a Workspace and publish this status to the ## Update the status for a Workspace and publish this status to the
## Workspace's status file and any output message handlers. ## Workspace's status file and any output message handlers.
wksp.status = BuildStatus( wksp.status = BuildStatus(
runId: $wksp.runRequest.runId, runId: $wksp.runRequest.runId, state: state, details: details)
state: state,
details: details,
version: wksp.version)
# Write to our run directory, and to our version status # Write to our run directory, and to our version status
writeFile(wksp.buildDataDir / "runs" / writeFile(wksp.buildDataDir / "runs" /
$wksp.runRequest.runId & ".status.json", $wksp.status) $wksp.runRequest.runId & ".status.json", $wksp.status)
# If we have our step we can save status to the step status # If we have our step we can save status to the step status
if wksp.step.name.len > 0: if not wksp.step.name.isNilOrEmpty():
let stepStatusDir = wksp.buildDataDir / "status" / wksp.step.name let stepStatusDir = wksp.buildDataDir / "status" / wksp.step.name
if not existsDir(stepStatusDir): createDir(stepStatusDir) if not existsDir(stepStatusDir): createDir(stepStatusDir)
writeFile(stepStatusDir / wksp.version & ".json", $wksp.status) writeFile(stepStatusDir / wksp.version & ".json", $wksp.status)
@ -280,7 +233,7 @@ proc getProjectConfig*(cfg: StrawBossConfig,
# If they didn't give us a version, let try to figure out what is the latest one. # If they didn't give us a version, let try to figure out what is the latest one.
var confFilePath: string var confFilePath: string
if version.len == 0: if version.isNilOrEmpty:
let candidatePaths = filesMatching( let candidatePaths = filesMatching(
cfg.buildDataDir / project.name / "configurations/*.json") cfg.buildDataDir / project.name / "configurations/*.json")
@ -473,8 +426,7 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
result = BuildStatus( result = BuildStatus(
runId: $req.runId, runId: $req.runId,
state: BuildState.setup, state: BuildState.setup,
details: "initializing build workspace", details: "initializing build workspace")
version: "")
outputHandler.sendStatusMsg(result) outputHandler.sendStatusMsg(result)
var wksp: Workspace var wksp: Workspace
@ -507,7 +459,7 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
wksp = Workspace( wksp = Workspace(
buildDataDir: cfg.buildDataDir / projectDef.name, buildDataDir: cfg.buildDataDir / projectDef.name,
buildRef: buildRef:
if req.buildRef.len > 0: req.buildRef if req.buildRef != nil and req.buildRef.len > 0: req.buildRef
else: projectDef.defaultBranch, else: projectDef.defaultBranch,
dir: req.workspaceDir, dir: req.workspaceDir,
env: env, env: env,
@ -519,12 +471,12 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
runRequest: req, runRequest: req,
status: result, status: result,
step: Step(), step: Step(),
version: "") version: nil)
except: except:
when not defined(release): echo getCurrentException().getStackTrace() when not defined(release): echo getCurrentException().getStackTrace()
result = BuildStatus(runId: $req.runId, state: BuildState.failed, result = BuildStatus(runId: $req.runId, state: BuildState.failed,
details: getCurrentExceptionMsg(), version: "") details: getCurrentExceptionMsg())
try: outputHandler.sendStatusMsg(result) try: outputHandler.sendStatusMsg(result)
except: discard "" except: discard ""
return return
@ -562,8 +514,7 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
wksp.publishStatus(BuildState.failed, msg) wksp.publishStatus(BuildState.failed, msg)
result = wksp.status result = wksp.status
except: except:
result = BuildStatus(runId: $req.runId, state: BuildState.failed, result = BuildStatus(runId: $req.runId, state: BuildState.failed, details: msg)
details: msg, version: "")
try: outputHandler.sendStatusMsg(result) try: outputHandler.sendStatusMsg(result)
except: discard "" except: discard ""
@ -594,8 +545,7 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
let queuedStatus = BuildStatus( let queuedStatus = BuildStatus(
runId: $req.runId, runId: $req.runId,
state: BuildState.queued, state: BuildState.queued,
details: "request queued for execution", details: "request queued for execution")
version: "")
writeFile(statusFile, $queuedStatus) writeFile(statusFile, $queuedStatus)
var args = @["run", reqFile, "-c", cfg.filePath] var args = @["run", reqFile, "-c", cfg.filePath]
@ -612,7 +562,6 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
let exMsg = "run request rejected: " & getCurrentExceptionMsg() let exMsg = "run request rejected: " & getCurrentExceptionMsg()
try: try:
writeFile(statusFile, writeFile(statusFile,
$(BuildStatus(runId: $req.runId, state: BuildState.rejected, $(BuildStatus(runId: $req.runId, state: BuildState.rejected, details: exMsg)))
details: exMsg, version: "")))
except: discard "" except: discard ""
raiseEx exMsg raiseEx exMsg

View File

@ -1,13 +1,12 @@
import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, md5, import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, md5,
options, os, osproc, sequtils, strutils, tempfile, times, unittest, uuids os, osproc, sequtils, strutils, tempfile, times, unittest, uuids
from mimetypes import getMimeType from mimetypes import getMimeType
from asyncfile import openAsync, readToStream, close from asyncfile import openAsync, readToStream, close
from asyncnet import send from asyncnet import send
from re import re, find from re import re, find
from timeutils import trimNanoSec
import ./configuration, ./core, ./version import ./configuration, ./core
type type
Session = object Session = object
@ -20,38 +19,24 @@ const JSON = "application/json"
proc newSession*(user: UserRef): Session = proc newSession*(user: UserRef): Session =
result = Session( result = Session(
user: user, user: user,
issuedAt: getTime().local.trimNanoSec.toTime, issuedAt: getTime(),
expires: daysForward(7).trimNanoSec.toTime) expires: daysForward(7).toTime())
template halt(code: HttpCode, proc buildJson(resp: Response, code: HttpCode, details: string = ""): void =
headers: RawHeaders, resp.data[0] = CallbackAction.TCActionSend
content: string): typed = resp.data[1] = code
## Immediately replies with the specified request. This means any further resp.data[2]["Content-Type"] = JSON
## code will not be executed after calling this template in the current resp.data[3] = $(%* {
## route. "statusCode": code.int,
bind TCActionSend, newHttpHeaders "status": $code,
result[0] = CallbackAction.TCActionSend "details": details
result[1] = code })
result[2] = some(headers)
result[3] = content
result.matched = true
break allRoutes
template jsonResp(code: HttpCode, details: string = "", headers: RawHeaders = @{:} ) = # Work-around for weirdness trying to use resp(Http500... in exception blocks
halt( proc build500Json(resp: Response, ex: ref Exception, msg: string): void =
code,
headers & @{"Content-Type": JSON},
$(%* {
"statusCode": code.int,
"status": $code,
"details": details
})
)
template json500Resp(ex: ref Exception, details: string = ""): void =
when not defined(release): debug ex.getStackTrace() when not defined(release): debug ex.getStackTrace()
error details & ":\n" & ex.msg error msg & ":\n" & ex.msg
jsonResp(Http500) resp.buildJson(Http500)
proc toJWT*(cfg: StrawBossConfig, session: Session): string = proc toJWT*(cfg: StrawBossConfig, session: Session): string =
## Make a JST token for this session. ## Make a JST token for this session.
@ -59,8 +44,8 @@ proc toJWT*(cfg: StrawBossConfig, session: Session): string =
header: JOSEHeader(alg: HS256, typ: "jwt"), header: JOSEHeader(alg: HS256, typ: "jwt"),
claims: toClaims(%*{ claims: toClaims(%*{
"sub": session.user.name, "sub": session.user.name,
"iat": session.issuedAt.toUnix.int, "iat": session.issuedAt.toSeconds().int,
"exp": session.expires.toUnix.int })) "exp": session.expires.toSeconds().int }))
jwt.sign(cfg.authSecret) jwt.sign(cfg.authSecret)
result = $jwt result = $jwt
@ -79,8 +64,8 @@ proc fromJWT*(cfg: StrawBossConfig, strTok: string): Session =
result = Session( result = Session(
user: users[0], user: users[0],
issuedAt: fromUnix(jwt.claims["iat"].node.num), issuedAt: fromSeconds(jwt.claims["iat"].node.num),
expires: fromUnix(jwt.claims["exp"].node.num)) expires: fromSeconds(jwt.claims["exp"].node.num))
proc extractSession(cfg: StrawBossConfig, request: Request): Session = proc extractSession(cfg: StrawBossConfig, request: Request): Session =
## Helper to extract a session from a reqest. ## Helper to extract a session from a reqest.
@ -108,7 +93,7 @@ proc makeAuthToken*(cfg: StrawBossConfig, uname, pwd: string): string =
## Given a username and pwd, validate the combination and generate a JWT ## Given a username and pwd, validate the combination and generate a JWT
## token string. ## token string.
if uname.len == 0 or pwd.len == 0: if uname == nil or pwd == nil:
raiseEx "fields 'username' and 'password' required" raiseEx "fields 'username' and 'password' required"
# find the user record # find the user record
@ -130,7 +115,7 @@ proc makeApiKey*(cfg: StrawBossConfig, uname: string): string =
## function for an administrator to setup a unsupervised account (git access ## function for an administrator to setup a unsupervised account (git access
## for example). ## for example).
if uname.len == 0: raiseEx "no username given" if uname == nil: raiseEx "no username given"
# find the user record # find the user record
let users = cfg.users.filterIt(it.name == uname) let users = cfg.users.filterIt(it.name == uname)
@ -145,15 +130,21 @@ proc makeApiKey*(cfg: StrawBossConfig, uname: string): string =
template checkAuth() = template checkAuth() =
## Check this request for authentication and authorization information. ## Check this request for authentication and authorization information.
## Injects the session into the running context. If the request is not ## Injects two variables into the running context: the session and authed:
## authorized, this template returns an appropriate 401 response. ## true if the request is authorized, false otherwise. If the request is not
## authorized, this template sets up the 401 response correctly. The calling
## context needs only to return from the route.
var session {.inject.}: Session var session {.inject.}: Session
var authed {.inject.} = false
try: session = extractSession(cfg, request) try:
session = extractSession(cfg, request)
authed = true
except: except:
debug "Auth failed: " & getCurrentExceptionMsg() debug "Auth failed: " & getCurrentExceptionMsg()
jsonResp(Http401, "Unauthorized", @{"WWW-Authenticate": "Bearer"}) response.data[2]["WWW-Authenticate"] = "Bearer"
response.buildJson(Http401)
proc start*(cfg: StrawBossConfig): void = proc start*(cfg: StrawBossConfig): void =
@ -161,13 +152,13 @@ proc start*(cfg: StrawBossConfig): void =
var workers: seq[Worker] = @[] var workers: seq[Worker] = @[]
settings: settings:
port = Port(cfg.port) port = Port(8180)
appName = "/api" appName = "/api"
routes: routes:
get "/version": get "/ping":
resp($(%("strawboss v" & SB_VERSION)), JSON) resp($(%"pong"), JSON)
post "/auth-token": post "/auth-token":
var uname, pwd: string var uname, pwd: string
@ -175,39 +166,37 @@ proc start*(cfg: StrawBossConfig): void =
let jsonBody = parseJson(request.body) let jsonBody = parseJson(request.body)
uname = jsonBody["username"].getStr uname = jsonBody["username"].getStr
pwd = jsonBody["password"].getStr pwd = jsonBody["password"].getStr
except: jsonResp(Http400) except: response.buildJson(Http400); return true
try: try:
let authToken = makeAuthToken(cfg, uname, pwd) let authToken = makeAuthToken(cfg, uname, pwd)
resp($(%authToken), JSON) resp($(%authToken), JSON)
except: except: response.buildJson(Http401, getCurrentExceptionMsg()); return true
jsonResp(Http401, getCurrentExceptionMsg())
if ctx.cfg.debug: echo getStackTrace()
get "/verify-auth": get "/verify-auth":
checkAuth() checkAuth(); if not authed: return true
resp(Http200, $(%*{ "username": session.user.name }), JSON) resp(Http200, $(%*{ "username": session.user.name }), JSON)
get "/projects": get "/projects":
## List project summaries (ProjectDefs only) ## List project summaries (ProjectDefs only)
checkAuth() checkAuth(); if not authed: return true
resp($(%cfg.projects), JSON) resp($(%cfg.projects), JSON)
post "/projects": post "/projects":
## Create a new project definition ## Create a new project definition
checkAuth() checkAuth(); if not authed: return true
# TODO # TODO
jsonResp(Http501) response.buildJson(Http501); return true
get "/project/@projectName": get "/project/@projectName":
## Return a project's configuration, as well as it's versions. ## Return a project's configuration, as well as it's versions.
checkAuth() checkAuth(); if not authed: return true
# Make sure we know about that project # Make sure we know about that project
var projDef: ProjectDef var projDef: ProjectDef
@ -215,10 +204,11 @@ proc start*(cfg: StrawBossConfig): void =
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: except NotFoundException:
jsonResp(Http404, getCurrentExceptionMsg()) response.buildJson(Http404, getCurrentExceptionMsg())
except: except:
let msg = "unable to load project definition for project " & @"projectName" response.build500Json(getCurrentException(),
json500Resp(getCurrentException(), msg) "unable to load project definition for project " & @"projectName")
return true
var projConf: ProjectConfig var projConf: ProjectConfig
try: projConf = getProjectConfig(cfg, @"projectName", "") try: projConf = getProjectConfig(cfg, @"projectName", "")
@ -227,7 +217,7 @@ proc start*(cfg: StrawBossConfig): void =
let respJson = newJObject() let respJson = newJObject()
respJson["definition"] = %projDef respJson["definition"] = %projDef
respJson["versions"] = %listVersions(cfg, @"projectName") respJson["versions"] = %listVersions(cfg, @"projectName")
if projConf.name.len > 0: if not projConf.name.isNil:
respJson["latestConfig"] = %projConf respJson["latestConfig"] = %projConf
resp(pretty(respJson), JSON) resp(pretty(respJson), JSON)
@ -235,89 +225,97 @@ proc start*(cfg: StrawBossConfig): void =
get "/project/@projectName/versions": get "/project/@projectName/versions":
## Get a list of all versions that we have built ## Get a list of all versions that we have built
checkAuth() checkAuth(); if not authed: return true
try: resp($(%listVersions(cfg, @"projectName")), JSON) try: resp($(%listVersions(cfg, @"projectName")), JSON)
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: except NotFoundException:
jsonResp(Http404, getCurrentExceptionMsg()) response.buildJson(Http404, getCurrentExceptionMsg())
except: except:
let msg = "unable to list versions for project " & @"projectName" response.build500Json(getCurrentException(),
json500Resp(getCurrentException(), msg) "unable to list versions for project " & @"projectName")
return true
get "/project/@projectName/version/@version?": get "/project/@projectName/version/@version?":
## Get a detailed project record including step definitions (ProjectConfig). ## Get a detailed project record including step definitions (ProjectConfig).
checkAuth() checkAuth(); if not authed: return true
# Make sure we know about that project # Make sure we know about that project
try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON) try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON)
except: jsonResp(Http404, getCurrentExceptionMsg()) except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
get "/project/@projectName/runs": get "/project/@projectName/runs":
## List all runs ## List all runs
checkAuth() checkAuth(); if not authed: return true
try: resp($(%listRuns(cfg, @"projectName")), JSON) try: resp($(%listRuns(cfg, @"projectName")), JSON)
except: jsonResp(Http404, getCurrentExceptionMsg()) except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
get "/project/@projectName/runs/active": get "/project/@projectName/runs/active":
## List all currently active runs ## List all currently active runs
checkAuth() checkAuth(); if not authed: return true
var details = ""
try: try:
let activeRuns = workers let activeRuns = workers
.filterIt(it.process.running and it.projectName == @"projectName") .filterIt(it.process.running and it.projectName == @"projectName")
.mapIt(cfg.getRun(@"projectName", $it.runId)); .mapIt(cfg.getRun(@"projecName", $it.runId));
resp($(%activeRuns), JSON) resp($(%activeRuns), JSON)
except NotFoundException:
jsonResp(Http404, getCurrentExceptionMsg())
except: except:
json500Resp(getCurrentException(), "problem loading active runs") try: raise getCurrentException()
except NotFoundException:
response.buildJson(Http404, getCurrentExceptionMsg())
except:
response.build500Json(getCurrentException(),
"problem loading active runs")
return true
get "/project/@projectName/run/@runId": get "/project/@projectName/run/@runId":
## Details for a specific run ## Details for a specific run
checkAuth() checkAuth(); if not authed: return true
# Make sure we know about that project # Make sure we know about that project
try: discard cfg.getProject(@"projectName") try: discard cfg.getProject(@"projectName")
except: jsonResp(Http404, getCurrentExceptionMsg()) except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
if not existsRun(cfg, @"projectName", @"runId"): if not existsRun(cfg, @"projectName", @"runId"):
jsonResp(Http404, "no such run for project") response.buildJson(Http404, "no such run for project"); return true
try: resp($getRun(cfg, @"projectName", @"runId"), JSON) try: resp($getRun(cfg, @"projectName", @"runId"), JSON)
except: except:
json500Resp(getCurrentException(), response.build500Json(getCurrentException(),
"unable to load run details for project " & @"projectName" & "unable to load run details for project " & @"projectName" &
" run " & @"runId") " run " & @"runId")
return true
get "/project/@projectName/run/@runId/logs": get "/project/@projectName/run/@runId/logs":
## Get logs from a specific run ## Get logs from a specific run
checkAuth() checkAuth(); if not authed: return true
try: discard cfg.getProject(@"projectName") try: discard cfg.getProject(@"projectName")
except: except:
jsonResp(Http404, getCurrentExceptionMsg()) response.buildJson(Http404, getCurrentExceptionMsg())
return true
if not existsRun(cfg, @"projectName", @"runId"): if not existsRun(cfg, @"projectName", @"runId"):
jsonResp(Http404, "no such run for project") response.buildJson(Http404, "no such run for project")
return true
try: resp($getLogs(cfg, @"projectName", @"runId")) try: resp($getLogs(cfg, @"projectName", @"runId"))
except: except:
json500Resp(getCurrentException(), response.build500Json(getCurrentException(),
"unable to load run logs for " & @"projectName" & " run " & @"runId") "unable to load run logs for " & @"projectName" & " run " & @"runId")
return true
get "/project/@projectName/step/@stepName/artifacts/@version": get "/project/@projectName/step/@stepName/artifacts/@version":
## Get the list of artifacts that were built for ## Get the list of artifacts that were built for
checkAuth() checkAuth(); if not authed: return true
debug "Matched artifacts list request: " & $(%*{ debug "Matched artifacts list request: " & $(%*{
"project": @"projectName", "project": @"projectName",
@ -329,15 +327,16 @@ proc start*(cfg: StrawBossConfig): void =
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: except NotFoundException:
jsonResp(Http404, getCurrentExceptionMsg()) response.buildJson(Http404, getCurrentExceptionMsg())
except: except:
json500Resp(getCurrentException(), "unable to list artifacts for " & response.build500Json(getCurrentException(), "unable to list artifacts for " &
@"projectName" & ":" & @"stepName" & "@" & @"buildRef") @"projectName" & ":" & @"stepName" & "@" & @"buildRef")
return true
get "/project/@projectName/step/@stepName/artifact/@version/@artifactName": get "/project/@projectName/step/@stepName/artifact/@version/@artifactName":
## Get a specific artifact that was built. ## Get a specific artifact that was built.
checkAuth() checkAuth(); if not authed: return true
var artifactPath: string var artifactPath: string
try: artifactPath = getArtifactPath(cfg, try: artifactPath = getArtifactPath(cfg,
@ -345,12 +344,11 @@ proc start*(cfg: StrawBossConfig): void =
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: except NotFoundException:
jsonResp(Http404, getCurrentExceptionMsg()) response.buildJson(Http404, getCurrentExceptionMsg())
except: except:
json500Resp(getCurrentException(), "unable to check artifact path for " & response.build500Json(getCurrentException(), "unable to check artifact path for " &
@"projectName" & ":" & @"stepName" & "@" & @"version") @"projectName" & ":" & @"stepName" & "@" & @"version")
return true
enableRawMode
debug "Preparing: " & artifactPath debug "Preparing: " & artifactPath
let fileSize = getFileSize(artifactPath) let fileSize = getFileSize(artifactPath)
@ -363,19 +361,19 @@ proc start*(cfg: StrawBossConfig): void =
# If the user has a cached version of this file and it matches our # If the user has a cached version of this file and it matches our
# version, let them use it # version, let them use it
if request.headers.hasKey("If-None-Match") and request.headers["If-None-Match"] == hashed: if request.headers.hasKey("If-None-Match") and request.headers["If-None-Match"] == hashed:
resp(Http304) resp(Http304, [], "")
else: else:
resp(Http200, [ resp(Http200, [
("Content-Disposition", "; filename=\"" & @"artifactName" & "\""), ("Content-Disposition", "; filename=\"" & @"artifactName" & "\""),
("Content-Type", mimetype), ("Content-Type", mimetype),
("ETag", hashed )], file) ("ETag", hashed )], file)
else: else:
let headers = @{ let headers = {
"Content-Disposition": "; filename=\"" & @"artifactName" & "\"", "Content-Disposition": "; filename=\"" & @"artifactName" & "\"",
"Content-Type": mimetype, "Content-Type": mimetype,
"Content-Length": $fileSize "Content-Length": $fileSize
} }.newStringTable
request.sendHeaders(Http200, headers) await response.sendHeaders(Http200, headers)
var fileStream = newFutureStream[string]("sendStaticIfExists") var fileStream = newFutureStream[string]("sendStaticIfExists")
var file = openAsync(artifactPath, fmRead) var file = openAsync(artifactPath, fmRead)
@ -386,22 +384,25 @@ proc start*(cfg: StrawBossConfig): void =
# `bodyStream` has been written to the file. # `bodyStream` has been written to the file.
while true: while true:
let (hasValue, value) = await fileStream.read() let (hasValue, value) = await fileStream.read()
if hasValue: request.send(value) if hasValue:
else: break await response.client.send(value)
else:
break
file.close() file.close()
get "/project/@projectName/step/@stepName/status/@buildRef": get "/project/@projectName/step/@stepName/status/@buildRef":
## Get detailed information about the status of a step (assuming it has been built) ## Get detailed information about the status of a step (assuming it has been built)
checkAuth() checkAuth(); if not authed: return true
try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON) try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON)
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: jsonResp(Http404, getCurrentExceptionMsg()) except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg())
except: except:
json500Resp(getCurrentException(), "unable to load the build state for " & response.build500Json(getCurrentException(), "unable to load the build state for " &
@"projectName" & ":" & @"stepName" & "@" & @"buildRef") @"projectName" & ":" & @"stepName" & "@" & @"buildRef")
return true
#get "/project/@projectName/step/@stepName/status/@buildRef.svg": #get "/project/@projectName/step/@stepName/status/@buildRef.svg":
## Get an image representing the status of a build ## Get an image representing the status of a build
@ -412,14 +413,14 @@ proc start*(cfg: StrawBossConfig): void =
post "/project/@projectName/step/@stepName/run/@buildRef?": post "/project/@projectName/step/@stepName/run/@buildRef?":
# Kick off a run # Kick off a run
checkAuth() checkAuth(); if not authed: return true
let runRequest = RunRequest( let runRequest = RunRequest(
runId: genUUID(), runId: genUUID(),
projectName: @"projectName", projectName: @"projectName",
stepName: @"stepName", stepName: @"stepName",
buildRef: if @"buildRef" != "": @"buildRef" else: "", buildRef: if @"buildRef" != "": @"buildRef" else: nil,
timestamp: getTime().local, timestamp: getLocalTime(getTime()),
forceRebuild: false) # TODO support this with optional query params forceRebuild: false) # TODO support this with optional query params
# TODO: instead of immediately spawning a worker, add the request to a # TODO: instead of immediately spawning a worker, add the request to a
@ -435,11 +436,13 @@ proc start*(cfg: StrawBossConfig): void =
status: status), JSON) status: status), JSON)
except: except:
try: raise getCurrentException() try: raise getCurrentException()
except NotFoundException: jsonResp(Http404, getCurrentExceptionMsg()) except NotFoundException:
except: jsonResp(Http400, getCurrentExceptionMsg()) response.buildJson(Http404, getCurrentExceptionMsg())
except: response.buildJson(Http400, getCurrentExceptionMsg())
return true
post "/service/debug/stop": post "/service/debug/stop":
if not cfg.debug: jsonResp(Http404) if not cfg.debug: response.buildJson(Http404); return true
else: else:
let shutdownFut = sleepAsync(100) let shutdownFut = sleepAsync(100)
shutdownFut.callback = proc(): void = complete(stopFuture) shutdownFut.callback = proc(): void = complete(stopFuture)
@ -447,10 +450,10 @@ proc start*(cfg: StrawBossConfig): void =
get re".*": get re".*":
jsonResp(Http404, "URL [" & request.path & "] is not present on this server.") response.buildJson(Http404); return true
post re".*": post re".*":
jsonResp(Http404) response.buildJson(Http404); return true
proc performMaintenance(cfg: StrawBossConfig): void = proc performMaintenance(cfg: StrawBossConfig): void =
# Prune workers # Prune workers

View File

@ -1,2 +0,0 @@
const SB_VERSION* = "0.5.1"

View File

@ -1,10 +1,8 @@
{ {
"name": "dummy-project", "name": "dummy-project",
"versionCmd": "git describe --all --always", "versionCmd": "git describe --all --always",
"containerImage": "ubuntu",
"steps": { "steps": {
"build": { "build": {
"containerImage": "alpine",
"depends": ["test"], "depends": ["test"],
"workingDir": "dir1", "workingDir": "dir1",
"stepCmd": "cust-build", "stepCmd": "cust-build",

View File

@ -6,7 +6,6 @@
{ "name": "bob@builder.com", "hashedPwd": "$2a$11$lVZ9U4optQMhzPh0E9A7Yu6XndXblUF3gCa.zmEvJy4F.4C4718b." }, { "name": "bob@builder.com", "hashedPwd": "$2a$11$lVZ9U4optQMhzPh0E9A7Yu6XndXblUF3gCa.zmEvJy4F.4C4718b." },
{ "name": "sam@sousa.com", "hashedPwd": "testvalue" } { "name": "sam@sousa.com", "hashedPwd": "testvalue" }
], ],
"port": 8180,
"pwdCost": 11, "pwdCost": 11,
"projects": [ "projects": [
{ "name": "dummy-project", { "name": "dummy-project",

View File

@ -1,4 +1,4 @@
import tempfile, times, unittest, untar import unittest
from langutils import sameContents from langutils import sameContents
@ -12,8 +12,8 @@ let TIMEOUT = 2.minutes
suite "strawboss core": suite "strawboss core":
# Suite setup: extract test project # Suite setup: extract test project
let testProjTempDir = mkdtemp() let testProjTempDir = mkdir()
let testProjTarFile = newTarFile("src/test/test-project.tar.gz") let testProjTarFile = newTarFile("src/test/test-project.tar.gz:)
let testProjName = "test-project" let testProjName = "test-project"
testProjTarFile.extract(testProjTempDir) testProjTarFile.extract(testProjTempDir)

View File

@ -2,7 +2,6 @@ import cliutils, httpclient, json, os, osproc, sequtils, strutils, tempfile,
times, unittest, untar, uuids times, unittest, untar, uuids
from langutils import sameContents from langutils import sameContents
from algorithm import sorted
import ../testutil import ../testutil
import ../../../main/nim/strawbosspkg/configuration import ../../../main/nim/strawbosspkg/configuration
@ -51,7 +50,7 @@ suite "strawboss server":
@["serve", "-c", tempCfgPath], loadEnv(), {poUsePath}) @["serve", "-c", tempCfgPath], loadEnv(), {poUsePath})
# give the server time to spin up # give the server time to spin up
sleep(200) sleep(100)
teardown: teardown:
discard newAsyncHttpClient().post(apiBase & "/service/debug/stop") discard newAsyncHttpClient().post(apiBase & "/service/debug/stop")
@ -61,7 +60,7 @@ suite "strawboss server":
removeFile(tempCfgPath) removeFile(tempCfgPath)
# give the server time to spin down but kill it after that # give the server time to spin down but kill it after that
sleep(200) sleep(100)
if serverProcess.running: kill(serverProcess) if serverProcess.running: kill(serverProcess)
test "handle missing project configuration": test "handle missing project configuration":
@ -143,61 +142,6 @@ suite "strawboss server":
let status = loadBuildStatus(statusFile) let status = loadBuildStatus(statusFile)
check status.state == step[1] check status.state == step[1]
test "run a build in docker":
let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
# Run the "build-docker" step
var resp = http.post(apiBase & "/project/" & testProjName & "/step/build-docker/run/0.3.0")
check resp.status.startsWith("200")
let queuedRun = parseRun(parseJson(resp.body))
check queuedRun.status.state == BuildState.queued
# Wait for the build to complete
let completedRun = http.waitForBuild(apiBase, testProjName, $queuedRun.id)
# check that the run directory, run request, status, and output logs exist
let runsDir = tempBuildDataDir & "/" & testProjName & "/runs"
let runId = $completedRun.id
check existsDir(runsDir)
for suffix in [".request.json", ".status.json", ".stdout.log", ".stderr.log"]:
check existsFile(runsDir & "/" & runId & suffix)
# check that the project directory has been created in the artifacts repo
let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build-docker/0.3.0"
check existsDir(runArtifactsDir)
# check that the build step status file has been created
let statusFile = tempBuildDataDir & "/" & testProjName & "/status/build-docker/0.3.0.json"
check fileExists(statusFile)
# check that the status is complete
var status = loadBuildStatus(statusFile)
check status.state == BuildState.complete
# check that the artifacts we expect are present
let binFile = runArtifactsDir & "/test_project"
check existsFile(binFile)
test "run a multi-step docker-based build":
let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
# Run the "test" step (depends on "build")
var resp = http.post(apiBase & "/project/" & testProjname & "/step/test-docker/run/0.3.0")
check resp.status.startsWith("200")
let queuedRun = parseRun(parseJson(resp.body))
let completedRun = http.waitForBuild(apiBase, testProjName, $queuedRun.id)
# there should be successful status files for both the build and test steps
for step in [("build-docker", BuildState.stepComplete), ("test-docker", BuildState.complete)]:
let statusFile = tempBuildDataDir & "/" & testProjName & "/status/" & step[0] & "/0.3.0.json"
check fileExists(statusFile)
let status = loadBuildStatus(statusFile)
check status.state == step[1]
# TODO
#test "already completed steps should not be rebuilt": #test "already completed steps should not be rebuilt":
# let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password") # let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
# let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build/0.2.1" # let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build/0.2.1"
@ -206,40 +150,9 @@ suite "strawboss server":
# Run the "build" step # Run the "build" step
# Kick off a build that depends on "build" (which was run in the last test) # Kick off a build that depends on "build" (which was run in the last test)
test "kick off multiple runs and check the list of active runs via the API": # TODO
let http = newAuthenticatedHttpClient(apiBase, "bob@builder.com", "password") #test "kick off multiple runs and check the list of active runs via the API":
# check false
# Kick off multiple runs of the "long-running" job
let queuedRuns = toSeq((1..3)).map(proc (idx: int): Run =
let resp = http.post(apiBase & "/project/" & testProjName & "/step/long-running/run/0.3.1")
check resp.status.startsWith("200")
return parseRun(parseJson(resp.body)))
# Collect run ids.
let runIds = queuedRuns.mapIt($(it.id)).sorted(cmpIgnoreCase)
# Check on the runs
let getActiveResp = http.get(apiBase & "/project/" & testProjName & "/runs/active")
check getActiveResp.status.startsWith("200")
let activeRuns = parseJson(getActiveResp.body).getElems().mapIt(parseRun(it))
let activeRunIds = activeRuns.mapIt($(it.id)).sorted(cmpIgnoreCase)
# Make sure we see all runs in the active state.
check runIds == activeRunIds
let completedRuns = runIds.map(proc (runId: string): Run =
return http.waitForBuild(apiBase, testProjName, runId))
# Make sure all are completed and all are accounted for
check completedRuns.allIt(it.status.state == BuildState.complete)
check completedRuns.mapIt($(it.id)).sorted(cmpIgnoreCase) == runIds;
# Check that there are no more active runs
let getActiveResp2 = http.get(apiBase & "/project/" & testProjName & "/runs/active")
let remainingActiveRuns = parseJson(getActiveResp2.body).getElems().mapIt(parseRun(it))
check remainingActiveRuns.len == 0
# Last-chance catch to kill the server in case some test err'ed and didn't # Last-chance catch to kill the server in case some test err'ed and didn't
# reach it's teardown handler # reach it's teardown handler

View File

@ -26,7 +26,7 @@ proc waitForBuild*(client: HttpClient, apiBase, projectName, runId: string,
#echo "Checking (" & $curElapsed & " has passed)." #echo "Checking (" & $curElapsed & " has passed)."
if curElapsed > toFloat(timeout): if curElapsed > toFloat(timeout):
raise newException(Exception, "Timeout exceeded waiting for build.") raise newException(SystemError, "Timeout exceeded waiting for build.")
let resp = client.get(apiBase & "/project/" & projectName & "/run/" & runId) let resp = client.get(apiBase & "/project/" & projectName & "/run/" & runId)

View File

@ -1,7 +1,6 @@
import json, strtabs, times, tables, unittest, uuids import json, strtabs, times, tables, unittest, uuids
from langutils import sameContents from langutils import sameContents
from timeutils import trimNanoSec
import ../../../main/nim/strawbosspkg/configuration import ../../../main/nim/strawbosspkg/configuration
suite "load and save configuration objects": suite "load and save configuration objects":
@ -27,7 +26,7 @@ suite "load and save configuration objects":
stepName: "build", stepName: "build",
buildRef: "master", buildRef: "master",
workspaceDir: "/no-real/dir", workspaceDir: "/no-real/dir",
timestamp: getTime().local.trimNanoSec, timestamp: getLocalTime(getTime()),
forceRebuild: true) forceRebuild: true)
let rrStr = $rr1 let rrStr = $rr1
@ -100,7 +99,6 @@ suite "load and save configuration objects":
check: check:
pc.name == "dummy-project" pc.name == "dummy-project"
pc.versionCmd == "git describe --all --always" pc.versionCmd == "git describe --all --always"
pc.containerImage == "ubuntu"
pc.steps.len == 2 pc.steps.len == 2
# Explicitly set properties # Explicitly set properties
@ -108,7 +106,6 @@ suite "load and save configuration objects":
pc.steps["build"].dontSkip == true pc.steps["build"].dontSkip == true
pc.steps["build"].stepCmd == "cust-build" pc.steps["build"].stepCmd == "cust-build"
pc.steps["build"].workingDir == "dir1" pc.steps["build"].workingDir == "dir1"
pc.steps["build"].containerImage == "alpine"
sameContents(pc.steps["build"].artifacts, @["bin1", "doc1"]) sameContents(pc.steps["build"].artifacts, @["bin1", "doc1"])
sameContents(pc.steps["build"].depends, @["test"]) sameContents(pc.steps["build"].depends, @["test"])
sameContents(pc.steps["build"].expectedEnv, @["VAR1"]) sameContents(pc.steps["build"].expectedEnv, @["VAR1"])
@ -119,7 +116,6 @@ suite "load and save configuration objects":
pc.steps["test"].dontSkip == false pc.steps["test"].dontSkip == false
pc.steps["test"].stepCmd == "true" pc.steps["test"].stepCmd == "true"
pc.steps["test"].workingDir == "." pc.steps["test"].workingDir == "."
pc.steps["test"].containerImage.len == 0
sameContents(pc.steps["test"].artifacts, @[]) sameContents(pc.steps["test"].artifacts, @[])
sameContents(pc.steps["test"].depends, @[]) sameContents(pc.steps["test"].depends, @[])
sameContents(pc.steps["test"].expectedEnv, @[]) sameContents(pc.steps["test"].expectedEnv, @[])

View File

@ -6,7 +6,6 @@ from langutils import sameContents
import ../testutil import ../testutil
import ../../../main/nim/strawbosspkg/configuration import ../../../main/nim/strawbosspkg/configuration
import ../../../main/nim/strawbosspkg/server import ../../../main/nim/strawbosspkg/server
import ../../../main/nim/strawbosspkg/version
let apiBase = "http://localhost:8180/api" let apiBase = "http://localhost:8180/api"
let cfgFilePath = "src/test/json/strawboss.config.json" let cfgFilePath = "src/test/json/strawboss.config.json"
@ -41,11 +40,11 @@ suite "strawboss server":
let tok = toJWT(cfg, session) let tok = toJWT(cfg, session)
check fromJWT(cfg, tok) == session check fromJWT(cfg, tok) == session
test "version": test "ping":
let resp = http.get(apiBase & "/version") let resp = http.get(apiBase & "/ping")
check: check:
resp.status.startsWith("200") resp.status.startsWith("200")
resp.body == "\"strawboss v" & SB_VERSION & "\"" resp.body == "\"pong\""
test "fail auth": test "fail auth":
let resp = http.post(apiBase & "/auth-token", let resp = http.post(apiBase & "/auth-token",

@ -1 +1 @@
Subproject commit ab883bd9602a1373347a23c8bee4ed28dd475aec Subproject commit 127be8f66fcc6d4d223acf56668d42ff9c37bfb0

Binary file not shown.

View File

@ -1,7 +1,7 @@
# Package # Package
bin = @["strawboss"] bin = @["strawboss"]
version = "0.5.1" version = "0.4.0"
author = "Jonathan Bernard" author = "Jonathan Bernard"
description = "My personal continious integration worker." description = "My personal continious integration worker."
license = "MIT" license = "MIT"
@ -9,8 +9,8 @@ srcDir = "src/main/nim"
# Dependencies # Dependencies
requires @["nim >= 0.19.0", "docopt >= 0.6.8", "isaac >= 0.1.3", "tempfile", "jester >= 0.4.1", "bcrypt", requires @["nim >= 0.16.1", "docopt >= 0.6.5", "isaac >= 0.1.2", "tempfile", "jester", "bcrypt",
"untar", "uuids >= 0.1.10", "jwt"] "untar", "uuids"]
# Hacky to point to a specific hash. But there is some bug building in the # Hacky to point to a specific hash. But there is some bug building in the
# docker image we use to build the project with the next version. It adds an # docker image we use to build the project with the next version. It adds an
@ -18,11 +18,10 @@ requires @["nim >= 0.19.0", "docopt >= 0.6.8", "isaac >= 0.1.3", "tempfile", "je
# wrong and it tries to build against the 1.1 API even though the image only # wrong and it tries to build against the 1.1 API even though the image only
# has the 1.0 API. I'm crossing my fingers and hoping that our base image # has the 1.0 API. I'm crossing my fingers and hoping that our base image
# supports libssl 1.1 before I need to update this library. # supports libssl 1.1 before I need to update this library.
#requires "https://github.com/yglukhov/nim-jwt#549aa1eb13b8ddc0c6861d15cc2cc5b52bcbef01" requires "https://github.com/yglukhov/nim-jwt#549aa1eb13b8ddc0c6861d15cc2cc5b52bcbef01"
requires "https://git.jdb-labs.com/jdb/nim-lang-utils.git >= 0.4.0" requires "https://git.jdb-labs.com/jdb/nim-lang-utils.git >= 0.3.0"
requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git >= 0.6.0" requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git >= 0.3.1"
requires "https://git.jdb-labs.com/jdb/nim-time-utils.git >= 0.4.0"
# Tasks # Tasks
task functest, "Runs the functional test suite.": task functest, "Runs the functional test suite.":
@ -44,8 +43,3 @@ task test, "Runs both the unit and functional test suites.":
echo "\nRunning functional tests." echo "\nRunning functional tests."
echo "-------------------------" echo "-------------------------"
exec "src/test/nim/run_functional_tests" exec "src/test/nim/run_functional_tests"
task dist, "Creates distributable package.":
exec "nimble build"
mkdir "dist"
exec "cp strawboss strawboss.config.json example.json dist/."

View File

@ -1,25 +1,24 @@
{ {
"name": "strawboss", "name": "strawboss",
"containerImage": "nimlang/nim:0.19.0",
"steps": { "steps": {
"compile": { "compile": {
"artifacts": ["strawboss"], "artifacts": ["strawboss"],
"stepCmd": "nimble build" "stepCmd": "docker run -v `pwd`:/usr/src/strawboss -w /usr/src/strawboss jdbernard/nim:0.17.2 nimble build"
}, },
"unittest": { "unittest": {
"depends": ["compile"], "depends": ["compile"],
"stepCmd": "/bin/bash", "stepCmd": "docker run -v `pwd`:/usr/src/strawboss -v $compile_DIR:/usr/build/strawboss -w /usr/src/strawboss -i jdbernard/nim:0.17.2 /bin/bash",
"cmdInput": [ "cmdInput": [
"cp $compile_DIR/strawboss .", "cp /usr/build/strawboss/strawboss .",
"nimble install --depsOnly", "nimble install --depsOnly",
"nim c -r src/test/nim/run_unit_tests" "nim c -r src/test/nim/run_unit_tests"
] ]
}, },
"functest": { "functest": {
"depends": ["compile"], "depends": ["compile"],
"stepCmd": "/bin/bash", "stepCmd": "docker run -v `pwd`:/usr/src/strawboss -v $compile_DIR:/usr/build/strawboss -w /usr/src/strawboss -i jdbernard/nim:0.17.2 /bin/bash",
"cmdInput": [ "cmdInput": [
"cp $compile_DIR/strawboss .", "cp /usr/build/strawboss/strawboss .",
"nimble install --depsOnly", "nimble install --depsOnly",
"nim c -r src/test/nim/run_functional_tests" "nim c -r src/test/nim/run_functional_tests"
] ]
@ -27,9 +26,9 @@
"build": { "build": {
"artifacts": ["strawboss-$VERSION.zip"], "artifacts": ["strawboss-$VERSION.zip"],
"depends": ["compile", "unittest", "functest"], "depends": ["compile", "unittest", "functest"],
"stepCmd": "/bin/bash", "stepCmd": "docker run -v `pwd`:/usr/src/strawboss -v $compile_DIR:/usr/build/strawboss -w /usr/src/strawboss -i jdbernard/nim:0.17.2 /bin/bash",
"cmdInput": [ "cmdInput": [
"cp $compile_DIR/strawboss .", "cp /usr/build/strawboss/strawboss .",
"zip strawboss-$VERSION.zip strawboss strawboss.config.json example.json src/main/systemd/strawboss.service" "zip strawboss-$VERSION.zip strawboss strawboss.config.json example.json src/main/systemd/strawboss.service"
] ]
} }

View File

@ -1,11 +0,0 @@
Run a build. Look for:
- Run request archived
- Output logs archived with the run request
- Artifacts archived in the build-data directory.
- Configuration for that version archived in configurations directory.
- Status for that version archived in the status directory
Run the build again for the same project and build ref:
- Build should be skipped.
- Run request should be archived.