4edae250ba
* Fixed the formatting of command line logging of strawboss workers. * Fixed a bug in the (de)serialization of log levels in the strawboss service config file. * Pulled `parseBuildStatus` logic out of `loadBuildStatus` so that we could parse a JSON that didn't come from a file. * Added `parseRun` for Run objects. * Moved `/ping` to `/service/debug/ping` for symmetry with `/service/debug/stop` * Added functional tests of full builds.
304 lines
9.9 KiB
Nim
304 lines
9.9 KiB
Nim
import cliutils, logging, json, os, sequtils, strtabs, strutils, tables, times, uuids
|
|
|
|
from langutils import sameContents
|
|
from typeinfo import toAny
|
|
from strutils import parseEnum
|
|
|
|
const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz"
|
|
|
|
# Types
|
|
#
|
|
type
|
|
BuildState* {.pure.} = enum
|
|
queued, complete, failed, running, setup, rejected
|
|
|
|
BuildStatus* = object
|
|
runId*, details*: string
|
|
state*: BuildState
|
|
|
|
Step* = object
|
|
name*, stepCmd*, workingDir*: string
|
|
artifacts*, cmdInput*, depends*, expectedEnv*: seq[string]
|
|
dontSkip*: bool
|
|
|
|
ProjectConfig* = object
|
|
name*: string
|
|
versionCmd*: string
|
|
steps*: Table[string, Step]
|
|
|
|
ProjectDef* = object
|
|
cfgFilePath*, defaultBranch*, name*, repo*: string
|
|
envVars*: StringTableRef
|
|
|
|
RunRequest* = object
|
|
runId*: UUID
|
|
projectName*, stepName*, buildRef*, workspaceDir*: string
|
|
timestamp*: TimeInfo
|
|
forceRebuild*: bool
|
|
|
|
Run* = object
|
|
id*: UUID
|
|
request*: RunRequest
|
|
status*: BuildStatus
|
|
|
|
User* = object
|
|
name*: string
|
|
hashedPwd*: string
|
|
|
|
UserRef* = ref User
|
|
|
|
StrawBossConfig* = object
|
|
buildDataDir*: string
|
|
authSecret*: string
|
|
filePath*: string
|
|
debug*: bool
|
|
logLevel*: Level
|
|
pathToExe*: string
|
|
projects*: seq[ProjectDef]
|
|
pwdCost*: int8
|
|
users*: seq[UserRef]
|
|
maintenancePeriod*: int
|
|
|
|
# Equality on custom types
|
|
proc `==`*(a, b: UserRef): bool = result = a.name == b.name
|
|
|
|
proc `==`*(a, b: ProjectDef): bool =
|
|
if a.envVars.len != b.envVars.len: return false
|
|
|
|
for k, v in a.envVars:
|
|
if not b.envVars.hasKey(k) or a.envVars[k] != b.envVars[k]: return false
|
|
|
|
return
|
|
a.name == b.name and
|
|
a.cfgFilePath == b.cfgFilePath and
|
|
a.defaultBranch == b.defaultBranch and
|
|
a.repo == b.repo
|
|
|
|
proc `==`*(a, b: StrawBossConfig): bool =
|
|
result =
|
|
a.buildDataDir == b.buildDataDir and
|
|
a.authSecret == b.authSecret and
|
|
a.pwdCost == b.pwdCost and
|
|
a.maintenancePeriod == b.maintenancePeriod and
|
|
a.logLevel == b.logLevel and
|
|
sameContents(a.users, b.users) and
|
|
sameContents(a.projects, b.projects)
|
|
|
|
proc `==`*(a, b: RunRequest): bool =
|
|
result =
|
|
a.runId == b.runId and
|
|
a.projectName == b.projectName and
|
|
a.stepName == b.stepName and
|
|
a.buildRef == b.buildRef and
|
|
a.timestamp == b.timestamp and
|
|
a.workspaceDir == b.workspaceDir and
|
|
a.forceRebuild == b.forceRebuild
|
|
|
|
# Useful utilities
|
|
proc filesMatching*(pat: string): seq[string] = toSeq(walkFiles(pat))
|
|
|
|
proc raiseEx*(reason: string): void =
|
|
raise newException(Exception, reason)
|
|
|
|
# internal utils
|
|
proc getIfExists(n: JsonNode, key: string): JsonNode =
|
|
## convenience method to get a key from a JObject or return null
|
|
result = if n.hasKey(key): n[key]
|
|
else: newJNull()
|
|
|
|
proc getOrFail(n: JsonNode, key: string, objName: string = ""): JsonNode =
|
|
## convenience method to get a key from a JObject or raise an exception
|
|
if not n.hasKey(key): raiseEx objName & " missing key '" & key & "'"
|
|
return n[key]
|
|
|
|
# Configuration parsing code
|
|
|
|
proc parseLogLevel*(level: string): Level =
|
|
let lvlStr = "lvl" & toUpper(level[0]) & level[1..^1]
|
|
result = parseEnum[Level](lvlStr)
|
|
|
|
proc parseProjectDef*(pJson: JsonNode): ProjectDef =
|
|
var envVars = newStringTable(modeCaseSensitive)
|
|
for k, v in pJson.getIfExists("envVars").getFields: envVars[k] = v.getStr("")
|
|
|
|
result = ProjectDef(
|
|
cfgFilePath: pJson.getIfExists("cfgFilePath").getStr("strawboss.json"),
|
|
defaultBranch: pJson.getIfExists("defaultBranch").getStr("master"),
|
|
name: pJson.getOrFail("name", "project definition").getStr,
|
|
envVars: envVars,
|
|
repo: pJson.getOrFail("repo", "project definition").getStr)
|
|
|
|
proc parseStrawBossConfig*(jsonCfg: JsonNode): StrawBossConfig =
|
|
var users: seq[UserRef] = @[]
|
|
|
|
for uJson in jsonCfg.getIfExists("users").getElems:
|
|
users.add(UserRef(
|
|
name: uJson.getOrFail("name", "user record").getStr,
|
|
hashedPwd: uJson.getOrFail("hashedPwd", "user record").getStr))
|
|
|
|
result = StrawBossConfig(
|
|
buildDataDir: jsonCfg.getIfExists("buildDataDir").getStr("build-data"),
|
|
authSecret: jsonCfg.getOrFail("authSecret", "strawboss config").getStr,
|
|
debug: jsonCfg.getIfExists("debug").getBVal(false),
|
|
pwdCost: int8(jsonCfg.getOrFail("pwdCost", "strawboss config").getNum),
|
|
projects: jsonCfg.getIfExists("projects").getElems.mapIt(parseProjectDef(it)),
|
|
maintenancePeriod: int(jsonCfg.getIfExists("maintenancePeriod").getNum(10000)),
|
|
logLevel: parseLogLevel(jsonCfg.getIfExists("logLevel").getStr("info")),
|
|
users: users)
|
|
|
|
|
|
proc loadStrawBossConfig*(cfgFile: string): StrawBossConfig =
|
|
if not existsFile(cfgFile):
|
|
raiseEx "strawboss config file not found: " & cfgFile
|
|
|
|
result = parseStrawBossConfig(parseFile(cfgFile))
|
|
result.filePath = cfgFile
|
|
|
|
proc loadProjectConfig*(cfgFile: string): ProjectConfig =
|
|
if not existsFile(cfgFile):
|
|
raiseEx "project config file not found: " & cfgFile
|
|
|
|
let jsonCfg = parseFile(cfgFile)
|
|
|
|
if not jsonCfg.hasKey("steps"):
|
|
raiseEx "project configuration is missing steps definition"
|
|
|
|
var steps = initTable[string, Step]()
|
|
for sName, pJson in jsonCfg.getOrFail("steps", "project configuration").getFields:
|
|
steps[sName] = Step(
|
|
name: sName,
|
|
workingDir: pJson.getIfExists("workingDir").getStr("."),
|
|
stepCmd: pJson.getIfExists("stepCmd").getStr("NOT GIVEN"),
|
|
depends: pJson.getIfExists("depends").getElems.mapIt(it.getStr),
|
|
artifacts: pJson.getIfExists("artifacts").getElems.mapIt(it.getStr),
|
|
cmdInput: pJson.getIfExists("cmdInput").getElems.mapIt(it.getStr),
|
|
expectedEnv: pJson.getIfExists("expectedEnv").getElems.mapIt(it.getStr),
|
|
dontSkip: pJson.getIfExists("dontSkip").getBVal(false))
|
|
|
|
# cmdInput and stepCmd are related, so we have a conditional defaulting.
|
|
# Four possibilities:
|
|
|
|
if steps[sName].stepCmd == "NOT GIVEN" and steps[sName].cmdInput.len == 0:
|
|
# 1. Neither given: default to no-op
|
|
steps[sName].stepCmd = "true"
|
|
|
|
if steps[sName].stepCmd == "NOT GIVEN" and steps[sName].cmdInput.len > 0:
|
|
# 2. cmdInput given but not stepCmd: default stepCmd to "sh"
|
|
steps[sName].stepCmd = "sh"
|
|
|
|
# 3. stepCmd given but not cmdInput & 4. both given: use them as-is
|
|
|
|
result = ProjectConfig(
|
|
name: jsonCfg.getOrFail("name", "project configuration").getStr,
|
|
versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"),
|
|
steps: steps)
|
|
|
|
proc parseBuildStatus*(statusJson: JsonNode): BuildStatus =
|
|
result = BuildStatus(
|
|
runId: statusJson.getOrFail("runId", "run ID").getStr,
|
|
state: parseEnum[BuildState](statusJson.getOrFail("state", "build status").getStr),
|
|
details: statusJson.getIfExists("details").getStr("") )
|
|
|
|
proc loadBuildStatus*(statusFile: string): BuildStatus =
|
|
if not existsFile(statusFile): raiseEx "status file not found: " & statusFile
|
|
let jsonObj = parseFile(statusFile)
|
|
|
|
result = parseBuildStatus(jsonObj)
|
|
|
|
proc parseRunRequest*(reqJson: JsonNode): RunRequest =
|
|
result = RunRequest(
|
|
runId: parseUUID(reqJson.getOrFail("runId", "RunRequest").getStr),
|
|
projectName: reqJson.getOrFail("projectName", "RunRequest").getStr,
|
|
stepName: reqJson.getOrFail("stepName", "RunRequest").getStr,
|
|
buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr,
|
|
workspaceDir: reqJson.getOrFail("workspaceDir", "RunRequest").getStr,
|
|
timestamp: times.parse(reqJson.getOrFail("timestamp", "RunRequest").getStr, ISO_TIME_FORMAT),
|
|
forceRebuild: reqJson.getOrFail("forceRebuild", "RunRequest").getBVal)
|
|
|
|
proc loadRunRequest*(reqFilePath: string): RunRequest =
|
|
if not existsFile(reqFilePath):
|
|
raiseEx "request file not found: " & reqFilePath
|
|
|
|
parseRunRequest(parseFile(reqFilePath))
|
|
|
|
proc parseRun*(runJson: JsonNode): Run =
|
|
result = Run(
|
|
id: parseUUID(runJson.getOrFail("id", "Run").getStr),
|
|
request: parseRunRequest(runJson.getOrFail("request", "Run")),
|
|
status: parseBuildStatus(runJson.getOrFail("status", "Run")))
|
|
|
|
# TODO: can we use the marshal module for this?
|
|
proc `%`*(s: BuildStatus): JsonNode =
|
|
result = %* {
|
|
"runId": s.runId,
|
|
"state": $s.state,
|
|
"details": s.details }
|
|
|
|
proc `%`*(p: ProjectDef): JsonNode =
|
|
result = %* {
|
|
"name": p.name,
|
|
"cfgFilePath": p.cfgFilePath,
|
|
"defaultBranch": p.defaultBranch,
|
|
"repo": p.repo }
|
|
|
|
result["envVars"] = newJObject()
|
|
for k, v in p.envVars: result["envVars"][k] = %v
|
|
|
|
proc `%`*(s: Step): JsonNode =
|
|
result = %* {
|
|
"name": s.name,
|
|
"stepCmd": s.stepCmd,
|
|
"workingDir": s.workingDir,
|
|
"artifacts": s.artifacts,
|
|
"cmdInput": s.cmdInput,
|
|
"depends": s.depends,
|
|
"expectedEnv": s.expectedEnv,
|
|
"dontSkip": s.dontSkip }
|
|
|
|
proc `%`*(p: ProjectConfig): JsonNode =
|
|
result = %* {
|
|
"name": p.name,
|
|
"versionCmd": p.versionCmd }
|
|
|
|
result["steps"] = newJObject()
|
|
for name, step in p.steps:
|
|
result["steps"][name] = %step
|
|
|
|
proc `%`*(req: RunRequest): JsonNode =
|
|
result = %* {
|
|
"runId": $(req.runId),
|
|
"projectName": req.projectName,
|
|
"stepName": req.stepName,
|
|
"buildRef": req.buildRef,
|
|
"workspaceDir": req.workspaceDir,
|
|
"forceRebuild": req.forceRebuild,
|
|
"timestamp": req.timestamp.format(ISO_TIME_FORMAT) }
|
|
|
|
proc `%`*(user: User): JsonNode =
|
|
result = %* {
|
|
"name": user.name,
|
|
"hashedPwd": user.hashedPwd }
|
|
|
|
proc `%`*(cfg: StrawBossConfig): JsonNode =
|
|
result = %* {
|
|
"buildDataDir": cfg.buildDataDir,
|
|
"authSecret": cfg.authSecret,
|
|
"debug": cfg.debug,
|
|
"projects": %cfg.projects,
|
|
"pwdCost": cfg.pwdCost,
|
|
"maintenancePeriod": cfg.maintenancePeriod,
|
|
"logLevel": toLower(($cfg.logLevel)[3]) & ($cfg.logLevel)[4..^1],
|
|
"users": %cfg.users }
|
|
|
|
proc `%`*(run: Run): JsonNode =
|
|
result = %* {
|
|
"id": $run.id,
|
|
"request": %run.request,
|
|
"status": %run.status }
|
|
|
|
proc `$`*(s: BuildStatus): string = result = pretty(%s)
|
|
proc `$`*(req: RunRequest): string = result = pretty(%req)
|
|
proc `$`*(pd: ProjectDef): string = result = pretty(%pd)
|
|
proc `$`*(cfg: StrawBossConfig): string = result = pretty(%cfg)
|
|
proc `$`*(run: Run): string = result = pretty(%run)
|