Finished refactor to base the build process around explicit run instances.
* Implemented periodic maintenance window. * Moved worker creation into the core module. * Worker processes no longer create run requests, but read queued requests from the file system. * Build status and logs have been moved into the StrawBoss data directory. * An initial build status is recorded when the job is queued. * Build status is recorded for build references as well as actual versions. So there will be a build status for "master", for example, that is overwritten whenever "master" is built for that step. * RunRequests now include a timestamp. * Added a Run object to contain both a RunRequest and the corresponding BuildStatus for that run. * API endpoints that talk about runs now return Run objects instead of RunRequests. * Moved all data layer operations into the core module so that the "database API" only lives in one place.
This commit is contained in:
Jonathan Bernard
@ -1,13 +1,20 @@
|
||||
import cliutils, logging, json, os, nre, sequtils, strtabs, tables, times, uuids
|
||||
import cliutils, logging, json, os, sequtils, strtabs, tables, times, uuids
|
||||
|
||||
from langutils import sameContents
|
||||
from typeinfo import toAny
|
||||
from strutils import parseEnum
|
||||
|
||||
const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz"
|
||||
|
||||
# Types
|
||||
#
|
||||
type
|
||||
BuildState* {.pure.} = enum
|
||||
queued, complete, failed, running, setup, rejected
|
||||
|
||||
BuildStatus* = object
|
||||
runId*, state*, details*: string
|
||||
runId*, details*: string
|
||||
state*: BuildState
|
||||
|
||||
Step* = object
|
||||
name*, stepCmd*, workingDir*: string
|
||||
@ -24,10 +31,16 @@ type
|
||||
envVars*: StringTableRef
|
||||
|
||||
RunRequest* = object
|
||||
id*: UUID
|
||||
runId*: UUID
|
||||
projectName*, stepName*, buildRef*, workspaceDir*: string
|
||||
timestamp*: TimeInfo
|
||||
forceRebuild*: bool
|
||||
|
||||
Run* = object
|
||||
id*: UUID
|
||||
request*: RunRequest
|
||||
status*: BuildStatus
|
||||
|
||||
User* = object
|
||||
name*: string
|
||||
hashedPwd*: string
|
||||
@ -43,6 +56,7 @@ type
|
||||
projects*: seq[ProjectDef]
|
||||
pwdCost*: int8
|
||||
users*: seq[UserRef]
|
||||
maintenancePeriod*: int
|
||||
|
||||
# Equality on custom types
|
||||
proc `==`*(a, b: UserRef): bool = result = a.name == b.name
|
||||
@ -64,38 +78,23 @@ proc `==`*(a, b: StrawBossConfig): bool =
|
||||
a.buildDataDir == b.buildDataDir and
|
||||
a.authSecret == b.authSecret and
|
||||
a.pwdCost == b.pwdCost and
|
||||
a.maintenancePeriod == b.maintenancePeriod and
|
||||
sameContents(a.users, b.users) and
|
||||
sameContents(a.projects, b.projects)
|
||||
|
||||
proc `==`*(a, b: RunRequest): bool =
|
||||
result =
|
||||
a.id == b.id and
|
||||
a.runId == b.runId and
|
||||
a.projectName == b.projectName and
|
||||
a.stepName == b.stepName and
|
||||
a.buildRef == b.buildRef and
|
||||
a.timestamp == b.timestamp and
|
||||
a.workspaceDir == b.workspaceDir and
|
||||
a.forceRebuild == b.forceRebuild
|
||||
|
||||
# Util methods on custom types
|
||||
proc findProject*(cfg: StrawBossConfig, projectName: string): ProjectDef =
|
||||
let candidates = cfg.projects.filterIt(it.name == projectName)
|
||||
if candidates.len == 0:
|
||||
raise newException(KeyError, "no project named " & projectName)
|
||||
elif candidates.len > 1:
|
||||
raise newException(KeyError, "multiple projects named " & projectName)
|
||||
else: result = candidates[0]
|
||||
# Useful utilities
|
||||
proc filesMatching*(pat: string): seq[string] = toSeq(walkFiles(pat))
|
||||
|
||||
proc setProject*(cfg: var StrawBossConfig, projectName: string, newDef: ProjectDef): void =
|
||||
var found = false
|
||||
for idx in 0..<cfg.projects.len:
|
||||
if cfg.projects[idx].name == projectName:
|
||||
cfg.projects[idx] = newDef
|
||||
found = true
|
||||
break
|
||||
|
||||
if not found: cfg.projects.add(newDef)
|
||||
|
||||
# other utils
|
||||
proc raiseEx*(reason: string): void =
|
||||
raise newException(Exception, reason)
|
||||
|
||||
@ -137,6 +136,7 @@ proc parseStrawBossConfig*(jsonCfg: JsonNode): StrawBossConfig =
|
||||
debug: jsonCfg.getIfExists("debug").getBVal(false),
|
||||
pwdCost: int8(jsonCfg.getOrFail("pwdCost", "strawboss config").getNum),
|
||||
projects: jsonCfg.getIfExists("projects").getElems.mapIt(parseProjectDef(it)),
|
||||
maintenancePeriod: int(jsonCfg.getIfExists("maintenancePeriod").getNum(10000)),
|
||||
users: users)
|
||||
|
||||
|
||||
@ -192,23 +192,30 @@ proc loadBuildStatus*(statusFile: string): BuildStatus =
|
||||
|
||||
result = BuildStatus(
|
||||
runId: jsonObj.getOrFail("runId", "run ID").getStr,
|
||||
state: jsonObj.getOrFail("state", "build status").getStr,
|
||||
state: parseEnum[BuildState](jsonObj.getOrFail("state", "build status").getStr),
|
||||
details: jsonObj.getIfExists("details").getStr("") )
|
||||
|
||||
proc parseRunRequest*(reqJson: JsonNode): RunRequest =
|
||||
result = RunRequest(
|
||||
id: parseUUID(reqJson.getOrFail("id", "RunRequest").getStr),
|
||||
runId: parseUUID(reqJson.getOrFail("runId", "RunRequest").getStr),
|
||||
projectName: reqJson.getOrFail("projectName", "RunRequest").getStr,
|
||||
stepName: reqJson.getOrFail("stepName", "RunRequest").getStr,
|
||||
buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr,
|
||||
workspaceDir: reqJson.getOrFail("workspaceDir", "RunRequest").getStr,
|
||||
timestamp: times.parse(reqJson.getOrFail("timestamp", "RunRequest").getStr, ISO_TIME_FORMAT),
|
||||
forceRebuild: reqJson.getOrFail("forceRebuild", "RunRequest").getBVal)
|
||||
|
||||
proc loadRunRequest*(reqFilePath: string): RunRequest =
|
||||
if not existsFile(reqFilePath):
|
||||
raiseEx "request file not found: " & reqFilePath
|
||||
|
||||
parseRunRequest(parseFile(reqFilePath))
|
||||
|
||||
# TODO: can we use the marshal module for this?
|
||||
proc `%`*(s: BuildStatus): JsonNode =
|
||||
result = %* {
|
||||
"runId": s.runId,
|
||||
"state": s.state,
|
||||
"state": $s.state,
|
||||
"details": s.details }
|
||||
|
||||
proc `%`*(p: ProjectDef): JsonNode =
|
||||
@ -243,12 +250,13 @@ proc `%`*(p: ProjectConfig): JsonNode =
|
||||
|
||||
proc `%`*(req: RunRequest): JsonNode =
|
||||
result = %* {
|
||||
"id": $(req.id),
|
||||
"runId": $(req.runId),
|
||||
"projectName": req.projectName,
|
||||
"stepName": req.stepName,
|
||||
"buildRef": req.buildRef,
|
||||
"workspaceDir": req.workspaceDir,
|
||||
"forceRebuild": req.forceRebuild }
|
||||
"forceRebuild": req.forceRebuild,
|
||||
"timestamp": req.timestamp.format(ISO_TIME_FORMAT) }
|
||||
|
||||
proc `%`*(user: User): JsonNode =
|
||||
result = %* {
|
||||
@ -262,9 +270,17 @@ proc `%`*(cfg: StrawBossConfig): JsonNode =
|
||||
"debug": cfg.debug,
|
||||
"projects": %cfg.projects,
|
||||
"pwdCost": cfg.pwdCost,
|
||||
"maintenancePeriod": cfg.maintenancePeriod,
|
||||
"users": %cfg.users }
|
||||
|
||||
proc `%`*(run: Run): JsonNode =
|
||||
result = %* {
|
||||
"id": $run.id,
|
||||
"request": %run.request,
|
||||
"status": %run.status }
|
||||
|
||||
proc `$`*(s: BuildStatus): string = result = pretty(%s)
|
||||
proc `$`*(req: RunRequest): string = result = pretty(%req)
|
||||
proc `$`*(pd: ProjectDef): string = result = pretty(%pd)
|
||||
proc `$`*(cfg: StrawBossConfig): string = result = pretty(%cfg)
|
||||
proc `$`*(run: Run): string = result = pretty(%run)
|
||||
|
||||
Reference in New Issue
Block a user