WIP Moving back towards using named runs.
* Rename artifactsRepo -> buildDataDir to be more explicit about the fact that it holds more than just the artifacts. * Revert removal of run ids. * Move Worker definition into core as part of making the core responsible for accepting run requests. * Make the core module more responsible for internal details of data structure and storage. External callers should not need to construct paths to artifacts, versions, etc. but should be able to call method in the core module to do this work for them. * The working directory no longer contains anything but the checked-out code. All StrawBoss-specific data is stored by StrawBoss elsewhere. * Add a regular maintenance cycle to the server module.
This commit is contained in:
Jonathan Bernard
parent
7aa0a69215
commit
e000b37c35
12
README.md
12
README.md
@ -53,8 +53,9 @@ the `strawboss` executable. This is the configuration file for StrawBoss
|
||||
itself. The contents are expected to be a valid JSON object. The top level keys
|
||||
are:
|
||||
|
||||
* `artifactsRepo`: A string denoting the path to the artifacts repository
|
||||
directory.
|
||||
* `buildDataDir`: A string denoting the path to the directory where StrawBoss
|
||||
keeps metadata about builds it has performed and the artifacts resulting from
|
||||
the builds.
|
||||
|
||||
* `authSecret`: Secret key used to sign JWT session tokens.
|
||||
|
||||
@ -113,12 +114,7 @@ object. The top level keys are:
|
||||
|
||||
* `versionCmd` *(optional)*: a command to be run in a shell (`sh`-compatible)
|
||||
that is expected to print the current version of the project on `stdout`.
|
||||
It is important to note that if you supply a verion command it should provide
|
||||
a unique result for every commit in the repository. StrawBoss is built around
|
||||
the assumptions that builds are repeatable and that every buildable point has
|
||||
a unique version id. This is the reason that StrawBoss does not create uniue
|
||||
IDs for individual builds. The combination of project name, build step, and
|
||||
version *is* the build ID. *(defaults to `git describe --tags --always`)*.
|
||||
*(defaults to `git describe --tags --always`)*.
|
||||
|
||||
#### Step Definition
|
||||
|
||||
|
||||
4
api.rst
4
api.rst
@ -6,10 +6,10 @@
|
||||
- GET /api/project/<proj-id> -- TODO
|
||||
* GET /api/project/<proj-id>/runs -- list summary information for all runs
|
||||
* GET /api/project/<proj-id>/runs/active -- list summary information about all currently active runs
|
||||
- GET /api/project/<proj-id>/runs/<run-id> -- list detailed information about a specific run
|
||||
✓ GET /api/project/<proj-id>/versions -- list the versions of this project that have been built
|
||||
* GET /api/project/<proj-id>/version/<ref> -- return detailed project definition (include steps) at a specific version
|
||||
- GET /api/project/<proj-id>/step/<step-id> -- return detailed step information (include runs for different versions)
|
||||
- GET /api/project/<proj-id>/step/<step-id>/run/<ref> -- list detailed information about a specific run
|
||||
- GET /api/project/<proj-id>/step/<step-id> -- return detailed step information (include runs)
|
||||
* POST /api/project/<proj-id>/step/<step-id>/run/<ref> -- kick off a run
|
||||
|
||||
|
||||
|
||||
17
file-structure.txt
Normal file
17
file-structure.txt
Normal file
@ -0,0 +1,17 @@
|
||||
build-data/
|
||||
<project-name>/
|
||||
configurations/
|
||||
<version>.json
|
||||
runs/
|
||||
<id>.request.json
|
||||
<id>.stdout.log
|
||||
<id>.stderr.log
|
||||
<id>.status.json
|
||||
status/
|
||||
<version>.json
|
||||
artifacts/
|
||||
<step-name>/
|
||||
<version>/
|
||||
<artifact-file>
|
||||
|
||||
workspace/
|
||||
@ -1,4 +1,4 @@
|
||||
import cliutils, docopt, os, sequtils, tempfile
|
||||
import cliutils, docopt, os, sequtils, tempfile, uuids
|
||||
|
||||
import strawbosspkg/configuration
|
||||
import strawbosspkg/core
|
||||
@ -30,6 +30,9 @@ Options
|
||||
|
||||
-r --reference <ref> Build the project at this commit reference.
|
||||
|
||||
-i --run-id <id> Use the given UUID as the run ID. If not given, a
|
||||
new UUID is generated for this run.
|
||||
|
||||
-w --workspace <workspace> Use the given directory as the build workspace.
|
||||
|
||||
"""
|
||||
@ -41,11 +44,11 @@ Options
|
||||
|
||||
var cfg = loadStrawBossConfig(cfgFile)
|
||||
cfg.pathToExe = paramStr(0)
|
||||
if not existsDir(cfg.artifactsRepo):
|
||||
echo "Artifacts repo (" & cfg.artifactsRepo & ") does not exist. Creating..."
|
||||
createDir(cfg.artifactsRepo)
|
||||
if not existsDir(cfg.buildDataDir):
|
||||
echo "Build data directory (" & cfg.buildDataDir & ") does not exist. Creating..."
|
||||
createDir(cfg.buildDataDir)
|
||||
|
||||
cfg.artifactsRepo = expandFilename(cfg.artifactsRepo)
|
||||
cfg.buildDataDir = expandFilename(cfg.buildDataDir)
|
||||
|
||||
|
||||
if args["run"]:
|
||||
@ -54,6 +57,7 @@ Options
|
||||
|
||||
try:
|
||||
let req = RunRequest(
|
||||
id: if args["--run-id"]: parseUUID($args["--run-id"]) else: genUUID(),
|
||||
projectName: $args["<project>"],
|
||||
stepName: $args["<step>"],
|
||||
buildRef: if args["--reference"]: $args["--reference"] else: nil,
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import cliutils, logging, json, os, nre, sequtils, strtabs, tables, times
|
||||
import cliutils, logging, json, os, nre, sequtils, strtabs, tables, times, uuids
|
||||
|
||||
from langutils import sameContents
|
||||
from typeinfo import toAny
|
||||
@ -7,7 +7,7 @@ from typeinfo import toAny
|
||||
#
|
||||
type
|
||||
BuildStatus* = object
|
||||
state*, details*: string
|
||||
runId*, state*, details*: string
|
||||
|
||||
Step* = object
|
||||
name*, stepCmd*, workingDir*: string
|
||||
@ -24,6 +24,7 @@ type
|
||||
envVars*: StringTableRef
|
||||
|
||||
RunRequest* = object
|
||||
id*: UUID
|
||||
projectName*, stepName*, buildRef*, workspaceDir*: string
|
||||
forceRebuild*: bool
|
||||
|
||||
@ -34,7 +35,7 @@ type
|
||||
UserRef* = ref User
|
||||
|
||||
StrawBossConfig* = object
|
||||
artifactsRepo*: string
|
||||
buildDataDir*: string
|
||||
authSecret*: string
|
||||
filePath*: string
|
||||
debug*: bool
|
||||
@ -60,7 +61,7 @@ proc `==`*(a, b: ProjectDef): bool =
|
||||
|
||||
proc `==`*(a, b: StrawBossConfig): bool =
|
||||
result =
|
||||
a.artifactsRepo == b.artifactsRepo and
|
||||
a.buildDataDir == b.buildDataDir and
|
||||
a.authSecret == b.authSecret and
|
||||
a.pwdCost == b.pwdCost and
|
||||
sameContents(a.users, b.users) and
|
||||
@ -68,6 +69,7 @@ proc `==`*(a, b: StrawBossConfig): bool =
|
||||
|
||||
proc `==`*(a, b: RunRequest): bool =
|
||||
result =
|
||||
a.id == b.id and
|
||||
a.projectName == b.projectName and
|
||||
a.stepName == b.stepName and
|
||||
a.buildRef == b.buildRef and
|
||||
@ -130,7 +132,7 @@ proc parseStrawBossConfig*(jsonCfg: JsonNode): StrawBossConfig =
|
||||
hashedPwd: uJson.getOrFail("hashedPwd", "user record").getStr))
|
||||
|
||||
result = StrawBossConfig(
|
||||
artifactsRepo: jsonCfg.getIfExists("artifactsRepo").getStr("artifacts"),
|
||||
buildDataDir: jsonCfg.getIfExists("buildDataDir").getStr("build-data"),
|
||||
authSecret: jsonCfg.getOrFail("authSecret", "strawboss config").getStr,
|
||||
debug: jsonCfg.getIfExists("debug").getBVal(false),
|
||||
pwdCost: int8(jsonCfg.getOrFail("pwdCost", "strawboss config").getNum),
|
||||
@ -189,11 +191,13 @@ proc loadBuildStatus*(statusFile: string): BuildStatus =
|
||||
let jsonObj = parseFile(statusFile)
|
||||
|
||||
result = BuildStatus(
|
||||
runId: jsonObj.getOrFail("runId", "run ID").getStr,
|
||||
state: jsonObj.getOrFail("state", "build status").getStr,
|
||||
details: jsonObj.getIfExists("details").getStr("") )
|
||||
|
||||
proc parseRunRequest*(reqJson: JsonNode): RunRequest =
|
||||
result = RunRequest(
|
||||
id: parseUUID(reqJson.getOrFail("id", "RunRequest").getStr),
|
||||
projectName: reqJson.getOrFail("projectName", "RunRequest").getStr,
|
||||
stepName: reqJson.getOrFail("stepName", "RunRequest").getStr,
|
||||
buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr,
|
||||
@ -203,6 +207,7 @@ proc parseRunRequest*(reqJson: JsonNode): RunRequest =
|
||||
# TODO: can we use the marshal module for this?
|
||||
proc `%`*(s: BuildStatus): JsonNode =
|
||||
result = %* {
|
||||
"runId": s.runId,
|
||||
"state": s.state,
|
||||
"details": s.details }
|
||||
|
||||
@ -238,6 +243,7 @@ proc `%`*(p: ProjectConfig): JsonNode =
|
||||
|
||||
proc `%`*(req: RunRequest): JsonNode =
|
||||
result = %* {
|
||||
"id": $(req.id),
|
||||
"projectName": req.projectName,
|
||||
"stepName": req.stepName,
|
||||
"buildRef": req.buildRef,
|
||||
@ -251,7 +257,7 @@ proc `%`*(user: User): JsonNode =
|
||||
|
||||
proc `%`*(cfg: StrawBossConfig): JsonNode =
|
||||
result = %* {
|
||||
"artifactsRepo": cfg.artifactsRepo,
|
||||
"buildDataDir": cfg.buildDataDir,
|
||||
"authSecret": cfg.authSecret,
|
||||
"debug": cfg.debug,
|
||||
"projects": %cfg.projects,
|
||||
|
||||
@ -1,14 +1,14 @@
|
||||
import cliutils, logging, json, options, os, osproc, sequtils, streams,
|
||||
strtabs, strutils, tables
|
||||
strtabs, strutils, tables, uuids
|
||||
|
||||
import nre except toSeq
|
||||
import ./configuration
|
||||
import nre except toSeq
|
||||
from posix import link
|
||||
|
||||
type
|
||||
Workspace = ref object ## Data needed by internal build process
|
||||
artifactsDir*: string ## absolute path to the directory for this version
|
||||
artifactsRepo*: string ## absolute path to the global artifacts repo
|
||||
buildDataDir*: string ## absolute path to the global build data directory for this project
|
||||
buildRef*: string ## git-style commit reference to the revision we are building
|
||||
dir*: string ## absolute path to the working directory
|
||||
env*: StringTableRef ## environment variables for all build processes
|
||||
@ -16,11 +16,16 @@ type
|
||||
outputHandler*: HandleProcMsgCB ## handler for process output
|
||||
project*: ProjectConfig ## the project configuration
|
||||
projectDef*: ProjectDef ## the StrawBoss project definition
|
||||
runRequest*: RunRequest ## the RunRequest that initated the current build
|
||||
status*: BuildStatus ## the current status of the build
|
||||
statusFile*: string ## absolute path to the build status file
|
||||
step*: Step ## the step we're building
|
||||
version*: string ## project version as returned by versionCmd
|
||||
|
||||
Worker* = object
|
||||
runId*: UUID
|
||||
process*: Process
|
||||
|
||||
proc sendMsg(h: HandleProcMsgCB, msg: TaintedString): void =
|
||||
h.sendMsg(msg, nil, "strawboss")
|
||||
|
||||
@ -45,21 +50,53 @@ proc emitStatus(status: BuildStatus, statusFilePath: string,
|
||||
proc publishStatus(wksp: Workspace, state, details: string) =
|
||||
## Update the status for a Workspace and publish this status to the
|
||||
## Workspace's status file and any output message handlers.
|
||||
let status = BuildStatus(state: state, details: details)
|
||||
let status = BuildStatus(
|
||||
runId: $wksp.runRequest.id, state: state, details: details)
|
||||
wksp.status = emitStatus(status, wksp.statusFile, wksp.outputHandler)
|
||||
|
||||
proc ensureProjectDirsExist(cfg: StrawBossConfig, p: ProjectDef): void =
|
||||
for subdir in ["configurations", "runs", "status", "artifacts"]:
|
||||
let fullPath = cfg.buildDataDir & "/" & p.name & "/" & subdir
|
||||
if not existsDir(fullPath):
|
||||
createDir(fullPath)
|
||||
|
||||
proc listVersions*(cfg: StrawBossConfig, project: ProjectDef): seq[string] =
|
||||
## List the versions that have been built for a project.
|
||||
ensureProjectDirsExist(cfg, project)
|
||||
|
||||
let versionFiles = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/configurations/*.json"))
|
||||
result = versionFiles.map(proc(s: string): string =
|
||||
let slashIdx = s.rfind('/')
|
||||
result = s[(slashIdx + 1)..^6])
|
||||
|
||||
proc listRuns*(cfg: StrawBossConfig, project: ProjectDef): seq[RunRequest] =
|
||||
## List the runs that have been performed for a project.
|
||||
ensureProjectDirsExist(cfg, project)
|
||||
|
||||
let runPaths = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/runs/*.request.json"))
|
||||
return runPaths.mapIt(parseRunRequest(parseFile(it)))
|
||||
|
||||
proc getCurrentProjectConfig*(cfg: StrawBossConfig, project: ProjectDef): Option[ProjectConfig] =
|
||||
let projCfgFile = "nope.json" # TODO
|
||||
if not existsFile(projCfgFile): result = none(ProjectConfig)
|
||||
else:
|
||||
try:
|
||||
let projectConfig: ProjectConfig = loadProjectConfig(projCfgFile) #ProjectConfig(name: "test")
|
||||
result = some(projectConfig)
|
||||
except: result = none(ProjectConfig)
|
||||
|
||||
proc setupProject(wksp: Workspace) =
|
||||
|
||||
# Clone the project into the $temp/repo directory
|
||||
let cloneResult = exec("git", wksp.dir,
|
||||
["clone", wksp.projectDef.repo, "repo"],
|
||||
# Clone the project into the $temp directory
|
||||
let cloneResult = exec("git", ".",
|
||||
["clone", wksp.projectDef.repo, wksp.dir],
|
||||
wksp.env, {poUsePath}, wksp.outputHandler)
|
||||
|
||||
if cloneResult != 0:
|
||||
raiseEx "unable to clone repo for '" & wksp.projectDef.name & "'"
|
||||
|
||||
# Checkout the requested ref
|
||||
let checkoutResult = exec("git", wksp.dir & "/repo",
|
||||
let checkoutResult = exec("git", wksp.dir,
|
||||
["checkout", wksp.buildRef],
|
||||
wksp.env, {poUsePath}, wksp.outputHandler)
|
||||
|
||||
@ -68,7 +105,7 @@ proc setupProject(wksp: Workspace) =
|
||||
" for '" & wksp.projectDef.name & "'"
|
||||
|
||||
# Find the strawboss project configuration
|
||||
let projCfgFile = wksp.dir & "/repo/" & wksp.projectDef.cfgFilePath
|
||||
let projCfgFile = wksp.dir & wksp.projectDef.cfgFilePath
|
||||
if not existsFile(projCfgFile):
|
||||
raiseEx "Cannot find strawboss project configuration in the project " &
|
||||
"repo (expected at '" & wksp.projectDef.cfgFilePath & "')."
|
||||
@ -81,7 +118,7 @@ proc setupProject(wksp: Workspace) =
|
||||
# Get the build version
|
||||
let versionResult = execWithOutput(
|
||||
wksp.project.versionCmd, # command
|
||||
wksp.dir & "/repo", # working dir
|
||||
wksp.dir, # working dir
|
||||
[], # args
|
||||
wksp.env, # environment
|
||||
{poUsePath, poEvalCommand}) # options
|
||||
@ -93,22 +130,6 @@ proc setupProject(wksp: Workspace) =
|
||||
wksp.version = versionResult.output.strip
|
||||
wksp.env["VERSION"] = wksp.version
|
||||
|
||||
proc listRuns*(cfg: StrawBossConfig, project: ProjectDef): seq[RunRequest] =
|
||||
let runsDir = cfg.artifactsRepo & "/" & project.name & "/runs"
|
||||
if not existsDir(runsDir): return @[]
|
||||
|
||||
let runPaths = toSeq(walkFiles(runsDir & "/*.json"))
|
||||
return runPaths.mapIt(parseRunRequest(parseFile(it)))
|
||||
|
||||
proc getCurrentProjectConfig*(cfg: StrawBossConfig, project: ProjectDef): Option[ProjectConfig] =
|
||||
let projCfgFile = cfg.artifactsRepo & "/" & project.name & "/" & project.cfgFilePath
|
||||
if not existsFile(projCfgFile): result = none(ProjectConfig)
|
||||
else:
|
||||
try:
|
||||
let projectConfig: ProjectConfig = loadProjectConfig(projCfgFile) #ProjectConfig(name: "test")
|
||||
result = some(projectConfig)
|
||||
except: result = none(ProjectConfig)
|
||||
|
||||
proc runStep*(wksp: Workspace, step: Step) =
|
||||
|
||||
## Lower-level method to execute a given step within the context of a project
|
||||
@ -139,14 +160,13 @@ proc runStep*(wksp: Workspace, step: Step) =
|
||||
|
||||
# Add the artifacts directory for the dependent step to our env so that
|
||||
# further steps can reference it via $<stepname>_DIR
|
||||
wksp.env[depStep.name & "_DIR"] = wksp.artifactsRepo & "/" &
|
||||
wksp.project.name & "/" & dep & "/" & wksp.version
|
||||
wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir & "/artifacts/" &
|
||||
"/" & dep & "/" & wksp.version
|
||||
|
||||
# Run the step command, piping in cmdInput
|
||||
wksp.outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd
|
||||
let cmdProc = startProcess(step.stepCmd,
|
||||
wksp.dir & "/repo/" & step.workingDir,
|
||||
[], wksp.env, {poUsePath, poEvalCommand})
|
||||
wksp.dir & step.workingDir, [], wksp.env, {poUsePath, poEvalCommand})
|
||||
|
||||
let cmdInStream = inputStream(cmdProc)
|
||||
|
||||
@ -167,11 +187,11 @@ proc runStep*(wksp: Workspace, step: Step) =
|
||||
let artifactPath = a.resolveEnvVars(wksp.env)
|
||||
let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
|
||||
try:
|
||||
wksp.outputHandler.sendMsg "copy " & wksp.dir & "/repo/" &
|
||||
wksp.outputHandler.sendMsg "copy " & wksp.dir &
|
||||
step.workingDir & "/" & artifactPath & " -> " &
|
||||
wksp.artifactsDir & "/" & artifactName
|
||||
|
||||
copyFile(wksp.dir & "/repo/" & step.workingDir & "/" & artifactPath,
|
||||
copyFile(wksp.dir & step.workingDir & "/" & artifactPath,
|
||||
wksp.artifactsDir & "/" & artifactName)
|
||||
except:
|
||||
raiseEx "step " & step.name & " failed: unable to copy artifact " &
|
||||
@ -186,6 +206,7 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
|
||||
## entrypoint to running a build step.
|
||||
|
||||
result = BuildStatus(
|
||||
runId: $req.id,
|
||||
state: "setup",
|
||||
details: "initializing build workspace")
|
||||
discard emitStatus(result, nil, outputHandler)
|
||||
@ -193,29 +214,33 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
|
||||
var wksp: Workspace
|
||||
|
||||
try:
|
||||
assert req.workspaceDir.isAbsolute
|
||||
if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
|
||||
|
||||
# Find the project definition
|
||||
let projectDef = cfg.findProject(req.projectName)
|
||||
|
||||
# Make sure our directory in the artifacts repo exists
|
||||
if not existsDir(cfg.artifactsRepo & "/" & projectDef.name & "/run-requests"):
|
||||
createDir(cfg.artifactsRepo & "/" & projectDef.name & "/run-requests")
|
||||
# Make sure the build data directories for this project exist.
|
||||
ensureProjectDirsExist(cfg, projectDef)
|
||||
|
||||
# Update our run status
|
||||
let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs"
|
||||
discard emitStatus(result, runDir & "/" & $req.id & ".status.json", nil)
|
||||
|
||||
# Read in the existing system environment
|
||||
var env = loadEnv()
|
||||
env["GIT_DIR"] = ".git"
|
||||
|
||||
# Make sure we have a workspace directory
|
||||
assert req.workspaceDir.isAbsolute
|
||||
if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
|
||||
|
||||
# Setup our STDOUT and STDERR files
|
||||
let stdoutFile = open(req.workspaceDir & "/stdout.log", fmWrite)
|
||||
let stderrFile = open(req.workspaceDir & "/stderr.log", fmWrite)
|
||||
let stdoutFile = open(runDir & "/" & $req.id & ".stdout.log", fmWrite)
|
||||
let stderrFile = open(runDir & "/" & $req.id & ".stderr.log", fmWrite)
|
||||
|
||||
let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile)
|
||||
|
||||
wksp = Workspace(
|
||||
artifactsDir: nil,
|
||||
artifactsRepo: cfg.artifactsRepo,
|
||||
buildDataDir: cfg.buildDataDir & "/" & projectDef.name,
|
||||
buildRef:
|
||||
if req.buildRef != nil and req.buildRef.len > 0: req.buildRef
|
||||
else: projectDef.defaultBranch,
|
||||
@ -225,14 +250,15 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
|
||||
outputHandler: combineProcMsgHandlers(outputHandler, logFilesOH),
|
||||
project: ProjectConfig(),
|
||||
projectDef: projectDef,
|
||||
runRequest: req,
|
||||
status: result,
|
||||
statusFile: req.workspaceDir & "/" & "status.json",
|
||||
statusFile: runDir & "/" & $req.id & ".status.json",
|
||||
step: Step(),
|
||||
version: nil)
|
||||
|
||||
except:
|
||||
when not defined(release): echo getCurrentException().getStackTrace()
|
||||
result = BuildStatus(state: "failed",
|
||||
result = BuildStatus(runId: $req.id, state: "failed",
|
||||
details: getCurrentExceptionMsg())
|
||||
try: discard emitStatus(result, nil, outputHandler)
|
||||
except: discard ""
|
||||
@ -244,33 +270,26 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
|
||||
"cloning project repo and preparing to run '" & req.stepName & "'")
|
||||
wksp.setupProject()
|
||||
|
||||
# Make sure our project directory exists in the artifacts repo
|
||||
if not existsDir(wksp.artifactsRepo & "/" & wksp.project.name):
|
||||
createDir(wksp.artifactsRepo & "/" & wksp.project.name)
|
||||
|
||||
# Update our cache of project configurations by copying the configuration
|
||||
# file to our artifacts directory.
|
||||
# Update our cache of project configurations.
|
||||
# TODO: what happens if this fails?
|
||||
copyFile(
|
||||
wksp.dir & "/repo/" & wksp.projectDef.cfgFilePath,
|
||||
cfg.artifactsRepo & "/" & wksp.project.name & "/configuration." &
|
||||
wksp.version & ".json")
|
||||
wksp.dir & "/" & wksp.projectDef.cfgFilePath,
|
||||
wksp.buildDataDir & "/configurations/" & wksp.version & ".json")
|
||||
|
||||
# Find the requested step
|
||||
if not wksp.project.steps.hasKey(req.stepName):
|
||||
raiseEx "no step name '" & req.stepName & "' for " & req.projectName
|
||||
|
||||
var step = wksp.project.steps[req.stepName]
|
||||
|
||||
# Enfore forceRebuild
|
||||
if req.forceRebuild: step.dontSkip = true
|
||||
|
||||
# Compose the path to the artifacts directory for this step and version
|
||||
wksp.artifactsDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" &
|
||||
wksp.artifactsDir = wksp.buildDataDir & "/artifacts/" &
|
||||
step.name & "/" & wksp.version
|
||||
|
||||
# Have we tried to build this before and are we caching the results?
|
||||
if existsFile(wksp.artifactsDir & "/status.json") and not step.dontSkip:
|
||||
let prevStatus = loadBuildStatus(wksp.artifactsDir & "/status.json")
|
||||
let statusFilePath = wksp.buildDataDir & "/status/" & wksp.version & ".json"
|
||||
if existsFile(statusFilePath) and not step.dontSkip:
|
||||
let prevStatus = loadBuildStatus(statusFilePath)
|
||||
|
||||
# If we succeeded last time, no need to rebuild
|
||||
if prevStatus.state == "complete":
|
||||
@ -283,21 +302,13 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
|
||||
"Rebuilding failed step '" & step.name & "' for version '" &
|
||||
wksp.version & "'.")
|
||||
|
||||
# Make the artifacts directory if it doesn't already exist
|
||||
if not existsDir(wksp.artifactsDir): createDir(wksp.artifactsDir)
|
||||
|
||||
# Link status file and output logs to the artifacts dir
|
||||
for fn in @["status.json", "stdout.log", "stderr.log"]:
|
||||
# TODO: roll old files instead of delete them?
|
||||
if existsFile(wksp.artifactsDir & "/" & fn):
|
||||
removeFile(wksp.artifactsDir & "/" & fn)
|
||||
|
||||
if link(wksp.dir & "/" & fn, wksp.artifactsDir & "/" & fn) != 0:
|
||||
wksp.outputHandler.sendErrMsg(
|
||||
"WARN: could not link " & fn & " to artifacts dir.")
|
||||
|
||||
runStep(wksp, step)
|
||||
|
||||
# Record the results of this build as the status for this version.
|
||||
writeFile(wksp.buildDataDir & "/status/" & wksp.version & ".json", $wksp.status)
|
||||
|
||||
result = wksp.status
|
||||
|
||||
except:
|
||||
@ -307,7 +318,7 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
|
||||
wksp.publishStatus("failed", msg)
|
||||
result = wksp.status
|
||||
except:
|
||||
result = BuildStatus(state: "failed", details: msg)
|
||||
result = BuildStatus(runId: $req.id, state: "failed", details: msg)
|
||||
try: discard emitStatus(result, nil, outputHandler)
|
||||
except: discard ""
|
||||
|
||||
@ -317,3 +328,38 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest,
|
||||
try: close(f)
|
||||
except: discard ""
|
||||
|
||||
proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest): Worker =
|
||||
|
||||
# Find the project definition (will throw appropriate exceptions)
|
||||
let projectDef = cfg.findProject(req.projectName)
|
||||
let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs"
|
||||
let reqFile = runDir & "/" & $req.id & ".request.json"
|
||||
let statusFile = runDir & "/" & $req.id & ".status.json"
|
||||
|
||||
try:
|
||||
# Make sure the build data directories for this project exist.
|
||||
ensureProjectDirsExist(cfg, projectDef)
|
||||
|
||||
# Save the run request
|
||||
writeFile(reqFile, $req)
|
||||
|
||||
# Write the initial build status (queued).
|
||||
let queuedStatus = BuildStatus(
|
||||
runId: $req.id,
|
||||
state: "queued",
|
||||
details: "request queued for execution")
|
||||
writeFile(statusFile, $queuedStatus)
|
||||
|
||||
var args = @["run", reqFile]
|
||||
debug "Launching worker: " & cfg.pathToExe & " " & args.join(" ")
|
||||
result = Worker(
|
||||
runId: req.id,
|
||||
process: startProcess(cfg.pathToExe, ".", args, loadEnv(), {poUsePath}))
|
||||
|
||||
except:
|
||||
let exMsg = "run request rejected: " & getCurrentExceptionMsg()
|
||||
raiseEx exMsg
|
||||
try:
|
||||
writeFile(statusFile,
|
||||
$(BuildStatus(runId: $req.id, state: "rejected", details: exMsg)))
|
||||
except: discard ""
|
||||
|
||||
@ -1,12 +1,8 @@
|
||||
import algorithm, asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging,
|
||||
options, os, osproc, sequtils, strutils, tempfile, times, unittest
|
||||
options, os, osproc, sequtils, strutils, tempfile, times, unittest, uuids
|
||||
|
||||
import ./configuration, ./core
|
||||
|
||||
type Worker = object
|
||||
process*: Process
|
||||
workingDir*: string
|
||||
|
||||
type
|
||||
Session = object
|
||||
user*: UserRef
|
||||
@ -14,6 +10,7 @@ type
|
||||
|
||||
#const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"
|
||||
const JSON = "application/json"
|
||||
const CLEANUP_PERIOD_MS = 1000
|
||||
|
||||
proc makeJsonResp(status: HttpCode, details: string = ""): string =
|
||||
result = $(%* {
|
||||
@ -71,18 +68,6 @@ proc extractSession(cfg: StrawBossConfig, request: Request): Session =
|
||||
|
||||
result = fromJWT(cfg, headerVal[7..^1])
|
||||
|
||||
proc spawnWorker(cfg: StrawBossConfig, req: RunRequest): Worker =
|
||||
## Kick off a new worker process with the given run information
|
||||
|
||||
let dir = mkdtemp()
|
||||
var args = @["run", req.projectName, req.stepName, "-r", req.buildRef,
|
||||
"-w", dir, "-c", cfg.filePath]
|
||||
if req.forceRebuild: args.add("-f")
|
||||
debug "Launching worker: " & cfg.pathToExe & " " & args.join(" ")
|
||||
result = Worker(
|
||||
process: startProcess(cfg.pathToExe, ".", args, loadEnv(), {poUsePath}),
|
||||
workingDir: dir)
|
||||
|
||||
proc hashPwd*(pwd: string, cost: int8): string =
|
||||
let salt = genSalt(cost)
|
||||
result = hash(pwd, salt)
|
||||
@ -124,9 +109,19 @@ template checkAuth() =
|
||||
debug "Auth failed: " & getCurrentExceptionMsg()
|
||||
resp(Http401, makeJsonResp(Http401), JSON)
|
||||
|
||||
proc performMaintenance(cfg: StrawBossConfig): void =
|
||||
# Prune workers
|
||||
workers = workers.filterIt(it.running())
|
||||
|
||||
let fut = sleepAsync(CLEANUP_PERIOD_MS)
|
||||
fut.callback =
|
||||
proc(): void =
|
||||
callSoon(proc(): void = performMaintenance(cfg))
|
||||
|
||||
|
||||
proc start*(cfg: StrawBossConfig): void =
|
||||
|
||||
let stopFuture = newFuture[void]()
|
||||
var stopFuture = newFuture[void]()
|
||||
var workers: seq[Worker] = @[]
|
||||
|
||||
# TODO: add recurring clean-up down to clear completed workers from the
|
||||
@ -138,7 +133,7 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
|
||||
routes:
|
||||
|
||||
get "/ping": resp($(%*"pong"), JSON)
|
||||
get "/ping": resp($(%"pong"), JSON)
|
||||
|
||||
post "/auth-token":
|
||||
var uname, pwd: string
|
||||
@ -150,7 +145,7 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
|
||||
try:
|
||||
let authToken = makeAuthToken(cfg, uname, pwd)
|
||||
resp("\"" & $authToken & "\"", JSON)
|
||||
resp($(%authToken), JSON)
|
||||
except: resp(Http401, makeJsonResp(Http401, getCurrentExceptionMsg()), JSON)
|
||||
|
||||
get "/verify-auth":
|
||||
@ -163,7 +158,7 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
|
||||
checkAuth(); if not authed: return true
|
||||
|
||||
resp($(%(cfg.projects)), JSON)
|
||||
resp($(%cfg.projects), JSON)
|
||||
|
||||
post "/projects":
|
||||
## Create a new project definition
|
||||
@ -183,13 +178,7 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
try: project = cfg.findProject(@"projectName")
|
||||
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
|
||||
|
||||
var versions: seq[string] = @[]
|
||||
for cfgFilePath in walkFiles(cfg.artifactsRepo & "/" & project.name & "/configuration.*.json"):
|
||||
let vstart = cfgFilePath.rfind("/configuration.") + 15
|
||||
versions.add(cfgFilePath[vstart..^6])
|
||||
|
||||
if versions.len == 0:
|
||||
resp(Http404, makeJsonResp(Http404, "I have not built any versions of " & project.name), JSON)
|
||||
var versions: seq[string] = listVersions(cfg, project)
|
||||
|
||||
resp($(%(versions)), JSON)
|
||||
|
||||
@ -206,8 +195,8 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
# Given version
|
||||
var cachedFilePath: string
|
||||
if @"version" != "":
|
||||
cachedFilePath = cfg.artifactsRepo & "/" & project.name &
|
||||
"/configuration." & @"version" & ".json"
|
||||
cachedFilePath = cfg.buildDataDir & "/" & project.name &
|
||||
"/configurations/" & @"version" & ".json"
|
||||
|
||||
if not existsFile(cachedFilePath):
|
||||
resp(Http404,
|
||||
@ -216,7 +205,7 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
|
||||
# No version requested, use "latest"
|
||||
else:
|
||||
let confFilePaths = toSeq(walkFiles(cfg.artifactsRepo & "/" & project.name & "/configuration.*.json"))
|
||||
let confFilePaths = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/configurations/*.json"))
|
||||
if confFilePaths.len == 0:
|
||||
resp(Http404, makeJsonResp(Http404, "I have not built any versions of " & project.name), JSON)
|
||||
let modTimes = confFilePaths.mapIt(it.getLastModificationTime)
|
||||
@ -230,7 +219,7 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
resp(Http500, makeJsonResp(Http500, "could not read cached project configuration"), JSON)
|
||||
|
||||
get "/project/@projectName":
|
||||
## TBD
|
||||
## Return a project's configuration, as well as it's versions.
|
||||
|
||||
checkAuth(); if not authed: return true
|
||||
|
||||
@ -239,17 +228,11 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
try: projDef = cfg.findProject(@"projectName")
|
||||
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
|
||||
|
||||
# Get the project configuration.
|
||||
let projConf = getCurrentProjectConfig(cfg, projDef)
|
||||
let respJson = newJObject()
|
||||
respJson["definition"] = %projDef
|
||||
respJson["versions"] = %listVersions(cfg, projDef)
|
||||
|
||||
var respObj = newJObject()
|
||||
respObj["definition"] = %projDef
|
||||
|
||||
if projConf.isSome():
|
||||
let pc: ProjectConfig = projConf.get()
|
||||
respObj["configuration"] = %pc
|
||||
|
||||
resp($respObj, JSON)
|
||||
resp(pretty(respJson), JSON)
|
||||
|
||||
get "/project/@projectName/runs":
|
||||
## List all runs
|
||||
@ -263,7 +246,7 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
|
||||
let runRequests = listRuns(cfg, project)
|
||||
|
||||
resp($(%runRequests), JSON)
|
||||
resp($runRequests, JSON)
|
||||
|
||||
get "/project/@projectName/runs/active":
|
||||
## List all currently active runs
|
||||
@ -275,6 +258,14 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
#resp($(%statuses), JSON)
|
||||
resp(Http501, makeJsonResp(Http501), JSON)
|
||||
|
||||
get "/project/@projectName/runs/@runId":
|
||||
## Details for a specific run
|
||||
|
||||
checkAuth(); if not authed: return true
|
||||
|
||||
# TODO
|
||||
resp(Http501, makeJsonResp(Http501), JSON)
|
||||
|
||||
get "/project/@projectName/step/@stepName":
|
||||
## Get step details including runs.
|
||||
|
||||
@ -297,6 +288,7 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
checkAuth(); if not authed: return true
|
||||
|
||||
let runRequest = RunRequest(
|
||||
id: genUUID(),
|
||||
projectName: @"projectName",
|
||||
stepName: @"stepName",
|
||||
buildRef: if @"buildRef" != "": @"buildRef" else: nil,
|
||||
@ -316,8 +308,9 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
post "/service/debug/stop":
|
||||
if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON)
|
||||
else:
|
||||
callSoon(proc(): void = complete(stopFuture))
|
||||
resp($(%*"shutting down"), JSON)
|
||||
let shutdownFut = sleepAsync(100)
|
||||
shutdownFut.callback = proc(): void = complete(stopFuture)
|
||||
resp($(%"shutting down"), JSON)
|
||||
|
||||
#[
|
||||
get re".*":
|
||||
@ -327,4 +320,5 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
resp(Http404, makeJsonResp(Http404), JSON)
|
||||
]#
|
||||
|
||||
callSoon(proc(): void = performMaintenance(cfg))
|
||||
waitFor(stopFuture)
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
{
|
||||
"runId": "90843e0c-6113-4462-af33-a89ff9731031",
|
||||
"state": "failed",
|
||||
"details": "some very good reason"
|
||||
}
|
||||
|
||||
@ -17,7 +17,7 @@ let TIMEOUT = 2.minutes
|
||||
# configuration and working files.
|
||||
template keepEnv(): untyped =
|
||||
preserveEnv = true
|
||||
echo "artifacts dir: " & tempArtifactsDir
|
||||
echo "artifacts dir: " & tempBuildDataDir
|
||||
echo "strawboss serve -c " & tempCfgPath
|
||||
|
||||
suite "strawboss server":
|
||||
@ -30,13 +30,13 @@ suite "strawboss server":
|
||||
|
||||
# per-test setup: spin up a fresh strawboss instance
|
||||
setup:
|
||||
let tempArtifactsDir = mkdtemp()
|
||||
let tempBuildDataDir = mkdtemp()
|
||||
let (_, tempCfgPath) = mkstemp()
|
||||
var preserveEnv = false
|
||||
|
||||
# copy our test config
|
||||
var newCfg = cfg
|
||||
newCfg.artifactsRepo = tempArtifactsDir
|
||||
newCfg.buildDataDir = tempBuildDataDir
|
||||
|
||||
# update the repo string for the extracted test project
|
||||
var testProjDef = newCfg.findProject(testProjName)
|
||||
@ -55,7 +55,7 @@ suite "strawboss server":
|
||||
discard newAsyncHttpClient().post(apiBase & "/service/debug/stop")
|
||||
|
||||
if not preserveEnv:
|
||||
removeDir(tempArtifactsDir)
|
||||
removeDir(tempBuildDataDir)
|
||||
removeFile(tempCfgPath)
|
||||
|
||||
# give the server time to spin down but kill it after that
|
||||
@ -73,14 +73,14 @@ suite "strawboss server":
|
||||
check resp.status.startsWith("404")
|
||||
|
||||
test "GET /api/project/@projectName/versions":
|
||||
let projArtifactsDir = tempArtifactsDir & "/" & testProjName
|
||||
let cachedConfsDir = tempBuildDataDir & "/" & testProjName & "/configurations"
|
||||
let expectedVersions = @["alpha", "beta", "1.0.0", "1.0.1"]
|
||||
|
||||
# Touch configuration files
|
||||
createDir(projArtifactsDir)
|
||||
createDir(cachedConfsDir)
|
||||
for v in expectedVersions:
|
||||
var f: File
|
||||
check open(f, projArtifactsDir & "/configuration." & v & ".json", fmWrite)
|
||||
check open(f, cachedConfsDir & "/" & v & ".json", fmWrite)
|
||||
close(f)
|
||||
|
||||
let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password")
|
||||
@ -96,12 +96,13 @@ suite "strawboss server":
|
||||
# give the filesystem time to create stuff
|
||||
sleep(100)
|
||||
|
||||
# check that the run request has been
|
||||
# check that the project directory has been created in the artifacts repo
|
||||
let runArtifactsDir = tempArtifactsDir & "/" & testProjName & "/build/0.1.0"
|
||||
let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build/0.1.0"
|
||||
check existsDir(runArtifactsDir)
|
||||
|
||||
# check that the run status file has been created in the artifacts repo
|
||||
let statusFile = runArtifactsDir & "/status.json"
|
||||
let statusFile = tempBuildDataDir & "/" & testProjName & "/status/0.1.0.json"
|
||||
check fileExists(statusFile)
|
||||
|
||||
# check that the run status is not failed
|
||||
|
||||
@ -21,6 +21,7 @@ suite "load and save configuration objects":
|
||||
|
||||
test "parseRunRequest":
|
||||
let rr1 = RunRequest(
|
||||
id: genUUID(),
|
||||
projectName: testProjDef.name,
|
||||
stepName: "build",
|
||||
buildRef: "master",
|
||||
@ -85,7 +86,7 @@ suite "load and save configuration objects":
|
||||
envVars: newStringTable(modeCaseSensitive))]
|
||||
|
||||
check:
|
||||
cfg.artifactsRepo == "artifacts"
|
||||
cfg.buildDataDir == "build-data"
|
||||
cfg.authSecret == "change me"
|
||||
cfg.pwdCost == 11
|
||||
sameContents(expectedUsers, cfg.users)
|
||||
@ -141,5 +142,6 @@ suite "load and save configuration objects":
|
||||
let st = loadBuildStatus("src/test/json/test-status.json")
|
||||
|
||||
check:
|
||||
st.runId == "90843e0c-6113-4462-af33-a89ff9731031"
|
||||
st.state == "failed"
|
||||
st.details == "some very good reason"
|
||||
|
||||
@ -19,7 +19,9 @@ requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git"
|
||||
# Tasks
|
||||
#
|
||||
task functest, "Runs the functional test suite.":
|
||||
exec "nimble build"
|
||||
exec "nim c -r src/test/nim/run_functional_tests.nim"
|
||||
|
||||
task unittest, "Runs the unit test suite.":
|
||||
exec "nimble build"
|
||||
exec "nim c -r src/test/nim/run_unit_tests.nim"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user