Artifact and log lookups. Bugfixes around failure scenarios.
* Added endopint to return the logs from a run.
* Refactored the `pathVar & "/" & pathvar` pattern into `pathVar / pathVar`
using the `ospaths` module. Cleaner code, more resistant to extra `/` bugs.
* Added endpoints and core methods to list artifacts for a build, as well as to
retrieve specific artifacts.
* Fixed a problem with the `complete` status being overloaded. The problem was
that in the case of a multi-step build all of the prerequisite steps will
return a state of `complete` which will get recorded in the status file for
the run. The run will continue, but anyone watching the run state file (via
the API for example) had no definitive way to tell the difference between a
sub-step completing and the requested (last) step completing. This was caught
in the functional tests (race condition based on when it polls for the run
status). The fix was to introduce a new build state: `stepComplete`. The
inner `doRun` procedure uses this instead of `complete`. Now the only place
that `complete` is set is at the end of the original call to `run`, right
before the worker terminates. It checks the last result (from the originally
requested step) and if this result is `stepComplete` it "finishes" the build
by setting the state to `complete`. Because this is the only place where
`complete` is set, an observer is now guaranteed not to see `complete` until
all steps have run successfully.
* Fixed a long-standing bug with the request handling logic in error cases
(like requested resources not being available). Issue has something to do
with the way that `except` blocks become special when in an async context.
The main jester routes block invokes the handlers in an async context. The
effect is that one `except` block is fine, but adding more than one (to catch
different exception types, for example) causes the return type of the route
handler to change and not match what the outer block is expecting (a Future).
The fix here is to wrap any exception discrimination within a single outer
except block, re-raise the exception, and catch it inside this new
synchronous context. Ex:
```nim
try: someCall(mayFail)
except:
try: raise getCurrentException()
except ExceptionType1:
# do whatever...
except ExceptionType2:
# do whatever
except:
# general catch-all
return true
```
The return at the end is also part of the story. Jester's match handler
allows a route to defer making a decision about whether it matches. If you
return true from a route block Jester accepts the result as a matched route.
If you return false, Jester discards the result and looks for another
matching route. Normally this is taken care of by the `resp` templates
provided by Jester, but invoking those templates within the except blocks
also causes problems, so we manually setup the response and `return true` to
tell Jester that, yes this route matched, use this response.
* Moved the `/service/debug/ping` endpoint back to `/ping` and removed the
debug-only fence. I envision this as being useful as a simple healthcheck URL.
This commit is contained in:
Jonathan Bernard
parent
07037616ac
commit
c6be698572
17
TODO.md
17
TODO.md
@ -1,8 +1,11 @@
|
||||
TODO
|
||||
|
||||
* Orchestration of docker containers for running builds.
|
||||
* GET /api/project/<project-name>/run/logs
|
||||
* Write a tool to convert JSON Schema into a human-readable format suitable for
|
||||
documentation. Should use the description, title, and other fields from the
|
||||
JSON spec. Use this for writing the JSON schema docs instead of duplicating
|
||||
the description of configuration files between JSON schema and the
|
||||
documentation. In other words, use the schemas as the single source of truth
|
||||
and generate everything else from that.
|
||||
* Write API docs.
|
||||
|
||||
NICE TO HAVE
|
||||
|
||||
* Use/create some json-schema -> nim code generator to auto-generate json
|
||||
handling code from schemas.
|
||||
* Use some json-schema -> docs generator to document the API.
|
||||
* Support unique UUID prefixes in URLs.
|
||||
|
||||
@ -10,7 +10,7 @@ const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz"
|
||||
#
|
||||
type
|
||||
BuildState* {.pure.} = enum
|
||||
queued, complete, failed, running, setup, rejected
|
||||
complete, failed, queued, rejected, running, setup, stepComplete
|
||||
|
||||
BuildStatus* = object
|
||||
runId*, details*: string
|
||||
@ -41,6 +41,10 @@ type
|
||||
request*: RunRequest
|
||||
status*: BuildStatus
|
||||
|
||||
RunLogs* = object
|
||||
runId*: UUID
|
||||
stdout*, stderr*: seq[string]
|
||||
|
||||
User* = object
|
||||
name*: string
|
||||
hashedPwd*: string
|
||||
@ -296,8 +300,15 @@ proc `%`*(run: Run): JsonNode =
|
||||
"request": %run.request,
|
||||
"status": %run.status }
|
||||
|
||||
proc `%`*(logs: RunLogs): JsonNode =
|
||||
result = %* {
|
||||
"runId": $logs.runId,
|
||||
"stdout": %logs.stdout,
|
||||
"stderr": %logs.stderr }
|
||||
|
||||
proc `$`*(s: BuildStatus): string = result = pretty(%s)
|
||||
proc `$`*(req: RunRequest): string = result = pretty(%req)
|
||||
proc `$`*(pd: ProjectDef): string = result = pretty(%pd)
|
||||
proc `$`*(cfg: StrawBossConfig): string = result = pretty(%cfg)
|
||||
proc `$`*(run: Run): string = result = pretty(%run)
|
||||
proc `$`*(logs: RunLogs): string = result = pretty(%logs)
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
import cliutils, logging, json, os, osproc, sequtils, streams,
|
||||
import cliutils, logging, json, os, ospaths, osproc, sequtils, streams,
|
||||
strtabs, strutils, tables, times, uuids
|
||||
|
||||
import ./configuration
|
||||
import nre except toSeq
|
||||
from posix import link
|
||||
from posix import link, realpath
|
||||
from algorithm import sorted
|
||||
|
||||
type
|
||||
@ -27,7 +27,7 @@ type
|
||||
projectName*: string
|
||||
process*: Process
|
||||
|
||||
NotFoundException = object of Exception
|
||||
NotFoundException* = object of Exception
|
||||
|
||||
proc newCopy(w: Workspace): Workspace =
|
||||
var newEnv: StringTableRef = newStringTable()
|
||||
@ -82,27 +82,27 @@ proc publishStatus(wksp: Workspace, state: BuildState, details: string): void =
|
||||
runId: $wksp.runRequest.runId, state: state, details: details)
|
||||
|
||||
# Write to our run directory, and to our version status
|
||||
writeFile(wksp.buildDataDir & "/runs/" &
|
||||
writeFile(wksp.buildDataDir / "runs" /
|
||||
$wksp.runRequest.runId & ".status.json", $wksp.status)
|
||||
|
||||
# If we have our step we can save status to the step status
|
||||
if not wksp.step.name.isNilOrEmpty():
|
||||
let stepStatusDir = wksp.buildDataDir & "/status/" & wksp.step.name
|
||||
let stepStatusDir = wksp.buildDataDir / "status" / wksp.step.name
|
||||
if not existsDir(stepStatusDir): createDir(stepStatusDir)
|
||||
writeFile(stepStatusDir & "/" & wksp.version & ".json", $wksp.status)
|
||||
writeFile(stepStatusDir / wksp.version & ".json", $wksp.status)
|
||||
|
||||
# If we were asked to build a ref that is not the version directly (like
|
||||
# "master" or something), then let's also save our status under that name.
|
||||
# We're probably overwriting a prior status, but that's OK.
|
||||
if wksp.runRequest.buildRef != wksp.version:
|
||||
writeFile(stepStatusDir & "/" & wksp.runRequest.buildRef & ".json",
|
||||
writeFile(stepStatusDir / wksp.runRequest.buildRef & ".json",
|
||||
$wksp.status)
|
||||
|
||||
wksp.outputHandler.sendStatusMsg(wksp.status)
|
||||
|
||||
proc ensureProjectDirsExist(cfg: StrawBossConfig, p: ProjectDef): void =
|
||||
for subdir in ["configurations", "runs", "status", "artifacts"]:
|
||||
let fullPath = cfg.buildDataDir & "/" & p.name & "/" & subdir
|
||||
let fullPath = cfg.buildDataDir / p.name / subdir
|
||||
if not existsDir(fullPath):
|
||||
createDir(fullPath)
|
||||
|
||||
@ -112,9 +112,9 @@ proc getProject*(cfg: StrawBossConfig, projectName: string): ProjectDef =
|
||||
## Get a project definition by name from the service configuration
|
||||
let candidates = cfg.projects.filterIt(it.name == projectName)
|
||||
if candidates.len == 0:
|
||||
raise newException(KeyError, "no project named " & projectName)
|
||||
raise newException(NotFoundException, "no project named " & projectName)
|
||||
elif candidates.len > 1:
|
||||
raise newException(KeyError, "multiple projects named " & projectName)
|
||||
raise newException(NotFoundException, "multiple projects named " & projectName)
|
||||
else: result = candidates[0]
|
||||
|
||||
proc setProject*(cfg: var StrawBossConfig, projectName: string, newDef: ProjectDef): void =
|
||||
@ -136,23 +136,67 @@ proc listVersions*(cfg: StrawBossConfig, projectName: string): seq[string] =
|
||||
ensureProjectDirsExist(cfg, project)
|
||||
|
||||
let versionFiles = filesMatching(
|
||||
cfg.buildDataDir & "/" & project.name & "/configurations/*.json")
|
||||
cfg.buildDataDir / project.name / "configurations/*.json")
|
||||
|
||||
result = versionFiles.map(proc(s: string): string =
|
||||
let slashIdx = s.rfind('/')
|
||||
result = s[(slashIdx + 1)..^6])
|
||||
|
||||
proc getBuildStatus*(cfg: StrawBossConfig,
|
||||
projectName, stepName, buildRef: string): BuildStatus =
|
||||
|
||||
let project = cfg.getProject(projectName)
|
||||
|
||||
let statusFile = cfg.buildDataDir / project.name / "status" /
|
||||
stepName / buildRef & ".json"
|
||||
|
||||
if not existsFile(statusFile):
|
||||
raise newException(NotFoundException,
|
||||
stepName & " has never been built for " & projectName & "@" & buildRef)
|
||||
|
||||
result = loadBuildStatus(statusFile)
|
||||
|
||||
|
||||
proc listArtifacts*(cfg: StrawBossConfig,
|
||||
projectName, stepName, version: string): seq[string] =
|
||||
## List the artifacts that have been built for a step.
|
||||
|
||||
let project = cfg.getProject(projectName)
|
||||
|
||||
ensureProjectDirsExist(cfg, project)
|
||||
|
||||
let buildStatus = cfg.getBuildStatus(projectName, stepName, version)
|
||||
|
||||
if buildStatus.state != BuildState.complete:
|
||||
raise newException(NotFoundException, "step " & stepName &
|
||||
" has never been successfully built for " & projectName & "@" & version)
|
||||
|
||||
result = filesMatching(
|
||||
cfg.buildDataDir / project.name / "artifacts" / stepName / version / "*")
|
||||
.mapIt(it.extractFilename)
|
||||
|
||||
proc getArtifactPath*(cfg: StrawBossConfig,
|
||||
projectName, stepName, version, artifactName: string): string =
|
||||
|
||||
let artifacts = cfg.listArtifacts(projectName, stepName, version)
|
||||
if not artifacts.contains(artifactName):
|
||||
raise newException(NotFoundException, "no artifact named " &
|
||||
artifactName & " exists for step " & stepName & " in project " &
|
||||
projectName & "@" & version)
|
||||
|
||||
result = cfg.buildDataDir / projectName / "artifacts" / stepName / version / artifactName
|
||||
|
||||
proc existsRun*(cfg: StrawBossConfig, projectName, runId: string): bool =
|
||||
existsFile(cfg.buildDataDir & "/" & projectName & "/runs/" & runId & ".request.json")
|
||||
existsFile(cfg.buildDataDir / projectName / "runs" / runId & ".request.json")
|
||||
|
||||
proc getRun*(cfg: StrawBossConfig, projectName, runId: string): Run =
|
||||
let project = cfg.getProject(projectName)
|
||||
let runsPath = cfg.buildDataDir & "/" & project.name & "/runs"
|
||||
let runsPath = cfg.buildDataDir / project.name / "runs"
|
||||
|
||||
try: result = Run(
|
||||
id: parseUUID(runId),
|
||||
request: loadRunRequest(runsPath & "/" & runId & ".request.json"),
|
||||
status: loadBuildStatus(runsPath & "/" & runId & ".status.json"))
|
||||
request: loadRunRequest(runsPath / runId & ".request.json"),
|
||||
status: loadBuildStatus(runsPath / runId & ".status.json"))
|
||||
except: raiseEx "unable to load run information for id " & runId
|
||||
|
||||
proc listRuns*(cfg: StrawBossConfig, projectName: string): seq[Run] =
|
||||
@ -160,29 +204,25 @@ proc listRuns*(cfg: StrawBossConfig, projectName: string): seq[Run] =
|
||||
let project = cfg.getProject(projectName)
|
||||
ensureProjectDirsExist(cfg, project)
|
||||
|
||||
let runsPath = cfg.buildDataDir & "/" & project.name & "/runs"
|
||||
let reqPaths = filesMatching(runsPath & "/*.request.json")
|
||||
let runsPath = cfg.buildDataDir / project.name / "runs"
|
||||
let reqPaths = filesMatching(runsPath / "*.request.json")
|
||||
|
||||
result = reqPaths.map(proc(reqPath: string): Run =
|
||||
let runId = reqPath[(runsPath.len + 1)..^14]
|
||||
result = Run(
|
||||
id: parseUUID(runId),
|
||||
request: loadRunRequest(reqPath),
|
||||
status: loadBuildStatus(runsPath & "/" & runId & ".status.json")))
|
||||
|
||||
proc getBuildStatus*(cfg: StrawBossConfig,
|
||||
projectName, stepName, buildRef: string): BuildStatus =
|
||||
status: loadBuildStatus(runsPath / runId & ".status.json")))
|
||||
|
||||
proc getLogs*(cfg: StrawBossConfig, projectname, runId: string): RunLogs =
|
||||
let project = cfg.getProject(projectName)
|
||||
let runsPath = cfg.buildDataDir / project.name / "runs"
|
||||
|
||||
let statusFile = cfg.buildDataDir & "/" & project.name & "/status/" &
|
||||
stepName & "/" & buildRef & ".json"
|
||||
|
||||
if not existsFile(statusFile):
|
||||
raise newException(NotFoundException,
|
||||
stepName & " has never been built for reference '" & buildRef)
|
||||
|
||||
result = loadBuildStatus(statusFile)
|
||||
try: result = RunLogs(
|
||||
runId: parseUUID(runId),
|
||||
stdout: toSeq(lines(runsPath / runId & ".stdout.log")),
|
||||
stderr: toSeq(lines(runsPath / runId & ".stderr.log")))
|
||||
except: raiseEx "unable to load logs for run " & runId
|
||||
|
||||
proc getProjectConfig*(cfg: StrawBossConfig,
|
||||
projectName, version: string): ProjectConfig =
|
||||
@ -196,7 +236,7 @@ proc getProjectConfig*(cfg: StrawBossConfig,
|
||||
if version.isNilOrEmpty:
|
||||
|
||||
let candidatePaths = filesMatching(
|
||||
cfg.buildDataDir & "/" & project.name & "/configurations/*.json")
|
||||
cfg.buildDataDir / project.name / "configurations/*.json")
|
||||
|
||||
if candidatePaths.len == 0:
|
||||
raise newException(NotFoundException,
|
||||
@ -212,8 +252,7 @@ proc getProjectConfig*(cfg: StrawBossConfig,
|
||||
# If they did, let's try to load that
|
||||
else:
|
||||
confFilePath =
|
||||
cfg.buildDataDir & "/" & project.name & "/configurations/" &
|
||||
version & ".json"
|
||||
cfg.buildDataDir / project.name / "configurations" / version & ".json"
|
||||
|
||||
if not existsFile(confFilePath):
|
||||
raise newException(NotFoundException,
|
||||
@ -249,7 +288,7 @@ proc setupProject(wksp: Workspace) =
|
||||
" for '" & wksp.projectDef.name & "'"
|
||||
|
||||
# Find the strawboss project configuration
|
||||
let projCfgFile = wksp.dir & "/" & wksp.projectDef.cfgFilePath
|
||||
let projCfgFile = wksp.dir / wksp.projectDef.cfgFilePath
|
||||
wksp.sendMsg(lvlDebug, "Looking for project configuration at '" & projCfgFile & "'")
|
||||
if not existsFile(projCfgFile):
|
||||
raiseEx "Cannot find strawboss project configuration in the project " &
|
||||
@ -283,21 +322,20 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
|
||||
|
||||
wksp.step = step
|
||||
|
||||
let artifactsDir = wksp.buildDataDir & "/artifacts/" &
|
||||
step.name & "/" & wksp.version
|
||||
let artifactsDir = wksp.buildDataDir / "artifacts" / step.name / wksp.version
|
||||
|
||||
if not existsDir(artifactsDir): createDir(artifactsDir)
|
||||
|
||||
# Have we tried to build this before and are we caching the results?
|
||||
let statusFilePath = wksp.buildDataDir & "/status/" & step.name &
|
||||
"/" & wksp.version & ".json"
|
||||
let statusFilePath = wksp.buildDataDir / "status" / step.name /
|
||||
wksp.version & ".json"
|
||||
|
||||
if existsFile(statusFilePath) and not step.dontSkip:
|
||||
let prevStatus = loadBuildStatus(statusFilePath)
|
||||
|
||||
# If we succeeded last time, no need to rebuild
|
||||
if prevStatus.state == BuildState.complete:
|
||||
wksp.publishStatus(BuildState.complete,
|
||||
wksp.publishStatus(BuildState.stepComplete,
|
||||
"Skipping step '" & step.name & "' for version '" & wksp.version &
|
||||
"': already completed.")
|
||||
return wksp.status
|
||||
@ -328,7 +366,7 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
|
||||
# Run that step (may get skipped)
|
||||
let runStatus = doStep(core.newCopy(wksp), depStep)
|
||||
|
||||
if not (runStatus.state == BuildState.complete):
|
||||
if not (runStatus.state == BuildState.stepComplete):
|
||||
raiseEx "dependent step failed: " & depStep.name
|
||||
|
||||
wksp.sendMsg(lvlDebug, "dependent step '" & depStep.name &
|
||||
@ -336,8 +374,8 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
|
||||
|
||||
# Add the artifacts directory for the dependent step to our env so that
|
||||
# further steps can reference it via $<stepname>_DIR
|
||||
wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir & "/artifacts/" &
|
||||
dep & "/" & wksp.version
|
||||
wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir / "artifacts" /
|
||||
dep / wksp.version
|
||||
|
||||
# Run the step command, piping in cmdInput
|
||||
let stepCmd = wksp.resolveEnvVars(step.stepCmd)
|
||||
@ -345,7 +383,7 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
|
||||
else: stepCmd
|
||||
wksp.sendMsg step.name & ": starting stepCmd: " & stepCmd
|
||||
let cmdProc = startProcess(stepCmd,
|
||||
wksp.dir & "/" & step.workingDir, [], wksp.env, {poUsePath, poEvalCommand})
|
||||
wksp.dir / step.workingDir, [], wksp.env, {poUsePath, poEvalCommand})
|
||||
|
||||
let cmdInStream = inputStream(cmdProc)
|
||||
|
||||
@ -367,16 +405,16 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
|
||||
let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
|
||||
try:
|
||||
wksp.sendMsg "copy " &
|
||||
wksp.dir & "/" & step.workingDir & "/" & artifactPath & " -> " &
|
||||
artifactsDir & "/" & artifactName
|
||||
wksp.dir / step.workingDir / artifactPath & " -> " &
|
||||
artifactsDir / artifactName
|
||||
|
||||
copyFileWithPermissions(wksp.dir & "/" & step.workingDir & "/" &
|
||||
artifactPath, artifactsDir & "/" & artifactName)
|
||||
copyFileWithPermissions(wksp.dir / step.workingDir / artifactPath,
|
||||
artifactsDir / artifactName)
|
||||
except:
|
||||
raiseEx "step " & step.name & " failed: unable to copy artifact " &
|
||||
artifactPath & ":\n" & getCurrentExceptionMsg()
|
||||
|
||||
wksp.publishStatus(BuildState.complete, "")
|
||||
wksp.publishStatus(BuildState.stepComplete, "step " & step.name & " complete")
|
||||
result = wksp.status
|
||||
|
||||
proc run*(cfg: StrawBossConfig, req: RunRequest,
|
||||
@ -401,8 +439,8 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
|
||||
ensureProjectDirsExist(cfg, projectDef)
|
||||
|
||||
# Update our run status
|
||||
let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs"
|
||||
writeFile(runDir & "/" & $req.runId & ".status.json", $result)
|
||||
let runDir = cfg.buildDataDir / projectDef.name / "runs"
|
||||
writeFile(runDir / $req.runId & ".status.json", $result)
|
||||
|
||||
# Read in the existing system environment
|
||||
var env = loadEnv()
|
||||
@ -413,13 +451,13 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
|
||||
if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
|
||||
|
||||
# Setup our STDOUT and STDERR files
|
||||
let stdoutFile = open(runDir & "/" & $req.runId & ".stdout.log", fmWrite)
|
||||
let stderrFile = open(runDir & "/" & $req.runId & ".stderr.log", fmWrite)
|
||||
let stdoutFile = open(runDir / $req.runId & ".stdout.log", fmWrite)
|
||||
let stderrFile = open(runDir / $req.runId & ".stderr.log", fmWrite)
|
||||
|
||||
let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile)
|
||||
|
||||
wksp = Workspace(
|
||||
buildDataDir: cfg.buildDataDir & "/" & projectDef.name,
|
||||
buildDataDir: cfg.buildDataDir / projectDef.name,
|
||||
buildRef:
|
||||
if req.buildRef != nil and req.buildRef.len > 0: req.buildRef
|
||||
else: projectDef.defaultBranch,
|
||||
@ -452,8 +490,8 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
|
||||
# Update our cache of project configurations.
|
||||
# TODO: what happens if this fails?
|
||||
copyFileWithPermissions(
|
||||
wksp.dir & "/" & wksp.projectDef.cfgFilePath,
|
||||
wksp.buildDataDir & "/configurations/" & wksp.version & ".json")
|
||||
wksp.dir / wksp.projectDef.cfgFilePath,
|
||||
wksp.buildDataDir / "configurations" / wksp.version & ".json")
|
||||
|
||||
# Find the requested step
|
||||
if not wksp.project.steps.hasKey(req.stepName):
|
||||
@ -462,7 +500,12 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
|
||||
|
||||
if req.forceRebuild: step.dontSkip = true
|
||||
|
||||
result = doStep(wksp, step)
|
||||
var buildStatus = doStep(wksp, step)
|
||||
if buildStatus.state == BuildState.stepComplete:
|
||||
buildStatus.state = BuildState.complete
|
||||
wksp.publishStatus(buildStatus.state, "all steps complete")
|
||||
|
||||
result = wksp.status
|
||||
|
||||
except:
|
||||
when not defined(release): echo getCurrentException().getStackTrace()
|
||||
@ -487,9 +530,9 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
|
||||
|
||||
# Find the project definition (will throw appropriate exceptions)
|
||||
let projectDef = cfg.getProject(req.projectName)
|
||||
let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs"
|
||||
let reqFile = runDir & "/" & $req.runId & ".request.json"
|
||||
let statusFile = runDir & "/" & $req.runId & ".status.json"
|
||||
let runDir = cfg.buildDataDir / projectDef.name / "runs"
|
||||
let reqFile = runDir / $req.runId & ".request.json"
|
||||
let statusFile = runDir / $req.runId & ".status.json"
|
||||
|
||||
try:
|
||||
# Make sure the build data directories for this project exist.
|
||||
@ -517,8 +560,8 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
|
||||
|
||||
except:
|
||||
let exMsg = "run request rejected: " & getCurrentExceptionMsg()
|
||||
raiseEx exMsg
|
||||
try:
|
||||
writeFile(statusFile,
|
||||
$(BuildStatus(runId: $req.runId, state: BuildState.rejected, details: exMsg)))
|
||||
except: discard ""
|
||||
raiseEx exMsg
|
||||
|
||||
@ -1,6 +1,11 @@
|
||||
import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging,
|
||||
import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, md5,
|
||||
os, osproc, sequtils, strutils, tempfile, times, unittest, uuids
|
||||
|
||||
from mimetypes import getMimeType
|
||||
from asyncfile import openAsync, readToStream, close
|
||||
from asyncnet import send
|
||||
from re import re, find
|
||||
|
||||
import ./configuration, ./core
|
||||
|
||||
type
|
||||
@ -17,13 +22,22 @@ proc newSession*(user: UserRef): Session =
|
||||
issuedAt: getTime(),
|
||||
expires: daysForward(7).toTime())
|
||||
|
||||
proc makeJsonResp(status: HttpCode, details: string = ""): string =
|
||||
result = $(%* {
|
||||
"statusCode": status.int,
|
||||
"status": $status,
|
||||
proc buildJson(resp: Response, code: HttpCode, details: string = ""): void =
|
||||
resp.data[0] = CallbackAction.TCActionSend
|
||||
resp.data[1] = code
|
||||
resp.data[2]["Content-Type"] = JSON
|
||||
resp.data[3] = $(%* {
|
||||
"statusCode": code.int,
|
||||
"status": $code,
|
||||
"details": details
|
||||
})
|
||||
|
||||
# Work-around for weirdness trying to use resp(Http500... in exception blocks
|
||||
proc build500Json(resp: Response, ex: ref Exception, msg: string): void =
|
||||
when not defined(release): debug ex.getStackTrace()
|
||||
error msg & ":\n" & ex.msg
|
||||
resp.buildJson(Http500)
|
||||
|
||||
proc toJWT*(cfg: StrawBossConfig, session: Session): string =
|
||||
## Make a JST token for this session.
|
||||
var jwt = JWT(
|
||||
@ -46,7 +60,6 @@ proc fromJWT*(cfg: StrawBossConfig, strTok: string): Session =
|
||||
# Find the user record (if authenticated)
|
||||
let username = jwt.claims["sub"].node.str
|
||||
let users = cfg.users.filterIt(it.name == username)
|
||||
debug "username: " & username & "\n\tusers: " & $users.mapIt(it.name) & "\n\tall users: " & cfg.users.mapIt(it.name)
|
||||
if users.len != 1: raiseEx "Could not find session user."
|
||||
|
||||
result = Session(
|
||||
@ -131,7 +144,8 @@ template checkAuth() =
|
||||
except:
|
||||
debug "Auth failed: " & getCurrentExceptionMsg()
|
||||
response.data[2]["WWW-Authenticate"] = "Bearer"
|
||||
resp(Http401, makeJsonResp(Http401), JSON)
|
||||
response.buildJson(Http401)
|
||||
return
|
||||
|
||||
proc start*(cfg: StrawBossConfig): void =
|
||||
|
||||
@ -144,18 +158,21 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
|
||||
routes:
|
||||
|
||||
get "/ping":
|
||||
resp($(%"pong"), JSON)
|
||||
|
||||
post "/auth-token":
|
||||
var uname, pwd: string
|
||||
try:
|
||||
let jsonBody = parseJson(request.body)
|
||||
uname = jsonBody["username"].getStr
|
||||
pwd = jsonBody["password"].getStr
|
||||
except: resp(Http400, makeJsonResp(Http400), JSON)
|
||||
except: response.buildJson(Http400); return
|
||||
|
||||
try:
|
||||
let authToken = makeAuthToken(cfg, uname, pwd)
|
||||
resp($(%authToken), JSON)
|
||||
except: resp(Http401, makeJsonResp(Http401, getCurrentExceptionMsg()), JSON)
|
||||
except: response.buildJson(Http401, getCurrentExceptionMsg()); return
|
||||
|
||||
get "/verify-auth":
|
||||
checkAuth(); if not authed: return true
|
||||
@ -175,7 +192,7 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
checkAuth(); if not authed: return true
|
||||
|
||||
# TODO
|
||||
resp(Http501, makeJsonResp(Http501), JSON)
|
||||
response.buildJson(Http501); return
|
||||
|
||||
get "/project/@projectName":
|
||||
## Return a project's configuration, as well as it's versions.
|
||||
@ -185,7 +202,14 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
# Make sure we know about that project
|
||||
var projDef: ProjectDef
|
||||
try: projDef = cfg.getProject(@"projectName")
|
||||
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
|
||||
except:
|
||||
try: raise getCurrentException()
|
||||
except NotFoundException:
|
||||
response.buildJson(Http404, getCurrentExceptionMsg())
|
||||
except:
|
||||
response.build500Json(getCurrentException(),
|
||||
"unable to load project definition for project " & @"projectName")
|
||||
return true
|
||||
|
||||
var projConf: ProjectConfig
|
||||
try: projConf = getProjectConfig(cfg, @"projectName", "")
|
||||
@ -206,13 +230,13 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
|
||||
try: resp($(%listVersions(cfg, @"projectName")), JSON)
|
||||
except:
|
||||
if getCurrentException() is KeyError:
|
||||
resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
|
||||
else:
|
||||
when not defined(release): debug getCurrentException().getStackTrace()
|
||||
error "unable to list versions for project " & @"projectName" &
|
||||
":\n" & getCurrentExceptionMsg()
|
||||
resp(Http500, makeJsonResp(Http500, "internal server error"), JSON)
|
||||
try: raise getCurrentException()
|
||||
except NotFoundException:
|
||||
response.buildJson(Http404, getCurrentExceptionMsg())
|
||||
except:
|
||||
response.build500Json(getCurrentException(),
|
||||
"unable to list versions for project " & @"projectName")
|
||||
return true
|
||||
|
||||
get "/project/@projectName/version/@version?":
|
||||
## Get a detailed project record including step definitions (ProjectConfig).
|
||||
@ -221,7 +245,7 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
|
||||
# Make sure we know about that project
|
||||
try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON)
|
||||
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
|
||||
except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
|
||||
|
||||
get "/project/@projectName/runs":
|
||||
## List all runs
|
||||
@ -229,7 +253,7 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
checkAuth(); if not authed: return true
|
||||
|
||||
try: resp($(%listRuns(cfg, @"projectName")), JSON)
|
||||
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
|
||||
except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
|
||||
|
||||
get "/project/@projectName/runs/active":
|
||||
## List all currently active runs
|
||||
@ -242,12 +266,13 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
.mapIt(cfg.getRun(@"projecName", $it.runId));
|
||||
resp($(%activeRuns), JSON)
|
||||
except:
|
||||
if getCurrentException() is KeyError:
|
||||
resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
|
||||
else:
|
||||
when not defined(release): debug getCurrentException().getStackTrace()
|
||||
error "problem loading active runs: " & getCurrentExceptionMsg()
|
||||
resp(Http500, makeJsonResp(Http500, "internal server error"), JSON)
|
||||
try: raise getCurrentException()
|
||||
except NotFoundException:
|
||||
response.buildJson(Http404, getCurrentExceptionMsg())
|
||||
except:
|
||||
response.build500Json(getCurrentException(),
|
||||
"problem loading active runs")
|
||||
return true
|
||||
|
||||
get "/project/@projectName/run/@runId":
|
||||
## Details for a specific run
|
||||
@ -256,13 +281,115 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
|
||||
# Make sure we know about that project
|
||||
try: discard cfg.getProject(@"projectName")
|
||||
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
|
||||
except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
|
||||
|
||||
if not existsRun(cfg, @"projectName", @"runId"):
|
||||
resp(Http404, makeJsonResp(Http404, "no such run for project"), JSON)
|
||||
response.buildJson(Http404, "no such run for project"); return true
|
||||
|
||||
try: resp($getRun(cfg, @"projectName", @"runId"), JSON)
|
||||
except: resp(Http500, makeJsonResp(Http500, getCurrentExceptionMsg()), JSON)
|
||||
except:
|
||||
response.build500Json(getCurrentException(),
|
||||
"unable to load run details for project " & @"projectName" &
|
||||
" run " & @"runId")
|
||||
return true
|
||||
|
||||
get "/project/@projectName/run/@runId/logs":
|
||||
## Get logs from a specific run
|
||||
|
||||
checkAuth(); if not authed: return true
|
||||
|
||||
try: discard cfg.getProject(@"projectName")
|
||||
except:
|
||||
response.buildJson(Http404, getCurrentExceptionMsg())
|
||||
return true
|
||||
|
||||
if not existsRun(cfg, @"projectName", @"runId"):
|
||||
response.buildJson(Http404, "no such run for project")
|
||||
return true
|
||||
|
||||
try: resp($getLogs(cfg, @"projectName", @"runId"))
|
||||
except:
|
||||
response.build500Json(getCurrentException(),
|
||||
"unable to load run logs for " & @"projectName" & " run " & @"runId")
|
||||
return true
|
||||
|
||||
get "/project/@projectName/step/@stepName/artifacts/@version":
|
||||
## Get the list of artifacts that were built for
|
||||
|
||||
checkAuth(); if not authed: return true
|
||||
|
||||
debug "Matched artifacts list request: " & $(%*{
|
||||
"project": @"projectName",
|
||||
"step": @"stepName",
|
||||
"version": @"version"
|
||||
})
|
||||
|
||||
try: resp($(%listArtifacts(cfg, @"projectName", @"stepName", @"version")), JSON)
|
||||
except:
|
||||
try: raise getCurrentException()
|
||||
except NotFoundException:
|
||||
response.buildJson(Http404, getCurrentExceptionMsg())
|
||||
except:
|
||||
response.build500Json(getCurrentException(), "unable to list artifacts for " &
|
||||
@"projectName" & ":" & @"stepName" & "@" & @"buildRef")
|
||||
return true
|
||||
|
||||
get "/project/@projectName/step/@stepName/artifact/@version/@artifactName":
|
||||
## Get a specific artifact that was built.
|
||||
|
||||
checkAuth(); if not authed: return true
|
||||
|
||||
var artifactPath: string
|
||||
try: artifactPath = getArtifactPath(cfg,
|
||||
@"projectName", @"stepName", @"version", @"artifactName")
|
||||
except:
|
||||
try: raise getCurrentException()
|
||||
except NotFoundException:
|
||||
response.buildJson(Http404, getCurrentExceptionMsg())
|
||||
except:
|
||||
response.build500Json(getCurrentException(), "unable to check artifact path for " &
|
||||
@"projectName" & ":" & @"stepName" & "@" & @"version")
|
||||
return true
|
||||
|
||||
debug "Preparing: " & artifactPath
|
||||
let fileSize = getFileSize(artifactPath)
|
||||
let mimetype = request.settings.mimes.getMimetype(artifactPath.splitFile.ext[1 .. ^1])
|
||||
if fileSize < 10_000_000: # 10 mb
|
||||
var file = readFile(artifactPath)
|
||||
|
||||
var hashed = getMD5(file)
|
||||
|
||||
# If the user has a cached version of this file and it matches our
|
||||
# version, let them use it
|
||||
if request.headers.hasKey("If-None-Match") and request.headers["If-None-Match"] == hashed:
|
||||
resp(Http304, [], "")
|
||||
else:
|
||||
resp(Http200, [
|
||||
("Content-Disposition", "; filename=\"" & @"artifactName" & "\""),
|
||||
("Content-Type", mimetype),
|
||||
("ETag", hashed )], file)
|
||||
else:
|
||||
let headers = {
|
||||
"Content-Disposition": "; filename=\"" & @"artifactName" & "\"",
|
||||
"Content-Type": mimetype,
|
||||
"Content-Length": $fileSize
|
||||
}.newStringTable
|
||||
await response.sendHeaders(Http200, headers)
|
||||
|
||||
var fileStream = newFutureStream[string]("sendStaticIfExists")
|
||||
var file = openAsync(artifactPath, fmRead)
|
||||
# Let `readToStream` write file data into fileStream in the
|
||||
# background.
|
||||
asyncCheck file.readToStream(fileStream)
|
||||
# The `writeFromStream` proc will complete once all the data in the
|
||||
# `bodyStream` has been written to the file.
|
||||
while true:
|
||||
let (hasValue, value) = await fileStream.read()
|
||||
if hasValue:
|
||||
await response.client.send(value)
|
||||
else:
|
||||
break
|
||||
file.close()
|
||||
|
||||
get "/project/@projectName/step/@stepName/status/@buildRef":
|
||||
## Get detailed information about the status of a step (assuming it has been built)
|
||||
@ -270,7 +397,19 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
checkAuth(); if not authed: return true
|
||||
|
||||
try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON)
|
||||
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
|
||||
except:
|
||||
try: raise getCurrentException()
|
||||
except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg())
|
||||
except:
|
||||
response.build500Json(getCurrentException(), "unable to load the build state for " &
|
||||
@"projectName" & ":" & @"stepName" & "@" & @"buildRef")
|
||||
return true
|
||||
|
||||
#get "/project/@projectName/step/@stepName/status/@buildRef.svg":
|
||||
## Get an image representing the status of a build
|
||||
|
||||
## TODO: how do we want to handle auth for this? Unlike
|
||||
#checkAuth(): if not authed: return true
|
||||
|
||||
post "/project/@projectName/step/@stepName/run/@buildRef?":
|
||||
# Kick off a run
|
||||
@ -296,26 +435,26 @@ proc start*(cfg: StrawBossConfig): void =
|
||||
id: runRequest.runId,
|
||||
request: runRequest,
|
||||
status: status), JSON)
|
||||
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON)
|
||||
|
||||
get "/service/debug/ping":
|
||||
if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON)
|
||||
else: resp($(%"pong"), JSON)
|
||||
except:
|
||||
try: raise getCurrentException()
|
||||
except NotFoundException:
|
||||
response.buildJson(Http404, getCurrentExceptionMsg())
|
||||
except: response.buildJson(Http400, getCurrentExceptionMsg())
|
||||
return true
|
||||
|
||||
post "/service/debug/stop":
|
||||
if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON)
|
||||
if not cfg.debug: response.buildJson(Http404); return
|
||||
else:
|
||||
let shutdownFut = sleepAsync(100)
|
||||
shutdownFut.callback = proc(): void = complete(stopFuture)
|
||||
resp($(%"shutting down"), JSON)
|
||||
|
||||
#[
|
||||
|
||||
get re".*":
|
||||
resp(Http404, makeJsonResp(Http404), JSON)
|
||||
response.buildJson(Http404); return true
|
||||
|
||||
post re".*":
|
||||
resp(Http404, makeJsonResp(Http404), JSON)
|
||||
]#
|
||||
response.buildJson(Http404); return true
|
||||
|
||||
proc performMaintenance(cfg: StrawBossConfig): void =
|
||||
# Prune workers
|
||||
|
||||
@ -41,7 +41,7 @@ suite "strawboss server":
|
||||
check fromJWT(cfg, tok) == session
|
||||
|
||||
test "ping":
|
||||
let resp = http.get(apiBase & "/service/debug/ping")
|
||||
let resp = http.get(apiBase & "/ping")
|
||||
check:
|
||||
resp.status.startsWith("200")
|
||||
resp.body == "\"pong\""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user