jdb/strawboss
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Artifact and log lookups. Bugfixes around failure scenarios.

Browse Source 
* Added endopint to return the logs from a run.
* Refactored the `pathVar & "/" & pathvar` pattern into `pathVar / pathVar`
  using the `ospaths` module. Cleaner code, more resistant to extra `/` bugs.
* Added endpoints and core methods to list artifacts for a build, as well as to
  retrieve specific artifacts.
* Fixed a problem with the `complete` status being overloaded. The problem was
  that in the case of a multi-step build all of the prerequisite steps will
  return a state of `complete` which will get recorded in the status file for
  the run. The run will continue, but anyone watching the run state file (via
  the API for example) had no definitive way to tell the difference between a
  sub-step completing and the requested (last) step completing. This was caught
  in the functional tests (race condition based on when it polls for the run
  status). The fix was to introduce a new build state: `stepComplete`. The
  inner `doRun` procedure uses this instead of `complete`. Now the only place
  that `complete` is set is at the end of the original call to `run`, right
  before the worker terminates. It checks the last result (from the originally
  requested step) and if this result is `stepComplete` it "finishes" the build
  by setting the state to `complete`. Because this is the only place where
  `complete` is set, an observer is now guaranteed not to see `complete` until
  all steps have run successfully.
* Fixed a long-standing bug with the request handling logic in error cases
  (like requested resources not being available). Issue has something to do
  with the way that `except` blocks become special when in an async context.
  The main jester routes block invokes the handlers in an async context. The
  effect is that one `except` block is fine, but adding more than one (to catch
  different exception types, for example) causes the return type of the route
  handler to change and not match what the outer block is expecting (a Future).
  The fix here is to wrap any exception discrimination within a single outer
  except block, re-raise the exception, and catch it inside this new
  synchronous context. Ex:

  ```nim
    try: someCall(mayFail)
    except:
      try: raise getCurrentException()
      except ExceptionType1:
        # do whatever...
      except ExceptionType2:
        # do whatever
      except:
        # general catch-all
    return true
  ```

  The return at the end is also part of the story. Jester's match handler
  allows a route to defer making a decision about whether it matches. If you
  return true from a route block Jester accepts the result as a matched route.
  If you return false, Jester discards the result and looks for another
  matching route. Normally this is taken care of by the `resp` templates
  provided by Jester, but invoking those templates within the except blocks
  also causes problems, so we manually setup the response and `return true` to
  tell Jester that, yes this route matched, use this response.
* Moved the `/service/debug/ping` endpoint back to `/ping` and removed the
  debug-only fence. I envision this as being useful as a simple healthcheck URL.
This commit is contained in:
Jonathan Bernard 2017-12-01 06:53:44 -06:00
parent 07037616ac
commit c6be698572
6 changed files with 309 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
TODO.md
View File

@ -1,8 +1,11 @@
TODO
* Orchestration of docker containers for running builds. * Orchestration of docker containers for running builds.
* GET /api/project/<project-name>/run/logs * Write API docs.
* Write a tool to convert JSON Schema into a human-readable format suitable for
documentation. Should use the description, title, and other fields from the NICE TO HAVE
JSON spec. Use this for writing the JSON schema docs instead of duplicating
the description of configuration files between JSON schema and the * Use/create some json-schema -> nim code generator to auto-generate json
documentation. In other words, use the schemas as the single source of truth handling code from schemas.
and generate everything else from that. * Use some json-schema -> docs generator to document the API.
* Support unique UUID prefixes in URLs.

4
src/main/nim/strawboss.nim
View File

@ -19,9 +19,9 @@ Usage:
strawboss run <requestFile> [options] strawboss run <requestFile> [options]
strawboss hashpwd <pwd> strawboss hashpwd <pwd>
strawboss api-key <username> strawboss api-key <username>
Options Options
-c --config-file <cfgFile> Use this config file instead of the default -c --config-file <cfgFile> Use this config file instead of the default
(strawboss.config.json). (strawboss.config.json).
""" """

13
src/main/nim/strawbosspkg/configuration.nim
View File

@ -10,7 +10,7 @@ const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz"
# #
type type
BuildState* {.pure.} = enum BuildState* {.pure.} = enum
queued, complete, failed, running, setup, rejected complete, failed, queued, rejected, running, setup, stepComplete
BuildStatus* = object BuildStatus* = object
runId*, details*: string runId*, details*: string
@ -41,6 +41,10 @@ type
request*: RunRequest request*: RunRequest
status*: BuildStatus status*: BuildStatus
RunLogs* = object
runId*: UUID
stdout*, stderr*: seq[string]
User* = object User* = object
name*: string name*: string
hashedPwd*: string hashedPwd*: string
@ -296,8 +300,15 @@ proc `%`*(run: Run): JsonNode =
"request": %run.request, "request": %run.request,
"status": %run.status } "status": %run.status }
proc `%`*(logs: RunLogs): JsonNode =
result = %* {
"runId": $logs.runId,
"stdout": %logs.stdout,
"stderr": %logs.stderr }
proc `$`*(s: BuildStatus): string = result = pretty(%s) proc `$`*(s: BuildStatus): string = result = pretty(%s)
proc `$`*(req: RunRequest): string = result = pretty(%req) proc `$`*(req: RunRequest): string = result = pretty(%req)
proc `$`*(pd: ProjectDef): string = result = pretty(%pd) proc `$`*(pd: ProjectDef): string = result = pretty(%pd)
proc `$`*(cfg: StrawBossConfig): string = result = pretty(%cfg) proc `$`*(cfg: StrawBossConfig): string = result = pretty(%cfg)
proc `$`*(run: Run): string = result = pretty(%run) proc `$`*(run: Run): string = result = pretty(%run)
proc `$`*(logs: RunLogs): string = result = pretty(%logs)

167
src/main/nim/strawbosspkg/core.nim
View File

@ -1,9 +1,9 @@
import cliutils, logging, json, os, osproc, sequtils, streams, import cliutils, logging, json, os, ospaths, osproc, sequtils, streams,
strtabs, strutils, tables, times, uuids strtabs, strutils, tables, times, uuids
import ./configuration import ./configuration
import nre except toSeq import nre except toSeq
from posix import link from posix import link, realpath
from algorithm import sorted from algorithm import sorted
type type
@ -27,7 +27,7 @@ type
projectName*: string projectName*: string
process*: Process process*: Process
NotFoundException = object of Exception NotFoundException* = object of Exception
proc newCopy(w: Workspace): Workspace = proc newCopy(w: Workspace): Workspace =
var newEnv: StringTableRef = newStringTable() var newEnv: StringTableRef = newStringTable()
@ -82,27 +82,27 @@ proc publishStatus(wksp: Workspace, state: BuildState, details: string): void =
runId: $wksp.runRequest.runId, state: state, details: details) runId: $wksp.runRequest.runId, state: state, details: details)
# Write to our run directory, and to our version status # Write to our run directory, and to our version status
writeFile(wksp.buildDataDir & "/runs/" & writeFile(wksp.buildDataDir / "runs" /
$wksp.runRequest.runId & ".status.json", $wksp.status) $wksp.runRequest.runId & ".status.json", $wksp.status)
# If we have our step we can save status to the step status # If we have our step we can save status to the step status
if not wksp.step.name.isNilOrEmpty(): if not wksp.step.name.isNilOrEmpty():
let stepStatusDir = wksp.buildDataDir & "/status/" & wksp.step.name let stepStatusDir = wksp.buildDataDir / "status" / wksp.step.name
if not existsDir(stepStatusDir): createDir(stepStatusDir) if not existsDir(stepStatusDir): createDir(stepStatusDir)
writeFile(stepStatusDir & "/" & wksp.version & ".json", $wksp.status) writeFile(stepStatusDir / wksp.version & ".json", $wksp.status)
# If we were asked to build a ref that is not the version directly (like # If we were asked to build a ref that is not the version directly (like
# "master" or something), then let's also save our status under that name. # "master" or something), then let's also save our status under that name.
# We're probably overwriting a prior status, but that's OK. # We're probably overwriting a prior status, but that's OK.
if wksp.runRequest.buildRef != wksp.version: if wksp.runRequest.buildRef != wksp.version:
writeFile(stepStatusDir & "/" & wksp.runRequest.buildRef & ".json", writeFile(stepStatusDir / wksp.runRequest.buildRef & ".json",
$wksp.status) $wksp.status)
wksp.outputHandler.sendStatusMsg(wksp.status) wksp.outputHandler.sendStatusMsg(wksp.status)
proc ensureProjectDirsExist(cfg: StrawBossConfig, p: ProjectDef): void = proc ensureProjectDirsExist(cfg: StrawBossConfig, p: ProjectDef): void =
for subdir in ["configurations", "runs", "status", "artifacts"]: for subdir in ["configurations", "runs", "status", "artifacts"]:
let fullPath = cfg.buildDataDir & "/" & p.name & "/" & subdir let fullPath = cfg.buildDataDir / p.name / subdir
if not existsDir(fullPath): if not existsDir(fullPath):
createDir(fullPath) createDir(fullPath)
@ -112,9 +112,9 @@ proc getProject*(cfg: StrawBossConfig, projectName: string): ProjectDef =
## Get a project definition by name from the service configuration ## Get a project definition by name from the service configuration
let candidates = cfg.projects.filterIt(it.name == projectName) let candidates = cfg.projects.filterIt(it.name == projectName)
if candidates.len == 0: if candidates.len == 0:
raise newException(KeyError, "no project named " & projectName) raise newException(NotFoundException, "no project named " & projectName)
elif candidates.len > 1: elif candidates.len > 1:
raise newException(KeyError, "multiple projects named " & projectName) raise newException(NotFoundException, "multiple projects named " & projectName)
else: result = candidates[0] else: result = candidates[0]
proc setProject*(cfg: var StrawBossConfig, projectName: string, newDef: ProjectDef): void = proc setProject*(cfg: var StrawBossConfig, projectName: string, newDef: ProjectDef): void =
@ -136,23 +136,67 @@ proc listVersions*(cfg: StrawBossConfig, projectName: string): seq[string] =
ensureProjectDirsExist(cfg, project) ensureProjectDirsExist(cfg, project)
let versionFiles = filesMatching( let versionFiles = filesMatching(
cfg.buildDataDir & "/" & project.name & "/configurations/*.json") cfg.buildDataDir / project.name / "configurations/*.json")
result = versionFiles.map(proc(s: string): string = result = versionFiles.map(proc(s: string): string =
let slashIdx = s.rfind('/') let slashIdx = s.rfind('/')
result = s[(slashIdx + 1)..^6]) result = s[(slashIdx + 1)..^6])
proc getBuildStatus*(cfg: StrawBossConfig,
projectName, stepName, buildRef: string): BuildStatus =
let project = cfg.getProject(projectName)
let statusFile = cfg.buildDataDir / project.name / "status" /
stepName / buildRef & ".json"
if not existsFile(statusFile):
raise newException(NotFoundException,
stepName & " has never been built for " & projectName & "@" & buildRef)
result = loadBuildStatus(statusFile)
proc listArtifacts*(cfg: StrawBossConfig,
projectName, stepName, version: string): seq[string] =
## List the artifacts that have been built for a step.
let project = cfg.getProject(projectName)
ensureProjectDirsExist(cfg, project)
let buildStatus = cfg.getBuildStatus(projectName, stepName, version)
if buildStatus.state != BuildState.complete:
raise newException(NotFoundException, "step " & stepName &
" has never been successfully built for " & projectName & "@" & version)
result = filesMatching(
cfg.buildDataDir / project.name / "artifacts" / stepName / version / "*")
.mapIt(it.extractFilename)
proc getArtifactPath*(cfg: StrawBossConfig,
projectName, stepName, version, artifactName: string): string =
let artifacts = cfg.listArtifacts(projectName, stepName, version)
if not artifacts.contains(artifactName):
raise newException(NotFoundException, "no artifact named " &
artifactName & " exists for step " & stepName & " in project " &
projectName & "@" & version)
result = cfg.buildDataDir / projectName / "artifacts" / stepName / version / artifactName
proc existsRun*(cfg: StrawBossConfig, projectName, runId: string): bool = proc existsRun*(cfg: StrawBossConfig, projectName, runId: string): bool =
existsFile(cfg.buildDataDir & "/" & projectName & "/runs/" & runId & ".request.json") existsFile(cfg.buildDataDir / projectName / "runs" / runId & ".request.json")
proc getRun*(cfg: StrawBossConfig, projectName, runId: string): Run = proc getRun*(cfg: StrawBossConfig, projectName, runId: string): Run =
let project = cfg.getProject(projectName) let project = cfg.getProject(projectName)
let runsPath = cfg.buildDataDir & "/" & project.name & "/runs" let runsPath = cfg.buildDataDir / project.name / "runs"
try: result = Run( try: result = Run(
id: parseUUID(runId), id: parseUUID(runId),
request: loadRunRequest(runsPath & "/" & runId & ".request.json"), request: loadRunRequest(runsPath / runId & ".request.json"),
status: loadBuildStatus(runsPath & "/" & runId & ".status.json")) status: loadBuildStatus(runsPath / runId & ".status.json"))
except: raiseEx "unable to load run information for id " & runId except: raiseEx "unable to load run information for id " & runId
proc listRuns*(cfg: StrawBossConfig, projectName: string): seq[Run] = proc listRuns*(cfg: StrawBossConfig, projectName: string): seq[Run] =
@ -160,29 +204,25 @@ proc listRuns*(cfg: StrawBossConfig, projectName: string): seq[Run] =
let project = cfg.getProject(projectName) let project = cfg.getProject(projectName)
ensureProjectDirsExist(cfg, project) ensureProjectDirsExist(cfg, project)
let runsPath = cfg.buildDataDir & "/" & project.name & "/runs" let runsPath = cfg.buildDataDir / project.name / "runs"
let reqPaths = filesMatching(runsPath & "/*.request.json") let reqPaths = filesMatching(runsPath / "*.request.json")
result = reqPaths.map(proc(reqPath: string): Run = result = reqPaths.map(proc(reqPath: string): Run =
let runId = reqPath[(runsPath.len + 1)..^14] let runId = reqPath[(runsPath.len + 1)..^14]
result = Run( result = Run(
id: parseUUID(runId), id: parseUUID(runId),
request: loadRunRequest(reqPath), request: loadRunRequest(reqPath),
status: loadBuildStatus(runsPath & "/" & runId & ".status.json"))) status: loadBuildStatus(runsPath / runId & ".status.json")))
proc getBuildStatus*(cfg: StrawBossConfig,
projectName, stepName, buildRef: string): BuildStatus =
proc getLogs*(cfg: StrawBossConfig, projectname, runId: string): RunLogs =
let project = cfg.getProject(projectName) let project = cfg.getProject(projectName)
let runsPath = cfg.buildDataDir / project.name / "runs"
let statusFile = cfg.buildDataDir & "/" & project.name & "/status/" & try: result = RunLogs(
stepName & "/" & buildRef & ".json" runId: parseUUID(runId),
stdout: toSeq(lines(runsPath / runId & ".stdout.log")),
if not existsFile(statusFile): stderr: toSeq(lines(runsPath / runId & ".stderr.log")))
raise newException(NotFoundException, except: raiseEx "unable to load logs for run " & runId
stepName & " has never been built for reference '" & buildRef)
result = loadBuildStatus(statusFile)
proc getProjectConfig*(cfg: StrawBossConfig, proc getProjectConfig*(cfg: StrawBossConfig,
projectName, version: string): ProjectConfig = projectName, version: string): ProjectConfig =
@ -196,7 +236,7 @@ proc getProjectConfig*(cfg: StrawBossConfig,
if version.isNilOrEmpty: if version.isNilOrEmpty:
let candidatePaths = filesMatching( let candidatePaths = filesMatching(
cfg.buildDataDir & "/" & project.name & "/configurations/*.json") cfg.buildDataDir / project.name / "configurations/*.json")
if candidatePaths.len == 0: if candidatePaths.len == 0:
raise newException(NotFoundException, raise newException(NotFoundException,
@ -212,8 +252,7 @@ proc getProjectConfig*(cfg: StrawBossConfig,
# If they did, let's try to load that # If they did, let's try to load that
else: else:
confFilePath = confFilePath =
cfg.buildDataDir & "/" & project.name & "/configurations/" & cfg.buildDataDir / project.name / "configurations" / version & ".json"
version & ".json"
if not existsFile(confFilePath): if not existsFile(confFilePath):
raise newException(NotFoundException, raise newException(NotFoundException,
@ -249,7 +288,7 @@ proc setupProject(wksp: Workspace) =
" for '" & wksp.projectDef.name & "'" " for '" & wksp.projectDef.name & "'"
# Find the strawboss project configuration # Find the strawboss project configuration
let projCfgFile = wksp.dir & "/" & wksp.projectDef.cfgFilePath let projCfgFile = wksp.dir / wksp.projectDef.cfgFilePath
wksp.sendMsg(lvlDebug, "Looking for project configuration at '" & projCfgFile & "'") wksp.sendMsg(lvlDebug, "Looking for project configuration at '" & projCfgFile & "'")
if not existsFile(projCfgFile): if not existsFile(projCfgFile):
raiseEx "Cannot find strawboss project configuration in the project " & raiseEx "Cannot find strawboss project configuration in the project " &
@ -283,21 +322,20 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
wksp.step = step wksp.step = step
let artifactsDir = wksp.buildDataDir & "/artifacts/" & let artifactsDir = wksp.buildDataDir / "artifacts" / step.name / wksp.version
step.name & "/" & wksp.version
if not existsDir(artifactsDir): createDir(artifactsDir) if not existsDir(artifactsDir): createDir(artifactsDir)
# Have we tried to build this before and are we caching the results? # Have we tried to build this before and are we caching the results?
let statusFilePath = wksp.buildDataDir & "/status/" & step.name & let statusFilePath = wksp.buildDataDir / "status" / step.name /
"/" & wksp.version & ".json" wksp.version & ".json"
if existsFile(statusFilePath) and not step.dontSkip: if existsFile(statusFilePath) and not step.dontSkip:
let prevStatus = loadBuildStatus(statusFilePath) let prevStatus = loadBuildStatus(statusFilePath)
# If we succeeded last time, no need to rebuild # If we succeeded last time, no need to rebuild
if prevStatus.state == BuildState.complete: if prevStatus.state == BuildState.complete:
wksp.publishStatus(BuildState.complete, wksp.publishStatus(BuildState.stepComplete,
"Skipping step '" & step.name & "' for version '" & wksp.version & "Skipping step '" & step.name & "' for version '" & wksp.version &
"': already completed.") "': already completed.")
return wksp.status return wksp.status
@ -328,7 +366,7 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
# Run that step (may get skipped) # Run that step (may get skipped)
let runStatus = doStep(core.newCopy(wksp), depStep) let runStatus = doStep(core.newCopy(wksp), depStep)
if not (runStatus.state == BuildState.complete): if not (runStatus.state == BuildState.stepComplete):
raiseEx "dependent step failed: " & depStep.name raiseEx "dependent step failed: " & depStep.name
wksp.sendMsg(lvlDebug, "dependent step '" & depStep.name & wksp.sendMsg(lvlDebug, "dependent step '" & depStep.name &
@ -336,8 +374,8 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
# Add the artifacts directory for the dependent step to our env so that # Add the artifacts directory for the dependent step to our env so that
# further steps can reference it via $<stepname>_DIR # further steps can reference it via $<stepname>_DIR
wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir & "/artifacts/" & wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir / "artifacts" /
dep & "/" & wksp.version dep / wksp.version
# Run the step command, piping in cmdInput # Run the step command, piping in cmdInput
let stepCmd = wksp.resolveEnvVars(step.stepCmd) let stepCmd = wksp.resolveEnvVars(step.stepCmd)
@ -345,7 +383,7 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
else: stepCmd else: stepCmd
wksp.sendMsg step.name & ": starting stepCmd: " & stepCmd wksp.sendMsg step.name & ": starting stepCmd: " & stepCmd
let cmdProc = startProcess(stepCmd, let cmdProc = startProcess(stepCmd,
wksp.dir & "/" & step.workingDir, [], wksp.env, {poUsePath, poEvalCommand}) wksp.dir / step.workingDir, [], wksp.env, {poUsePath, poEvalCommand})
let cmdInStream = inputStream(cmdProc) let cmdInStream = inputStream(cmdProc)
@ -353,7 +391,7 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
for line in step.cmdInput: cmdInStream.writeLine(wksp.resolveEnvVars(line)) for line in step.cmdInput: cmdInStream.writeLine(wksp.resolveEnvVars(line))
cmdInStream.flush() cmdInStream.flush()
cmdInStream.close() cmdInStream.close()
let cmdResult = waitFor(cmdProc, wksp.outputHandler, cmdName) let cmdResult = waitFor(cmdProc, wksp.outputHandler, cmdName)
if cmdResult != 0: if cmdResult != 0:
@ -367,16 +405,16 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus =
let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1] let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
try: try:
wksp.sendMsg "copy " & wksp.sendMsg "copy " &
wksp.dir & "/" & step.workingDir & "/" & artifactPath & " -> " & wksp.dir / step.workingDir / artifactPath & " -> " &
artifactsDir & "/" & artifactName artifactsDir / artifactName
copyFileWithPermissions(wksp.dir & "/" & step.workingDir & "/" & copyFileWithPermissions(wksp.dir / step.workingDir / artifactPath,
artifactPath, artifactsDir & "/" & artifactName) artifactsDir / artifactName)
except: except:
raiseEx "step " & step.name & " failed: unable to copy artifact " & raiseEx "step " & step.name & " failed: unable to copy artifact " &
artifactPath & ":\n" & getCurrentExceptionMsg() artifactPath & ":\n" & getCurrentExceptionMsg()
wksp.publishStatus(BuildState.complete, "") wksp.publishStatus(BuildState.stepComplete, "step " & step.name & " complete")
result = wksp.status result = wksp.status
proc run*(cfg: StrawBossConfig, req: RunRequest, proc run*(cfg: StrawBossConfig, req: RunRequest,
@ -401,8 +439,8 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
ensureProjectDirsExist(cfg, projectDef) ensureProjectDirsExist(cfg, projectDef)
# Update our run status # Update our run status
let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs" let runDir = cfg.buildDataDir / projectDef.name / "runs"
writeFile(runDir & "/" & $req.runId & ".status.json", $result) writeFile(runDir / $req.runId & ".status.json", $result)
# Read in the existing system environment # Read in the existing system environment
var env = loadEnv() var env = loadEnv()
@ -413,13 +451,13 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
if not existsDir(req.workspaceDir): createDir(req.workspaceDir) if not existsDir(req.workspaceDir): createDir(req.workspaceDir)
# Setup our STDOUT and STDERR files # Setup our STDOUT and STDERR files
let stdoutFile = open(runDir & "/" & $req.runId & ".stdout.log", fmWrite) let stdoutFile = open(runDir / $req.runId & ".stdout.log", fmWrite)
let stderrFile = open(runDir & "/" & $req.runId & ".stderr.log", fmWrite) let stderrFile = open(runDir / $req.runId & ".stderr.log", fmWrite)
let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile) let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile)
wksp = Workspace( wksp = Workspace(
buildDataDir: cfg.buildDataDir & "/" & projectDef.name, buildDataDir: cfg.buildDataDir / projectDef.name,
buildRef: buildRef:
if req.buildRef != nil and req.buildRef.len > 0: req.buildRef if req.buildRef != nil and req.buildRef.len > 0: req.buildRef
else: projectDef.defaultBranch, else: projectDef.defaultBranch,
@ -452,8 +490,8 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
# Update our cache of project configurations. # Update our cache of project configurations.
# TODO: what happens if this fails? # TODO: what happens if this fails?
copyFileWithPermissions( copyFileWithPermissions(
wksp.dir & "/" & wksp.projectDef.cfgFilePath, wksp.dir / wksp.projectDef.cfgFilePath,
wksp.buildDataDir & "/configurations/" & wksp.version & ".json") wksp.buildDataDir / "configurations" / wksp.version & ".json")
# Find the requested step # Find the requested step
if not wksp.project.steps.hasKey(req.stepName): if not wksp.project.steps.hasKey(req.stepName):
@ -462,7 +500,12 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
if req.forceRebuild: step.dontSkip = true if req.forceRebuild: step.dontSkip = true
result = doStep(wksp, step) var buildStatus = doStep(wksp, step)
if buildStatus.state == BuildState.stepComplete:
buildStatus.state = BuildState.complete
wksp.publishStatus(buildStatus.state, "all steps complete")
result = wksp.status
except: except:
when not defined(release): echo getCurrentException().getStackTrace() when not defined(release): echo getCurrentException().getStackTrace()
@ -477,19 +520,19 @@ proc run*(cfg: StrawBossConfig, req: RunRequest,
finally: finally:
if wksp != nil: if wksp != nil:
# Close open files # Close open files
for f in wksp.openedFiles: for f in wksp.openedFiles:
try: close(f) try: close(f)
except: discard "" except: discard ""
proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest): proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
tuple[status: BuildStatus, worker: Worker] = tuple[status: BuildStatus, worker: Worker] =
# Find the project definition (will throw appropriate exceptions) # Find the project definition (will throw appropriate exceptions)
let projectDef = cfg.getProject(req.projectName) let projectDef = cfg.getProject(req.projectName)
let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs" let runDir = cfg.buildDataDir / projectDef.name / "runs"
let reqFile = runDir & "/" & $req.runId & ".request.json" let reqFile = runDir / $req.runId & ".request.json"
let statusFile = runDir & "/" & $req.runId & ".status.json" let statusFile = runDir / $req.runId & ".status.json"
try: try:
# Make sure the build data directories for this project exist. # Make sure the build data directories for this project exist.
@ -517,8 +560,8 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
except: except:
let exMsg = "run request rejected: " & getCurrentExceptionMsg() let exMsg = "run request rejected: " & getCurrentExceptionMsg()
raiseEx exMsg
try: try:
writeFile(statusFile, writeFile(statusFile,
$(BuildStatus(runId: $req.runId, state: BuildState.rejected, details: exMsg))) $(BuildStatus(runId: $req.runId, state: BuildState.rejected, details: exMsg)))
except: discard "" except: discard ""
raiseEx exMsg

219
src/main/nim/strawbosspkg/server.nim
View File

@ -1,6 +1,11 @@
import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, md5,
os, osproc, sequtils, strutils, tempfile, times, unittest, uuids os, osproc, sequtils, strutils, tempfile, times, unittest, uuids
from mimetypes import getMimeType
from asyncfile import openAsync, readToStream, close
from asyncnet import send
from re import re, find
import ./configuration, ./core import ./configuration, ./core
type type
@ -17,13 +22,22 @@ proc newSession*(user: UserRef): Session =
issuedAt: getTime(), issuedAt: getTime(),
expires: daysForward(7).toTime()) expires: daysForward(7).toTime())
proc makeJsonResp(status: HttpCode, details: string = ""): string = proc buildJson(resp: Response, code: HttpCode, details: string = ""): void =
result = $(%* { resp.data[0] = CallbackAction.TCActionSend
"statusCode": status.int, resp.data[1] = code
"status": $status, resp.data[2]["Content-Type"] = JSON
resp.data[3] = $(%* {
"statusCode": code.int,
"status": $code,
"details": details "details": details
}) })
# Work-around for weirdness trying to use resp(Http500... in exception blocks
proc build500Json(resp: Response, ex: ref Exception, msg: string): void =
when not defined(release): debug ex.getStackTrace()
error msg & ":\n" & ex.msg
resp.buildJson(Http500)
proc toJWT*(cfg: StrawBossConfig, session: Session): string = proc toJWT*(cfg: StrawBossConfig, session: Session): string =
## Make a JST token for this session. ## Make a JST token for this session.
var jwt = JWT( var jwt = JWT(
@ -46,7 +60,6 @@ proc fromJWT*(cfg: StrawBossConfig, strTok: string): Session =
# Find the user record (if authenticated) # Find the user record (if authenticated)
let username = jwt.claims["sub"].node.str let username = jwt.claims["sub"].node.str
let users = cfg.users.filterIt(it.name == username) let users = cfg.users.filterIt(it.name == username)
debug "username: " & username & "\n\tusers: " & $users.mapIt(it.name) & "\n\tall users: " & cfg.users.mapIt(it.name)
if users.len != 1: raiseEx "Could not find session user." if users.len != 1: raiseEx "Could not find session user."
result = Session( result = Session(
@ -131,7 +144,8 @@ template checkAuth() =
except: except:
debug "Auth failed: " & getCurrentExceptionMsg() debug "Auth failed: " & getCurrentExceptionMsg()
response.data[2]["WWW-Authenticate"] = "Bearer" response.data[2]["WWW-Authenticate"] = "Bearer"
resp(Http401, makeJsonResp(Http401), JSON) response.buildJson(Http401)
return
proc start*(cfg: StrawBossConfig): void = proc start*(cfg: StrawBossConfig): void =
@ -144,18 +158,21 @@ proc start*(cfg: StrawBossConfig): void =
routes: routes:
get "/ping":
resp($(%"pong"), JSON)
post "/auth-token": post "/auth-token":
var uname, pwd: string var uname, pwd: string
try: try:
let jsonBody = parseJson(request.body) let jsonBody = parseJson(request.body)
uname = jsonBody["username"].getStr uname = jsonBody["username"].getStr
pwd = jsonBody["password"].getStr pwd = jsonBody["password"].getStr
except: resp(Http400, makeJsonResp(Http400), JSON) except: response.buildJson(Http400); return
try: try:
let authToken = makeAuthToken(cfg, uname, pwd) let authToken = makeAuthToken(cfg, uname, pwd)
resp($(%authToken), JSON) resp($(%authToken), JSON)
except: resp(Http401, makeJsonResp(Http401, getCurrentExceptionMsg()), JSON) except: response.buildJson(Http401, getCurrentExceptionMsg()); return
get "/verify-auth": get "/verify-auth":
checkAuth(); if not authed: return true checkAuth(); if not authed: return true
@ -175,7 +192,7 @@ proc start*(cfg: StrawBossConfig): void =
checkAuth(); if not authed: return true checkAuth(); if not authed: return true
# TODO # TODO
resp(Http501, makeJsonResp(Http501), JSON) response.buildJson(Http501); return
get "/project/@projectName": get "/project/@projectName":
## Return a project's configuration, as well as it's versions. ## Return a project's configuration, as well as it's versions.
@ -185,7 +202,14 @@ proc start*(cfg: StrawBossConfig): void =
# Make sure we know about that project # Make sure we know about that project
var projDef: ProjectDef var projDef: ProjectDef
try: projDef = cfg.getProject(@"projectName") try: projDef = cfg.getProject(@"projectName")
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) except:
try: raise getCurrentException()
except NotFoundException:
response.buildJson(Http404, getCurrentExceptionMsg())
except:
response.build500Json(getCurrentException(),
"unable to load project definition for project " & @"projectName")
return true
var projConf: ProjectConfig var projConf: ProjectConfig
try: projConf = getProjectConfig(cfg, @"projectName", "") try: projConf = getProjectConfig(cfg, @"projectName", "")
@ -206,13 +230,13 @@ proc start*(cfg: StrawBossConfig): void =
try: resp($(%listVersions(cfg, @"projectName")), JSON) try: resp($(%listVersions(cfg, @"projectName")), JSON)
except: except:
if getCurrentException() is KeyError: try: raise getCurrentException()
resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) except NotFoundException:
else: response.buildJson(Http404, getCurrentExceptionMsg())
when not defined(release): debug getCurrentException().getStackTrace() except:
error "unable to list versions for project " & @"projectName" & response.build500Json(getCurrentException(),
":\n" & getCurrentExceptionMsg() "unable to list versions for project " & @"projectName")
resp(Http500, makeJsonResp(Http500, "internal server error"), JSON) return true
get "/project/@projectName/version/@version?": get "/project/@projectName/version/@version?":
## Get a detailed project record including step definitions (ProjectConfig). ## Get a detailed project record including step definitions (ProjectConfig).
@ -221,7 +245,7 @@ proc start*(cfg: StrawBossConfig): void =
# Make sure we know about that project # Make sure we know about that project
try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON) try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON)
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
get "/project/@projectName/runs": get "/project/@projectName/runs":
## List all runs ## List all runs
@ -229,7 +253,7 @@ proc start*(cfg: StrawBossConfig): void =
checkAuth(); if not authed: return true checkAuth(); if not authed: return true
try: resp($(%listRuns(cfg, @"projectName")), JSON) try: resp($(%listRuns(cfg, @"projectName")), JSON)
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
get "/project/@projectName/runs/active": get "/project/@projectName/runs/active":
## List all currently active runs ## List all currently active runs
@ -242,12 +266,13 @@ proc start*(cfg: StrawBossConfig): void =
.mapIt(cfg.getRun(@"projecName", $it.runId)); .mapIt(cfg.getRun(@"projecName", $it.runId));
resp($(%activeRuns), JSON) resp($(%activeRuns), JSON)
except: except:
if getCurrentException() is KeyError: try: raise getCurrentException()
resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) except NotFoundException:
else: response.buildJson(Http404, getCurrentExceptionMsg())
when not defined(release): debug getCurrentException().getStackTrace() except:
error "problem loading active runs: " & getCurrentExceptionMsg() response.build500Json(getCurrentException(),
resp(Http500, makeJsonResp(Http500, "internal server error"), JSON) "problem loading active runs")
return true
get "/project/@projectName/run/@runId": get "/project/@projectName/run/@runId":
## Details for a specific run ## Details for a specific run
@ -256,13 +281,115 @@ proc start*(cfg: StrawBossConfig): void =
# Make sure we know about that project # Make sure we know about that project
try: discard cfg.getProject(@"projectName") try: discard cfg.getProject(@"projectName")
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) except: response.buildJson(Http404, getCurrentExceptionMsg()); return true
if not existsRun(cfg, @"projectName", @"runId"): if not existsRun(cfg, @"projectName", @"runId"):
resp(Http404, makeJsonResp(Http404, "no such run for project"), JSON) response.buildJson(Http404, "no such run for project"); return true
try: resp($getRun(cfg, @"projectName", @"runId"), JSON) try: resp($getRun(cfg, @"projectName", @"runId"), JSON)
except: resp(Http500, makeJsonResp(Http500, getCurrentExceptionMsg()), JSON) except:
response.build500Json(getCurrentException(),
"unable to load run details for project " & @"projectName" &
" run " & @"runId")
return true
get "/project/@projectName/run/@runId/logs":
## Get logs from a specific run
checkAuth(); if not authed: return true
try: discard cfg.getProject(@"projectName")
except:
response.buildJson(Http404, getCurrentExceptionMsg())
return true
if not existsRun(cfg, @"projectName", @"runId"):
response.buildJson(Http404, "no such run for project")
return true
try: resp($getLogs(cfg, @"projectName", @"runId"))
except:
response.build500Json(getCurrentException(),
"unable to load run logs for " & @"projectName" & " run " & @"runId")
return true
get "/project/@projectName/step/@stepName/artifacts/@version":
## Get the list of artifacts that were built for
checkAuth(); if not authed: return true
debug "Matched artifacts list request: " & $(%*{
"project": @"projectName",
"step": @"stepName",
"version": @"version"
})
try: resp($(%listArtifacts(cfg, @"projectName", @"stepName", @"version")), JSON)
except:
try: raise getCurrentException()
except NotFoundException:
response.buildJson(Http404, getCurrentExceptionMsg())
except:
response.build500Json(getCurrentException(), "unable to list artifacts for " &
@"projectName" & ":" & @"stepName" & "@" & @"buildRef")
return true
get "/project/@projectName/step/@stepName/artifact/@version/@artifactName":
## Get a specific artifact that was built.
checkAuth(); if not authed: return true
var artifactPath: string
try: artifactPath = getArtifactPath(cfg,
@"projectName", @"stepName", @"version", @"artifactName")
except:
try: raise getCurrentException()
except NotFoundException:
response.buildJson(Http404, getCurrentExceptionMsg())
except:
response.build500Json(getCurrentException(), "unable to check artifact path for " &
@"projectName" & ":" & @"stepName" & "@" & @"version")
return true
debug "Preparing: " & artifactPath
let fileSize = getFileSize(artifactPath)
let mimetype = request.settings.mimes.getMimetype(artifactPath.splitFile.ext[1 .. ^1])
if fileSize < 10_000_000: # 10 mb
var file = readFile(artifactPath)
var hashed = getMD5(file)
# If the user has a cached version of this file and it matches our
# version, let them use it
if request.headers.hasKey("If-None-Match") and request.headers["If-None-Match"] == hashed:
resp(Http304, [], "")
else:
resp(Http200, [
("Content-Disposition", "; filename=\"" & @"artifactName" & "\""),
("Content-Type", mimetype),
("ETag", hashed )], file)
else:
let headers = {
"Content-Disposition": "; filename=\"" & @"artifactName" & "\"",
"Content-Type": mimetype,
"Content-Length": $fileSize
}.newStringTable
await response.sendHeaders(Http200, headers)
var fileStream = newFutureStream[string]("sendStaticIfExists")
var file = openAsync(artifactPath, fmRead)
# Let `readToStream` write file data into fileStream in the
# background.
asyncCheck file.readToStream(fileStream)
# The `writeFromStream` proc will complete once all the data in the
# `bodyStream` has been written to the file.
while true:
let (hasValue, value) = await fileStream.read()
if hasValue:
await response.client.send(value)
else:
break
file.close()
get "/project/@projectName/step/@stepName/status/@buildRef": get "/project/@projectName/step/@stepName/status/@buildRef":
## Get detailed information about the status of a step (assuming it has been built) ## Get detailed information about the status of a step (assuming it has been built)
@ -270,7 +397,19 @@ proc start*(cfg: StrawBossConfig): void =
checkAuth(); if not authed: return true checkAuth(); if not authed: return true
try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON) try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON)
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) except:
try: raise getCurrentException()
except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg())
except:
response.build500Json(getCurrentException(), "unable to load the build state for " &
@"projectName" & ":" & @"stepName" & "@" & @"buildRef")
return true
#get "/project/@projectName/step/@stepName/status/@buildRef.svg":
## Get an image representing the status of a build
## TODO: how do we want to handle auth for this? Unlike
#checkAuth(): if not authed: return true
post "/project/@projectName/step/@stepName/run/@buildRef?": post "/project/@projectName/step/@stepName/run/@buildRef?":
# Kick off a run # Kick off a run
@ -296,26 +435,26 @@ proc start*(cfg: StrawBossConfig): void =
id: runRequest.runId, id: runRequest.runId,
request: runRequest, request: runRequest,
status: status), JSON) status: status), JSON)
except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) except:
try: raise getCurrentException()
get "/service/debug/ping": except NotFoundException:
if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON) response.buildJson(Http404, getCurrentExceptionMsg())
else: resp($(%"pong"), JSON) except: response.buildJson(Http400, getCurrentExceptionMsg())
return true
post "/service/debug/stop": post "/service/debug/stop":
if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON) if not cfg.debug: response.buildJson(Http404); return
else: else:
let shutdownFut = sleepAsync(100) let shutdownFut = sleepAsync(100)
shutdownFut.callback = proc(): void = complete(stopFuture) shutdownFut.callback = proc(): void = complete(stopFuture)
resp($(%"shutting down"), JSON) resp($(%"shutting down"), JSON)
#[
get re".*": get re".*":
resp(Http404, makeJsonResp(Http404), JSON) response.buildJson(Http404); return true
post re".*": post re".*":
resp(Http404, makeJsonResp(Http404), JSON) response.buildJson(Http404); return true
]#
proc performMaintenance(cfg: StrawBossConfig): void = proc performMaintenance(cfg: StrawBossConfig): void =
# Prune workers # Prune workers

2
src/test/nim/unit/tserver.nim
View File

@ -41,7 +41,7 @@ suite "strawboss server":
check fromJWT(cfg, tok) == session check fromJWT(cfg, tok) == session
test "ping": test "ping":
let resp = http.get(apiBase & "/service/debug/ping") let resp = http.get(apiBase & "/ping")
check: check:
resp.status.startsWith("200") resp.status.startsWith("200")
resp.body == "\"pong\"" resp.body == "\"pong\""