From c6be6985727a336f6ba9d3fa13849874c75a380c Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Fri, 1 Dec 2017 06:53:44 -0600 Subject: [PATCH] Artifact and log lookups. Bugfixes around failure scenarios. * Added endopint to return the logs from a run. * Refactored the `pathVar & "/" & pathvar` pattern into `pathVar / pathVar` using the `ospaths` module. Cleaner code, more resistant to extra `/` bugs. * Added endpoints and core methods to list artifacts for a build, as well as to retrieve specific artifacts. * Fixed a problem with the `complete` status being overloaded. The problem was that in the case of a multi-step build all of the prerequisite steps will return a state of `complete` which will get recorded in the status file for the run. The run will continue, but anyone watching the run state file (via the API for example) had no definitive way to tell the difference between a sub-step completing and the requested (last) step completing. This was caught in the functional tests (race condition based on when it polls for the run status). The fix was to introduce a new build state: `stepComplete`. The inner `doRun` procedure uses this instead of `complete`. Now the only place that `complete` is set is at the end of the original call to `run`, right before the worker terminates. It checks the last result (from the originally requested step) and if this result is `stepComplete` it "finishes" the build by setting the state to `complete`. Because this is the only place where `complete` is set, an observer is now guaranteed not to see `complete` until all steps have run successfully. * Fixed a long-standing bug with the request handling logic in error cases (like requested resources not being available). Issue has something to do with the way that `except` blocks become special when in an async context. The main jester routes block invokes the handlers in an async context. The effect is that one `except` block is fine, but adding more than one (to catch different exception types, for example) causes the return type of the route handler to change and not match what the outer block is expecting (a Future). The fix here is to wrap any exception discrimination within a single outer except block, re-raise the exception, and catch it inside this new synchronous context. Ex: ```nim try: someCall(mayFail) except: try: raise getCurrentException() except ExceptionType1: # do whatever... except ExceptionType2: # do whatever except: # general catch-all return true ``` The return at the end is also part of the story. Jester's match handler allows a route to defer making a decision about whether it matches. If you return true from a route block Jester accepts the result as a matched route. If you return false, Jester discards the result and looks for another matching route. Normally this is taken care of by the `resp` templates provided by Jester, but invoking those templates within the except blocks also causes problems, so we manually setup the response and `return true` to tell Jester that, yes this route matched, use this response. * Moved the `/service/debug/ping` endpoint back to `/ping` and removed the debug-only fence. I envision this as being useful as a simple healthcheck URL. --- TODO.md | 17 +- src/main/nim/strawboss.nim | 4 +- src/main/nim/strawbosspkg/configuration.nim | 13 +- src/main/nim/strawbosspkg/core.nim | 167 +++++++++------ src/main/nim/strawbosspkg/server.nim | 219 ++++++++++++++++---- src/test/nim/unit/tserver.nim | 2 +- 6 files changed, 309 insertions(+), 113 deletions(-) diff --git a/TODO.md b/TODO.md index 6f537c6..1804fc1 100644 --- a/TODO.md +++ b/TODO.md @@ -1,8 +1,11 @@ +TODO + * Orchestration of docker containers for running builds. -* GET /api/project//run/logs -* Write a tool to convert JSON Schema into a human-readable format suitable for - documentation. Should use the description, title, and other fields from the - JSON spec. Use this for writing the JSON schema docs instead of duplicating - the description of configuration files between JSON schema and the - documentation. In other words, use the schemas as the single source of truth - and generate everything else from that. +* Write API docs. + +NICE TO HAVE + +* Use/create some json-schema -> nim code generator to auto-generate json + handling code from schemas. +* Use some json-schema -> docs generator to document the API. +* Support unique UUID prefixes in URLs. diff --git a/src/main/nim/strawboss.nim b/src/main/nim/strawboss.nim index f66b3d8..fc81860 100644 --- a/src/main/nim/strawboss.nim +++ b/src/main/nim/strawboss.nim @@ -19,9 +19,9 @@ Usage: strawboss run [options] strawboss hashpwd strawboss api-key - + Options - + -c --config-file Use this config file instead of the default (strawboss.config.json). """ diff --git a/src/main/nim/strawbosspkg/configuration.nim b/src/main/nim/strawbosspkg/configuration.nim index d949cca..1ab100d 100644 --- a/src/main/nim/strawbosspkg/configuration.nim +++ b/src/main/nim/strawbosspkg/configuration.nim @@ -10,7 +10,7 @@ const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:sszzz" # type BuildState* {.pure.} = enum - queued, complete, failed, running, setup, rejected + complete, failed, queued, rejected, running, setup, stepComplete BuildStatus* = object runId*, details*: string @@ -41,6 +41,10 @@ type request*: RunRequest status*: BuildStatus + RunLogs* = object + runId*: UUID + stdout*, stderr*: seq[string] + User* = object name*: string hashedPwd*: string @@ -296,8 +300,15 @@ proc `%`*(run: Run): JsonNode = "request": %run.request, "status": %run.status } +proc `%`*(logs: RunLogs): JsonNode = + result = %* { + "runId": $logs.runId, + "stdout": %logs.stdout, + "stderr": %logs.stderr } + proc `$`*(s: BuildStatus): string = result = pretty(%s) proc `$`*(req: RunRequest): string = result = pretty(%req) proc `$`*(pd: ProjectDef): string = result = pretty(%pd) proc `$`*(cfg: StrawBossConfig): string = result = pretty(%cfg) proc `$`*(run: Run): string = result = pretty(%run) +proc `$`*(logs: RunLogs): string = result = pretty(%logs) diff --git a/src/main/nim/strawbosspkg/core.nim b/src/main/nim/strawbosspkg/core.nim index e99d228..cd35e6a 100644 --- a/src/main/nim/strawbosspkg/core.nim +++ b/src/main/nim/strawbosspkg/core.nim @@ -1,9 +1,9 @@ -import cliutils, logging, json, os, osproc, sequtils, streams, +import cliutils, logging, json, os, ospaths, osproc, sequtils, streams, strtabs, strutils, tables, times, uuids import ./configuration import nre except toSeq -from posix import link +from posix import link, realpath from algorithm import sorted type @@ -27,7 +27,7 @@ type projectName*: string process*: Process - NotFoundException = object of Exception + NotFoundException* = object of Exception proc newCopy(w: Workspace): Workspace = var newEnv: StringTableRef = newStringTable() @@ -82,27 +82,27 @@ proc publishStatus(wksp: Workspace, state: BuildState, details: string): void = runId: $wksp.runRequest.runId, state: state, details: details) # Write to our run directory, and to our version status - writeFile(wksp.buildDataDir & "/runs/" & + writeFile(wksp.buildDataDir / "runs" / $wksp.runRequest.runId & ".status.json", $wksp.status) # If we have our step we can save status to the step status if not wksp.step.name.isNilOrEmpty(): - let stepStatusDir = wksp.buildDataDir & "/status/" & wksp.step.name + let stepStatusDir = wksp.buildDataDir / "status" / wksp.step.name if not existsDir(stepStatusDir): createDir(stepStatusDir) - writeFile(stepStatusDir & "/" & wksp.version & ".json", $wksp.status) + writeFile(stepStatusDir / wksp.version & ".json", $wksp.status) # If we were asked to build a ref that is not the version directly (like # "master" or something), then let's also save our status under that name. # We're probably overwriting a prior status, but that's OK. if wksp.runRequest.buildRef != wksp.version: - writeFile(stepStatusDir & "/" & wksp.runRequest.buildRef & ".json", + writeFile(stepStatusDir / wksp.runRequest.buildRef & ".json", $wksp.status) wksp.outputHandler.sendStatusMsg(wksp.status) proc ensureProjectDirsExist(cfg: StrawBossConfig, p: ProjectDef): void = for subdir in ["configurations", "runs", "status", "artifacts"]: - let fullPath = cfg.buildDataDir & "/" & p.name & "/" & subdir + let fullPath = cfg.buildDataDir / p.name / subdir if not existsDir(fullPath): createDir(fullPath) @@ -112,9 +112,9 @@ proc getProject*(cfg: StrawBossConfig, projectName: string): ProjectDef = ## Get a project definition by name from the service configuration let candidates = cfg.projects.filterIt(it.name == projectName) if candidates.len == 0: - raise newException(KeyError, "no project named " & projectName) + raise newException(NotFoundException, "no project named " & projectName) elif candidates.len > 1: - raise newException(KeyError, "multiple projects named " & projectName) + raise newException(NotFoundException, "multiple projects named " & projectName) else: result = candidates[0] proc setProject*(cfg: var StrawBossConfig, projectName: string, newDef: ProjectDef): void = @@ -136,23 +136,67 @@ proc listVersions*(cfg: StrawBossConfig, projectName: string): seq[string] = ensureProjectDirsExist(cfg, project) let versionFiles = filesMatching( - cfg.buildDataDir & "/" & project.name & "/configurations/*.json") + cfg.buildDataDir / project.name / "configurations/*.json") result = versionFiles.map(proc(s: string): string = let slashIdx = s.rfind('/') result = s[(slashIdx + 1)..^6]) +proc getBuildStatus*(cfg: StrawBossConfig, + projectName, stepName, buildRef: string): BuildStatus = + + let project = cfg.getProject(projectName) + + let statusFile = cfg.buildDataDir / project.name / "status" / + stepName / buildRef & ".json" + + if not existsFile(statusFile): + raise newException(NotFoundException, + stepName & " has never been built for " & projectName & "@" & buildRef) + + result = loadBuildStatus(statusFile) + + +proc listArtifacts*(cfg: StrawBossConfig, + projectName, stepName, version: string): seq[string] = + ## List the artifacts that have been built for a step. + + let project = cfg.getProject(projectName) + + ensureProjectDirsExist(cfg, project) + + let buildStatus = cfg.getBuildStatus(projectName, stepName, version) + + if buildStatus.state != BuildState.complete: + raise newException(NotFoundException, "step " & stepName & + " has never been successfully built for " & projectName & "@" & version) + + result = filesMatching( + cfg.buildDataDir / project.name / "artifacts" / stepName / version / "*") + .mapIt(it.extractFilename) + +proc getArtifactPath*(cfg: StrawBossConfig, + projectName, stepName, version, artifactName: string): string = + + let artifacts = cfg.listArtifacts(projectName, stepName, version) + if not artifacts.contains(artifactName): + raise newException(NotFoundException, "no artifact named " & + artifactName & " exists for step " & stepName & " in project " & + projectName & "@" & version) + + result = cfg.buildDataDir / projectName / "artifacts" / stepName / version / artifactName + proc existsRun*(cfg: StrawBossConfig, projectName, runId: string): bool = - existsFile(cfg.buildDataDir & "/" & projectName & "/runs/" & runId & ".request.json") + existsFile(cfg.buildDataDir / projectName / "runs" / runId & ".request.json") proc getRun*(cfg: StrawBossConfig, projectName, runId: string): Run = let project = cfg.getProject(projectName) - let runsPath = cfg.buildDataDir & "/" & project.name & "/runs" + let runsPath = cfg.buildDataDir / project.name / "runs" try: result = Run( id: parseUUID(runId), - request: loadRunRequest(runsPath & "/" & runId & ".request.json"), - status: loadBuildStatus(runsPath & "/" & runId & ".status.json")) + request: loadRunRequest(runsPath / runId & ".request.json"), + status: loadBuildStatus(runsPath / runId & ".status.json")) except: raiseEx "unable to load run information for id " & runId proc listRuns*(cfg: StrawBossConfig, projectName: string): seq[Run] = @@ -160,29 +204,25 @@ proc listRuns*(cfg: StrawBossConfig, projectName: string): seq[Run] = let project = cfg.getProject(projectName) ensureProjectDirsExist(cfg, project) - let runsPath = cfg.buildDataDir & "/" & project.name & "/runs" - let reqPaths = filesMatching(runsPath & "/*.request.json") + let runsPath = cfg.buildDataDir / project.name / "runs" + let reqPaths = filesMatching(runsPath / "*.request.json") result = reqPaths.map(proc(reqPath: string): Run = let runId = reqPath[(runsPath.len + 1)..^14] result = Run( id: parseUUID(runId), request: loadRunRequest(reqPath), - status: loadBuildStatus(runsPath & "/" & runId & ".status.json"))) - -proc getBuildStatus*(cfg: StrawBossConfig, - projectName, stepName, buildRef: string): BuildStatus = + status: loadBuildStatus(runsPath / runId & ".status.json"))) +proc getLogs*(cfg: StrawBossConfig, projectname, runId: string): RunLogs = let project = cfg.getProject(projectName) + let runsPath = cfg.buildDataDir / project.name / "runs" - let statusFile = cfg.buildDataDir & "/" & project.name & "/status/" & - stepName & "/" & buildRef & ".json" - - if not existsFile(statusFile): - raise newException(NotFoundException, - stepName & " has never been built for reference '" & buildRef) - - result = loadBuildStatus(statusFile) + try: result = RunLogs( + runId: parseUUID(runId), + stdout: toSeq(lines(runsPath / runId & ".stdout.log")), + stderr: toSeq(lines(runsPath / runId & ".stderr.log"))) + except: raiseEx "unable to load logs for run " & runId proc getProjectConfig*(cfg: StrawBossConfig, projectName, version: string): ProjectConfig = @@ -196,7 +236,7 @@ proc getProjectConfig*(cfg: StrawBossConfig, if version.isNilOrEmpty: let candidatePaths = filesMatching( - cfg.buildDataDir & "/" & project.name & "/configurations/*.json") + cfg.buildDataDir / project.name / "configurations/*.json") if candidatePaths.len == 0: raise newException(NotFoundException, @@ -212,8 +252,7 @@ proc getProjectConfig*(cfg: StrawBossConfig, # If they did, let's try to load that else: confFilePath = - cfg.buildDataDir & "/" & project.name & "/configurations/" & - version & ".json" + cfg.buildDataDir / project.name / "configurations" / version & ".json" if not existsFile(confFilePath): raise newException(NotFoundException, @@ -249,7 +288,7 @@ proc setupProject(wksp: Workspace) = " for '" & wksp.projectDef.name & "'" # Find the strawboss project configuration - let projCfgFile = wksp.dir & "/" & wksp.projectDef.cfgFilePath + let projCfgFile = wksp.dir / wksp.projectDef.cfgFilePath wksp.sendMsg(lvlDebug, "Looking for project configuration at '" & projCfgFile & "'") if not existsFile(projCfgFile): raiseEx "Cannot find strawboss project configuration in the project " & @@ -283,21 +322,20 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus = wksp.step = step - let artifactsDir = wksp.buildDataDir & "/artifacts/" & - step.name & "/" & wksp.version + let artifactsDir = wksp.buildDataDir / "artifacts" / step.name / wksp.version if not existsDir(artifactsDir): createDir(artifactsDir) # Have we tried to build this before and are we caching the results? - let statusFilePath = wksp.buildDataDir & "/status/" & step.name & - "/" & wksp.version & ".json" + let statusFilePath = wksp.buildDataDir / "status" / step.name / + wksp.version & ".json" if existsFile(statusFilePath) and not step.dontSkip: let prevStatus = loadBuildStatus(statusFilePath) # If we succeeded last time, no need to rebuild if prevStatus.state == BuildState.complete: - wksp.publishStatus(BuildState.complete, + wksp.publishStatus(BuildState.stepComplete, "Skipping step '" & step.name & "' for version '" & wksp.version & "': already completed.") return wksp.status @@ -328,7 +366,7 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus = # Run that step (may get skipped) let runStatus = doStep(core.newCopy(wksp), depStep) - if not (runStatus.state == BuildState.complete): + if not (runStatus.state == BuildState.stepComplete): raiseEx "dependent step failed: " & depStep.name wksp.sendMsg(lvlDebug, "dependent step '" & depStep.name & @@ -336,8 +374,8 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus = # Add the artifacts directory for the dependent step to our env so that # further steps can reference it via $_DIR - wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir & "/artifacts/" & - dep & "/" & wksp.version + wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir / "artifacts" / + dep / wksp.version # Run the step command, piping in cmdInput let stepCmd = wksp.resolveEnvVars(step.stepCmd) @@ -345,7 +383,7 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus = else: stepCmd wksp.sendMsg step.name & ": starting stepCmd: " & stepCmd let cmdProc = startProcess(stepCmd, - wksp.dir & "/" & step.workingDir, [], wksp.env, {poUsePath, poEvalCommand}) + wksp.dir / step.workingDir, [], wksp.env, {poUsePath, poEvalCommand}) let cmdInStream = inputStream(cmdProc) @@ -353,7 +391,7 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus = for line in step.cmdInput: cmdInStream.writeLine(wksp.resolveEnvVars(line)) cmdInStream.flush() cmdInStream.close() - + let cmdResult = waitFor(cmdProc, wksp.outputHandler, cmdName) if cmdResult != 0: @@ -367,16 +405,16 @@ proc doStep*(wksp: Workspace, step: Step): BuildStatus = let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1] try: wksp.sendMsg "copy " & - wksp.dir & "/" & step.workingDir & "/" & artifactPath & " -> " & - artifactsDir & "/" & artifactName + wksp.dir / step.workingDir / artifactPath & " -> " & + artifactsDir / artifactName - copyFileWithPermissions(wksp.dir & "/" & step.workingDir & "/" & - artifactPath, artifactsDir & "/" & artifactName) + copyFileWithPermissions(wksp.dir / step.workingDir / artifactPath, + artifactsDir / artifactName) except: raiseEx "step " & step.name & " failed: unable to copy artifact " & artifactPath & ":\n" & getCurrentExceptionMsg() - wksp.publishStatus(BuildState.complete, "") + wksp.publishStatus(BuildState.stepComplete, "step " & step.name & " complete") result = wksp.status proc run*(cfg: StrawBossConfig, req: RunRequest, @@ -401,8 +439,8 @@ proc run*(cfg: StrawBossConfig, req: RunRequest, ensureProjectDirsExist(cfg, projectDef) # Update our run status - let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs" - writeFile(runDir & "/" & $req.runId & ".status.json", $result) + let runDir = cfg.buildDataDir / projectDef.name / "runs" + writeFile(runDir / $req.runId & ".status.json", $result) # Read in the existing system environment var env = loadEnv() @@ -413,13 +451,13 @@ proc run*(cfg: StrawBossConfig, req: RunRequest, if not existsDir(req.workspaceDir): createDir(req.workspaceDir) # Setup our STDOUT and STDERR files - let stdoutFile = open(runDir & "/" & $req.runId & ".stdout.log", fmWrite) - let stderrFile = open(runDir & "/" & $req.runId & ".stderr.log", fmWrite) + let stdoutFile = open(runDir / $req.runId & ".stdout.log", fmWrite) + let stderrFile = open(runDir / $req.runId & ".stderr.log", fmWrite) let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile) wksp = Workspace( - buildDataDir: cfg.buildDataDir & "/" & projectDef.name, + buildDataDir: cfg.buildDataDir / projectDef.name, buildRef: if req.buildRef != nil and req.buildRef.len > 0: req.buildRef else: projectDef.defaultBranch, @@ -452,8 +490,8 @@ proc run*(cfg: StrawBossConfig, req: RunRequest, # Update our cache of project configurations. # TODO: what happens if this fails? copyFileWithPermissions( - wksp.dir & "/" & wksp.projectDef.cfgFilePath, - wksp.buildDataDir & "/configurations/" & wksp.version & ".json") + wksp.dir / wksp.projectDef.cfgFilePath, + wksp.buildDataDir / "configurations" / wksp.version & ".json") # Find the requested step if not wksp.project.steps.hasKey(req.stepName): @@ -462,7 +500,12 @@ proc run*(cfg: StrawBossConfig, req: RunRequest, if req.forceRebuild: step.dontSkip = true - result = doStep(wksp, step) + var buildStatus = doStep(wksp, step) + if buildStatus.state == BuildState.stepComplete: + buildStatus.state = BuildState.complete + wksp.publishStatus(buildStatus.state, "all steps complete") + + result = wksp.status except: when not defined(release): echo getCurrentException().getStackTrace() @@ -477,19 +520,19 @@ proc run*(cfg: StrawBossConfig, req: RunRequest, finally: if wksp != nil: - # Close open files + # Close open files for f in wksp.openedFiles: try: close(f) except: discard "" proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest): tuple[status: BuildStatus, worker: Worker] = - + # Find the project definition (will throw appropriate exceptions) let projectDef = cfg.getProject(req.projectName) - let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs" - let reqFile = runDir & "/" & $req.runId & ".request.json" - let statusFile = runDir & "/" & $req.runId & ".status.json" + let runDir = cfg.buildDataDir / projectDef.name / "runs" + let reqFile = runDir / $req.runId & ".request.json" + let statusFile = runDir / $req.runId & ".status.json" try: # Make sure the build data directories for this project exist. @@ -517,8 +560,8 @@ proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest): except: let exMsg = "run request rejected: " & getCurrentExceptionMsg() - raiseEx exMsg try: writeFile(statusFile, $(BuildStatus(runId: $req.runId, state: BuildState.rejected, details: exMsg))) except: discard "" + raiseEx exMsg diff --git a/src/main/nim/strawbosspkg/server.nim b/src/main/nim/strawbosspkg/server.nim index d2f0e41..4278282 100644 --- a/src/main/nim/strawbosspkg/server.nim +++ b/src/main/nim/strawbosspkg/server.nim @@ -1,6 +1,11 @@ -import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, +import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, md5, os, osproc, sequtils, strutils, tempfile, times, unittest, uuids +from mimetypes import getMimeType +from asyncfile import openAsync, readToStream, close +from asyncnet import send +from re import re, find + import ./configuration, ./core type @@ -17,13 +22,22 @@ proc newSession*(user: UserRef): Session = issuedAt: getTime(), expires: daysForward(7).toTime()) -proc makeJsonResp(status: HttpCode, details: string = ""): string = - result = $(%* { - "statusCode": status.int, - "status": $status, +proc buildJson(resp: Response, code: HttpCode, details: string = ""): void = + resp.data[0] = CallbackAction.TCActionSend + resp.data[1] = code + resp.data[2]["Content-Type"] = JSON + resp.data[3] = $(%* { + "statusCode": code.int, + "status": $code, "details": details }) +# Work-around for weirdness trying to use resp(Http500... in exception blocks +proc build500Json(resp: Response, ex: ref Exception, msg: string): void = + when not defined(release): debug ex.getStackTrace() + error msg & ":\n" & ex.msg + resp.buildJson(Http500) + proc toJWT*(cfg: StrawBossConfig, session: Session): string = ## Make a JST token for this session. var jwt = JWT( @@ -46,7 +60,6 @@ proc fromJWT*(cfg: StrawBossConfig, strTok: string): Session = # Find the user record (if authenticated) let username = jwt.claims["sub"].node.str let users = cfg.users.filterIt(it.name == username) - debug "username: " & username & "\n\tusers: " & $users.mapIt(it.name) & "\n\tall users: " & cfg.users.mapIt(it.name) if users.len != 1: raiseEx "Could not find session user." result = Session( @@ -131,7 +144,8 @@ template checkAuth() = except: debug "Auth failed: " & getCurrentExceptionMsg() response.data[2]["WWW-Authenticate"] = "Bearer" - resp(Http401, makeJsonResp(Http401), JSON) + response.buildJson(Http401) + return proc start*(cfg: StrawBossConfig): void = @@ -144,18 +158,21 @@ proc start*(cfg: StrawBossConfig): void = routes: + get "/ping": + resp($(%"pong"), JSON) + post "/auth-token": var uname, pwd: string try: let jsonBody = parseJson(request.body) uname = jsonBody["username"].getStr pwd = jsonBody["password"].getStr - except: resp(Http400, makeJsonResp(Http400), JSON) + except: response.buildJson(Http400); return try: let authToken = makeAuthToken(cfg, uname, pwd) resp($(%authToken), JSON) - except: resp(Http401, makeJsonResp(Http401, getCurrentExceptionMsg()), JSON) + except: response.buildJson(Http401, getCurrentExceptionMsg()); return get "/verify-auth": checkAuth(); if not authed: return true @@ -175,7 +192,7 @@ proc start*(cfg: StrawBossConfig): void = checkAuth(); if not authed: return true # TODO - resp(Http501, makeJsonResp(Http501), JSON) + response.buildJson(Http501); return get "/project/@projectName": ## Return a project's configuration, as well as it's versions. @@ -185,7 +202,14 @@ proc start*(cfg: StrawBossConfig): void = # Make sure we know about that project var projDef: ProjectDef try: projDef = cfg.getProject(@"projectName") - except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) + except: + try: raise getCurrentException() + except NotFoundException: + response.buildJson(Http404, getCurrentExceptionMsg()) + except: + response.build500Json(getCurrentException(), + "unable to load project definition for project " & @"projectName") + return true var projConf: ProjectConfig try: projConf = getProjectConfig(cfg, @"projectName", "") @@ -206,13 +230,13 @@ proc start*(cfg: StrawBossConfig): void = try: resp($(%listVersions(cfg, @"projectName")), JSON) except: - if getCurrentException() is KeyError: - resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) - else: - when not defined(release): debug getCurrentException().getStackTrace() - error "unable to list versions for project " & @"projectName" & - ":\n" & getCurrentExceptionMsg() - resp(Http500, makeJsonResp(Http500, "internal server error"), JSON) + try: raise getCurrentException() + except NotFoundException: + response.buildJson(Http404, getCurrentExceptionMsg()) + except: + response.build500Json(getCurrentException(), + "unable to list versions for project " & @"projectName") + return true get "/project/@projectName/version/@version?": ## Get a detailed project record including step definitions (ProjectConfig). @@ -221,7 +245,7 @@ proc start*(cfg: StrawBossConfig): void = # Make sure we know about that project try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON) - except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) + except: response.buildJson(Http404, getCurrentExceptionMsg()); return true get "/project/@projectName/runs": ## List all runs @@ -229,7 +253,7 @@ proc start*(cfg: StrawBossConfig): void = checkAuth(); if not authed: return true try: resp($(%listRuns(cfg, @"projectName")), JSON) - except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) + except: response.buildJson(Http404, getCurrentExceptionMsg()); return true get "/project/@projectName/runs/active": ## List all currently active runs @@ -242,12 +266,13 @@ proc start*(cfg: StrawBossConfig): void = .mapIt(cfg.getRun(@"projecName", $it.runId)); resp($(%activeRuns), JSON) except: - if getCurrentException() is KeyError: - resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) - else: - when not defined(release): debug getCurrentException().getStackTrace() - error "problem loading active runs: " & getCurrentExceptionMsg() - resp(Http500, makeJsonResp(Http500, "internal server error"), JSON) + try: raise getCurrentException() + except NotFoundException: + response.buildJson(Http404, getCurrentExceptionMsg()) + except: + response.build500Json(getCurrentException(), + "problem loading active runs") + return true get "/project/@projectName/run/@runId": ## Details for a specific run @@ -256,13 +281,115 @@ proc start*(cfg: StrawBossConfig): void = # Make sure we know about that project try: discard cfg.getProject(@"projectName") - except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) + except: response.buildJson(Http404, getCurrentExceptionMsg()); return true if not existsRun(cfg, @"projectName", @"runId"): - resp(Http404, makeJsonResp(Http404, "no such run for project"), JSON) + response.buildJson(Http404, "no such run for project"); return true try: resp($getRun(cfg, @"projectName", @"runId"), JSON) - except: resp(Http500, makeJsonResp(Http500, getCurrentExceptionMsg()), JSON) + except: + response.build500Json(getCurrentException(), + "unable to load run details for project " & @"projectName" & + " run " & @"runId") + return true + + get "/project/@projectName/run/@runId/logs": + ## Get logs from a specific run + + checkAuth(); if not authed: return true + + try: discard cfg.getProject(@"projectName") + except: + response.buildJson(Http404, getCurrentExceptionMsg()) + return true + + if not existsRun(cfg, @"projectName", @"runId"): + response.buildJson(Http404, "no such run for project") + return true + + try: resp($getLogs(cfg, @"projectName", @"runId")) + except: + response.build500Json(getCurrentException(), + "unable to load run logs for " & @"projectName" & " run " & @"runId") + return true + + get "/project/@projectName/step/@stepName/artifacts/@version": + ## Get the list of artifacts that were built for + + checkAuth(); if not authed: return true + + debug "Matched artifacts list request: " & $(%*{ + "project": @"projectName", + "step": @"stepName", + "version": @"version" + }) + + try: resp($(%listArtifacts(cfg, @"projectName", @"stepName", @"version")), JSON) + except: + try: raise getCurrentException() + except NotFoundException: + response.buildJson(Http404, getCurrentExceptionMsg()) + except: + response.build500Json(getCurrentException(), "unable to list artifacts for " & + @"projectName" & ":" & @"stepName" & "@" & @"buildRef") + return true + + get "/project/@projectName/step/@stepName/artifact/@version/@artifactName": + ## Get a specific artifact that was built. + + checkAuth(); if not authed: return true + + var artifactPath: string + try: artifactPath = getArtifactPath(cfg, + @"projectName", @"stepName", @"version", @"artifactName") + except: + try: raise getCurrentException() + except NotFoundException: + response.buildJson(Http404, getCurrentExceptionMsg()) + except: + response.build500Json(getCurrentException(), "unable to check artifact path for " & + @"projectName" & ":" & @"stepName" & "@" & @"version") + return true + + debug "Preparing: " & artifactPath + let fileSize = getFileSize(artifactPath) + let mimetype = request.settings.mimes.getMimetype(artifactPath.splitFile.ext[1 .. ^1]) + if fileSize < 10_000_000: # 10 mb + var file = readFile(artifactPath) + + var hashed = getMD5(file) + + # If the user has a cached version of this file and it matches our + # version, let them use it + if request.headers.hasKey("If-None-Match") and request.headers["If-None-Match"] == hashed: + resp(Http304, [], "") + else: + resp(Http200, [ + ("Content-Disposition", "; filename=\"" & @"artifactName" & "\""), + ("Content-Type", mimetype), + ("ETag", hashed )], file) + else: + let headers = { + "Content-Disposition": "; filename=\"" & @"artifactName" & "\"", + "Content-Type": mimetype, + "Content-Length": $fileSize + }.newStringTable + await response.sendHeaders(Http200, headers) + + var fileStream = newFutureStream[string]("sendStaticIfExists") + var file = openAsync(artifactPath, fmRead) + # Let `readToStream` write file data into fileStream in the + # background. + asyncCheck file.readToStream(fileStream) + # The `writeFromStream` proc will complete once all the data in the + # `bodyStream` has been written to the file. + while true: + let (hasValue, value) = await fileStream.read() + if hasValue: + await response.client.send(value) + else: + break + file.close() get "/project/@projectName/step/@stepName/status/@buildRef": ## Get detailed information about the status of a step (assuming it has been built) @@ -270,7 +397,19 @@ proc start*(cfg: StrawBossConfig): void = checkAuth(); if not authed: return true try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON) - except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) + except: + try: raise getCurrentException() + except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg()) + except: + response.build500Json(getCurrentException(), "unable to load the build state for " & + @"projectName" & ":" & @"stepName" & "@" & @"buildRef") + return true + + #get "/project/@projectName/step/@stepName/status/@buildRef.svg": + ## Get an image representing the status of a build + + ## TODO: how do we want to handle auth for this? Unlike + #checkAuth(): if not authed: return true post "/project/@projectName/step/@stepName/run/@buildRef?": # Kick off a run @@ -296,26 +435,26 @@ proc start*(cfg: StrawBossConfig): void = id: runRequest.runId, request: runRequest, status: status), JSON) - except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) - - get "/service/debug/ping": - if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON) - else: resp($(%"pong"), JSON) + except: + try: raise getCurrentException() + except NotFoundException: + response.buildJson(Http404, getCurrentExceptionMsg()) + except: response.buildJson(Http400, getCurrentExceptionMsg()) + return true post "/service/debug/stop": - if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON) + if not cfg.debug: response.buildJson(Http404); return else: let shutdownFut = sleepAsync(100) shutdownFut.callback = proc(): void = complete(stopFuture) resp($(%"shutting down"), JSON) - #[ + get re".*": - resp(Http404, makeJsonResp(Http404), JSON) + response.buildJson(Http404); return true post re".*": - resp(Http404, makeJsonResp(Http404), JSON) - ]# + response.buildJson(Http404); return true proc performMaintenance(cfg: StrawBossConfig): void = # Prune workers diff --git a/src/test/nim/unit/tserver.nim b/src/test/nim/unit/tserver.nim index 0db62be..46926ab 100644 --- a/src/test/nim/unit/tserver.nim +++ b/src/test/nim/unit/tserver.nim @@ -41,7 +41,7 @@ suite "strawboss server": check fromJWT(cfg, tok) == session test "ping": - let resp = http.get(apiBase & "/service/debug/ping") + let resp = http.get(apiBase & "/ping") check: resp.status.startsWith("200") resp.body == "\"pong\""