From e000b37c35c9769bdb980341965b3b7fcda454b5 Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Wed, 22 Nov 2017 10:47:00 -0600 Subject: [PATCH] WIP Moving back towards using named runs. * Rename artifactsRepo -> buildDataDir to be more explicit about the fact that it holds more than just the artifacts. * Revert removal of run ids. * Move Worker definition into core as part of making the core responsible for accepting run requests. * Make the core module more responsible for internal details of data structure and storage. External callers should not need to construct paths to artifacts, versions, etc. but should be able to call method in the core module to do this work for them. * The working directory no longer contains anything but the checked-out code. All StrawBoss-specific data is stored by StrawBoss elsewhere. * Add a regular maintenance cycle to the server module. --- README.md | 12 +- api.rst | 4 +- file-structure.txt | 17 ++ src/main/nim/strawboss.nim | 14 +- src/main/nim/strawbosspkg/configuration.nim | 18 +- src/main/nim/strawbosspkg/core.nim | 186 ++++++++++++-------- src/main/nim/strawbosspkg/server.nim | 84 ++++----- src/test/json/test-status.json | 1 + src/test/nim/functional/tserver.nim | 19 +- src/test/nim/unit/tconfiguration.nim | 4 +- strawboss.nimble | 2 + 11 files changed, 215 insertions(+), 146 deletions(-) create mode 100644 file-structure.txt diff --git a/README.md b/README.md index 2a607f6..e7e7397 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,9 @@ the `strawboss` executable. This is the configuration file for StrawBoss itself. The contents are expected to be a valid JSON object. The top level keys are: -* `artifactsRepo`: A string denoting the path to the artifacts repository - directory. +* `buildDataDir`: A string denoting the path to the directory where StrawBoss + keeps metadata about builds it has performed and the artifacts resulting from + the builds. * `authSecret`: Secret key used to sign JWT session tokens. @@ -113,12 +114,7 @@ object. The top level keys are: * `versionCmd` *(optional)*: a command to be run in a shell (`sh`-compatible) that is expected to print the current version of the project on `stdout`. - It is important to note that if you supply a verion command it should provide - a unique result for every commit in the repository. StrawBoss is built around - the assumptions that builds are repeatable and that every buildable point has - a unique version id. This is the reason that StrawBoss does not create uniue - IDs for individual builds. The combination of project name, build step, and - version *is* the build ID. *(defaults to `git describe --tags --always`)*. + *(defaults to `git describe --tags --always`)*. #### Step Definition diff --git a/api.rst b/api.rst index 3c2d2a3..69e785c 100644 --- a/api.rst +++ b/api.rst @@ -6,10 +6,10 @@ - GET /api/project/ -- TODO * GET /api/project//runs -- list summary information for all runs * GET /api/project//runs/active -- list summary information about all currently active runs +- GET /api/project//runs/ -- list detailed information about a specific run ✓ GET /api/project//versions -- list the versions of this project that have been built * GET /api/project//version/ -- return detailed project definition (include steps) at a specific version -- GET /api/project//step/ -- return detailed step information (include runs for different versions) -- GET /api/project//step//run/ -- list detailed information about a specific run +- GET /api/project//step/ -- return detailed step information (include runs) * POST /api/project//step//run/ -- kick off a run diff --git a/file-structure.txt b/file-structure.txt new file mode 100644 index 0000000..42738be --- /dev/null +++ b/file-structure.txt @@ -0,0 +1,17 @@ +build-data/ + / + configurations/ + .json + runs/ + .request.json + .stdout.log + .stderr.log + .status.json + status/ + .json + artifacts/ + / + / + + +workspace/ diff --git a/src/main/nim/strawboss.nim b/src/main/nim/strawboss.nim index a917a56..e3436ef 100644 --- a/src/main/nim/strawboss.nim +++ b/src/main/nim/strawboss.nim @@ -1,4 +1,4 @@ -import cliutils, docopt, os, sequtils, tempfile +import cliutils, docopt, os, sequtils, tempfile, uuids import strawbosspkg/configuration import strawbosspkg/core @@ -30,6 +30,9 @@ Options -r --reference Build the project at this commit reference. + -i --run-id Use the given UUID as the run ID. If not given, a + new UUID is generated for this run. + -w --workspace Use the given directory as the build workspace. """ @@ -41,11 +44,11 @@ Options var cfg = loadStrawBossConfig(cfgFile) cfg.pathToExe = paramStr(0) - if not existsDir(cfg.artifactsRepo): - echo "Artifacts repo (" & cfg.artifactsRepo & ") does not exist. Creating..." - createDir(cfg.artifactsRepo) + if not existsDir(cfg.buildDataDir): + echo "Build data directory (" & cfg.buildDataDir & ") does not exist. Creating..." + createDir(cfg.buildDataDir) - cfg.artifactsRepo = expandFilename(cfg.artifactsRepo) + cfg.buildDataDir = expandFilename(cfg.buildDataDir) if args["run"]: @@ -54,6 +57,7 @@ Options try: let req = RunRequest( + id: if args["--run-id"]: parseUUID($args["--run-id"]) else: genUUID(), projectName: $args[""], stepName: $args[""], buildRef: if args["--reference"]: $args["--reference"] else: nil, diff --git a/src/main/nim/strawbosspkg/configuration.nim b/src/main/nim/strawbosspkg/configuration.nim index 651f9f6..52d21ca 100644 --- a/src/main/nim/strawbosspkg/configuration.nim +++ b/src/main/nim/strawbosspkg/configuration.nim @@ -1,4 +1,4 @@ -import cliutils, logging, json, os, nre, sequtils, strtabs, tables, times +import cliutils, logging, json, os, nre, sequtils, strtabs, tables, times, uuids from langutils import sameContents from typeinfo import toAny @@ -7,7 +7,7 @@ from typeinfo import toAny # type BuildStatus* = object - state*, details*: string + runId*, state*, details*: string Step* = object name*, stepCmd*, workingDir*: string @@ -24,6 +24,7 @@ type envVars*: StringTableRef RunRequest* = object + id*: UUID projectName*, stepName*, buildRef*, workspaceDir*: string forceRebuild*: bool @@ -34,7 +35,7 @@ type UserRef* = ref User StrawBossConfig* = object - artifactsRepo*: string + buildDataDir*: string authSecret*: string filePath*: string debug*: bool @@ -60,7 +61,7 @@ proc `==`*(a, b: ProjectDef): bool = proc `==`*(a, b: StrawBossConfig): bool = result = - a.artifactsRepo == b.artifactsRepo and + a.buildDataDir == b.buildDataDir and a.authSecret == b.authSecret and a.pwdCost == b.pwdCost and sameContents(a.users, b.users) and @@ -68,6 +69,7 @@ proc `==`*(a, b: StrawBossConfig): bool = proc `==`*(a, b: RunRequest): bool = result = + a.id == b.id and a.projectName == b.projectName and a.stepName == b.stepName and a.buildRef == b.buildRef and @@ -130,7 +132,7 @@ proc parseStrawBossConfig*(jsonCfg: JsonNode): StrawBossConfig = hashedPwd: uJson.getOrFail("hashedPwd", "user record").getStr)) result = StrawBossConfig( - artifactsRepo: jsonCfg.getIfExists("artifactsRepo").getStr("artifacts"), + buildDataDir: jsonCfg.getIfExists("buildDataDir").getStr("build-data"), authSecret: jsonCfg.getOrFail("authSecret", "strawboss config").getStr, debug: jsonCfg.getIfExists("debug").getBVal(false), pwdCost: int8(jsonCfg.getOrFail("pwdCost", "strawboss config").getNum), @@ -189,11 +191,13 @@ proc loadBuildStatus*(statusFile: string): BuildStatus = let jsonObj = parseFile(statusFile) result = BuildStatus( + runId: jsonObj.getOrFail("runId", "run ID").getStr, state: jsonObj.getOrFail("state", "build status").getStr, details: jsonObj.getIfExists("details").getStr("") ) proc parseRunRequest*(reqJson: JsonNode): RunRequest = result = RunRequest( + id: parseUUID(reqJson.getOrFail("id", "RunRequest").getStr), projectName: reqJson.getOrFail("projectName", "RunRequest").getStr, stepName: reqJson.getOrFail("stepName", "RunRequest").getStr, buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr, @@ -203,6 +207,7 @@ proc parseRunRequest*(reqJson: JsonNode): RunRequest = # TODO: can we use the marshal module for this? proc `%`*(s: BuildStatus): JsonNode = result = %* { + "runId": s.runId, "state": s.state, "details": s.details } @@ -238,6 +243,7 @@ proc `%`*(p: ProjectConfig): JsonNode = proc `%`*(req: RunRequest): JsonNode = result = %* { + "id": $(req.id), "projectName": req.projectName, "stepName": req.stepName, "buildRef": req.buildRef, @@ -251,7 +257,7 @@ proc `%`*(user: User): JsonNode = proc `%`*(cfg: StrawBossConfig): JsonNode = result = %* { - "artifactsRepo": cfg.artifactsRepo, + "buildDataDir": cfg.buildDataDir, "authSecret": cfg.authSecret, "debug": cfg.debug, "projects": %cfg.projects, diff --git a/src/main/nim/strawbosspkg/core.nim b/src/main/nim/strawbosspkg/core.nim index 96dc064..83a54ee 100644 --- a/src/main/nim/strawbosspkg/core.nim +++ b/src/main/nim/strawbosspkg/core.nim @@ -1,14 +1,14 @@ import cliutils, logging, json, options, os, osproc, sequtils, streams, - strtabs, strutils, tables + strtabs, strutils, tables, uuids -import nre except toSeq import ./configuration +import nre except toSeq from posix import link type Workspace = ref object ## Data needed by internal build process artifactsDir*: string ## absolute path to the directory for this version - artifactsRepo*: string ## absolute path to the global artifacts repo + buildDataDir*: string ## absolute path to the global build data directory for this project buildRef*: string ## git-style commit reference to the revision we are building dir*: string ## absolute path to the working directory env*: StringTableRef ## environment variables for all build processes @@ -16,11 +16,16 @@ type outputHandler*: HandleProcMsgCB ## handler for process output project*: ProjectConfig ## the project configuration projectDef*: ProjectDef ## the StrawBoss project definition + runRequest*: RunRequest ## the RunRequest that initated the current build status*: BuildStatus ## the current status of the build statusFile*: string ## absolute path to the build status file step*: Step ## the step we're building version*: string ## project version as returned by versionCmd + Worker* = object + runId*: UUID + process*: Process + proc sendMsg(h: HandleProcMsgCB, msg: TaintedString): void = h.sendMsg(msg, nil, "strawboss") @@ -45,21 +50,53 @@ proc emitStatus(status: BuildStatus, statusFilePath: string, proc publishStatus(wksp: Workspace, state, details: string) = ## Update the status for a Workspace and publish this status to the ## Workspace's status file and any output message handlers. - let status = BuildStatus(state: state, details: details) + let status = BuildStatus( + runId: $wksp.runRequest.id, state: state, details: details) wksp.status = emitStatus(status, wksp.statusFile, wksp.outputHandler) +proc ensureProjectDirsExist(cfg: StrawBossConfig, p: ProjectDef): void = + for subdir in ["configurations", "runs", "status", "artifacts"]: + let fullPath = cfg.buildDataDir & "/" & p.name & "/" & subdir + if not existsDir(fullPath): + createDir(fullPath) + +proc listVersions*(cfg: StrawBossConfig, project: ProjectDef): seq[string] = + ## List the versions that have been built for a project. + ensureProjectDirsExist(cfg, project) + + let versionFiles = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/configurations/*.json")) + result = versionFiles.map(proc(s: string): string = + let slashIdx = s.rfind('/') + result = s[(slashIdx + 1)..^6]) + +proc listRuns*(cfg: StrawBossConfig, project: ProjectDef): seq[RunRequest] = + ## List the runs that have been performed for a project. + ensureProjectDirsExist(cfg, project) + + let runPaths = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/runs/*.request.json")) + return runPaths.mapIt(parseRunRequest(parseFile(it))) + +proc getCurrentProjectConfig*(cfg: StrawBossConfig, project: ProjectDef): Option[ProjectConfig] = + let projCfgFile = "nope.json" # TODO + if not existsFile(projCfgFile): result = none(ProjectConfig) + else: + try: + let projectConfig: ProjectConfig = loadProjectConfig(projCfgFile) #ProjectConfig(name: "test") + result = some(projectConfig) + except: result = none(ProjectConfig) + proc setupProject(wksp: Workspace) = - # Clone the project into the $temp/repo directory - let cloneResult = exec("git", wksp.dir, - ["clone", wksp.projectDef.repo, "repo"], + # Clone the project into the $temp directory + let cloneResult = exec("git", ".", + ["clone", wksp.projectDef.repo, wksp.dir], wksp.env, {poUsePath}, wksp.outputHandler) if cloneResult != 0: raiseEx "unable to clone repo for '" & wksp.projectDef.name & "'" # Checkout the requested ref - let checkoutResult = exec("git", wksp.dir & "/repo", + let checkoutResult = exec("git", wksp.dir, ["checkout", wksp.buildRef], wksp.env, {poUsePath}, wksp.outputHandler) @@ -68,7 +105,7 @@ proc setupProject(wksp: Workspace) = " for '" & wksp.projectDef.name & "'" # Find the strawboss project configuration - let projCfgFile = wksp.dir & "/repo/" & wksp.projectDef.cfgFilePath + let projCfgFile = wksp.dir & wksp.projectDef.cfgFilePath if not existsFile(projCfgFile): raiseEx "Cannot find strawboss project configuration in the project " & "repo (expected at '" & wksp.projectDef.cfgFilePath & "')." @@ -81,7 +118,7 @@ proc setupProject(wksp: Workspace) = # Get the build version let versionResult = execWithOutput( wksp.project.versionCmd, # command - wksp.dir & "/repo", # working dir + wksp.dir, # working dir [], # args wksp.env, # environment {poUsePath, poEvalCommand}) # options @@ -93,22 +130,6 @@ proc setupProject(wksp: Workspace) = wksp.version = versionResult.output.strip wksp.env["VERSION"] = wksp.version -proc listRuns*(cfg: StrawBossConfig, project: ProjectDef): seq[RunRequest] = - let runsDir = cfg.artifactsRepo & "/" & project.name & "/runs" - if not existsDir(runsDir): return @[] - - let runPaths = toSeq(walkFiles(runsDir & "/*.json")) - return runPaths.mapIt(parseRunRequest(parseFile(it))) - -proc getCurrentProjectConfig*(cfg: StrawBossConfig, project: ProjectDef): Option[ProjectConfig] = - let projCfgFile = cfg.artifactsRepo & "/" & project.name & "/" & project.cfgFilePath - if not existsFile(projCfgFile): result = none(ProjectConfig) - else: - try: - let projectConfig: ProjectConfig = loadProjectConfig(projCfgFile) #ProjectConfig(name: "test") - result = some(projectConfig) - except: result = none(ProjectConfig) - proc runStep*(wksp: Workspace, step: Step) = ## Lower-level method to execute a given step within the context of a project @@ -139,14 +160,13 @@ proc runStep*(wksp: Workspace, step: Step) = # Add the artifacts directory for the dependent step to our env so that # further steps can reference it via $_DIR - wksp.env[depStep.name & "_DIR"] = wksp.artifactsRepo & "/" & - wksp.project.name & "/" & dep & "/" & wksp.version + wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir & "/artifacts/" & + "/" & dep & "/" & wksp.version # Run the step command, piping in cmdInput wksp.outputHandler.sendMsg step.name & ": starting stepCmd: " & step.stepCmd let cmdProc = startProcess(step.stepCmd, - wksp.dir & "/repo/" & step.workingDir, - [], wksp.env, {poUsePath, poEvalCommand}) + wksp.dir & step.workingDir, [], wksp.env, {poUsePath, poEvalCommand}) let cmdInStream = inputStream(cmdProc) @@ -167,11 +187,11 @@ proc runStep*(wksp: Workspace, step: Step) = let artifactPath = a.resolveEnvVars(wksp.env) let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1] try: - wksp.outputHandler.sendMsg "copy " & wksp.dir & "/repo/" & + wksp.outputHandler.sendMsg "copy " & wksp.dir & step.workingDir & "/" & artifactPath & " -> " & wksp.artifactsDir & "/" & artifactName - copyFile(wksp.dir & "/repo/" & step.workingDir & "/" & artifactPath, + copyFile(wksp.dir & step.workingDir & "/" & artifactPath, wksp.artifactsDir & "/" & artifactName) except: raiseEx "step " & step.name & " failed: unable to copy artifact " & @@ -186,6 +206,7 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest, ## entrypoint to running a build step. result = BuildStatus( + runId: $req.id, state: "setup", details: "initializing build workspace") discard emitStatus(result, nil, outputHandler) @@ -193,29 +214,33 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest, var wksp: Workspace try: - assert req.workspaceDir.isAbsolute - if not existsDir(req.workspaceDir): createDir(req.workspaceDir) - # Find the project definition let projectDef = cfg.findProject(req.projectName) - # Make sure our directory in the artifacts repo exists - if not existsDir(cfg.artifactsRepo & "/" & projectDef.name & "/run-requests"): - createDir(cfg.artifactsRepo & "/" & projectDef.name & "/run-requests") + # Make sure the build data directories for this project exist. + ensureProjectDirsExist(cfg, projectDef) + + # Update our run status + let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs" + discard emitStatus(result, runDir & "/" & $req.id & ".status.json", nil) # Read in the existing system environment var env = loadEnv() env["GIT_DIR"] = ".git" + # Make sure we have a workspace directory + assert req.workspaceDir.isAbsolute + if not existsDir(req.workspaceDir): createDir(req.workspaceDir) + # Setup our STDOUT and STDERR files - let stdoutFile = open(req.workspaceDir & "/stdout.log", fmWrite) - let stderrFile = open(req.workspaceDir & "/stderr.log", fmWrite) + let stdoutFile = open(runDir & "/" & $req.id & ".stdout.log", fmWrite) + let stderrFile = open(runDir & "/" & $req.id & ".stderr.log", fmWrite) let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile) wksp = Workspace( artifactsDir: nil, - artifactsRepo: cfg.artifactsRepo, + buildDataDir: cfg.buildDataDir & "/" & projectDef.name, buildRef: if req.buildRef != nil and req.buildRef.len > 0: req.buildRef else: projectDef.defaultBranch, @@ -225,14 +250,15 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest, outputHandler: combineProcMsgHandlers(outputHandler, logFilesOH), project: ProjectConfig(), projectDef: projectDef, + runRequest: req, status: result, - statusFile: req.workspaceDir & "/" & "status.json", + statusFile: runDir & "/" & $req.id & ".status.json", step: Step(), version: nil) except: when not defined(release): echo getCurrentException().getStackTrace() - result = BuildStatus(state: "failed", + result = BuildStatus(runId: $req.id, state: "failed", details: getCurrentExceptionMsg()) try: discard emitStatus(result, nil, outputHandler) except: discard "" @@ -244,33 +270,26 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest, "cloning project repo and preparing to run '" & req.stepName & "'") wksp.setupProject() - # Make sure our project directory exists in the artifacts repo - if not existsDir(wksp.artifactsRepo & "/" & wksp.project.name): - createDir(wksp.artifactsRepo & "/" & wksp.project.name) - - # Update our cache of project configurations by copying the configuration - # file to our artifacts directory. + # Update our cache of project configurations. + # TODO: what happens if this fails? copyFile( - wksp.dir & "/repo/" & wksp.projectDef.cfgFilePath, - cfg.artifactsRepo & "/" & wksp.project.name & "/configuration." & - wksp.version & ".json") + wksp.dir & "/" & wksp.projectDef.cfgFilePath, + wksp.buildDataDir & "/configurations/" & wksp.version & ".json") # Find the requested step if not wksp.project.steps.hasKey(req.stepName): raiseEx "no step name '" & req.stepName & "' for " & req.projectName - var step = wksp.project.steps[req.stepName] - # Enfore forceRebuild if req.forceRebuild: step.dontSkip = true - # Compose the path to the artifacts directory for this step and version - wksp.artifactsDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & + wksp.artifactsDir = wksp.buildDataDir & "/artifacts/" & step.name & "/" & wksp.version # Have we tried to build this before and are we caching the results? - if existsFile(wksp.artifactsDir & "/status.json") and not step.dontSkip: - let prevStatus = loadBuildStatus(wksp.artifactsDir & "/status.json") + let statusFilePath = wksp.buildDataDir & "/status/" & wksp.version & ".json" + if existsFile(statusFilePath) and not step.dontSkip: + let prevStatus = loadBuildStatus(statusFilePath) # If we succeeded last time, no need to rebuild if prevStatus.state == "complete": @@ -283,21 +302,13 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest, "Rebuilding failed step '" & step.name & "' for version '" & wksp.version & "'.") - # Make the artifacts directory if it doesn't already exist if not existsDir(wksp.artifactsDir): createDir(wksp.artifactsDir) - # Link status file and output logs to the artifacts dir - for fn in @["status.json", "stdout.log", "stderr.log"]: - # TODO: roll old files instead of delete them? - if existsFile(wksp.artifactsDir & "/" & fn): - removeFile(wksp.artifactsDir & "/" & fn) - - if link(wksp.dir & "/" & fn, wksp.artifactsDir & "/" & fn) != 0: - wksp.outputHandler.sendErrMsg( - "WARN: could not link " & fn & " to artifacts dir.") - runStep(wksp, step) + # Record the results of this build as the status for this version. + writeFile(wksp.buildDataDir & "/status/" & wksp.version & ".json", $wksp.status) + result = wksp.status except: @@ -307,7 +318,7 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest, wksp.publishStatus("failed", msg) result = wksp.status except: - result = BuildStatus(state: "failed", details: msg) + result = BuildStatus(runId: $req.id, state: "failed", details: msg) try: discard emitStatus(result, nil, outputHandler) except: discard "" @@ -317,3 +328,38 @@ proc initiateRun*(cfg: StrawBossConfig, req: RunRequest, try: close(f) except: discard "" +proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest): Worker = + + # Find the project definition (will throw appropriate exceptions) + let projectDef = cfg.findProject(req.projectName) + let runDir = cfg.buildDataDir & "/" & projectDef.name & "/runs" + let reqFile = runDir & "/" & $req.id & ".request.json" + let statusFile = runDir & "/" & $req.id & ".status.json" + + try: + # Make sure the build data directories for this project exist. + ensureProjectDirsExist(cfg, projectDef) + + # Save the run request + writeFile(reqFile, $req) + + # Write the initial build status (queued). + let queuedStatus = BuildStatus( + runId: $req.id, + state: "queued", + details: "request queued for execution") + writeFile(statusFile, $queuedStatus) + + var args = @["run", reqFile] + debug "Launching worker: " & cfg.pathToExe & " " & args.join(" ") + result = Worker( + runId: req.id, + process: startProcess(cfg.pathToExe, ".", args, loadEnv(), {poUsePath})) + + except: + let exMsg = "run request rejected: " & getCurrentExceptionMsg() + raiseEx exMsg + try: + writeFile(statusFile, + $(BuildStatus(runId: $req.id, state: "rejected", details: exMsg))) + except: discard "" diff --git a/src/main/nim/strawbosspkg/server.nim b/src/main/nim/strawbosspkg/server.nim index 0c1a435..5d2a03b 100644 --- a/src/main/nim/strawbosspkg/server.nim +++ b/src/main/nim/strawbosspkg/server.nim @@ -1,12 +1,8 @@ import algorithm, asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, - options, os, osproc, sequtils, strutils, tempfile, times, unittest + options, os, osproc, sequtils, strutils, tempfile, times, unittest, uuids import ./configuration, ./core -type Worker = object - process*: Process - workingDir*: string - type Session = object user*: UserRef @@ -14,6 +10,7 @@ type #const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss" const JSON = "application/json" +const CLEANUP_PERIOD_MS = 1000 proc makeJsonResp(status: HttpCode, details: string = ""): string = result = $(%* { @@ -71,18 +68,6 @@ proc extractSession(cfg: StrawBossConfig, request: Request): Session = result = fromJWT(cfg, headerVal[7..^1]) -proc spawnWorker(cfg: StrawBossConfig, req: RunRequest): Worker = - ## Kick off a new worker process with the given run information - - let dir = mkdtemp() - var args = @["run", req.projectName, req.stepName, "-r", req.buildRef, - "-w", dir, "-c", cfg.filePath] - if req.forceRebuild: args.add("-f") - debug "Launching worker: " & cfg.pathToExe & " " & args.join(" ") - result = Worker( - process: startProcess(cfg.pathToExe, ".", args, loadEnv(), {poUsePath}), - workingDir: dir) - proc hashPwd*(pwd: string, cost: int8): string = let salt = genSalt(cost) result = hash(pwd, salt) @@ -124,9 +109,19 @@ template checkAuth() = debug "Auth failed: " & getCurrentExceptionMsg() resp(Http401, makeJsonResp(Http401), JSON) +proc performMaintenance(cfg: StrawBossConfig): void = + # Prune workers + workers = workers.filterIt(it.running()) + + let fut = sleepAsync(CLEANUP_PERIOD_MS) + fut.callback = + proc(): void = + callSoon(proc(): void = performMaintenance(cfg)) + + proc start*(cfg: StrawBossConfig): void = - let stopFuture = newFuture[void]() + var stopFuture = newFuture[void]() var workers: seq[Worker] = @[] # TODO: add recurring clean-up down to clear completed workers from the @@ -138,7 +133,7 @@ proc start*(cfg: StrawBossConfig): void = routes: - get "/ping": resp($(%*"pong"), JSON) + get "/ping": resp($(%"pong"), JSON) post "/auth-token": var uname, pwd: string @@ -150,7 +145,7 @@ proc start*(cfg: StrawBossConfig): void = try: let authToken = makeAuthToken(cfg, uname, pwd) - resp("\"" & $authToken & "\"", JSON) + resp($(%authToken), JSON) except: resp(Http401, makeJsonResp(Http401, getCurrentExceptionMsg()), JSON) get "/verify-auth": @@ -163,7 +158,7 @@ proc start*(cfg: StrawBossConfig): void = checkAuth(); if not authed: return true - resp($(%(cfg.projects)), JSON) + resp($(%cfg.projects), JSON) post "/projects": ## Create a new project definition @@ -183,13 +178,7 @@ proc start*(cfg: StrawBossConfig): void = try: project = cfg.findProject(@"projectName") except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) - var versions: seq[string] = @[] - for cfgFilePath in walkFiles(cfg.artifactsRepo & "/" & project.name & "/configuration.*.json"): - let vstart = cfgFilePath.rfind("/configuration.") + 15 - versions.add(cfgFilePath[vstart..^6]) - - if versions.len == 0: - resp(Http404, makeJsonResp(Http404, "I have not built any versions of " & project.name), JSON) + var versions: seq[string] = listVersions(cfg, project) resp($(%(versions)), JSON) @@ -206,8 +195,8 @@ proc start*(cfg: StrawBossConfig): void = # Given version var cachedFilePath: string if @"version" != "": - cachedFilePath = cfg.artifactsRepo & "/" & project.name & - "/configuration." & @"version" & ".json" + cachedFilePath = cfg.buildDataDir & "/" & project.name & + "/configurations/" & @"version" & ".json" if not existsFile(cachedFilePath): resp(Http404, @@ -216,7 +205,7 @@ proc start*(cfg: StrawBossConfig): void = # No version requested, use "latest" else: - let confFilePaths = toSeq(walkFiles(cfg.artifactsRepo & "/" & project.name & "/configuration.*.json")) + let confFilePaths = toSeq(walkFiles(cfg.buildDataDir & "/" & project.name & "/configurations/*.json")) if confFilePaths.len == 0: resp(Http404, makeJsonResp(Http404, "I have not built any versions of " & project.name), JSON) let modTimes = confFilePaths.mapIt(it.getLastModificationTime) @@ -230,7 +219,7 @@ proc start*(cfg: StrawBossConfig): void = resp(Http500, makeJsonResp(Http500, "could not read cached project configuration"), JSON) get "/project/@projectName": - ## TBD + ## Return a project's configuration, as well as it's versions. checkAuth(); if not authed: return true @@ -239,17 +228,11 @@ proc start*(cfg: StrawBossConfig): void = try: projDef = cfg.findProject(@"projectName") except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) - # Get the project configuration. - let projConf = getCurrentProjectConfig(cfg, projDef) + let respJson = newJObject() + respJson["definition"] = %projDef + respJson["versions"] = %listVersions(cfg, projDef) - var respObj = newJObject() - respObj["definition"] = %projDef - - if projConf.isSome(): - let pc: ProjectConfig = projConf.get() - respObj["configuration"] = %pc - - resp($respObj, JSON) + resp(pretty(respJson), JSON) get "/project/@projectName/runs": ## List all runs @@ -263,7 +246,7 @@ proc start*(cfg: StrawBossConfig): void = let runRequests = listRuns(cfg, project) - resp($(%runRequests), JSON) + resp($runRequests, JSON) get "/project/@projectName/runs/active": ## List all currently active runs @@ -275,6 +258,14 @@ proc start*(cfg: StrawBossConfig): void = #resp($(%statuses), JSON) resp(Http501, makeJsonResp(Http501), JSON) + get "/project/@projectName/runs/@runId": + ## Details for a specific run + + checkAuth(); if not authed: return true + + # TODO + resp(Http501, makeJsonResp(Http501), JSON) + get "/project/@projectName/step/@stepName": ## Get step details including runs. @@ -297,6 +288,7 @@ proc start*(cfg: StrawBossConfig): void = checkAuth(); if not authed: return true let runRequest = RunRequest( + id: genUUID(), projectName: @"projectName", stepName: @"stepName", buildRef: if @"buildRef" != "": @"buildRef" else: nil, @@ -316,8 +308,9 @@ proc start*(cfg: StrawBossConfig): void = post "/service/debug/stop": if not cfg.debug: resp(Http404, makeJsonResp(Http404), JSON) else: - callSoon(proc(): void = complete(stopFuture)) - resp($(%*"shutting down"), JSON) + let shutdownFut = sleepAsync(100) + shutdownFut.callback = proc(): void = complete(stopFuture) + resp($(%"shutting down"), JSON) #[ get re".*": @@ -327,4 +320,5 @@ proc start*(cfg: StrawBossConfig): void = resp(Http404, makeJsonResp(Http404), JSON) ]# + callSoon(proc(): void = performMaintenance(cfg)) waitFor(stopFuture) diff --git a/src/test/json/test-status.json b/src/test/json/test-status.json index da2a735..5f862bf 100644 --- a/src/test/json/test-status.json +++ b/src/test/json/test-status.json @@ -1,4 +1,5 @@ { + "runId": "90843e0c-6113-4462-af33-a89ff9731031", "state": "failed", "details": "some very good reason" } diff --git a/src/test/nim/functional/tserver.nim b/src/test/nim/functional/tserver.nim index 5e3a458..e8a7c79 100644 --- a/src/test/nim/functional/tserver.nim +++ b/src/test/nim/functional/tserver.nim @@ -17,7 +17,7 @@ let TIMEOUT = 2.minutes # configuration and working files. template keepEnv(): untyped = preserveEnv = true - echo "artifacts dir: " & tempArtifactsDir + echo "artifacts dir: " & tempBuildDataDir echo "strawboss serve -c " & tempCfgPath suite "strawboss server": @@ -30,13 +30,13 @@ suite "strawboss server": # per-test setup: spin up a fresh strawboss instance setup: - let tempArtifactsDir = mkdtemp() + let tempBuildDataDir = mkdtemp() let (_, tempCfgPath) = mkstemp() var preserveEnv = false # copy our test config var newCfg = cfg - newCfg.artifactsRepo = tempArtifactsDir + newCfg.buildDataDir = tempBuildDataDir # update the repo string for the extracted test project var testProjDef = newCfg.findProject(testProjName) @@ -55,7 +55,7 @@ suite "strawboss server": discard newAsyncHttpClient().post(apiBase & "/service/debug/stop") if not preserveEnv: - removeDir(tempArtifactsDir) + removeDir(tempBuildDataDir) removeFile(tempCfgPath) # give the server time to spin down but kill it after that @@ -73,14 +73,14 @@ suite "strawboss server": check resp.status.startsWith("404") test "GET /api/project/@projectName/versions": - let projArtifactsDir = tempArtifactsDir & "/" & testProjName + let cachedConfsDir = tempBuildDataDir & "/" & testProjName & "/configurations" let expectedVersions = @["alpha", "beta", "1.0.0", "1.0.1"] # Touch configuration files - createDir(projArtifactsDir) + createDir(cachedConfsDir) for v in expectedVersions: var f: File - check open(f, projArtifactsDir & "/configuration." & v & ".json", fmWrite) + check open(f, cachedConfsDir & "/" & v & ".json", fmWrite) close(f) let http = newAuthenticatedHttpClient(apibase, "bob@builder.com", "password") @@ -96,12 +96,13 @@ suite "strawboss server": # give the filesystem time to create stuff sleep(100) + # check that the run request has been # check that the project directory has been created in the artifacts repo - let runArtifactsDir = tempArtifactsDir & "/" & testProjName & "/build/0.1.0" + let runArtifactsDir = tempBuildDataDir & "/" & testProjName & "/artifacts/build/0.1.0" check existsDir(runArtifactsDir) # check that the run status file has been created in the artifacts repo - let statusFile = runArtifactsDir & "/status.json" + let statusFile = tempBuildDataDir & "/" & testProjName & "/status/0.1.0.json" check fileExists(statusFile) # check that the run status is not failed diff --git a/src/test/nim/unit/tconfiguration.nim b/src/test/nim/unit/tconfiguration.nim index 34d5d01..6c901c0 100644 --- a/src/test/nim/unit/tconfiguration.nim +++ b/src/test/nim/unit/tconfiguration.nim @@ -21,6 +21,7 @@ suite "load and save configuration objects": test "parseRunRequest": let rr1 = RunRequest( + id: genUUID(), projectName: testProjDef.name, stepName: "build", buildRef: "master", @@ -85,7 +86,7 @@ suite "load and save configuration objects": envVars: newStringTable(modeCaseSensitive))] check: - cfg.artifactsRepo == "artifacts" + cfg.buildDataDir == "build-data" cfg.authSecret == "change me" cfg.pwdCost == 11 sameContents(expectedUsers, cfg.users) @@ -141,5 +142,6 @@ suite "load and save configuration objects": let st = loadBuildStatus("src/test/json/test-status.json") check: + st.runId == "90843e0c-6113-4462-af33-a89ff9731031" st.state == "failed" st.details == "some very good reason" diff --git a/strawboss.nimble b/strawboss.nimble index bcc2bde..1d3d52b 100644 --- a/strawboss.nimble +++ b/strawboss.nimble @@ -19,7 +19,9 @@ requires "https://git.jdb-labs.com/jdb/nim-cli-utils.git" # Tasks # task functest, "Runs the functional test suite.": + exec "nimble build" exec "nim c -r src/test/nim/run_functional_tests.nim" task unittest, "Runs the unit test suite.": + exec "nimble build" exec "nim c -r src/test/nim/run_unit_tests.nim"