From 3d8454d486d84c0a430a22bfb1a1df974144578a Mon Sep 17 00:00:00 2001 From: Jonathan Bernard Date: Thu, 11 May 2017 10:51:06 -0500 Subject: [PATCH] Reworking runs to include an id, save the run request. --- src/main/nim/strawboss.nim | 22 ++++++++++++------- src/main/nim/strawbosspkg/configuration.nim | 20 +++++++++++------ src/main/nim/strawbosspkg/core.nim | 8 +++++++ src/main/nim/strawbosspkg/server.nim | 24 +++++++++++++++++---- src/test/nim/unit/tconfiguration.nim | 15 ++++++++++++- strawboss.nimble | 6 +++++- 6 files changed, 75 insertions(+), 20 deletions(-) diff --git a/src/main/nim/strawboss.nim b/src/main/nim/strawboss.nim index 4e086a1..c0b7215 100644 --- a/src/main/nim/strawboss.nim +++ b/src/main/nim/strawboss.nim @@ -1,4 +1,4 @@ -import docopt, os, sequtils, tempfile +import docopt, os, sequtils, tempfile, uuids import strawbosspkg/private/util import strawbosspkg/configuration @@ -31,6 +31,9 @@ Options -r --reference Build the project at this commit reference. + -i --run-id Use the given UUID as the run ID. If not given, a + new UUID is generated for this run. + -w --workspace Use the given directory as the build workspace. """ @@ -51,14 +54,17 @@ Options if args["run"]: - let req = RunRequest( - projectName: $args[""], - stepName: $args[""], - buildRef: if args["--reference"]: $args["--reference"] else: nil, - forceRebuild: args["--force-rebuild"], - workspaceDir: if args["--workspace"]: $args[""] else: mkdtemp()) + let wkspDir = if args["--workspace"]: $args["--workspace"] else: mkdtemp() try: + let req = RunRequest( + id: if args["--run-id"]: parseUUID($args["--run-id"]) else: genUUID(), + projectName: $args[""], + stepName: $args[""], + buildRef: if args["--reference"]: $args["--reference"] else: nil, + forceRebuild: args["--force-rebuild"], + workspaceDir: wkspDir) + let status = core.runStep(cfg, req, logProcOutput) if status.state == "failed": raiseEx status.details echo "strawboss: build passed." @@ -66,7 +72,7 @@ Options echo "strawboss: build FAILED: " & getCurrentExceptionMsg() & "." quit(QuitFailure) finally: - if existsDir(req.workspaceDir): removeDir(req.workspaceDir) + if existsDir(wkspDir): removeDir(wkspDir) elif args["serve"]: server.start(cfg) diff --git a/src/main/nim/strawbosspkg/configuration.nim b/src/main/nim/strawbosspkg/configuration.nim index fbc604f..8ebdfa1 100644 --- a/src/main/nim/strawbosspkg/configuration.nim +++ b/src/main/nim/strawbosspkg/configuration.nim @@ -1,4 +1,4 @@ -import logging, json, os, nre, sequtils, strtabs, tables, times +import logging, json, os, nre, sequtils, strtabs, tables, times, uuids import private/util from langutils import sameContents @@ -25,6 +25,7 @@ type envVars*: StringTableRef RunRequest* = object + id*: UUID projectName*, stepName*, buildRef*, workspaceDir*: string forceRebuild*: bool @@ -67,6 +68,15 @@ proc `==`*(a, b: StrawBossConfig): bool = sameContents(a.users, b.users) and sameContents(a.projects, b.projects) +proc `==`*(a, b: RunRequest): bool = + result = + a.id == b.id and + a.projectName == b.projectName and + a.stepName == b.stepName and + a.buildRef == b.buildRef and + a.workspaceDir == b.workspaceDir and + a.forceRebuild == b.forceRebuild + # Util methods on custom types proc findProject*(cfg: StrawBossConfig, projectName: string): ProjectDef = let candidates = cfg.projects.filterIt(it.name == projectName) @@ -182,12 +192,9 @@ proc loadBuildStatus*(statusFile: string): BuildStatus = state: jsonObj.getOrFail("state", "build status").getStr, details: jsonObj.getIfExists("details").getStr("") ) - -# TODO: unused and untested, add tests if we start using this -proc parseRunRequest*(reqStr: string): RunRequest = - let reqJson = parseJson(reqStr) - +proc parseRunRequest*(reqJson: JsonNode): RunRequest = result = RunRequest( + id: parseUUID(reqJson.getOrFail("id", "RunRequest").getStr), projectName: reqJson.getOrFail("projectName", "RunRequest").getStr, stepName: reqJson.getOrFail("stepName", "RunRequest").getStr, buildRef: reqJson.getOrFail("buildRef", "RunRequest").getStr, @@ -213,6 +220,7 @@ proc `%`*(p: ProjectDef): JsonNode = proc `%`*(req: RunRequest): JsonNode = result = %* { + "id": $req.id, "projectName": req.projectName, "stepName": req.stepName, "buildRef": req.buildRef, diff --git a/src/main/nim/strawbosspkg/core.nim b/src/main/nim/strawbosspkg/core.nim index b3fe890..1aca01e 100644 --- a/src/main/nim/strawbosspkg/core.nim +++ b/src/main/nim/strawbosspkg/core.nim @@ -168,6 +168,14 @@ proc runStep*(cfg: StrawBossConfig, req: RunRequest, # Find the project definition let projectDef = cfg.findProject(req.projectName) + # Make sure our directory in the artifacts repo exists + if not existsDir(cfg.artifactsRepo & "/" & projectDef.name & "/run-requests"): + createDir(cfg.artifactsRepo & "/" & projectDef.name & "/run-requests") + + # Save the run request + writeFile(cfg.artifactsRepo & "/" & projectDef.name & + "/run-requests/" & $req.id & ".json", $req) + # Read in the existing system environment var env = loadEnv() env["GIT_DIR"] = ".git" diff --git a/src/main/nim/strawbosspkg/server.nim b/src/main/nim/strawbosspkg/server.nim index bc8b53a..a71b2bb 100644 --- a/src/main/nim/strawbosspkg/server.nim +++ b/src/main/nim/strawbosspkg/server.nim @@ -1,9 +1,10 @@ import algorithm, asyncdispatch, bcrypt, jester, json, jwt, logging, os, osproc, - sequtils, strutils, tempfile, times, unittest + sequtils, strutils, tempfile, times, unittest, uuids import ./configuration, ./core, private/util type Worker = object + runId*: UUID process*: Process workingDir*: string @@ -76,10 +77,11 @@ proc spawnWorker(cfg: StrawBossConfig, req: RunRequest): Worker = let dir = mkdtemp() var args = @["run", req.projectName, req.stepName, "-r", req.buildRef, - "-w", dir, "-c", cfg.filePath] + "-w", dir, "-c", cfg.filePath, "-i", req.id] if req.forceRebuild: args.add("-f") debug "Launching worker: " & cfg.pathToExe & " " & args.join(" ") result = Worker( + runId: req.id, process: startProcess(cfg.pathToExe, ".", args, loadEnv(), {poUsePath}), workingDir: dir) @@ -238,7 +240,20 @@ proc start*(cfg: StrawBossConfig): void = ## List all runs checkAuth(); if not authed: return true - + + # Make sure we know about that project + var project: ProjectDef + try: project = cfg.findProject(@"projectName") + except: resp(Http404, makeJsonResp(Http404, getCurrentExceptionMsg()), JSON) + + let runsDir = cfg.artifactsRepo & "/" & project.name & "/runs" + if not existsDir(runsDir): resp("[]", JSON) + + let runPaths = toSeq(walkFiles(runsDir & "/*.json")) + let runRequests = runPaths.mapIt(parseRunRequest(parseFile(it))) + + resp($(%runRequests), JSON) + # TODO resp(Http501, makeJsonResp(Http501), JSON) @@ -279,7 +294,8 @@ proc start*(cfg: StrawBossConfig): void = checkAuth(); if not authed: return true - workers.add(spawnWorker(RunRequest( + let runRequest = RunRequest( + id: genUUID(), projectName: @"projectName", stepName: @"stepName", buildRef: if @"buildRef" != "": @"buildRef" else: nil, diff --git a/src/test/nim/unit/tconfiguration.nim b/src/test/nim/unit/tconfiguration.nim index 148317f..c0c42d8 100644 --- a/src/test/nim/unit/tconfiguration.nim +++ b/src/test/nim/unit/tconfiguration.nim @@ -1,4 +1,5 @@ -import json, strtabs, tables, unittest +import json, strtabs, tables, unittest, uuids + from langutils import sameContents import ../../../main/nim/strawbosspkg/configuration @@ -18,6 +19,18 @@ suite "load and save configuration objects": defaultBranch: "deploy", envVars: newStringTable("VAR1", "value", modeCaseInsensitive)) + test "parseRunRequest": + let rr1 = RunRequest( + id: genUUID(), + projectName: testProjDef.name, + stepName: "build", + buildRef: "master", + workspaceDir: "/no-real/dir", + forceRebuild: true) + + let rrStr = $rr1 + let rr2 = parseRunRequest(parseJson(rrStr)) + check rr1 == rr2 test "parseProjectDef": let pd = parseProjectDef(parseJson(testProjDefStr)) diff --git a/strawboss.nimble b/strawboss.nimble index 8be08dd..bb27586 100644 --- a/strawboss.nimble +++ b/strawboss.nimble @@ -9,10 +9,14 @@ srcDir = "src/main/nim" # Dependencies -requires @["nim >= 0.16.1", "docopt >= 0.1.0", "tempfile", "jester", "bcrypt", "untar"] +requires @["nim >= 0.16.1", "docopt >= 0.1.0", "tempfile", "jester", "bcrypt", + "untar", "uuids"] + requires "https://github.com/yglukhov/nim-jwt" requires "https://git.jdb-labs.com/jdb/nim-lang-utils.git" +# Tasks +# task functest, "Runs the functional test suite.": exec "nim c -r src/test/nim/run_functional_tests.nim"