import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, md5, os, osproc, sequtils, strutils, tempfile, times, unittest, uuids from mimetypes import getMimeType from asyncfile import openAsync, readToStream, close from asyncnet import send from re import re, find import ./configuration, ./core type Session = object user*: UserRef issuedAt*, expires*: Time #const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss" const JSON = "application/json" proc newSession*(user: UserRef): Session = result = Session( user: user, issuedAt: getTime(), expires: daysForward(7).toTime()) proc buildJson(resp: Response, code: HttpCode, details: string = ""): void = resp.data[0] = CallbackAction.TCActionSend resp.data[1] = code resp.data[2]["Content-Type"] = JSON resp.data[3] = $(%* { "statusCode": code.int, "status": $code, "details": details }) # Work-around for weirdness trying to use resp(Http500... in exception blocks proc build500Json(resp: Response, ex: ref Exception, msg: string): void = when not defined(release): debug ex.getStackTrace() error msg & ":\n" & ex.msg resp.buildJson(Http500) proc toJWT*(cfg: StrawBossConfig, session: Session): string = ## Make a JST token for this session. var jwt = JWT( header: JOSEHeader(alg: HS256, typ: "jwt"), claims: toClaims(%*{ "sub": session.user.name, "iat": session.issuedAt.toSeconds().int, "exp": session.expires.toSeconds().int })) jwt.sign(cfg.authSecret) result = $jwt proc fromJWT*(cfg: StrawBossConfig, strTok: string): Session = ## Validate a given JWT and extract the session data. let jwt = toJWT(strTok) var secret = cfg.authSecret if not jwt.verify(secret): raiseEx "Unable to verify auth token." jwt.verifyTimeClaims() # Find the user record (if authenticated) let username = jwt.claims["sub"].node.str let users = cfg.users.filterIt(it.name == username) if users.len != 1: raiseEx "Could not find session user." result = Session( user: users[0], issuedAt: fromSeconds(jwt.claims["iat"].node.num), expires: fromSeconds(jwt.claims["exp"].node.num)) proc extractSession(cfg: StrawBossConfig, request: Request): Session = ## Helper to extract a session from a reqest. # Find the auth header if not request.headers.hasKey("Authorization"): raiseEx "No auth token." # Read and verify the JWT token let headerVal = request.headers["Authorization"] if not headerVal.startsWith("Bearer "): raiseEx "Invalid Authentication type (only 'Bearer' is supported)." result = fromJWT(cfg, headerVal[7..^1]) proc hashPwd*(pwd: string, cost: int8): string = let salt = genSalt(cost) result = hash(pwd, salt) proc validatePwd*(u: UserRef, givenPwd: string): bool = let salt = u.hashedPwd[0..28] # TODO: magic numbers result = compare(u.hashedPwd, hash(givenPwd, salt)) proc makeAuthToken*(cfg: StrawBossConfig, uname, pwd: string): string = ## Given a username and pwd, validate the combination and generate a JWT ## token string. if uname == nil or pwd == nil: raiseEx "fields 'username' and 'password' required" # find the user record let users = cfg.users.filterIt(it.name == uname) if users.len != 1: raiseEx "invalid username or password" let user = users[0] if not validatePwd(user, pwd): raiseEx "invalid username or password" let session = newSession(user) result = toJWT(cfg, session) proc makeApiKey*(cfg: StrawBossConfig, uname: string): string = ## Given a username, make an API token (JWT token string that does not ## expire). Note that this does not validate the username/pwd combination. It ## is not intended to be exposed publicly via the API, but serve as a utility ## function for an administrator to setup a unsupervised account (git access ## for example). if uname == nil: raiseEx "no username given" # find the user record let users = cfg.users.filterIt(it.name == uname) if users.len != 1: raiseEx "invalid username" let session = Session( user: users[0], issuedAt: getTime(), expires: daysForward(365 * 1000).toTime()) result = toJWT(cfg, session); template checkAuth() = ## Check this request for authentication and authorization information. ## Injects two variables into the running context: the session and authed: ## true if the request is authorized, false otherwise. If the request is not ## authorized, this template sets up the 401 response correctly. The calling ## context needs only to return from the route. var session {.inject.}: Session var authed {.inject.} = false try: session = extractSession(cfg, request) authed = true except: debug "Auth failed: " & getCurrentExceptionMsg() response.data[2]["WWW-Authenticate"] = "Bearer" response.buildJson(Http401) proc start*(cfg: StrawBossConfig): void = var stopFuture = newFuture[void]() var workers: seq[Worker] = @[] settings: port = Port(8180) appName = "/api" routes: get "/ping": resp($(%"pong"), JSON) post "/auth-token": var uname, pwd: string try: let jsonBody = parseJson(request.body) uname = jsonBody["username"].getStr pwd = jsonBody["password"].getStr except: response.buildJson(Http400); return true try: let authToken = makeAuthToken(cfg, uname, pwd) resp($(%authToken), JSON) except: response.buildJson(Http401, getCurrentExceptionMsg()); return true get "/verify-auth": checkAuth(); if not authed: return true resp(Http200, $(%*{ "username": session.user.name }), JSON) get "/projects": ## List project summaries (ProjectDefs only) checkAuth(); if not authed: return true resp($(%cfg.projects), JSON) post "/projects": ## Create a new project definition checkAuth(); if not authed: return true # TODO response.buildJson(Http501); return true get "/project/@projectName": ## Return a project's configuration, as well as it's versions. checkAuth(); if not authed: return true # Make sure we know about that project var projDef: ProjectDef try: projDef = cfg.getProject(@"projectName") except: try: raise getCurrentException() except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg()) except: response.build500Json(getCurrentException(), "unable to load project definition for project " & @"projectName") return true var projConf: ProjectConfig try: projConf = getProjectConfig(cfg, @"projectName", "") except: discard "" let respJson = newJObject() respJson["definition"] = %projDef respJson["versions"] = %listVersions(cfg, @"projectName") if not projConf.name.isNil: respJson["latestConfig"] = %projConf resp(pretty(respJson), JSON) get "/project/@projectName/versions": ## Get a list of all versions that we have built checkAuth(); if not authed: return true try: resp($(%listVersions(cfg, @"projectName")), JSON) except: try: raise getCurrentException() except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg()) except: response.build500Json(getCurrentException(), "unable to list versions for project " & @"projectName") return true get "/project/@projectName/version/@version?": ## Get a detailed project record including step definitions (ProjectConfig). checkAuth(); if not authed: return true # Make sure we know about that project try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON) except: response.buildJson(Http404, getCurrentExceptionMsg()); return true get "/project/@projectName/runs": ## List all runs checkAuth(); if not authed: return true try: resp($(%listRuns(cfg, @"projectName")), JSON) except: response.buildJson(Http404, getCurrentExceptionMsg()); return true get "/project/@projectName/runs/active": ## List all currently active runs checkAuth(); if not authed: return true try: let activeRuns = workers .filterIt(it.process.running and it.projectName == @"projectName") .mapIt(cfg.getRun(@"projecName", $it.runId)); resp($(%activeRuns), JSON) except: try: raise getCurrentException() except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg()) except: response.build500Json(getCurrentException(), "problem loading active runs") return true get "/project/@projectName/run/@runId": ## Details for a specific run checkAuth(); if not authed: return true # Make sure we know about that project try: discard cfg.getProject(@"projectName") except: response.buildJson(Http404, getCurrentExceptionMsg()); return true if not existsRun(cfg, @"projectName", @"runId"): response.buildJson(Http404, "no such run for project"); return true try: resp($getRun(cfg, @"projectName", @"runId"), JSON) except: response.build500Json(getCurrentException(), "unable to load run details for project " & @"projectName" & " run " & @"runId") return true get "/project/@projectName/run/@runId/logs": ## Get logs from a specific run checkAuth(); if not authed: return true try: discard cfg.getProject(@"projectName") except: response.buildJson(Http404, getCurrentExceptionMsg()) return true if not existsRun(cfg, @"projectName", @"runId"): response.buildJson(Http404, "no such run for project") return true try: resp($getLogs(cfg, @"projectName", @"runId")) except: response.build500Json(getCurrentException(), "unable to load run logs for " & @"projectName" & " run " & @"runId") return true get "/project/@projectName/step/@stepName/artifacts/@version": ## Get the list of artifacts that were built for checkAuth(); if not authed: return true debug "Matched artifacts list request: " & $(%*{ "project": @"projectName", "step": @"stepName", "version": @"version" }) try: resp($(%listArtifacts(cfg, @"projectName", @"stepName", @"version")), JSON) except: try: raise getCurrentException() except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg()) except: response.build500Json(getCurrentException(), "unable to list artifacts for " & @"projectName" & ":" & @"stepName" & "@" & @"buildRef") return true get "/project/@projectName/step/@stepName/artifact/@version/@artifactName": ## Get a specific artifact that was built. checkAuth(); if not authed: return true var artifactPath: string try: artifactPath = getArtifactPath(cfg, @"projectName", @"stepName", @"version", @"artifactName") except: try: raise getCurrentException() except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg()) except: response.build500Json(getCurrentException(), "unable to check artifact path for " & @"projectName" & ":" & @"stepName" & "@" & @"version") return true debug "Preparing: " & artifactPath let fileSize = getFileSize(artifactPath) let mimetype = request.settings.mimes.getMimetype(artifactPath.splitFile.ext[1 .. ^1]) if fileSize < 10_000_000: # 10 mb var file = readFile(artifactPath) var hashed = getMD5(file) # If the user has a cached version of this file and it matches our # version, let them use it if request.headers.hasKey("If-None-Match") and request.headers["If-None-Match"] == hashed: resp(Http304, [], "") else: resp(Http200, [ ("Content-Disposition", "; filename=\"" & @"artifactName" & "\""), ("Content-Type", mimetype), ("ETag", hashed )], file) else: let headers = { "Content-Disposition": "; filename=\"" & @"artifactName" & "\"", "Content-Type": mimetype, "Content-Length": $fileSize }.newStringTable await response.sendHeaders(Http200, headers) var fileStream = newFutureStream[string]("sendStaticIfExists") var file = openAsync(artifactPath, fmRead) # Let `readToStream` write file data into fileStream in the # background. asyncCheck file.readToStream(fileStream) # The `writeFromStream` proc will complete once all the data in the # `bodyStream` has been written to the file. while true: let (hasValue, value) = await fileStream.read() if hasValue: await response.client.send(value) else: break file.close() get "/project/@projectName/step/@stepName/status/@buildRef": ## Get detailed information about the status of a step (assuming it has been built) checkAuth(); if not authed: return true try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON) except: try: raise getCurrentException() except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg()) except: response.build500Json(getCurrentException(), "unable to load the build state for " & @"projectName" & ":" & @"stepName" & "@" & @"buildRef") return true #get "/project/@projectName/step/@stepName/status/@buildRef.svg": ## Get an image representing the status of a build ## TODO: how do we want to handle auth for this? Unlike #checkAuth(): if not authed: return true post "/project/@projectName/step/@stepName/run/@buildRef?": # Kick off a run checkAuth(); if not authed: return true let runRequest = RunRequest( runId: genUUID(), projectName: @"projectName", stepName: @"stepName", buildRef: if @"buildRef" != "": @"buildRef" else: nil, timestamp: getLocalTime(getTime()), forceRebuild: false) # TODO support this with optional query params # TODO: instead of immediately spawning a worker, add the request to a # queue to be picked up by a worker. Allows capping the number of worker # prcesses, distributing, etc. try: let (status, worker) = spawnWorker(cfg, runRequest) workers.add(worker) resp($Run( id: runRequest.runId, request: runRequest, status: status), JSON) except: try: raise getCurrentException() except NotFoundException: response.buildJson(Http404, getCurrentExceptionMsg()) except: response.buildJson(Http400, getCurrentExceptionMsg()) return true post "/service/debug/stop": if not cfg.debug: response.buildJson(Http404); return true else: let shutdownFut = sleepAsync(100) shutdownFut.callback = proc(): void = complete(stopFuture) resp($(%"shutting down"), JSON) get re".*": response.buildJson(Http404); return true post re".*": response.buildJson(Http404); return true proc performMaintenance(cfg: StrawBossConfig): void = # Prune workers workers = workers.filterIt(it.process.running()) debug "Performing maintanance: " & $len(workers) & " active workers after pruning." let fut = sleepAsync(cfg.maintenancePeriod) fut.callback = proc(): void = callSoon(proc(): void = performMaintenance(cfg)) info "StrawBoss is bossing people around." callSoon(proc(): void = performMaintenance(cfg)) waitFor(stopFuture)