b2d4df0aac
* Addressing breaking changes in migration from Nim 0.18 to 0.19. * Finishing the initial pass at the refactor required to include docker-based builds. * Regaining confidence in the existing functionality by getting all tests passing again after docker introduction (still need new tests to cover new docker functionality).
469 lines
15 KiB
Nim
469 lines
15 KiB
Nim
import asyncdispatch, bcrypt, cliutils, jester, json, jwt, logging, md5,
|
|
options, os, osproc, sequtils, strutils, tempfile, times, unittest, uuids
|
|
|
|
from mimetypes import getMimeType
|
|
from asyncfile import openAsync, readToStream, close
|
|
from asyncnet import send
|
|
from re import re, find
|
|
from timeutils import trimNanoSec
|
|
|
|
import ./configuration, ./core
|
|
|
|
type
|
|
Session = object
|
|
user*: UserRef
|
|
issuedAt*, expires*: Time
|
|
|
|
#const ISO_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"
|
|
const JSON = "application/json"
|
|
|
|
proc newSession*(user: UserRef): Session =
|
|
result = Session(
|
|
user: user,
|
|
issuedAt: getTime().local.trimNanoSec.toTime,
|
|
expires: daysForward(7).trimNanoSec.toTime)
|
|
|
|
template halt(code: HttpCode,
|
|
headers: RawHeaders,
|
|
content: string): typed =
|
|
## Immediately replies with the specified request. This means any further
|
|
## code will not be executed after calling this template in the current
|
|
## route.
|
|
bind TCActionSend, newHttpHeaders
|
|
result[0] = CallbackAction.TCActionSend
|
|
result[1] = code
|
|
result[2] = some(headers)
|
|
result[3] = content
|
|
result.matched = true
|
|
break allRoutes
|
|
|
|
template jsonResp(code: HttpCode, details: string = "", headers: RawHeaders = @{:} ) =
|
|
halt(
|
|
code,
|
|
headers & @{"Content-Type": JSON},
|
|
$(%* {
|
|
"statusCode": code.int,
|
|
"status": $code,
|
|
"details": details
|
|
})
|
|
)
|
|
|
|
template json500Resp(ex: ref Exception, details: string = ""): void =
|
|
when not defined(release): debug ex.getStackTrace()
|
|
error details & ":\n" & ex.msg
|
|
jsonResp(Http500)
|
|
|
|
proc toJWT*(cfg: StrawBossConfig, session: Session): string =
|
|
## Make a JST token for this session.
|
|
var jwt = JWT(
|
|
header: JOSEHeader(alg: HS256, typ: "jwt"),
|
|
claims: toClaims(%*{
|
|
"sub": session.user.name,
|
|
"iat": session.issuedAt.toUnix.int,
|
|
"exp": session.expires.toUnix.int }))
|
|
|
|
jwt.sign(cfg.authSecret)
|
|
result = $jwt
|
|
|
|
proc fromJWT*(cfg: StrawBossConfig, strTok: string): Session =
|
|
## Validate a given JWT and extract the session data.
|
|
let jwt = toJWT(strTok)
|
|
var secret = cfg.authSecret
|
|
if not jwt.verify(secret): raiseEx "Unable to verify auth token."
|
|
jwt.verifyTimeClaims()
|
|
|
|
# Find the user record (if authenticated)
|
|
let username = jwt.claims["sub"].node.str
|
|
let users = cfg.users.filterIt(it.name == username)
|
|
if users.len != 1: raiseEx "Could not find session user."
|
|
|
|
result = Session(
|
|
user: users[0],
|
|
issuedAt: fromUnix(jwt.claims["iat"].node.num),
|
|
expires: fromUnix(jwt.claims["exp"].node.num))
|
|
|
|
proc extractSession(cfg: StrawBossConfig, request: Request): Session =
|
|
## Helper to extract a session from a reqest.
|
|
|
|
# Find the auth header
|
|
if not request.headers.hasKey("Authorization"):
|
|
raiseEx "No auth token."
|
|
|
|
# Read and verify the JWT token
|
|
let headerVal = request.headers["Authorization"]
|
|
if not headerVal.startsWith("Bearer "):
|
|
raiseEx "Invalid Authentication type (only 'Bearer' is supported)."
|
|
|
|
result = fromJWT(cfg, headerVal[7..^1])
|
|
|
|
proc hashPwd*(pwd: string, cost: int8): string =
|
|
let salt = genSalt(cost)
|
|
result = hash(pwd, salt)
|
|
|
|
proc validatePwd*(u: UserRef, givenPwd: string): bool =
|
|
let salt = u.hashedPwd[0..28] # TODO: magic numbers
|
|
result = compare(u.hashedPwd, hash(givenPwd, salt))
|
|
|
|
proc makeAuthToken*(cfg: StrawBossConfig, uname, pwd: string): string =
|
|
## Given a username and pwd, validate the combination and generate a JWT
|
|
## token string.
|
|
|
|
if uname.len == 0 or pwd.len == 0:
|
|
raiseEx "fields 'username' and 'password' required"
|
|
|
|
# find the user record
|
|
let users = cfg.users.filterIt(it.name == uname)
|
|
if users.len != 1: raiseEx "invalid username or password"
|
|
|
|
let user = users[0]
|
|
|
|
if not validatePwd(user, pwd): raiseEx "invalid username or password"
|
|
|
|
let session = newSession(user)
|
|
|
|
result = toJWT(cfg, session)
|
|
|
|
proc makeApiKey*(cfg: StrawBossConfig, uname: string): string =
|
|
## Given a username, make an API token (JWT token string that does not
|
|
## expire). Note that this does not validate the username/pwd combination. It
|
|
## is not intended to be exposed publicly via the API, but serve as a utility
|
|
## function for an administrator to setup a unsupervised account (git access
|
|
## for example).
|
|
|
|
if uname.len == 0: raiseEx "no username given"
|
|
|
|
# find the user record
|
|
let users = cfg.users.filterIt(it.name == uname)
|
|
if users.len != 1: raiseEx "invalid username"
|
|
|
|
let session = Session(
|
|
user: users[0],
|
|
issuedAt: getTime(),
|
|
expires: daysForward(365 * 1000).toTime())
|
|
|
|
result = toJWT(cfg, session);
|
|
|
|
template checkAuth() =
|
|
## Check this request for authentication and authorization information.
|
|
## Injects the session into the running context. If the request is not
|
|
## authorized, this template returns an appropriate 401 response.
|
|
|
|
var session {.inject.}: Session
|
|
|
|
try: session = extractSession(cfg, request)
|
|
except:
|
|
debug "Auth failed: " & getCurrentExceptionMsg()
|
|
jsonResp(Http401, "Unauthorized", @{"WWW-Authenticate": "Bearer"})
|
|
|
|
proc start*(cfg: StrawBossConfig): void =
|
|
|
|
var stopFuture = newFuture[void]()
|
|
var workers: seq[Worker] = @[]
|
|
|
|
settings:
|
|
port = Port(8180)
|
|
appName = "/api"
|
|
|
|
routes:
|
|
|
|
get "/ping":
|
|
resp($(%"pong"), JSON)
|
|
|
|
post "/auth-token":
|
|
var uname, pwd: string
|
|
try:
|
|
let jsonBody = parseJson(request.body)
|
|
uname = jsonBody["username"].getStr
|
|
pwd = jsonBody["password"].getStr
|
|
except: jsonResp(Http400)
|
|
|
|
try:
|
|
let authToken = makeAuthToken(cfg, uname, pwd)
|
|
resp($(%authToken), JSON)
|
|
except: jsonResp(Http401, getCurrentExceptionMsg())
|
|
|
|
get "/verify-auth":
|
|
checkAuth()
|
|
|
|
resp(Http200, $(%*{ "username": session.user.name }), JSON)
|
|
|
|
get "/projects":
|
|
## List project summaries (ProjectDefs only)
|
|
|
|
checkAuth()
|
|
|
|
resp($(%cfg.projects), JSON)
|
|
|
|
post "/projects":
|
|
## Create a new project definition
|
|
|
|
checkAuth()
|
|
|
|
# TODO
|
|
jsonResp(Http501)
|
|
|
|
get "/project/@projectName":
|
|
## Return a project's configuration, as well as it's versions.
|
|
|
|
checkAuth()
|
|
|
|
# Make sure we know about that project
|
|
var projDef: ProjectDef
|
|
try: projDef = cfg.getProject(@"projectName")
|
|
except:
|
|
try: raise getCurrentException()
|
|
except NotFoundException:
|
|
jsonResp(Http404, getCurrentExceptionMsg())
|
|
except:
|
|
let msg = "unable to load project definition for project " & @"projectName"
|
|
json500Resp(getCurrentException(), msg)
|
|
|
|
var projConf: ProjectConfig
|
|
try: projConf = getProjectConfig(cfg, @"projectName", "")
|
|
except: discard ""
|
|
|
|
let respJson = newJObject()
|
|
respJson["definition"] = %projDef
|
|
respJson["versions"] = %listVersions(cfg, @"projectName")
|
|
if projConf.name.len > 0:
|
|
respJson["latestConfig"] = %projConf
|
|
|
|
resp(pretty(respJson), JSON)
|
|
|
|
get "/project/@projectName/versions":
|
|
## Get a list of all versions that we have built
|
|
|
|
checkAuth()
|
|
|
|
try: resp($(%listVersions(cfg, @"projectName")), JSON)
|
|
except:
|
|
try: raise getCurrentException()
|
|
except NotFoundException:
|
|
jsonResp(Http404, getCurrentExceptionMsg())
|
|
except:
|
|
let msg = "unable to list versions for project " & @"projectName"
|
|
json500Resp(getCurrentException(), msg)
|
|
|
|
get "/project/@projectName/version/@version?":
|
|
## Get a detailed project record including step definitions (ProjectConfig).
|
|
|
|
checkAuth()
|
|
|
|
# Make sure we know about that project
|
|
try: resp($(%getProjectConfig(cfg, @"projectName", @"version")), JSON)
|
|
except: jsonResp(Http404, getCurrentExceptionMsg())
|
|
|
|
get "/project/@projectName/runs":
|
|
## List all runs
|
|
|
|
checkAuth()
|
|
|
|
try: resp($(%listRuns(cfg, @"projectName")), JSON)
|
|
except: jsonResp(Http404, getCurrentExceptionMsg())
|
|
|
|
get "/project/@projectName/runs/active":
|
|
## List all currently active runs
|
|
|
|
checkAuth()
|
|
|
|
try:
|
|
let activeRuns = workers
|
|
.filterIt(it.process.running and it.projectName == @"projectName")
|
|
.mapIt(cfg.getRun(@"projecName", $it.runId));
|
|
resp($(%activeRuns), JSON)
|
|
except:
|
|
try: raise getCurrentException()
|
|
except NotFoundException:
|
|
jsonResp(Http404, getCurrentExceptionMsg())
|
|
except:
|
|
json500Resp(getCurrentException(), "problem loading active runs")
|
|
|
|
get "/project/@projectName/run/@runId":
|
|
## Details for a specific run
|
|
|
|
checkAuth()
|
|
|
|
# Make sure we know about that project
|
|
try: discard cfg.getProject(@"projectName")
|
|
except: jsonResp(Http404, getCurrentExceptionMsg())
|
|
|
|
if not existsRun(cfg, @"projectName", @"runId"):
|
|
jsonResp(Http404, "no such run for project")
|
|
|
|
try: resp($getRun(cfg, @"projectName", @"runId"), JSON)
|
|
except:
|
|
json500Resp(getCurrentException(),
|
|
"unable to load run details for project " & @"projectName" &
|
|
" run " & @"runId")
|
|
|
|
get "/project/@projectName/run/@runId/logs":
|
|
## Get logs from a specific run
|
|
|
|
checkAuth()
|
|
|
|
try: discard cfg.getProject(@"projectName")
|
|
except:
|
|
jsonResp(Http404, getCurrentExceptionMsg())
|
|
|
|
if not existsRun(cfg, @"projectName", @"runId"):
|
|
jsonResp(Http404, "no such run for project")
|
|
|
|
try: resp($getLogs(cfg, @"projectName", @"runId"))
|
|
except:
|
|
json500Resp(getCurrentException(),
|
|
"unable to load run logs for " & @"projectName" & " run " & @"runId")
|
|
|
|
get "/project/@projectName/step/@stepName/artifacts/@version":
|
|
## Get the list of artifacts that were built for
|
|
|
|
checkAuth()
|
|
|
|
debug "Matched artifacts list request: " & $(%*{
|
|
"project": @"projectName",
|
|
"step": @"stepName",
|
|
"version": @"version"
|
|
})
|
|
|
|
try: resp($(%listArtifacts(cfg, @"projectName", @"stepName", @"version")), JSON)
|
|
except:
|
|
try: raise getCurrentException()
|
|
except NotFoundException:
|
|
jsonResp(Http404, getCurrentExceptionMsg())
|
|
except:
|
|
json500Resp(getCurrentException(), "unable to list artifacts for " &
|
|
@"projectName" & ":" & @"stepName" & "@" & @"buildRef")
|
|
|
|
get "/project/@projectName/step/@stepName/artifact/@version/@artifactName":
|
|
## Get a specific artifact that was built.
|
|
|
|
checkAuth()
|
|
|
|
var artifactPath: string
|
|
try: artifactPath = getArtifactPath(cfg,
|
|
@"projectName", @"stepName", @"version", @"artifactName")
|
|
except:
|
|
try: raise getCurrentException()
|
|
except NotFoundException:
|
|
jsonResp(Http404, getCurrentExceptionMsg())
|
|
except:
|
|
json500Resp(getCurrentException(), "unable to check artifact path for " &
|
|
@"projectName" & ":" & @"stepName" & "@" & @"version")
|
|
|
|
enableRawMode
|
|
|
|
debug "Preparing: " & artifactPath
|
|
let fileSize = getFileSize(artifactPath)
|
|
let mimetype = request.settings.mimes.getMimetype(artifactPath.splitFile.ext[1 .. ^1])
|
|
if fileSize < 10_000_000: # 10 mb
|
|
var file = readFile(artifactPath)
|
|
|
|
var hashed = getMD5(file)
|
|
|
|
# If the user has a cached version of this file and it matches our
|
|
# version, let them use it
|
|
if request.headers.hasKey("If-None-Match") and request.headers["If-None-Match"] == hashed:
|
|
resp(Http304)
|
|
else:
|
|
resp(Http200, [
|
|
("Content-Disposition", "; filename=\"" & @"artifactName" & "\""),
|
|
("Content-Type", mimetype),
|
|
("ETag", hashed )], file)
|
|
else:
|
|
let headers = @{
|
|
"Content-Disposition": "; filename=\"" & @"artifactName" & "\"",
|
|
"Content-Type": mimetype,
|
|
"Content-Length": $fileSize
|
|
}
|
|
request.sendHeaders(Http200, headers)
|
|
|
|
var fileStream = newFutureStream[string]("sendStaticIfExists")
|
|
var file = openAsync(artifactPath, fmRead)
|
|
# Let `readToStream` write file data into fileStream in the
|
|
# background.
|
|
asyncCheck file.readToStream(fileStream)
|
|
# The `writeFromStream` proc will complete once all the data in the
|
|
# `bodyStream` has been written to the file.
|
|
while true:
|
|
let (hasValue, value) = await fileStream.read()
|
|
if hasValue: request.send(value)
|
|
else: break
|
|
file.close()
|
|
|
|
get "/project/@projectName/step/@stepName/status/@buildRef":
|
|
## Get detailed information about the status of a step (assuming it has been built)
|
|
|
|
checkAuth()
|
|
|
|
try: resp($cfg.getBuildStatus(@"projectName", @"stepName", @"buildRef"), JSON)
|
|
except:
|
|
try: raise getCurrentException()
|
|
except NotFoundException: jsonResp(Http404, getCurrentExceptionMsg())
|
|
except:
|
|
json500Resp(getCurrentException(), "unable to load the build state for " &
|
|
@"projectName" & ":" & @"stepName" & "@" & @"buildRef")
|
|
|
|
#get "/project/@projectName/step/@stepName/status/@buildRef.svg":
|
|
## Get an image representing the status of a build
|
|
|
|
## TODO: how do we want to handle auth for this? Unlike
|
|
#checkAuth(): if not authed: return true
|
|
|
|
post "/project/@projectName/step/@stepName/run/@buildRef?":
|
|
# Kick off a run
|
|
|
|
checkAuth()
|
|
|
|
let runRequest = RunRequest(
|
|
runId: genUUID(),
|
|
projectName: @"projectName",
|
|
stepName: @"stepName",
|
|
buildRef: if @"buildRef" != "": @"buildRef" else: "",
|
|
timestamp: getTime().local,
|
|
forceRebuild: false) # TODO support this with optional query params
|
|
|
|
# TODO: instead of immediately spawning a worker, add the request to a
|
|
# queue to be picked up by a worker. Allows capping the number of worker
|
|
# prcesses, distributing, etc.
|
|
try:
|
|
let (status, worker) = spawnWorker(cfg, runRequest)
|
|
workers.add(worker)
|
|
|
|
resp($Run(
|
|
id: runRequest.runId,
|
|
request: runRequest,
|
|
status: status), JSON)
|
|
except:
|
|
try: raise getCurrentException()
|
|
except NotFoundException: jsonResp(Http404, getCurrentExceptionMsg())
|
|
except: jsonResp(Http400, getCurrentExceptionMsg())
|
|
|
|
post "/service/debug/stop":
|
|
if not cfg.debug: jsonResp(Http404)
|
|
else:
|
|
let shutdownFut = sleepAsync(100)
|
|
shutdownFut.callback = proc(): void = complete(stopFuture)
|
|
resp($(%"shutting down"), JSON)
|
|
|
|
|
|
get re".*":
|
|
jsonResp(Http404)
|
|
|
|
post re".*":
|
|
jsonResp(Http404)
|
|
|
|
proc performMaintenance(cfg: StrawBossConfig): void =
|
|
# Prune workers
|
|
workers = workers.filterIt(it.process.running())
|
|
debug "Performing maintanance: " & $len(workers) & " active workers after pruning."
|
|
|
|
let fut = sleepAsync(cfg.maintenancePeriod)
|
|
fut.callback =
|
|
proc(): void =
|
|
callSoon(proc(): void = performMaintenance(cfg))
|
|
|
|
|
|
info "StrawBoss is bossing people around."
|
|
|
|
callSoon(proc(): void = performMaintenance(cfg))
|
|
waitFor(stopFuture)
|