Initial version.

.gitignore

@@ -0,0 +1,3 @@
+*.sw?
+nimcache/
+/strawboss

example.json

@@ -0,0 +1,22 @@
+{
+  "name": "new-life-intro-band",
+  "steps": {
+    "build": {
+      "workingDir": "website",
+      "expectedEnv": ["VERSION"],
+      "artifacts": ["nlib-web-$VERSION.zip"],
+      "cmdInput": [
+        "nim c -r make_site",
+        "(cd rendered && zip -r ../nlib-web-$VERSION.zip *)"
+      ]
+    },
+    "deploy": {
+      "dontCache": "true",
+      "depends": ["build"],
+      "expectedEnv": ["VERSION"],
+      "cmdInput": [
+        "netlify deploy -s newlifeintroband -p '$build_DIR/nlib-web-$VERSION.zip'"
+      ]
+    }
+  }
+}

strawboss.config.json

@@ -0,0 +1,6 @@
+{
+  "artifactsRepo": "artifacts",
+  "projects": [
+    { "name": "new-life-intro-band",
+      "repo": "/home/jdb/projects/new-life-introductory-band" } ]
+}

strawboss.nim

@@ -0,0 +1,320 @@
+import docopt, json, logging, nre, os, osproc, sequtils, streams, strtabs, strutils, tables, tempfile
+
+let SB_VER = "0.1.0"
+
+# Types
+#
+type
+  Step* = object
+    name*, stepCmd*, workingDir*: string
+    artifacts*, cmdInput*, depends*, expectedEnv*: seq[string]
+    dontCache*: bool
+
+  ProjectCfg* = object
+    name*: string
+    versionCmd*: string
+    steps*: Table[string, Step]
+
+  ProjectDef* = object
+    cfgFilePath*, defaultBranch*, name*, repo*: string
+    envVars*: StringTableRef
+
+  StrawBossCfg* = object
+    artifactsRepo*: string
+    projects*: seq[ProjectDef]
+
+  Workspace* = ref object
+    artifactsRepo*: string
+    env*: StringTableRef
+    workingDir*: string
+    project: ProjectCfg
+
+  HandleProcMsgCB = proc (cmd: string, outMsg: TaintedString, errMsg: TaintedString): void
+
+# Misc. helpers
+
+proc raiseEx(reason: string): void =
+  raise newException(Exception, reason)
+
+let nullNode = newJNull()
+proc getIfExists(n: JsonNode, key: string): JsonNode =
+  result = if n.hasKey(key): n[key]
+           else: nullNode
+
+
+proc loadEnv(): StringTableRef =
+  result = newStringTable()
+
+  for k, v in envPairs():
+    result[k] = v
+
+proc waitForWithOutput(p: Process, msgCB: HandleProcMsgCB,
+                       procCmd: string = ""):
+      tuple[output: TaintedString, error: TaintedString, exitCode: int] =
+
+  var pout = outputStream(p)
+  var perr = errorStream(p)
+
+  result = (TaintedString"", TaintedString"", -1)
+  var line = newStringOfCap(120).TaintedString
+  while true:
+    if pout.readLine(line):
+      if msgCB != nil: msgCB(procCmd, line, nil)
+      result[0].string.add(line.string)
+      result[0].string.add("\n")
+    elif perr.readLine(line):
+      if msgCB != nil: msgCB(procCmd, nil, line)
+      result[1].string.add(line.string)
+      result[1].string.add("\n")
+    else:
+      result[2] = peekExitCode(p)
+      if result[2] != -1: break
+  close(p)
+  
+proc exec(command: string, workingDir: string = "",
+          args: openArray[string] = [], env: StringTableRef = nil,
+          options: set[ProcessOption] = {poUsePath},
+          msgCB: HandleProcMsgCB = nil):
+            tuple[output: TaintedString, error: TaintedString, exitCode: int]
+            {.tags: [ExecIOEffect, ReadIOEffect], gcsafe.} =
+
+  var p = startProcess(command, workingDir, args, env, options)
+  result = waitForWithOutput(p, msgCb, command)
+
+let ENV = loadEnv()
+let logProcOutput: HandleProcMsgCB = proc (cmd: string, outMsg: TaintedString, errMsg: TaintedString) =
+  if outMsg != nil: info cmd & "(stdout): " & outMsg
+  if errMsg != nil: info cmd & "(stderr): " & errMsg
+
+let envVarRe = re"\$\w+|\$\{[^}]+\}"
+proc resolveEnvVars(line: string, env: StringTableRef): string =
+  result = line
+  for found in line.findAll(envVarRe):
+    let key = if found[1] == '{': found[2..^2] else: found[1..^1]
+    if env.hasKey(key): result = result.replace(found, env[key])
+
+let SB_EXPECTED_VARS = ["VERSION"]
+# Configuration parsing code
+
+proc loadStrawBossConfig(cfgFile: string): StrawBossCfg =
+  let jsonCfg = parseFile(cfgFile)
+
+  var projectDefs: seq[ProjectDef] = @[]
+
+  for pJson in jsonCfg.getIfExists("projects").getElems:
+    if not pJson.hasKey("name"):
+      raiseEx "a project definition is missing the project name"
+
+    if not pJson.hasKey("repo"):
+      raiseEx "a project definition is missing the project repo configuration"
+
+    var envVars = newStringTable(modeCaseSensitive)
+    for k, v in pJson.getIfExists("envVars").getFields: envVars[k] = v.getStr("")
+
+    projectDefs.add(
+      ProjectDef(
+        cfgFilePath: pJson.getIfExists("cfgFilePath").getStr("strawboss.json"),
+        defaultBranch: pJson.getIfExists("defaultBranch").getStr("master"),
+        name: pJson["name"].getStr,
+        envVars: envVars,
+        repo: pJson["repo"].getStr))
+
+  result = StrawBossCfg(
+    artifactsRepo: jsonCfg.getIfExists("artifactsRepo").getStr("artifacts"),
+    projects: projectDefs)
+
+proc loadProjectConfig(cfgFile: string): ProjectCfg =
+  let jsonCfg = parseFile(cfgFile)
+
+  if not jsonCfg.hasKey("name"):
+    raise newException(Exception, "project configuration is missing a name")
+
+  if not jsonCfg.hasKey("steps"):
+    raise newException(Exception, "project configuration is missing steps definition")
+
+  var steps = initTable[string, Step]()
+  for sName, pJson in jsonCfg["steps"].getFields:
+    steps[sName] = Step(
+      name:         sName,
+      workingDir:   pJson.getIfExists("workingDir").getStr("."),
+      stepCmd:      pJson.getIfExists("stepCmd").getStr("sh"),
+      depends:      pJson.getIfExists("depends").getElems.mapIt(it.getStr),
+      artifacts:    pJson.getIfExists("artifacts").getElems.mapIt(it.getStr),
+      cmdInput:     pJson.getIfExists("cmdInput").getElems.mapIt(it.getStr),
+      expectedEnv:  pJson.getIfExists("expectedEnv").getElems.mapIt(it.getStr),
+      dontCache:    pJson.getIfExists("dontCache").getStr("false") != "false")
+
+    if steps[sName].stepCmd == "sh" and steps[sName].cmdInput.len == 0:
+      warn "Step " & sName & " uses 'sh' as its command but has no cmdInput."
+
+  result = ProjectCfg(
+    name: jsonCfg["name"].getStr,
+    versionCmd: jsonCfg.getIfExists("versionCmd").getStr("git describe --tags --always"),
+    steps: steps)
+
+proc setupProjectForWork(projectDef: ProjectDef, buildRef, artifactsRepo: string): Workspace =
+
+  info "Setting up to do work for '" & projectDef.name & "' at ref " & buildRef & "."
+
+  # Create a temp directory that we'll work in
+  let projDir = mkdtemp()
+
+  # Clone the project into the $temp/repo directory
+  let cloneResult = exec("git", projDir, ["clone", projectDef.repo, "repo"],
+    ENV, {poUsePath}, logProcOutput)
+
+  if cloneResult.exitCode != 0:
+    removeDir(projDir)
+    raiseEx "unable to clone repo for '" & projectDef.name & "'"
+
+  # Checkout the requested ref
+  let checkoutResult = exec("git", projDir & "/repo", ["checkout", buildRef],
+      ENV, {poUsePath}, logProcOutput)
+
+  if checkoutResult.exitCode != 0:
+    removeDir(projDir)
+    raiseEx "unable to checkout ref " & buildRef & " for '" & projectDef.name & "'"
+
+  # Find the strawboss project configuration
+  let projCfgFile = projDir & "/repo/" & projectDef.cfgFilePath
+  if not existsFile(projCfgFile):
+    removeDir(projDir)
+    raiseEx "Cannot find strawboss project configuration in the project " &
+      "repo (expected at '" & projectDef.cfgFilePath & "')."
+
+  let projectCfg = loadProjectConfig(projCfgFile)
+  result = Workspace(env: ENV, workingDir: projDir, project: projectCfg,
+                     artifactsRepo: artifactsRepo)
+
+  # Merge in the project-defined env vars
+  for k, v in projectDef.envVars: result.env[k] = v
+
+  # Get the build version
+  let versionProc = startProcess(
+    projectCfg.versionCmd,      # command
+    projDir & "/repo",          # working dir
+    [],                         # args
+    result.env,                 # environment
+    {poUsePath, poEvalCommand}) # options
+
+  let versionResult = waitForWithOutput(versionProc, logProcOutput,
+                                        projectCfg.versionCmd)
+
+  if versionResult.exitCode != 0:
+    removeDir(projDir)
+    raiseEx "Version command (" & projectCfg.versionCmd & ") returned non-zero exit code."
+
+  debug "Building version " & versionResult.output.strip
+  result.env["VERSION"] = versionResult.output.strip
+
+  debug "Workspace for '" & projectCfg.name & ":  " & projDir
+
+## TODO
+proc runStep(step: Step, wksp: Workspace): void =
+
+  let stepArtifactDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & step.name & "/" & wksp.env["VERSION"]
+
+  if existsDir(stepArtifactDir) and not step.dontCache:
+    info "Skipping step '" & step.name & "': already completed."
+    return
+
+  info "Running step '" & step.name & "' for " & wksp.project.name
+
+  # Ensure all expected environment variables are present.
+  for k in (step.expectedEnv & @SB_EXPECTED_VARS):
+    if not wksp.env.hasKey(k):
+      debug "workspace.env = " & $(wksp.env)
+      raiseEx "step " & step.name & " failed: missing required env variable: " & k
+
+  # Ensure that artifacts in steps we depend on are present
+  for dep in step.depends:
+    if not wksp.project.steps.hasKey(dep):
+      raiseEx step.name & " depends on " & dep & " but there is no step named " & dep
+    let depStep = wksp.project.steps[dep]
+
+    # Run that step (may get skipped)
+    let depDir = wksp.artifactsRepo & "/" & wksp.project.name & "/" & dep & "/" & wksp.env["VERSION"]
+    runStep(depStep, wksp)
+
+    # Add the artifacts directory for the dependent step to our env so that
+    # further steps can reference it via $<stepname>_DIR
+    echo "FP: " & depDir
+    wksp.env[depStep.name & "_DIR"] = depDir
+
+  # Run the step command, piping in cmdInput
+  debug step.name & ": starting stepCmd: " & step.stepCmd
+  let cmdProc = startProcess(step.stepCmd, wksp.workingDir & "/repo/" & step.workingDir,
+    [], wksp.env, {poUsePath, poEvalCommand})
+
+  let cmdInStream = inputStream(cmdProc)
+
+  # Replace env variables in step cmdInput as we pipe it in
+  for line in step.cmdInput: cmdInStream.writeLine(line.resolveEnvVars(wksp.env))
+  cmdInStream.flush()
+  cmdInStream.close()
+  
+  let cmdResult = waitForWithOutput(cmdProc, logProcOutput, step.stepCmd)
+
+  if cmdResult.exitCode != 0:
+    raiseEx "step " & step.name & " failed: step command returned non-zero exit code"
+
+  # Gather the output artifacts (if we have any)
+  if not existsDir(stepArtifactDir): createDir(stepArtifactDir)
+  if step.artifacts.len > 0:
+    for a in step.artifacts:
+      let artifactPath = a.resolveEnvVars(wksp.env)
+      let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
+      try:
+        copyFile(wksp.workingDir & "/repo/" & step.workingDir & "/" & artifactPath,
+                 stepArtifactDir & "/" & artifactName)
+      except:
+        removeDir(stepArtifactDir)
+        raiseEx "step " & step.name & " failed: unable to copy artifact " &
+          artifactPath & ":\n" & getCurrentExceptionMsg()
+
+when isMainModule:
+
+  logging.addHandler(newConsoleLogger())
+
+  let cfg = loadStrawBossConfig("strawboss.config.json")
+  let artifactsRepo = expandFilename(cfg.artifactsRepo)
+  
+  if not existsDir(artifactsRepo):
+    info "Artifacts repo (" & artifactsRepo & ") does not exist. Creating..."
+    createDir(artifactsRepo)
+
+  let doc = """
+Usage:
+  strawboss serve
+  strawboss run <project> <step> [-r <ref>]"""
+
+  let args = docopt(doc, version = "strawboss v" & SB_VER)
+
+  if args["run"]:
+
+    # Find the project
+    let projName = $args["<project>"]
+    let matching = cfg.projects.filterIt(it.name == projName)
+    if matching.len == 0:
+      fatal "strawboss: no such project: " & projName
+      quit(QuitFailure)
+    elif matching.len > 1:
+      fatal "strawboss: more than one project named : " & projName
+      quit(QuitFailure)
+
+    let projectDef = matching[0]
+    let buildRef = if args["-r"]: $args["<ref>"] else: projectDef.defaultBranch
+
+    try:
+      let wksp = setupProjectForWork(projectDef, buildRef, artifactsRepo)
+
+      # Find the step
+      let stepName = $args["<step>"]
+      if not wksp.project.steps.hasKey(stepName):
+        raiseEx "no step name '" & stepName & "' for " & projName
+
+      let step = wksp.project.steps[stepName]
+      runStep(step, wksp)
+    except:
+      fatal "strawboss: " & getCurrentExceptionMsg()
+      quit(QuitFailure)

strawboss.nimble

@@ -8,5 +8,5 @@ license       = "MIT"
 
 # Dependencies
 
-requires @["nim >= 0.16.1", "docopt >= 0.1.0"]
+requires @["nim >= 0.16.1", "docopt >= 0.1.0", "tempfile"]